From 65ba1374af2526f8fe9d950d1a946056fc7bdb9d Mon Sep 17 00:00:00 2001 From: t Date: Thu, 9 Jul 2026 21:43:38 -0500 Subject: [PATCH 01/52] feat(node,git): cap concurrent served git ops with a 503 load-shed (#62) PR3 of the #62 served-git hardening stack (timeout #165 and teardown wiring #150 are merged). A bounded semaphore caps how many upload-pack / receive-pack / info-refs operations run at once; past the cap a request is shed with a clean 503 + Retry-After before spawning another git subprocess, instead of exhausting the PID/thread table. A permit is acquired at the top of each of the three handlers and held for the whole op, releasing on return. The cap is a portable backstop: the compose pids_limit is absent on Fly, whose 500-connection cap is a different axis. Size --max-concurrent-git-ops (GITLAWB_MAX_CONCURRENT_GIT_OPS, default 128) below the process budget. Range 1..=1_048_576 so 0 (shed everything) and an oversized value that would panic tokio's Semaphore at boot are clean CLI errors. Known gap, tracked separately: info/refs and the withheld-blob (upload_pack_excluding) path are not duration-bounded and do not reap their git child on client disconnect, so a hung git on those two paths holds its slot until it exits and live git can briefly exceed the cap. The main pack path (run_git_service) tears its group down on drop. Tests: Overloaded maps to 503 + Retry-After; the config knob defaults and rejects out-of-range; git_permit sheds at capacity and releases; and each of the three endpoints sheds with 503 when the semaphore is exhausted (load-bearing: drop the permit line and the endpoint test goes red). --- crates/gitlawb-node/src/api/repos.rs | 37 ++++++++ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/config.rs | 61 ++++++++++++ crates/gitlawb-node/src/error.rs | 31 +++++- crates/gitlawb-node/src/main.rs | 1 + crates/gitlawb-node/src/state.rs | 4 + crates/gitlawb-node/src/test_support.rs | 119 ++++++++++++++++++++++++ 7 files changed, 253 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b9cbc352..4bc1f737 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -507,6 +507,9 @@ pub async fn git_info_refs( headers: axum::http::HeaderMap, auth: Option>, ) -> Result { + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. + let _permit = git_permit(&state.git_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state @@ -589,6 +592,22 @@ pub async fn git_info_refs( }) } +/// Acquire a permit from the served-git concurrency semaphore, or shed the +/// request with a 503 + Retry-After when every slot is in use. Bind the returned +/// permit to a named local so it is held for the whole git op (it releases on +/// drop); a bare `_` would release it immediately. +fn git_permit( + sem: &std::sync::Arc, +) -> Result { + sem.clone().try_acquire_owned().map_err(|_| { + // Surface the shed so operators can see the cap engaging, mirroring the + // receive-pack rate-limit warn above. A silent 503 makes a saturated or + // misconfigured cap look like a client problem instead of a capacity one. + tracing::warn!("served-git concurrency cap reached; shedding request with 503"); + AppError::Overloaded("git service at capacity, retry shortly".into()) + }) +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -616,6 +635,9 @@ pub async fn git_upload_pack( auth: Option>, body: Bytes, ) -> Result { + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. + let _permit = git_permit(&state.git_semaphore)?; let name = smart_http_repo_name(&repo)?; let record = state .db @@ -853,6 +875,10 @@ pub async fn git_receive_pack( Extension(auth): Extension, body: Bytes, ) -> Result { + // Shed with a 503 before spawning git when the concurrency cap is saturated. + // Acquired at the very top so it wraps the write-guard below; held for the + // whole op (incl. the smart_http call), released on return. + let _permit = git_permit(&state.git_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state @@ -1862,6 +1888,17 @@ mod tests { assert!(matches!(git_service_app_error(&other), AppError::Git(_))); } + #[test] + fn git_permit_sheds_at_capacity_and_releases() { + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let p1 = git_permit(&sem).expect("first acquire succeeds"); + // At capacity the next request is shed with Overloaded (-> 503), not queued. + assert!(matches!(git_permit(&sem), Err(AppError::Overloaded(_)))); + // Releasing the permit frees the slot for the next request. + drop(p1); + assert!(git_permit(&sem).is_ok()); + } + fn repo_owned_by(owner_did: &str) -> crate::db::RepoRecord { let now = chrono::Utc::now(); crate::db::RepoRecord { diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 720fb3ae..b541b57b 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -520,6 +520,7 @@ mod tests { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..688e99f4 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -234,6 +234,40 @@ pub struct Config { value_parser = clap::value_parser!(u64).range(1..) )] pub db_retry_max_secs: u64, + + /// Maximum number of served git operations (upload-pack / receive-pack / + /// info-refs) allowed to run concurrently. Beyond this the node sheds the + /// request with a clean 503 + Retry-After instead of spawning another git + /// subprocess and risking PID/thread exhaustion. Portable backstop: the + /// compose `pids_limit` is not present on Fly, whose connection-concurrency + /// cap is a different axis (500 connections each fan out to git + + /// pack-objects + threads). Size below the process budget with headroom. + /// + /// A permit is held for the whole op. upload-pack/receive-pack are + /// duration-bounded by `git_service_timeout_secs`, but the `info/refs` + /// advertisement and the withheld-blob (`upload_pack_excluding`) path are + /// not, so a hung git on those two paths holds its slot until it exits, so a + /// stuck advertisement permanently costs one unit of capacity. Bounding + /// those paths' duration is tracked as separate follow-up. + /// + /// The same two paths have the mirror gap on cancellation: the permit frees + /// when the handler future drops (client disconnect), but neither reaps its + /// git child — `info/refs` runs a bare `Command`, and `upload_pack_excluding` + /// a blocking `spawn_blocking` a dropped future cannot cancel — so the child + /// runs to completion after its slot is freed and live git can briefly exceed + /// the cap. The main pack path (`run_git_service`) tears its process group + /// down on drop, so this is confined to the same two follow-up paths. + /// + /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value + /// well under tokio's `Semaphore` permit limit so an oversized value is a + /// clean CLI error rather than a boot-time panic. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_OPS", + default_value_t = 128, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_ops: usize, } impl Config { @@ -272,4 +306,31 @@ mod tests { Config::try_parse_from(["gitlawb-node", "--git-service-timeout-secs", "0"]).is_err() ); } + + #[test] + fn max_concurrent_git_ops_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_ops, + 128 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "8"]) + .max_concurrent_git_ops, + 8 + ); + // 0 permits would shed every served-git request with a 503; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "0"]).is_err()); + // Above the ceiling would panic tokio's Semaphore::new at boot (permits > + // usize::MAX >> 3); clap must reject it as a clean CLI error instead. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048577"]) + .is_err() + ); + // The ceiling itself is accepted. + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048576"]) + .max_concurrent_git_ops, + 1_048_576 + ); + } } diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index bdd5aec9..a07235fa 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -47,6 +47,9 @@ pub enum AppError { #[error("git service timed out: {0}")] Timeout(String), + #[error("server overloaded: {0}")] + Overloaded(String), + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -147,6 +150,12 @@ impl IntoResponse for AppError { DB_UNAVAILABLE_CODE, DB_UNAVAILABLE_MESSAGE.into(), ), + // 503 with a Retry-After (attached after this match — the shared tail + // can't carry per-variant headers). This is the single place Overloaded + // becomes a response, so it can never ship a 503 without the retry hint. + AppError::Overloaded(msg) => { + (StatusCode::SERVICE_UNAVAILABLE, "overloaded", msg.clone()) + } AppError::Db(e) => (StatusCode::INTERNAL_SERVER_ERROR, "db_error", e.to_string()), AppError::Internal(e) => ( StatusCode::INTERNAL_SERVER_ERROR, @@ -160,7 +169,17 @@ impl IntoResponse for AppError { "message": message, })); - (status, body).into_response() + let mut resp = (status, body).into_response(); + // Overloaded advertises when to retry. It rides the shared tail above for + // its body/status, so the header is attached here rather than in a bespoke + // early return — keeping the variant handled in exactly one place. + if matches!(self, AppError::Overloaded(_)) { + resp.headers_mut().insert( + axum::http::header::RETRY_AFTER, + axum::http::HeaderValue::from_static("1"), + ); + } + resp } } @@ -182,4 +201,14 @@ mod tests { StatusCode::INTERNAL_SERVER_ERROR ); } + + #[test] + fn overloaded_maps_to_503_with_retry_after() { + let resp = AppError::Overloaded("x".into()).into_response(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1" + ); + } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..ee36c783 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -378,6 +378,7 @@ async fn main() -> Result<()> { sync_trigger_rate_limiter, peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), + git_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 6a84b3c2..e6e69ea6 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -88,6 +88,10 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, + /// Bounds concurrent served git operations. A handler acquires a permit + /// before spawning git and holds it for the op; when none are free the + /// request is shed with a 503 rather than exhausting the PID/thread table. + pub git_semaphore: Arc, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 4fe52f64..cc322a67 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -82,6 +82,8 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + // Generous — no test drives the handler-level shed (git_permit is unit-tested). + git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } @@ -187,6 +189,123 @@ mod tests { ); } + /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer. One test + /// per git handler drives the `let _permit = git_permit(...)` wiring end to end + /// (this one plus the git_upload_pack / git_receive_pack siblings below); the + /// git_permit unit test covers the helper in isolation. DB-free: an exhausted + /// semaphore sheds before any DB/disk access, so a lazy state works. Remove the + /// permit line from git_info_refs and this goes red (the request falls through + /// to the DB and returns something other than 503). + #[tokio::test] + async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-upload-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed info/refs with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack also acquires a + /// permit at the top, so an exhausted semaphore must shed it with a 503 before + /// any DB/disk work. Anonymous-reachable, so no auth injection is needed. Remove + /// the permit line from git_upload_pack and this goes red. + #[tokio::test] + async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed git-upload-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling for the push path: git-receive-pack requires an + /// AuthenticatedDid extension (production: require_signature injects it), so the + /// request carries one via signed_request_as — without it the Extension + /// extractor 500s before the handler body reaches git_permit. The permit is the + /// first statement, so an exhausted semaphore still sheds 503 before any DB + /// work. Remove the permit line from git_receive_pack and this goes red. + #[tokio::test] + async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVSHEDOWNERAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed git-receive-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + /// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner /// before any git work (so no on-disk repo is needed for the rejection). #[sqlx::test] From db4de844add8e1eeb61f2e5b5118751a677ca6f3 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 12:18:07 -0500 Subject: [PATCH 02/52] feat(node,config): add write-pool and per-caller read-cap knobs (#62) Add max_concurrent_git_pushes (default 32) and max_concurrent_reads_per_caller (default 16), both clap range(1..=1_048_576) so an oversized value is a clean CLI error, not a Semaphore::new boot panic. The per-caller knob documents that per-source-IP keying is only as granular as GITLAWB_TRUSTED_PROXY. Wiring lands in the following commits; these are the config surface for the #174 concurrency-fairness fix. Resolves jatmn P1a/P1b groundwork on #174. --- .env.example | 15 +++++ crates/gitlawb-node/src/config.rs | 96 +++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/.env.example b/.env.example index bbd9a342..d7c6a30b 100644 --- a/.env.example +++ b/.env.example @@ -113,6 +113,21 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # advertisement or the withheld-blob path, which remain unbounded. Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max concurrent git-receive-pack (push) operations, in a pool separate from the +# read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an +# authenticated push at admission. Over-cap sheds a 503 + Retry-After. Default 32. +GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 + +# Max concurrent read ops (upload-pack + info/refs advertisements) a single caller +# may hold, so one caller cannot monopolize the read pool. Keyed per-DID when +# signed, else per-source-IP. The per-IP key is only as granular as +# GITLAWB_TRUSTED_PROXY below: left unset, a node behind an edge/NAT keys all +# anonymous callers on the edge IP and this collapses to one global anonymous cap. +# Set GITLAWB_TRUSTED_PROXY for per-client keying; a high-fanout caller (CI behind +# one NAT) should authenticate for a per-DID budget or the operator raises this. +# Default 16. +GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 + # ── Push rate limiting (git-receive-pack flood brake) ───────────────────── # Max receive-pack requests (info/refs advertisement + push POST) per client # IP per hour. 0 disables. Default 600. diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 688e99f4..ff2c2556 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -268,6 +268,42 @@ pub struct Config { value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) )] pub max_concurrent_git_ops: usize, + + /// Maximum number of concurrent `git-receive-pack` (push) operations. Pushes + /// draw from this dedicated pool, separate from `max_concurrent_git_ops` + /// (reads), so a flood of anonymous reads cannot shed an authenticated push at + /// admission (#174). Beyond this a push sheds a clean 503 + Retry-After. + /// + /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value + /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI + /// error rather than a boot-time panic). + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_pushes: usize, + + /// Maximum concurrent read operations (`upload-pack` and the `info/refs` + /// advertisements) a single caller may hold at once, so one caller cannot + /// monopolize the `max_concurrent_git_ops` read pool (#174). Callers are keyed + /// per-DID when signed, else per-source-IP. IMPORTANT: the per-source-IP key is + /// only as granular as `GITLAWB_TRUSTED_PROXY`. Left unset (the default), a node + /// behind an edge/NAT keys all anonymous callers on the edge IP, so this cap + /// collapses to a single global anonymous cap rather than per-client. Set + /// `GITLAWB_TRUSTED_PROXY` to key on the real client; a known high-fanout caller + /// (a CI fleet behind one NAT) should authenticate for a per-DID budget or the + /// operator raises this. Over-cap for a caller sheds a clean 503 + Retry-After. + /// + /// Default: 16. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_READS_PER_CALLER", + default_value_t = 16, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_reads_per_caller: usize, } impl Config { @@ -333,4 +369,64 @@ mod tests { 1_048_576 ); } + + #[test] + fn max_concurrent_git_pushes_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_pushes, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "8"]) + .max_concurrent_git_pushes, + 8 + ); + // 0 permits would shed every push with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048576"]) + .max_concurrent_git_pushes, + 1_048_576 + ); + } + + #[test] + fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_reads_per_caller, + 16 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "4"]) + .max_concurrent_reads_per_caller, + 4 + ); + // 0 would shed every read from a keyed caller; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "0"]) + .is_err() + ); + assert!(Config::try_parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048577" + ]) + .is_err()); + assert_eq!( + Config::parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048576" + ]) + .max_concurrent_reads_per_caller, + 1_048_576 + ); + } } From 2434dde42f1ac776da08df8186b7f4a3b54073ae Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 12:29:30 -0500 Subject: [PATCH 03/52] fix(node,git): isolate authed pushes in a dedicated write pool (#62) git-receive-pack now draws from a separate git_write_semaphore (max_concurrent_git_pushes) instead of the shared pool, so a flood of anonymous reads can no longer shed an authenticated push at admission (jatmn P1a). The shared field is renamed git_read_semaphore and continues to gate upload-pack and both info/refs advertisements. The write permit stays above acquire_write so it precedes the Tigris fresh-acquire (INV-10). Handler-layer tests: write-pool shed (503), and a cross-boundary proof that an exhausted read pool does NOT shed a push; both mutation-checked (routing receive-pack back to the read pool flips each RED). 497 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 12 ++++--- crates/gitlawb-node/src/auth/mod.rs | 3 +- crates/gitlawb-node/src/main.rs | 5 ++- crates/gitlawb-node/src/state.rs | 14 +++++--- crates/gitlawb-node/src/test_support.rs | 47 ++++++++++++++++++++++--- 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4bc1f737..dd030a36 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -509,7 +509,7 @@ pub async fn git_info_refs( ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated; // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_semaphore)?; + let _permit = git_permit(&state.git_read_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state @@ -637,7 +637,7 @@ pub async fn git_upload_pack( ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated; // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_semaphore)?; + let _permit = git_permit(&state.git_read_semaphore)?; let name = smart_http_repo_name(&repo)?; let record = state .db @@ -876,9 +876,11 @@ pub async fn git_receive_pack( body: Bytes, ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated. - // Acquired at the very top so it wraps the write-guard below; held for the - // whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_semaphore)?; + // Pushes draw from the dedicated WRITE pool, separate from reads, so a flood of + // anonymous reads cannot shed an authenticated push (#174). Acquired at the very + // top so it wraps the write-guard below (and precedes the Tigris acquire_write, + // bounding concurrent fresh acquires — INV-10); held for the whole op. + let _permit = git_permit(&state.git_write_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index b541b57b..1d169e1c 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -520,7 +520,8 @@ mod tests { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, - git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index ee36c783..b8f687b1 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -378,7 +378,10 @@ async fn main() -> Result<()> { sync_trigger_rate_limiter, peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), - git_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index e6e69ea6..b43f4de2 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -88,10 +88,16 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, - /// Bounds concurrent served git operations. A handler acquires a permit - /// before spawning git and holds it for the op; when none are free the - /// request is shed with a 503 rather than exhausting the PID/thread table. - pub git_semaphore: Arc, + /// Bounds concurrent served git READ operations (upload-pack + both info/refs + /// advertisements). A read handler acquires a permit before spawning git and + /// holds it for the op; when none are free the request is shed with a 503. + /// Writes draw from `git_write_semaphore` so a read flood cannot shed an + /// authenticated push at admission (#174). + pub git_read_semaphore: Arc, + /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate + /// from `git_read_semaphore` so anonymous reads can never shed an authenticated + /// push (#174). Sized by `max_concurrent_git_pushes`. + pub git_write_semaphore: Arc, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index cc322a67..fd68c8e6 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -83,7 +83,8 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, // Generous — no test drives the handler-level shed (git_permit is unit-tested). - git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } @@ -199,7 +200,7 @@ mod tests { #[tokio::test] async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); - state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -235,7 +236,7 @@ mod tests { #[tokio::test] async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); - state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -273,7 +274,7 @@ mod tests { #[tokio::test] async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); - state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -295,7 +296,7 @@ mod tests { assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "an exhausted git semaphore must shed git-receive-pack with 503 before touching the DB" + "an exhausted write pool must shed git-receive-pack with 503 before touching the DB" ); assert_eq!( resp.headers() @@ -306,6 +307,42 @@ mod tests { ); } + /// #174 (SC1, load-bearing): a saturated READ pool must NOT shed an + /// authenticated push — the write pool is a separate budget. Read pool at zero, + /// write pool with capacity: the push proceeds PAST admission (it then errors on + /// the placeholder DB, but crucially it is not a 503). Route git-receive-pack + /// back to the read pool and this goes red — that is the isolation proof. + #[tokio::test] + async fn git_receive_pack_not_shed_by_exhausted_read_pool() { + let mut state = test_state_lazy(); + // Read pool exhausted as if a flood of anonymous clones held every slot. + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + // Write pool keeps its default capacity from test_state_lazy. + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVCROSSBOUNDARYAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted READ pool must not shed a push — the write pool is a separate budget (#174)" + ); + } + /// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner /// before any git work (so no on-disk repo is needed for the rejection). #[sqlx::test] From fc4c206c3a1ccf38a433b2a4d8255b83910f4357 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 12:50:46 -0500 Subject: [PATCH 04/52] fix(node,git): per-caller concurrency sub-cap on the read pool (#62) Adds PerCallerConcurrency, a bounded-keyed in-flight limiter (distinct from the request-rate RateLimiter) so no single caller monopolizes the served-git read pool. Each caller (per-DID when signed via optional_signature, else per-source-IP via client_key) may hold at most max_concurrent_reads_per_caller concurrent reads; over that it sheds 503. The key map is self-bounding (a key is dropped when its in-flight count hits zero) with a reject-before-insert max_keys backstop so a key farm can't grow it (INV-15). Applied in git_upload_pack and both info/refs advertisements, acquired after the visibility gate so a denied request never consumes a slot (KTD7). Primitive unit-tested (cap + self-bounding + reject-before-insert) and mutation-checked. Handler-layer SC2: same-caller sheds while a different caller passes, proven on BOTH git_info_refs and git_upload_pack with independent mutation probes; plus a None-key bypass test. Per-source-IP keying is trust-config dependent, documented on the config knob. 502 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 210 ++++++++++++++++++++++++ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 3 + crates/gitlawb-node/src/rate_limit.rs | 142 ++++++++++++++++ crates/gitlawb-node/src/state.rs | 5 + crates/gitlawb-node/src/test_support.rs | 1 + 6 files changed, 362 insertions(+) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index dd030a36..e51b62e6 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -566,6 +566,28 @@ pub async fn git_info_refs( } } + // Per-caller read sub-cap (#174): acquired AFTER the visibility + push-rate + // gates (KTD7) so a denied or rate-limited request never consumes a caller's + // scarce read slot. Applies to BOTH advertisements. Held for the whole op. + let caller_key = read_caller_key( + auth.as_ref().map(|e| e.0 .0.as_str()), + &headers, + peer, + state.push_limiter_trust, + ); + let _caller_permit = match &caller_key { + Some(k) => match state.git_read_per_caller.try_acquire(k) { + Some(p) => Some(p), + None => { + tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding info/refs with 503"); + return Err(AppError::Overloaded( + "git read service at capacity for this caller, retry shortly".into(), + )); + } + }, + None => None, + }; + // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. let disk_path = if service == "git-receive-pack" { @@ -608,6 +630,23 @@ fn git_permit( }) } +/// Resolve the per-caller key for the read sub-cap (#174): the authenticated DID +/// when the caller signed (via `optional_signature`), else the trusted client IP +/// (`client_key`). `None` when neither is available — such a request is bounded by +/// the global read pool only, never a 500. The per-source-IP key is only as +/// granular as `trust`; see the `max_concurrent_reads_per_caller` config doc. +fn read_caller_key( + caller_did: Option<&str>, + headers: &axum::http::HeaderMap, + peer: Option, + trust: crate::rate_limit::TrustedProxy, +) -> Option { + match caller_did { + Some(did) => Some(did.to_string()), + None => crate::rate_limit::client_key(headers, peer, trust), + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -633,6 +672,8 @@ pub async fn git_upload_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, auth: Option>, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated; @@ -658,6 +699,23 @@ pub async fn git_upload_pack( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } + // Per-caller read sub-cap (#174): after the visibility gate (KTD7) so a + // visibility-denied caller never consumes a scarce read slot. Keyed per-DID + // when signed, else per-source-IP; no resolvable key -> global read pool only. + let caller_key = read_caller_key(caller, &headers, peer, state.push_limiter_trust); + let _caller_permit = match &caller_key { + Some(k) => match state.git_read_per_caller.try_acquire(k) { + Some(p) => Some(p), + None => { + tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding upload-pack with 503"); + return Err(AppError::Overloaded( + "git read service at capacity for this caller, retry shortly".into(), + )); + } + }, + None => None, + }; + let disk_path = state .repo_store .acquire(&record.owner_did, &record.name) @@ -2618,6 +2676,158 @@ mod tests { ); } + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that + /// is already at its concurrency budget on the upload-pack advertisement, while + /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and + /// the same-caller assertion goes green-not-503 — this is the info_refs half of + /// the two-handler mutation probe. + #[sqlx::test] + async fn info_refs_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcadv", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.31:5000".parse().unwrap(); + // Fill this caller's single read slot (a clone shares the Arc-backed map). + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + // Same caller (IP) at its cap -> shed 503 before the git/Tigris work. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed the advertisement with 503" + ); + + // A DIFFERENT caller (IP) has its own budget -> not shed by the per-caller cap. + let other: SocketAddr = "203.0.113.32:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 SC2 (upload_pack probe): the same per-caller shed on the POST + /// upload-pack path. Remove the sub-cap from `git_upload_pack` and this goes + /// green-not-503 — the upload_pack half of the two-handler mutation probe. + #[sqlx::test] + async fn upload_pack_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcupl", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.41:5000".parse().unwrap(); + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed upload-pack with 503" + ); + + let other: SocketAddr = "203.0.113.42:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 SC2 (None-key): a request with no resolvable caller key (no ConnectInfo, + /// no trusted header) must NOT be shed by the per-caller cap even when another + /// caller's budget is full — it is bounded by the global read pool only. A None + /// key never keys into the map, so it never 503s from the per-caller sub-cap. + #[sqlx::test] + async fn info_refs_none_key_bypasses_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcnone", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + // Saturate an unrelated caller's budget; the None-key request must be + // unaffected because it never keys into the per-caller map. + let _slot = state + .git_read_per_caller + .try_acquire("203.0.113.99") + .expect("hold an unrelated caller's slot"); + + // No ConnectInfo inserted -> PeerAddr is None -> no per-caller key. + let router = crate::server::build_router(state.clone()); + let req = Request::builder() + .method(Method::GET) + .uri("/z6pcnone/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a request with no resolvable caller key must not be shed by the per-caller cap" + ); + } + /// Repo creation must be throttled by the per-IP creation limiter BEFORE /// signature verification — otherwise a DID farm (one throwaway did:key per /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 1d169e1c..253b3210 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -522,6 +522,7 @@ mod tests { shutdown_tx: tokio::sync::watch::channel(false).0, git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index b8f687b1..c617b434 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -382,6 +382,9 @@ async fn main() -> Result<()> { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.max_concurrent_reads_per_caller, + ), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d40e2691..d505f1bb 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -132,6 +132,91 @@ impl RateLimiter { } } +/// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which +/// caps request RATE. Each caller key may hold at most `per_caller` in-flight +/// permits at once; beyond that [`try_acquire`](Self::try_acquire) returns `None` +/// and the caller sheds. Used to stop one caller (a single anonymous source-IP or +/// DID) monopolizing the served-git read pool (#174). +/// +/// The key map is self-bounding: a key is removed the instant its in-flight count +/// reaches zero, so it never holds more keys than there are concurrently-active +/// callers (itself bounded by the read semaphore). A `max_keys` reject-before-insert +/// backstop guarantees a key farm can never grow the map even if that invariant +/// weakened — a NEW key at the cap is rejected WITHOUT allocating an entry (INV-15). +/// +/// Uses a `std::sync::Mutex` (not the file's `tokio::sync::Mutex`) because the +/// permit's `Drop` must release synchronously; the critical section holds no await. +#[derive(Clone)] +pub struct PerCallerConcurrency { + state: Arc>>, + per_caller: usize, + max_keys: usize, +} + +/// RAII permit from [`PerCallerConcurrency::try_acquire`]. On drop it decrements +/// the caller's in-flight count and removes the key when it reaches zero. +pub struct PerCallerPermit { + state: Arc>>, + key: String, +} + +impl PerCallerConcurrency { + pub fn new(per_caller: usize, max_keys: usize) -> Self { + Self { + state: Arc::new(std::sync::Mutex::new(HashMap::new())), + per_caller: per_caller.max(1), + max_keys: max_keys.max(1), + } + } + + /// Convenience constructor with the default key bound. + pub fn with_default_max_keys(per_caller: usize) -> Self { + Self::new(per_caller, DEFAULT_MAX_KEYS) + } + + /// `Some(permit)` when the caller is under its cap and the map has room; + /// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is + /// rejected without allocating. + pub fn try_acquire(&self, key: &str) -> Option { + let mut map = self.state.lock().expect("per-caller mutex poisoned"); + match map.get_mut(key) { + Some(count) => { + if *count >= self.per_caller { + return None; + } + *count += 1; + } + None => { + if map.len() >= self.max_keys { + return None; + } + map.insert(key.to_string(), 1); + } + } + Some(PerCallerPermit { + state: self.state.clone(), + key: key.to_string(), + }) + } + + #[cfg(test)] + pub fn tracked_keys(&self) -> usize { + self.state.lock().unwrap().len() + } +} + +impl Drop for PerCallerPermit { + fn drop(&mut self) { + let mut map = self.state.lock().expect("per-caller mutex poisoned"); + if let Some(count) = map.get_mut(&self.key) { + *count -= 1; + if *count == 0 { + map.remove(&self.key); + } + } + } +} + pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { let limiter = request.extensions().get::().cloned(); @@ -288,6 +373,63 @@ pub async fn rate_limit_by_ip(request: Request, next: Next) -> Response { mod tests { use super::*; + #[test] + fn per_caller_concurrency_caps_one_caller_and_frees_on_drop() { + let lim = PerCallerConcurrency::new(2, 100); + let p1 = lim.try_acquire("did:key:zA").expect("first under cap"); + let p2 = lim.try_acquire("did:key:zA").expect("second under cap"); + assert!( + lim.try_acquire("did:key:zA").is_none(), + "a third in-flight op for the same caller sheds (over the per-caller cap)" + ); + // A DIFFERENT caller is unaffected — the cap is per-caller, not global. + let _other = lim + .try_acquire("did:key:zB") + .expect("a different caller has its own budget"); + drop(p1); + assert!( + lim.try_acquire("did:key:zA").is_some(), + "freeing one in-flight slot lets the same caller back in" + ); + drop(p2); + } + + #[test] + fn per_caller_concurrency_map_is_self_bounding_and_reject_before_insert() { + // Self-bounding: acquire+drop many distinct keys — the map never grows + // because a key is removed the instant its in-flight count hits zero. + let lim = PerCallerConcurrency::new(4, 3); + for i in 0..50 { + let _p = lim.try_acquire(&format!("k{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "keys with zero in-flight ops are removed, so an acquire+drop flood leaves the map empty" + ); + // Reject-before-insert: HOLD max_keys distinct keys, then a new key sheds + // WITHOUT growing the map past the cap (INV-15 — a rejected request never + // allocates an entry). + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct callers held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected new key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + #[tokio::test] async fn allows_within_limit() { let limiter = RateLimiter::new(3, Duration::from_secs(60)); diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index b43f4de2..01007ac4 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -98,6 +98,11 @@ pub struct AppState { /// from `git_read_semaphore` so anonymous reads can never shed an authenticated /// push (#174). Sized by `max_concurrent_git_pushes`. pub git_write_semaphore: Arc, + /// Per-caller concurrency sub-cap on the read pool: each caller (per-DID when + /// signed, else per-source-IP) may hold at most `max_concurrent_reads_per_caller` + /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` + /// (#174). Applied by `git_upload_pack` and both `info/refs` advertisements. + pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index fd68c8e6..7745d233 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -85,6 +85,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { // Generous — no test drives the handler-level shed (git_permit is unit-tested). git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), } } From 2c264e08e391f5675cbfd3edeff0238434e9bb15 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:09:34 -0500 Subject: [PATCH 05/52] fix(node,git): bound and reap the info/refs advertisement (#62) info_refs ran a bare Command::output() with no timeout and no process-group teardown, so a hung git pinned its concurrency slot indefinitely and a client disconnect orphaned the child (jatmn P1b). Extract the timeout + process_group(0) + KillGroupOnDrop core from run_git_service into a shared drive_git_child, and route info_refs through it with an injectable git_bin. A hung advertisement now aborts with GitServiceTimeout (mapped to 504); disconnect reaps the group. run_git_service's teardown tests all pass through the shared core (proving the group teardown info_refs inherits), the real-git filter tests cover the advertisement happy path, and a new watchdog-bounded test proves a hung advertisement times out. 503 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 15 +++- crates/gitlawb-node/src/git/smart_http.rs | 87 +++++++++++++++++++---- 2 files changed, 85 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index e51b62e6..c42a6989 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -606,11 +606,20 @@ pub async fn git_info_refs( AppError::Git(e.to_string()) })?; - smart_http::info_refs(&disk_path, &service) + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + smart_http::info_refs("git", &service, &disk_path, git_timeout) .await .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed"); - AppError::Git(e.to_string()) + let app = git_service_app_error(&e); + match &app { + AppError::Timeout(_) => { + tracing::warn!(repo = %name, service = %service, "info/refs advertisement timed out") + } + _ => { + tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed") + } + } + app }) } diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index eeb35b99..b4e05a01 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -14,21 +14,29 @@ use tokio::process::Command; /// or `?service=git-receive-pack` /// /// This is the ref advertisement — the first step of a clone or push. -pub async fn info_refs(repo_path: &Path, service: &str) -> Result { +/// +/// `git_bin` is injectable purely so the process-group teardown can be driven by a +/// fake `git` in tests (production passes `"git"`). `timeout` bounds the whole child +/// interaction: previously the advertisement ran a bare `Command::output()` with no +/// deadline and no teardown, so a hung git pinned its concurrency slot indefinitely +/// and a client disconnect orphaned the child (#174). It now shares +/// [`drive_git_child`]'s timeout + `process_group(0)` + [`KillGroupOnDrop`] teardown. +pub async fn info_refs( + git_bin: &str, + service: &str, + repo_path: &Path, + timeout: Duration, +) -> Result { validate_service(service)?; - let output = Command::new("git") + let mut command = Command::new(git_bin); + command .arg(service_to_command(service)) .arg("--stateless-rpc") .arg("--advertise-refs") - .arg(repo_path) - .output() - .await?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git {service} --advertise-refs failed: {stderr}"); - } + .arg(repo_path); + // No request body — advertise-refs does not read stdin. + let stdout = drive_git_child(command, Bytes::new(), timeout, "advertise-refs").await?; let content_type = format!("application/x-{service}-advertisement"); @@ -38,7 +46,7 @@ pub async fn info_refs(repo_path: &Path, service: &str) -> Result { let mut body = Vec::new(); body.extend_from_slice(&pkt_service); body.extend_from_slice(flush); - body.extend_from_slice(&output.stdout); + body.extend_from_slice(&stdout); Ok(Response::builder() .status(StatusCode::OK) @@ -207,7 +215,24 @@ async fn run_git_service( command .arg(service_to_command(service)) .arg("--stateless-rpc") - .arg(repo_path) + .arg(repo_path); + drive_git_child(command, input, timeout, service).await +} + +/// Drive a spawned git child under `timeout` with process-group teardown, returning +/// its stdout. Shared core for [`run_git_service`] and [`info_refs`]: the caller +/// passes a `Command` with its args set; this adds piped stdio and `process_group(0)`. +/// On the deadline the whole group is torn down and reaped before returning +/// [`GitServiceTimeout`]; on a dropped future (client disconnect) the +/// [`KillGroupOnDrop`] guard fires. `input` is written to the child's stdin (empty +/// for the advertise-refs path, which has no request body); `what` labels errors. +async fn drive_git_child( + mut command: Command, + input: Bytes, + timeout: Duration, + what: &str, +) -> Result> { + command .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -294,7 +319,7 @@ async fn run_git_service( // the real cause. if !status.success() { let stderr = String::from_utf8_lossy(&err); - bail!("{service} failed: {stderr}"); + bail!("{what} failed: {stderr}"); } write_result.context("failed to write to git stdin")?; @@ -650,7 +675,9 @@ mod tests { axum::extract::Query(q): axum::extract::Query>, ) -> Response { let service = q.get("service").cloned().unwrap_or_default(); - info_refs(&st.repo, &service).await.unwrap() + info_refs("git", &service, &st.repo, Duration::from_secs(30)) + .await + .unwrap() } async fn pack_handler( @@ -1225,6 +1252,38 @@ mod tests { ); } + // #174 (SC3): the info/refs advertisement is now duration-bounded — previously + // it ran a bare `Command::output()` with no deadline, so a hung git pinned its + // concurrency slot forever. A hung fake git must abort with GitServiceTimeout + // (which the handler maps to 504). The outer watchdog turns a missing timeout + // into a loud failure instead of hanging the suite. info_refs shares + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped above. + #[cfg(unix)] + #[tokio::test] + async fn info_refs_times_out_a_hung_advertisement() { + let tmp = tempfile::TempDir::new().unwrap(); + // A fake git that hangs forever instead of advertising refs. + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\nsleep 300\n"); + + let result = tokio::time::timeout( + Duration::from_secs(10), + info_refs( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Duration::from_millis(200), + ), + ) + .await + .expect("info_refs must return within the watchdog — its own timeout must fire"); + let err = result.expect_err("a hung advertisement must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung advertisement must abort with GitServiceTimeout (-> 504), got: {err}" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 57c6e01c26ac1e3032cb06b57ebc640e18582a85 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:16:41 -0500 Subject: [PATCH 06/52] fix(node,git): bound and reap the withheld-blob pack build (#62) The filtered-pack path ran the whole rev-list + pack-objects build inside a spawn_blocking, so an outer tokio timeout could not cancel the blocking thread and a client disconnect orphaned the git child while the permit freed (jatmn P1b, the second gap path). Split it: rev-list enumeration stays blocking off the runtime (rev_list_keep), but the streaming pack-objects stage now runs under the shared drive_git_child on the async side, so it is duration-bounded (GitServiceTimeout -> 504) and its process group is reaped on disconnect. build_filtered_pack becomes async and takes a git_bin seam + timeout; upload_pack_excluding threads the git_service_timeout through. A watchdog-bounded test proves a hung pack-objects times out (rev-list fast, pack-objects hangs). The refactor's happy path is covered by the existing filtered-pack correctness and real-git partial-clone/fetch tests, all still green; disconnect/group-teardown is the shared drive_git_child code proven by the run_git_service tests. 504 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 2 +- crates/gitlawb-node/src/git/smart_http.rs | 145 ++++++++++++++-------- 2 files changed, 95 insertions(+), 52 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index c42a6989..dba8ea71 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -765,7 +765,7 @@ pub async fn git_upload_pack( smart_http::upload_pack(&disk_path, body, git_timeout).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - smart_http::upload_pack_excluding(&disk_path, body, &withheld).await + smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await } } .map_err(|e| { diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index b4e05a01..5586a2c7 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -349,12 +349,17 @@ fn pkt_line(data: &str) -> Vec { format!("{len:04x}{data}").into_bytes() } -/// Build a packfile containing every object reachable from all refs EXCEPT the -/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; -/// only the named blobs are dropped. -pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Result> { - // All reachable objects as "oid [path]" lines. - let rev = std::process::Command::new("git") +/// Run `rev-list --objects --all` and return the reachable object ids minus the +/// withheld blobs. Blocking (shells out to git); the caller runs it off the async +/// runtime. Kept separate from the pack-objects stage so that stage can live on the +/// async side under [`drive_git_child`] (`git_bin` is injectable for the same +/// fake-git testing reason as `run_git_service`). +fn rev_list_keep( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, +) -> Result> { + let rev = std::process::Command::new(git_bin) .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path) .output()?; @@ -372,39 +377,39 @@ pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Resu } keep.push(oid.to_string()); } - let mut child = std::process::Command::new("git") - .args(["pack-objects", "--stdout"]) - .current_dir(repo_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - // Feed the object ids on stdin, but always reap the child afterward even if - // the write fails or stdin is missing, so an error can't drop the Child - // unwaited and leak a zombie (#53). - let write_result: std::io::Result<()> = { - use std::io::Write as _; - match child.stdin.take() { - Some(mut stdin) => { - let mut data = keep.join("\n").into_bytes(); - data.push(b'\n'); - stdin.write_all(&data) - } - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git pack-objects stdin unavailable", - )), - } + Ok(keep) +} + +/// Build a packfile containing every object reachable from all refs EXCEPT the +/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; +/// only the named blobs are dropped. +/// +/// The rev-list enumeration runs blocking off the async runtime; the streaming +/// `pack-objects` stage runs under [`drive_git_child`], so a hung/slow pack build is +/// duration-bounded and its process group is reaped on client disconnect. An outer +/// `tokio::time::timeout` around a `spawn_blocking` cannot cancel the blocking +/// thread, so the streaming stage MUST live on the async side (#174, KTD5). +pub async fn build_filtered_pack( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, + timeout: Duration, +) -> Result> { + let keep = { + let git_bin = git_bin.to_string(); + let repo_path = repo_path.to_path_buf(); + let withheld = withheld.clone(); + tokio::task::spawn_blocking(move || rev_list_keep(&git_bin, &repo_path, &withheld)) + .await + .context("rev-list task panicked")?? }; - let out = child.wait_with_output()?; - write_result.context("failed to write object ids to git pack-objects stdin")?; - if !out.status.success() { - bail!( - "git pack-objects failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - } - Ok(out.stdout) + let mut data = keep.join("\n").into_bytes(); + data.push(b'\n'); + let mut command = Command::new(git_bin); + command + .args(["pack-objects", "--stdout"]) + .current_dir(repo_path); + drive_git_child(command, Bytes::from(data), timeout, "pack-objects").await } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -434,17 +439,13 @@ pub async fn upload_pack_excluding( repo_path: &Path, request_body: Bytes, withheld: &HashSet, + timeout: Duration, ) -> Result { - // build_filtered_pack shells out to git (rev-list, pack-objects) with - // blocking std::process I/O; run it off the async worker so a large repo's - // pack build does not stall the tokio runtime. - let pack = { - let repo_path = repo_path.to_path_buf(); - let withheld = withheld.clone(); - tokio::task::spawn_blocking(move || build_filtered_pack(&repo_path, &withheld)) - .await - .context("filtered-pack build task panicked")?? - }; + // The rev-list enumeration runs blocking off the runtime; the streaming + // pack-objects stage is duration-bounded and its process group is reaped on + // disconnect via drive_git_child (#174), so a hung build no longer pins its + // concurrency slot and a client disconnect no longer orphans the git child. + let pack = build_filtered_pack("git", repo_path, withheld, timeout).await?; // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -563,7 +564,9 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack(&bare, &withheld).unwrap(); + let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30)) + .await + .unwrap(); let ids = pack_object_ids(&pack); assert!(ids.contains(&public), "public blob must be in the pack"); assert!( @@ -625,7 +628,9 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld).await.unwrap(); + let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30)) + .await + .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); let ids = pack_object_ids(&extract_pack(&body)); assert!( @@ -684,7 +689,7 @@ mod tests { axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld) + upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30)) .await .unwrap() } @@ -1284,6 +1289,44 @@ mod tests { ); } + // #174 (SC3, KTD5): the filtered-pack build's streaming pack-objects stage is + // duration-bounded on the ASYNC side. The old build ran the whole thing in a + // spawn_blocking, so an outer tokio timeout could not cancel the blocking thread + // and a disconnect orphaned the git child. A fake git that returns objects fast + // on rev-list but hangs on pack-objects must now abort with GitServiceTimeout; + // the watchdog turns a missing bound into a loud failure. The stage runs under + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_pack_objects() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list returns one oid fast; pack-objects hangs forever. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the pack-objects \ + stage must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung pack-objects must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung pack-objects must abort with GitServiceTimeout, got: {err}" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 6336657efcdaf3babc8234224e78e5867d2ebf25 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:20:18 -0500 Subject: [PATCH 07/52] docs(node,config): reconcile the concurrency-cap comments with the fix (#62) The max_concurrent_git_ops and git_service_timeout_secs doc-comments (and .env.example) described the info/refs and withheld-blob paths as unbounded follow-up gaps. Both are now closed (#174): the comments reflect the read/write pool split, the per-caller sub-cap, and that every capped path is duration-bounded with process-group teardown. Verified the pattern-doc pre-ship checklist: every git_permit / write-permit / per-caller site holds only a timeout+teardown git path, and all three size knobs are range(1..=1_048_576). Closes the #174 work. No behavior change. --- .env.example | 4 ++-- crates/gitlawb-node/src/config.rs | 32 +++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.env.example b/.env.example index d7c6a30b..69865920 100644 --- a/.env.example +++ b/.env.example @@ -109,8 +109,8 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Max seconds a served git upload-pack / receive-pack (clone / push) may run # before it is aborted with a 504. Bounds a hung git that would otherwise pin a -# worker and, on push, the repo write lock. Does NOT cover the info/refs -# advertisement or the withheld-blob path, which remain unbounded. Default 600. +# worker and, on push, the repo write lock. Also bounds the info/refs +# advertisement and the withheld-blob pack build (#174). Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 # Max concurrent git-receive-pack (push) operations, in a pool separate from the diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index ff2c2556..93c49449 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -172,9 +172,9 @@ pub struct Config { /// its process group torn down, in seconds. Bounds a git that neither /// finishes nor disconnects. Must be positive; set it very large to /// effectively disable the bound. Default: 600s (10 min), generous for large - /// clones. Does not cover the ref advertisement (`info/refs`) or the - /// withheld-blob fetch path (`upload_pack_excluding`, a blocking - /// `spawn_blocking` a tokio timeout cannot cancel); both remain unbounded. + /// clones. Also bounds the ref advertisement (`info/refs`) and the withheld-blob + /// pack build (`upload_pack_excluding`'s pack-objects stage), which now share the + /// same timeout + process-group teardown (#174). #[arg( long, env = "GITLAWB_GIT_SERVICE_TIMEOUT_SECS", @@ -243,20 +243,20 @@ pub struct Config { /// cap is a different axis (500 connections each fan out to git + /// pack-objects + threads). Size below the process budget with headroom. /// - /// A permit is held for the whole op. upload-pack/receive-pack are - /// duration-bounded by `git_service_timeout_secs`, but the `info/refs` - /// advertisement and the withheld-blob (`upload_pack_excluding`) path are - /// not, so a hung git on those two paths holds its slot until it exits, so a - /// stuck advertisement permanently costs one unit of capacity. Bounding - /// those paths' duration is tracked as separate follow-up. + /// This is the READ pool (`git_read_semaphore`): upload-pack and both info/refs + /// advertisements. Authenticated pushes draw from a separate write pool + /// (`max_concurrent_git_pushes`) and each caller is additionally bounded by + /// `max_concurrent_reads_per_caller`, so an anonymous flood can neither shed a + /// push nor monopolize reads (#174). /// - /// The same two paths have the mirror gap on cancellation: the permit frees - /// when the handler future drops (client disconnect), but neither reaps its - /// git child — `info/refs` runs a bare `Command`, and `upload_pack_excluding` - /// a blocking `spawn_blocking` a dropped future cannot cancel — so the child - /// runs to completion after its slot is freed and live git can briefly exceed - /// the cap. The main pack path (`run_git_service`) tears its process group - /// down on drop, so this is confined to the same two follow-up paths. + /// A permit is held for the whole op, and every capped path is now + /// duration-bounded and reaps its process group on disconnect: upload-pack, + /// receive-pack, and both info/refs advertisements run under + /// `git_service_timeout_secs` with `process_group(0)` teardown, and the + /// withheld-blob (`upload_pack_excluding`) pack-objects stage runs on the async + /// side under the same teardown. So a hung git can no longer pin a slot and a + /// client disconnect no longer orphans its git child (#174 closed the two + /// duration/cancellation gaps this comment previously tracked as follow-up). /// /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value /// well under tokio's `Semaphore` permit limit so an oversized value is a From 5069cd13ca18eeaf7f6be59ace288fa30ca35472 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:45:38 -0500 Subject: [PATCH 08/52] fix(node,concurrency): resolve #174 code-review findings Review of the served-git concurrency cap found no P0/P1; these are the verified P2 follow-ups. config: the max_concurrent_git_ops doc overclaimed that "every capped path is duration-bounded." The rev-list object enumeration in the withheld-blob path still runs in an uncancellable spawn_blocking, so a stuck rev-list can hold its slot until git exits. Scope the guarantee to the streaming stages and name the residual. Also tighten the fairness claim: the receive-pack advertisement shares the read pool (a shed advertisement is a cheap retryable GET); only the push POST is on the isolated write pool. api/repos: add info_refs_per_caller_cap_keys_on_did_not_ip, the missing handler proof that a signed caller is keyed by its DID, not its source IP. Filling the DID slot sheds a request from a free IP; collapsing read_caller_key to its IP arm turns the assertion green-not-503 (mutation-verified RED). api/repos: extract acquire_read_caller_permit so both read handlers share one shed path instead of a duplicated match block. rate_limit: recover from a poisoned PerCallerConcurrency mutex instead of panicking. The critical section is pure counter arithmetic and cannot poison the lock, but a panic there would brick the limiter for every caller. 505 tests pass; clippy -D warnings and fmt clean. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 127 +++++++++++++++++++++----- crates/gitlawb-node/src/config.rs | 20 ++-- crates/gitlawb-node/src/rate_limit.rs | 9 +- 3 files changed, 122 insertions(+), 34 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index dba8ea71..f739c04b 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -575,18 +575,12 @@ pub async fn git_info_refs( peer, state.push_limiter_trust, ); - let _caller_permit = match &caller_key { - Some(k) => match state.git_read_per_caller.try_acquire(k) { - Some(p) => Some(p), - None => { - tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding info/refs with 503"); - return Err(AppError::Overloaded( - "git read service at capacity for this caller, retry shortly".into(), - )); - } - }, - None => None, - }; + let _caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "info/refs", + )?; // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. @@ -656,6 +650,30 @@ fn read_caller_key( } } +/// Acquire the per-caller read sub-cap permit (#174), or shed with a 503. `key` is +/// `None` when no caller key resolves — that request is bounded by the global read +/// pool only and is never shed here (returns `Ok(None)`). `handler` labels the shed +/// log line. Shared by both read handlers so the two acquire sites cannot drift. +fn acquire_read_caller_permit( + limiter: &crate::rate_limit::PerCallerConcurrency, + key: Option<&str>, + repo: &str, + handler: &str, +) -> Result> { + match key { + Some(k) => match limiter.try_acquire(k) { + Some(p) => Ok(Some(p)), + None => { + tracing::warn!(repo = %repo, caller = %k, handler, "per-caller read cap reached; shedding with 503"); + Err(AppError::Overloaded( + "git read service at capacity for this caller, retry shortly".into(), + )) + } + }, + None => Ok(None), + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -712,18 +730,12 @@ pub async fn git_upload_pack( // visibility-denied caller never consumes a scarce read slot. Keyed per-DID // when signed, else per-source-IP; no resolvable key -> global read pool only. let caller_key = read_caller_key(caller, &headers, peer, state.push_limiter_trust); - let _caller_permit = match &caller_key { - Some(k) => match state.git_read_per_caller.try_acquire(k) { - Some(p) => Some(p), - None => { - tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding upload-pack with 503"); - return Err(AppError::Overloaded( - "git read service at capacity for this caller, retry shortly".into(), - )); - } - }, - None => None, - }; + let _caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "upload-pack", + )?; let disk_path = state .repo_store @@ -2798,6 +2810,73 @@ mod tests { ); } + /// #174 SC2 (per-DID key): a SIGNED caller is keyed by its DID, not its source + /// IP. Fill the DID's single read slot, then drive a request carrying that + /// `AuthenticatedDid` from an IP whose own slot is free — it must shed 503, + /// proving the key is the DID. A second request from the SAME IP but a DIFFERENT + /// DID is not shed. Collapse `read_caller_key` to its IP arm and the first + /// assertion goes green-not-503 (the IP slot is free) — this is the per-DID half + /// of the keying proof that the per-IP probes above do not cover. + #[sqlx::test] + async fn info_refs_per_caller_cap_keys_on_did_not_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcdid", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let did_a = "did:key:z6MkPerCallerKeyingProofDidAAAAAAAAAAAAAAAA"; + let did_b = "did:key:z6MkPerCallerKeyingProofDidBBBBBBBBBBBBBBBB"; + let peer: SocketAddr = "203.0.113.51:5000".parse().unwrap(); + + // Fill DID_A's single read slot; the IP's own slot is left untouched. + let _slot = state + .git_read_per_caller + .try_acquire(did_a) + .expect("first slot for this DID"); + + // Signed as DID_A, from `peer` (whose IP slot is free): keyed by DID -> shed. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller at its per-DID read cap must shed with 503 (keyed by DID, not the free IP slot)" + ); + + // Same IP, DIFFERENT DID -> its own budget -> not shed by DID_A's saturation. + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(peer)); + req2.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_b.to_string())); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different DID from the same IP must not be shed by another DID's saturated budget" + ); + } + /// #174 SC2 (None-key): a request with no resolvable caller key (no ConnectInfo, /// no trusted header) must NOT be shed by the per-caller cap even when another /// caller's budget is full — it is bounded by the global read pool only. A None diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 93c49449..90a061e0 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -244,19 +244,23 @@ pub struct Config { /// pack-objects + threads). Size below the process budget with headroom. /// /// This is the READ pool (`git_read_semaphore`): upload-pack and both info/refs - /// advertisements. Authenticated pushes draw from a separate write pool - /// (`max_concurrent_git_pushes`) and each caller is additionally bounded by - /// `max_concurrent_reads_per_caller`, so an anonymous flood can neither shed a - /// push nor monopolize reads (#174). + /// advertisements. The authenticated push POST draws from a separate write pool + /// (`max_concurrent_git_pushes`) that anonymous reads can never reach, and each + /// caller is additionally bounded by `max_concurrent_reads_per_caller`, so an + /// anonymous flood cannot shed the actual push nor monopolize reads (#174). (The + /// receive-pack advertisement itself shares the read pool; a shed advertisement + /// is a cheap retryable GET, and the write POST it precedes always has capacity.) /// - /// A permit is held for the whole op, and every capped path is now + /// A permit is held for the whole op. Every git subprocess that STREAMS is /// duration-bounded and reaps its process group on disconnect: upload-pack, /// receive-pack, and both info/refs advertisements run under /// `git_service_timeout_secs` with `process_group(0)` teardown, and the /// withheld-blob (`upload_pack_excluding`) pack-objects stage runs on the async - /// side under the same teardown. So a hung git can no longer pin a slot and a - /// client disconnect no longer orphans its git child (#174 closed the two - /// duration/cancellation gaps this comment previously tracked as follow-up). + /// side under the same teardown (#174 closed the two duration/cancellation gaps + /// this comment previously tracked). The one remaining blocking stage is the + /// `rev-list` object enumeration in the withheld-blob path — a bounded walk that + /// terminates, run off the async runtime; it is not process-group-reaped on + /// disconnect, so a stuck rev-list can still hold its slot until git exits. /// /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value /// well under tokio's `Semaphore` permit limit so an oversized value is a diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d505f1bb..22281691 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -178,7 +178,12 @@ impl PerCallerConcurrency { /// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is /// rejected without allocating. pub fn try_acquire(&self, key: &str) -> Option { - let mut map = self.state.lock().expect("per-caller mutex poisoned"); + // Recover from a poisoned lock rather than panicking: the critical section + // is pure counter arithmetic and cannot itself panic, so a poisoned mutex + // would only ever come from an unrelated abort, and a slightly-off count + // self-heals as permits drop. A panic here would instead brick the limiter + // for every caller (each subsequent lock re-panics). + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); match map.get_mut(key) { Some(count) => { if *count >= self.per_caller { @@ -207,7 +212,7 @@ impl PerCallerConcurrency { impl Drop for PerCallerPermit { fn drop(&mut self) { - let mut map = self.state.lock().expect("per-caller mutex poisoned"); + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); if let Some(count) = map.get_mut(&self.key) { *count -= 1; if *count == 0 { From 3b1fa54b73c2af9280b89042946cb4c7b26892e4 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 16:05:29 -0500 Subject: [PATCH 09/52] docs(config): add GITLAWB_MAX_CONCURRENT_GIT_OPS to .env.example (#174) The read-pool knob was referenced by the push and per-caller entries' comments but had no example line of its own, so operators couldn't discover it from the template. Add it with the config default (128). --- .env.example | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.env.example b/.env.example index 69865920..cbf3db3a 100644 --- a/.env.example +++ b/.env.example @@ -113,6 +113,13 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # advertisement and the withheld-blob pack build (#174). Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max concurrent git read ops (upload-pack + info/refs advertisements) served at +# once, a global pool separate from the push pool below. Over-cap sheds a clean +# 503 + Retry-After. Anonymous reads draw from here, so pair it with +# GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one caller cannot monopolize +# the pool. Default 128. +GITLAWB_MAX_CONCURRENT_GIT_OPS=128 + # Max concurrent git-receive-pack (push) operations, in a pool separate from the # read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an # authenticated push at admission. Over-cap sheds a 503 + Retry-After. Default 32. From 362ee3462e84015913c3f57012e29a972e6c1d8d Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 13:02:24 -0500 Subject: [PATCH 10/52] fix(node,git): bound the rev-list stage under drive_git_child so a hung enumeration can't pin a git permit (#174) --- crates/gitlawb-node/src/git/smart_http.rs | 108 +++++++++++++++------- 1 file changed, 77 insertions(+), 31 deletions(-) diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 5586a2c7..206e7b27 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -6,7 +6,7 @@ use bytes::Bytes; use std::collections::HashSet; use std::path::Path; use std::process::Stdio; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; @@ -349,28 +349,27 @@ fn pkt_line(data: &str) -> Vec { format!("{len:04x}{data}").into_bytes() } -/// Run `rev-list --objects --all` and return the reachable object ids minus the -/// withheld blobs. Blocking (shells out to git); the caller runs it off the async -/// runtime. Kept separate from the pack-objects stage so that stage can live on the -/// async side under [`drive_git_child`] (`git_bin` is injectable for the same -/// fake-git testing reason as `run_git_service`). -fn rev_list_keep( +/// Run `rev-list --objects --all` under `timeout` and return the reachable object +/// ids minus the withheld blobs. Runs under [`drive_git_child`] (async, with the +/// `tokio::time::timeout` + `process_group(0)` + [`KillGroupOnDrop`] teardown), so a +/// hung/slow enumeration is duration-bounded and reaped on client disconnect just +/// like the pack-objects stage. A bare blocking `Command::output()` inside a +/// `spawn_blocking` is uncancellable, so a hung rev-list would pin the endpoint's +/// concurrency permit for the whole hang (#174). `git_bin` is injectable for the +/// same fake-git testing reason as `run_git_service`. +async fn rev_list_keep( git_bin: &str, repo_path: &Path, withheld: &HashSet, + timeout: Duration, ) -> Result> { - let rev = std::process::Command::new(git_bin) + let mut command = Command::new(git_bin); + command .args(["rev-list", "--objects", "--all"]) - .current_dir(repo_path) - .output()?; - if !rev.status.success() { - bail!( - "git rev-list failed: {}", - String::from_utf8_lossy(&rev.stderr) - ); - } + .current_dir(repo_path); + let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list").await?; let mut keep = Vec::new(); - for line in String::from_utf8_lossy(&rev.stdout).lines() { + for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); if oid.is_empty() || withheld.contains(oid) { continue; @@ -384,32 +383,41 @@ fn rev_list_keep( /// given blob OIDs. Commits and trees are always included, so SHAs stay intact; /// only the named blobs are dropped. /// -/// The rev-list enumeration runs blocking off the async runtime; the streaming -/// `pack-objects` stage runs under [`drive_git_child`], so a hung/slow pack build is -/// duration-bounded and its process group is reaped on client disconnect. An outer -/// `tokio::time::timeout` around a `spawn_blocking` cannot cancel the blocking -/// thread, so the streaming stage MUST live on the async side (#174, KTD5). +/// Both git stages — the `rev-list` enumeration and the streaming `pack-objects` +/// build — run under [`drive_git_child`] sharing one deadline, so a hung/slow git at +/// either stage is duration-bounded and its process group is reaped on client +/// disconnect. An outer `tokio::time::timeout` around a `spawn_blocking` cannot +/// cancel the blocking thread, so neither stage may live off the async side +/// (#174, KTD5). pub async fn build_filtered_pack( git_bin: &str, repo_path: &Path, withheld: &HashSet, timeout: Duration, ) -> Result> { - let keep = { - let git_bin = git_bin.to_string(); - let repo_path = repo_path.to_path_buf(); - let withheld = withheld.clone(); - tokio::task::spawn_blocking(move || rev_list_keep(&git_bin, &repo_path, &withheld)) - .await - .context("rev-list task panicked")?? - }; + // One deadline spans both git stages so a slow rev-list eats into the pack + // budget rather than granting each stage a fresh `timeout` (2x the permit hold). + let deadline = Instant::now() + timeout; + let keep = rev_list_keep( + git_bin, + repo_path, + withheld, + deadline.saturating_duration_since(Instant::now()), + ) + .await?; let mut data = keep.join("\n").into_bytes(); data.push(b'\n'); let mut command = Command::new(git_bin); command .args(["pack-objects", "--stdout"]) .current_dir(repo_path); - drive_git_child(command, Bytes::from(data), timeout, "pack-objects").await + drive_git_child( + command, + Bytes::from(data), + deadline.saturating_duration_since(Instant::now()), + "pack-objects", + ) + .await } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -1327,6 +1335,44 @@ mod tests { ); } + // #174 (SC3, KTD5): the filtered-pack build's rev-list ENUMERATION stage is now + // duration-bounded on the ASYNC side too. It previously ran inside a + // spawn_blocking via a bare `Command::output()`, which an outer tokio timeout + // cannot cancel, so a hung/slow rev-list pinned the endpoint's concurrency permit + // for the whole hang. A fake git that hangs on rev-list must now abort with + // GitServiceTimeout; the watchdog turns a missing bound into a loud failure. + // rev-list runs under drive_git_child, so its disconnect/group-teardown is the + // same code proven by run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_rev_list() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list hangs forever; pack-objects would return fast if it were reached. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 300 ;;\n pack-objects) printf '' ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the rev-list \ + stage must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung rev-list must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung rev-list must abort with GitServiceTimeout, got: {err}" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 4983b854d1a25c43354d08f420490555e4bce728 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 23:44:13 -0500 Subject: [PATCH 11/52] fix(node): key the per-caller read cap on source IP, not the signed DID (#174) read_caller_key returned the authenticated DID when a caller signed, dropping the source-IP key. Public read routes accept any valid did:key via optional_signature with no admission step, so one host could mint N disposable DIDs and hold max_concurrent_reads_per_caller slots under each, multiplying its budget N-fold and filling the global read pool, while the same host unsigned was capped on its IP. Key the read sub-cap on the resolved source IP for every caller, signed or not, mirroring the push path's IpRateLimiter which already throttles on source IP for this exact DID-farm reason. Drops the now-unused caller_did parameter at both call sites (git_info_refs, git_upload_pack). Inverts info_refs_per_caller_cap_keys_on_did_not_ip into info_refs_per_caller_cap_keys_on_ip_not_did: fill one source IP's slot, then two requests signed under different DIDs from that same IP both shed 503 (farm defeated), while a signed request from a different IP keeps its own budget. RED on the DID-keyed tree, GREEN after. --- crates/gitlawb-node/src/api/repos.rs | 91 ++++++++++++++++------------ 1 file changed, 53 insertions(+), 38 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index f739c04b..e95910ef 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -569,12 +569,7 @@ pub async fn git_info_refs( // Per-caller read sub-cap (#174): acquired AFTER the visibility + push-rate // gates (KTD7) so a denied or rate-limited request never consumes a caller's // scarce read slot. Applies to BOTH advertisements. Held for the whole op. - let caller_key = read_caller_key( - auth.as_ref().map(|e| e.0 .0.as_str()), - &headers, - peer, - state.push_limiter_trust, - ); + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = acquire_read_caller_permit( &state.git_read_per_caller, caller_key.as_deref(), @@ -633,21 +628,21 @@ fn git_permit( }) } -/// Resolve the per-caller key for the read sub-cap (#174): the authenticated DID -/// when the caller signed (via `optional_signature`), else the trusted client IP -/// (`client_key`). `None` when neither is available — such a request is bounded by -/// the global read pool only, never a 500. The per-source-IP key is only as -/// granular as `trust`; see the `max_concurrent_reads_per_caller` config doc. +/// Resolve the per-caller key for the read sub-cap (#174): always the resolved +/// source IP (`client_key`), never the signed DID. Public read routes accept any +/// valid `did:key` via `optional_signature` with no admission step, so keying on +/// the DID would let one host mint disposable DIDs to multiply its per-source +/// budget; the push path already throttles on the resolved source IP for exactly +/// this DID-farm reason (`rate_limit.rs`, `IpRateLimiter`). `None` when no key +/// resolves (no trusted header and no peer): such a request is bounded by the +/// global read pool only, never a 500. The per-source-IP key is only as granular +/// as `trust`; see the `max_concurrent_reads_per_caller` config doc. fn read_caller_key( - caller_did: Option<&str>, headers: &axum::http::HeaderMap, peer: Option, trust: crate::rate_limit::TrustedProxy, ) -> Option { - match caller_did { - Some(did) => Some(did.to_string()), - None => crate::rate_limit::client_key(headers, peer, trust), - } + crate::rate_limit::client_key(headers, peer, trust) } /// Acquire the per-caller read sub-cap permit (#174), or shed with a 503. `key` is @@ -727,9 +722,10 @@ pub async fn git_upload_pack( } // Per-caller read sub-cap (#174): after the visibility gate (KTD7) so a - // visibility-denied caller never consumes a scarce read slot. Keyed per-DID - // when signed, else per-source-IP; no resolvable key -> global read pool only. - let caller_key = read_caller_key(caller, &headers, peer, state.push_limiter_trust); + // visibility-denied caller never consumes a scarce read slot. Keyed on the + // resolved source IP (never the signed DID, #174 U1); no resolvable key -> + // global read pool only. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = acquire_read_caller_permit( &state.git_read_per_caller, caller_key.as_deref(), @@ -2810,15 +2806,16 @@ mod tests { ); } - /// #174 SC2 (per-DID key): a SIGNED caller is keyed by its DID, not its source - /// IP. Fill the DID's single read slot, then drive a request carrying that - /// `AuthenticatedDid` from an IP whose own slot is free — it must shed 503, - /// proving the key is the DID. A second request from the SAME IP but a DIFFERENT - /// DID is not shed. Collapse `read_caller_key` to its IP arm and the first - /// assertion goes green-not-503 (the IP slot is free) — this is the per-DID half - /// of the keying proof that the per-IP probes above do not cover. + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the + /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot + /// multiply its budget. Fill the source IP's single read slot, then drive two + /// requests signed under DIFFERENT DIDs from that SAME IP: both must shed 503 + /// (keyed by the saturated IP, not their own free DID slots). A signed request + /// from a DIFFERENT source IP keeps its own budget. Revert `read_caller_key` to + /// prefer the DID and the same-IP assertions go green-not-503 (each fresh DID + /// gets a free slot) -- the farm-defeat mutation probe. #[sqlx::test] - async fn info_refs_per_caller_cap_keys_on_did_not_ip(pool: sqlx::PgPool) { + async fn info_refs_per_caller_cap_keys_on_ip_not_did(pool: sqlx::PgPool) { use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; @@ -2830,7 +2827,7 @@ mod tests { state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; state .db - .upsert_mirror_repo("z6pcdid", "pc", "/tmp/pc-nonexistent", None, false) + .upsert_mirror_repo("z6pcip", "pc", "/tmp/pc-nonexistent", None, false) .await .unwrap(); @@ -2838,17 +2835,17 @@ mod tests { let did_b = "did:key:z6MkPerCallerKeyingProofDidBBBBBBBBBBBBBBBB"; let peer: SocketAddr = "203.0.113.51:5000".parse().unwrap(); - // Fill DID_A's single read slot; the IP's own slot is left untouched. + // Fill the SOURCE IP's single read slot; both DIDs' own slots stay free. let _slot = state .git_read_per_caller - .try_acquire(did_a) - .expect("first slot for this DID"); + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this source IP"); - // Signed as DID_A, from `peer` (whose IP slot is free): keyed by DID -> shed. + // Signed as DID_A from `peer`: keyed by the saturated source IP -> shed 503. let router = crate::server::build_router(state.clone()); let mut req = Request::builder() .method(Method::GET) - .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") .body(Body::empty()) .unwrap(); req.extensions_mut().insert(ConnectInfo(peer)); @@ -2857,23 +2854,41 @@ mod tests { assert_eq!( router.oneshot(req).await.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE, - "a signed caller at its per-DID read cap must shed with 503 (keyed by DID, not the free IP slot)" + "a signed caller must be keyed by its source IP, not its DID: the saturated IP must shed it 503" ); - // Same IP, DIFFERENT DID -> its own budget -> not shed by DID_A's saturation. + // Same IP, a DIFFERENT DID: still keyed by the same saturated IP -> also shed. + // The farm defeat: minting a fresh DID buys no fresh per-source budget. let router2 = crate::server::build_router(state.clone()); let mut req2 = Request::builder() .method(Method::GET) - .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") .body(Body::empty()) .unwrap(); req2.extensions_mut().insert(ConnectInfo(peer)); req2.extensions_mut() .insert(crate::auth::AuthenticatedDid(did_b.to_string())); - assert_ne!( + assert_eq!( router2.oneshot(req2).await.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE, - "a different DID from the same IP must not be shed by another DID's saturated budget" + "a second DID from the same source IP must also shed 503: a DID farm cannot multiply the per-source budget" + ); + + // A signed caller from a DIFFERENT source IP keeps its own budget -> not shed. + let other: SocketAddr = "203.0.113.52:5000".parse().unwrap(); + let router3 = crate::server::build_router(state.clone()); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(other)); + req3.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller from a different source IP must keep its own per-source budget" ); } From e444bc0ae5f3813c93c7e57b147c6ff974f5f497 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 23:54:21 -0500 Subject: [PATCH 12/52] fix(node): draw the receive-pack advertisement from the write pool (#174) git_info_refs acquired git_read_semaphore for BOTH services, so the push handshake (GET /info/refs?service=git-receive-pack) competed in the global read pool. An anonymous clone flood could exhaust that pool and shed a legitimate push with 503 during its required advertisement phase, before it ever reached git_write_semaphore on the POST. The write pool exists precisely so anonymous reads cannot shed an authenticated push, but only the POST drew from it. Select the pool by service: the receive-pack advertisement (phase one of a push) now draws from git_write_semaphore, like the git-receive-pack POST, so a saturated read pool cannot starve it. The per-IP push_rate_limiter that already brakes the advertisement stays as the anti-flood control, and the advertisement stays reader-visible with no new auth requirement. Because the receive-pack branch is now a write-path op, it no longer consumes a read per-caller slot. Handler-layer proofs: with the read pool at zero the receive-pack advertisement survives while the upload-pack advertisement sheds; with the write pool at zero the receive-pack advertisement sheds while upload-pack is unaffected; and a receive-pack advertisement from an IP whose read per-caller budget is full still gets through (mutation-checked, RED when the skip is neutralized). --- crates/gitlawb-node/src/api/repos.rs | 164 ++++++++++++++++++++++++--- 1 file changed, 148 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index e95910ef..3be94bfc 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -507,10 +507,21 @@ pub async fn git_info_refs( headers: axum::http::HeaderMap, auth: Option>, ) -> Result { - // Shed with a 503 before spawning git when the concurrency cap is saturated; - // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_read_semaphore)?; let name = smart_http_repo_name(&repo)?; + let service = query + .service + .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. The + // receive-pack advertisement is phase one of a push, so it draws from the WRITE + // pool (like the git-receive-pack POST), not the global read pool an anonymous + // clone flood can exhaust and thereby starve the push handshake (#174 U2). The + // upload-pack advertisement stays on the read pool with its per-caller sub-cap. + let _permit = if service == "git-receive-pack" { + git_permit(&state.git_write_semaphore)? + } else { + git_permit(&state.git_read_semaphore)? + }; tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state .db @@ -524,9 +535,6 @@ pub async fn git_info_refs( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } - let service = query - .service - .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; tracing::debug!(service = %service, repo = %name, "info/refs service"); // Enforce read visibility on the ref advertisement, for BOTH services. The @@ -566,16 +574,23 @@ pub async fn git_info_refs( } } - // Per-caller read sub-cap (#174): acquired AFTER the visibility + push-rate - // gates (KTD7) so a denied or rate-limited request never consumes a caller's - // scarce read slot. Applies to BOTH advertisements. Held for the whole op. - let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); - let _caller_permit = acquire_read_caller_permit( - &state.git_read_per_caller, - caller_key.as_deref(), - name, - "info/refs", - )?; + // Per-caller read sub-cap (#174): upload-pack advertisement only. The + // receive-pack advertisement is a write-path op (it drew from git_write_semaphore + // above and is braked per-IP by push_rate_limiter), so it must NOT consume a + // read caller's scarce slot (#174 U2). Acquired AFTER the visibility + push-rate + // gates (KTD7) so a denied or rate-limited request never consumes a slot. Held + // for the whole op. + let _caller_permit = if service == "git-receive-pack" { + None + } else { + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "info/refs", + )? + }; // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. @@ -2693,6 +2708,123 @@ mod tests { ); } + /// #174 U2: the receive-pack advertisement (`GET info/refs?service=git-receive-pack`) + /// is phase one of a push, so it draws from the WRITE pool, not the global read + /// pool an anonymous clone flood can exhaust. Proven at the handler by saturating + /// each pool to zero and checking who shares it (INV-10). Revert the branch to the + /// read pool and the read-saturated receive-pack assertion goes 503 (the exact + /// push-handshake starvation jatmn flagged). + #[sqlx::test] + async fn receive_pack_advertisement_draws_from_write_pool(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + // Build a fresh state with the two pools sized independently, then drive one + // info/refs advertisement for `service` and return its handler status. + async fn advert_status( + pool: &sqlx::PgPool, + read_permits: usize, + write_permits: usize, + service: &str, + ) -> StatusCode { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_read_semaphore = Arc::new(Semaphore::new(read_permits)); + state.git_write_semaphore = Arc::new(Semaphore::new(write_permits)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6wpadv", "wp", "/tmp/wp-nonexistent", None, false) + .await + .unwrap(); + let peer: SocketAddr = "203.0.113.61:6000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/z6wpadv/wp/info/refs?service={service}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + router.oneshot(req).await.unwrap().status() + } + + // Read pool saturated, write pool free: the push handshake SURVIVES. + assert_ne!( + advert_status(&pool, 0, 8, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must draw from the write pool: a saturated read pool must not shed it" + ); + // Write pool saturated, read pool free: the receive-pack advertisement SHEDS, + // proving it now consumes the write pool (not the read pool). + assert_eq!( + advert_status(&pool, 8, 0, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement draws from the write pool, so a saturated write pool sheds it 503" + ); + // Read pool saturated: the upload-pack advertisement still SHEDS (unchanged). + assert_eq!( + advert_status(&pool, 0, 8, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement stays on the read pool: a saturated read pool sheds it 503" + ); + // Write pool saturated, read pool free: the upload-pack advertisement is + // UNAFFECTED, proving reads never touch the write pool in either direction. + assert_ne!( + advert_status(&pool, 8, 0, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement never touches the write pool" + ); + } + + /// #174 U2: the receive-pack advertisement is a write-path op, so it must not be + /// shed by the READ per-caller sub-cap even when the caller's source IP has + /// exhausted its read budget (e.g. concurrent clones from the same host). Fill + /// the IP's read per-caller slot, then the receive-pack advertisement from that + /// same IP must still get through. Restore the unconditional read-cap acquire on + /// the receive-pack branch and this goes 503. + #[sqlx::test] + async fn receive_pack_advertisement_ignores_read_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6wpc", "wp", "/tmp/wp-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.71:6000".parse().unwrap(); + // Exhaust the source IP's single READ per-caller slot, as concurrent clones + // from the same host would. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("fill the IP's read per-caller slot"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6wpc/wp/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must not be shed by the read per-caller cap: it is a write-path op" + ); + } + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that /// is already at its concurrency budget on the upload-pack advertisement, while /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and From 71389e621a233a30c97f959b9d5ccc5c9f1bd57b Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:14:39 -0500 Subject: [PATCH 13/52] fix(node): bound the withheld-blob walk at the shared blob_paths seam (#174) The withheld-blob classification walk (blob_paths) fanned out blocking git children with no deadline and no process-group teardown: git for-each-ref, git cat-file, git rev-list, a git ls-tree per commit, and an uncounted git rev-parse (via store::head_commit). A hung or pathologically slow child pinned the caller's served-git permit for the whole hang, and on client disconnect the spawn_blocking task and its git children ran on, orphaned. blob_paths is the shared core of five callers: the upload-pack serve path (holds a read permit) AND, inside git_receive_pack, the post-push replication and encrypt-then-pin walks (hold the write permit U2 reserves for pushes). So the same unbounded walk could pin either pool, and leaving the write-side twin unbounded would have made U2's reservation a claim that does not match behavior. Bound every git child at the blob_paths spawn seam on the blocking side: each child runs in its own process group with a watchdog thread that SIGTERMs (then SIGKILLs) the group on one shared deadline spanning the whole walk, and retains admission until the group is reaped. This is the blocking-side counterpart of smart_http::drive_git_child (spawn_blocking cannot be cancelled by an async timeout). blob_paths stays sync, so all five callers keep their signatures and the 32 classification tests are unchanged; because every caller funnels through blob_paths, one seam bounds both the serve and replication paths. The previously unbounded store::head_commit child becomes a bounded git rev-parse inside the walk. A walk that hits its deadline carries GitServiceTimeout, which the serve handler now maps to 504 rather than a generic 500. Proof: a fake git that hangs on rev-list makes blob_paths return GitServiceTimeout within the watchdog budget (not block on the child) and the recorded process-group leader is reaped, not orphaned; neutralizing the watchdog kill makes it hang past the budget (RED). The 32 real-git classification tests stay green through the refactor, including detached-HEAD, non-standard-ref, and deleted-in-history cases. --- crates/gitlawb-node/src/api/repos.rs | 4 +- .../gitlawb-node/src/git/visibility_pack.rs | 333 +++++++++++++----- 2 files changed, 242 insertions(+), 95 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 3be94bfc..7a8aa5c2 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -781,7 +781,9 @@ pub async fn git_upload_pack( }) .await .map_err(|e| AppError::Git(e.to_string()))? - .map_err(|e| AppError::Git(e.to_string()))? + // A walk that hit its deadline carries GitServiceTimeout; map it to 504 + // like the smart_http paths, not a generic 500 (#174 U3). + .map_err(|e| git_service_app_error(&e))? }; if withheld.is_empty() { diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index cb70e39c..7b9bda69 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -4,11 +4,122 @@ //! content is held back. use crate::db::VisibilityRule; -use crate::git::store; use crate::visibility::{visibility_check, Decision}; use anyhow::{Context, Result}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::Path; +use std::process::Stdio; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +/// Fixed budget bounding the whole withheld-blob classification walk (#174 U3). +/// The walk is fast for a real repo; this bound exists to reap a hung or +/// pathologically slow git child so it cannot pin a served-git permit (the read +/// permit on the upload-pack serve path, the write permit on the receive-pack +/// post-push replication path) past the deadline. Every caller funnels through +/// `blob_paths`, so bounding here bounds both paths at one seam. +const WALK_TIMEOUT: Duration = Duration::from_secs(600); + +/// Run one git child under a shared `deadline` with process-group teardown, +/// BLOCKING, and return its stdout. The child runs in its own process group; a +/// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, +/// the whole group if the deadline passes before the child is reaped, so a hung or +/// slow git can pin neither a served-git permit nor a blocking thread past the +/// deadline (jatmn's "retain admission until they are reaped"). This is the +/// blocking-side counterpart of `smart_http::drive_git_child`, needed because the +/// walk's callers run it inside `spawn_blocking`, which an async timeout cannot +/// cancel. Returns [`crate::git::smart_http::GitServiceTimeout`] on the deadline so +/// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can +/// drive the teardown in tests without mutating the process-global PATH; +/// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). +#[cfg(unix)] +fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + use std::io::{Read, Write}; + use std::os::unix::process::CommandExt; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + // With process_group(0) the child leads its own group, so pgid == its pid. + let pgid = child.id() as i32; + + // Watchdog: on the deadline, SIGTERM the whole group, poll until it exits, + // escalate to SIGKILL after a grace, and report whether it killed. Signalled to + // stand down when the child is reaped in time. Kept off the main thread because + // the main thread's stdout drain is exactly what blocks until a hung child is + // torn down. + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + for step in 0..200u32 { + if unsafe { libc::kill(-pgid, 0) } != 0 { + return true; // ESRCH: every group member has exited + } + if step == 100 { + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + } + std::thread::sleep(Duration::from_millis(10)); + } + true + } + } + }); + + // Feed stdin on a writer thread and drain stderr on a reader thread so the main + // thread can drain stdout concurrently; writing all of stdin (or draining one + // pipe) before the others can deadlock once a pipe buffer fills. + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut out = Vec::new(); + let read_result = stdout.read_to_end(&mut out); + let status = child.wait().context("git wait failed")?; + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + + // Reaped: stand the watchdog down and learn whether it killed on the deadline. + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); + if killed { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + read_result.context("failed to read git stdout")?; + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} /// Fail closed unless every ref ultimately resolves to a commit (a ref pointing /// directly at a blob or tree, or an annotated tag — even a nested one — of such @@ -21,19 +132,15 @@ use std::path::Path; /// Full peeling is why this is not `for-each-ref %(*objecttype)`, which /// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a- /// commit as a non-commit. -fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { - let refs = std::process::Command::new("git") - .args(["for-each-ref", "--format=%(refname)"]) - .current_dir(repo_path) - .output() - .context("git for-each-ref failed")?; - if !refs.status.success() { - anyhow::bail!( - "git for-each-ref failed: {}", - String::from_utf8_lossy(&refs.stderr) - ); - } - let refs_stdout = String::from_utf8_lossy(&refs.stdout); +fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<()> { + let refs_out = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(refname)"], + repo_path, + b"", + deadline, + )?; + let refs_stdout = String::from_utf8_lossy(&refs_out); let refnames: Vec<&str> = refs_stdout .lines() .map(str::trim) @@ -43,59 +150,25 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { return Ok(()); } - // Peel every ref in one `git cat-file --batch-check` pass: one - // `^{}` query per line, one output line per input line, in order. - // The stdin write runs on a separate thread so this thread can drain stdout - // concurrently. cat-file echoes the full query on a ` missing` line, - // so output scales with refname length (not a fixed size per ref); writing - // all of stdin before reading any stdout would deadlock both pipes once the - // child's stdout buffer fills. Dropping `stdin` at the end of the closure - // sends EOF. + // Peel every ref in one `git cat-file --batch-check` pass: one `^{}` + // query per line, one output line per input line, in order. cat-file echoes the + // full query on a ` missing` line, so output scales with refname length; + // run_bounded_git drains stdout concurrently with the stdin write, so the pipe + // cannot deadlock, and the whole peel is bounded by the shared walk deadline. let queries = refnames .iter() .map(|r| format!("{r}^{{}}")) .collect::>() .join("\n"); - use std::io::Write; - let mut child = std::process::Command::new("git") - .args(["cat-file", "--batch-check=%(objecttype)"]) - .current_dir(repo_path) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .context("failed to spawn git cat-file")?; - // Feed stdin on a writer thread so this thread can drain stdout via - // wait_with_output concurrently; a None handle (the pipe vanished) becomes a - // broken-pipe write error. wait_with_output reaps the child unconditionally - // before any error is surfaced, so no path drops it unwaited (#53), and the - // writer is joined only after the drain so the join cannot deadlock. - let writer = child - .stdin - .take() - .map(|mut stdin| std::thread::spawn(move || stdin.write_all(queries.as_bytes()))); - let peel_result = child.wait_with_output(); - let write_result = match writer { - Some(handle) => handle - .join() - .map_err(|_| anyhow::anyhow!("git cat-file stdin writer thread panicked"))?, - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git cat-file stdin unavailable", - )), - }; - // Surface a write error only if the process didn't already fail with a - // clearer status. - let peel = peel_result.context("git cat-file failed")?; - if !peel.status.success() { - anyhow::bail!( - "git cat-file --batch-check failed: {}", - String::from_utf8_lossy(&peel.stderr) - ); - } - write_result.context("failed to write to git cat-file stdin")?; - - let peel_stdout = String::from_utf8_lossy(&peel.stdout); + let peel_out = run_bounded_git( + git_bin, + &["cat-file", "--batch-check=%(objecttype)"], + repo_path, + queries.as_bytes(), + deadline, + )?; + + let peel_stdout = String::from_utf8_lossy(&peel_out); let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect(); // A short read means at least one ref went unclassified — fail closed. if types.len() != refnames.len() { @@ -147,47 +220,46 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { /// Fails closed: if commit enumeration or any tree walk fails, returns an error so /// the caller aborts the serve/pin rather than producing a partial (under-withheld) /// set. -fn blob_paths(repo_path: &Path) -> Result> { - assert_all_refs_are_commits(repo_path)?; +fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result> { + // One deadline spans the whole walk (the ref check, the HEAD probe, rev-list, + // and every per-commit ls-tree), so a slow or hung walk is bounded as a unit + // rather than granting each git child a fresh timeout. + let deadline = Instant::now() + timeout; + assert_all_refs_are_commits(repo_path, git_bin, deadline)?; // Enumerate every reachable commit, not just ref tips. `--all` walks all refs; // append HEAD so a detached HEAD (reachable by rev-list/upload-pack but in no // ref) is still classified. When HEAD does not resolve (unborn branch on an - // empty repo) `--all` alone yields nothing, which is correct — no objects exist. - let head = store::head_commit(repo_path).context("resolve HEAD failed")?; + // empty repo) `--all` alone yields nothing, which is correct: no objects exist. + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), replacing the previously unbounded `store::head_commit` child. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); let mut rev_args = vec!["rev-list", "--all"]; - if head.is_some() { + if head_resolves { rev_args.push("HEAD"); } - let commits = std::process::Command::new("git") - .args(&rev_args) - .current_dir(repo_path) - .output() - .context("git rev-list --all failed")?; - if !commits.status.success() { - anyhow::bail!( - "git rev-list --all failed: {}", - String::from_utf8_lossy(&commits.stderr) - ); - } - let commits_stdout = String::from_utf8_lossy(&commits.stdout); + let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let commits_stdout = String::from_utf8_lossy(&commits_out); let mut out: HashSet<(String, String)> = HashSet::new(); for commit in commits_stdout.lines() { let commit = commit.trim(); if commit.is_empty() { continue; } - let listing = std::process::Command::new("git") - .args(["ls-tree", "-rz", commit]) - .current_dir(repo_path) - .output() - .context("git ls-tree -rz failed")?; - if !listing.status.success() { - anyhow::bail!( - "git ls-tree -rz {commit} failed: {}", - String::from_utf8_lossy(&listing.stderr) - ); - } + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rz", commit], + repo_path, + b"", + deadline, + )?; // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes // "secret/caf\303\251.txt"), and that quoted literal would not match a @@ -198,7 +270,7 @@ fn blob_paths(repo_path: &Path) -> Result> { // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string // would no longer match its deny rule — the same under-withholding class, one // layer down. Fail closed instead so the caller aborts rather than leaks. - let Ok(listing_stdout) = std::str::from_utf8(&listing.stdout) else { + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { anyhow::bail!( "git ls-tree -rz {commit} returned a non-UTF-8 path; \ refusing to produce a partial (under-withheld) set" @@ -236,7 +308,7 @@ pub fn withheld_blob_oids( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; Ok(withheld_from_pairs( &pairs, rules, is_public, owner_did, caller, )) @@ -332,7 +404,7 @@ pub fn allowed_blob_set_for_caller( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { @@ -372,7 +444,7 @@ pub fn withheld_blob_recipients( owner_did: &str, ) -> Result>> { // One history walk feeds both the withheld set and the recipient mapping. - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None); if withheld.is_empty() { return Ok(HashMap::new()); @@ -402,6 +474,79 @@ pub fn withheld_blob_recipients( #[cfg(test)] mod tests { use super::*; + + /// Write an executable fake `git` shell script into `dir` and return its path, + /// so a test can drive the walk's process-group teardown without a real git and + /// without mutating the process-global PATH (the crate's only injection seam). + #[cfg(unix)] + fn write_fake_git(dir: &Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 U3: the withheld-blob walk is bounded at the shared `blob_paths` seam, so + /// a hung git child cannot pin the caller's permit past the deadline. A fake git + /// that hangs on `rev-list` must make `blob_paths` return `GitServiceTimeout` + /// within the watchdog budget (not block for the child's lifetime), and the + /// child's process group must be reaped (its recorded leader PID gone). Every + /// caller (upload-pack serve, receive-pack replication) funnels through + /// `blob_paths`, so this seam-level proof covers both permit pools. Neutralize + /// the watchdog SIGTERM and this hangs past the recv budget (RED). + #[cfg(unix)] + #[test] + fn blob_paths_times_out_and_reaps_a_hung_walk() { + use std::time::Duration; + let tmp = TempDir::new().unwrap(); + // Fast on every stage except rev-list, which records its own (group-leader) + // PID and then hangs. `sleep 30` bounds the worst case if the watchdog is + // ever broken, so a regression cannot wedge the suite for 300s. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > revlist.pid ; sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + + // Run the walk on a thread with a short budget; the recv_timeout succeeding + // is itself proof the walk did not block on the hung child. + let (tx, rx) = mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(blob_paths(&path, &git_bin, Duration::from_millis(200))); + }); + let result = rx.recv_timeout(Duration::from_secs(10)).expect( + "blob_paths must return within the watchdog budget, not hang on a stuck git child", + ); + let err = result.expect_err("a hung rev-list must abort the walk with an error"); + assert!( + err.downcast_ref::() + .is_some(), + "a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err}" + ); + + // The recorded process-group leader must be gone: the watchdog reaps the + // whole group before blob_paths returns, so no orphaned git lingers. + let pid: i32 = std::fs::read_to_string(tmp.path().join("revlist.pid")) + .expect("the fake git must have recorded its rev-list PID") + .trim() + .parse() + .expect("recorded PID must parse"); + let mut gone = false; + for _ in 0..200 { + // SAFETY: kill(2) with signal 0 only probes existence; ESRCH (-1) means + // the process is gone. Borrows no Rust memory. + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts" + ); + } use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; From ab0ea240fcca3087bb7e3c08302885aede32627a Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:20:07 -0500 Subject: [PATCH 14/52] docs(node): correct the git-service-timeout coverage note (#174) GITLAWB_GIT_SERVICE_TIMEOUT_SECS bounds the info/refs advertisement too: smart_http::info_refs drives it through drive_git_child under this timeout, with a passing test proving the 504. The old note claimed it does not. It also claimed the withheld-blob path is unbounded; after the blob_paths seam bound (this PR) the walk is bounded and reaped, by a fixed internal deadline rather than this env var, so the line now states that precisely instead of overclaiming this setting covers it. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57ce2885..d02c4a0b 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack/receive-pack may run before it is aborted (504). Default 600. Does not bound `info/refs` or the withheld-blob path. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. The withheld-blob classification walk is also bounded and reaped, by a fixed internal deadline rather than this setting. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | From 87a7e40513b018b335d8f3a2e46084d323484798 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:36:42 -0500 Subject: [PATCH 15/52] fix(review): close the withheld-walk watchdog reused-pgid race (#174) Code review found run_bounded_git's watchdog could return a spurious 504 and signal a recycled process group. The watchdog runs off a wall clock on its own thread; done_tx.send() only fires after child.wait() reaps the leader, so a walk that finished within microseconds of the deadline took the watchdog's Timeout branch, discarded a fully-captured successful result, and returned GitServiceTimeout (a 504 for a walk that actually completed). Worse, the Timeout branch SIGTERMed -pgid unconditionally after the leader was reaped, so a recycled pgid could be signalled, the exact hazard smart_http guards via disarm-after-wait. Set a reaped AtomicBool the instant the main thread reaps the child; the watchdog checks it before every kill and stands down if the leader is already reaped. Gate the timeout verdict on !status.success(), so a child that exited on its own is never reported as a timeout even if the watchdog fired late. Add the survived-SIGKILL warn smart_http's reap already emits, for operator visibility on a wedged (D-state) git. The hung-walk test stays green (a killed child exits by signal, not success, so it still surfaces GitServiceTimeout and reaps the group) and the 32 real-git classification tests stay green (a fast walk is never spuriously killed). --- .../gitlawb-node/src/git/visibility_pack.rs | 66 +++++++++++++------ 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 7b9bda69..0d6661e4 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -63,27 +63,46 @@ fn run_bounded_git( // the main thread's stdout drain is exactly what blocks until a hung child is // torn down. let (done_tx, done_rx) = mpsc::channel::<()>(); - let watchdog = std::thread::spawn(move || -> bool { - let wait = deadline.saturating_duration_since(Instant::now()); - match done_rx.recv_timeout(wait) { - Ok(()) | Err(RecvTimeoutError::Disconnected) => false, - Err(RecvTimeoutError::Timeout) => { - // SAFETY: kill(2) takes only integers and borrows no Rust memory; - // ESRCH on an already-gone group is ignored. - unsafe { libc::kill(-pgid, libc::SIGTERM) }; - for step in 0..200u32 { - if unsafe { libc::kill(-pgid, 0) } != 0 { - return true; // ESRCH: every group member has exited + // Set true the instant the main thread reaps the child, so the watchdog never + // SIGTERMs a pgid whose leader is already reaped and whose pid the kernel may + // recycle (the reused-pgid hazard smart_http guards via disarm-after-wait). + let reaped = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let watchdog = { + let reaped = reaped.clone(); + std::thread::spawn(move || -> bool { + use std::sync::atomic::Ordering; + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // The child was reaped right as the deadline fired: do not signal + // its (possibly-recycled) pgid. + if reaped.load(Ordering::SeqCst) { + return false; } - if step == 100 { - unsafe { libc::kill(-pgid, libc::SIGKILL) }; + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + for step in 0..200u32 { + if reaped.load(Ordering::SeqCst) || unsafe { libc::kill(-pgid, 0) } != 0 { + return true; // reaped, or ESRCH: every group member has exited + } + if step == 100 { + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + } + std::thread::sleep(Duration::from_millis(10)); } - std::thread::sleep(Duration::from_millis(10)); + // Survived SIGKILL past the cap: a wedged (D-state) git, the same + // condition smart_http's reap logs. Warn so an operator sees it. + tracing::warn!( + pgid, + "withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)" + ); + true } - true } - } - }); + }) + }; // Feed stdin on a writer thread and drain stderr on a reader thread so the main // thread can drain stdout concurrently; writing all of stdin (or draining one @@ -105,16 +124,21 @@ fn run_bounded_git( let mut out = Vec::new(); let read_result = stdout.read_to_end(&mut out); let status = child.wait().context("git wait failed")?; + // Reaped: bar the watchdog from signalling the now-reaped (possibly recycled) + // pgid before it can fire, then stand it down. + reaped.store(true, std::sync::atomic::Ordering::SeqCst); let err = err_reader.join().unwrap_or_default(); let _ = writer.join(); - - // Reaped: stand the watchdog down and learn whether it killed on the deadline. let _ = done_tx.send(()); let killed = watchdog.join().unwrap_or(false); - if killed { + read_result.context("failed to read git stdout")?; + // The watchdog runs off a wall clock that can race a child finishing right at the + // deadline. A child that exited on its own (success) is not a timeout even if the + // watchdog fired late; only a child that did not exit successfully is a genuine + // timeout, which keeps a walk completing at its budget from a spurious 504. + if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } - read_result.context("failed to read git stdout")?; if !status.success() { anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); } From e750c3042dff5dc0aa8b1959f5bb72ca7be68137 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:56:24 -0500 Subject: [PATCH 16/52] fix(review): cap the receive-pack advertisement per source so anon cannot starve the write pool (#174) U2 moved the receive-pack info/refs advertisement onto git_write_semaphore to keep an anonymous read flood from starving the push handshake. But the advertisement is anon-reachable on public repos and holds its write permit across the slow acquire_fresh Tigris download, and the only per-source brake on it was the push RATE limiter, not a concurrency cap. So a multi-source flood of receive-pack advertisements could hold the write pool's slots across those downloads and shed authenticated pushes (both the advertisement and the owner-gated git-receive-pack POST draw from the same pool). U2 thus introduced the first anonymous consumer of the write pool the state doc promised anon could never reach; the plan's residual note (no worse than the POST) was wrong, because the POST is owner-gated and the advertisement is not. Add git_push_advert_per_caller, a per-source concurrency sub-cap on the receive-pack advertisement keyed on the resolved source IP (the same PerCallerConcurrency mechanism U1 uses for reads), sized to an eighth of the write pool so a single source holds at most that share and saturating the pool takes many distinct source IPs, each also braked by the per-IP push rate limiter. The upload-pack advertisement keeps its read-pool per-caller cap; the owner-gated POST is unchanged. Correct the state doc for git_write_semaphore accordingly. Handler-layer proof: a source at its receive-pack advertisement cap sheds 503 (RED before the acquisition, 500-not-503), while a different source and the upload-pack advertisement are unaffected. Full suite 510 green. --- crates/gitlawb-node/src/api/repos.rs | 101 +++++++++++++++++++++--- crates/gitlawb-node/src/auth/mod.rs | 2 + crates/gitlawb-node/src/main.rs | 7 ++ crates/gitlawb-node/src/state.rs | 22 ++++-- crates/gitlawb-node/src/test_support.rs | 3 + 5 files changed, 121 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7a8aa5c2..796558cf 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -574,16 +574,24 @@ pub async fn git_info_refs( } } - // Per-caller read sub-cap (#174): upload-pack advertisement only. The - // receive-pack advertisement is a write-path op (it drew from git_write_semaphore - // above and is braked per-IP by push_rate_limiter), so it must NOT consume a - // read caller's scarce slot (#174 U2). Acquired AFTER the visibility + push-rate - // gates (KTD7) so a denied or rate-limited request never consumes a slot. Held - // for the whole op. + // Per-source concurrency sub-cap (#174), keyed on the resolved source IP and + // acquired AFTER the visibility + push-rate gates (KTD7) so a denied or + // rate-limited request never consumes a slot; held for the whole op. The + // upload-pack advertisement is bounded on the read pool (git_read_per_caller). + // The receive-pack advertisement draws from the write pool, so it is bounded per + // source (git_push_advert_per_caller) instead: without this, an anonymous + // multi-source flood of push-handshake advertisements could hold the write pool's + // slots across acquire_fresh and shed authenticated pushes, since the per-IP push + // rate limiter caps rate, not concurrency (#174 review fix). + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = if service == "git-receive-pack" { - None + acquire_read_caller_permit( + &state.git_push_advert_per_caller, + caller_key.as_deref(), + name, + "receive-pack advert", + )? } else { - let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); acquire_read_caller_permit( &state.git_read_per_caller, caller_key.as_deref(), @@ -676,7 +684,7 @@ fn acquire_read_caller_permit( None => { tracing::warn!(repo = %repo, caller = %k, handler, "per-caller read cap reached; shedding with 503"); Err(AppError::Overloaded( - "git read service at capacity for this caller, retry shortly".into(), + "git service at capacity for this caller, retry shortly".into(), )) } }, @@ -2827,6 +2835,81 @@ mod tests { ); } + /// #174 (review fix): the anon-reachable receive-pack advertisement draws from + /// the write pool, so it is bounded per source by `git_push_advert_per_caller` to + /// stop one source from monopolizing the write pool and shedding authenticated + /// pushes. Fill one source IP's advert slot; its next receive-pack advertisement + /// sheds 503, while a different source and the upload-pack advertisement are + /// unaffected. Remove the advert-cap acquisition and the same-source assertion + /// goes green-not-503. + #[sqlx::test] + async fn receive_pack_advertisement_capped_per_source(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6advcap", "ac", "/tmp/ac-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:6000".parse().unwrap(); + // Fill this source IP's single receive-pack-advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + // Same source: the receive-pack advertisement sheds 503 (advert cap full). + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its receive-pack advertisement cap must shed 503, so it cannot monopolize the write pool" + ); + + // A DIFFERENT source keeps its own advert budget -> not shed. + let other: SocketAddr = "203.0.113.82:6000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must keep its own receive-pack advertisement budget" + ); + + // The upload-pack advertisement is NOT bounded by the receive-pack advert cap. + let router3 = crate::server::build_router(state); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "the upload-pack advertisement must not be shed by the receive-pack advert cap" + ); + } + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that /// is already at its concurrency budget on the upload-pack advertisement, while /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 253b3210..b9e28e57 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -523,6 +523,8 @@ mod tests { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index c617b434..0f33fde4 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -385,6 +385,13 @@ async fn main() -> Result<()> { git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), + // Per-source cap on the receive-pack advertisement, sized to an eighth of the + // write pool (min 1): a single source can hold at most this many write-pool + // slots via the anon advertisement, so saturating the pool takes ~8 distinct + // source IPs, each also rate-limited (#174). + git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + (config.max_concurrent_git_pushes / 8).max(1), + ), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 01007ac4..f55abeb7 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -95,14 +95,26 @@ pub struct AppState { /// authenticated push at admission (#174). pub git_read_semaphore: Arc, /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate - /// from `git_read_semaphore` so anonymous reads can never shed an authenticated - /// push (#174). Sized by `max_concurrent_git_pushes`. + /// from `git_read_semaphore` so an anonymous READ flood can never shed an + /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from + /// by the `git-receive-pack` POST (owner-gated) and the receive-pack `info/refs` + /// advertisement (anon-reachable); the advertisement is additionally bounded per + /// source by `git_push_advert_per_caller` so no single source can monopolize this + /// pool and shed pushes. pub git_write_semaphore: Arc, - /// Per-caller concurrency sub-cap on the read pool: each caller (per-DID when - /// signed, else per-source-IP) may hold at most `max_concurrent_reads_per_caller` + /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the + /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` - /// (#174). Applied by `git_upload_pack` and both `info/refs` advertisements. + /// (#174). Applied by `git_upload_pack` and the upload-pack `info/refs` + /// advertisement. pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the anon-reachable receive-pack `info/refs` + /// advertisement: each source IP may hold at most a small share of the write + /// pool, so a multi-source flood of push-handshake advertisements cannot + /// saturate `git_write_semaphore` and shed authenticated pushes (#174). Sized as + /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many + /// distinct source IPs (each also braked by the per-IP push rate limiter). + pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 7745d233..03d3d5c4 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -86,6 +86,9 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 8, + ), } } From c35452ed41091ba38f1b294a96f44d1d7b827f74 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 02:12:08 -0500 Subject: [PATCH 17/52] test(node): vet the withheld-walk bound end to end; use the configured timeout (#174) Close the reasoned-not-run gaps from the code review by making the walk's git binary and timeout injectable, then driving the missing branches with a real handler and a fake git instead of reasoning about them. - Add state.git_bin and *_bounded variants of the walk entry points taking (git_bin, timeout); the served handlers (upload-pack serve, receive-pack replication and full-scan and encrypt-pin, and the ipfs gate) now pass the operator-configured GITLAWB_GIT_SERVICE_TIMEOUT_SECS, so the whole walk is bounded by the same budget as the other served-git ops rather than a fixed constant. The git_bin-less wrappers stay for the real-git classification tests. Newly vetted by execution (not reasoning): - receive-pack replication path is bounded: replication_withheld_set with an injected hung git returns within the budget and fails closed, so it cannot pin the write permit git_receive_pack holds across it. - a hung withheld-blob walk on the upload-pack POST returns 504 (real handler, real repo on disk, injected hung git), proving the GitServiceTimeout -> git_service_app_error wiring end to end. - the watchdog status-gate: a child that exits successfully is not reported as a timeout even when the watchdog fired (mutation-checked: drop the guard -> RED). - SIGKILL escalation: a SIGTERM-ignoring child is still reaped via SIGKILL and the group is gone; a truly uninterruptible D-state child (unreapable by any signal) is the documented residual, matching the async teardown. - the advertisement per-source cap sizing never derives 0. Full gitlawb-node suite 515 green. --- crates/gitlawb-node/src/api/ipfs.rs | 12 +- crates/gitlawb-node/src/api/repos.rs | 208 +++++++++++++++++- crates/gitlawb-node/src/auth/mod.rs | 1 + .../gitlawb-node/src/git/visibility_pack.rs | 173 ++++++++++++++- crates/gitlawb-node/src/main.rs | 1 + crates/gitlawb-node/src/state.rs | 5 + crates/gitlawb-node/src/test_support.rs | 1 + 7 files changed, 383 insertions(+), 18 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f3de7570..e921c39f 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -27,7 +27,7 @@ use std::str::FromStr; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; use crate::git::store; -use crate::git::visibility_pack::{allowed_blob_set_for_caller, has_path_scoped_rule}; +use crate::git::visibility_pack::{allowed_blob_set_for_caller_bounded, has_path_scoped_rule}; use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; @@ -142,10 +142,16 @@ pub async fn get_by_cid( let is_public = repo.is_public; let owner = repo.owner_did.clone(); let caller_for_walk = caller_owned.clone(); - // Full-history walk shells out to git — keep it off the async runtime. + let git_bin = state.git_bin.clone(); + let walk_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + // Full-history walk shells out to git — keep it off the async runtime, + // bounded and reaped like the served-git ops (#174). let walk = tokio::task::spawn_blocking(move || { - allowed_blob_set_for_caller( + allowed_blob_set_for_caller_bounded( &rp, + &git_bin, + walk_timeout, &r, is_public, &owner, diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 796558cf..1532be58 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -44,6 +44,8 @@ async fn replication_withheld_set( owner_did: &str, is_public: bool, disk_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, ) -> (bool, Option>) { let announce = match &rules { Some(rules) => crate::visibility::listable_at_root(rules, is_public, owner_did, None), @@ -65,8 +67,8 @@ async fn replication_withheld_set( Some(rules) => { let owner_did = owner_did.to_string(); tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_oids( - &disk_path, &rules, is_public, &owner_did, None, + crate::git::visibility_pack::withheld_blob_oids_bounded( + &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, ) }) .await @@ -106,10 +108,12 @@ async fn fail_closed_full_scan_objects( is_public: bool, owner_did: String, candidates: Vec, + git_bin: String, + timeout: std::time::Duration, ) -> Vec { tokio::task::spawn_blocking(move || -> anyhow::Result> { - let allowed = crate::git::visibility_pack::replicable_blob_set( - &disk_path, &rules, is_public, &owner_did, + let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( + &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, )?; let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( @@ -778,9 +782,12 @@ pub async fn git_upload_pack( let owner_did = record.owner_did.clone(); let caller_owned = caller.map(str::to_string); let is_public = record.is_public; + let git_bin = state.git_bin.clone(); tokio::task::spawn_blocking(move || { - visibility_pack::withheld_blob_oids( + visibility_pack::withheld_blob_oids_bounded( &path, + &git_bin, + git_timeout, &rules, is_public, &owner_did, @@ -1185,6 +1192,8 @@ pub async fn git_receive_pack( &record.owner_did, record.is_public, disk_path.clone(), + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await; @@ -1219,6 +1228,8 @@ pub async fn git_receive_pack( record.is_public, record.owner_did.clone(), pin_set.candidates, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await } else { @@ -1244,6 +1255,8 @@ pub async fn git_receive_pack( let node_did_str = state.node_did.to_string(); let node_seed = state.node_keypair.to_seed(); let repo_name = record.name.clone(); + let enc_git_bin = state.git_bin.clone(); + let enc_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); tokio::spawn(async move { let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, @@ -1268,9 +1281,15 @@ pub async fn git_receive_pack( { let p = repo_path_clone.clone(); let owner = owner_did.clone(); + let git_bin = enc_git_bin.clone(); let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients( - &p, &rules, is_public, &owner, + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &p, + &git_bin, + enc_timeout, + &rules, + is_public, + &owner, ) }) .await; @@ -2031,16 +2050,39 @@ mod tests { let dummy = std::path::PathBuf::from("/nonexistent"); // Private: no rules at all. - let (announce, _) = replication_withheld_set(None, OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + None, + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (no rules) must not announce"); // Private: empty rule set, is_public=false → still not listable at root. - let (announce, _) = - replication_withheld_set(Some(vec![]), OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + Some(vec![]), + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (empty rules) must not announce"); // Public: empty rule set, is_public=true → listable at root, announces. - let (announce, _) = replication_withheld_set(Some(vec![]), OWNER_DID, true, dummy).await; + let (announce, _) = replication_withheld_set( + Some(vec![]), + OWNER_DID, + true, + dummy, + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(announce, "public repo must announce"); } @@ -2132,6 +2174,150 @@ mod tests { } } + #[cfg(unix)] + fn write_fake_git(dir: &std::path::Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 (write-pool twin, vetted by execution not reasoning): the receive-pack + /// post-push replication walk is bounded. Drive `replication_withheld_set` with an + /// injected fake git that hangs on `rev-list` and a short budget: it must RETURN + /// within the budget (so `git_receive_pack` releases the write permit it holds + /// across this await, rather than pinning it for the hang) AND fail closed + /// (announce suppressed) because the walk could not be vetted. Proves this path + /// funnels through the bounded `blob_paths`, on the write-permit-holding side. + #[cfg(unix)] + #[tokio::test] + async fn replication_walk_is_bounded_and_fails_closed_on_a_hung_git() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + // Public root (announceable) + a path-scoped rule, so the walk actually runs + // rather than taking the has_path_scoped_rule short-circuit. + let rules = Some(vec![vis_rule("/secret/**", &[])]); + + let result = tokio::time::timeout( + Duration::from_secs(10), + replication_withheld_set( + rules, + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin, + Duration::from_millis(200), + ), + ) + .await + .expect( + "replication_withheld_set must return within the budget; a hung walk must \ + not pin the write permit git_receive_pack holds across it", + ); + assert_eq!( + result, + (false, None), + "a walk that could not be vetted must suppress the announce (fail closed)" + ); + } + + /// #174 (serve-path 504, vetted by execution): a hung withheld-blob walk on the + /// upload-pack POST maps to 504, not a generic 500. Real repo dir on disk (so + /// acquire's fast path returns it) + a path-scoped rule (so the walk runs) + + /// an injected fake git that hangs on rev-list. The handler must return 504, + /// proving git_upload_pack routes the walk's GitServiceTimeout through + /// git_service_app_error end to end. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_hung_withheld_walk_returns_504(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let fake = write_fake_git(tmp.path(), body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 1; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6srv504", "sv", "/tmp/z6srv504-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6srv504", "sv").await.unwrap().unwrap(); + // Path-scoped rule so has_path_scoped_rule() is true and the walk runs; the + // public root still lets an anonymous caller past the "/" gate. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + // acquire()'s fast path returns the local path when it exists on disk. + let disk = std::path::Path::new("/tmp/z6srv504/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.91:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6srv504/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6srv504"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "a hung withheld-blob walk must surface as 504, not a generic 500" + ); + } + + /// #174 (F2 sizing edge, vetted by execution): the receive-pack advertisement + /// per-source cap is derived in main.rs as `(max_concurrent_git_pushes / 8).max(1)`, + /// so it is never 0 even at the minimum write-pool size (1). A 0 cap would make + /// PerCallerConcurrency shed EVERY receive-pack advertisement and break all pushes. + #[test] + fn advert_per_caller_cap_sizing_is_never_zero() { + let cap = |pushes: usize| (pushes / 8).max(1); + for pushes in [1usize, 4, 8, 32, 256] { + assert!( + cap(pushes) >= 1, + "advert cap must be >= 1 for pushes={pushes}" + ); + } + assert_eq!(cap(1), 1, "minimum write pool must derive cap 1, not 0"); + assert_eq!( + cap(32), + 4, + "default write pool 32 derives cap 4 (~8 source IPs to fill)" + ); + // A cap of 1 admits one and sheds the second from the same source. + let lim = crate::rate_limit::PerCallerConcurrency::new(cap(1), 100); + let _held = lim.try_acquire("src").expect("first advert admitted"); + assert!( + lim.try_acquire("src").is_none(), + "second advert from the same source is shed" + ); + } + #[test] fn fork_owner_full_did_with_path_rule_allowed() { // Owner reads everything (implicit reader), so nothing is withheld. diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index b9e28e57..5befa7f3 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -525,6 +525,7 @@ mod tests { git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 0d6661e4..09a80aba 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -17,7 +17,10 @@ use std::time::{Duration, Instant}; /// pathologically slow git child so it cannot pin a served-git permit (the read /// permit on the upload-pack serve path, the write permit on the receive-pack /// post-push replication path) past the deadline. Every caller funnels through -/// `blob_paths`, so bounding here bounds both paths at one seam. +/// `blob_paths`, so bounding here bounds both paths at one seam. Production callers +/// pass the operator-configured `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` instead; this +/// fixed budget only backs the `git_bin`-less test wrappers. +#[cfg(test)] const WALK_TIMEOUT: Duration = Duration::from_secs(600); /// Run one git child under a shared `deadline` with process-group teardown, @@ -122,6 +125,12 @@ fn run_bounded_git( }); let mut stdout = child.stdout.take().context("git stdout was not piped")?; let mut out = Vec::new(); + // Blocking drain, unblocked by the child closing stdout on exit. The watchdog's + // SIGTERM/SIGKILL is what makes a hung child exit; a git wedged in uninterruptible + // (D-state) I/O survives even SIGKILL, so this drain and the wait below can block + // until the kernel returns, pinning the walk thread and its permit. That residual + // is unreachable in userspace (no signal reaps a D-state process) and matches the + // async `reap_group_on_timeout`, which likewise only warns and gives up there. let read_result = stdout.read_to_end(&mut out); let status = child.wait().context("git wait failed")?; // Reaped: bar the watchdog from signalling the now-reaped (possibly recycled) @@ -325,6 +334,7 @@ fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result, ) -> Result> { - let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; + withheld_blob_oids_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`withheld_blob_oids`] with an injectable `git_bin` and walk `timeout`. Served +/// handlers call this with the operator-configured git binary and +/// `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`, so the whole walk is bounded by the same +/// budget as the other served-git ops and a fake `git` can drive its teardown in +/// tests. The `git_bin`-less wrapper above keeps the fixed [`WALK_TIMEOUT`] for the +/// classification tests that run against real git. +pub fn withheld_blob_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; Ok(withheld_from_pairs( &pairs, rules, is_public, owner_did, caller, )) @@ -400,6 +436,7 @@ pub fn replicable_objects(all: Vec, withheld: &HashSet) -> Vec Result> { + allowed_blob_set_for_caller_bounded( + repo_path, git_bin, timeout, rules, is_public, owner_did, None, + ) +} + /// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The /// caller-aware generalization of `replicable_blob_set` (which is the anonymous /// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against @@ -421,6 +473,7 @@ pub fn replicable_blob_set( /// elsewhere (its content is readable to this caller elsewhere). Trees and /// commits are NOT included here; the caller decides per object type whether /// the allow-set applies (it does not for trees/commits — KTD3). +#[cfg(test)] pub fn allowed_blob_set_for_caller( repo_path: &Path, rules: &[VisibilityRule], @@ -428,7 +481,29 @@ pub fn allowed_blob_set_for_caller( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; + allowed_blob_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_blob_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` gate. +pub fn allowed_blob_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { @@ -461,14 +536,28 @@ pub fn replicable_objects_fail_closed( /// owner plus any reader DID that `visibility_check` Allows at some path the /// blob appears at. Least-privilege: a reader of one private subtree is not a /// recipient of a blob that only lives in another. +#[cfg(test)] pub fn withheld_blob_recipients( repo_path: &Path, rules: &[VisibilityRule], is_public: bool, owner_did: &str, +) -> Result>> { + withheld_blob_recipients_bounded(repo_path, "git", WALK_TIMEOUT, rules, is_public, owner_did) +} + +/// [`withheld_blob_recipients`] with an injectable `git_bin` and walk `timeout`, for +/// the receive-pack encrypt-then-pin path. +pub fn withheld_blob_recipients_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, ) -> Result>> { // One history walk feeds both the withheld set and the recipient mapping. - let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; + let pairs = blob_paths(repo_path, git_bin, timeout)?; let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None); if withheld.is_empty() { return Ok(HashMap::new()); @@ -571,6 +660,82 @@ mod tests { "the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts" ); } + + /// #174 (F1 status-gate, vetted by execution): a child that exits SUCCESSFULLY is + /// never reported as a timeout even when the watchdog fires, so a walk finishing + /// right at its deadline is not a spurious 504. The fake only exits when signalled + /// and exits 0 on SIGTERM, so with a deadline already elapsed the watchdog always + /// reaches its kill path (killed == true) yet the child's status is success. + /// Drop the `!status.success()` guard and this returns GitServiceTimeout (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_success_at_the_deadline_is_not_a_timeout() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap 'exit 0' TERM\nsleep 30 &\nwait\n"; + let git_bin = write_fake_git(tmp.path(), body); + let out = run_bounded_git( + &git_bin, + &["rev-list"], + tmp.path(), + b"", + Instant::now() + Duration::from_millis(100), + ); + assert!( + out.is_ok(), + "a child that exited successfully must not be reported as a timeout even if the watchdog fired: {out:?}" + ); + } + + /// #174 (F3, vetted by execution): a child that IGNORES SIGTERM is still reaped + /// via the watchdog's SIGKILL escalation, so it cannot pin the walk thread or its + /// permit. The fake traps SIGTERM and keeps sleeping; run_bounded_git must still + /// return (via SIGKILL at the grace step) with a timeout error and the group must + /// be gone. (A truly uninterruptible D-state child, which no signal can reap, is + /// the documented residual this teardown, like the async twin, cannot cover.) + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_sigterm_ignoring_child_via_sigkill() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nwhile true; do sleep 1; done\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return via SIGKILL even for a SIGTERM-ignoring child"); + assert!( + out.is_err(), + "a SIGTERM-ignoring child killed by SIGKILL is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running" + ); + } use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 0f33fde4..9b646c2b 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -392,6 +392,7 @@ async fn main() -> Result<()> { git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), + git_bin: "git".to_string(), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index f55abeb7..94952120 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -115,6 +115,11 @@ pub struct AppState { /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many /// distinct source IPs (each also braked by the per-IP push rate limiter). pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, + /// The `git` executable the served-git withheld-blob walk spawns. Production is + /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's + /// process-group teardown in handler tests without mutating the process-global + /// PATH (#174). + pub git_bin: String, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 03d3d5c4..a43a79a0 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -89,6 +89,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, ), + git_bin: "git".to_string(), } } From 454e59ba533535168a6eb763447275ef107564ad Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 02:12:37 -0500 Subject: [PATCH 18/52] docs(node): the withheld-walk is bounded by the service timeout, not a fixed const (#174) Follow-up to threading the configured timeout into the walk: the walk now honors GITLAWB_GIT_SERVICE_TIMEOUT_SECS on both the serve and replication paths, so the README no longer says a fixed internal deadline. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d02c4a0b..0f760449 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. The withheld-blob classification walk is also bounded and reaped, by a fixed internal deadline rather than this setting. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths), which is reaped at the deadline. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | From 79b9ee5c89070f2c9b8d534d28195803110385a1 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 14:32:24 -0500 Subject: [PATCH 19/52] fix(node): shed the served-git pool before the DB and cap per-source before it (#174) CodeRabbit review: the held git_permit was acquired at the top of git_info_refs and git_upload_pack, before the per-source cap, so one source could occupy the global pool during the DB/visibility window before its sub-cap rejected the excess. Move the held git_permit to after acquire_read_caller_permit (still before acquire_fresh/git, so INV-10's bound on the fresh Tigris acquire holds), and add a cheap pre-DB early shed (available_permits() == 0 -> 503, holds no permit) so the #62 shed-before-DB property is preserved without holding a permit across the DB work. The receive-pack POST is unchanged (owner-only; its top-of-handler git_permit is deliberate for acquire_write bounding). Tests: the two #62 shed-before-DB tests now exercise the explicit early check (load-bearing: disable it and they fall through to the DB); three new *_per_source_cap_sheds_with_global_capacity tests prove the per-source sub-cap sheds a capped source even with global capacity free (load-bearing on the caller cap). Full workspace green. --- crates/gitlawb-node/src/api/repos.rs | 253 ++++++++++++++++++++++-- crates/gitlawb-node/src/test_support.rs | 62 ++++-- 2 files changed, 290 insertions(+), 25 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 1532be58..85eaa227 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -515,17 +515,27 @@ pub async fn git_info_refs( let service = query .service .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; - // Shed with a 503 before spawning git when the concurrency cap is saturated; - // held for the whole op (incl. the smart_http call), released on return. The - // receive-pack advertisement is phase one of a push, so it draws from the WRITE - // pool (like the git-receive-pack POST), not the global read pool an anonymous - // clone flood can exhaust and thereby starve the push handshake (#174 U2). The - // upload-pack advertisement stays on the read pool with its per-caller sub-cap. - let _permit = if service == "git-receive-pack" { - git_permit(&state.git_write_semaphore)? - } else { - git_permit(&state.git_read_semaphore)? - }; + // #62 cheap pre-DB load shed: if the pool this service draws from is already + // saturated, shed with 503 before any DB/disk work. Best-effort (holds no + // permit); the authoritative hold is `git_permit` below, after the per-source + // cap. Restores the shed-before-DB property the reordered held acquire alone + // would drop, while the reorder still prevents one source from occupying global + // slots during the DB/visibility window. + { + let pool = if service == "git-receive-pack" { + &state.git_write_semaphore + } else { + &state.git_read_semaphore + }; + if pool.available_permits() == 0 { + tracing::warn!( + "served-git concurrency cap reached; shedding request with 503 (pre-DB)" + ); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } + } tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state .db @@ -604,6 +614,22 @@ pub async fn git_info_refs( )? }; + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire_fresh/git so it bounds the fresh Tigris acquire + // and git exec (INV-10). The receive-pack advertisement is phase one of a push, + // so it draws from the WRITE pool (like the git-receive-pack POST), not the + // global read pool an anonymous clone flood can exhaust and thereby starve the + // push handshake (#174 U2). The upload-pack advertisement stays on the read + // pool with its per-caller sub-cap. + let _permit = if service == "git-receive-pack" { + git_permit(&state.git_write_semaphore)? + } else { + git_permit(&state.git_read_semaphore)? + }; + // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. let disk_path = if service == "git-receive-pack" { @@ -725,9 +751,15 @@ pub async fn git_upload_pack( headers: axum::http::HeaderMap, body: Bytes, ) -> Result { - // Shed with a 503 before spawning git when the concurrency cap is saturated; - // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_read_semaphore)?; + // #62 cheap pre-DB load shed (see git_info_refs): shed before DB when the read + // pool is saturated; the authoritative hold is `git_permit` below, after the + // per-source cap. + if state.git_read_semaphore.available_permits() == 0 { + tracing::warn!("served-git concurrency cap reached; shedding request with 503 (pre-DB)"); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } let name = smart_http_repo_name(&repo)?; let record = state .db @@ -760,6 +792,14 @@ pub async fn git_upload_pack( "upload-pack", )?; + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire/git so it bounds the Tigris acquire and git + // exec (INV-10). + let _permit = git_permit(&state.git_read_semaphore)?; + let disk_path = state .repo_store .acquire(&record.owner_did, &record.name) @@ -3209,6 +3249,191 @@ mod tests { ); } + /// #174 (review fix): the per-source caller cap is an independent brake that + /// sheds a capped source even when the global pool has free capacity — the + /// sub-cap is not a mere pre-filter for pool exhaustion. Proven by leaving the + /// global read pool with capacity (so the pre-DB early shed passes) AND + /// pre-holding the source's upload-pack read sub-cap: the request reaches the + /// caller cap and sheds there, so its 503 body reads "for this caller". Remove + /// the `acquire_read_caller_permit` call and the capped source falls through to + /// the git op instead of shedding with "for this caller" — this is the + /// caller-cap acquire probe for the info/refs upload-pack branch. + #[sqlx::test] + async fn info_refs_upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordir", "oi", "/tmp/oi-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.91:5000".parse().unwrap(); + // Pin this source at its single upload-pack read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordir/oi/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the receive-pack + /// advertisement branch of info/refs — its per-source cap + /// (`git_push_advert_per_caller`) sheds a capped source even when the global + /// write pool has capacity. Leave the global write pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's advert slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". The push rate limiter is left permissive so the + /// request reaches the caller cap. + #[sqlx::test] + async fn info_refs_receive_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has free capacity (early shed passes); source pre-held at + // its advert sub-cap so it sheds on the caller cap, not the global pool. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + // Permissive push rate limiter so the advertisement passes the rate gate and + // reaches the per-source concurrency cap. + state.push_rate_limiter = crate::rate_limit::RateLimiter::new(100, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordrp", "or", "/tmp/or-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.92:5000".parse().unwrap(); + // Pin this source at its single receive-pack advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordrp/or/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its advert sub-cap must shed 503 even with global write pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source advert cap is an independent brake: with global write capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the POST upload-pack + /// handler — its per-source read cap sheds a capped source even when the global + /// read pool has capacity. Leave the global read pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's read slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". + #[sqlx::test] + async fn upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordup", "ou", "/tmp/ou-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.93:5000".parse().unwrap(); + // Pin this source at its single read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6ordup/ou/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index a43a79a0..765545f1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -195,13 +195,14 @@ mod tests { ); } - /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer. One test - /// per git handler drives the `let _permit = git_permit(...)` wiring end to end - /// (this one plus the git_upload_pack / git_receive_pack siblings below); the - /// git_permit unit test covers the helper in isolation. DB-free: an exhausted - /// semaphore sheds before any DB/disk access, so a lazy state works. Remove the - /// permit line from git_info_refs and this goes red (the request falls through - /// to the DB and returns something other than 503). + /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer before the + /// DB. The held `git_permit` acquire now sits after the per-source cap, so the + /// shed-before-DB property is carried by an explicit `available_permits() == 0` + /// early check at the top of the handler (the held permit remains the + /// authoritative bound further down). DB-free: an exhausted semaphore sheds + /// before any DB/disk access, so a lazy state works. Remove the early-shed block + /// from git_info_refs and this goes red (the request falls through to the DB and + /// returns something other than 503). #[tokio::test] async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); @@ -234,10 +235,11 @@ mod tests { ); } - /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack also acquires a - /// permit at the top, so an exhausted semaphore must shed it with a 503 before - /// any DB/disk work. Anonymous-reachable, so no auth injection is needed. Remove - /// the permit line from git_upload_pack and this goes red. + /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack carries the same + /// explicit `available_permits() == 0` early-shed check at the top, so an + /// exhausted semaphore must shed it with a 503 before any DB/disk work. + /// Anonymous-reachable, so no auth injection is needed. Remove the early-shed + /// block from git_upload_pack and this goes red. #[tokio::test] async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); @@ -270,6 +272,44 @@ mod tests { ); } + /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed + /// selects the WRITE pool for a git-receive-pack advertisement (phase one of a + /// push, #174 U2), so an exhausted write pool sheds the advert with 503 before + /// any DB/disk work — even while the read pool is free (only the write pool is + /// zeroed here). Flip the pool selection to the read pool, or remove the + /// early-shed block, and this goes red. + #[tokio::test] + async fn git_info_refs_receive_pack_sheds_with_503_when_write_pool_exhausted() { + let mut state = test_state_lazy(); + state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-receive-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted WRITE pool must shed the receive-pack advertisement with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + /// PR3 (#62) sibling for the push path: git-receive-pack requires an /// AuthenticatedDid extension (production: require_signature injects it), so the /// request carries one via signed_request_as — without it the Extension From f19f2813a2b73c326001e5724e0ae175d0b9d72c Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 09:44:53 -0500 Subject: [PATCH 20/52] fix(node): give the receive-pack advert its own pool so it can't shed authed pushes (#174) jatmn P1: the anon-reachable GET info/refs?service=git-receive-pack drew from the same git_write_semaphore as the authenticated git-receive-pack POST. The per-source advert cap reserves no global slots, so ~8 sources (each within the per-IP push rate limit) could hold all 32 write permits across acquire_fresh + the advertisement, and the next authenticated push then 503s at admission. That is an anonymous caller shedding an authenticated push across the auth boundary (INV-10). Add a dedicated git_push_advert_semaphore, disjoint from the write pool. Both the pre-DB early-shed peek and the held acquire for the receive-pack advertisement now draw from it; the write pool is left exclusively for the POST. An advert flood can at worst exhaust the advert pool (each source still capped by git_push_advert_per_caller and the per-IP push rate limiter); the authenticated push pool is untouched. Test receive_pack_advertisement_draws_from_dedicated_advert_pool proves at the handler layer that the advert sheds when its own pool is saturated AND survives when the write pool is saturated (the reservation). Both flip under revert, verified independently for the pre-DB peek and the held acquire. --- crates/gitlawb-node/src/api/repos.rs | 74 ++++++++++++++++--------- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 7 +++ crates/gitlawb-node/src/state.rs | 16 ++++-- crates/gitlawb-node/src/test_support.rs | 17 +++--- 5 files changed, 76 insertions(+), 39 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 85eaa227..719246e0 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -522,8 +522,11 @@ pub async fn git_info_refs( // would drop, while the reorder still prevents one source from occupying global // slots during the DB/visibility window. { + // The receive-pack advertisement peeks its DEDICATED advert pool, not the + // write pool the authenticated POST uses (#174) — matching the held acquire + // below, so the pre-DB shed and the authoritative hold agree on the pool. let pool = if service == "git-receive-pack" { - &state.git_write_semaphore + &state.git_push_advert_semaphore } else { &state.git_read_semaphore }; @@ -620,12 +623,14 @@ pub async fn git_info_refs( // would be sub-cap-denied for during the DB/visibility window and starve other // sources; still before acquire_fresh/git so it bounds the fresh Tigris acquire // and git exec (INV-10). The receive-pack advertisement is phase one of a push, - // so it draws from the WRITE pool (like the git-receive-pack POST), not the - // global read pool an anonymous clone flood can exhaust and thereby starve the - // push handshake (#174 U2). The upload-pack advertisement stays on the read - // pool with its per-caller sub-cap. + // but it is ANON-reachable, so it draws from the dedicated advert pool + // (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses: + // an advert flood can at worst exhaust the advert pool, never a permit a push + // POST needs at admission (#174 U2). A clone flood on the read pool likewise + // can't touch either. The upload-pack advertisement stays on the read pool with + // its per-caller sub-cap. let _permit = if service == "git-receive-pack" { - git_permit(&state.git_write_semaphore)? + git_permit(&state.git_push_advert_semaphore)? } else { git_permit(&state.git_read_semaphore)? }; @@ -2944,14 +2949,19 @@ mod tests { ); } - /// #174 U2: the receive-pack advertisement (`GET info/refs?service=git-receive-pack`) - /// is phase one of a push, so it draws from the WRITE pool, not the global read - /// pool an anonymous clone flood can exhaust. Proven at the handler by saturating - /// each pool to zero and checking who shares it (INV-10). Revert the branch to the - /// read pool and the read-saturated receive-pack assertion goes 503 (the exact - /// push-handshake starvation jatmn flagged). + /// #174 (jatmn P1): the anon-reachable receive-pack advertisement + /// (`GET info/refs?service=git-receive-pack`) draws from a DEDICATED advert pool + /// (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses. + /// Proven at the handler by saturating each pool to zero and checking who shares + /// it (INV-10, across the auth boundary). The load-bearing pair: + /// * advert pool at 0 -> the advert SHEDS 503 (it is bound to that pool); + /// * write pool at 0 -> the advert SURVIVES (it can NOT consume a permit the + /// authenticated POST needs — the reservation jatmn asked for). + /// Revert the branch to `git_write_semaphore` and BOTH flip: the advert-pool-0 + /// case stops shedding and the write-pool-0 case starts shedding (the exact + /// anon-sheds-authed-push starvation). #[sqlx::test] - async fn receive_pack_advertisement_draws_from_write_pool(pool: sqlx::PgPool) { + async fn receive_pack_advertisement_draws_from_dedicated_advert_pool(pool: sqlx::PgPool) { use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; @@ -2960,17 +2970,19 @@ mod tests { use tokio::sync::Semaphore; use tower::ServiceExt; - // Build a fresh state with the two pools sized independently, then drive one + // Build a fresh state with the three pools sized independently, then drive one // info/refs advertisement for `service` and return its handler status. async fn advert_status( pool: &sqlx::PgPool, read_permits: usize, write_permits: usize, + advert_permits: usize, service: &str, ) -> StatusCode { let mut state = crate::test_support::test_state(pool.clone()).await; state.git_read_semaphore = Arc::new(Semaphore::new(read_permits)); state.git_write_semaphore = Arc::new(Semaphore::new(write_permits)); + state.git_push_advert_semaphore = Arc::new(Semaphore::new(advert_permits)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; state .db @@ -2988,31 +3000,39 @@ mod tests { router.oneshot(req).await.unwrap().status() } - // Read pool saturated, write pool free: the push handshake SURVIVES. + // Advert pool saturated (read + write free): the receive-pack advert SHEDS, + // proving it is bound to the dedicated advert pool. + assert_eq!( + advert_status(&pool, 8, 8, 0, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement draws from the dedicated advert pool: a saturated advert pool sheds it 503" + ); + // WRITE pool saturated (advert + read free): the advert SURVIVES. This is the + // reservation — an advert flood can never occupy a permit the authenticated + // push POST relies on at admission. assert_ne!( - advert_status(&pool, 0, 8, "git-receive-pack").await, + advert_status(&pool, 8, 0, 8, "git-receive-pack").await, StatusCode::SERVICE_UNAVAILABLE, - "receive-pack advertisement must draw from the write pool: a saturated read pool must not shed it" + "receive-pack advertisement must NOT draw from the write pool: a saturated write pool must not shed it" ); - // Write pool saturated, read pool free: the receive-pack advertisement SHEDS, - // proving it now consumes the write pool (not the read pool). - assert_eq!( - advert_status(&pool, 8, 0, "git-receive-pack").await, + // Read pool saturated (advert + write free): the advert SURVIVES (never on the read pool). + assert_ne!( + advert_status(&pool, 0, 8, 8, "git-receive-pack").await, StatusCode::SERVICE_UNAVAILABLE, - "receive-pack advertisement draws from the write pool, so a saturated write pool sheds it 503" + "receive-pack advertisement must not draw from the read pool" ); // Read pool saturated: the upload-pack advertisement still SHEDS (unchanged). assert_eq!( - advert_status(&pool, 0, 8, "git-upload-pack").await, + advert_status(&pool, 0, 8, 8, "git-upload-pack").await, StatusCode::SERVICE_UNAVAILABLE, "upload-pack advertisement stays on the read pool: a saturated read pool sheds it 503" ); - // Write pool saturated, read pool free: the upload-pack advertisement is - // UNAFFECTED, proving reads never touch the write pool in either direction. + // Write + advert pools saturated, read free: the upload-pack advertisement is + // UNAFFECTED, proving reads never touch either write-side pool. assert_ne!( - advert_status(&pool, 8, 0, "git-upload-pack").await, + advert_status(&pool, 8, 0, 0, "git-upload-pack").await, StatusCode::SERVICE_UNAVAILABLE, - "upload-pack advertisement never touches the write pool" + "upload-pack advertisement never touches the write or advert pool" ); } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 5befa7f3..5d827794 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -522,6 +522,7 @@ mod tests { shutdown_tx: tokio::sync::watch::channel(false).0, git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 9b646c2b..18019395 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -382,6 +382,13 @@ async fn main() -> Result<()> { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Anon receive-pack advertisements get their OWN pool, same size as the + // write pool but disjoint, so filling it (which takes many source IPs, each + // capped by git_push_advert_per_caller) never occupies a permit the + // authenticated POST needs (#174). + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 94952120..eb07a3e4 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -97,11 +97,19 @@ pub struct AppState { /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate /// from `git_read_semaphore` so an anonymous READ flood can never shed an /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from - /// by the `git-receive-pack` POST (owner-gated) and the receive-pack `info/refs` - /// advertisement (anon-reachable); the advertisement is additionally bounded per - /// source by `git_push_advert_per_caller` so no single source can monopolize this - /// pool and shed pushes. + /// by the `git-receive-pack` POST (owner-gated) ONLY. The anon-reachable + /// receive-pack `info/refs` advertisement draws from the SEPARATE + /// `git_push_advert_semaphore` below, never this pool, so a multi-source flood + /// of push-handshake advertisements can never occupy a permit an authenticated + /// POST needs at admission (#174). pub git_write_semaphore: Arc, + /// Bounds concurrent anon-reachable `git-receive-pack` `info/refs` + /// advertisements — a pool SEPARATE from `git_write_semaphore` so adverts (which + /// hold a permit across `acquire_fresh` + `info/refs`) can never consume a slot + /// the authenticated POST relies on. A per-source flood can at worst exhaust this + /// advert pool (each source also capped by `git_push_advert_per_caller` and the + /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). + pub git_push_advert_semaphore: Arc, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 765545f1..0805fec1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -85,6 +85,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { // Generous — no test drives the handler-level shed (git_permit is unit-tested). git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, @@ -273,15 +274,15 @@ mod tests { } /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed - /// selects the WRITE pool for a git-receive-pack advertisement (phase one of a - /// push, #174 U2), so an exhausted write pool sheds the advert with 503 before - /// any DB/disk work — even while the read pool is free (only the write pool is - /// zeroed here). Flip the pool selection to the read pool, or remove the - /// early-shed block, and this goes red. + /// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174), + /// so an exhausted advert pool sheds the advert with 503 before any DB/disk work + /// — while the write pool (reserved for authenticated POSTs) is left free here. + /// Flip the pool selection back to the write pool, or remove the early-shed + /// block, and this goes red. #[tokio::test] - async fn git_info_refs_receive_pack_sheds_with_503_when_write_pool_exhausted() { + async fn git_info_refs_receive_pack_sheds_with_503_when_advert_pool_exhausted() { let mut state = test_state_lazy(); - state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_push_advert_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -299,7 +300,7 @@ mod tests { assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "an exhausted WRITE pool must shed the receive-pack advertisement with 503 before touching the DB" + "an exhausted ADVERT pool must shed the receive-pack advertisement with 503 before touching the DB" ); assert_eq!( resp.headers() From 473924dcfec11c22b8ea5da5f125478bbcd5f845 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 09:49:33 -0500 Subject: [PATCH 21/52] fix(node): compile the bounded visibility git runner on non-Unix targets (#174) jatmn P1: run_bounded_git was gated #[cfg(unix)], but its callers (assert_all_refs_are_commits, blob_paths) and the module are compiled on all targets, so a non-Unix build (the continue-on-error Windows release target) could not resolve the function. Add a #[cfg(not(unix))] twin with the same signature and result semantics. It bounds a single child (no process-group teardown, which is Unix-only): threads feed stdin and drain stderr, the main thread drains stdout, and a watchdog thread kills the child at the deadline. The child is shared with the watchdog behind a mutex the main thread does not hold while draining, so the watchdog can always acquire it to kill, and killing closes stdout to unblock the drain. Best-effort (reaps only the direct child), which is why the hardened group-aware path stays Unix-only. Verified the twin type-checks by compiling its (portable-std) body on the native target; the full non-Unix link is CI's Windows job. --- .../gitlawb-node/src/git/visibility_pack.rs | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 09a80aba..0eaeed2d 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -154,6 +154,98 @@ fn run_bounded_git( Ok(out) } +/// Non-Unix fallback for [`run_bounded_git`]. Windows and other non-Unix targets +/// have no process-group teardown (`process_group(0)` / `kill(-pgid)` are Unix-only), +/// so this bounds a single child on its own: threads feed stdin and drain stderr +/// while the main thread drains stdout, and a watchdog thread kills the child at the +/// deadline (which closes stdout and unblocks the drain). The child is shared with +/// the watchdog behind a mutex that the main thread does NOT hold while draining, so +/// the watchdog can always acquire it to kill. Best-effort — it reaps only the direct +/// child, not a descendant group — which is why the hardened, group-aware path above +/// is gated to Unix, the only target the served node actually runs on (the Windows +/// release binary is best-effort / `continue-on-error` in CI). Kept in lockstep with +/// the Unix version's signature and result semantics so every caller compiles on all +/// targets (#174). +#[cfg(not(unix))] +fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + use std::io::{Read, Write}; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + + // Share the child with the watchdog. The main thread drains stdout WITHOUT + // holding this lock, so the watchdog can always acquire it to kill on timeout; + // killing closes stdout and unblocks the drain below. + let child = std::sync::Arc::new(std::sync::Mutex::new(child)); + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = { + let child = child.clone(); + std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + if let Ok(mut c) = child.lock() { + let _ = c.kill(); + } + true + } + } + }) + }; + + let mut out = Vec::new(); + let read_result = stdout.read_to_end(&mut out); + // The drain has returned (child exited or was killed), so taking the lock here + // cannot deadlock against the watchdog. + let status = child + .lock() + .expect("git child mutex poisoned") + .wait() + .context("git wait failed")?; + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + read_result.context("failed to read git stdout")?; + if killed && !status.success() { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} + /// Fail closed unless every ref ultimately resolves to a commit (a ref pointing /// directly at a blob or tree, or an annotated tag — even a nested one — of such /// an object is refused). `git rev-list --all` silently *skips* such refs, but From 7a6bd359eb915285ba4e459ae3339f0ed8716891 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:11:41 -0500 Subject: [PATCH 22/52] fix(node): SIGKILL a SIGTERM-ignoring member of the visibility-walk process group (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jatmn/CodeRabbit P1: the walk watchdog stood down as soon as the group LEADER was reaped. A leader that exits cleanly on SIGTERM while a background member ignores SIGTERM and closes its inherited pipes let the main drain unblock, the leader be reaped, and the watchdog skip its SIGKILL escalation — so the handler returned while the descendant kept running. jatmn's suggested "continue until ESRCH" could not work: the main thread reaping the leader is what frees the pgid, so continuing to kill(-pgid) after that races a recycled group. Restructure the coordination instead: the main thread now defers reaping the leader until the watchdog thread returns. The watchdog always escalates through an unconditional group SIGKILL (no leader-reap short-circuit), and because the leader is still unreaped while every kill(-pgid) fires, the pgid cannot have been recycled. The grace before SIGKILL is a fixed budget (only paid on the exceptional timeout path). Regression run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits: a leader that exits 0 on SIGTERM spawns an sh -c descendant (its own $$, not a ( ) subshell's parent pid) that ignores SIGTERM and closes its pipes; the descendant must be dead when the runner returns. RED on the old teardown (survives), GREEN now. The existing leader-ignores-SIGTERM and hung-walk teardown tests still pass. --- .../gitlawb-node/src/git/visibility_pack.rs | 162 +++++++++++++----- 1 file changed, 118 insertions(+), 44 deletions(-) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 0eaeed2d..ec55423e 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -23,6 +23,12 @@ use std::time::{Duration, Instant}; #[cfg(test)] const WALK_TIMEOUT: Duration = Duration::from_secs(600); +/// How long the process-group watchdog waits after SIGTERM before escalating to +/// SIGKILL, giving a well-behaved git child time to clean up its `*.lock` files. Only +/// paid on a timeout (already the exceptional path). +#[cfg(unix)] +const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); + /// Run one git child under a shared `deadline` with process-group teardown, /// BLOCKING, and return its stdout. The child runs in its own process group; a /// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, @@ -60,52 +66,46 @@ fn run_bounded_git( // With process_group(0) the child leads its own group, so pgid == its pid. let pgid = child.id() as i32; - // Watchdog: on the deadline, SIGTERM the whole group, poll until it exits, - // escalate to SIGKILL after a grace, and report whether it killed. Signalled to - // stand down when the child is reaped in time. Kept off the main thread because - // the main thread's stdout drain is exactly what blocks until a hung child is - // torn down. + // Watchdog: on the deadline, tear the WHOLE process group down — SIGTERM, a grace + // for a well-behaved child to clean up its `*.lock` files, then an UNCONDITIONAL + // SIGKILL of the group. It never stands down on leader-reap alone: a group member + // that ignores SIGTERM while the leader exits cleanly would otherwise escape the + // SIGKILL and keep running past the deadline (finding 3, #174). The main thread + // defers reaping the leader until this thread returns (see below), so the leader's + // pid is still unreaped while every `kill(-pgid)` fires and the pgid cannot have + // been recycled — which is why this no longer needs the old `reaped` short-circuit. + // Kept off the main thread because the main thread's stdout drain is exactly what + // blocks until a hung child is torn down. let (done_tx, done_rx) = mpsc::channel::<()>(); - // Set true the instant the main thread reaps the child, so the watchdog never - // SIGTERMs a pgid whose leader is already reaped and whose pid the kernel may - // recycle (the reused-pgid hazard smart_http guards via disarm-after-wait). - let reaped = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let watchdog = { - let reaped = reaped.clone(); - std::thread::spawn(move || -> bool { - use std::sync::atomic::Ordering; - let wait = deadline.saturating_duration_since(Instant::now()); - match done_rx.recv_timeout(wait) { - Ok(()) | Err(RecvTimeoutError::Disconnected) => false, - Err(RecvTimeoutError::Timeout) => { - // The child was reaped right as the deadline fired: do not signal - // its (possibly-recycled) pgid. - if reaped.load(Ordering::SeqCst) { - return false; - } - // SAFETY: kill(2) takes only integers and borrows no Rust memory; - // ESRCH on an already-gone group is ignored. - unsafe { libc::kill(-pgid, libc::SIGTERM) }; - for step in 0..200u32 { - if reaped.load(Ordering::SeqCst) || unsafe { libc::kill(-pgid, 0) } != 0 { - return true; // reaped, or ESRCH: every group member has exited - } - if step == 100 { - unsafe { libc::kill(-pgid, libc::SIGKILL) }; - } - std::thread::sleep(Duration::from_millis(10)); - } - // Survived SIGKILL past the cap: a wedged (D-state) git, the same - // condition smart_http's reap logs. Warn so an operator sees it. + let watchdog = std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + // Fixed grace: because the main thread defers the leader's reap, a + // fully-exited group still shows a zombie leader here, so polling for + // ESRCH cannot detect early completion — just wait the grace, then + // SIGKILL. On a group of only zombies the SIGKILL is a harmless no-op; + // on a SIGTERM-ignoring member it is what actually kills it. + std::thread::sleep(WATCHDOG_TERM_GRACE); + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + // Brief settle so the SIGKILL is delivered before the main thread + // reaps the leader and frees the pgid. A wedged (D-state) member + // survives even SIGKILL — the documented residual, as in smart_http. + std::thread::sleep(Duration::from_millis(20)); + if unsafe { libc::kill(-pgid, 0) } == 0 { tracing::warn!( pgid, "withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)" ); - true } + true } - }) - }; + } + }); // Feed stdin on a writer thread and drain stderr on a reader thread so the main // thread can drain stdout concurrently; writing all of stdin (or draining one @@ -132,14 +132,18 @@ fn run_bounded_git( // is unreachable in userspace (no signal reaps a D-state process) and matches the // async `reap_group_on_timeout`, which likewise only warns and gives up there. let read_result = stdout.read_to_end(&mut out); + // The drain has returned: either the child exited on its own, or the watchdog's + // teardown closed its pipes. Tell the watchdog the drain is done, then WAIT for it + // to return BEFORE reaping the leader. On the timeout path the watchdog runs the + // full SIGTERM -> grace -> SIGKILL teardown; joining it before `child.wait()` keeps + // the leader's pid unreaped while every `kill(-pgid)` fires (so the pgid can't be + // recycled), and guarantees a SIGTERM-ignoring group member has already been + // SIGKILLed rather than left running past the deadline (finding 3, #174). + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); let status = child.wait().context("git wait failed")?; - // Reaped: bar the watchdog from signalling the now-reaped (possibly recycled) - // pgid before it can fire, then stand it down. - reaped.store(true, std::sync::atomic::Ordering::SeqCst); let err = err_reader.join().unwrap_or_default(); let _ = writer.join(); - let _ = done_tx.send(()); - let killed = watchdog.join().unwrap_or(false); read_result.context("failed to read git stdout")?; // The watchdog runs off a wall clock that can race a child finishing right at the // deadline. A child that exited on its own (success) is not a timeout even if the @@ -828,6 +832,76 @@ mod tests { "the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running" ); } + + /// #174 finding 3 (jatmn/CodeRabbit): a group MEMBER that ignores SIGTERM must + /// still be SIGKILLed even when the group LEADER exits cleanly on SIGTERM. The + /// leader traps SIGTERM to exit 0, but first spawns a descendant (`sh -c`, so its + /// `$$` is its OWN pid — a `( )` subshell's `$$` is the parent's) that ignores + /// SIGTERM and closes its inherited stdout/stderr. When the watchdog SIGTERMs the + /// group, the leader exits, its stdout closes, the main drain unblocks, and the + /// leader is reaped — the exact window a `reaped`-gated watchdog stands down in, + /// before escalating to SIGKILL. The descendant must be dead when run_bounded_git + /// returns; a teardown that stands down on leader-reap leaves it running (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // Both loops are bounded (~30s) so a broken teardown cannot leak a permanent + // orphan or wedge the suite; the assertion fires well before then. + let body = "#!/bin/sh\n\ +case \"$1\" in\n\ + rev-list)\n\ + sh -c 'trap \"\" TERM; echo $$ > desc.pid; exec 1>&- 2>&-; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + trap 'exit 0' TERM\n\ + i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n\ + *) : ;;\n\ +esac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let _ = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return within the watchdog budget"); + + // Wait for the descendant to record its OWN pid, then assert it is gone. + let desc_pid_path = tmp.path().join("desc.pid"); + let mut desc: Option = None; + for _ in 0..200 { + if let Some(p) = std::fs::read_to_string(&desc_pid_path) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(desc, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(desc, libc::SIGKILL) }; + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned" + ); + } use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; From 9412aa9c5bdc56d5c27cbf5be3dfebea995e7039 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:34:44 -0500 Subject: [PATCH 23/52] fix(node): bound and reap push-candidate git children under the write permit (#174) Route push_delta's candidate-discovery git children (cat-file / rev-list) through the bounded run_bounded_git (deadline + process-group SIGKILL teardown) instead of bare Command::output(). A hung scan no longer pins the git-receive-pack write permit past the deadline or a client disconnect. Threads git_bin + a shared per-scan deadline (git_service_timeout_secs) through resolve_candidates_for_push and all_blob_oids. --- crates/gitlawb-node/src/api/repos.rs | 4 +- crates/gitlawb-node/src/git/push_delta.rs | 278 ++++++++++++------ .../gitlawb-node/src/git/visibility_pack.rs | 11 +- 3 files changed, 193 insertions(+), 100 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 719246e0..ea6f126e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -115,7 +115,7 @@ async fn fail_closed_full_scan_objects( let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path)?; + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, std::time::Instant::now() + timeout)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( candidates, &allowed, &all_blobs, )) @@ -1264,6 +1264,8 @@ pub async fn git_receive_pack( disk_path.clone(), new_tips, old_tips, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await; if pin_set.full_scan { diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 7ab00816..51b6f836 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -29,9 +29,9 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::time::{Duration, Instant}; -use anyhow::{bail, Context, Result}; +use anyhow::Result; /// Env var that forces the push path to always full-scan, bypassing the delta /// optimization (KTD7 kill-switch). Reuses the already-tested fallback branch @@ -64,11 +64,21 @@ pub enum PinCandidates { /// Fail-closed: any condition where the introduced set cannot be safely /// determined returns [`PinCandidates::FullScanRequired`] rather than a partial /// set, so the caller full-scans instead of silently under-pinning. -pub fn resolve_push_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> PinCandidates { - // KTD7 kill-switch: force the (already-tested) full-scan fallback. The env - // read is split out from the pure logic so the resolver stays unit-testable - // without touching process-global state. - resolve_push_delta_inner(repo_path, new_tips, old_tips, force_full_scan()) +pub fn resolve_push_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> PinCandidates { + resolve_push_delta_inner( + repo_path, + new_tips, + old_tips, + force_full_scan(), + git_bin, + deadline, + ) } /// Whether the force-full-scan kill-switch env var is set. @@ -83,6 +93,8 @@ fn resolve_push_delta_inner( new_tips: &[&str], old_tips: &[&str], force_full_scan: bool, + git_bin: &str, + deadline: Instant, ) -> PinCandidates { if force_full_scan { tracing::debug!("{FORCE_FULL_SCAN_ENV} set — forcing full scan"); @@ -102,7 +114,7 @@ fn resolve_push_delta_inner( // commit). Bare `cat-file -t` returns `tag` for an annotated tag, and // `for-each-ref %(*objecttype)` peels only one level — neither is correct. for tip in new_tips { - match peeled_object_type(repo_path, tip) { + match peeled_object_type(repo_path, tip, git_bin, deadline) { Some(t) if t == "commit" => {} other => { tracing::debug!( @@ -115,7 +127,7 @@ fn resolve_push_delta_inner( } } - match rev_list_delta(repo_path, new_tips, old_tips) { + match rev_list_delta(repo_path, new_tips, old_tips, git_bin, deadline) { Ok(oids) => PinCandidates::Delta(oids), Err(e) => { tracing::debug!(err = %e, "push-delta rev-list failed — forcing full scan"); @@ -126,41 +138,43 @@ fn resolve_push_delta_inner( /// Return the fully-peeled object type of `sha` (e.g. `commit`, `tree`, /// `blob`), or `None` if the object is missing/unpeelable or git errored. -fn peeled_object_type(repo_path: &Path, sha: &str) -> Option { - let output = Command::new("git") - .args(["cat-file", "-t", &format!("{sha}^{{}}")]) - .current_dir(repo_path) - .output() - .ok()?; - if !output.status.success() { - return None; - } - Some(String::from_utf8_lossy(&output.stdout).trim().to_string()) +fn peeled_object_type( + repo_path: &Path, + sha: &str, + git_bin: &str, + deadline: Instant, +) -> Option { + let peel = format!("{sha}^{{}}"); + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-t", &peel], + repo_path, + b"", + deadline, + ) + .ok()?; + Some(String::from_utf8_lossy(&out).trim().to_string()) } /// Run `git rev-list --objects --no-object-names --not ` and return /// the bare OID set. Decides on `status.success()` *before* parsing stdout, so /// a walk that prints a valid prefix then errors mid-walk is discarded. -fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Result> { +fn rev_list_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> Result> { let mut args: Vec<&str> = vec!["rev-list", "--objects", "--no-object-names"]; args.extend_from_slice(new_tips); if !old_tips.is_empty() { args.push("--not"); args.extend_from_slice(old_tips); } - - let output = Command::new("git") - .args(&args) - .current_dir(repo_path) - .output() - .context("failed to run git rev-list for push delta")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git rev-list failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + let out = + crate::git::visibility_pack::run_bounded_git(git_bin, &args, repo_path, b"", deadline)?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -175,23 +189,19 @@ fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Res /// reconciliation sweep relies on. It returns *all* objects (including /// unreachable/dangling ones), which is what the sweep needs to catch /// stragglers — do not swap it for a reachability walk. -pub fn list_all_objects(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -203,23 +213,23 @@ pub fn list_all_objects(repo_path: &Path) -> Result> { /// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed /// filter needs to tell blobs (content, withholdable) from commits/trees /// (structural, never withheld) without typing the candidate list itself. -pub fn list_all_objects_with_type(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects_with_type( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname) %(objecttype)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .filter_map(|l| { @@ -235,8 +245,12 @@ pub fn list_all_objects_with_type(repo_path: &Path) -> Result Result> { - Ok(list_all_objects_with_type(repo_path)? +pub fn all_blob_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + Ok(list_all_objects_with_type(repo_path, git_bin, deadline)? .into_iter() .filter(|(_, ty)| ty == "blob") .map(|(oid, _)| oid) @@ -272,18 +286,22 @@ pub async fn resolve_candidates_for_push( repo_path: PathBuf, new_tips: Vec, old_tips: Vec, + git_bin: String, + timeout: Duration, ) -> PinCandidateSet { tokio::task::spawn_blocking(move || { + // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). + let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); let old_refs: Vec<&str> = old_tips.iter().map(String::as_str).collect(); - match resolve_push_delta(&repo_path, &new_refs, &old_refs) { + match resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) { PinCandidates::Delta(objs) => { tracing::info!(delta = objs.len(), repo = %repo_path.display(), "pin candidate set from push delta"); PinCandidateSet { candidates: objs, full_scan: false } } PinCandidates::FullScanRequired => { tracing::warn!(repo = %repo_path.display(), "pin delta unavailable (non-commit tip, git error, or force-full-scan) — full-scan fallback"); - match list_all_objects(&repo_path) { + match list_all_objects(&repo_path, &git_bin, deadline) { Ok(objs) => PinCandidateSet { candidates: objs, full_scan: true }, Err(e) => { tracing::warn!(repo = %repo_path.display(), err = %e, "full-scan fallback failed; pinning nothing this push (reconciliation sweep backstops)"); @@ -304,8 +322,13 @@ pub async fn resolve_candidates_for_push( mod tests { use super::*; use std::collections::HashSet; + use std::process::Command; use tempfile::TempDir; + fn td() -> std::time::Instant { + std::time::Instant::now() + std::time::Duration::from_secs(600) + } + /// Minimal git helper for building test repos. struct Repo { _td: TempDir, @@ -363,9 +386,10 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c2], &[&c1])) - .into_iter() - .collect(); + let got: HashSet = + delta(resolve_push_delta(&repo.path, &[&c2], &[&c1], "git", td())) + .into_iter() + .collect(); // The new blob b.txt and commit c2 are in the delta; the old blob a.txt // and commit c1 are not. let new_blob = repo.rev("HEAD:b.txt"); @@ -383,7 +407,7 @@ mod tests { // genuinely new objects, never fewer. let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[], "git", td())) .into_iter() .collect(); assert!(got.contains(&c1)); @@ -399,9 +423,15 @@ mod tests { // Rewrite history: reset to base, commit a different file. repo.git(&["reset", "-q", "--hard", &base]); let new_tip = repo.commit_file("c.txt", "three\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&new_tip], &[&old_tip])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&new_tip], + &[&old_tip], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&new_tip), "new tip in delta"); assert!(got.contains(&repo.rev(&format!("{new_tip}:c.txt")))); // No error; force-push computes new-minus-old cleanly. @@ -413,7 +443,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); // All updates were deletions => new_tips empty after the caller strips zeros. assert_eq!( - resolve_push_delta(&repo.path, &[], &[ZERO]), + resolve_push_delta(&repo.path, &[], &[ZERO], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -424,7 +454,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); assert_eq!( - resolve_push_delta(&repo.path, &[&blob], &[]), + resolve_push_delta(&repo.path, &[&blob], &[], "git", td()), PinCandidates::FullScanRequired, "a blob tip must force full scan (rev-list would exit 0 and walk it)" ); @@ -436,7 +466,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let tree = repo.rev("HEAD^{tree}"); assert_eq!( - resolve_push_delta(&repo.path, &[&tree], &[]), + resolve_push_delta(&repo.path, &[&tree], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -449,7 +479,7 @@ mod tests { repo.git(&["tag", "-a", "treetag", "-m", "x", &tree]); let tag = repo.rev("treetag"); assert_eq!( - resolve_push_delta(&repo.path, &[&tag], &[]), + resolve_push_delta(&repo.path, &[&tag], &[], "git", td()), PinCandidates::FullScanRequired, "annotated tag peeling to a tree must force full scan" ); @@ -465,7 +495,7 @@ mod tests { repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); assert_eq!( - resolve_push_delta(&repo.path, &[&t2], &[]), + resolve_push_delta(&repo.path, &[&t2], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -480,7 +510,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); repo.git(&["tag", "-a", "rel", "-m", "release", &c1]); let tag = repo.rev("rel"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[], "git", td())) .into_iter() .collect(); assert!( @@ -501,7 +531,7 @@ mod tests { let t1 = repo.rev("t1"); repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[], "git", td())) .into_iter() .collect(); assert!( @@ -516,9 +546,14 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let set = - resolve_candidates_for_push(repo.path.clone(), vec![c2.clone()], vec![c1.clone()]) - .await; + let set = resolve_candidates_for_push( + repo.path.clone(), + vec![c2.clone()], + vec![c1.clone()], + "git".to_string(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!set.full_scan, "happy-path delta is not a full scan"); let got: HashSet = set.candidates.into_iter().collect(); let new_blob = repo.rev("HEAD:b.txt"); @@ -536,8 +571,18 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); - let all: HashSet = list_all_objects(&repo.path).unwrap().into_iter().collect(); - let set = resolve_candidates_for_push(repo.path.clone(), vec![blob], vec![]).await; + let all: HashSet = list_all_objects(&repo.path, "git", td()) + .unwrap() + .into_iter() + .collect(); + let set = resolve_candidates_for_push( + repo.path.clone(), + vec![blob], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + ) + .await; assert!(set.full_scan, "non-commit tip is signalled as a full scan"); let got: HashSet = set.candidates.into_iter().collect(); assert_eq!(got, all, "non-commit tip falls back to full repo scan"); @@ -549,7 +594,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let bogus = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; assert_eq!( - resolve_push_delta(&repo.path, &[bogus], &[]), + resolve_push_delta(&repo.path, &[bogus], &[], "git", td()), PinCandidates::FullScanRequired, "a missing/corrupt tip OID must force full scan" ); @@ -564,7 +609,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); let old_tree = repo.rev(&format!("{c1}^{{tree}}")); - let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree]); + let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree], "git", td()); match result { PinCandidates::FullScanRequired => {} // safe: caller full-scans PinCandidates::Delta(objs) => { @@ -583,10 +628,15 @@ mod tests { // branch2 from base advances independently repo.git(&["checkout", "-q", "-b", "branch2", &base]); let b2 = repo.commit_file("c.txt", "three\n"); - let got: HashSet = - delta(resolve_push_delta(&repo.path, &[&b1, &b2], &[&base, &base])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&b1, &b2], + &[&base, &base], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&b1), "branch1 new commit"); assert!(got.contains(&b2), "branch2 new commit"); } @@ -595,7 +645,7 @@ mod tests { fn empty_repo_no_tips_yields_empty_delta() { let repo = Repo::new(); assert_eq!( - resolve_push_delta(&repo.path, &[], &[]), + resolve_push_delta(&repo.path, &[], &[], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -609,12 +659,12 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); assert_eq!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], true), + resolve_push_delta_inner(&repo.path, &[&c1], &[], true, "git", td()), PinCandidates::FullScanRequired ); // And with the flag off, the same push yields a Delta. assert!(matches!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], false), + resolve_push_delta_inner(&repo.path, &[&c1], &[], false, "git", td()), PinCandidates::Delta(_) )); } @@ -624,7 +674,7 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); repo.commit_file("b.txt", "two\n"); - let all = list_all_objects(&repo.path).unwrap(); + let all = list_all_objects(&repo.path, "git", td()).unwrap(); // 2 commits + 2 trees + 2 blobs = 6 objects. assert_eq!(all.len(), 6, "got: {all:?}"); } @@ -642,7 +692,7 @@ mod tests { std::fs::write(repo.path.join("orphan.bin"), b"dangling\n").unwrap(); let dangling = repo.git(&["hash-object", "-w", "orphan.bin"]); - let blobs = all_blob_oids(&repo.path).unwrap(); + let blobs = all_blob_oids(&repo.path, "git", td()).unwrap(); assert!(blobs.contains(&reachable_blob), "reachable blob present"); assert!( blobs.contains(&dangling), @@ -652,7 +702,7 @@ mod tests { assert!(!blobs.contains(&tree), "tree is not a blob"); // The typed lister tags each object; spot-check the dangling blob's type. - let typed = list_all_objects_with_type(&repo.path).unwrap(); + let typed = list_all_objects_with_type(&repo.path, "git", td()).unwrap(); assert!( typed .iter() @@ -660,4 +710,40 @@ mod tests { "dangling object is typed as a blob" ); } + + #[cfg(unix)] + #[test] + fn list_all_objects_times_out_on_a_hung_git_instead_of_blocking() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git: hang on cat-file (bounded to 30s so a broken test can't wedge). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + + let (tx, rx) = std::sync::mpsc::channel(); + let path = dir.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(list_all_objects( + &path, + &git_bin, + Instant::now() + Duration::from_millis(150), + )); + }); + let res = rx.recv_timeout(Duration::from_secs(10)).expect( + "list_all_objects must return within the watchdog budget, not block on a hung git", + ); + assert!( + res.is_err(), + "a hung git must make list_all_objects error out, not hang" + ); + } } diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index ec55423e..9f40cd4b 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -42,7 +42,7 @@ const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); /// drive the teardown in tests without mutating the process-global PATH; /// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). #[cfg(unix)] -fn run_bounded_git( +pub(crate) fn run_bounded_git( git_bin: &str, args: &[&str], repo_path: &Path, @@ -171,7 +171,7 @@ fn run_bounded_git( /// the Unix version's signature and result semantics so every caller compiles on all /// targets (#174). #[cfg(not(unix))] -fn run_bounded_git( +pub(crate) fn run_bounded_git( git_bin: &str, args: &[&str], repo_path: &Path, @@ -1235,7 +1235,12 @@ esac\n"; String::from_utf8_lossy(&out.stdout).trim().to_string() }; - let all_blobs = crate::git::push_delta::all_blob_oids(&work).unwrap(); + let all_blobs = crate::git::push_delta::all_blob_oids( + &work, + "git", + std::time::Instant::now() + std::time::Duration::from_secs(600), + ) + .unwrap(); assert!( all_blobs.contains(&dangling_oid), "precondition: the dangling blob is in the all-objects universe" From 4410b834df04163c6121629e3e3b05b227501e27 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:46:24 -0500 Subject: [PATCH 24/52] docs(node): correct concurrency + timeout docs to match the final git routing (#174) jatmn P2 (INV-24): the config help, .env.example, and README described routing the handler no longer does. - The read pool serves upload-pack and the UPLOAD-PACK info/refs advertisement only; the anon receive-pack info/refs advertisement draws from its own dedicated pool (sized like the write pool but disjoint), not the read pool and not the write pool. - The write pool is the authenticated push POST only. - The per-caller read cap is keyed on the resolved source IP, never the DID: a signature does not move a caller off it (the docs claimed a per-DID budget, which would tell a NATed authenticated client it escapes the shared cap when it does not). - git_service_timeout_secs now also bounds the push-side pin-candidate discovery (rev-list / cat-file), reaped via process-group teardown, alongside the info/refs advertisements and the withheld-blob walk. --- .env.example | 40 ++++++++++++--------- README.md | 2 +- crates/gitlawb-node/src/config.rs | 60 +++++++++++++++++-------------- 3 files changed, 57 insertions(+), 45 deletions(-) diff --git a/.env.example b/.env.example index cbf3db3a..0ef193f8 100644 --- a/.env.example +++ b/.env.example @@ -109,29 +109,35 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Max seconds a served git upload-pack / receive-pack (clone / push) may run # before it is aborted with a 504. Bounds a hung git that would otherwise pin a -# worker and, on push, the repo write lock. Also bounds the info/refs -# advertisement and the withheld-blob pack build (#174). Default 600. +# worker and, on push, the repo write lock. Also bounds both info/refs +# advertisements, the withheld-blob pack build, and the push-side candidate +# discovery (rev-list / cat-file), all reaped via process-group teardown (#174). +# Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 -# Max concurrent git read ops (upload-pack + info/refs advertisements) served at -# once, a global pool separate from the push pool below. Over-cap sheds a clean -# 503 + Retry-After. Anonymous reads draw from here, so pair it with -# GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one caller cannot monopolize -# the pool. Default 128. +# Max concurrent git READ ops (upload-pack + the upload-pack info/refs +# advertisement) served at once, a global pool separate from the push pool below. +# The anon receive-pack info/refs advertisement has its OWN pool (see below), not +# this one. Over-cap sheds a clean 503 + Retry-After. Anonymous reads draw from +# here, so pair it with GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one +# caller cannot monopolize the pool. Default 128. GITLAWB_MAX_CONCURRENT_GIT_OPS=128 -# Max concurrent git-receive-pack (push) operations, in a pool separate from the -# read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an -# authenticated push at admission. Over-cap sheds a 503 + Retry-After. Default 32. +# Max concurrent git-receive-pack (push) POST operations, in a pool separate from +# the read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an +# authenticated push at admission. The anon receive-pack info/refs advertisement +# runs in a SEPARATE pool of the same size (disjoint from this one), so an +# advertisement flood cannot shed a push either. Over-cap sheds a 503 + +# Retry-After. Default 32. GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 -# Max concurrent read ops (upload-pack + info/refs advertisements) a single caller -# may hold, so one caller cannot monopolize the read pool. Keyed per-DID when -# signed, else per-source-IP. The per-IP key is only as granular as -# GITLAWB_TRUSTED_PROXY below: left unset, a node behind an edge/NAT keys all -# anonymous callers on the edge IP and this collapses to one global anonymous cap. -# Set GITLAWB_TRUSTED_PROXY for per-client keying; a high-fanout caller (CI behind -# one NAT) should authenticate for a per-DID budget or the operator raises this. +# Max concurrent read ops (upload-pack + the upload-pack info/refs advertisement) +# a single caller may hold, so one caller cannot monopolize the read pool. Keyed +# on the resolved SOURCE IP, never the DID: a signature does not move a caller off +# this cap. The source-IP key is only as granular as GITLAWB_TRUSTED_PROXY below: +# left unset, a node behind an edge/NAT keys all callers on the edge IP and this +# collapses to one global cap. Set GITLAWB_TRUSTED_PROXY for per-client keying; a +# high-fanout caller (CI behind one NAT) then needs the operator to raise this. # Default 16. GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 diff --git a/README.md b/README.md index 0f760449..542a988e 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths), which is reaped at the deadline. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 90a061e0..d0a4c319 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -243,24 +243,25 @@ pub struct Config { /// cap is a different axis (500 connections each fan out to git + /// pack-objects + threads). Size below the process budget with headroom. /// - /// This is the READ pool (`git_read_semaphore`): upload-pack and both info/refs - /// advertisements. The authenticated push POST draws from a separate write pool - /// (`max_concurrent_git_pushes`) that anonymous reads can never reach, and each - /// caller is additionally bounded by `max_concurrent_reads_per_caller`, so an - /// anonymous flood cannot shed the actual push nor monopolize reads (#174). (The - /// receive-pack advertisement itself shares the read pool; a shed advertisement - /// is a cheap retryable GET, and the write POST it precedes always has capacity.) + /// This is the READ pool (`git_read_semaphore`): upload-pack and the UPLOAD-PACK + /// `info/refs` advertisement only. The authenticated push POST draws from a + /// separate write pool (`max_concurrent_git_pushes`) that anonymous reads can + /// never reach, and each read caller is additionally bounded by + /// `max_concurrent_reads_per_caller`, so an anonymous flood cannot shed the actual + /// push nor monopolize reads (#174). The anon-reachable RECEIVE-PACK `info/refs` + /// advertisement draws from its OWN dedicated pool (sized like the write pool but + /// disjoint), so an advertisement flood can never occupy a permit the + /// authenticated push POST needs at admission (#174). /// /// A permit is held for the whole op. Every git subprocess that STREAMS is /// duration-bounded and reaps its process group on disconnect: upload-pack, /// receive-pack, and both info/refs advertisements run under /// `git_service_timeout_secs` with `process_group(0)` teardown, and the - /// withheld-blob (`upload_pack_excluding`) pack-objects stage runs on the async - /// side under the same teardown (#174 closed the two duration/cancellation gaps - /// this comment previously tracked). The one remaining blocking stage is the - /// `rev-list` object enumeration in the withheld-blob path — a bounded walk that - /// terminates, run off the async runtime; it is not process-group-reaped on - /// disconnect, so a stuck rev-list can still hold its slot until git exits. + /// withheld-blob (`upload_pack_excluding`) pack-objects stage plus the push-side + /// candidate-discovery children (`rev-list` / `cat-file`) now run under the same + /// bounded runner with process-group teardown, so a stuck git child no longer + /// holds its slot indefinitely (#174 closed the duration/cancellation gaps this + /// comment previously tracked). /// /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value /// well under tokio's `Semaphore` permit limit so an oversized value is a @@ -273,10 +274,14 @@ pub struct Config { )] pub max_concurrent_git_ops: usize, - /// Maximum number of concurrent `git-receive-pack` (push) operations. Pushes - /// draw from this dedicated pool, separate from `max_concurrent_git_ops` - /// (reads), so a flood of anonymous reads cannot shed an authenticated push at - /// admission (#174). Beyond this a push sheds a clean 503 + Retry-After. + /// Maximum number of concurrent `git-receive-pack` (push) operations. The + /// authenticated push POST draws from this dedicated pool, separate from + /// `max_concurrent_git_ops` (reads), so a flood of anonymous reads cannot shed an + /// authenticated push at admission (#174). The anon-reachable receive-pack + /// `info/refs` advertisement runs in a SEPARATE pool of the same size (derived + /// from this knob), disjoint from this one, so an advertisement flood cannot + /// occupy a POST's slot either (#174). Beyond this a push sheds a clean 503 + + /// Retry-After. /// /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI @@ -289,16 +294,17 @@ pub struct Config { )] pub max_concurrent_git_pushes: usize, - /// Maximum concurrent read operations (`upload-pack` and the `info/refs` - /// advertisements) a single caller may hold at once, so one caller cannot - /// monopolize the `max_concurrent_git_ops` read pool (#174). Callers are keyed - /// per-DID when signed, else per-source-IP. IMPORTANT: the per-source-IP key is - /// only as granular as `GITLAWB_TRUSTED_PROXY`. Left unset (the default), a node - /// behind an edge/NAT keys all anonymous callers on the edge IP, so this cap - /// collapses to a single global anonymous cap rather than per-client. Set - /// `GITLAWB_TRUSTED_PROXY` to key on the real client; a known high-fanout caller - /// (a CI fleet behind one NAT) should authenticate for a per-DID budget or the - /// operator raises this. Over-cap for a caller sheds a clean 503 + Retry-After. + /// Maximum concurrent read operations (`upload-pack` and the upload-pack + /// `info/refs` advertisement) a single caller may hold at once, so one caller + /// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are + /// keyed on the RESOLVED SOURCE IP, never the DID — a signature does not move a + /// caller off this cap, so an authenticated client cannot mint DIDs to escape it. + /// IMPORTANT: the source-IP key is only as granular as `GITLAWB_TRUSTED_PROXY`. + /// Left unset (the default), a node behind an edge/NAT keys all callers on the + /// edge IP, so this cap collapses to a single global cap rather than per-client. + /// Set `GITLAWB_TRUSTED_PROXY` to key on the real client; a high-fanout caller (a + /// CI fleet behind one NAT) then needs the operator to raise this. Over-cap for a + /// caller sheds a clean 503 + Retry-After. /// /// Default: 16. Must be between 1 and 1_048_576. #[arg( From cf97405bc188e67beaff37c6a708694dda04fe94 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:55:57 -0500 Subject: [PATCH 25/52] test(node): execution-cover the delta-path push-scan timeout wiring (#174) The finding-2 wiring test covered only list_all_objects (the full-scan fallback). Add delta_path_exec_fns_time_out_on_a_hung_git covering the common push path: peeled_object_type (cat-file), rev_list_delta (rev-list), and list_all_objects_with_type (cat-file, which all_blob_oids delegates to). Each must return within the watchdog budget on a hung git; reverting any to a bare Command::output() blocks past the recv budget (RED-verified). --- crates/gitlawb-node/src/git/push_delta.rs | 56 +++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 51b6f836..962c9c65 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -746,4 +746,60 @@ mod tests { "a hung git must make list_all_objects error out, not hang" ); } + + /// #174 finding 2: the DELTA-path children — `peeled_object_type` (cat-file), + /// `rev_list_delta` (rev-list) — and `list_all_objects_with_type` (cat-file) are + /// the remaining candidate-discovery execs (the common push path). Each must + /// return within the watchdog budget on a hung git rather than block and pin the + /// write permit. Revert any one to a bare `Command::output()` and its arm blocks + /// past the recv budget (RED). + #[cfg(unix)] + #[test] + fn delta_path_exec_fns_time_out_on_a_hung_git() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git hangs on BOTH cat-file and rev-list (bounded 30s so a broken + // test can't leak a permanent orphan or wedge the suite). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file|rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + let path = dir.path().to_path_buf(); + + // Run `f` on a thread and require it to RETURN (not block) within 10s. + fn returns_within(f: impl FnOnce() -> T + Send + 'static) -> T { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(Duration::from_secs(10)).expect( + "a delta-path exec fn must return within the watchdog budget, not block on a hung git", + ) + } + let dl = || Instant::now() + Duration::from_millis(150); + let sha = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || peeled_object_type(&p, sha, &g, d)).is_none(), + "peeled_object_type must time out to None on a hung cat-file" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || rev_list_delta(&p, &[sha], &[], &g, d)).is_err(), + "rev_list_delta must error out on a hung rev-list" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || list_all_objects_with_type(&p, &g, d)).is_err(), + "list_all_objects_with_type must error out on a hung cat-file" + ); + } } From a030eefb704d68a008e760a01077abc823cbe258 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 16:01:49 -0500 Subject: [PATCH 26/52] test(node): retry the ETXTBSY exec race in build_filtered_pack timeout tests (#174) build_filtered_pack_times_out_a_hung_rev_list and _a_hung_pack_objects spawn a freshly-written fake git directly, bypassing the fake_git_run_with_pids retry the rest of the module uses. Under cargo test fork-storm load a concurrent worker forks while the fake's write fd is still open, so execve returns ETXTBSY and the raw io::Error surfaces where the test asserts GitServiceTimeout, failing test (stable) intermittently. Route both tests through build_filtered_pack_or_exec_race_retry, which retries ONLY errno 26 (ETXTBSY) and keeps the outer 10s watchdog per attempt, so a genuinely-missing async timeout bound still trips the watchdog and a wrong error type still fails at the assertion. Production paths unchanged. --- crates/gitlawb-node/src/git/smart_http.rs | 99 ++++++++++++++++------- 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 206e7b27..f885aef3 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -1160,6 +1160,61 @@ mod tests { ); } + /// True when `err` is the transient ETXTBSY exec race a freshly-written fake + /// `git` hits under fork-storm load — a concurrent worker forked while this + /// file's write fd was still open, so `execve` sees it as busy (the same race + /// `fake_git_run_with_pids` retries). Deliberately narrow: only this raw-OS + /// error is retried, so a wrong error *type* (e.g. a missing async bound + /// surfacing as anything other than `GitServiceTimeout`) still fails loudly at + /// the caller's assertion instead of being swallowed. `spawn()?` propagates the + /// `io::Error` with no context, so it survives as the anyhow root. + #[cfg(unix)] + fn is_transient_exec_race(err: &anyhow::Error) -> bool { + // ETXTBSY == 26 on Linux and the BSDs/macOS; std has no stable ErrorKind. + err.downcast_ref::() + .and_then(std::io::Error::raw_os_error) + == Some(26) + } + + /// Drive `build_filtered_pack` under a per-attempt watchdog, retrying ONLY the + /// transient ETXTBSY exec race and returning the terminal error for the caller + /// to classify. Every attempt keeps its own outer `tokio::time::timeout`, so a + /// MISSING async bound — the regression the `build_filtered_pack_times_out_*` + /// tests guard — still trips the watchdog loudly on every attempt: the retry + /// can only absorb a fast exec failure, never a hang (a hang never returns, so + /// it can never reach the retry decision). + #[cfg(unix)] + async fn build_filtered_pack_or_exec_race_retry( + git_bin: &str, + repo_path: &std::path::Path, + withheld: &HashSet, + stage_timeout: Duration, + ) -> anyhow::Error { + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the git stage \ + must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung git stage must return an error, not hang"); + if is_transient_exec_race(&err) { + // Fresh-fake-git ETXTBSY: back off (growing) so a bursty fork-pressure + // spike subsides before retrying, per fake_git_run_with_pids. + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + return err; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + // Dropping the request future mid-flight (client disconnect) must SIGTERM the // whole group so git AND its pack-objects grandchild die together. Goes RED // if `process_group(0)` or the guard-arming is removed: without its own @@ -1314,21 +1369,15 @@ mod tests { let git_bin = write_fake_git(tmp.path(), body); let withheld = HashSet::new(); - let result = tokio::time::timeout( - Duration::from_secs(10), - build_filtered_pack( - git_bin.to_str().unwrap(), - tmp.path(), - &withheld, - Duration::from_millis(200), - ), + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), ) - .await - .expect( - "build_filtered_pack must return within the watchdog — the pack-objects \ - stage must be timeout-bounded, not an uncancellable spawn_blocking", - ); - let err = result.expect_err("a hung pack-objects must return an error, not hang"); + .await; assert!( err.downcast_ref::().is_some(), "a hung pack-objects must abort with GitServiceTimeout, got: {err}" @@ -1352,21 +1401,15 @@ mod tests { let git_bin = write_fake_git(tmp.path(), body); let withheld = HashSet::new(); - let result = tokio::time::timeout( - Duration::from_secs(10), - build_filtered_pack( - git_bin.to_str().unwrap(), - tmp.path(), - &withheld, - Duration::from_millis(200), - ), + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), ) - .await - .expect( - "build_filtered_pack must return within the watchdog — the rev-list \ - stage must be timeout-bounded, not an uncancellable spawn_blocking", - ); - let err = result.expect_err("a hung rev-list must return an error, not hang"); + .await; assert!( err.downcast_ref::().is_some(), "a hung rev-list must abort with GitServiceTimeout, got: {err}" From 3cc01890fbb3b395a336df951cff0e5b89393530 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 19:27:12 -0500 Subject: [PATCH 27/52] fix(node): keep the visibility-walk watchdog armed until the child is reaped (#174) A group member (or the leader itself) that closes stdout before the deadline made the stdout drain return EOF early, which stood the watchdog down via done_tx before it could ever fire; child.wait() then blocked on the still-live child, pinning the walk thread and its read/write permit past GITLAWB_GIT_SERVICE_TIMEOUT_SECS. This is distinct from the descendant case already covered: there the leader sleeps until the deadline so the watchdog does time out; here the drain-EOF races ahead of it. Stand the watchdog down only once the child has actually terminated, detected without reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its pgid un-recycled until the watchdog finishes its teardown and is joined. Past the deadline the watchdog still owns the full SIGTERM -> grace -> SIGKILL sequence. Regression: run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs, RED before (blocks on child.wait past the recv budget), GREEN after; the three existing teardown tests still pass (no recycle-hazard regression). --- .../gitlawb-node/src/git/visibility_pack.rs | 116 ++++++++++++++++-- 1 file changed, 108 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 9f40cd4b..9f07020d 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -41,6 +41,31 @@ const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); /// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can /// drive the teardown in tests without mutating the process-global PATH; /// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). +/// Returns true if `pid` (a process-group leader we spawned) has terminated, WITHOUT +/// reaping it. `waitid(..., WNOWAIT)` reports the exit state but leaves the child +/// waitable, so the caller's later `child.wait()` still collects the status and the +/// pid/pgid stays live until then — which is what keeps the watchdog's `kill(-pgid)` +/// teardown from ever racing a recycled pgid. Used to distinguish "the child actually +/// exited" from "the child merely closed stdout" after the drain returns (#174 P1-a). +#[cfg(unix)] +fn child_terminated_without_reaping(pid: i32) -> bool { + // SAFETY: waitid writes only into the zeroed siginfo and borrows no Rust memory; + // WNOWAIT leaves the child unreaped, WNOHANG makes the probe non-blocking. + let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; + let rc = unsafe { + libc::waitid( + libc::P_PID, + pid as libc::id_t, + &mut info, + libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, + ) + }; + // rc == 0 with si_pid == 0 means "no state change yet" (still running); a non-zero + // si_pid means the child has entered a waitable, exited state. EINTR/other errors + // (rc != 0) are treated as "not yet terminated" and the caller re-polls. + rc == 0 && unsafe { info.si_pid() } != 0 +} + #[cfg(unix)] pub(crate) fn run_bounded_git( git_bin: &str, @@ -132,14 +157,28 @@ pub(crate) fn run_bounded_git( // is unreachable in userspace (no signal reaps a D-state process) and matches the // async `reap_group_on_timeout`, which likewise only warns and gives up there. let read_result = stdout.read_to_end(&mut out); - // The drain has returned: either the child exited on its own, or the watchdog's - // teardown closed its pipes. Tell the watchdog the drain is done, then WAIT for it - // to return BEFORE reaping the leader. On the timeout path the watchdog runs the - // full SIGTERM -> grace -> SIGKILL teardown; joining it before `child.wait()` keeps - // the leader's pid unreaped while every `kill(-pgid)` fires (so the pgid can't be - // recycled), and guarantees a SIGTERM-ignoring group member has already been - // SIGKILLed rather than left running past the deadline (finding 3, #174). - let _ = done_tx.send(()); + // The drain has returned, but that only means all stdout write ends are closed — + // NOT that the child has exited. A group member, or the leader itself, can close + // stdout and keep running; standing the watchdog down on the drain alone (as the + // old code did) would then let `child.wait()` block forever on that live child, + // past the deadline, pinning the walk thread and its permit (finding P1-a, #174). + // So stand the watchdog down only once the child has ACTUALLY terminated, detected + // WITHOUT reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its + // pgid un-recycled until the watchdog finishes and we join it below. Past the + // deadline the watchdog owns the teardown, so we stop polling and let it run the + // full SIGTERM -> grace -> SIGKILL; joining it before `child.wait()` keeps every + // `kill(-pgid)` firing while the pid is still unreaped and guarantees a + // stdout-closing-then-hanging member has been SIGKILLed rather than left running. + loop { + if child_terminated_without_reaping(pgid) { + let _ = done_tx.send(()); + break; + } + if Instant::now() >= deadline { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } let killed = watchdog.join().unwrap_or(false); let status = child.wait().context("git wait failed")?; let err = err_reader.join().unwrap_or_default(); @@ -902,6 +941,67 @@ esac\n"; "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned" ); } + + /// #174 U1 (P1-a, RED-before/GREEN-after): the group LEADER closes its own + /// stdout/stderr BEFORE the deadline and then keeps running. On the pre-fix code + /// the stdout drain returns EOF early, `done_tx.send` stands the watchdog down + /// before it ever fires (`recv` gets `Ok` -> `false`, no kill), and `child.wait()` + /// then blocks on the still-alive leader — pinning the walk thread and its read/ + /// write permit past the deadline, bypassing GITLAWB_GIT_SERVICE_TIMEOUT_SECS. + /// This is distinct from the descendant case above: there the leader sleeps until + /// the deadline so the watchdog DOES time out; here the drain-EOF races ahead of + /// the deadline. The fix keeps the watchdog armed until the child is actually + /// reaped, so the deadline SIGTERM still fires and the call returns within budget. + /// A pre-fix build blocks on `child.wait()` past the recv budget (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // rev-list records its (leader) pid, closes stdout+stderr so the drain EOFs + // immediately, then sleeps without trapping TERM. `sleep 30` bounds the worst + // case so a RED run cannot wedge the suite; the recv budget fires first. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > leader.pid; exec 1>&- 2>&-; sleep 30 ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx.recv_timeout(Duration::from_secs(10)).expect( + "run_bounded_git must return within the watchdog budget when the leader closes stdout then hangs, not block on child.wait()", + ); + assert!( + out.is_err(), + "a leader killed at the deadline (no TERM trap) is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("leader.pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(pid, libc::SIGKILL) }; + assert!( + gone, + "the hung leader (pid {pid}) must be killed and reaped at the deadline, not left running" + ); + } + use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; From ac59bd7e01bf42b7d42b7adb57ccb927c0309993 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 19:50:41 -0500 Subject: [PATCH 28/52] fix(node): reap the served-git process group on disconnect, not just SIGTERM (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a client disconnect KillGroupOnDrop sent a lone SIGTERM with no escalation or reap, so a group member that ignores SIGTERM (a wedged pack-objects, a malicious helper) survived while the handler released its concurrency permit — letting disconnect-spam accumulate live git processes and defeat the load shed (P1-c). The timeout path already ran the full teardown. Make the disconnect teardown as strong as the timeout path: the guard now owns the child and, on drop, launches a detached reaper that runs the same TERM -> grace -> SIGKILL -> reap-the-group sequence (reap_group_on_timeout). Owning the tokio Child in the reaper keeps it the sole reaper of the leader, so it never races tokio's own orphan reaper (a raw waitpid(-pgid) would). Regression: run_git_service_sigkills_a_sigterm_ignoring_child_on_disconnect, RED before (the descendant survives the lone SIGTERM), GREEN after; all existing teardown/timeout/disarm tests still pass (the three hand-built-guard tests were updated to the owned-child shape). Residual: capacity (the handler's semaphore permit) still releases at disconnect rather than strictly after the reaper confirms the group dead, so accumulation is now bounded to ~one teardown window (<=4s) rather than eliminated; gating the permit on the reap needs threading it through the serve handlers and is left for a follow-up (it also touches the receive-pack permit lifetime U4 changes). --- crates/gitlawb-node/src/git/smart_http.rs | 235 +++++++++++++++++----- 1 file changed, 188 insertions(+), 47 deletions(-) diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index f885aef3..a5649a42 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -108,24 +108,60 @@ pub async fn receive_pack( /// `wait_with_output()` returns, so a request that completed cleanly never signals. #[cfg(unix)] struct KillGroupOnDrop { + // Holds the child + its pgid while armed. The interaction drives the child through + // `child_mut()`; the success/timeout paths call `disarm` once they have reaped it. + // On drop — a client disconnect that drops the whole request future — the guard + // launches a detached reaper that OWNS the child and runs the full + // SIGTERM -> grace -> SIGKILL -> reap sequence (`reap_group_on_timeout`), as strong + // as the timeout path, so a SIGTERM-ignoring group member is SIGKILLed and reaped + // rather than left running until EPIPE to accumulate past the concurrency cap + // (#174 P1-c). Owning the tokio `Child` in the reaper (rather than a raw + // `waitpid(-pgid)`) keeps a single reaper of the leader, so it never races tokio's + // own SIGCHLD-driven orphan reaper. + child: Option, pgid: Option, } #[cfg(unix)] impl KillGroupOnDrop { + /// The child, while armed. The interaction drives stdin/stdout/`wait` through this. + fn child_mut(&mut self) -> &mut tokio::process::Child { + self.child.as_mut().expect("child present while armed") + } + + /// Disarm on the success/timeout path: the body has already reaped the child (its + /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear + /// the pgid, leaving the guard's Drop a no-op. fn disarm(&mut self) { self.pgid = None; + self.child = None; } } #[cfg(unix)] impl Drop for KillGroupOnDrop { fn drop(&mut self) { - if let Some(pgid) = self.pgid { - // SAFETY: kill(2) takes only integer arguments and borrows no Rust - // memory. Signalling a stale group just returns ESRCH, which we ignore. - unsafe { - libc::kill(-pgid, libc::SIGTERM); + let (Some(mut child), Some(pgid)) = (self.child.take(), self.pgid) else { + return; + }; + // A sync Drop cannot await, so launch a detached reaper that owns the child and + // runs the same teardown as the timeout path (TERM -> grace -> SIGKILL -> reap + // the whole group). Owning the tokio `Child` means this task is the sole reaper + // of the leader, so it cannot race tokio's orphan reaper. Prefer the current + // runtime handle; if dropped outside a runtime, fall back to a best-effort + // synchronous SIGTERM so the group is at least signalled. + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + handle.spawn(async move { + reap_group_on_timeout(&mut child).await; + }); + } + Err(_) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { + libc::kill(-pgid, libc::SIGTERM); + } } } } @@ -244,20 +280,28 @@ async fn drive_git_child( let mut child = command.spawn()?; - // Arm the group-kill guard for the lifetime of the request. With - // process_group(0) the child is its own group leader, so pgid == its pid. - // This fires on a client disconnect (the whole future is dropped mid-request). + // Own the pipes so the child stays reap-able: wait_with_output would consume it, + // but on a timeout we must actively reap the group first (see + // reap_group_on_timeout), and on a disconnect the guard's detached reaper does. + // Take the pipes before the child moves into the guard below. + let mut stdin = child.stdin.take(); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + + // Arm the group-kill guard for the lifetime of the request. With process_group(0) + // the child is its own group leader, so pgid == its pid. On a client disconnect + // (the whole future is dropped mid-request) the guard's Drop launches a detached + // reaper that OWNS the child and runs the full TERM/grace/SIGKILL/reap — as strong + // as the timeout path below (#174 P1-c). The guard owns the child; the interaction + // drives it through `child_mut()`. + #[cfg(unix)] + let pgid = child.id().map(|id| id as i32); #[cfg(unix)] let mut group_guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid, }; - // Own the pipes so `child` stays reap-able after a timeout: wait_with_output - // would consume it, but on timeout we must actively reap the group before - // returning (see reap_group_on_timeout). - let mut stdin = child.stdin.take(); - let mut stdout = child.stdout.take().context("git stdout was not piped")?; - let mut stderr = child.stderr.take().context("git stderr was not piped")?; let mut out = Vec::new(); let mut err = Vec::new(); @@ -274,11 +318,15 @@ async fn drive_git_child( None => Ok(()), } }; + #[cfg(unix)] + let child_ref = group_guard.child_mut(); + #[cfg(not(unix))] + let child_ref = &mut child; let (write_result, r_out, r_err, status) = tokio::join!( write, stdout.read_to_end(&mut out), stderr.read_to_end(&mut err), - child.wait(), + child_ref.wait(), ); r_out?; r_err?; @@ -290,7 +338,7 @@ async fn drive_git_child( Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the - // guard's drop would fire SIGTERM on the reaped, possibly-reused pgid. + // guard's drop would reap an already-reaped child / signal a reused pgid. #[cfg(unix)] group_guard.disarm(); result? @@ -301,7 +349,7 @@ async fn drive_git_child( // the (now redundant) guard so its drop can't hit a reused pgid. #[cfg(unix)] { - reap_group_on_timeout(&mut child).await; + reap_group_on_timeout(group_guard.child_mut()).await; group_guard.disarm(); } #[cfg(not(unix))] @@ -960,23 +1008,36 @@ mod tests { #[cfg(unix)] #[tokio::test] async fn kill_group_guard_terminates_child_on_drop() { - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; { - let _guard = KillGroupOnDrop { pgid }; - } // guard drops here -> SIGTERM to the group - - use std::os::unix::process::ExitStatusExt; - let status = child.wait().await.unwrap(); - assert_eq!( - status.signal(), - Some(libc::SIGTERM), - "child must be terminated by SIGTERM via its process group" + // The guard owns the child; on drop its detached reaper TERM/grace/KILL/ + // reaps the group (a plain sleep dies on the first SIGTERM). + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + }; + } + + let mut gone = false; + for _ in 0..300 { + if !alive(pid) { + gone = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + gone, + "child must be terminated and reaped via its process group on guard drop" ); } @@ -994,9 +1055,10 @@ mod tests { .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; - // Read the backgrounded grandchild's pid from the first stdout line. + // Read the backgrounded grandchild's pid from the first stdout line, before + // the child moves into the guard. let mut stdout = child.stdout.take().unwrap(); let mut buf = Vec::new(); loop { @@ -1011,20 +1073,25 @@ mod tests { assert!(alive(grandchild), "grandchild should be running"); { - let _guard = KillGroupOnDrop { pgid }; - } // group SIGTERM reaches sh AND the sleep grandchild - - let _ = child.wait().await; // reap sh + // The guard owns the child; on drop the detached reaper group-kills sh AND + // the sleep grandchild. + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + }; + } - // The grandchild reparents to init and is reaped; poll until it's gone. let mut gone = false; - for _ in 0..200 { + for _ in 0..300 { if !alive(grandchild) { gone = true; break; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } + unsafe { + libc::kill(grandchild, libc::SIGKILL); + } assert!( gone, "grandchild must be terminated by the group signal (#53)" @@ -1036,27 +1103,31 @@ mod tests { async fn kill_group_guard_disarmed_does_not_kill() { // A request that completed cleanly disarms the guard; dropping it must not // signal anything. - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); + let pid = child.id().unwrap() as i32; { let mut guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid: Some(pid), }; guard.disarm(); - } // disarmed -> no kill + } // disarmed -> no reaper, no kill - assert!( - child.try_wait().unwrap().is_none(), - "disarmed guard must not kill the child" - ); + // Give any erroneously-spawned reaper a chance to run, then assert alive. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!(alive(pid), "disarmed guard must not kill the child"); - // Clean up the still-running child. - let _ = child.kill().await; - let _ = child.wait().await; + // Clean up the still-running child (disarm dropped the handle, so reap by pid). + unsafe { + libc::kill(pid, libc::SIGKILL); + let mut status = 0; + libc::waitpid(pid, &mut status, 0); + } } // ── #62 PR1: end-to-end teardown wiring through run_git_service ───────── @@ -1320,6 +1391,76 @@ mod tests { ); } + /// #174 U2 (P1-c, RED-before/GREEN-after): on a client disconnect the teardown must + /// be as strong as the timeout path — a group member that IGNORES SIGTERM is still + /// SIGKILLed and reaped, not left running to accumulate past the concurrency cap. + /// The old `KillGroupOnDrop::drop` sent a lone SIGTERM with no escalation or reap, + /// so a SIGTERM-ignoring descendant survived the disconnect (RED). The fix launches + /// a detached reaper owning the child that runs the full TERM/grace/SIGKILL/reap. + /// This is distinct from the well-behaved-grandchild test above, whose `sleep` dies + /// on the first SIGTERM. + #[cfg(unix)] + #[tokio::test] + async fn run_git_service_sigkills_a_sigterm_ignoring_child_on_disconnect() { + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Leader (dies on the group SIGTERM) spawns a descendant that traps SIGTERM, + // records its own pid, and loops ~30s (bounded so a RED run leaks no permanent + // orphan; the assertion fires well before then). + let body = format!( + "#!/bin/sh\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n", + descfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut fut = Box::pin(run_git_service( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Bytes::new(), + Duration::from_secs(60), + )); + // Drive the future a slice at a time until the descendant records its pid. + let mut desc: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let _cleanup = ReapOnPanic(vec![desc]); + assert!(alive(desc), "descendant should be running before the drop"); + + // Client disconnect: drop the request future. The guard's detached reaper must + // escalate SIGTERM -> SIGKILL and reap the SIGTERM-ignoring descendant. + drop(fut); + + let mut gone = false; + for _ in 0..500 { + if !alive(desc) { + gone = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + // Kill regardless so a RED run leaks no orphan. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed and reaped on \ + disconnect, not left running (old KillGroupOnDrop sent SIGTERM only)" + ); + } + // #174 (SC3): the info/refs advertisement is now duration-bounded — previously // it ran a bare `Command::output()` with no deadline, so a hung git pinned its // concurrency slot forever. A hung fake git must abort with GitServiceTimeout From be0cdd6a9eae1b39618f16fdf4e28eae6e83ca51 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:09:40 -0500 Subject: [PATCH 29/52] fix(node): hold upload-pack admission through the walk on disconnect (#174) On the path-scoped upload-pack path the per-source and global read permits were handler locals, so a client disconnect dropped the handler future and released both permits while the awaited spawn_blocking withheld-blob walk kept running (spawn_blocking can't be cancelled). A permitless caller could disconnect-spam to exceed the read cap while real git work continued (P1-b). Move both permits into the blocking task and return them out: on success they flow back so the serve phase keeps them; on a dropped future the returned tuple (with the permits) is discarded only when the blocking task completes, so admission tracks the walk's real duration. Regression: upload_pack_permit_held_through_walk_after_disconnect drives the handler to mid-walk (fake git hangs on rev-list), disconnects, and asserts the global read slot stays held (available_permits == 0). RED before (the slot frees to 1 the instant the future drops), GREEN after. The test roots the repo store at its own TempDir so it is isolated per run. --- crates/gitlawb-node/src/api/repos.rs | 166 +++++++++++++++++++++++++-- 1 file changed, 158 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index ea6f126e..8612709c 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -819,9 +819,16 @@ pub async fn git_upload_pack( let resp = if !visibility_pack::has_path_scoped_rule(&rules) { smart_http::upload_pack(&disk_path, body, git_timeout).await } else { - // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep - // that off the async worker thread. - let withheld = { + // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep that + // off the async worker thread. Move BOTH admission permits INTO the blocking + // task so they are held for the walk's real duration: spawn_blocking cannot be + // cancelled, so on a client disconnect the handler future drops but the walk + // keeps running — and now so do its permits, released only when the walk + // finishes rather than the instant the future drops (#174 P1-b). On success the + // task hands the permits back so the serve phase below keeps them; on a + // dropped future the returned tuple (with the permits) is discarded only when + // the blocking task completes, so admission tracks the real git work. + let (withheld, _permit, _caller_permit) = { let path = disk_path.clone(); let rules = rules.clone(); let owner_did = record.owner_did.clone(); @@ -829,7 +836,7 @@ pub async fn git_upload_pack( let is_public = record.is_public; let git_bin = state.git_bin.clone(); tokio::task::spawn_blocking(move || { - visibility_pack::withheld_blob_oids_bounded( + let withheld = visibility_pack::withheld_blob_oids_bounded( &path, &git_bin, git_timeout, @@ -837,14 +844,15 @@ pub async fn git_upload_pack( is_public, &owner_did, caller_owned.as_deref(), - ) + ); + (withheld, _permit, _caller_permit) }) .await .map_err(|e| AppError::Git(e.to_string()))? - // A walk that hit its deadline carries GitServiceTimeout; map it to 504 - // like the smart_http paths, not a generic 500 (#174 U3). - .map_err(|e| git_service_app_error(&e))? }; + // A walk that hit its deadline carries GitServiceTimeout; map it to 504 like + // the smart_http paths, not a generic 500 (#174 U3). + let withheld = withheld.map_err(|e| git_service_app_error(&e))?; if withheld.is_empty() { smart_http::upload_pack(&disk_path, body, git_timeout).await @@ -3456,6 +3464,148 @@ mod tests { ); } + /// #174 U3 (P1-b, RED-before/GREEN-after): a client disconnect during the + /// path-scoped withheld-blob walk must NOT release the read admission while the + /// uncancellable `spawn_blocking` walk is still running. The handler takes the + /// global read permit, enters the walk (a fake git hangs on rev-list), then the + /// request future is dropped mid-walk. With both permits moved into the blocking + /// task the global slot stays occupied until the walk finishes; on the pre-fix code + /// the handler-local permits drop on future-drop and the slot frees instantly (RED), + /// letting disconnect-spam exceed the cap while real git work keeps running. + #[sqlx::test] + async fn upload_pack_permit_held_through_walk_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git: resolve refs fast, hang on rev-list (recording its pid first). The + // ~6s sleep bounds the walk so a broken fix cannot wedge the suite. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > \"{}\" ; sleep 6 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Root the repo store at this test's TempDir so the bare repo is isolated per + // run (the default for_testing store uses a fixed /tmp path that would collide + // across runs). + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up3rd"; + let name = "up3"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the walk. + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + // A path-scoped rule so has_path_scoped_rule() is true (the walk path) without + // denying the "/" gate for the public repo. + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.77:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let mut fut = Box::pin(router.oneshot(req)); + // Drive until the walk's rev-list starts (its pidfile appears) — i.e. the + // request is inside the spawn_blocking walk, holding the global read permit. + let mut in_walk = false; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if revlist_pid.exists() { + in_walk = true; + break; + } + } + assert!( + in_walk, + "the walk's rev-list must start (request reached the spawn_blocking walk)" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the walk runs" + ); + + // Client disconnect: drop the request future mid-walk. + drop(fut); + + // Load-bearing: the slot must STAY held while the uncancellable walk runs. On + // the pre-fix code the handler-local permits drop here and the slot frees at + // once (RED). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be held until the spawn_blocking walk \ + finishes, not released the instant the future drops (P1-b)" + ); + + // Cleanup: let the walk finish so the slot releases and no blocking task leaks. + for _ in 0..400 { + if sem.available_permits() == 1 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + unsafe { + libc::kill(p, libc::SIGKILL); + } + } + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two From 12643573f6c9f4805bd1876008141edda58f9192 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:20:49 -0500 Subject: [PATCH 30/52] fix(node): cap concurrent receive-pack pushes per source IP (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The authenticated git-receive-pack POST acquired only the global write semaphore — no per-source sub-cap, unlike the read and anon-advert pools. Owner enforcement defaults off, so one host minting disposable did:key identities could open max_concurrent_git_pushes slow POSTs from a single source IP and 503 every other source's push; the 600/hour push limiter bounds arrival rate, not in-flight concurrency (P1-d). Add git_write_per_caller (a PerCallerConcurrency sized like the advert cap, max_concurrent_git_pushes/8), and acquire it before the global write permit, keyed on the resolved source IP via read_caller_key — never the signed DID (a DID farm defeats a DID key). This required adding the PeerAddr + HeaderMap extractors to git_receive_pack, which it lacked (without them the key is None and the cap is inert). Neutralized the shared acquire helper's log wording now that it serves both read and write paths. Regression: receive_pack_per_source_write_cap_sheds_capped_source_not_others — a source at its write sub-cap sheds Overloaded/503 (proving the extractors resolve a key), while a different source is not shed. RED before (the capped source proceeds past admission to a git error instead of shedding), GREEN after; the existing receive-pack advert cap test still passes. --- crates/gitlawb-node/src/api/repos.rs | 103 ++++++++++++++++++++++-- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 7 ++ crates/gitlawb-node/src/state.rs | 7 ++ crates/gitlawb-node/src/test_support.rs | 1 + 5 files changed, 114 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 8612709c..67c6c82a 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -717,7 +717,7 @@ fn acquire_read_caller_permit( Some(k) => match limiter.try_acquire(k) { Some(p) => Ok(Some(p)), None => { - tracing::warn!(repo = %repo, caller = %k, handler, "per-caller read cap reached; shedding with 503"); + tracing::warn!(repo = %repo, caller = %k, handler, "per-caller cap reached; shedding with 503"); Err(AppError::Overloaded( "git service at capacity for this caller, retry shortly".into(), )) @@ -1033,15 +1033,32 @@ pub async fn git_receive_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, Extension(auth): Extension, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { + let name = smart_http_repo_name(&repo)?; + // Per-source write sub-cap (#174 P1-d): before the global write permit so one + // source IP cannot occupy the whole write pool via many slow authenticated pushes + // and 503 every other source. Owner enforcement defaults off, so any valid did:key + // is accepted (auth != authz), and the 600/hour push limiter bounds arrival RATE, + // not in-flight concurrency — so without this a single host minting disposable DIDs + // saturates the pool. Keyed on the resolved source IP, NEVER the signed DID (a DID + // farm defeats a DID key); no resolvable key -> global write pool only. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_write_per_caller, + caller_key.as_deref(), + name, + "receive-pack", + )?; // Shed with a 503 before spawning git when the concurrency cap is saturated. // Pushes draw from the dedicated WRITE pool, separate from reads, so a flood of - // anonymous reads cannot shed an authenticated push (#174). Acquired at the very - // top so it wraps the write-guard below (and precedes the Tigris acquire_write, - // bounding concurrent fresh acquires — INV-10); held for the whole op. + // anonymous reads cannot shed an authenticated push (#174). Taken after the + // per-source cap above so one source cannot occupy global slots it would be + // sub-cap-denied for; still before the Tigris acquire_write, bounding concurrent + // fresh acquires (INV-10); held for the whole op. let _permit = git_permit(&state.git_write_semaphore)?; - let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state .db @@ -3606,6 +3623,82 @@ mod tests { } } + /// #174 U4 (P1-d, RED-before/GREEN-after): the authenticated receive-pack POST + /// carries a per-source WRITE sub-cap so one source IP cannot monopolize the write + /// pool with many slow pushes (owner enforcement defaults off, so disposable DIDs + /// are free). Global write pool has capacity; the source is pre-held at its single + /// write slot. A push from THAT source sheds (Overloaded/503) — which also proves + /// the PeerAddr+HeaderMap extractors resolve a key (without them the key is None and + /// the cap is inert, never shedding). A push from a DIFFERENT source is NOT shed by + /// the cap. Called directly so the test needs no signed request; the handler is + /// where the cap lives. Remove the `git_write_per_caller` acquire and the capped + /// source no longer sheds (RED). + #[sqlx::test] + async fn receive_pack_per_source_write_cap_sheds_capped_source_not_others(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has capacity; the per-source cap is 1. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6rp4wr", "rp4", "/tmp/rp4-nonexistent", None, false) + .await + .unwrap(); + + let did = "did:key:z6MkReceivePackWriteCapProofDidAAAAAAAAAA"; + let capped: SocketAddr = "203.0.113.44:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.45:5000".parse().unwrap(); + + // Pin the capped source at its single write slot. + let _slot = state + .git_write_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first write slot for the capped source IP"); + + // A push from the capped source must shed on the per-source write cap even with + // global write capacity free. The shed also proves the source-IP key resolved + // via the extractors (an inert None key would fall through to Ok(None)). + let capped_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(capped)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + matches!(capped_result, Err(AppError::Overloaded(_))), + "a source at its per-source write cap must shed (Overloaded/503) with global \ + pool capacity free; got {capped_result:?}" + ); + + // A push from a DIFFERENT source must NOT be shed by the per-source cap — it + // proceeds past admission (and fails later on the nonexistent repo, which is not + // an Overloaded error). + let other_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(other)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(other_result, Err(AppError::Overloaded(_))), + "a different source must not be shed by the per-source write cap while the \ + capped source holds its slot; got {other_result:?}" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 5d827794..5597cd8d 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -526,6 +526,7 @@ mod tests { git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 18019395..74504f90 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -399,6 +399,13 @@ async fn main() -> Result<()> { git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), + // Per-source cap on the authenticated receive-pack POST, sized like the advert + // cap: one source IP can hold at most this many write-pool slots, so + // monopolizing the pool takes ~8 distinct source IPs, each also rate-limited + // (#174 P1-d). + git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + (config.max_concurrent_git_pushes / 8).max(1), + ), git_bin: "git".to_string(), }; diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index eb07a3e4..6ce158e3 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -123,6 +123,13 @@ pub struct AppState { /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many /// distinct source IPs (each also braked by the per-IP push rate limiter). pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the authenticated `git-receive-pack` POST: + /// each source IP may hold at most a small share of `git_write_semaphore`, so one + /// host minting disposable `did:key` identities cannot open enough slow pushes to + /// monopolize the write pool and 503 every other source's push (#174 P1-d). Keyed + /// on the resolved source IP (never the DID — a DID farm defeats a DID key). Sized + /// like `git_push_advert_per_caller`, a fraction of `max_concurrent_git_pushes`. + pub git_write_per_caller: crate::rate_limit::PerCallerConcurrency, /// The `git` executable the served-git withheld-blob walk spawns. Production is /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's /// process-group teardown in handler tests without mutating the process-global diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 0805fec1..9241b518 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -90,6 +90,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, ), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), git_bin: "git".to_string(), } } From 2a54c159e6984032205545816e0e44e0881f996c Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:28:52 -0500 Subject: [PATCH 31/52] fix(node): bound concurrent post-push encryption walks with an admission pool (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a successful path-scoped push the handler released its write permit and then ran a DETACHED tokio task whose spawn_blocking(withheld_blob_recipients_bounded) performed another full-history git walk under no admission. N fast completed pushes spawned N concurrent full-history walks past GITLAWB_MAX_CONCURRENT_GIT_PUSHES (which bounds only the in-handler phase), and the per-child bounding from 71389e6 caps each walk's duration but not their count (P1-e). Add git_encrypt_semaphore, a pool of its own (sized from max_concurrent_git_pushes, no new knob — Q1) so a long background walk never holds a foreground write slot and a handler holding a write permit can't self-deadlock. Route the walk through a new withheld_recipients_gated helper that acquires the pool before the walk and DEFERS (blocks) when full rather than shedding — dropping the walk would lose the withheld-blob recovery copy, so durability stays fail-closed. Regression: encrypt_walk_defers_when_pool_exhausted drives the gated helper with the pool exhausted and asserts the walk blocks and does not run its rev-list, then runs once a permit frees. RED before (the walk runs regardless of the pool), GREEN after. The detached push task calls this exact helper. --- crates/gitlawb-node/src/api/repos.rs | 138 +++++++++++++++++++++--- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 6 ++ crates/gitlawb-node/src/state.rs | 11 ++ crates/gitlawb-node/src/test_support.rs | 1 + 5 files changed, 144 insertions(+), 13 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 67c6c82a..430b2f7f 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -727,6 +727,37 @@ fn acquire_read_caller_permit( } } +/// Acquire an encryption-walk admission permit, then run the bounded withheld-blob +/// recipients walk. Blocks (defers) when `git_encrypt_semaphore` is full rather than +/// shedding — the walk is background so added latency is fine, and dropping it would +/// lose the withheld-blob recovery copy (#174 P1-e). Bounds the number of concurrent +/// post-push encryption walks so N fast completed pushes cannot spawn N concurrent +/// full-history git walks. Mirrors the original `spawn_blocking(...).await` return +/// shape so the caller's `Ok(Ok(recipients))` match is unchanged. +async fn withheld_recipients_gated( + encrypt_sem: std::sync::Arc, + repo_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, + rules: Vec, + is_public: bool, + owner_did: String, +) -> std::result::Result< + anyhow::Result>>, + tokio::task::JoinError, +> { + let _permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tokio::task::spawn_blocking(move || { + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &repo_path, &git_bin, timeout, &rules, is_public, &owner_did, + ) + }) + .await +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1329,6 +1360,7 @@ pub async fn git_receive_pack( let repo_name = record.name.clone(); let enc_git_bin = state.git_bin.clone(); let enc_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let encrypt_sem = state.git_encrypt_semaphore.clone(); tokio::spawn(async move { let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, @@ -1351,19 +1383,18 @@ pub async fn git_receive_pack( // the has_path_scoped_rule gate on the other two withheld-walk sites. if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) { - let p = repo_path_clone.clone(); - let owner = owner_did.clone(); - let git_bin = enc_git_bin.clone(); - let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients_bounded( - &p, - &git_bin, - enc_timeout, - &rules, - is_public, - &owner, - ) - }) + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + encrypt_sem.clone(), + repo_path_clone.clone(), + enc_git_bin.clone(), + enc_timeout, + rules, + is_public, + owner_did.clone(), + ) .await; if let Ok(Ok(recipients)) = recip { let delta = crate::encrypted_pin::encrypt_and_pin( @@ -3699,6 +3730,87 @@ mod tests { ); } + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a + /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn + /// unbounded concurrent full-history walks. With the pool exhausted the gated walk + /// must DEFER (block on admission) and NOT run its rev-list; on the pre-fix code + /// (no acquire) the walk runs regardless of the pool (RED). It defers rather than + /// sheds — releasing the permit lets the SAME walk run and pin (durability stays + /// fail-closed). Exercises the gating seam directly; the detached push task calls + /// this exact helper. + #[tokio::test] + async fn encrypt_walk_defers_when_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("revlist.ran"); + // Fake git records when rev-list runs (the walk's first git call). + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo ran > \"{}\" ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let git_bin = git_path.to_str().unwrap().to_string(); + let owner = "did:key:z6MkEncWalkOwnerAAAAAAAAAAAAAAAAAAAAAAAA".to_string(); + + // Exhaust the pool: hold its only permit so a gated walk must defer. + let sem = Arc::new(Semaphore::new(1)); + let held = sem.clone().acquire_owned().await.unwrap(); + + // Blocked: the gated walk must NOT complete or run rev-list while exhausted. + let blocked = tokio::time::timeout( + Duration::from_millis(500), + withheld_recipients_gated( + sem.clone(), + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + Vec::new(), + true, + owner.clone(), + ), + ) + .await; + assert!( + blocked.is_err(), + "the encryption walk must defer (block on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the walk's rev-list must not run while its admission permit is unavailable (P1-e)" + ); + + // Release admission: the SAME walk now runs (defer, not shed) — rev-list fires. + drop(held); + let ran = withheld_recipients_gated( + sem, + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(5), + Vec::new(), + true, + owner, + ) + .await; + assert!( + ran.is_ok(), + "with a permit the walk runs and joins: {ran:?}" + ); + assert!( + marker.exists(), + "once admission is available the deferred walk runs its rev-list" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 5597cd8d..f18b4963 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -523,6 +523,7 @@ mod tests { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 74504f90..380ab5f4 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -389,6 +389,12 @@ async fn main() -> Result<()> { git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Bounds concurrent detached post-push encryption walks, sized from the push + // pool (no separate knob — Q1): completed pushes cannot outnumber active + // encryption walks past this (#174 P1-e). + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 6ce158e3..3ff86923 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -110,6 +110,17 @@ pub struct AppState { /// advert pool (each source also capped by `git_push_advert_per_caller` and the /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). pub git_push_advert_semaphore: Arc, + /// Bounds concurrent post-push encrypt-then-pin history walks. Each successful + /// path-scoped push releases its handler write permit and then runs a DETACHED + /// full-history walk (`withheld_blob_recipients_bounded`) to seal withheld blobs; + /// without a cap, N fast pushes spawn N concurrent full-history git walks past + /// `max_concurrent_git_pushes` (which only bounds the in-handler phase). The walk + /// acquires a permit here and DEFERS (blocks) when the pool is full rather than + /// shedding — the work is background and dropping it would lose the recovery copy + /// (#174 P1-e). A pool of its own, not `git_write_semaphore`: a long background + /// walk must not hold a foreground write slot, and a handler already holding a + /// write permit that needed a second would self-deadlock at pool size 1. + pub git_encrypt_semaphore: Arc, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 9241b518..a27be946 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -86,6 +86,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, From 5749df6e67be0d06fd99b69f0a64156ba3df29ca Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:33:01 -0500 Subject: [PATCH 32/52] test(node): INV-22 completeness guard for the served-git concurrency surface (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A source-scan tripwire that fails when a new site reintroduces the concurrency-cap class PR #174 fixed, or when one of the five gates (U1-U5) is removed. INV-22: a permit held per op recovers only if every path is duration-bounded and reaps the group before releasing admission, and every detached git task carries admission. Lives in tests/ (a separate crate) on purpose — a guard scanning the file it lives in would match its own identifier literals and pass vacuously; scanning src/ from here keeps every check load-bearing. Checks: run_bounded_git confirms child exit via child_terminated_without_reaping (U1); KillGroupOnDrop launches the reaper via Handle::try_current (U2); git_receive_pack acquires state.git_write_per_caller (U4); the encryption walk runs through withheld_recipients_gated over git_encrypt_semaphore (U5); and the bounded recipients walk is invoked exactly once — only inside the gated helper — so a new spawn_blocking bypass (count > 1) fails (P1-e non-bypass). Proven load-bearing: a synthetic second call site turns the count tripwire RED, and each named gate maps to a fix whose own regression already goes RED on revert. --- crates/gitlawb-node/tests/inv22_gates.rs | 80 ++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 crates/gitlawb-node/tests/inv22_gates.rs diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs new file mode 100644 index 00000000..5f576300 --- /dev/null +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -0,0 +1,80 @@ +//! #174 U6 — INV-22 completeness guard (rung-raising). +//! +//! INV-22: a permit held per op recovers only if every path that holds it is also +//! duration-bounded and reaps the process group before releasing admission, and every +//! detached git/blocking task carries its own admission. PR #174 fixed five paths +//! (U1-U5) that violated this. Each fix has a per-unit RED/GREEN regression; together +//! those form the five-revert matrix. This guard adds the missing piece: a source-scan +//! tripwire that fails when a NEW site reintroduces the class, or when one of the five +//! gates is removed. +//! +//! It lives in `tests/` (a separate crate) on purpose: a guard that scanned the same +//! file it lives in would match its own identifier literals and pass vacuously. Here +//! the scanned `src/` files never contain this file's literals, so each check is +//! load-bearing — reverting the named gate turns the assertion red. +//! +//! These are deliberately coarse structural checks, not a parser. They cannot prove a +//! gate is *correct* (the per-unit tests do that); they prove a gate is *present and +//! not bypassed*, which is what stops the class from silently regressing. + +use std::path::Path; + +fn src(rel: &str) -> String { + let p = Path::new(env!("CARGO_MANIFEST_DIR")).join("src").join(rel); + std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())) +} + +#[test] +fn inv22_concurrency_gates_present_and_not_bypassed() { + let repos = src("api/repos.rs"); + let smart_http = src("git/smart_http.rs"); + let vis = src("git/visibility_pack.rs"); + + // U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the + // child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise + // a child that closes stdout then hangs pins the permit past the deadline. The + // probe is defined and called, so >= 2 occurrences; reverting the fix removes both. + assert!( + vis.matches("child_terminated_without_reaping").count() >= 2, + "U1/P1-a gate missing: run_bounded_git must confirm child exit via \ + child_terminated_without_reaping before signalling the watchdog" + ); + + // U2 / P1-c: on client disconnect KillGroupOnDrop must launch a detached reaper + // that runs the full TERM/grace/KILL/reap, not a lone SIGTERM. The reaper is spawned + // via a runtime handle in Drop; `Handle::try_current` is unique to that launch (the + // timeout path already has an async context and never calls it). + assert!( + smart_http.contains("Handle::try_current"), + "U2/P1-c gate missing: KillGroupOnDrop::drop must launch the full \ + TERM/grace/KILL reaper on disconnect (Handle::try_current), not a lone SIGTERM" + ); + + // U4 / P1-d: git_receive_pack must acquire the per-source write sub-cap before the + // global write permit. The acquire reads `state.git_write_per_caller`; comments name + // the field without the `state.` prefix, so this targets the real acquire site. + assert!( + repos.contains("state.git_write_per_caller"), + "U4/P1-d gate missing: git_receive_pack must acquire the per-source write cap \ + (state.git_write_per_caller) before the global write permit" + ); + + // U5 / P1-e: the detached post-push encryption walk must run through the + // admission-gated helper, which is wired to the shared encrypt pool. + assert!( + repos.contains("fn withheld_recipients_gated") + && repos.contains("state.git_encrypt_semaphore"), + "U5/P1-e gate missing: the encryption walk must run through \ + withheld_recipients_gated, which acquires git_encrypt_semaphore" + ); + + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere + // but inside withheld_recipients_gated. A second call site (count > 1) is a new + // detached git walk that skips the admission gate — exactly the class U5 closed. + assert_eq!( + repos.matches("withheld_blob_recipients_bounded").count(), + 1, + "P1-e bypass: the bounded recipients walk must be invoked only inside \ + withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap" + ); +} From 72e899dae022dbc757be4198161629dad5245a92 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 11:33:20 -0500 Subject: [PATCH 33/52] fix(node): hold served-git admission until the process group is reaped on the plain spawn paths (#174) On the plain (non-path-scoped) info_refs / upload-pack / receive-pack paths the global + per-source admission permits were handler-locals that dropped the instant the handler future dropped on a client disconnect, while KillGroupOnDrop reaps the git process group in a DETACHED task (SIGTERM -> grace -> SIGKILL -> reap). So a disconnect-spammer admitted replacements while prior groups were still alive, defeating the process cap and the per-source cap during teardown. be0cdd6 already fixed this for the path-scoped upload-pack walk; this closes the residual plain paths (P1-a). Introduce AdmissionGuard, a move-only holder of the permits, threaded through info_refs/upload_pack/receive_pack/run_git_service into drive_git_child's KillGroupOnDrop. On disconnect the guard moves into the detached reaper and drops only after the group is ESRCH-confirmed reaped; on success/timeout disarm() returns it for the earliest provably-free drop. The rev-list/pack-objects visibility-walk callers hold no admission and pass None. Thread git_bin through upload_pack/receive_pack (they hardcoded "git") so the plain POST path is drivable by an injected fake git, matching the existing injectable seam on info_refs/run_git_service/upload_pack_excluding. Handler-layer regression upload_pack_plain_permit_held_through_group_reap_after_disconnect: isolates the global read pool (size 1, per-source + rate limiter permissive), a SIGTERM-ignoring descendant keeps the group alive across the reap window, and asserts available_permits()==0 held on disconnect + a cross-source replacement sheds 503, then frees after ESRCH. Reverting the guard-into-reaper move (release immediately) turns it RED at the held-through-reap assertion. Plus a None-key arm. --- crates/gitlawb-node/src/api/repos.rs | 269 +++++++++++++++++++++- crates/gitlawb-node/src/git/smart_http.rs | 130 +++++++++-- 2 files changed, 382 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 430b2f7f..16a8ed0d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -653,8 +653,14 @@ pub async fn git_info_refs( AppError::Git(e.to_string()) })?; + // Move the admission permits into the guard so they release only after the spawned + // git process group is confirmed reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper is still tearing + // the group down (#174 P1-a). The handler keeps no copy: `_permit`/`_caller_permit` + // are moved in, so admission tracks the real process lifetime. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - smart_http::info_refs("git", &service, &disk_path, git_timeout) + smart_http::info_refs("git", &service, &disk_path, git_timeout, Some(admission)) .await .map_err(|e| { let app = git_service_app_error(&e); @@ -848,7 +854,12 @@ pub async fn git_upload_pack( // withheld walk and serve the pack directly. let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); let resp = if !visibility_pack::has_path_scoped_rule(&rules) { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // Plain (non-path-scoped) serve: move both admission permits into the guard so + // they release only after the spawned git group is reaped, on + // complete/timeout/disconnect — not the instant a disconnect drops this future + // (#174 P1-a). The handler keeps no copy. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep that // off the async worker thread. Move BOTH admission permits INTO the blocking @@ -886,9 +897,17 @@ pub async fn git_upload_pack( let withheld = withheld.map_err(|e| git_service_app_error(&e))?; if withheld.is_empty() { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // No blobs to withhold: serve the plain pack, moving the permits returned by + // the walk into the guard so admission tracks the served git group's reap + // (the walk already held them per be0cdd6; this hands them to the serve). + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); + // upload_pack_excluding runs its own rev-list/pack-objects (both pass `None` + // admission internally); the walk's permits stay handler-locals held across + // this serve, as be0cdd6 established, and drop when the handler returns. + let _hold = (_permit, _caller_permit); smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await } } @@ -1172,7 +1191,14 @@ pub async fn git_receive_pack( tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let receive_result = smart_http::receive_pack(&disk_path, body, git_timeout).await; + // Move both admission permits into the guard so they release only after the spawned + // receive-pack process group is reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). + // The handler keeps no copy. This is independent of the write-lock `guard.release` + // below: admission tracks the git process lifetime, the write lock tracks the repo. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + let receive_result = + smart_http::receive_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await; // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push @@ -3654,6 +3680,241 @@ mod tests { } } + /// #174 U1 (P1-a, plain-spawn residual, RED-before/GREEN-after): on the PLAIN + /// (non-path-scoped) upload-pack path a client disconnect must NOT release the + /// global read admission while the detached process-group reaper is still tearing + /// down a git group that ignores SIGTERM. The `be0cdd6` fix moved permits into the + /// path-scoped `spawn_blocking` walk; this closes the residual plain path, where the + /// permits were handler-locals that dropped the instant the future was dropped. + /// + /// Isolate the GLOBAL pool: read pool = 1, per-source cap + rate limiter permissive, + /// so the only thing that can shed a replacement is the leaked global permit. Drive + /// the handler until git spawns, disconnect, then assert the global slot stays held + /// (`available_permits() == 0`) AND a replacement sheds 503 while the group is alive; + /// after the reaper SIGKILLs+reaps the group the slot frees and a replacement is no + /// longer shed by the global cap. On the pre-fix code the handler-local permit drops + /// on future-drop and the slot frees at once (RED). + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_plain_permit_held_through_group_reap_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Fake git for the plain upload-pack path (invoked as `git upload-pack + // --stateless-rpc `). It forks a descendant that TRAPS SIGTERM, records its + // pid, and loops ~20s, then `wait`s — so on disconnect the group leader dies on + // the reaper's SIGTERM but the descendant survives until the reaper escalates to + // SIGKILL, keeping the group alive (ESRCH not reached) across the observation + // window. Bounded so a broken fix leaks no permanent orphan. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + upload-pack)\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + descfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global read pool: size 1; per-source cap + rate limiter permissive + // so only the leaked global permit can shed the replacement. + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up1st"; + let name = "up1"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the + // spawn. No path-scoped rule -> the PLAIN serve branch (this test's target). + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!(sem.available_permits(), 1, "one read slot before the request"); + + let router = crate::server::build_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until git spawns (the descendant records its pid) — the request is + // inside the plain serve, holding the global read permit. Stop polling the + // instant the future completes (re-polling a completed oneshot panics); read the + // descfile first so a spawn that recorded its pid then returned is still caught. + let mut spawned: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + spawned = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let desc = spawned + .unwrap_or_else(|| panic!("the fake git must have spawned; early finish: {early:?}")); + // Kill the descendant regardless of outcome so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(desc); + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "descendant should be running before the disconnect" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the git op runs" + ); + + // Client disconnect: drop the request future. The detached reaper now owns the + // AdmissionGuard and will not drop it until the group is ESRCH-confirmed reaped. + drop(fut); + + // Load-bearing: the slot must STAY held while the SIGTERM-ignoring group is still + // alive. On the pre-fix code the handler-local permit drops here and the slot + // frees at once (RED). Check quickly (before the reaper's ~2s SIGKILL escalation). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be HELD until the process group is \ + reaped, not released the instant the future drops (P1-a)" + ); + // A replacement request from a DIFFERENT source must shed 503 — the only pool + // that can shed it is the leaked global permit (per-source cap is permissive). + let peer2: SocketAddr = "203.0.113.72:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "while the prior group is still alive the held global permit must shed a \ + replacement with 503" + ); + + // After the reaper SIGKILLs + reaps the group the AdmissionGuard drops and the + // slot frees. Poll for recovery. + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the reaper confirms the group gone the admission guard must drop and \ + free the global slot" + ); + // A replacement is now no longer shed by the global cap (it proceeds past + // admission; it then fails downstream on the fake git, which is not a 503). + let peer3: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let resp = router.oneshot(make_req(peer3)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "after the group is reaped the freed slot must admit a replacement" + ); + } + + /// #174 U1 (P1-a): the `None`-key arm — a request with no resolvable source key + /// (no trusted-proxy header, no peer) is bounded by the GLOBAL read pool only, never + /// a per-source cap. With the global read pool exhausted such a request still sheds + /// 503, proving the plain path admits/sheds on the global pool for the `None` arm + /// (the counterpart to the `Some(ip)` arm above). Complements the resolver-arm rule: + /// neither arm is vacuous. + #[tokio::test] + async fn upload_pack_plain_none_key_arm_sheds_on_global_pool() { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use axum::Router; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state_lazy(); + // Global read pool exhausted; per-source cap permissive so only the global pool + // can shed. No ConnectInfo + no trusted header -> read_caller_key resolves None. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + // No ConnectInfo extension and no XFF header: the caller key is None. + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL read pool" + ); + } + /// #174 U4 (P1-d, RED-before/GREEN-after): the authenticated receive-pack POST /// carries a per-source WRITE sub-cap so one source IP cannot monopolize the write /// pool with many slow pushes (owner enforcement defaults off, so disposable DIDs diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index a5649a42..64d3cb7f 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -10,6 +10,40 @@ use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; +/// Owns the served-git admission permits (global + per-source) for the lifetime of +/// the work they admitted, so admission is released only when that work is truly +/// done — not the instant the handler future drops on a client disconnect. +/// +/// A move-only wrapper: no methods beyond construction and `Drop`. The handler +/// MOVEs its permits in and keeps no copy (a retained copy would drop early and +/// release admission the moment the future is dropped, defeating the guard). It is +/// threaded into `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the +/// detached reaper on disconnect, so both permits drop only after the process group +/// is confirmed reaped (`kill(-pgid,0)==ESRCH`) rather than while the group is still +/// alive holding PIDs past the concurrency cap (#174 P1-a, plain-spawn residual). +/// +/// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by +/// moving its permits into the `spawn_blocking`; this generalizes it to the plain +/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths. +pub struct AdmissionGuard { + // Boxed so the concrete permit types stay private to the caller (repos.rs) — the + // guard only needs to hold them and drop them, never inspect them. `Send + + // 'static` so the guard can move into the detached reaper task. + _global: Option>, + _caller: Option>, +} + +impl AdmissionGuard { + /// Take ownership of the global permit and an optional per-caller permit. Both are + /// erased to `Box` — the guard's only job is to hold them until it drops. + pub fn new(global: impl Send + 'static, caller: Option) -> Self { + Self { + _global: Some(Box::new(global)), + _caller: caller.map(|c| Box::new(c) as Box), + } + } +} + /// Handle `GET /:owner/:repo/info/refs?service=git-upload-pack` /// or `?service=git-receive-pack` /// @@ -26,6 +60,7 @@ pub async fn info_refs( service: &str, repo_path: &Path, timeout: Duration, + admission: Option, ) -> Result { validate_service(service)?; @@ -36,7 +71,8 @@ pub async fn info_refs( .arg("--advertise-refs") .arg(repo_path); // No request body — advertise-refs does not read stdin. - let stdout = drive_git_child(command, Bytes::new(), timeout, "advertise-refs").await?; + let stdout = + drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; let content_type = format!("application/x-{service}-advertisement"); @@ -61,12 +97,21 @@ pub async fn info_refs( /// Serves pack data for a clone or fetch. This is stateless — the entire /// negotiation happens in a single request/response. pub async fn upload_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-upload-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-upload-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -80,12 +125,21 @@ pub async fn upload_pack( /// Accepts a push. The caller MUST verify HTTP Signature auth before /// calling this function. pub async fn receive_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-receive-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-receive-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -120,6 +174,12 @@ struct KillGroupOnDrop { // own SIGCHLD-driven orphan reaper. child: Option, pgid: Option, + // The global + per-source admission permits for this op, if any. On the plain + // (non-path-scoped) spawn paths repos.rs moves its permits in here so admission is + // released only when the group is confirmed reaped, on every exit — complete, + // timeout, or client-disconnect (#174 P1-a). The rev-list/pack-objects + // visibility-walk callers hold no admission and pass `None`. + admission: Option, } #[cfg(unix)] @@ -131,17 +191,25 @@ impl KillGroupOnDrop { /// Disarm on the success/timeout path: the body has already reaped the child (its /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear - /// the pgid, leaving the guard's Drop a no-op. - fn disarm(&mut self) { + /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard so the + /// caller drops it at the earliest provably-free point (the group is already reaped + /// on both callers of this) rather than at end-of-bookkeeping. + fn disarm(&mut self) -> Option { self.pgid = None; self.child = None; + self.admission.take() } } #[cfg(unix)] impl Drop for KillGroupOnDrop { fn drop(&mut self) { + // Move any admission guard out first so it travels with the reaper (or drops + // here on the no-child / no-runtime fallback), never before the group is gone. + let admission = self.admission.take(); let (Some(mut child), Some(pgid)) = (self.child.take(), self.pgid) else { + // Nothing to reap (already disarmed); dropping `admission` here is correct — + // the group is already gone. return; }; // A sync Drop cannot await, so launch a detached reaper that owns the child and @@ -154,6 +222,13 @@ impl Drop for KillGroupOnDrop { Ok(handle) => { handle.spawn(async move { reap_group_on_timeout(&mut child).await; + // Release admission only now: the group is ESRCH-confirmed gone (or + // hit the ~4s D-state hard cap, past which nothing in userspace can + // free the PIDs anyway). Holding the permits until here is what + // stops disconnect-spam from admitting replacements while the prior + // group is still alive (#174 P1-a). `admission` is moved in and + // dropped when this closure ends. + drop(admission); }); } Err(_) => { @@ -162,6 +237,9 @@ impl Drop for KillGroupOnDrop { unsafe { libc::kill(-pgid, libc::SIGTERM); } + // No runtime to await the reap; drop admission best-effort after the + // synchronous signal (the fallback path is not on the hot request path). + drop(admission); } } } @@ -246,13 +324,14 @@ async fn run_git_service( repo_path: &Path, input: Bytes, timeout: Duration, + admission: Option, ) -> Result> { let mut command = Command::new(git_bin); command .arg(service_to_command(service)) .arg("--stateless-rpc") .arg(repo_path); - drive_git_child(command, input, timeout, service).await + drive_git_child(command, input, timeout, service, admission).await } /// Drive a spawned git child under `timeout` with process-group teardown, returning @@ -267,6 +346,7 @@ async fn drive_git_child( input: Bytes, timeout: Duration, what: &str, + admission: Option, ) -> Result> { command .stdin(Stdio::piped()) @@ -300,7 +380,12 @@ async fn drive_git_child( let mut group_guard = KillGroupOnDrop { child: Some(child), pgid, + admission, }; + // On non-unix there is no process-group teardown, so hold the admission guard here + // for the child's whole interaction; it drops when this function returns. + #[cfg(not(unix))] + let _admission = admission; let mut out = Vec::new(); let mut err = Vec::new(); @@ -339,18 +424,21 @@ async fn drive_git_child( // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the // guard's drop would reap an already-reaped child / signal a reused pgid. + // Dropping the returned admission guard here releases the permits at the + // earliest provably-free point on the success path (the op finished). #[cfg(unix)] - group_guard.disarm(); + drop(group_guard.disarm()); result? } Err(_elapsed) => { // Timeout: tear the whole group down and reap it before returning so a // caller releasing a write lock can't race a still-live git. Then disarm - // the (now redundant) guard so its drop can't hit a reused pgid. + // the (now redundant) guard so its drop can't hit a reused pgid. The + // returned admission guard drops here, AFTER the group is confirmed reaped. #[cfg(unix)] { reap_group_on_timeout(group_guard.child_mut()).await; - group_guard.disarm(); + drop(group_guard.disarm()); } #[cfg(not(unix))] { @@ -415,7 +503,9 @@ async fn rev_list_keep( command .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path); - let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list").await?; + // The visibility-walk callers intentionally hold no admission permit (their + // admission is governed elsewhere), so pass `None` (#174 KTD2). + let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list", None).await?; let mut keep = Vec::new(); for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); @@ -464,6 +554,8 @@ pub async fn build_filtered_pack( Bytes::from(data), deadline.saturating_duration_since(Instant::now()), "pack-objects", + // Visibility-walk pack build: no admission permit here (#174 KTD2). + None, ) .await } @@ -736,7 +828,7 @@ mod tests { axum::extract::Query(q): axum::extract::Query>, ) -> Response { let service = q.get("service").cloned().unwrap_or_default(); - info_refs("git", &service, &st.repo, Duration::from_secs(30)) + info_refs("git", &service, &st.repo, Duration::from_secs(30), None) .await .unwrap() } @@ -1021,6 +1113,7 @@ mod tests { let _guard = KillGroupOnDrop { child: Some(child), pgid: Some(pid), + admission: None, }; } @@ -1078,6 +1171,7 @@ mod tests { let _guard = KillGroupOnDrop { child: Some(child), pgid: Some(pid), + admission: None, }; } @@ -1114,6 +1208,7 @@ mod tests { let mut guard = KillGroupOnDrop { child: Some(child), pgid: Some(pid), + admission: None, }; guard.disarm(); } // disarmed -> no reaper, no kill @@ -1320,6 +1415,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, )); // Advance the future a slice at a time until the fake records its @@ -1421,6 +1517,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, )); // Drive the future a slice at a time until the descendant records its pid. let mut desc: Option = None; @@ -1482,6 +1579,7 @@ mod tests { "git-upload-pack", tmp.path(), Duration::from_millis(200), + None, ), ) .await @@ -1589,6 +1687,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1633,6 +1732,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1683,6 +1783,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1757,6 +1858,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1794,6 +1896,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_millis(200), + None, ), ) .await; @@ -1834,6 +1937,7 @@ mod tests { tmp.path(), big, Duration::from_secs(60), + None, ) .await; From 8d17038e5bc1e37cc76c931288c4d993a17b39c3 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 12:00:02 -0500 Subject: [PATCH 34/52] fix(node): bound the served-git storage-acquisition phase with a release-on-expiry deadline (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The served-git admission permit is acquired, then RepoStore::acquire*/advisory-lock runs before the bounded git runner starts — with no local deadline. acquire/acquire_fresh await Tigris HEAD/GET, and acquire_write's advisory-lock loop awaits a per-iteration pg_try_advisory_lock that can block indefinitely on a hung Postgres pool. Since GITLAWB_GIT_SERVICE_TIMEOUT_SECS only starts once git spawns, a stalled backend pins the permit and drains the pool until every later request 503s (P1-2). Wrap each git-path acquire (git_info_refs read/advert, git_upload_pack, git_receive_pack) and the /ipfs per-repo acquire loop in tokio::time::timeout(git_acquire_timeout_secs, ..). On expiry the git paths return AppError::Overloaded (503 + Retry-After); the permit is a handler-local at that point (moved into the AdmissionGuard only after acquire) so the early return frees the slot. The /ipfs per-repo acquire keeps its fail-closed `continue` on timeout (never serves an un-acquired repo). No repo_store.rs signature change: the outer timeout cancels a mid-sleep/mid-fetch_one future, so no in-loop deadline is needed. New knob GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS (default 30, range 1..), modeled on git_service_timeout_secs, kept separate because acquisition and git execution are distinct cost centers. Handler-layer regression receive_pack_acquire_deadline_sheds_and_releases_permit: holds the same pg advisory lock acquire_write derives on a second connection so the real loop must retry, drives git-receive-pack with the deadline at 2s and write pool size 1, and asserts 503 at ~2s + the permit recovers + a follow-up admits once the lock frees. Removing the timeout wrapper turns it RED (blocks ~31s to the test ceiling on the ~59s advisory loop). --- crates/gitlawb-node/src/api/ipfs.rs | 21 +- crates/gitlawb-node/src/api/repos.rs | 307 ++++++++++++++++++++++++--- crates/gitlawb-node/src/config.rs | 21 ++ 3 files changed, 315 insertions(+), 34 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index e921c39f..284c9398 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -114,9 +114,24 @@ pub async fn get_by_cid( continue; } - let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { - Ok(p) => p, - Err(_) => continue, + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares + // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise + // block the whole /ipfs request). On expiry keep the existing fail-closed skip — + // never serve an un-acquired repo; a public copy (if any) still gets its turn. + let acquire_deadline = + std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let repo_path = match tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&repo.owner_did, &repo.name), + ) + .await + { + Ok(Ok(p)) => p, + Ok(Err(_)) => continue, + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs walk; skipping repo"); + continue; + } }; // Check whether the object exists in this repo before any expensive diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 16a8ed0d..5b11404d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -637,21 +637,37 @@ pub async fn git_info_refs( // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. - let disk_path = if service == "git-receive-pack" { - state - .repo_store - .acquire_fresh(&record.owner_did, &record.name) - .await - } else { - state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - } - .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); - AppError::Git(e.to_string()) - })?; + // + // Bound the acquire under `git_acquire_timeout_secs`: the concurrency permit is + // already held above, and `git_service_timeout_secs` only starts once git spawns, + // so an un-deadlined acquire (a hung Tigris HEAD/GET here) pins the permit until + // the pool drains (#174 P1-2). On expiry the handler-local `_permit`/`_caller_permit` + // drop on the early return (the AdmissionGuard is not built until after acquire), + // so the shed frees the slot; return a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let acquire_fut = async { + if service == "git-receive-pack" { + state + .repo_store + .acquire_fresh(&record.owner_did, &record.name) + .await + } else { + state + .repo_store + .acquire(&record.owner_did, &record.name) + .await + } + }; + let disk_path = tokio::time::timeout(acquire_deadline, acquire_fut) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, service = %service, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| { + tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); + AppError::Git(e.to_string()) + })?; // Move the admission permits into the guard so they release only after the spawned // git process group is confirmed reaped, on complete/timeout/disconnect — not the @@ -842,11 +858,21 @@ pub async fn git_upload_pack( // exec (INV-10). let _permit = git_permit(&state.git_read_semaphore)?; - let disk_path = state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; + // Bound the acquire under `git_acquire_timeout_secs` so a hung Tigris HEAD/GET + // cannot pin the read permit indefinitely (#174 P1-2). The permit is a handler + // local here (moved into the AdmissionGuard only below, once git is spawned), so + // the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let disk_path = tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| AppError::Git(e.to_string()))?; let body_len = body.len(); // No path-scoped rule can withhold an individual blob, and the whole-repo @@ -1179,14 +1205,31 @@ pub async fn git_receive_pack( } tracing::debug!(repo = %name, "acquiring write lock"); - let guard = state - .repo_store - .acquire_write(&record.owner_did, &record.name) - .await - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - })?; + // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's + // advisory-lock loop already caps at ~60s, but its per-iteration + // `pg_try_advisory_lock().fetch_one(&pool)` can block indefinitely on a hung / + // exhausted Postgres pool (so the 60-count never advances) — and the write permit + // is held the whole time, draining the pool (#174 P1-2). The outer + // `tokio::time::timeout` cancels a mid-sleep/mid-`fetch_one` future, so it bounds + // both the loop and a hung iteration without any repo_store.rs change (KTD3). The + // permit is a handler local here (moved into the AdmissionGuard only after this), + // so the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let guard = tokio::time::timeout( + acquire_deadline, + state + .repo_store + .acquire_write(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "acquire_write timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| { + tracing::error!(repo = %name, err = %e, "acquire_write failed"); + AppError::Git(e.to_string()) + })?; let disk_path = guard.path().to_path_buf(); tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); @@ -1197,8 +1240,14 @@ pub async fn git_receive_pack( // The handler keeps no copy. This is independent of the write-lock `guard.release` // below: admission tracks the git process lifetime, the write lock tracks the repo. let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); - let receive_result = - smart_http::receive_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await; + let receive_result = smart_http::receive_pack( + &state.git_bin, + &disk_path, + body, + git_timeout, + Some(admission), + ) + .await; // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push @@ -3761,7 +3810,11 @@ mod tests { .unwrap(); let sem = state.git_read_semaphore.clone(); - assert_eq!(sem.available_permits(), 1, "one read slot before the request"); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); let router = crate::server::build_router(state); let make_req = |peer: SocketAddr| { @@ -3991,6 +4044,198 @@ mod tests { ); } + /// #174 U2 (P1-2, RED-before/GREEN-after): the storage-acquisition phase is bounded + /// by `git_acquire_timeout_secs`, so a stalled backend releases the admission permit + /// and sheds a 503 instead of pinning the pool. The permit is taken BEFORE + /// `acquire_write`, whose advisory-lock loop can spin ~60s (and whose per-iteration + /// `pg_try_advisory_lock` can block indefinitely on a hung pool), so without the + /// `tokio::time::timeout` wrapper the permit is held far past the deadline. + /// + /// Real stall (no `RepoStore` trait to fake): hold the SAME session-level advisory + /// lock `acquire_write` derives (`advisory_lock_key(owner_slug, repo_name)`, where + /// `owner_slug = owner_did.replace([':','/'], "_")`) on a second pooled connection, + /// so the handler's `pg_try_advisory_lock` returns false every iteration and the loop + /// must retry against the deadline. `git_acquire_timeout_secs = 2`; the request must + /// return 503 (Overloaded) at ~2s (NOT ~59s), and the write permit must be released + /// (`available_permits()` recovers to full once the shed returns). Covers R2. + /// + /// Load-bearing / mutation: remove the `tokio::time::timeout` wrapper on + /// `acquire_write` and the loop runs to ~59s with the permit held the whole time — + /// the `< DEADLINE_CEILING` timing assertion goes RED (observed ~59s) and the permit + /// stays pinned past the deadline. Restore to return GREEN. + #[sqlx::test] + async fn receive_pack_acquire_deadline_sheds_and_releases_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + // Reproduce acquire_write's session-level advisory-lock key exactly so the + // second-connection lock collides with the handler's pg_try_advisory_lock + // (repo_store.rs: advisory_lock_key over owner_slug then repo_name). + fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + owner_slug.hash(&mut hasher); + repo_name.hash(&mut hasher); + hasher.finish() as i64 + } + + let owner = "z6acqdead"; + let name = "acq1"; + // owner_slug as local_path() computes it from the record's owner_did. The + // mirror row stores the short owner as owner_did, so slug == owner (no ':'/'/'). + let owner_slug = owner.replace([':', '/'], "_"); + let lock_key = advisory_lock_key(&owner_slug, name); + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Isolate the write pool at size 1 so available_permits() cleanly reports + // held (0) vs released (1). Per-source cap + trust permissive so only the + // write pool / acquire path can gate. + state.git_write_semaphore = Arc::new(Semaphore::new(1)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Short acquire deadline: the fix must shed here, well before acquire_write's + // ~59s advisory-lock loop would bail on its own. + const ACQUIRE_TIMEOUT_SECS: u64 = 2; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = ACQUIRE_TIMEOUT_SECS; + // Keep the git-service timeout large so the deadline under test is the acquire + // one, not git execution (which is never reached on the stalled path anyway). + cfg.git_service_timeout_secs = 600; + state.config = std::sync::Arc::new(cfg); + + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6acqdead-acq1", None, false) + .await + .unwrap(); + + // Hold the advisory lock on a dedicated pooled connection (a distinct session), + // so the handler's pg_try_advisory_lock($lock_key) returns false every iteration + // and acquire_write's real loop must retry against the deadline. Released when + // this connection drops at end of test. + let mut lock_conn = pool + .acquire() + .await + .expect("second connection for the lock"); + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("hold the advisory lock on the second connection"); + + let did = "did:key:z6MkAcquireDeadlineProofDidAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + + let sem = state.git_write_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one write slot before the request" + ); + + // Drive the authenticated push in the background so we can observe the permit is + // held while acquire_write stalls, then that it is released on the shed. + let state_for_task = state.clone(); + let start = std::time::Instant::now(); + let handle = tokio::spawn(async move { + git_receive_pack( + State(state_for_task), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + + // The handler takes the write permit BEFORE acquire_write, so once it is stalled + // in the advisory-lock loop the pool reports 0 available. Wait for that to prove + // the permit is genuinely held during the stall (and the request really reached + // acquire_write, not an earlier reject). + let mut held = false; + for _ in 0..200 { + if sem.available_permits() == 0 { + held = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + held, + "the write permit must be held while acquire_write stalls on the advisory lock" + ); + + // The bounded acquire deadline must shed with 503 (Overloaded), NOT wait out the + // ~59s advisory-lock loop. Ceiling is comfortably above the 2s deadline + task + // scheduling but far below 59s, so a RED run (no wrapper -> ~59s) fails here. + const DEADLINE_CEILING: std::time::Duration = std::time::Duration::from_secs(20); + let result = tokio::time::timeout( + DEADLINE_CEILING + std::time::Duration::from_secs(10), + handle, + ) + .await + .expect("the handler must return within the ceiling — a hang means the acquire deadline is missing (RED)") + .expect("the receive-pack task must not panic"); + let elapsed = start.elapsed(); + + assert!( + matches!(result, Err(AppError::Overloaded(_))), + "a stalled acquire_write must shed with Overloaded/503 at the acquire deadline; \ + got {result:?}" + ); + assert!( + elapsed < DEADLINE_CEILING, + "the shed must land at ~{ACQUIRE_TIMEOUT_SECS}s (the acquire deadline), not ~59s \ + (the advisory-lock loop). Observed {elapsed:?}; without the timeout wrapper this \ + is ~59s (RED)" + ); + + // Permit release on expiry: the Overloaded return drops the handler-local permit, + // so the isolated write pool must recover to full. A leaked permit here means the + // pool drains under a stalled backend (the #174 P1-2 bug). + let mut freed = false; + for _ in 0..200 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + freed, + "on the acquire-deadline shed the write permit must be released; the pool did \ + not recover to full (permit leaked)" + ); + + // Follow-up admits once the contended lock is released: release the second-conn + // lock, then a fresh push proceeds PAST admission (it fails later on the + // nonexistent on-disk repo, which is NOT an Overloaded/503). Proves the freed + // slot is usable, not merely counted. + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("release the advisory lock"); + let followup = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.62:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(followup, Err(AppError::Overloaded(_))), + "once the lock frees, a follow-up push must admit past the (recovered) write \ + pool and acquire; got {followup:?}" + ); + } + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn /// unbounded concurrent full-history walks. With the pool exhausted the gated walk diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d0a4c319..68125364 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -183,6 +183,27 @@ pub struct Config { )] pub git_service_timeout_secs: u64, + /// Maximum wall-clock time the storage-acquisition phase of a served git + /// operation may run before the request is shed with a 503, in seconds. This + /// bounds `RepoStore::{acquire,acquire_fresh,acquire_write}` — the Tigris + /// HEAD/GET on a read/advert acquire and the advisory-lock retry loop (incl. a + /// per-iteration `pg_try_advisory_lock` that can block on a hung Postgres pool) + /// on a write acquire. A concurrency permit is taken BEFORE this phase, and + /// `git_service_timeout_secs` only starts once git spawns, so without this the + /// acquire phase is unbounded: a stalled backend pins the permit and drains the + /// pool until every later request 503s. On expiry the permit is released and a + /// bounded 503 + Retry-After is returned (fail-closed). Kept separate from + /// `git_service_timeout_secs` because acquisition and git execution are distinct + /// cost centers — one shared budget would let a slow acquire starve git. Must be + /// positive; set it very large to effectively disable the bound. Default: 30s. + #[arg( + long, + env = "GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS", + default_value_t = 30, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub git_acquire_timeout_secs: u64, + /// Maximum connections in the PostgreSQL pool. This is a cap, not a floor /// (connections open lazily). Size against the database server's /// max_connections, remembering admin tooling opens its own pool. From f424ccbbcde49260932d776f3e8b560ec2a674c8 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 12:42:55 -0500 Subject: [PATCH 35/52] fix(node): gate the /ipfs walk with bounded concurrency admission + per-source cap + route rate limit (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /ipfs/{cid} (get_by_cid, publicly reachable via optional_signature) ran allowed_blob_set_for_caller_bounded in spawn_blocking inside a per-repo loop with NO global/per-source admission and NO route rate limit — the surface had zero concurrency or rate control. A permissionless caller could fan out concurrent full-history git walks (each up to the walk timeout) exhausting blocking-pool threads and PIDs outside every served-git pool (P1-3). Acquire a global git_ipfs_walk permit (try_acquire_owned, shed 503) plus a per-source sub-cap (with_default_max_keys, reject-before-insert, keyed via client_key/TrustedProxy so XFF can't spoof it) ONCE after CID validation, held as handler locals across the whole request — including every spawn_blocking walk, so the slot reflects real blocking-thread occupancy (a tokio timeout cannot cancel spawn_blocking). None key -> global pool only. Cap repos walked per request so one request can't serialize N full-history walks. Attach rate_limit_by_ip + IpRateLimiter to the /ipfs route (mirrors the git/create/peer routers; the extension is required or the middleware is a silent no-op). Visibility/authz semantics unchanged. New knobs: GITLAWB_MAX_CONCURRENT_IPFS_WALKS (32), GITLAWB_IPFS_WALK_PER_SOURCE (4), GITLAWB_IPFS_MAX_REPOS_WALKED (64), GITLAWB_IPFS_RATE_LIMIT (600/hr, 0 disables) — all range-validated and enforced on-path. Handler-layer regressions (all mutation-verified RED->GREEN): shed-at-capacity 503 (delete acquire -> falls through), per-source cap sheds same source / admits another, None-key arm sheds on the global pool, map self-bounds reject-before-insert, the walk permit is held while the spawn_blocking walk runs (available_permits()==0; dropping it before the loop -> RED), repos-walked cap (remove break -> 2 walks), and the route IP rate limit actually fires 429 (drop the Extension -> RED no-op). --- crates/gitlawb-node/src/api/ipfs.rs | 691 ++++++++++++++++++++++++ crates/gitlawb-node/src/auth/mod.rs | 4 + crates/gitlawb-node/src/config.rs | 120 ++++ crates/gitlawb-node/src/main.rs | 17 + crates/gitlawb-node/src/server.rs | 11 + crates/gitlawb-node/src/state.rs | 27 + crates/gitlawb-node/src/test_support.rs | 6 + 7 files changed, 876 insertions(+) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 284c9398..ac8e615c 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -58,6 +58,11 @@ pub async fn get_by_cid( Path(cid_str): Path, State(state): State, auth: Option>, + // Per-source keying for the walk concurrency sub-cap. Infallible extractors + // (mirror the git handlers in `repos.rs`): `PeerAddr` yields `None` under + // `oneshot` with no `ConnectInfo`, and the header map falls back per `client_key`. + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + req_headers: HeaderMap, ) -> Result { // 1. Decode the CID and extract the SHA-256 digest let cid = CidGeneric::<64>::from_str(&cid_str) @@ -76,6 +81,36 @@ pub async fn get_by_cid( let caller = auth.as_ref().map(|e| e.0 .0.as_str()); let caller_owned = caller.map(|c| c.to_string()); + // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds + // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with + // no served-git admission of its own; a permissionless caller could otherwise fan + // out concurrent walks past every git pool, exhausting the blocking pool + PIDs. + // Acquire the global permit (and, for a resolvable source, the per-source + // sub-permit) ONCE here and hold BOTH for the whole request — across every + // `spawn_blocking` walk in the loop below — so the slot reflects real blocking-thread + // occupancy (a tokio walk-timeout cannot free it while the blocking work still runs) + // and one request cannot open more than its share of concurrent walks. On + // unavailability shed a clean 503. The per-source key is the resolved source IP + // (`client_key`), never the DID (`/ipfs` admits any `did:key` unthrottled, so a DID + // key would be free to mint around); a `None` key (no trusted header, no peer) is + // bounded by the global pool only, never the per-source sub-cap. + let _ipfs_walk_permit = state + .git_ipfs_walk_semaphore + .clone() + .try_acquire_owned() + .map_err(|_| { + tracing::warn!("/ipfs walk concurrency cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity, retry shortly".into()) + })?; + let source_key = crate::rate_limit::client_key(&req_headers, peer, state.push_limiter_trust); + let _ipfs_caller_permit = match &source_key { + Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { + tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity for this source, retry shortly".into()) + })?), + None => None, + }; + // 2. Search all repos for an object with this SHA-256 let repos = state .db @@ -104,6 +139,11 @@ pub async fn get_by_cid( // deny entry (#126). let mut allowed_memo: HashMap> = HashMap::new(); + // Cap the number of candidate repos one request walks (it already short-circuits on + // serve): a CID present in — or path-gated out of — many repos must not serialize an + // unbounded number of full-history walks inside the single held admission slot. + let mut repos_walked: usize = 0; + for repo in &repos { // Repo-level read gate against THIS row's own rules (KTD2a). let rules: &[crate::db::VisibilityRule] = rules_by_repo @@ -114,6 +154,19 @@ pub async fn get_by_cid( continue; } + // Loop bound (#174 P1-3): once this request has walked its cap of candidate + // repos (each a git subprocess, up to a full-history walk), stop and fall + // through to the opaque 404 rather than serialize an unbounded number under the + // single held admission slot. + if repos_walked >= state.config.ipfs_max_repos_walked { + tracing::warn!( + cap = state.config.ipfs_max_repos_walked, + "/ipfs request hit the per-request repo-walk cap; stopping the scan" + ); + break; + } + repos_walked += 1; + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise // block the whole /ipfs request). On expiry keep the existing fail-closed skip — @@ -249,3 +302,641 @@ pub async fn list_pins(State(state): State) -> Result Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state) + } + + /// A syntactically valid CIDv1(raw, sha2-256) string the handler decodes past its + /// CID/hash-code validation, so the request reaches the walk admission (not a 400). + fn valid_cid() -> String { + gitlawb_core::cid::Cid::from_git_object_bytes(b"blob 5\0hello") + .as_str() + .to_string() + } + + fn get_cid(cid: &str, peer: Option) -> Request { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + if let Some(p) = peer { + req.extensions_mut().insert(ConnectInfo(p)); + } + req + } + + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` + /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID + /// validation), so a lazy DB-free state suffices — exactly like the served-git shed + /// tests. MUTATION (RED): delete the `git_ipfs_walk_semaphore` acquire in + /// `get_by_cid` and the request no longer sheds here (it falls through to the DB / + /// walk and returns something other than 503). + #[tokio::test] + async fn get_by_cid_sheds_with_503_when_walk_pool_exhausted() { + let mut state = crate::test_support::test_state_lazy(); + // Global /ipfs walk pool exhausted; per-source cap permissive so only the global + // pool can shed. Route rate limit is applied as a layer in production, not here. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.9:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted /ipfs walk pool must shed the request with 503" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// Per-source sub-cap, the `Some(ip)` arm: with per-source = 1 and the source pinned + /// at its single slot, a request from THAT source sheds 503 (global pool has room), + /// while a request from a DIFFERENT source is NOT shed by the cap (it proceeds past + /// admission). Pinning proves the `PeerAddr`/`HeaderMap` extractors resolved the key + /// — an inert `None` key would never shed on the per-source cap. MUTATION (RED): + /// delete the `git_ipfs_walk_per_caller` acquire and the capped source no longer + /// sheds. + #[tokio::test] + async fn get_by_cid_per_source_cap_sheds_same_source_admits_other() { + let mut state = crate::test_support::test_state_lazy(); + // Global pool has room; the per-source cap is 1. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(8)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let capped: SocketAddr = "203.0.113.20:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.21:5000".parse().unwrap(); + + // Pin the capped source at its single walk slot. + let _slot = state + .git_ipfs_walk_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first walk slot for the capped source IP"); + + let cid = valid_cid(); + // The capped source sheds on the per-source cap even with global capacity free. + let resp = ipfs_router(state.clone()) + .oneshot(get_cid(&cid, Some(capped))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its per-source /ipfs walk cap must shed 503 with global capacity free" + ); + + // A DIFFERENT source is NOT shed by the per-source cap: it clears admission and + // proceeds (then errors on the lazy DB, which is not a 503). + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(other))) + .await + .unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must not be shed by the per-source cap" + ); + } + + /// The `None`-key arm: a request with no resolvable source key (no trusted-proxy + /// header, no `ConnectInfo`) is bounded by the GLOBAL pool only, never the per-source + /// sub-cap. With the global pool exhausted it still sheds 503 (the counterpart to the + /// `Some(ip)` arm above, so neither arm is vacuous). + #[tokio::test] + async fn get_by_cid_none_key_arm_sheds_on_global_pool() { + let mut state = crate::test_support::test_state_lazy(); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + // Per-source cap permissive so only the global pool can shed. + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // No ConnectInfo + no trusted header -> client_key resolves None. + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), None)) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL /ipfs walk pool" + ); + } + + /// Map self-bound (INV-15): the `/ipfs` per-source map is a `PerCallerConcurrency` + /// built via `with_default_max_keys`, so a distinct-source-key flood cannot grow it + /// past the cap and a rejected key never allocates (reject-before-insert). Mirrors + /// `per_caller_concurrency_map_is_self_bounding_and_reject_before_insert` for the + /// pool U3 adds. + #[tokio::test] + async fn ipfs_walk_per_caller_map_is_self_bounding_and_reject_before_insert() { + let lim = crate::rate_limit::PerCallerConcurrency::new(4, 3); + // Acquire+drop a flood of distinct keys — the map self-empties (a key is removed + // the instant its in-flight count hits zero). + for i in 0..50 { + let _p = lim.try_acquire(&format!("src{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "an acquire+drop flood of distinct sources leaves the /ipfs map empty" + ); + // Reject-before-insert: hold max_keys distinct sources, then a new one sheds + // without growing the map. + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct sources held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new source key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + + /// Retain-through-blocking (R3, the load-bearing async property): the walk + /// admission is held until the `spawn_blocking` walk actually RETURNS, not when a + /// tokio timeout fires. With the global pool at size 1, drive a request until its + /// walk (a fake git that hangs on `rev-list`) is in flight; the slot must stay held + /// (`available_permits() == 0`) and a replacement from a DIFFERENT source must shed + /// 503 for as long as the blocking walk runs — even though the request future is + /// only `.await`ing the blocking join. When the blocking walk ends the permit frees + /// and a replacement is admitted. The permit lives INSIDE the handler across the + /// blocking `.await`; move it out (drop before the walk) and the replacement would + /// be admitted while the walk still burns a blocking thread (the bug this guards). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_permit_held_through_blocking_walk(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git for the /ipfs WALK only (object_type/read_object_content use the real + // `git`, so the object must genuinely exist below). Empty refs (so + // assert_all_refs_are_commits returns Ok without the peel), `rev-parse` resolves, + // and `rev-list` records its pid then sleeps ~6s so the walk BLOCKS + // deterministically. The sleep bounds the walk so a broken fix cannot wedge the + // suite. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo $$ > \"{}\"; sleep 6 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global walk pool at size 1; per-source cap permissive so only the + // held global permit can shed the replacement. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6ipfs1"; + let name = "ip1"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 bare + // repo there with a committed blob under `src/`, so real `git cat-file -t ` classifies it as a blob (the CID digest IS the sha256 object id in + // object-format=sha256) and the handler reaches the path-scoped walk branch. + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let work = tmp.path().join("work"); + std::fs::create_dir_all(work.join("src")).unwrap(); + std::fs::write(work.join("src/secret.txt"), b"ipfs walk retain proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + // The blob's SHA-256 object id (= the CID's digest); build the CID from it. + let oid = { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + assert!(out.status.success(), "rev-parse failed"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); + let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string(); + // Precondition: real git classifies the object as a blob (so the handler reaches + // the walk branch, not an early `continue`). + assert_eq!( + crate::git::store::object_type(&bare, &oid) + .unwrap() + .as_deref(), + Some("blob"), + "the seeded sha256 blob must exist so the handler reaches the walk" + ); + // A path-scoped rule so has_path_scoped_rule() is true (the walk branch) without + // denying the "/" gate on the public repo. + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_ipfs_walk_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one walk slot before the request" + ); + + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until the fake git's rev-list records its pid — the walk is now in the + // blocking pool and the request future is `.await`ing its join, holding the walk + // permit. Stop polling the instant the future completes (re-polling a completed + // oneshot panics). + let mut walk_pid: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let pid = walk_pid + .unwrap_or_else(|| panic!("the fake git rev-list must have spawned; early: {early:?}")); + // Reap the sleeping child on drop so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Load-bearing: while the blocking walk runs, the slot is HELD and a replacement + // from a DIFFERENT source sheds 503 — proving the permit is retained across the + // spawn_blocking join, not freed by a tokio timeout. + assert_eq!( + sem.available_permits(), + 0, + "the walk slot must be held while the spawn_blocking walk runs" + ); + let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a replacement must shed 503 while the prior request's blocking walk still runs" + ); + + // Drop the in-flight request; the detached blocking walk keeps running (a + // spawn_blocking cannot be cancelled), but on the fix the permit is a handler + // local, so dropping the future releases it once the blocking join is abandoned. + // Either way, kill the sleeping child so the slot frees promptly and poll for + // recovery — the point already proven above is that the slot stayed held for the + // duration of the blocking work. + drop(fut); + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the blocking walk ends the walk permit must free the global slot" + ); + } + + /// Loop bound (cap N): one `/ipfs/{cid}` request against a CID present in many repos + /// must not serialize an unbounded number of full-history walks. With + /// `ipfs_max_repos_walked = 1` and TWO public, path-scoped repos both carrying the + /// blob at the requested CID, the handler walks only the FIRST candidate then stops + /// (the second is cut by the cap), so the fake git's `rev-list` (one per walk) runs + /// exactly once. MUTATION (RED): remove the `repos_walked >= cap` break and both + /// repos are walked (count 2). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_caps_repos_walked_per_request(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + // Fake git for the WALK: empty refs, `rev-parse` resolves, and each `rev-list` + // appends one line to a log (so the number of walks == the line count) and exits + // with EMPTY output (the allowed-set is empty, so every repo path-gates to a + // `continue` and the request 404s after walking). object_type uses the REAL git, + // so the seeded blob below must genuinely exist. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // The bound under test: walk at most one candidate repo per request. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed TWO public repos, each with the SAME blob (same content -> same sha256 OID + // -> same CID) under a path-scoped rule, so both are walk candidates for one CID. + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let mut oid = String::new(); + for (i, name) in ["ipa", "ipb"].iter().enumerate() { + let owner = "z6ipfsN"; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.path().join(format!("work{i}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + // Identical content in both repos -> identical sha256 blob OID -> one CID. + std::fs::write(work.join("src/secret.txt"), b"loop bound proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + if oid.is_empty() { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + } + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderBBBBBBBBBBBBBBBBBBBBBBBB".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); + let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string(); + + let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = ipfs_router(state).oneshot(req).await.unwrap(); + // The empty allowed-set path-gates both repos to a `continue`, so a 404; the + // point is HOW MANY walks ran to get there. + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "with the per-request repo-walk cap at 1, only the first candidate repo is \ + walked (the second is cut by the cap), so exactly one walk runs; got {walks}" + ); + } + + /// Route rate limit is WIRED (not a silent no-op): the production `build_router` + /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP + /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does + /// nothing, so this proves the extension is attached. Drive it through the real + /// router with a tight limiter (1/hr): the second request from the same IP is 429. + /// MUTATION (RED): drop the `axum::Extension(ipfs_limiter)` layer in `server.rs` and + /// the second request is no longer braked (it reaches the handler, 404, not 429). + #[sqlx::test] + async fn ipfs_route_ip_rate_limit_is_attached(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + // Tight per-IP /ipfs bucket so the second request from one IP trips 429. + state.ipfs_rate_limiter = + crate::rate_limit::RateLimiter::new(1, std::time::Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = crate::server::build_router(state); + let cid = valid_cid(); + let make = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + let peer: SocketAddr = "203.0.113.99:5000".parse().unwrap(); + + // First request from this IP passes the brake and reaches the handler (404 — no + // such object anywhere), debiting the single-slot bucket. + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "the first /ipfs request from an IP must pass the rate brake" + ); + // Second request from the SAME IP is braked with 429 — proving the limiter + // extension is attached (a bare no-op layer would let it through to 404). + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "an exhausted per-IP /ipfs bucket must brake with 429 — the IpRateLimiter \ + extension must be attached to the route" + ); + // A DIFFERENT IP still has its own budget (independent bucket). + let other: SocketAddr = "203.0.113.100:5000".parse().unwrap(); + let resp = router.oneshot(make(other)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a different IP must not be braked by another IP's exhausted bucket" + ); + } +} diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index f18b4963..eed49253 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -528,6 +528,10 @@ mod tests { git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 68125364..b3667024 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -335,6 +335,66 @@ pub struct Config { value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) )] pub max_concurrent_reads_per_caller: usize, + + /// Maximum number of concurrent `GET /ipfs/{cid}` requests that may run their + /// visibility walk at once. The publicly-reachable `/ipfs/{cid}` route runs + /// `allowed_blob_set_for_caller_bounded` in `spawn_blocking` — a full-history + /// git walk (up to `git_service_timeout_secs`) — for each candidate repo. It + /// draws from THIS pool, not any served-git pool: a distinct public cost center + /// on a distinct surface, so sharing a git pool would let anonymous /ipfs + /// traffic shed authenticated git ops (the auth-boundary trap). A permit is + /// held for the whole request (across the repo loop) so it reflects real + /// blocking-thread occupancy, not merely the tokio wait. Beyond this the request + /// sheds a clean 503 + Retry-After. Must be between 1 and 1_048_576; the ceiling + /// keeps the value under tokio's `Semaphore` permit limit so an oversized value + /// is a clean CLI error rather than a boot-time panic. Default: 32. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_IPFS_WALKS", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_ipfs_walks: usize, + + /// Maximum concurrent `/ipfs/{cid}` walk requests a single source may hold at + /// once, so one source cannot monopolize `max_concurrent_ipfs_walks` (#174). + /// Callers are keyed on the RESOLVED SOURCE IP (`client_key`/`GITLAWB_TRUSTED_PROXY`), + /// never the DID — `/ipfs` accepts any `did:key` via `optional_signature` with no + /// admission step, so keying on the DID would let one host mint disposable DIDs to + /// multiply its budget. A request with no resolvable key (no trusted header, no + /// peer) is bounded by the global pool only, never this sub-cap. Over-cap sheds a + /// clean 503 + Retry-After. Must be between 1 and 1_048_576. Default: 4. + #[arg( + long, + env = "GITLAWB_IPFS_WALK_PER_SOURCE", + default_value_t = 4, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_walk_per_source: usize, + + /// Upper bound on the number of candidate repos a single `/ipfs/{cid}` request + /// will walk before giving up (returning the opaque 404). The handler already + /// short-circuits the moment it serves the object, but a CID that is present in + /// (or path-gated out of) many repos could otherwise serialize one full-history + /// walk per repo inside a single held admission slot. Capping the count bounds + /// the worst-case work one request can pin its slot with. Must be between 1 and + /// 1_048_576. Default: 64. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_REPOS_WALKED", + default_value_t = 64, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_repos_walked: usize, + + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The + /// route is publicly reachable (`optional_signature`) and each request can drive + /// a full-history git walk, so it carries a per-IP flood brake in addition to the + /// concurrency cap above (a rate limit bounds request *rate*, the semaphore + /// bounds concurrent slow holds — different axes). Keyed on the resolved client + /// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600. + #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] + pub ipfs_rate_limit: usize, } impl Config { @@ -428,6 +488,66 @@ mod tests { ); } + #[test] + fn max_concurrent_ipfs_walks_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_ipfs_walks, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "4"]) + .max_concurrent_ipfs_walks, + 4 + ); + // 0 permits would shed every /ipfs walk with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048576"]) + .max_concurrent_ipfs_walks, + 1_048_576 + ); + } + + #[test] + fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() { + assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-walk-per-source", "2"]) + .ipfs_walk_per_source, + 2 + ); + // 0 would shed every /ipfs walk from a keyed source; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "1048577"]).is_err() + ); + } + + #[test] + fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked, + 64 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"]) + .ipfs_max_repos_walked, + 8 + ); + // 0 would walk no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1048577"]).is_err() + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 380ab5f4..8727aae1 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -412,8 +412,25 @@ async fn main() -> Result<()> { git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), + // Bounds concurrent /ipfs visibility walks — a distinct public cost center, so + // its own pool + per-source sub-cap + per-IP rate limiter, never a git pool + // (#174 P1-3). The per-source map is bounded (reject-before-insert, INV-15). + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_ipfs_walks, + )), + git_ipfs_walk_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.ipfs_walk_per_source, + ), + ipfs_rate_limiter: rate_limit::RateLimiter::new_bounded( + config.ipfs_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ), git_bin: "git".to_string(), }; + if config.ipfs_rate_limit == 0 { + tracing::warn!("GITLAWB_IPFS_RATE_LIMIT=0 — per-IP /ipfs rate limiting disabled"); + } // Periodic peer-count poll for the metrics gauge. If p2p is disabled // we still set the gauge to 0 so dashboards don't show "no data". diff --git a/crates/gitlawb-node/src/server.rs b/crates/gitlawb-node/src/server.rs index f4c0d3e3..de61fcbe 100644 --- a/crates/gitlawb-node/src/server.rs +++ b/crates/gitlawb-node/src/server.rs @@ -214,9 +214,20 @@ pub fn build_router(state: AppState) -> Router { // identity and can apply per-repo visibility (#110); anonymous callers stay // anonymous and still read genuinely public content. `/api/v1/ipfs/pins` // stays unsigned — gating the pin index is tracked separately (#121). + // `/ipfs/{cid}` also carries a per-IP flood brake: it is anon-reachable and each + // request can drive a full-history git walk, so the per-IP rate limiter is the + // outermost layer (rejects a flood before the walk-admission work), mirroring the + // push/create routers. The extension MUST be attached or rate_limit_by_ip is a + // silent no-op. `/api/v1/ipfs/pins` (no walk) is merged in unbraked, as before. + let ipfs_limiter = rate_limit::IpRateLimiter { + limiter: state.ipfs_rate_limiter.clone(), + trust: state.push_limiter_trust, + }; let ipfs_routes = Router::new() .route("/ipfs/{cid}", get(ipfs::get_by_cid)) .layer(middleware::from_fn(auth::optional_signature)) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(ipfs_limiter)) .merge(Router::new().route("/api/v1/ipfs/pins", get(ipfs::list_pins))); // ── Arweave permanent anchors ────────────────────────────────────────── diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 3ff86923..3b8ba250 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -141,6 +141,33 @@ pub struct AppState { /// on the resolved source IP (never the DID — a DID farm defeats a DID key). Sized /// like `git_push_advert_per_caller`, a fraction of `max_concurrent_git_pushes`. pub git_write_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Bounds concurrent `GET /ipfs/{cid}` visibility-walk requests. The public + /// `/ipfs/{cid}` route runs `allowed_blob_set_for_caller_bounded` in + /// `spawn_blocking` (a full-history git walk) with NO served-git admission of its + /// own; without this a permissionless caller fans out concurrent walks past every + /// git pool, exhausting the blocking pool + PIDs (#174 P1-3). A request acquires a + /// permit before the repo loop and holds it for the whole request (across every + /// `spawn_blocking` walk), so the slot reflects real thread occupancy — a tokio + /// walk-timeout cannot free it while the blocking work still runs. A pool of its + /// own (`max_concurrent_ipfs_walks`), NOT a git pool: distinct cost center + public + /// surface, so anonymous /ipfs traffic can never shed an authenticated git op. + pub git_ipfs_walk_semaphore: Arc, + /// Per-source concurrency sub-cap on the `/ipfs/{cid}` walk pool: each source + /// (keyed on the resolved source IP, never the DID — `/ipfs` admits any `did:key` + /// unthrottled, so a DID key would be free to mint around) may hold at most + /// `ipfs_walk_per_source` in-flight walk slots, so one source cannot monopolize + /// `git_ipfs_walk_semaphore` (#174 P1-3). A request with no resolvable key is + /// bounded by the global pool only, never this sub-cap. The key map is bounded + /// (`with_default_max_keys`, reject-before-insert) so a source-key farm cannot grow + /// it (INV-15). + pub git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-client-IP rate limiter for `GET /ipfs/{cid}`. The route is publicly + /// reachable and each request can drive a full-history git walk, so it carries a + /// per-IP flood brake in addition to the concurrency cap above — a rate limit + /// bounds request *rate*, the semaphore bounds concurrent slow holds (different + /// axes). Keyed on the resolved client IP via `push_limiter_trust`. Layered on the + /// `/ipfs` route via `rate_limit_by_ip`. + pub ipfs_rate_limiter: RateLimiter, /// The `git` executable the served-git withheld-blob walk spawns. Production is /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's /// process-group teardown in handler tests without mutating the process-global diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index a27be946..cfa676af 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -92,6 +92,12 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { 8, ), git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + // Generous — a test that drives the /ipfs walk shed overrides these directly. + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 16, + ), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } From 394768711ec6afa834d010734c34cafd1f6063ad Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 13:10:04 -0500 Subject: [PATCH 36/52] fix(node): bound the post-push encryption task set by per-repo coalescing without dropping recovery copies (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a successful path-scoped push the handler spawns a DETACHED task that runs pin_new_objects then withheld_recipients_gated, which acquire_owned().await's on git_encrypt_semaphore — it DEFERS (blocks) when the pool is full. The semaphore caps active walks but nothing capped how many detached tasks were spawned and PARKED on that await: N rapid path-scoped pushes spawned N tasks, each holding cloned object lists/rules/paths/keys — an unbounded parked-waiter set (P2-2). 2a54c15 deliberately chose defer-not-shed because dropping the walk loses the withheld-blob recovery copy and there is no reconciliation sweep to rebuild it. So bound the OUTSTANDING-TASK set by per-repo coalescing rather than shedding: EncryptInflight (a bounded Arc>>) + try_begin — before the spawn, if a task for the repo is already in-flight skip the duplicate (the pending walk covers the newer objects); otherwise spawn and move an RAII EncryptInflightGuard into the task that removes the key on drop (completion, error, or panic-unwind). This bounds the set to <=1 pending task per repo and never drops work; withheld_recipients_gated's defer is unchanged. The second detached post-push spawn (Pinata replication, also runs pin_new_objects) is scoped out with rationale: it parks on no semaphore (no unbounded waiter set) and does per-push per-ref work (branch->CID, gossip, subscription, Arweave, peer-notify) keyed to this push's ref_updates — coalescing it would DROP a later push's announcements, a correctness regression. Regressions (mutation-verified RED->GREEN): the outstanding set is bounded to 1 per repo under saturation (never-coalesce -> 32 tasks RED); a coalesced repo's key is released when its task ends so it is reprocessed, never permanently skipped (guard-drop no-op -> repo locked out, recovery copy lost forever, RED — the exact durability regression the fix prevents); distinct repos each admit; cold-set first push admits; plus an inv22_gates structural tripwire that fails if the handler gate is removed. --- crates/gitlawb-node/src/api/repos.rs | 383 +++++++++++++++++------ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 5 + crates/gitlawb-node/src/state.rs | 96 ++++++ crates/gitlawb-node/src/test_support.rs | 1 + crates/gitlawb-node/tests/inv22_gates.rs | 10 + 6 files changed, 407 insertions(+), 89 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 5b11404d..6b52273f 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1419,109 +1419,145 @@ pub async fn git_receive_pack( // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). // Skipped entirely when the public cannot read the repo (withheld == None). + // + // Coalesce per repo (#174 P2-2): this task parks on `git_encrypt_semaphore` + // (which DEFERS when the pool is full rather than dropping the recovery copy). To + // bound the OUTSTANDING parked-task set, only spawn if no encryption task for this + // repo is already in flight; otherwise skip — the pending/next walk over this + // repo's history already covers the newer push's objects, so the recovery copy is + // delayed, never lost. The guard removes the repo key when the task ends (success, + // error, or panic), so a later push is re-admitted (no permanent skip). if withheld.is_some() { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - let enc_git_bin = state.git_bin.clone(); - let enc_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let encrypt_sem = state.git_encrypt_semaphore.clone(); - tokio::spawn(async move { - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } + match state.encrypt_inflight.try_begin(&record.id) { + None => { + tracing::debug!( + repo = %record.id, + "post-push encryption task already in flight for this repo; coalescing \ + (the pending recovery-copy walk covers this push's objects)" + ); } - - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - // Bound the number of concurrent post-push encryption walks (#174 P1-e): - // acquire an admission permit before the full-history walk, deferring - // when the pool is full rather than shedding the recovery pin. - let recip = withheld_recipients_gated( - encrypt_sem.clone(), - repo_path_clone.clone(), - enc_git_bin.clone(), - enc_timeout, - rules, - is_public, - owner_did.clone(), - ) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( + Some(inflight_guard) => { + let object_list_ipfs = object_list.clone(); + let ipfs_api = state.config.ipfs_api.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let rules_for_enc = rules_opt.clone(); + let repo_id = record.id.clone(); + let owner_did = record.owner_did.clone(); + let is_public = record.is_public; + let irys_url = state.config.irys_url.clone(); + let http_client = std::sync::Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let node_seed = state.node_keypair.to_seed(); + let repo_name = record.name.clone(); + let enc_git_bin = state.git_bin.clone(); + let enc_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let encrypt_sem = state.git_encrypt_semaphore.clone(); + tokio::spawn(async move { + // Held for the whole task; drop (on completion/error/panic) releases the + // repo's coalescing key so the next push for this repo can spawn again. + let _inflight_guard = inflight_guard; + let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, &repo_path_clone, + object_list_ipfs, &db_clone, - &repo_id, - &node_seed, - &recipients, ) .await; + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); + for (sha, cid) in &pinned { + tracing::info!(sha = %sha, %cid, "pinned"); + } + } - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, + // Option B1: encrypt-then-pin the withheld blobs so authorized + // readers can recover them when the origin cannot serve them. + // No path-scoped rule can withhold a blob, so withheld_blob_recipients + // would return an empty map after a full per-ref walk; skip it. Mirrors + // the has_path_scoped_rule gate on the other two withheld-walk sites. + if let Some(rules) = + rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) + { + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + encrypt_sem.clone(), + repo_path_clone.clone(), + enc_git_bin.clone(), + enc_timeout, + rules, + is_public, + owner_did.clone(), ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), + .await; + if let Ok(Ok(recipients)) = recip { + let delta = crate::encrypted_pin::encrypt_and_pin( + &ipfs_api, + &repo_path_clone, + &db_clone, + &repo_id, + &node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the blobs sealed + // this push to Arweave, so the oid->cid index survives total + // node loss. Best-effort; never fails the push. + if !delta.is_empty() && !irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&owner_did); + let repo_slug = format!("{owner_short}/{repo_name}"); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &owner_did, + node_did: &node_did_str, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &http_client, + &irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } } } - } + }); } - }); + } } - // Pin new git objects to Pinata, then record branch→CID and gossip + // Pin new git objects to Pinata, then record branch→CID and gossip. + // + // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought + // under the per-repo encryption coalescing above. Two reasons: (1) it does not + // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` + // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter + // set that is the P2-2 residual; it runs to completion under the HTTP client's + // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does + // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription + // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's + // ref_updates. Coalescing it against an in-flight task for the same repo would DROP + // a later push's ref-update announcements (a correctness regression), not merely + // delay a duplicate. So it is scoped out with rationale, not brought under the bound. { let pinata_jwt = state.config.pinata_jwt.clone(); let pinata_upload_url = state.config.pinata_upload_url.clone(); @@ -4317,6 +4353,175 @@ mod tests { ); } + // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- + // + // The residual jatmn found is not the WALK (bounded by `git_encrypt_semaphore`, + // proven by `encrypt_walk_defers_when_pool_exhausted` above) but the OUTER + // `tokio::spawn` + its parked `acquire_owned().await` waiters: N rapid pushes to a + // repo spawn N tasks that each park holding cloned object lists/rules/keys — an + // unbounded outstanding set. U4 bounds it by coalescing per repo: before spawning, + // if a task for the repo is in flight, skip the duplicate. Crucially this DEFERS a + // duplicate walk (the newer push's objects are covered by the pending one) and does + // NOT shed — there is no reconciliation sweep, so a dropped job would permanently + // lose the withheld-blob recovery copy (`2a54c15`'s fail-closed durability stance). + // + // These drive the coalescing seam (`EncryptInflight`) that the detached spawn at + // `repos.rs` consults directly (the try_begin gate on the in-flight set, guarded by + // `withheld.is_some()`). Observing `encrypt_and_pin`'s IPFS effect end-to-end needs a live IPFS node + // (`pin_git_object` hits the API), so the durability property is proven at this + // layer: a coalesced repo's key is released when its task ends, so a later push for + // that repo is processed once — NOT permanently skipped, which is exactly what a + // coalesce->shed mutation would break by dropping the job with no sweep to recover it. + + /// Bounded outstanding set under saturation (R4). Simulate K rapid path-scoped + /// pushes to the SAME repo while the encrypt pool is saturated (every spawned task + /// would park, so none has finished and removed its key): the first `try_begin` + /// admits (spawns), the rest coalesce (skip). The in-flight set holds at 1, not K. + /// + /// MUTATION (RED): removing the coalescing check makes every push spawn — modeled by + /// `simulate_without_coalescing`, which reaches K. If the coalesced count equaled the + /// un-coalesced one the gate would be a no-op; the strict inequality proves it bites. + #[test] + fn u4_outstanding_encrypt_set_is_bounded_to_one_per_repo_under_saturation() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkRepoOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA/proj"; + const K: usize = 32; + + // Hold every admitted guard so the tasks are "still in flight" (the saturated + // case: all parked on acquire_owned().await, none finished, none removed a key). + let mut admitted = Vec::new(); + let mut coalesced = 0usize; + for _ in 0..K { + match inflight.try_begin(repo) { + Some(g) => admitted.push(g), + None => coalesced += 1, + } + } + + assert_eq!( + admitted.len(), + 1, + "exactly ONE detached task may spawn per repo while one is in flight — the \ + outstanding set is bounded to 1, not K parked waiters" + ); + assert_eq!( + coalesced, + K - 1, + "the other K-1 rapid pushes to the same repo coalesce (skip spawning)" + ); + assert_eq!( + inflight.len(), + 1, + "the in-flight set holds at most one entry per repo under saturation" + ); + + let no_coalesce = simulate_without_coalescing(K); + assert_eq!( + no_coalesce, K, + "sanity: without the coalescing check all K pushes spawn (the unbounded set \ + the fix prevents) — proves the bound above is not vacuously 1" + ); + assert!( + admitted.len() < no_coalesce, + "coalesced set ({}) must be strictly smaller than the un-coalesced one ({})", + admitted.len(), + no_coalesce + ); + } + + /// Coalescing is PER-REPO: distinct repos are never coalesced against each other, so + /// one repo in flight cannot starve a second repo's recovery copy. + #[test] + fn u4_distinct_repos_each_admit_one_encrypt_task() { + let inflight = crate::state::EncryptInflight::new(); + let a = inflight.try_begin("owner/repo-a"); + let b = inflight.try_begin("owner/repo-b"); + let c = inflight.try_begin("owner/repo-c"); + assert!( + a.is_some() && b.is_some() && c.is_some(), + "three distinct repos each admit their own encryption task" + ); + assert_eq!(inflight.len(), 3, "one in-flight entry per distinct repo"); + } + + /// NO LOST RECOVERY COPY — the security guard (R4/R6). Coalescing must DELAY a + /// duplicate walk, never permanently drop a repo's recovery copy. Observable + /// property: once an in-flight task ENDS (its guard drops — completion, error, or + /// panic-unwind) the repo key is released, so the NEXT push for that repo is admitted + /// and processed again. A coalesce->shed mutation would drop the job AND never + /// re-admit — with no reconciliation sweep the copy is lost forever. Here re-admission + /// survives normal completion AND a panic, so no permanent skip / no leaked key. + #[test] + fn u4_coalesced_repo_is_reprocessed_after_task_ends_not_permanently_skipped() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkDurableRepoBBBBBBBBBBBBBBBBBBBBBBBBB/repo"; + + // Push #1 admits and "spawns". A concurrent push #2 (task #1 still in flight) + // coalesces — no duplicate spawn. + let guard1 = inflight.try_begin(repo).expect("first push admits"); + assert!( + inflight.try_begin(repo).is_none(), + "while task #1 is in flight, push #2 to the same repo coalesces" + ); + + // Task #1 finishes (encrypt_and_pin ran or errored): guard drops, key released. + drop(guard1); + assert_eq!( + inflight.len(), + 0, + "when the in-flight task ends its repo key is released — the set does not leak" + ); + + // A LATER push for the SAME repo is admitted again (processed, not skipped + // forever). This is what coalesce->shed breaks: shed drops the job and no sweep + // re-derives the missing copy, so the recovery copy is permanently lost. + let guard2 = inflight.try_begin(repo).expect( + "a later push for a coalesced repo MUST be re-admitted — durability: the \ + deferred recovery copy is produced eventually, never dropped", + ); + drop(guard2); + assert_eq!(inflight.len(), 0); + + // Durability across PANIC: a task that panics mid-walk must still release its key + // (Drop runs on unwind), so one crashed walk never permanently locks a repo out + // of future recovery copies. + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _g = inflight.try_begin(repo).expect("admit before the panic"); + assert_eq!(inflight.len(), 1); + panic!("simulate the detached encryption task panicking mid-walk"); + })); + assert!(panicked.is_err(), "the simulated task panicked"); + assert_eq!( + inflight.len(), + 0, + "a panicked encryption task still releases its repo key (Drop on unwind) — no \ + permanent leak that would block every future recovery copy for the repo" + ); + assert!( + inflight.try_begin(repo).is_some(), + "after a panicked task the repo can still be admitted — durability preserved" + ); + } + + /// Degenerate state: the first push on a cold/empty in-flight set always admits + /// (never a false coalesce on an empty set). + #[test] + fn u4_first_push_on_a_cold_set_always_admits() { + let inflight = crate::state::EncryptInflight::new(); + assert!(inflight.is_empty(), "cold set is empty"); + assert!( + inflight.try_begin("owner/first").is_some(), + "the first push on a cold in-flight set must admit (never falsely coalesce)" + ); + } + + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. + /// Returns the count of tasks spawned (== the size of the unbounded outstanding set + /// the fix prevents), used as the RED comparison in the bound test above. + fn simulate_without_coalescing(pushes: usize) -> usize { + (0..pushes).count() + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index eed49253..912108a0 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -524,6 +524,7 @@ mod tests { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 8727aae1..ffac65a2 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -395,6 +395,11 @@ async fn main() -> Result<()> { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher + // cannot grow the outstanding parked-waiter set past one task per repo (#174 + // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a + // sized pool. + encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 3b8ba250..deccf4c7 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -121,6 +121,16 @@ pub struct AppState { /// walk must not hold a foreground write slot, and a handler already holding a /// write permit that needed a second would self-deadlock at pool size 1. pub git_encrypt_semaphore: Arc, + /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing + /// (#174 P2-2). `git_encrypt_semaphore` caps *active* walks; this caps how many + /// detached tasks *spawn and park* on that semaphore's `acquire_owned().await`. + /// Before spawning a per-push encryption task, the receive-pack handler consults + /// this set: if the repo already has a task in flight it coalesces (skips the + /// duplicate spawn) rather than parking a new unbounded waiter. Coalescing only + /// delays a duplicate walk — it never drops the withheld-blob recovery copy, which + /// `2a54c15` deliberately kept fail-closed (there is no reconciliation sweep to + /// re-derive a dropped copy). See [`EncryptInflight`]. + pub encrypt_inflight: EncryptInflight, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` @@ -203,3 +213,89 @@ impl AppState { *self.shutdown_tx.borrow() } } + +/// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing +/// (#174 P2-2). Each successful path-scoped push `tokio::spawn`s a DETACHED task that +/// parks on `git_encrypt_semaphore.acquire_owned().await` (which DEFERS when the pool +/// is full rather than shedding — `2a54c15` kept it fail-closed so the withheld-blob +/// recovery copy is never dropped). The semaphore caps *active* walks, but nothing +/// capped how many detached tasks *spawn and park* on that await: N rapid pushes to a +/// repo spawn N parked tasks, each holding cloned object lists/rules/paths/keys — an +/// unbounded outstanding set. +/// +/// This tracks the repo keys with an in-flight encryption task. Before spawning, the +/// handler calls [`try_begin`](Self::try_begin): if a task for the repo is already +/// in-flight it returns `None` and the handler SKIPS spawning a duplicate (coalesce — +/// the newer push's objects are covered by the pending/next walk over the same repo's +/// history). This bounds the outstanding set to <=1 pending task per repo WITHOUT +/// dropping work: coalescing only delays a duplicate walk, it never sheds the recovery +/// copy (there is no reconciliation sweep, so a *dropped* job would be lost forever). +/// +/// The returned [`EncryptInflightGuard`] is moved into the detached task and removes +/// the repo key on drop — on normal completion, error, OR panic (Drop runs on unwind) +/// — so one crashed walk can never permanently lock a repo out of future recovery +/// copies. +#[derive(Clone, Default)] +pub struct EncryptInflight { + // std::sync::Mutex: only ever held for O(1) HashSet insert/remove in a sync + // context (right before `tokio::spawn`, and in the guard's Drop) — never across + // an await, so a std Mutex is correct and cheaper than a tokio one. + repos: Arc>>, +} + +impl EncryptInflight { + pub fn new() -> Self { + Self::default() + } + + /// Try to begin an encryption task for `repo_id`. Returns `Some(guard)` if no task + /// for the repo was in-flight (the caller should spawn), or `None` if one already + /// is (the caller should COALESCE — skip spawning a duplicate). The guard releases + /// the repo key on drop. + pub fn try_begin(&self, repo_id: &str) -> Option { + let mut set = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + if set.insert(repo_id.to_string()) { + Some(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + }) + } else { + None + } + } + + /// Number of repos with an in-flight encryption task. Test/metrics observability; + /// the bound under saturation is `len() <= number of distinct repos`, i.e. at most + /// one task per repo. + #[allow(dead_code)] + pub fn len(&self) -> usize { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .len() + } + + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// RAII guard removing a repo key from [`EncryptInflight`] when the detached +/// encryption task finishes (drop on completion, error, or panic-unwind). Move-only — +/// there is no reason to clone a guard, and cloning would double-remove. +pub struct EncryptInflightGuard { + repos: Arc>>, + repo_id: String, +} + +impl Drop for EncryptInflightGuard { + fn drop(&mut self) { + // A poisoned lock (a prior panic while holding it) still lets us take the inner + // set via into_inner-on-guard; but poisoning here is not expected because the + // only critical sections are the O(1) ops above. Remove best-effort. + if let Ok(mut set) = self.repos.lock() { + set.remove(&self.repo_id); + } + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index cfa676af..92f41a8d 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -87,6 +87,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 5f576300..7d3fd1c8 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -77,4 +77,14 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { "P1-e bypass: the bounded recipients walk must be invoked only inside \ withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap" ); + + // U4 / P2-2: the detached post-push encryption task must be gated by the per-repo + // coalescing set (`encrypt_inflight.try_begin`) so the OUTSTANDING parked-task set is + // bounded to <=1 per repo. Removing the gate lets N rapid pushes spawn N parked + // waiters (the unbounded set U4 closed); the semaphore only caps active walks. + assert!( + repos.contains("state.encrypt_inflight.try_begin"), + "U4/P2-2 gate missing: the detached post-push encryption spawn must consult \ + encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" + ); } From 413d6cf0bcf1a8c587e01d6c9384de70e2876ecb Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 13:16:40 -0500 Subject: [PATCH 37/52] fix(node): reject unsupported git service before the read slot; document the new admission knobs (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P2-1: git_info_refs treated every service other than git-receive-pack as a read op and did the DB/visibility/Tigris work under a read permit, validating the service string only downstream in smart_http. So an unauthenticated `?service=anything` to a public repo consumed a read slot + storage work before rejection. Validate the service is exactly git-upload-pack or git-receive-pack immediately after parsing, returning 400 before the pre-DB shed and the permit. Handler-layer regression: with the read pool exhausted, `?service=git-explode` returns 400 (validated first), not 503 — removing the validation makes it 503 (RED). U5/INV-24 docs: document the new knobs from U2/U3 in .env.example and the README env-var table — GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS, GITLAWB_MAX_CONCURRENT_IPFS_WALKS, GITLAWB_IPFS_WALK_PER_SOURCE, GITLAWB_IPFS_MAX_REPOS_WALKED, GITLAWB_IPFS_RATE_LIMIT. Each is already enforced on the path it names (U2/U3 tests), so the docs match the implemented routing. --- .env.example | 29 ++++++++++++++++ README.md | 5 +++ crates/gitlawb-node/src/api/repos.rs | 51 ++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/.env.example b/.env.example index 0ef193f8..149ea8b1 100644 --- a/.env.example +++ b/.env.example @@ -115,6 +115,16 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max seconds the storage-ACQUISITION phase of a served git op may run before the +# request is shed with a 503, separate from the git-run timeout above. A +# concurrency permit is taken before this phase and GITLAWB_GIT_SERVICE_TIMEOUT_SECS +# only starts once git spawns, so without this a stalled backend (a hung Tigris +# HEAD/GET, or a hung pg advisory-lock iteration on push) pins the permit and drains +# the pool until every later request 503s. On expiry the permit is released +# (fail-closed). Kept separate because acquisition and git execution are distinct +# cost centers. Must be positive; set very large to effectively disable. Default 30. +GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS=30 + # Max concurrent git READ ops (upload-pack + the upload-pack info/refs # advertisement) served at once, a global pool separate from the push pool below. # The anon receive-pack info/refs advertisement has its OWN pool (see below), not @@ -146,6 +156,25 @@ GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 # IP per hour. 0 disables. Default 600. GITLAWB_PUSH_RATE_LIMIT=600 +# ── /ipfs/{cid} visibility-walk admission (#174) ────────────────────────── +# GET /ipfs/{cid} runs a per-repo full-history git walk in a blocking thread to +# decide whether the caller may read a path-scoped blob. It is publicly reachable, +# so it is bounded to keep a permissionless caller from fanning out unbounded +# concurrent walks and exhausting blocking-pool threads + PIDs. +# Max concurrent /ipfs walks across all callers (a pool of its own, disjoint from +# the served-git pools). Over-cap sheds a 503. Default 32. +GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 +# Max concurrent /ipfs walks a single SOURCE IP may hold (keyed like the git +# per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). +# Default 4. +GITLAWB_IPFS_WALK_PER_SOURCE=4 +# Max repos walked per single /ipfs request, so one request cannot serialize a +# full-history walk over every repo carrying the CID. Default 64. +GITLAWB_IPFS_MAX_REPOS_WALKED=64 +# Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct +# from the concurrency caps above). 0 disables. Default 600. +GITLAWB_IPFS_RATE_LIMIT=600 + # ── Creation rate limiting (repo/agent/issue/PR flood brake) ────────────── # Max creation requests (POST /api/v1/repos, /api/register, fork, issues, # pulls) per client IP per hour, in addition to the per-DID limit. The per-DID diff --git a/README.md b/README.md index 542a988e..e9765876 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,11 @@ Important node settings: | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | | `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | +| `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | +| `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | +| `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max repos walked per `/ipfs/{cid}` request, bounding one request's fan-out. Default 64. | +| `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 6b52273f..9b05ece1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -515,6 +515,16 @@ pub async fn git_info_refs( let service = query .service .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; + // Reject an unsupported service BEFORE taking a read slot or doing any DB/Tigris + // work (#174 P2-1). git_info_refs otherwise treats everything that is not + // git-receive-pack as a read op, so an unauthenticated `?service=anything` to a + // public repo would consume a read permit and the visibility/Tigris work before + // validate_service rejected it downstream in smart_http. + if service != "git-upload-pack" && service != "git-receive-pack" { + return Err(AppError::BadRequest(format!( + "unsupported git service: {service}" + ))); + } // #62 cheap pre-DB load shed: if the pool this service draws from is already // saturated, shed with 503 before any DB/disk work. Best-effort (holds no // permit); the authoritative hold is `git_permit` below, after the per-source @@ -3118,6 +3128,47 @@ mod tests { ); } + /// #174 P2-1: an unsupported `?service=` must be rejected with 400 BEFORE taking a + /// read slot or doing DB/Tigris work. Isolate it: exhaust the read pool so a read + /// op WOULD shed 503 at the pre-DB check — a garbage service must still return 400 + /// (validation runs first), proving `?service=anything` cannot consume the read + /// pool. Removing the validation makes this 503 (RED). + #[sqlx::test] + async fn info_refs_rejects_unsupported_service_before_the_read_slot(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state + .db + .upsert_mirror_repo("z6svcowner", "svc", "/tmp/svc", None, false) + .await + .unwrap(); + // Exhaust the read pool: a read op would shed 503 at the pre-DB check. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); + + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.90:7000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6svcowner/svc/info/refs?service=git-explode") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "an unsupported ?service= must be 400 before the read-pool shed, not 503" + ); + } + /// #174 (jatmn P1): the anon-reachable receive-pack advertisement /// (`GET info/refs?service=git-receive-pack`) draws from a DEDICATED advert pool /// (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses. From 433c8915bf85a81db469dfead356d3c54a83220a Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 10:15:56 -0500 Subject: [PATCH 38/52] feat(node): integrate the #173 IPFS CID tree-gate onto the #174 walk-concurrency base Rebased onto #174 (fix/served-git-concurrency-cap). #173's incremental history cannot replay commit-by-commit because #174 rewrote the same handler files in parallel; this single commit carries the fully-integrated tree (identical to the verified merge result), with the follow-up fixes as separate commits on top. Brings in #173: GET /ipfs/{cid} CID->oid resolution via pinned_cids, per-caller path-scoped blob/tree/commit-tag gating (#135/#173 F1-F6), and the pin-source provenance table. Adapts #173's tree/commit-tag walks to #174's run_bounded_git so the held /ipfs walk-concurrency permit is duration-safe (F5). --- Cargo.lock | 11 +- crates/gitlawb-node/src/api/ipfs.rs | 1131 +++-- crates/gitlawb-node/src/api/mod.rs | 24 +- crates/gitlawb-node/src/api/repos.rs | 3 + crates/gitlawb-node/src/auth/mod.rs | 5 +- crates/gitlawb-node/src/db/mod.rs | 282 +- crates/gitlawb-node/src/error.rs | 12 + crates/gitlawb-node/src/git/store.rs | 20 + .../gitlawb-node/src/git/visibility_pack.rs | 1079 +++++ crates/gitlawb-node/src/ipfs_pin.rs | 42 +- crates/gitlawb-node/src/main.rs | 17 +- crates/gitlawb-node/src/pinata.rs | 31 +- crates/gitlawb-node/src/rate_limit.rs | 29 + crates/gitlawb-node/src/state.rs | 44 +- crates/gitlawb-node/src/test_support.rs | 3764 +++++++++++++++-- crates/gitlawb-node/src/visibility.rs | 25 + crates/gl/src/ipfs_cmd.rs | 202 +- 17 files changed, 5942 insertions(+), 779 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d12daa43..24beb9ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "git-remote-gitlawb" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "gitlawb-core", @@ -3312,7 +3312,7 @@ dependencies = [ [[package]] name = "gitlawb-attest" -version = "0.5.0" +version = "0.5.1" dependencies = [ "base64", "ed25519-dalek", @@ -3329,7 +3329,7 @@ dependencies = [ [[package]] name = "gitlawb-core" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "base64", @@ -3356,7 +3356,7 @@ dependencies = [ [[package]] name = "gitlawb-node" -version = "0.5.0" +version = "0.5.1" dependencies = [ "alloy", "anyhow", @@ -3412,7 +3412,7 @@ dependencies = [ [[package]] name = "gl" -version = "0.5.0" +version = "0.5.1" dependencies = [ "alloy", "anyhow", @@ -3824,6 +3824,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "tracing", ] diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index ac8e615c..a764901b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -1,14 +1,17 @@ //! GET /ipfs/{cid} — content-addressed retrieval of git objects by CIDv1. //! -//! Every git object stored on this node is addressable by its IPFS CIDv1. +//! Every git object pinned on this node is addressable by its IPFS CIDv1. //! The CID is computed as: //! //! CIDv1(codec=raw, multihash=sha2-256(content_bytes)) //! //! where `content_bytes` is the raw object content as returned by -//! `git cat-file ` (i.e. without the git framing header). -//! This is consistent with how `gitlawb_core::cid::Cid::from_git_object_bytes` -//! computes CIDs when objects are pushed. +//! `git cat-file ` (i.e. without the git framing header) — the +//! same bytes `gitlawb_core::cid::Cid::from_git_object_bytes` hashes when the +//! object is pinned. That digest is NOT the object's git oid: git frames the +//! content with a `" \0"` header before hashing, so `sha2-256(content)` +//! and the git oid differ. The handler therefore maps the CID back to its oid via +//! the `pinned_cids` table rather than treating the digest as an oid (#173). //! //! Serving is access-controlled: an object is returned only from a repo row the //! requesting caller is permitted to read (per-caller path-scoped visibility, @@ -27,14 +30,66 @@ use std::str::FromStr; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; use crate::git::store; -use crate::git::visibility_pack::{allowed_blob_set_for_caller_bounded, has_path_scoped_rule}; +use crate::git::visibility_pack::{ + allowed_blob_set_for_caller_bounded, allowed_tree_set_for_caller_bounded, has_path_scoped_rule, + reachable_commit_tag_oids_bounded, +}; use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; +/// Hard ceiling on the number of full-history reachability walks a single +/// `GET /ipfs/{cid}` request may spawn. The per-request `ipfs_rate_limiter` +/// check brakes *repeat* requests, but within one request the object can exist +/// under path-scoped rules in many repos, and each distinct repo pays its own +/// `spawn_blocking` walk (the memo only dedups the same repo). Without a ceiling +/// a single request fans out to O(repos) walks for one rate-limiter token — an +/// amplification sink (INV-10). Once this many walks have run, no further walk is +/// spawned for the rest of the request: any remaining candidate that still needs +/// a walk is skipped (and, with nothing else readable, the request falls through +/// to the opaque 404). The bound is deliberately generous: a legitimate caller +/// serves on the first repo that grants them, so reaching it requires being +/// denied by this many path-scoped repos first, which real traffic effectively +/// never does. Tunable if that assumption stops holding. +pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = 16; + +/// Hard per-request ceiling on how many legacy (NULL-provenance) repositories +/// the CID resolver's scan fallback may PROBE (`acquire` + `git cat-file -t`). +/// The provenance path targets one repo; the legacy scan, absent this bound, +/// fans one anonymous request out to O(repos) subprocess spawns and cold-cache +/// Tigris fetches for a CID enumerable from the public pins index (#173 round 3, +/// F1, INV-10). Deliberately generous: a normal node has far fewer repos than +/// this, so a genuine miss still completes the whole scan and returns a truthful +/// 404; only a node larger than the cap truncates, and a truncated search +/// surfaces as a retryable 503 (never a false "absent"). Legacy pins are a +/// shrinking set — each re-pin backfills provenance — so this fallback is a +/// transitional path, not the steady state. Tunable via `AppState`. +pub(crate) const MAX_LEGACY_PROBES_PER_REQUEST: u32 = 256; + +/// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` buffers and serves +/// (#173 round 8, F6, INV-10). The serve reads via a blocking `git cat-file` and +/// buffers the whole object; unbounded, a large public blob (enumerable from the pins +/// index) could exhaust memory or block a runtime worker. A content-addressed serve +/// must verify the whole object hashes to the requested CID before any byte egresses +/// (F2), so it cannot stream — it buffers up to this cap and withholds anything larger +/// (raise the cap if a class of legitimate objects legitimately exceeds it; never +/// stream unverified). 32 MiB is generous for git blobs/trees/commits. Tunable via +/// `AppState` for the test seam, like the sibling caps. +pub(crate) const MAX_SERVED_OBJECT_BYTES: u64 = 32 * 1024 * 1024; + +/// Lazily-loaded context for the legacy (NULL-provenance) scan fallback in +/// `get_by_cid`: all repos, their visibility rules keyed by repo id, and the set of +/// quarantined repo ids. Loaded once per request only if a legacy pin is hit. +type LegacyScanCtx = ( + Vec, + HashMap>, + HashSet, +); + /// GET /ipfs/{cid} /// -/// Search all repos on the node for a git object whose SHA-256 hash matches -/// the given CIDv1, returning its raw content if the caller may read it. +/// Resolve the CIDv1 to its git oid via the `pinned_cids` table, then search all +/// repos on the node for that object, returning its raw content if the caller may +/// read it. /// /// Visibility (#110, #126): the object is served only from a repo row the /// caller passes. For each iterated row we gate against that row's OWN rules @@ -43,13 +98,16 @@ use crate::visibility::{visibility_check, Decision}; /// row than the one read (KTD2a). We check object existence via /// `store::object_type` *before* the expensive reachability walk so random-CID /// spray cannot trigger full-history git walks on repos that don't carry the -/// object. When the row carries path-scoped rules (KTD4) the served object -/// must be either a non-blob (trees/commits are structural; KTD3) OR a blob -/// in the caller's *reachable* allowed-set (`allowed_blob_set_for_caller`). -/// The reachable allowed-set excludes dangling blobs — a blob written via -/// `git hash-object -w` and never committed has no path to gate, so it is -/// fail-closed 404'd under path-scoped rules (#126). Denial and genuine -/// not-found both fall through to an opaque 404. +/// object. When the row carries path-scoped rules (KTD4) the served object is +/// gated by type: a `blob`/`tree` must be in the caller's *reachable* allowed-set +/// (`allowed_blob_set_for_caller` / `allowed_tree_set_for_caller`), and a +/// `commit`/`tag` must be in the repo's *reachable* commit/tag set +/// (`reachable_commit_tag_oids`, #173). A withheld subtree's tree object is denied +/// here exactly as `get_tree` denies its path, so its child names and oids cannot +/// leak by CID (#135). All these sets exclude dangling objects — a blob, tree, +/// commit, or tag written via plumbing and never referenced has no reachable path, +/// so it is fail-closed 404'd under path-scoped rules (#126, #173). Denial and +/// genuine not-found both fall through to an opaque 404. /// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked @@ -57,14 +115,12 @@ use crate::visibility::{visibility_check, Decision}; pub async fn get_by_cid( Path(cid_str): Path, State(state): State, - auth: Option>, - // Per-source keying for the walk concurrency sub-cap. Infallible extractors - // (mirror the git handlers in `repos.rs`): `PeerAddr` yields `None` under - // `oneshot` with no `ConnectInfo`, and the header map falls back per `client_key`. crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, - req_headers: HeaderMap, + headers: HeaderMap, + auth: Option>, ) -> Result { - // 1. Decode the CID and extract the SHA-256 digest + // 1. Decode and validate the CID (uniform 400 on a malformed / non-sha2-256 + // CID, before any DB or git work). let cid = CidGeneric::<64>::from_str(&cid_str) .map_err(|e| AppError::BadRequest(format!("invalid CID: {e}")))?; @@ -77,9 +133,13 @@ pub async fn get_by_cid( )); } - let sha256_hex = hex::encode(mh.digest()); - let caller = auth.as_ref().map(|e| e.0 .0.as_str()); - let caller_owned = caller.map(|c| c.to_string()); + // Canonicalize the CID for the pinned_cids lookup. Pins are stored under the + // canonical base32 `cid.to_string()`, but a client may send any equivalent + // multibase spelling (base58/base64) of the same CID; those parse and pass + // the sha2-256 check yet miss the canonical key, so they must be normalized + // before the DB lookup (#173). Response headers and error messages still echo + // the original `cid_str` the client sent. + let canonical_cid = cid.to_string(); // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with @@ -87,13 +147,16 @@ pub async fn get_by_cid( // out concurrent walks past every git pool, exhausting the blocking pool + PIDs. // Acquire the global permit (and, for a resolvable source, the per-source // sub-permit) ONCE here and hold BOTH for the whole request — across every - // `spawn_blocking` walk in the loop below — so the slot reflects real blocking-thread + // `spawn_blocking` walk below — so the slot reflects real blocking-thread // occupancy (a tokio walk-timeout cannot free it while the blocking work still runs) - // and one request cannot open more than its share of concurrent walks. On - // unavailability shed a clean 503. The per-source key is the resolved source IP - // (`client_key`), never the DID (`/ipfs` admits any `did:key` unthrottled, so a DID - // key would be free to mint around); a `None` key (no trusted header, no peer) is - // bounded by the global pool only, never the per-source sub-cap. + // and one request cannot open more than its share of concurrent walks. Holding a + // slot across a walk is only safe because every walk child is duration-bounded + // (`*_bounded` + `run_bounded_git` teardown), so a hung git cannot pin the slot + // past `git_service_timeout_secs`. On unavailability shed a clean 503. The + // per-source key is the resolved source IP (`client_key`), never the DID (`/ipfs` + // admits any `did:key` unthrottled, so a DID key would be free to mint around); a + // `None` key (no trusted header, no peer) is bounded by the global pool only, + // never the per-source sub-cap. let _ipfs_walk_permit = state .git_ipfs_walk_semaphore .clone() @@ -102,7 +165,7 @@ pub async fn get_by_cid( tracing::warn!("/ipfs walk concurrency cap reached; shedding request with 503"); AppError::Overloaded("ipfs service at capacity, retry shortly".into()) })?; - let source_key = crate::rate_limit::client_key(&req_headers, peer, state.push_limiter_trust); + let source_key = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust); let _ipfs_caller_permit = match &source_key { Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); @@ -111,178 +174,553 @@ pub async fn get_by_cid( None => None, }; - // 2. Search all repos for an object with this SHA-256 - let repos = state + // Resolve the content-addressed CID to the object's git oid(s). A real pin + // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the + // git oid (git frames content with a `" \0"` header first), so we + // map it back through `pinned_cids` rather than treating the digest as an oid + // (#173). The cid index is non-unique, so one CID can map to several oids (a + // tree and a blob whose raw bytes collide, or content pinned under two oids); + // we try each candidate below rather than pick one arbitrarily and false-404 + // when the chosen one is withheld or absent while another is readable (#173). + // An empty result is an opaque 404, uniform with a genuine not-found and a + // visibility denial. + let oids = state .db - .list_all_repos() + .oids_for_cid(&canonical_cid) .await .map_err(AppError::Internal)?; + if oids.is_empty() { + return Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))); + } + let caller = auth.as_ref().map(|e| e.0 .0.as_str()); + let caller_owned = caller.map(|c| c.to_string()); - // Fetch every repo's visibility rules in one query rather than one per row - // (the gate runs each row against its OWN rules — KTD2a). A row absent from - // the map has no rules. - let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = state - .db - .list_visibility_rules_for_repos(&repo_ids) - .await - .map_err(AppError::Internal)?; + // Per-request walk budget + memos + throttle flag, shared by the provenance path + // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and + // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. + let mut walk = WalkState { + walks: 0, + probes: 0, + truncated: false, + allowed_blob_memo: HashMap::new(), + allowed_tree_memo: HashMap::new(), + reachable_ct_memo: HashMap::new(), + }; + // Set when a walk-requiring candidate is skipped because the source IP's walk quota + // is spent (#173 review, F-C): the scan keeps going so a later walk-free copy still + // serves; only if nothing is servable is it turned into the 429. + let mut throttled = false; + let rctx = ResolveCtx { + caller, + caller_owned: &caller_owned, + headers: &headers, + peer, + cid_str: &cid_str, + canonical_cid: &canonical_cid, + }; - // Request-scoped memo of the per-repo allowed-blob set (KTD1, #126). The - // caller is constant for one request, so `repo.id` alone is a safe, - // sufficient key — never a coarse caller "class", which - // `visibility_check`'s exact full-DID reader match would make unsafe. - // - // We flipped from a deny-set (`withheld_blob_oids`) to an allowed-set - // (`allowed_blob_set_for_caller`) so dangling blobs — never enumerated by - // the reachable walk — fail closed instead of slipping through an empty - // deny entry (#126). - let mut allowed_memo: HashMap> = HashMap::new(); - - // Cap the number of candidate repos one request walks (it already short-circuits on - // serve): a CID present in — or path-gated out of — many repos must not serialize an - // unbounded number of full-history walks inside the single held admission slot. - let mut repos_walked: usize = 0; - - for repo in &repos { - // Repo-level read gate against THIS row's own rules (KTD2a). - let rules: &[crate::db::VisibilityRule] = rules_by_repo - .get(&repo.id) - .map(Vec::as_slice) - .unwrap_or(&[]); - if visibility_check(rules, repo.is_public, &repo.owner_did, caller, "/") == Decision::Deny { - continue; - } + // Legacy scan context (repos + rules + quarantined ids), loaded LAZILY only when a + // legacy NULL-provenance pin is hit — the provenance path must never trigger the + // O(repos) load (that fan-out is exactly what provenance removes, #173 round 2). + let mut scan_ctx: Option = None; - // Loop bound (#174 P1-3): once this request has walked its cap of candidate - // repos (each a git subprocess, up to a full-history walk), stop and fall - // through to the opaque 404 rather than serialize an unbounded number under the - // single held admission slot. - if repos_walked >= state.config.ipfs_max_repos_walked { - tracing::warn!( - cap = state.config.ipfs_max_repos_walked, - "/ipfs request hit the per-request repo-walk cap; stopping the scan" - ); - break; - } - repos_walked += 1; - - // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares - // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise - // block the whole /ipfs request). On expiry keep the existing fail-closed skip — - // never serve an un-acquired repo; a public copy (if any) still gets its turn. - let acquire_deadline = - std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); - let repo_path = match tokio::time::timeout( - acquire_deadline, - state.repo_store.acquire(&repo.owner_did, &repo.name), - ) - .await - { - Ok(Ok(p)) => p, - Ok(Err(_)) => continue, - Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs walk; skipping repo"); - continue; + for sha256_hex in &oids { + // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). + // Resolve a PROVENANCED pin by trying each source repo (bounded to + // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no + // scan fan-out. A shared object first pinned from a private/quarantined repo + // still serves from a later PUBLIC source. Deterministic (ORDER BY on the + // union), so no ordering can turn an authorized copy into a 404. + let sources = state + .db + .pin_sources_for_oid(sha256_hex) + .await + .map_err(AppError::Internal)?; + if sources.is_empty() { + // F3 (#173, INV-10/INV-15): the legacy NULL-provenance scan builds an + // O(repos) preload (repos + rules + quarantine) BEFORE gate_and_serve's + // per-probe brake can bite, so a throttled source could still force O(repos) + // DB work on every replay. Peek the per-IP limiter WITHOUT consuming a token: + // an already-throttled source is shed here, before the preload runs. The + // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is + // load-bearing for the across-request bound), so this adds no double-charge — + // a non-consuming peek plus the existing per-probe charge, never two charges. + if let Some(key) = + crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) + { + if state.ipfs_rate_limiter.is_throttled(&key).await { + throttled = true; + continue; + } } - }; - - // Check whether the object exists in this repo before any expensive - // reachability walk. This prevents random-CID spray from triggering - // full-history git walks on repos that don't carry the object. - let obj_type = match store::object_type(&repo_path, &sha256_hex) { - Ok(Some(t)) => t, - Ok(None) => continue, - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); - continue; + // Legacy pin (recorded before provenance existed): fall back to the repo + // scan, gating each repo through the SAME gate. Load the scan context + // once, lazily. + if scan_ctx.is_none() { + #[cfg(test)] + bump_preload_queries(); + let repos = state + .db + .list_all_repos() + .await + .map_err(AppError::Internal)?; + let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); + let rules_by_repo = state + .db + .list_visibility_rules_for_repos(&repo_ids) + .await + .map_err(AppError::Internal)?; + let quarantined: HashSet = state + .db + .list_quarantined_repos() + .await + .map_err(AppError::Internal)? + .into_iter() + .map(|r| r.id) + .collect(); + scan_ctx = Some((repos, rules_by_repo, quarantined)); } - }; - - // Per-blob gating only applies when a path-scoped rule exists (KTD4). - // Without any path-scoped rule, the "/" gate above is the whole story. - // Trees/commits are always served under path-scoped rules (KTD3). - let path_scoped = has_path_scoped_rule(rules); - if path_scoped && obj_type == "blob" { - if !allowed_memo.contains_key(&repo.id) { - let rp = repo_path.clone(); - let r = rules.to_vec(); - let is_public = repo.is_public; - let owner = repo.owner_did.clone(); - let caller_for_walk = caller_owned.clone(); - let git_bin = state.git_bin.clone(); - let walk_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); - // Full-history walk shells out to git — keep it off the async runtime, - // bounded and reaped like the served-git ops (#174). - let walk = tokio::task::spawn_blocking(move || { - allowed_blob_set_for_caller_bounded( - &rp, - &git_bin, - walk_timeout, - &r, - is_public, - &owner, - caller_for_walk.as_deref(), - ) - }) - .await; - // Fail closed on EITHER a task panic (JoinError) or a walk error: - // we cannot prove the caller may read here, so skip this repo and - // let a public copy (if any) serve. Never serve on an unproven gate. - let set = match walk { - Ok(Ok(set)) => set, - Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed; skipping repo"); - continue; - } - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked; skipping repo"); + let (repos, rules_by_repo, quarantined) = scan_ctx.as_ref().unwrap(); + for repo in repos { + let rules = rules_by_repo + .get(&repo.id) + .map(Vec::as_slice) + .unwrap_or(&[]); + let is_quar = quarantined.contains(&repo.id); + match gate_and_serve( + &state, repo, rules, is_quar, sha256_hex, &rctx, &mut walk, true, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + // A throttled walk-requiring candidate is skipped, not fatal: + // keep scanning for a later walk-free copy (#173 review, F-C). + GateOutcome::Throttled => throttled = true, + GateOutcome::Skip => {} + } + } + } else { + for repo_id in &sources { + let repo = match state + .db + .get_repo_by_id(repo_id) + .await + .map_err(AppError::Internal)? + { + Some(r) => r, + // A source repo is gone: skip this source. Do NOT fall back to the + // scan (that would reopen the fan-out); a later source or oid + // candidate may still resolve. + None => continue, + }; + let quarantined = state + .db + .is_repo_quarantined(repo_id) + .await + .map_err(AppError::Internal)?; + let rules_map = state + .db + .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) + .await + .map_err(AppError::Internal)?; + let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); + match gate_and_serve( + &state, + &repo, + rules, + quarantined, + sha256_hex, + &rctx, + &mut walk, + false, // provenance path: bounded source set, no scan fan-out + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + GateOutcome::Throttled => { + throttled = true; continue; } - }; - allowed_memo.insert(repo.id.clone(), set); + GateOutcome::Skip => continue, + } } - let in_allowed = allowed_memo - .get(&repo.id) - .is_some_and(|set| set.contains(&sha256_hex)); - if !in_allowed { - continue; + } + } + + // Nothing served — three distinct tails, in precedence order: + // 1. The scan was cut short by a cap (legacy probe ceiling or walk ceiling), so + // the object was NOT proven absent/unreadable everywhere → 503, retryable, and + // explicitly NOT a definitive not-found (#173, F2). This outranks the throttle: + // an incomplete search must not masquerade as a clean rate-limit outcome, and + // it carries only the caller-supplied CID (no object/OID/metadata leak). + // 2. A walk-requiring candidate was skipped for a spent IP quota while the scan + // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). + // 3. A full scan under the caps found nothing readable → opaque 404, uniform with + // a genuine not-found and a visibility denial. + if walk.truncated { + return Err(AppError::SearchIncomplete(format!( + "CID {cid_str} search incomplete — retry" + ))); + } + if throttled { + return Err(AppError::TooManyRequests( + "ipfs retrieval rate limit exceeded — try again later".into(), + )); + } + Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))) +} + +/// Outcome of gating one repo for one candidate oid. +enum GateOutcome { + /// The object passed the gate; serve this response. + Served(Response), + /// This repo does not serve the object (absent, denied, quarantined, walk-capped, + /// or a walk error) — try the next candidate. + Skip, + /// A walk-requiring candidate hit the per-IP walk quota; skip it but let the caller + /// record the throttle so a later walk-free copy can still serve. + Throttled, +} + +/// Outcome of the bounded, off-worker object read for one gated candidate (F6, #173). +enum ServedRead { + /// Verified: the object's bytes hash to the requested CID; serve them. + Ok(Vec), + /// The bytes do not hash to the requested CID (a legacy provider-CID row); withhold. + Mismatch(String), + /// The object exceeds the served-object size cap; withhold rather than buffer it. + TooLarge(u64), + /// The object is genuinely absent (git reported it does not exist); try the next + /// candidate. Distinct from `ReadErr` so an infra failure is never silently rendered + /// as a clean not-found. + Gone, + /// A git subprocess failed to run (spawn/IO error, not a "no such object"). Logged at + /// the handler layer and skipped — an infra failure must surface as an error, not a + /// silent 404 for an authorized caller (INV-25 spirit, #173). + ReadErr(String), +} + +/// Immutable per-request context threaded into the gate. +struct ResolveCtx<'a> { + caller: Option<&'a str>, + caller_owned: &'a Option, + headers: &'a HeaderMap, + peer: Option, + cid_str: &'a str, + /// Canonical base32 form of the requested CID (`cid.to_string()`), used by the + /// serve-side integrity check to confirm the served bytes actually hash to the + /// requested content address (F2, #173). Compared against the recomputed CID, NOT + /// `cid_str` — a client may send an equivalent non-canonical multibase spelling. + canonical_cid: &'a str, +} + +/// Per-request walk budget + memos, shared across the provenance path and the legacy +/// scan so the fan-out ceiling and per-repo memoization span the whole request. +struct WalkState { + walks: u32, + /// Count of legacy (NULL-provenance) repos actually probed this request, so the + /// scan can stop at `ipfs_max_legacy_probes` instead of fanning out to O(repos) + /// `acquire` + `cat-file` (#173, F1, INV-10). Only the legacy path bumps it. + probes: u32, + /// Set when any cap (the legacy probe ceiling or the walk ceiling) cut the scan + /// short. A truncated scan did NOT prove the object absent/unreadable everywhere, + /// so the tail returns a retryable 503 rather than a definitive 404 (#173, F2). + truncated: bool, + allowed_blob_memo: HashMap>, + allowed_tree_memo: HashMap>, + reachable_ct_memo: HashMap>, +} + +/// Gate ONE repo for ONE candidate oid and, if the caller may read it, serve it. The +/// SINGLE gate both the provenance path and the legacy scan call, so INV-11 (quarantine +/// hard-drops before visibility), INV-2 (the repo's own "/" gate), and the per-object +/// reachability walk hold identically on both paths (KTD5). Never re-resolves via +/// `authorize_repo_read`, whose fuzzy match could authorize a different physical row +/// than the one read (KTD2a). +// The per-repo gate genuinely needs the row, its rules, its quarantine bit, the oid, +// the request context, the shared walk budget, and whether this is the fan-out-bounded +// legacy scan; bundling them buys nothing over the existing threshold. +#[allow(clippy::too_many_arguments)] +async fn gate_and_serve( + state: &AppState, + repo: &crate::db::RepoRecord, + rules: &[crate::db::VisibilityRule], + quarantined: bool, + sha256_hex: &str, + ctx: &ResolveCtx<'_>, + walk: &mut WalkState, + // True only for the legacy NULL-provenance scan, which iterates every repo. The + // provenance path targets one repo (no fan-out) and passes false, so it does not + // consume the per-request probe budget below. + legacy_scan: bool, +) -> GateOutcome { + // Quarantine gate (INV-11): a quarantined mirror is hidden from every reader, owner + // included, BEFORE any visibility check — so an owner whom visibility would Allow + // still 404s. + if quarantined { + return GateOutcome::Skip; + } + // Repo-level "/" read gate against THIS row's own rules (INV-2, KTD2a). + if visibility_check(rules, repo.is_public, &repo.owner_did, ctx.caller, "/") == Decision::Deny { + return GateOutcome::Skip; + } + // Legacy-scan fan-out control (#173, F1/F3, INV-10). The legacy path probes every + // root-visible repo, and the probe below (`acquire` — a possible cold-cache + // Tigris fetch — plus a `git cat-file -t` subprocess) is the expensive part. + // Cap it per request BEFORE that work runs, so an anonymous caller wielding a + // CID from the public pins index cannot amplify one request into O(repos) + // subprocesses. A legacy scan is inherently fan-out (unlike a targeted + // provenance fetch), so EVERY legacy probe is charged to the source IP from the + // first one, not just the ones past a free budget. A per-request-only budget + // reset each request, leaving a NULL-provenance CID open to unbounded ACROSS- + // request amplification: N requests spending N x budget cold `acquire` calls + // against Tigris with zero limiter contact (#173, F3, jatmn). Charging the first + // probe makes those requests accumulate against the per-IP `ipfs_rate_limiter`, + // closing that path. The per-request cap below stays as the second bound (a + // single request's ceiling). A spent quota is the same non-fatal Throttled as the + // walk brake: keep scanning for a walk-free copy, and only a wholly-unservable + // request becomes the 429. No resolvable key (a test oneshot with no peer/header) + // skips the brake, as the walk brake does. The provenance path targets one repo + // (no fan-out) and is exempt (`legacy_scan == false`). + if legacy_scan { + if walk.probes >= state.ipfs_max_legacy_probes { + // Budget spent: stop probing and mark the scan truncated so the tail + // reports an incomplete search (503), not a false 404 (#173, F2). + walk.truncated = true; + return GateOutcome::Skip; + } + if let Some(key) = + crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) + { + if !state.ipfs_rate_limiter.check(&key).await { + return GateOutcome::Throttled; } } + walk.probes += 1; + } + let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { + Ok(p) => p, + Err(_) => return GateOutcome::Skip, + }; - // Now that we've passed the gate, read the content. - let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { - Ok(c) => c, + // Existence probe before any walk (random-CID spray must not trigger a walk on a + // repo that lacks the object). Off the async runtime — it shells out to + // `git cat-file -t`. Fail closed (skip) on a task panic. + let obj_type = { + let rp = repo_path.clone(); + let sha = sha256_hex.to_string(); + match tokio::task::spawn_blocking(move || store::object_type(&rp, &sha)).await { + Ok(Ok(Some(t))) => t, + Ok(Ok(None)) => return GateOutcome::Skip, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + return GateOutcome::Skip; + } Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); - continue; + tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); + return GateOutcome::Skip; } - }; + } + }; - // 3. Return the content with IPFS-style headers - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_static("content-type"), - HeaderValue::from_static("application/octet-stream"), - ); - headers.insert( - HeaderName::from_static("x-content-cid"), - HeaderValue::from_str(&cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), - ); - headers.insert( - HeaderName::from_static("x-git-hash"), - HeaderValue::from_str(&sha256_hex) - .unwrap_or_else(|_| HeaderValue::from_static("invalid")), - ); + // Per-object gating applies only under a path-scoped rule (KTD4); otherwise the "/" + // gate above is the whole story. A blob is gated on the caller's allowed-blob set, a + // tree on the allowed-tree set (#135), a commit/tag on the repo's reachable + // commit/tag set (#173) — each a full-history walk sharing the per-request cap and + // per-walk IP quota. + let path_scoped = has_path_scoped_rule(rules); + let gated = path_scoped && matches!(obj_type.as_str(), "blob" | "tree" | "commit" | "tag"); + if gated { + let already = match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.contains_key(&repo.id), + "tree" => walk.allowed_tree_memo.contains_key(&repo.id), + "commit" | "tag" => walk.reachable_ct_memo.contains_key(&repo.id), + other => unreachable!("gated admits only blob/tree/commit/tag, got {other}"), + }; + if !already { + // Per-request fan-out ceiling (INV-10): once this many walks have run, skip + // THIS walk-requiring candidate and keep scanning (a later walk-free copy + // must still serve). `walks` is bumped only inside this block, so walk-free + // candidates never consume budget. + if walk.walks >= state.ipfs_max_history_walks { + // The walk ceiling truncated the search: a later repo (possibly one that + // authorizes this caller) is left unwalked, so absence is unproven — + // record it so the tail returns 503, not a false 404 (#173, F2). + walk.truncated = true; + return GateOutcome::Skip; + } + // Brake each spawned walk on the source IP (#173, F3, INV-15), BEFORE + // spending walk budget: a throttled candidate neither walks nor consumes + // budget and must not end the request — skip it and keep scanning + // (#173 review, F-C). No key (a test oneshot with no peer/header) skips the + // brake, as the other IP brakes do. On the LEGACY path the probe brake + // above already charged THIS candidate to the source (#173, F3, jatmn), so + // the walk brake must not double-charge it: only the provenance path + // (`legacy_scan == false`, no probe toll) charges here. + if !legacy_scan { + if let Some(key) = + crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) + { + if !state.ipfs_rate_limiter.check(&key).await { + return GateOutcome::Throttled; + } + } + } + walk.walks += 1; - return Ok((StatusCode::OK, headers, content).into_response()); + let rp = repo_path.clone(); + let r = rules.to_vec(); + let is_public = repo.is_public; + let owner = repo.owner_did.clone(); + let caller_for_walk = ctx.caller_owned.clone(); + let kind = obj_type.clone(); + // Every walk is the DURATION-BOUNDED twin (`run_bounded_git` teardown under + // `git_service_timeout_secs`): the handler holds its /ipfs walk permit + // across this spawn_blocking, and a held permit is only safe if no walk + // child can outlive the deadline (#174 F5). + let git_bin = state.git_bin.clone(); + let walk_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let result = tokio::task::spawn_blocking(move || match kind.as_str() { + "blob" => allowed_blob_set_for_caller_bounded( + &rp, + &git_bin, + walk_timeout, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ), + "tree" => allowed_tree_set_for_caller_bounded( + &rp, + &git_bin, + walk_timeout, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ), + "commit" | "tag" => reachable_commit_tag_oids_bounded(&rp, &git_bin, walk_timeout), + other => unreachable!("gated admits only blob/tree/commit/tag, got {other}"), + }) + .await; + // Fail closed on a walk error or task panic: we cannot prove readability, so + // skip rather than serve on an unproven gate. + let set = match result { + Ok(Ok(set)) => set, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk failed; skipping repo"); + return GateOutcome::Skip; + } + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk task panicked; skipping repo"); + return GateOutcome::Skip; + } + }; + match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.insert(repo.id.clone(), set), + "tree" => walk.allowed_tree_memo.insert(repo.id.clone(), set), + _ => walk.reachable_ct_memo.insert(repo.id.clone(), set), + }; + } + let in_set = match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.get(&repo.id), + "tree" => walk.allowed_tree_memo.get(&repo.id), + _ => walk.reachable_ct_memo.get(&repo.id), + } + .is_some_and(|set| set.contains(sha256_hex)); + if !in_set { + return GateOutcome::Skip; + } } - // Not found in any repo - Err(AppError::RepoNotFound(format!( - "no git object found for CID {cid_str}" - ))) + // Passed the gate — bound the object, read it OFF the async worker, and verify the + // content address, all before any byte egresses. F6 (#173): read_object_content runs a + // blocking `git cat-file` and buffers the whole object; called directly on the Axum + // worker (the type-probe and walk are already off-worker) it blocks a runtime thread, + // and unbounded it can exhaust memory for a large public blob (enumerable from the pins + // index). Precheck the SIZE and run size + read + verify inside spawn_blocking. A + // content-addressed serve cannot verify a STREAMED body (the digest is known only after + // the last byte, by which point the prefix has already egressed), so we never stream: + // buffer-verify-then-serve up to the cap and withhold anything larger. F2's integrity + // check moves in here too, so no unverified bytes are ever assembled into a response. + let max_bytes = state.ipfs_max_served_object_bytes; + let read_repo = repo_path.clone(); + let read_sha = sha256_hex.to_string(); + let read_type = obj_type.clone(); + let want_cid = ctx.canonical_cid.to_string(); + let read = tokio::task::spawn_blocking(move || -> ServedRead { + match store::object_size(&read_repo, &read_sha) { + Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(Some(_)) => {} + // git ran and reported no such object (or an unparseable size): genuine + // not-found for this candidate. + Ok(None) => return ServedRead::Gone, + // git itself failed to run: an infra failure, not a not-found. + Err(e) => return ServedRead::ReadErr(e.to_string()), + } + let content = match store::read_object_content(&read_repo, &read_sha, &read_type) { + Ok(c) => c, + Err(e) => return ServedRead::ReadErr(e.to_string()), + }; + let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); + if served != want_cid { + return ServedRead::Mismatch(served); + } + ServedRead::Ok(content) + }) + .await; + let content = match read { + Ok(ServedRead::Ok(c)) => c, + Ok(ServedRead::TooLarge(size)) => { + tracing::warn!( + repo = %repo.name, size, max = max_bytes, + "withholding object: exceeds the served-object size cap (F6)" + ); + #[cfg(test)] + note_oversize_reject(); + return GateOutcome::Skip; + } + Ok(ServedRead::Mismatch(served)) => { + tracing::warn!( + repo = %repo.name, requested = %ctx.canonical_cid, served = %served, + "withholding object: served bytes do not hash to the requested CID (legacy provider-CID row?)" + ); + return GateOutcome::Skip; + } + Ok(ServedRead::Gone) => return GateOutcome::Skip, + Ok(ServedRead::ReadErr(e)) => { + // Infra failure (git spawn/IO), NOT a not-found: mark the search truncated so + // a wholly-unserved request tails to a retryable 503, never a definitive 404 + // for an authorized caller (INV-25 spirit — logging alone is not surfacing). + tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); + walk.truncated = true; + return GateOutcome::Skip; + } + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); + walk.truncated = true; + return GateOutcome::Skip; + } + }; + let mut resp_headers = HeaderMap::new(); + resp_headers.insert( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/octet-stream"), + ); + resp_headers.insert( + HeaderName::from_static("x-content-cid"), + HeaderValue::from_str(ctx.cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + resp_headers.insert( + HeaderName::from_static("x-git-hash"), + HeaderValue::from_str(sha256_hex).unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + GateOutcome::Served((StatusCode::OK, resp_headers, content).into_response()) } /// GET /api/v1/ipfs/pins @@ -303,6 +741,58 @@ pub async fn list_pins(State(state): State) -> Result = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_preload_queries() { + PRELOAD_QUERIES.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn preload_queries() -> usize { + PRELOAD_QUERIES.with(|c| c.get()) +} + +#[cfg(test)] +fn bump_preload_queries() { + PRELOAD_QUERIES.with(|c| c.set(c.get() + 1)); +} + +// Test-only INV-10 cost counter (F6, U6/U7): how many times the serve path withheld an +// object because it exceeded `ipfs_max_served_object_bytes`. The bounded read must reject +// an oversized object rather than buffer it on the worker; the counter is the both-ways +// guard (a removed size precheck stops incrementing it and serves the oversized object). +// Set from the match arm after `spawn_blocking` resolves, i.e. on the test's runtime +// thread, so the thread-local is read on the same thread it is written. +#[cfg(test)] +thread_local! { + static OVERSIZE_REJECTS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_oversize_rejects() { + OVERSIZE_REJECTS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn oversize_rejects() -> usize { + OVERSIZE_REJECTS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_oversize_reject() { + OVERSIZE_REJECTS.with(|c| c.set(c.get() + 1)); +} + #[cfg(test)] mod tests { //! #174 P1-3 (U3): the public `GET /ipfs/{cid}` walk carries bounded CONCURRENCY @@ -310,7 +800,9 @@ mod tests { //! walk, plus a per-IP route rate limit. These are handler-layer proofs: mount the //! real handler/router, drive one request, assert the exact 503 shed, then name the //! mutation that turns each RED. The per-source key resolves an IP only (`Some(ip)` - //! vs `None`), never a DID — both arms are driven so neither is vacuous. + //! vs `None`), never a DID — both arms are driven so neither is vacuous. The + //! CID-resolution / visibility-gate behavior of the handler itself is covered by the + //! `#[sqlx::test]` suite in `test_support.rs`. use axum::body::Body; use axum::extract::ConnectInfo; @@ -500,29 +992,33 @@ mod tests { drop(held); } - /// Retain-through-blocking (R3, the load-bearing async property): the walk - /// admission is held until the `spawn_blocking` walk actually RETURNS, not when a - /// tokio timeout fires. With the global pool at size 1, drive a request until its - /// walk (a fake git that hangs on `rev-list`) is in flight; the slot must stay held - /// (`available_permits() == 0`) and a replacement from a DIFFERENT source must shed - /// 503 for as long as the blocking walk runs — even though the request future is - /// only `.await`ing the blocking join. When the blocking walk ends the permit frees - /// and a replacement is admitted. The permit lives INSIDE the handler across the - /// blocking `.await`; move it out (drop before the walk) and the replacement would - /// be admitted while the walk still burns a blocking thread (the bug this guards). + /// Retain-through-blocking (#174 F5, the load-bearing async property, on the + /// NEWLY-BOUNDED TREE path): the walk admission is held until the `spawn_blocking` + /// walk actually RETURNS, not when a tokio timeout fires. The requested CID + /// resolves to a TREE object under a path-scoped rule, so the gate runs + /// `allowed_tree_set_for_caller_bounded` — the walk this integration converts to + /// `run_bounded_git` — rather than the blob walk #174 already proved. With the + /// global pool at size 1, drive a request until its walk (a fake git that hangs on + /// `rev-list`) is in flight; the slot must stay held (`available_permits() == 0`) + /// and a replacement from a DIFFERENT source must shed 503 for as long as the + /// blocking walk runs — even though the request future is only `.await`ing the + /// blocking join. When the blocking walk ends the permit frees and a replacement + /// is admitted. The permit lives INSIDE the handler across the blocking `.await`; + /// move it out (drop before the walk) and the replacement would be admitted while + /// the walk still burns a blocking thread (the bug this guards). #[cfg(unix)] #[sqlx::test] - async fn get_by_cid_walk_permit_held_through_blocking_walk(pool: sqlx::PgPool) { + async fn get_by_cid_walk_permit_held_through_bounded_tree_walk(pool: sqlx::PgPool) { use std::process::Command; let tmp = tempfile::TempDir::new().unwrap(); let revlist_pid = tmp.path().join("revlist.pid"); - // Fake git for the /ipfs WALK only (object_type/read_object_content use the real - // `git`, so the object must genuinely exist below). Empty refs (so - // assert_all_refs_are_commits returns Ok without the peel), `rev-parse` resolves, - // and `rev-list` records its pid then sleeps ~6s so the walk BLOCKS - // deterministically. The sleep bounds the walk so a broken fix cannot wedge the - // suite. + // Fake git for the /ipfs TREE walk only (object_type/read_object_content use + // the real `git`, so the tree must genuinely exist below). `rev-parse` + // resolves (so the lenient enumeration appends HEAD) and `rev-list` records + // its pid then sleeps ~6s so the walk BLOCKS deterministically inside + // `run_bounded_git`. The sleep bounds the walk so a broken fix cannot wedge + // the suite. let body = format!( "#!/bin/sh\n\ case \"$1\" in\n\ @@ -554,18 +1050,18 @@ mod tests { state.git_bin = git_path.to_str().unwrap().to_string(); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - let owner = "z6ipfs1"; - let name = "ip1"; + let owner = "z6ipfstree"; + let name = "iptree"; state .db .upsert_mirror_repo(owner, name, "/unused", None, false) .await .unwrap(); let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); - // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 bare - // repo there with a committed blob under `src/`, so real `git cat-file -t ` classifies it as a blob (the CID digest IS the sha256 object id in - // object-format=sha256) and the handler reaches the path-scoped walk branch. + // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 + // bare repo there with a committed `src/` directory, so real + // `git cat-file -t ` classifies the requested object as a TREE and the + // handler routes into the tree-walk arm of the gate. let bare = state .repo_store .acquire(&rec.owner_did, &rec.name) @@ -587,7 +1083,11 @@ mod tests { }; let work = tmp.path().join("work"); std::fs::create_dir_all(work.join("src")).unwrap(); - std::fs::write(work.join("src/secret.txt"), b"ipfs walk retain proof\n").unwrap(); + std::fs::write( + work.join("src/secret.txt"), + b"ipfs tree walk retain proof\n", + ) + .unwrap(); run( &["init", "-q", "--object-format=sha256", "-b", "main"], &work, @@ -606,38 +1106,46 @@ mod tests { ], tmp.path(), ); - // The blob's SHA-256 object id (= the CID's digest); build the CID from it. - let oid = { + // The `src` directory's TREE oid — the object the request asks for. + let tree_oid = { let out = Command::new("git") - .args(["rev-parse", "HEAD:src/secret.txt"]) + .args(["rev-parse", "HEAD:src"]) .current_dir(&work) .output() .expect("git rev-parse runs"); assert!(out.status.success(), "rev-parse failed"); String::from_utf8_lossy(&out.stdout).trim().to_string() }; - let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); - let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) - .as_str() - .to_string(); - // Precondition: real git classifies the object as a blob (so the handler reaches - // the walk branch, not an early `continue`). + // Precondition: real git classifies the object as a TREE (so the handler + // reaches the tree-walk arm, not the blob arm or an early `continue`). assert_eq!( - crate::git::store::object_type(&bare, &oid) + crate::git::store::object_type(&bare, &tree_oid) .unwrap() .as_deref(), - Some("blob"), - "the seeded sha256 blob must exist so the handler reaches the walk" + Some("tree"), + "the seeded sha256 tree must exist so the handler reaches the tree walk" ); - // A path-scoped rule so has_path_scoped_rule() is true (the walk branch) without - // denying the "/" gate on the public repo. + // Pin the tree's content CID WITH provenance so the resolver targets this one + // repo (no legacy scan). A real pin CID digests the raw object content, not + // the git oid, so build it exactly as the pin path does (#173). + let (_ty, raw) = crate::git::store::read_object(&bare, &tree_oid) + .unwrap() + .expect("tree object readable"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + state + .db + .record_pinned_cid(&tree_oid, &cid, Some(&rec.id)) + .await + .unwrap(); + // A path-scoped rule so has_path_scoped_rule() is true (the tree-gate branch) + // without denying the "/" gate on the public repo. state .db .set_visibility_rule( &rec.id, - "src/**", + "/src/**", crate::db::VisibilityMode::B, - &["did:key:z6MkU3IpfsReaderAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &["did:key:z6MkF5IpfsTreeReaderAAAAAAAAAAAAAAAAAAAA".to_string()], &rec.owner_did, ) .await @@ -663,10 +1171,10 @@ mod tests { let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); - // Drive until the fake git's rev-list records its pid — the walk is now in the - // blocking pool and the request future is `.await`ing its join, holding the walk - // permit. Stop polling the instant the future completes (re-polling a completed - // oneshot panics). + // Drive until the fake git's rev-list records its pid — the TREE walk is now in + // the blocking pool and the request future is `.await`ing its join, holding the + // walk permit. Stop polling the instant the future completes (re-polling a + // completed oneshot panics). let mut walk_pid: Option = None; let mut early = None; for _ in 0..500 { @@ -696,28 +1204,28 @@ mod tests { } let _cleanup = ReapOnDrop(pid); - // Load-bearing: while the blocking walk runs, the slot is HELD and a replacement - // from a DIFFERENT source sheds 503 — proving the permit is retained across the - // spawn_blocking join, not freed by a tokio timeout. + // Load-bearing: while the blocking TREE walk runs, the slot is HELD and a + // replacement from a DIFFERENT source sheds 503 — proving the permit is + // retained across the spawn_blocking join, not freed by a tokio timeout. assert_eq!( sem.available_permits(), 0, - "the walk slot must be held while the spawn_blocking walk runs" + "the walk slot must be held while the spawn_blocking tree walk runs" ); let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "a replacement must shed 503 while the prior request's blocking walk still runs" + "a replacement must shed 503 while the prior request's blocking tree walk still runs" ); // Drop the in-flight request; the detached blocking walk keeps running (a - // spawn_blocking cannot be cancelled), but on the fix the permit is a handler - // local, so dropping the future releases it once the blocking join is abandoned. - // Either way, kill the sleeping child so the slot frees promptly and poll for - // recovery — the point already proven above is that the slot stayed held for the - // duration of the blocking work. + // spawn_blocking cannot be cancelled), but the permit is a handler local, so + // dropping the future releases it once the blocking join is abandoned. Either + // way, kill the sleeping child so the slot frees promptly and poll for + // recovery — the point already proven above is that the slot stayed held for + // the duration of the blocking work. drop(fut); unsafe { libc::kill(pid, libc::SIGKILL); @@ -736,155 +1244,6 @@ mod tests { ); } - /// Loop bound (cap N): one `/ipfs/{cid}` request against a CID present in many repos - /// must not serialize an unbounded number of full-history walks. With - /// `ipfs_max_repos_walked = 1` and TWO public, path-scoped repos both carrying the - /// blob at the requested CID, the handler walks only the FIRST candidate then stops - /// (the second is cut by the cap), so the fake git's `rev-list` (one per walk) runs - /// exactly once. MUTATION (RED): remove the `repos_walked >= cap` break and both - /// repos are walked (count 2). - #[cfg(unix)] - #[sqlx::test] - async fn get_by_cid_caps_repos_walked_per_request(pool: sqlx::PgPool) { - use std::process::Command; - - let tmp = tempfile::TempDir::new().unwrap(); - let walk_log = tmp.path().join("walks.log"); - // Fake git for the WALK: empty refs, `rev-parse` resolves, and each `rev-list` - // appends one line to a log (so the number of walks == the line count) and exits - // with EMPTY output (the allowed-set is empty, so every repo path-gates to a - // `continue` and the request 404s after walking). object_type uses the REAL git, - // so the seeded blob below must genuinely exist. - let body = format!( - "#!/bin/sh\n\ - case \"$1\" in\n\ - for-each-ref) : ;;\n\ - rev-parse) echo deadbeef ;;\n\ - rev-list) echo walk >> \"{}\" ;;\n\ - *) : ;;\n\ - esac\n\ - exit 0\n", - walk_log.display() - ); - let git_path = tmp.path().join("fakegit"); - std::fs::write(&git_path, &body).unwrap(); - { - use std::os::unix::fs::PermissionsExt; - let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); - perm.set_mode(0o755); - std::fs::set_permissions(&git_path, perm).unwrap(); - } - - let mut state = crate::test_support::test_state(pool.clone()).await; - let repos_dir = tmp.path().join("repos"); - std::fs::create_dir_all(&repos_dir).unwrap(); - state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); - state.git_bin = git_path.to_str().unwrap().to_string(); - state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - // The bound under test: walk at most one candidate repo per request. - let mut cfg = (*state.config).clone(); - cfg.ipfs_max_repos_walked = 1; - state.config = Arc::new(cfg); - - // Seed TWO public repos, each with the SAME blob (same content -> same sha256 OID - // -> same CID) under a path-scoped rule, so both are walk candidates for one CID. - let run = |args: &[&str], cwd: &std::path::Path| { - let out = Command::new("git") - .args(args) - .current_dir(cwd) - .output() - .expect("git runs"); - assert!( - out.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - }; - let mut oid = String::new(); - for (i, name) in ["ipa", "ipb"].iter().enumerate() { - let owner = "z6ipfsN"; - state - .db - .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) - .await - .unwrap(); - let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); - let bare = state - .repo_store - .acquire(&rec.owner_did, &rec.name) - .await - .unwrap(); - let _ = std::fs::remove_dir_all(&bare); - std::fs::create_dir_all(&bare).unwrap(); - let work = tmp.path().join(format!("work{i}")); - std::fs::create_dir_all(work.join("src")).unwrap(); - // Identical content in both repos -> identical sha256 blob OID -> one CID. - std::fs::write(work.join("src/secret.txt"), b"loop bound proof\n").unwrap(); - run( - &["init", "-q", "--object-format=sha256", "-b", "main"], - &work, - ); - run(&["config", "user.email", "t@t"], &work); - run(&["config", "user.name", "t"], &work); - run(&["add", "src/secret.txt"], &work); - run(&["commit", "-q", "-m", "seed"], &work); - run( - &[ - "clone", - "--bare", - "-q", - work.to_str().unwrap(), - bare.to_str().unwrap(), - ], - tmp.path(), - ); - if oid.is_empty() { - let out = Command::new("git") - .args(["rev-parse", "HEAD:src/secret.txt"]) - .current_dir(&work) - .output() - .expect("git rev-parse runs"); - oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); - } - state - .db - .set_visibility_rule( - &rec.id, - "src/**", - crate::db::VisibilityMode::B, - &["did:key:z6MkU3IpfsReaderBBBBBBBBBBBBBBBBBBBBBBBB".to_string()], - &rec.owner_did, - ) - .await - .unwrap(); - } - let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); - let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) - .as_str() - .to_string(); - - let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); - let mut req = Request::builder() - .method(Method::GET) - .uri(format!("/ipfs/{cid}")) - .body(Body::empty()) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); - let resp = ipfs_router(state).oneshot(req).await.unwrap(); - // The empty allowed-set path-gates both repos to a `continue`, so a 404; the - // point is HOW MANY walks ran to get there. - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - let walks = std::fs::read_to_string(&walk_log) - .map(|s| s.lines().count()) - .unwrap_or(0); - assert_eq!( - walks, 1, - "with the per-request repo-walk cap at 1, only the first candidate repo is \ - walked (the second is cut by the cap), so exactly one walk runs; got {walks}" - ); - } - /// Route rate limit is WIRED (not a silent no-op): the production `build_router` /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does diff --git a/crates/gitlawb-node/src/api/mod.rs b/crates/gitlawb-node/src/api/mod.rs index 10d76daf..86594c03 100644 --- a/crates/gitlawb-node/src/api/mod.rs +++ b/crates/gitlawb-node/src/api/mod.rs @@ -200,9 +200,11 @@ mod authz_guard { (issues, "create_issue", "authorize_repo_read("), (bounties, "create_bounty", "authorize_repo_read("), (repos, "fork_repo", "authorize_repo_read("), - // get_by_cid gates each iterated repo row directly via visibility_check - // (KTD2a: it must NOT route through authorize_repo_read's fuzzy re-resolve). - (ipfs, "get_by_cid", "visibility_check("), + // get_by_cid resolves each candidate (provenance path + legacy scan) through + // the shared `gate_and_serve` (#173 round 2); the gate markers themselves are + // asserted below. This row proves the delegation is real — the gate is + // actually reached, not dead code. + (ipfs, "get_by_cid", "gate_and_serve("), // Bucket C — signer-self: the acting DID is matched/bound to auth.0 (tasks, "create_task", "did_matches("), (tasks, "claim_task", "did_matches("), @@ -236,6 +238,22 @@ mod authz_guard { "visibility::require_owner must use did_matches for DID-safe owner matching" ); + // The CID read surface (#173) enforces its gate inside the shared + // `gate_and_serve`, which BOTH the provenance path and the legacy scan call, so + // the markers must live there (the get_by_cid row above only proves delegation). + // The repo's own "/" visibility check (KTD2a — never authorize_repo_read's fuzzy + // re-resolve) and the quarantine hard-drop BEFORE visibility (INV-11) are both + // load-bearing: removing either re-opens a leak on the provenance path. + let gate_body = fn_body(ipfs, "gate_and_serve"); + assert!( + gate_body.contains("visibility_check("), + "gate_and_serve must gate the CID read surface via visibility_check (KTD2a)" + ); + assert!( + gate_body.contains("if quarantined"), + "gate_and_serve must hard-drop a quarantined repo before the visibility gate (INV-11)" + ); + for (src, func, marker) in rows { let body = fn_body(src, func); assert!( diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 9b05ece1..cb6ed541 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1473,6 +1473,7 @@ pub async fn git_receive_pack( &repo_path_clone, object_list_ipfs, &db_clone, + &repo_id, ) .await; if !pinned.is_empty() { @@ -1573,6 +1574,7 @@ pub async fn git_receive_pack( let pinata_upload_url = state.config.pinata_upload_url.clone(); let repo_path_clone = disk_path.clone(); let db_clone = state.db.clone(); + let repo_id = record.id.clone(); let http_client = Arc::clone(&state.http_client); let node_did_str = state.node_did.to_string(); let repo_slug = format!( @@ -1603,6 +1605,7 @@ pub async fn git_receive_pack( &repo_path_clone, object_list_pinata, &db_clone, + &repo_id, ) .await } else { diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 912108a0..c03eb2e5 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -516,6 +516,10 @@ mod tests { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), @@ -532,7 +536,6 @@ mod tests { git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), - ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 5324a4b9..6b0d394e 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -872,8 +872,63 @@ const MIGRATIONS: &[Migration] = &[ "CREATE UNIQUE INDEX IF NOT EXISTS idx_ref_certs_repo_ref ON ref_certificates(repo_id, ref_name)", ], }, + Migration { + version: 11, + name: "pinned_cids_cid_index", + stmts: &[ + // GET /ipfs/{cid} resolves an incoming CID -> git oid via pinned_cids.cid + // (#173); index it so the per-request lookup is not a table scan. This is + // a NEW versioned migration (not appended to the applied v1 bundle) so a + // node already past v1 actually gets the index. Non-unique on purpose: cid + // is a function of raw content, so a UNIQUE index could reject a legitimate + // record_pinned_cid insert, and colliding rows serve byte-identical content. + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_cid ON pinned_cids(cid)", + ], + }, + Migration { + version: 12, + name: "pinned_cids_repo_provenance", + stmts: &[ + // Record the repository a pin came from so GET /ipfs/{cid} resolves a + // provenanced pin straight to its ONE source repo instead of scanning every + // repo (#173, jatmn round 2 — bounds the anonymous fan-out and removes the + // updated_at-ordering false-404). NEW versioned migration (never appended to + // the applied v1 pinned_cids table) so a node past v1 gets the column. + // Nullable: pins recorded before this migration have no provenance and fall + // back to the legacy repo scan; new pins carry repo_id and resolve to one + // repo. Indexed for the resolver's oid -> repo_id lookup. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS repo_id TEXT", + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_repo_id ON pinned_cids(repo_id)", + ], + }, + Migration { + version: 13, + name: "pin_repo_sources", + stmts: &[ + // F1 (#173, jatmn round 8): a shared object (a blob/tree/commit common to + // forks and mirrors) can be pinned from more than one repo. `pinned_cids` + // keeps only the FIRST pinner's `repo_id`, so a shared object first pinned + // from a private/quarantined repo 404s by CID even when a later PUBLIC repo + // also pinned it. Record EVERY pin-path source so `GET /ipfs/{cid}` can try + // each. NEW versioned migration (never appended to an applied block, INV-7). + // Bounded per object at insert time (MAX_PIN_SOURCES) so an adversary pushing + // one object from N repos cannot make resolution O(repos) (R2, INV-10). + "CREATE TABLE IF NOT EXISTS pin_repo_sources ( + sha256_hex TEXT NOT NULL, + repo_id TEXT NOT NULL, + PRIMARY KEY (sha256_hex, repo_id) + )", + "CREATE INDEX IF NOT EXISTS idx_pin_repo_sources_sha ON pin_repo_sources(sha256_hex)", + ], + }, ]; +/// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). +/// Bounds both the resolver's per-OID source loop and the `pin_repo_sources` growth, +/// so an adversary re-pushing one object from many repos cannot make resolution +/// O(repos) (R2, INV-10). +pub const MAX_PIN_SOURCES: i64 = 16; + // ── Repos ───────────────────────────────────────────────────────────────────── pub(crate) fn normalize_owner_key(did: &str) -> &str { @@ -1046,6 +1101,22 @@ impl Db { Ok(row.map(row_to_repo)) } + /// Fetch a repo by its stable `id`. Used by the `/ipfs/{cid}` provenance path, + /// which resolves a pin straight to its ONE source repo (#173) instead of + /// scanning `list_all_repos`. `id` is exact, so unlike `get_repo`'s fuzzy + /// owner/name match there is no mirror-vs-canonical disambiguation. + pub async fn get_repo_by_id(&self, id: &str) -> Result> { + let row = sqlx::query( + "SELECT id, name, owner_did, description, is_public, default_branch, + created_at, updated_at, disk_path, forked_from, machine_id + FROM repos WHERE id = $1 LIMIT 1", + ) + .bind(id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(row_to_repo)) + } + #[allow(dead_code)] pub async fn list_repos(&self, owner_did: &str) -> Result> { let rows = sqlx::query( @@ -2166,20 +2237,148 @@ impl Db { Ok(row.get::("cnt") > 0) } - pub async fn record_pinned_cid(&self, sha256_hex: &str, cid: &str) -> Result<()> { + /// Every git oid a pinned CID maps to (`pinned_cids.cid` -> `sha256_hex`). + /// `GET /ipfs/{cid}` resolves the content-addressed CID a client sends back to + /// the object's git oid this way: a real pin CID digests the raw object + /// content, not the git oid, so the digest cannot be `git cat-file`d directly + /// (#173). The index is unique on the git oid but NON-unique on cid, so two + /// distinct oids can share one content-CID (a tree and a blob whose raw bytes + /// collide, or byte-identical content pinned under two oids). Returning every + /// candidate lets the handler try each rather than pick one arbitrarily and + /// false-404 when the chosen one is withheld or absent while another is + /// readable (#173). Empty when the CID was never pinned on this node. + pub async fn oids_for_cid(&self, cid: &str) -> Result> { + let rows = sqlx::query("SELECT sha256_hex FROM pinned_cids WHERE cid = $1") + .bind(cid) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("sha256_hex")) + .collect()) + } + + /// Record a pinned object's CID and the repository it was pinned from + /// (`repo_id`, #173). On conflict the `COALESCE` backfills a NULL provenance + /// from a known source while keeping first-pinner-owns: an existing non-NULL + /// `repo_id` is never rewritten by a later push of the same oid, but a legacy + /// pin (or a pin recorded before provenance existed) whose `repo_id` is NULL + /// gets it filled the next time the object is re-pinned with a known source. + /// `cid`/`pinned_at` are left untouched on conflict. `repo_id` is `None` only + /// for a legacy pin with no known source; those fall back to the resolver's scan. + pub async fn record_pinned_cid( + &self, + sha256_hex: &str, + cid: &str, + repo_id: Option<&str>, + ) -> Result<()> { sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) - VALUES ($1, $2, $3) - ON CONFLICT(sha256_hex) DO NOTHING", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT(sha256_hex) DO UPDATE SET + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) .bind(cid) .bind(Utc::now().to_rfc3339()) + .bind(repo_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// The repository a pinned object was recorded from (`pinned_cids.repo_id`), + /// or `None` for a legacy pin (recorded before provenance existed) or an + /// unpinned oid. `GET /ipfs/{cid}` uses this to gate+serve the ONE source + /// repo instead of scanning every repo (#173). + pub async fn provenance_for_oid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT repo_id FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.and_then(|r| r.get::, _>("repo_id"))) + } + + /// Backfill the source repo on an already-pinned object whose provenance is + /// NULL (a legacy pin recorded before provenance existed, #173, jatmn). The + /// `AND repo_id IS NULL` guard keeps first-pinner-owns: an existing non-NULL + /// provenance is left untouched. Touches only `repo_id` and never re-pins the + /// object's bytes, so it is safe to call on the already-pinned skip path. + pub async fn backfill_pin_provenance(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + sqlx::query( + "UPDATE pinned_cids SET repo_id = $2 WHERE sha256_hex = $1 AND repo_id IS NULL", + ) + .bind(sha256_hex) + .bind(repo_id) .execute(&self.pool) .await?; Ok(()) } + /// Record a repository as a source for a pinned object (F1, #173 jatmn round 8), + /// bounded to about `MAX_PIN_SOURCES` distinct repos per object. The count guard + /// lives inside the INSERT (a single statement), which suppresses a re-push of the + /// SAME `(oid, repo)` via `ON CONFLICT DO NOTHING`. It does NOT hard-serialize + /// concurrent inserts of DIFFERENT repos for the same object: under Postgres READ + /// COMMITTED each concurrent writer's count subquery reads a snapshot that omits the + /// others' uncommitted rows, so N concurrent pushers can each see `count < cap` and + /// overshoot by up to N-1 rows. The overshoot is a small constant (bounded by + /// concurrent-pusher count, never O(repos)), and the RESOLVER read side + /// (`pin_sources_for_oid`) caps the ADDITIONAL sources at `MAX_PIN_SOURCES` (always + /// keeping the first-pinner), so the INV-10 bound on serve-time work holds at + /// `O(MAX_PIN_SOURCES + 1)` regardless of a table overshoot. + pub async fn record_pin_source(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Every source repository recorded for a pinned object (F1, #173 jatmn round 8): + /// the union of the first-pinner `pinned_cids.repo_id` and the `pin_repo_sources` + /// rows, deduped and ordered for a deterministic resolver walk. + /// + /// The first-pinner (a single row by `pinned_cids`' PK on `sha256_hex`) is ALWAYS + /// included; the `LIMIT MAX_PIN_SOURCES` caps only the ADDITIONAL `pin_repo_sources` + /// rows. This keeps the resolver's per-source work a bounded `O(MAX_PIN_SOURCES + 1)` + /// ceiling (INV-10) while never letting the cap evict the original source. A prior + /// version applied the `LIMIT` to the whole UNION with a lexicographic `ORDER BY`, + /// which let an attacker 404 a legacy public CID (first-pinner in `pinned_cids` but + /// not yet in `pin_repo_sources`) by pushing the same object from `MAX_PIN_SOURCES` + /// repos whose grindable ids sort before it, evicting the public source from the + /// window. Empty for a legacy pin with no known source (it falls back to the repo + /// scan) or an unpinned oid. + pub async fn pin_sources_for_oid(&self, sha256_hex: &str) -> Result> { + let rows = sqlx::query( + "SELECT repo_id FROM pinned_cids + WHERE sha256_hex = $1 AND repo_id IS NOT NULL + UNION + SELECT repo_id FROM ( + SELECT repo_id FROM pin_repo_sources + WHERE sha256_hex = $1 + ORDER BY repo_id + LIMIT $2 + ) capped + ORDER BY repo_id", + ) + .bind(sha256_hex) + .bind(MAX_PIN_SOURCES) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("repo_id")) + .collect()) + } + pub async fn record_encrypted_blob( &self, repo_id: &str, @@ -2279,18 +2478,34 @@ impl Db { } /// Record the Pinata CID for a git object. - /// Inserts the row if it doesn't exist (objects pinned directly to Pinata - /// without a prior local IPFS pin get cid = pinata_cid). - pub async fn record_pinata_cid(&self, sha256_hex: &str, pinata_cid: &str) -> Result<()> { + /// + /// `raw_cid` is the locally-computed raw-content CID (`Cid::from_git_object_bytes`, + /// CIDv1/raw/sha2-256), the resolver key `GET /ipfs/{cid}` looks up; `pinata_cid` + /// is the provider CID Pinata returned (a dag-pb/UnixFS CID for gateway retrieval). + /// Inserts the row if it doesn't exist (an object pinned directly to Pinata with + /// no prior local IPFS pin gets `cid = raw_cid`, never the provider CID — a dag-pb + /// provider CID must never become an alias that serves raw bytes that do not hash + /// to it, #173). On conflict `cid` is left untouched: a prior local pin already + /// stored the correct raw CID, and the COALESCE backfills a NULL provenance from a + /// known source while keeping first-pinner-owns. + pub async fn record_pinata_cid( + &self, + sha256_hex: &str, + raw_cid: &str, + pinata_cid: &str, + repo_id: Option<&str>, + ) -> Result<()> { sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) - VALUES ($1, $2, $3, $4) - ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid, repo_id) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid, + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) - .bind(pinata_cid) // fallback local cid if row is new + .bind(raw_cid) // resolver-key cid: locally-computed raw CID, never the provider CID .bind(Utc::now().to_rfc3339()) .bind(pinata_cid) + .bind(repo_id) .execute(&self.pool) .await?; Ok(()) @@ -5037,6 +5252,51 @@ mod ref_certificate_tests { ); } + /// INV-7: upgrade-path test — an existing node already past v1 must still get + /// the `pinned_cids.cid` index. It ships as its OWN v11 migration (not appended + /// to the applied v1 bundle), so dropping the index + its `schema_migrations` + /// row and re-running migrations must recreate it, exercising the real code + /// path rather than hand-copying the SQL. + #[sqlx::test] + async fn v11_pinned_cids_cid_index_applies_on_upgrade(pool: PgPool) { + async fn index_exists(pool: &PgPool) -> bool { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_indexes WHERE indexname = 'idx_pinned_cids_cid')", + ) + .fetch_one(pool) + .await + .unwrap() + } + + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "fresh migration chain creates the index" + ); + + // Simulate a node at v10 (pre-v11): drop the index and its migration record. + sqlx::query("DROP INDEX IF EXISTS idx_pinned_cids_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 11") + .execute(&pool) + .await + .unwrap(); + assert!( + !index_exists(&pool).await, + "precondition: index and its migration record removed" + ); + + // Re-run migrations: v11 re-applies and recreates the index on the upgrade. + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "v11 must recreate idx_pinned_cids_cid on an upgrading node" + ); + } + /// INV-7: upgrade-path test — seed a database at v9 with duplicate /// ref_certificates, then let the real v10 migration fire via /// run_migrations(). This exercises the migration code path rather than diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index a07235fa..371f0cf6 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -41,6 +41,9 @@ pub enum AppError { #[error("incomplete: {0}")] Incomplete(String), + #[error("search incomplete: {0}")] + SearchIncomplete(String), + #[error("git error: {0}")] Git(String), @@ -141,6 +144,15 @@ impl IntoResponse for AppError { AppError::Incomplete(msg) => { (StatusCode::UNPROCESSABLE_ENTITY, "incomplete", msg.clone()) } + // A bounded search that could not complete (the CID resolver hit its + // legacy-probe or walk ceiling), distinct from the 404 that asserts a + // definitive not-found: absence was NOT proven, so the caller should + // retry rather than treat it as gone (#173, F2). 503, retryable. + AppError::SearchIncomplete(msg) => ( + StatusCode::SERVICE_UNAVAILABLE, + "search_incomplete", + msg.clone(), + ), AppError::Git(msg) => (StatusCode::INTERNAL_SERVER_ERROR, "git_error", msg.clone()), // 504, distinct from the 500 git_error and from the read-gate's 404 / // the auth 401, so the client can tell a deadline from a failure. diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 229ee695..25f11246 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -290,6 +290,26 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> )) } +/// Object size in bytes (`git cat-file -s`) WITHOUT reading the content, so an +/// oversized object can be rejected before it is buffered into memory (#173, F6). +/// `None` if the object does not exist or the size is unparseable. +pub fn object_size(repo_path: &Path, sha256_hex: &str) -> Result> { + // allow-unbounded-git: cheap cat-file -s header read (no content), holds no served-git + // permit and cannot hang; exact twin of object_type above. Not an INV-22 lifecycle op. + let out = Command::new("git") + .args(["cat-file", "-s", sha256_hex]) + .current_dir(repo_path) + .output() + .context("failed to run git cat-file -s")?; + if !out.status.success() { + return Ok(None); + } + Ok(String::from_utf8_lossy(&out.stdout) + .trim() + .parse::() + .ok()) +} + /// Read an object's content if its type is already known. pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) -> Result> { let content_output = Command::new("git") diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 9f07020d..d463360c 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -648,6 +648,477 @@ pub fn allowed_blob_set_for_caller_bounded( Ok(allowed) } +/// The reachable-commit enumeration for the LENIENT walks (the `/ipfs/{cid}` tree +/// gate and the commit/tag reachability set): bounded `git rev-list --all [HEAD]` +/// under the caller's shared `deadline`, deliberately WITHOUT +/// `assert_all_refs_are_commits`. That guard fail-closes a repo's whole walk when +/// any ref peels to a non-commit (an annotated tag of a tree is pushable through +/// receive-pack), which would 404 every reachable tree/commit/tag CID here for a +/// legitimate reader. `rev-list --all` skips such refs cleanly, so the commit set +/// stays complete; an object reachable only via such a ref is simply excluded — +/// correctly fail-closed. Fails closed on a rev-list error. +/// +/// Safe ONLY for a caller whose output feeds a fail-closed allow-list where absence +/// = withhold: a tolerant walk there over-withholds, never leaks. NOT safe for a +/// serve/replication filter, where a missed reachable object under-withholds — +/// those go through `blob_paths`, which runs the guard first. +fn reachable_commit_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), matching `blob_paths`. When HEAD does not resolve (unborn + // branch on an empty repo) `--all` alone yields nothing, which is correct. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); + let mut rev_args = vec!["rev-list", "--all"]; + if head_resolves { + rev_args.push("HEAD"); + } + let out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + Ok(String::from_utf8_lossy(&out) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect()) +} + +/// Every `(oid, "/repo/relative/path", kind)` triple reachable from the given +/// `commits` — the shared ls-tree seam the tree walk filters (`kind == "tree"`). +/// One bounded `git ls-tree -rzt` per commit under the caller's shared `deadline`: +/// `-rzt` is byte-identical to `-rz` for blob records and additionally emits the +/// tree object for each directory at its own path. `kind` is git's object-type +/// string ("blob", "tree", or "commit" for a gitlink). The commit's ROOT tree is +/// not emitted by `ls-tree` (it lists entries *under* a tree); `tree_paths` adds +/// it. Triples are de-duplicated across commits and paths carry a leading "/" to +/// match the glob form of visibility rules ("/secret/**"). +/// +/// Fails closed: if any tree walk fails — or a path is not valid UTF-8 — it +/// returns an error so the caller aborts rather than producing a partial +/// (under-withheld) set. +fn object_paths( + repo_path: &Path, + git_bin: &str, + commits: &[String], + deadline: Instant, +) -> Result> { + let mut out: HashSet<(String, String, String)> = HashSet::new(); + for commit in commits { + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rzt", commit], + repo_path, + b"", + deadline, + )?; + // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` + // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes + // "secret/caf\303\251.txt"), and that quoted literal would not match a + // visibility rule like "/secret/**", under-withholding the object. The TAB + // field separator survives `-z`, so the per-record parse is unchanged. + // + // Parse strictly: a lossy decode would replace an invalid byte in a denied + // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string + // would no longer match its deny rule — the same under-withholding class, one + // layer down. Fail closed instead so the caller aborts rather than leaks. + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { + anyhow::bail!( + "git ls-tree -rzt {commit} returned a non-UTF-8 path; \ + refusing to produce a partial (under-withheld) set" + ); + }; + for record in listing_stdout.split('\0') { + // " \t" + let Some((meta, path)) = record.split_once('\t') else { + continue; + }; + let mut parts = meta.split_whitespace(); + let _mode = parts.next(); + let kind = parts.next(); + let oid = parts.next(); + if let (Some(kind), Some(oid)) = (kind, oid) { + out.insert((oid.to_string(), format!("/{path}"), kind.to_string())); + } + } + } + Ok(out) +} + +/// Root tree oid of every reachable commit, at "/". `ls-tree` never emits a commit's +/// own root tree (it lists entries *under* a tree), so it is added explicitly here. +/// Resolved in ONE bounded `git log --no-walk --format=%T --stdin` pass over the +/// shared commit set — not a per-commit `rev-parse` — so a tree-set walk costs the +/// same subprocess order as the blob walk. The commit oids go on STDIN, not argv: a +/// long history has tens of thousands of reachable commits, and passing them all as +/// arguments overflows ARG_MAX so `git log` fails to spawn — which the caller treats +/// as a walk error and fail-closed 404s an authorized reader of a reachable/root +/// tree (#173 P2). `run_bounded_git` drains stdout concurrently with the stdin +/// write, so a large history cannot deadlock the pipes. A commit whose root tree git +/// cannot resolve fails the pass (bail), failing closed. +fn root_tree_pairs( + repo_path: &Path, + git_bin: &str, + commits: &[String], + deadline: Instant, +) -> Result> { + if commits.is_empty() { + return Ok(HashSet::new()); + } + let mut buf = String::with_capacity(commits.len() * 65); + for c in commits { + buf.push_str(c); + buf.push('\n'); + } + let out = run_bounded_git( + git_bin, + &["log", "--no-walk=unsorted", "--format=%T", "--stdin"], + repo_path, + buf.as_bytes(), + deadline, + )?; + let mut set = HashSet::new(); + for line in String::from_utf8_lossy(&out).lines() { + let oid = line.trim(); + if !oid.is_empty() { + set.insert((oid.to_string(), "/".to_string())); + } + } + Ok(set) +} + +/// Every `(tree_oid, "/path")` pair reachable in `repo_path`: the `kind == "tree"` +/// slice of [`object_paths`] (subtree trees at their directory paths) PLUS every +/// reachable commit's root tree at "/" (see [`root_tree_pairs`]). Computes the +/// reachable-commit set ONCE (leniently — see [`reachable_commit_oids`]; the tree +/// allowed-set feeds ONLY the `/ipfs/{cid}` tree gate, where absence = fail-closed +/// 404) and drives both the ls-tree walk and the root-tree pass from it, so the two +/// cannot diverge and neither re-enumerates. The tree analog of [`blob_paths`], +/// bounded by the same shared `deadline`. +fn tree_paths( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let commits = reachable_commit_oids(repo_path, git_bin, deadline)?; + let mut out: HashSet<(String, String)> = object_paths(repo_path, git_bin, &commits, deadline)? + .into_iter() + .filter(|(_, _, kind)| kind == "tree") + .map(|(oid, path, _)| (oid, path)) + .collect(); + out.extend(root_tree_pairs(repo_path, git_bin, &commits, deadline)?); + Ok(out) +} + +/// The OIDs from a `(oid, "/path")` listing that visibility ALLOWS `caller` at some +/// path — the shared inner loop of the blob and tree allowed-sets. An oid reachable +/// at an allowed path is kept even when also reachable at a denied one. +fn allowed_set_from_pairs<'a>( + pairs: impl IntoIterator, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> HashSet { + pairs + .into_iter() + .filter(|(_, path)| { + visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow + }) + .map(|(oid, _)| oid.clone()) + .collect() +} + +/// Reachable tree OIDs that visibility ALLOWS `caller` at some path — the tree +/// analog of [`allowed_blob_set_for_caller`]. `GET /ipfs/{cid}` gates tree objects +/// with this so the CID surface matches `get_tree`: a tree reachable only at a +/// withheld path is absent from the set and 404'd; the root tree ("/") and any tree +/// on the path to an allowed subtree are present. Fails closed on a +/// dangling/unreachable tree (never enumerated by the reachable walk, so never in +/// the set — the #126 geometry, for trees). A tree reachable at an allowed path is +/// included even when also reachable at a withheld one (its structure is visible to +/// this caller elsewhere). +#[cfg(test)] +pub fn allowed_tree_set_for_caller( + repo_path: &Path, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + allowed_tree_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_tree_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` tree gate. One deadline spans the whole walk (the HEAD +/// probe, rev-list, every per-commit ls-tree, and the root-tree pass), matching +/// `blob_paths`, so a slow or hung walk is bounded as a unit while the handler holds +/// its /ipfs walk permit (#174 F5). +pub fn allowed_tree_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let deadline = Instant::now() + timeout; + Ok(allowed_set_from_pairs( + &tree_paths(repo_path, git_bin, deadline)?, + rules, + is_public, + owner_did, + caller, + )) +} + +/// Object bound for the annotated-tag reachability walk (#173, jatmn tag fan-out). +/// A path-scoped pinned-CID request drives this walk while holding one per-request +/// and one per-IP walk slot, so the total tag work must be finite regardless of how +/// many tag refs the repo has. 8192 is far past any real repo's annotated-tag count +/// (the Linux kernel has a few hundred), yet finite: a repo beyond it fails closed +/// (Err), matching this function's fail-closed-on-any-git-error contract, rather than +/// truncating silently (which would under-withhold a still-reachable tag object). +const MAX_TAG_OBJECTS: usize = 8192; + +/// Walk the annotated-tag chains rooted at `seeds`, inserting every tag object they +/// pass through into `set`. A tag whose target is itself a tag (tag-of-a-tag) +/// discovers the inner tag, which is walked in a later round. +/// +/// #173 (jatmn): the tag inspection is BATCHED, not one process per tag. Each round +/// feeds every not-yet-inspected tag oid to a SINGLE `git cat-file --batch` child on +/// stdin and reads back framed ` \n\n` records, so the +/// number of child processes is bounded by the tag-chain DEPTH (rounds), not the tag +/// COUNT. Oids go on stdin, never argv, so a large tag set cannot overflow ARG_MAX. +/// The child runs through [`run_bounded_git`], which drains stdout concurrently with +/// the stdin write (subsuming #173's F4 writer-thread drain — a round large enough to +/// fill both pipes cannot deadlock) and tears the child down at `deadline`, so a hung +/// cat-file cannot pin the caller's /ipfs walk permit (#174 F5). Total tag objects +/// inspected are capped at `max_tag_objects`; exceeding it is an error (fail closed), +/// not a silent truncation. Takes the bound as a parameter so a test can drive a tiny +/// value while the caller passes the real `MAX_TAG_OBJECTS`. +fn walk_tag_chain( + repo_path: &Path, + git_bin: &str, + seeds: Vec, + set: &mut HashSet, + max_tag_objects: usize, + deadline: Instant, +) -> Result<()> { + // Tag oids known but not yet inspected. Seeds may repeat / already be present; + // the `set.insert` gate below is what actually dedups and terminates cycles. + let mut pending: Vec = seeds; + let mut inspected: usize = 0; + + while !pending.is_empty() { + // Inspect only oids new to `set`; a re-seen oid was already walked. + let round: Vec = pending + .drain(..) + .filter(|oid| set.insert(oid.clone())) + .collect(); + if round.is_empty() { + break; + } + inspected += round.len(); + if inspected > max_tag_objects { + anyhow::bail!( + "annotated-tag walk exceeded the object bound ({max_tag_objects}); refusing to serve" + ); + } + + // One bounded child for the whole round: feed all oids on stdin, read the + // framed records from the returned stdout. + let mut buf = String::with_capacity(round.len() * 65); + for oid in &round { + buf.push_str(oid); + buf.push('\n'); + } + let stdout = run_bounded_git( + git_bin, + &["cat-file", "--batch"], + repo_path, + buf.as_bytes(), + deadline, + )?; + + // Parse one record per requested oid: ` \n\n`. + // A ` missing\n` record has no size/body and is anomalous here (every + // oid came from a ref tip or a prior tag body), so fail closed. + let mut i = 0usize; + for _ in 0..round.len() { + let hdr_end = stdout[i..] + .iter() + .position(|&b| b == b'\n') + .map(|p| i + p) + .context("git cat-file --batch: truncated record header")?; + let header = std::str::from_utf8(&stdout[i..hdr_end]) + .context("git cat-file --batch: non-utf8 record header")?; + i = hdr_end + 1; + let mut fields = header.split(' '); + let _oid = fields.next().unwrap_or(""); + let ty = fields.next().unwrap_or(""); + if ty == "missing" || fields.clone().next().is_none() { + anyhow::bail!("git cat-file --batch: object {header:?} missing or malformed"); + } + let size: usize = fields + .next() + .unwrap_or("") + .parse() + .context("git cat-file --batch: bad record size")?; + let body_end = i + .checked_add(size) + .filter(|&e| e <= stdout.len()) + .context("git cat-file --batch: truncated record body")?; + // Only a tag object can point at an inner tag; walk its header. + if ty == "tag" { + let body = std::str::from_utf8(&stdout[i..body_end]) + .context("git cat-file --batch: non-utf8 tag body")?; + let mut target = None; + let mut is_tag = false; + for line in body.lines() { + if let Some(oid) = line.strip_prefix("object ") { + target = Some(oid.trim().to_string()); + } else if line == "type tag" { + is_tag = true; + } else if line.is_empty() { + break; // end of header + } + } + if is_tag { + if let Some(t) = target { + pending.push(t); + } + } + } + // Skip body plus its trailing newline to the next record. + i = body_end + 1; + } + } + Ok(()) +} + +/// The reachable-commit/tag gate set for the `/ipfs/{cid}` resolver (#173, F2): +/// every reachable commit oid UNION every reachable annotated-tag OBJECT oid. A +/// DANGLING commit/tag (referenced by no ref, directly or via a tag chain) is in +/// neither part, so the resolver denies it under a path-scoped rule instead of +/// leaking its message; a reachable one still serves. +#[cfg(test)] +pub fn reachable_commit_tag_oids(repo_path: &Path) -> Result> { + reachable_commit_tag_oids_bounded(repo_path, "git", WALK_TIMEOUT) +} + +/// [`reachable_commit_tag_oids`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` commit/tag gate. One deadline spans the whole walk. +/// +/// Reachable commits come from bounded `git rev-list --all` (+ HEAD for the +/// detached case). Unlike the blob allowed-set, this does NOT run +/// `assert_all_refs_are_commits`: that guard fail-closes a repo's whole walk when +/// any ref peels to a non-commit (an annotated tag of a tree is pushable through +/// receive-pack), which would 404 every reachable commit/tag CID here for a +/// legitimate reader. The guard exists to stop blob/tree UNDER-withholding; it is +/// unnecessary for reachability, since a dangling object is absent from +/// `rev-list --all` and the ref walk below regardless of odd refs — so dropping it +/// recovers availability without admitting any dangling object (no leak). +/// +/// Reachable tag OBJECTS: `rev-list --all` dereferences annotated tags to commits, +/// so the tag objects are absent from it. Collect them by walking every ref tip and +/// peeling each tag's chain, so a nested tag-of-a-tag's INNER tag object (reachable +/// and pinnable, but not itself a ref tip) is included too. Fails closed on any git +/// error. +pub fn reachable_commit_tag_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, +) -> Result> { + let deadline = Instant::now() + timeout; + // Reachable commits — no ref-commit assertion (see docstring). The HEAD probe + // doubles as the seed source for the tag-valued detached HEAD below: + // `rev-parse --verify HEAD` returns the tag oid UNPEELED when HEAD names a tag + // object. Failing to resolve HEAD (unborn/absent) is not fatal — there is + // simply no HEAD to walk or seed. + let head_oid: Option = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .ok() + .map(|out| String::from_utf8_lossy(&out).trim().to_string()) + .filter(|s| !s.is_empty()); + let mut rev_args = vec!["rev-list", "--all"]; + if head_oid.is_some() { + rev_args.push("HEAD"); + } + let rev = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let mut set: HashSet = String::from_utf8_lossy(&rev) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect(); + + // Ref tips that are annotated tag objects seed the tag-chain walk. + let refs = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(objectname) %(objecttype)"], + repo_path, + b"", + deadline, + )?; + let mut worklist: Vec = Vec::new(); + for line in String::from_utf8_lossy(&refs).lines() { + let mut it = line.split_whitespace(); + if let (Some(oid), Some("tag")) = (it.next(), it.next()) { + worklist.push(oid.to_string()); + } + } + // A detached/direct HEAD may name an annotated tag object with no ref at that tag + // (#173 review, finding 3): `rev-list --all HEAD` above peels it to its commit and + // `for-each-ref` has no tag row, so the tag OBJECT would be omitted and its pinned + // CID would 404 for an authorized reader. Seed a tag-valued HEAD into the tag-chain + // walk; a `commit` HEAD adds nothing. A cat-file failure here only skips the seed + // (over-withholds that one tag — fail-closed), matching the original's tolerance. + if let Some(head_oid) = head_oid { + if let Ok(ty) = run_bounded_git( + git_bin, + &["cat-file", "-t", &head_oid], + repo_path, + b"", + deadline, + ) { + if String::from_utf8_lossy(&ty).trim() == "tag" { + worklist.push(head_oid); + } + } + } + // Peel every tag object's chain into `set`, adding each tag object it passes + // through. Bounded and batched (#173, jatmn tag fan-out): see `walk_tag_chain`. + walk_tag_chain( + repo_path, + git_bin, + worklist, + &mut set, + MAX_TAG_OBJECTS, + deadline, + )?; + Ok(set) +} + /// Objects safe to replicate, failing closed on blobs (#99). A candidate /// replicates iff it is NOT a blob (`all_blob_oids` — commits and trees are /// structural, never content-withheld) OR it is in `allowed_blobs` (reachable @@ -1068,6 +1539,614 @@ esac\n"; (td, bare, secret, public) } + /// #173 (jatmn round 8, F4 — load-bearing): a repo with enough annotated tags that + /// one `cat-file --batch` round fills BOTH pipes (stdin > 64 KiB of oids while the + /// child blocks on a full stdout) must not deadlock. The old order wrote the whole + /// round to stdin before draining stdout and hung indefinitely, stranding a blocking- + /// pool thread; `run_bounded_git`'s concurrent writer/drain completes. Driven with a + /// completion timeout: GREEN finishes in well under a second, RED (old order) hangs + /// and the recv_timeout fires. ~3000 tags is well past the ~2030-oid deadlock + /// threshold (41 bytes/oid, 64 KiB pipes) and under MAX_TAG_OBJECTS (8192). + /// Bulk-created via one fast-import stream so the fixture cost is one git process, + /// not 3000 `git tag -a` spawns. + #[test] + fn walk_tag_chain_large_batch_does_not_deadlock() { + use std::io::Write; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(&work).unwrap(); + std::fs::write(work.join("f.txt"), b"x\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + let head = { + let out = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Bulk-create ~3000 annotated tags via one fast-import stream. + const N: usize = 3000; + let mut stream = String::new(); + for i in 0..N { + let msg = format!("annotated tag {i}\n"); + stream.push_str(&format!("tag t{i}\n")); + stream.push_str(&format!("from {head}\n")); + stream.push_str("tagger t 1700000000 +0000\n"); + stream.push_str(&format!("data {}\n", msg.len())); + stream.push_str(&msg); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&work) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + + // Drive the walk on a worker thread with a completion timeout. The old + // write-all-before-drain order hangs here; the fix completes near-instantly. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(reachable_commit_tag_oids(&bare).map(|s| s.len())); + }); + match rx.recv_timeout(std::time::Duration::from_secs(20)) { + Ok(Ok(n)) => assert!( + n >= N, + "the walk must resolve every annotated tag object (got {n}, expected >= {N})" + ), + Ok(Err(e)) => panic!("walk errored: {e}"), + Err(_) => panic!("walk_tag_chain deadlocked on a large tag batch (F4 regression)"), + } + } + + /// #173 review (finding 3): an annotated tag reachable ONLY through a tag-valued + /// detached HEAD (raw HEAD naming a tag object, with no ref at that tag) must still + /// enter `reachable_commit_tag_oids`. `rev-list --all HEAD` peels such a HEAD to its + /// commit and `for-each-ref` has no tag row, so without a HEAD tag-seed the tag + /// OBJECT is omitted and its pinned CID would 404 for an authorized reader. RED + /// before the HEAD tag-seed (the tag oid is absent); GREEN after. + #[test] + fn reachable_commit_tag_oids_includes_tag_valued_detached_head() { + use std::io::Write; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::create_dir_all(&work).unwrap(); + std::fs::write(work.join("a.txt"), b"hi\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "seed"], &work); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + let commit = run(&["rev-parse", "HEAD"], &bare); + + // An annotated tag OBJECT in the bare ODB, with NO ref pointing at it. + let tag_body = format!( + "object {commit}\ntype commit\ntag htag\ntagger t 0 +0000\n\nHEAD-only tag\n" + ); + let tag_oid = { + let mut child = Command::new("git") + .args(["hash-object", "-t", "tag", "-w", "--stdin"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .as_mut() + .unwrap() + .write_all(tag_body.as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + assert_eq!(run(&["cat-file", "-t", &tag_oid], &bare), "tag"); + // Raw-write HEAD directly to the tag object (the only way this state arises; + // update-ref / checkout both refuse a non-commit HEAD). + std::fs::write(bare.join("HEAD"), format!("{tag_oid}\n")).unwrap(); + + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!( + set.contains(&tag_oid), + "a tag reachable only via a tag-valued detached HEAD must be in the reachable set" + ); + assert!( + set.contains(&commit), + "the commit the HEAD tag peels to stays reachable (no regression)" + ); + } + + /// #173: `reachable_commit_tag_oids` on an empty repo (unborn HEAD) must return an + /// empty set, not error — exercising the `rev-parse HEAD` fail branch of the + /// detached-HEAD tag seed (there is simply no HEAD to seed). + #[test] + fn reachable_commit_tag_oids_handles_unborn_head() { + let td = TempDir::new().unwrap(); + let bare = td.path().join("empty.git"); + let ok = Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success(); + assert!(ok, "git init --bare failed"); + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!( + set.is_empty(), + "an empty repo (unborn HEAD) yields an empty reachable set with no error" + ); + } + + #[test] + fn object_paths_emits_trees_and_blob_paths_is_the_blob_slice() { + let (_td, bare, secret_oid, public_oid) = fixture(); + let deadline = Instant::now() + WALK_TIMEOUT; + // The lenient enumeration; on this clean fixture it matches the strict one. + let commits = reachable_commit_oids(&bare, "git", deadline).unwrap(); + let objs = object_paths(&bare, "git", &commits, deadline).unwrap(); + + // Blob records survive the `-rzt` change, at their paths (unchanged). + assert!(objs.contains(&(secret_oid.clone(), "/secret/b.txt".into(), "blob".into()))); + assert!(objs.contains(&(public_oid.clone(), "/public/a.txt".into(), "blob".into()))); + + // The #135 addition: subtree tree objects at their directory paths. + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/secret"), + "the /secret subtree tree must be emitted at its dir path" + ); + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/public"), + "the /public subtree tree must be emitted at its dir path" + ); + + // blob_paths must equal the blob slice of object_paths exactly — compared as + // SETS (both walks dedup via HashSet; the collected order is nondeterministic). + let bp: HashSet<(String, String)> = blob_paths(&bare, "git", WALK_TIMEOUT) + .unwrap() + .into_iter() + .collect(); + let bp_from_obj: HashSet<(String, String)> = objs + .iter() + .filter(|(_, _, k)| k == "blob") + .map(|(o, p, _)| (o.clone(), p.clone())) + .collect(); + assert_eq!( + bp, bp_from_obj, + "blob_paths output must be byte-identical to object_paths' blob slice" + ); + } + + #[test] + fn allowed_tree_set_gates_withheld_subtree_tree() { + let (_td, bare, _s, _p) = fixture(); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let secret_tree = oid("HEAD:secret"); + let public_tree = oid("HEAD:public"); + let root_tree = oid("HEAD^{tree}"); + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", &[reader])]; + + // anon: the withheld /secret tree is excluded; root ("/") and /public are in. + let anon = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, None).unwrap(); + assert!( + !anon.contains(&secret_tree), + "withheld /secret subtree tree excluded for anon" + ); + assert!(anon.contains(&root_tree), "root tree included (path /)"); + assert!(anon.contains(&public_tree), "/public subtree tree included"); + + // listed reader: sees the /secret tree (caller-aware, not a blanket deny). + let rd = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(reader)).unwrap(); + assert!( + rd.contains(&secret_tree), + "listed reader sees the /secret tree" + ); + + // owner: sees every reachable tree. + let ow = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(OWNER)).unwrap(); + assert!( + ow.contains(&secret_tree) && ow.contains(&public_tree) && ow.contains(&root_tree), + "owner sees all reachable trees" + ); + } + + #[test] + fn allowed_tree_set_excludes_dangling_tree() { + use std::io::Write; + let (_td, bare, secret_oid, _p) = fixture(); + // A DANGLING tree: written to the ODB but referenced by no commit. Uses a + // UNIQUE entry name so its oid is content-distinct from every reachable tree + // (a content-identical tree would dedup to a reachable oid — that is T2, not + // danglingness). The reachable-only walk never enumerates it -> fail closed. + let mut child = Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {secret_oid}\tdangling-only-unreferenced.txt" + ) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "git mktree"); + let dangling = String::from_utf8_lossy(&out.stdout).trim().to_string(); + + let rules = [rule("/secret/**", &[])]; + for caller in [None, Some(OWNER)] { + let set = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, caller).unwrap(); + assert!( + !set.contains(&dangling), + "dangling tree must never be in the reachable allowed-set (caller={caller:?})" + ); + } + } + + #[test] + fn allowed_tree_set_includes_tree_shared_across_allowed_and_denied_paths() { + // T2 (content-dedup): the SAME tree oid reachable at both an allowed and a + // withheld path is INCLUDED for anon (allowed-wins) — its structure is + // visible to the caller at the allowed path. Mirrors the blob analog + // `same_blob_at_allowed_and_denied_path_is_not_withheld`. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(work.join("pub/sub")).unwrap(); + std::fs::create_dir_all(work.join("sec/sub")).unwrap(); + std::fs::write(work.join("pub/sub/f.txt"), b"same bytes\n").unwrap(); + std::fs::write(work.join("sec/sub/f.txt"), b"same bytes\n").unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + run(&["add", "."]); + run(&["commit", "-qm", "seed"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let pub_sub = oid("HEAD:pub/sub"); + let sec_sub = oid("HEAD:sec/sub"); + assert_eq!(pub_sub, sec_sub, "identical content dedups to one tree oid"); + + // Withhold /sec from anon; the shared oid is still reachable at /pub/sub. + let rules = [rule("/sec/**", &[])]; + let anon = allowed_tree_set_for_caller(&work, &rules, true, OWNER, None).unwrap(); + assert!( + anon.contains(&pub_sub), + "a tree reachable at an allowed path is included even when also at a withheld path" + ); + } + + #[test] + fn allowed_tree_set_includes_root_trees_of_all_reachable_commits() { + // The batched root-tree pass (root_tree_pairs) must return EVERY reachable + // commit's root tree, not just HEAD's — two commits with distinct root trees + // both land in the set. Guards the git-log-over-N-commits root derivation. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::write(work.join("a.txt"), b"one\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c1"]); + let root1 = oid("HEAD^{tree}"); + std::fs::write(work.join("b.txt"), b"two\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c2"]); + let root2 = oid("HEAD^{tree}"); + assert_ne!(root1, root2, "the two commits have distinct root trees"); + + // Public repo, no rules: every reachable tree is allowed for anon. + let set = allowed_tree_set_for_caller(&work, &[], true, OWNER, None).unwrap(); + assert!( + set.contains(&root1) && set.contains(&root2), + "root trees of BOTH reachable commits are in the set (batched root pass)" + ); + } + + #[test] + fn root_tree_pairs_returns_every_root_tree_at_scale() { + // Parity + liveness at scale for root_tree_pairs (#173 P2): feed every + // reachable commit oid to `git log --format=%T --stdin` and collect each + // commit's root tree. With N commits that is ~N*41 bytes of oids in and + // ~N*41 bytes of %T out — past the ~64 KiB pipe buffer in both directions — + // so this exercises the large-bidirectional-IO path the 2-commit test above + // cannot, and asserts parity: every distinct root tree comes back. + // + // NOTE: this is NOT a deadlock guard. `git log --stdin` reads its whole + // revision list to EOF before emitting any %T, so the naive "write all of + // stdin, then drain stdout" form does not deadlock at any scale for this + // invocation. `run_bounded_git`'s concurrent writer/drain is cheap defensive + // isolation, not load-bearing, and this test does not claim otherwise. The + // 30s watchdog is a general liveness bound so a future regression that + // genuinely hangs fails fast here rather than stalling the suite. + const N: usize = 2500; + let td = TempDir::new().unwrap(); + let bare = td.path().join("many.git"); + assert!(Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success()); + + // fast-import a linear chain of N commits, each adding a distinct file so + // every root tree is distinct (dedup cannot shrink the output). One + // subprocess, ~1s — far cheaper than N `git commit` spawns. + let mut stream = String::new(); + for i in 0..N { + let (b, cm) = (2 * i + 1, 2 * i + 2); + let content = format!("v{i}"); + let msg = format!("c{i}"); + stream.push_str(&format!( + "blob\nmark :{b}\ndata {}\n{content}\n", + content.len() + )); + stream.push_str(&format!( + "commit refs/heads/main\nmark :{cm}\ncommitter t 0 +0000\ndata {}\n{msg}\n", + msg.len() + )); + if i > 0 { + stream.push_str(&format!("from :{}\n", 2 * (i - 1) + 2)); + } + stream.push_str(&format!("M 100644 :{b} f{i}\n\n")); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + { + use std::io::Write; + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + } + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + let commits = reachable_commit_oids(&bare, "git", Instant::now() + WALK_TIMEOUT).unwrap(); + assert_eq!(commits.len(), N, "all {N} commits reachable"); + + // Call root_tree_pairs directly (private, same module) under a liveness + // watchdog, then assert it returned every distinct root tree. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send( + root_tree_pairs(&bare, "git", &commits, Instant::now() + WALK_TIMEOUT) + .map(|s| s.len()), + ); + }); + match rx.recv_timeout(std::time::Duration::from_secs(30)) { + Ok(Ok(len)) => assert_eq!(len, N, "every distinct root tree returned"), + Ok(Err(e)) => panic!("root_tree_pairs errored: {e}"), + Err(_) => panic!("root_tree_pairs did not return within 30s"), + } + } + + /// #173 (jatmn tag fan-out): the batched `git cat-file --batch` tag walk must + /// return the SAME reachable set as the old per-tag `cat-file tag` loop — every + /// commit, the outer tag object, AND the inner tag object of a tag-of-a-tag chain + /// (the inner tag is reachable but is not itself a ref tip, so it is only found by + /// peeling the outer tag's target). Behavior-preservation proof for the rewrite. + #[test] + fn reachable_commit_tag_oids_includes_nested_tag_objects() { + let (_td, bare, _secret, _public) = fixture(); + let run = |args: &[&str]| -> String { + let out = Command::new("git") + .args(args) + .current_dir(&bare) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + // v1 -> commit, v2 -> v1 (tag-of-a-tag), plus a couple of sibling tags so the + // round batches more than one oid. Capture v1's oid, then DELETE the v1 ref so + // the inner tag object survives in the ODB but is NOT a ref tip: it is then + // reachable ONLY by peeling v2's target chain. That makes the peel load-bearing + // (breaking the inner-tag enqueue drops v1 from the set), unlike leaving v1 as + // its own ref where `for-each-ref` would seed it directly. + run(&["tag", "-a", "-m", "inner", "v1", "HEAD"]); + run(&["tag", "-a", "-m", "outer", "v2", "v1"]); + run(&["tag", "-a", "-m", "s1", "s1", "HEAD"]); + run(&["tag", "-a", "-m", "s2", "s2", "HEAD"]); + let commit = run(&["rev-parse", "HEAD"]); + let v1 = run(&["rev-parse", "v1"]); + let v2 = run(&["rev-parse", "v2"]); + let s1 = run(&["rev-parse", "s1"]); + let s2 = run(&["rev-parse", "s2"]); + run(&["tag", "-d", "v1"]); + + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!(set.contains(&commit), "the commit must be reachable"); + assert!( + set.contains(&v2), + "the outer tag object (ref tip) must be present" + ); + assert!( + set.contains(&v1), + "the INNER tag object of a tag-of-a-tag must be present (peeled from v2, no ref)" + ); + assert!(set.contains(&s1), "sibling tag s1 must be present"); + assert!(set.contains(&s2), "sibling tag s2 must be present"); + } + + /// #173 (jatmn tag fan-out): the object bound is load-bearing. A repo whose tag + /// count exceeds the bound must FAIL CLOSED (Err), not return a truncated set that + /// would under-withhold a still-reachable tag. Drives `walk_tag_chain` with a tiny + /// injected bound (the public fn uses the real `MAX_TAG_OBJECTS`); with the bound + /// check removed this would collect all tags and return Ok. + #[test] + fn walk_tag_chain_fails_closed_over_object_bound() { + let (_td, bare, _secret, _public) = fixture(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&bare) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let mut seeds = Vec::new(); + for n in 0..5 { + let name = format!("t{n}"); + run(&["tag", "-a", "-m", &name, &name, "HEAD"]); + let oid = Command::new("git") + .args(["rev-parse", &name]) + .current_dir(&bare) + .output() + .unwrap(); + seeds.push(String::from_utf8_lossy(&oid.stdout).trim().to_string()); + } + + // Within a generous bound: the walk succeeds and collects the tags. + let mut ok_set = HashSet::new(); + walk_tag_chain( + &bare, + "git", + seeds.clone(), + &mut ok_set, + 8192, + Instant::now() + WALK_TIMEOUT, + ) + .unwrap(); + assert!( + seeds.iter().all(|s| ok_set.contains(s)), + "all 5 tags collected under a generous bound" + ); + + // Under a bound of 2 with 5 tags: fail closed (Err), not a partial set. + let mut small_set = HashSet::new(); + let result = walk_tag_chain( + &bare, + "git", + seeds, + &mut small_set, + 2, + Instant::now() + WALK_TIMEOUT, + ); + assert!( + result.is_err(), + "a tag count exceeding the object bound must fail closed (Err), not truncate" + ); + } + #[test] fn anonymous_caller_withholds_only_private_blob() { let (_td, bare, secret_oid, public_oid) = fixture(); diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3b346190..a4ed8e4d 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -91,7 +91,8 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { /// `..._fail_closed` filter on the full-scan path before calling, so this /// function never sees a withheld blob. `repo_path` is still needed to read each /// object's bytes. The twin in `pinata.rs` mirrors this shape — change both in -/// lockstep. +/// lockstep. `repo_id` records the pin's provenance so `GET /ipfs/{cid}` resolves +/// straight to this repo instead of scanning every repo (#173). /// /// Returns a list of `(sha256_hex, cid)` pairs for objects pinned this call. pub async fn pin_new_objects( @@ -99,6 +100,7 @@ pub async fn pin_new_objects( repo_path: &std::path::Path, object_list: Vec, db: &crate::db::Db, + repo_id: &str, ) -> Vec<(String, String)> { if ipfs_api.is_empty() { return vec![]; @@ -107,9 +109,36 @@ pub async fn pin_new_objects( let mut pinned = Vec::new(); for sha in object_list { - // Skip if already pinned + // Skip if already pinned — but first backfill provenance if the existing + // pin has none. A legacy pin (recorded before repo_id existed, #173, jatmn) + // is skipped here before record_pinned_cid ever runs, so its NULL provenance + // would never resolve to one repo and known CIDs keep hitting the scan. The + // backfill only sets repo_id (AND repo_id IS NULL guard preserves + // first-pinner-owns) and never re-pins the bytes — the object is already on IPFS. match db.is_pinned(&sha).await { - Ok(true) => continue, + Ok(true) => { + match db.provenance_for_oid(&sha).await { + Ok(None) => { + if let Err(e) = db.backfill_pin_provenance(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); + } + } + Ok(Some(_)) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error reading pin provenance"); + } + } + // F1 (#173 round 8): record this repo as an ADDITIONAL source for the + // already-pinned object. This is the load-bearing skip-branch insert — + // a later repo pushing a shared object hits this path (already pinned), + // and without it `GET /ipfs/{cid}` only ever knows the first pinner, so a + // shared object first pinned from a private/quarantined repo 404s even + // when this repo would serve it. Bounded per object (MAX_PIN_SOURCES). + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinned status"); @@ -130,9 +159,14 @@ pub async fn pin_new_objects( // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinned_cid(&sha, &cid).await { + if let Err(e) = db.record_pinned_cid(&sha, &cid, Some(repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } + // F1 (#173 round 8): also record the first pinner in pin_repo_sources so + // every source (first and subsequent) is tried uniformly by the resolver. + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } pinned.push((sha, cid)); } Ok(_) => {} diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index ffac65a2..37190c76 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -374,6 +374,9 @@ async fn main() -> Result<()> { rate_limiter, create_ip_rate_limiter, push_rate_limiter, + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, sync_trigger_rate_limiter, peer_write_rate_limiter, @@ -465,22 +468,16 @@ async fn main() -> Result<()> { // Periodic cleanup of expired rate limit entries + consumed-proof ledger { - let rl = state.rate_limiter.clone(); - let create_ip_rl = state.create_ip_rate_limiter.clone(); - let push_rl = state.push_rate_limiter.clone(); - let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); - let peer_write_rl = state.peer_write_rate_limiter.clone(); + let cleanup_state = state.clone(); let db = state.db.clone(); let mut shutdown_rx = state.subscribe_shutdown(); tokio::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - rl.cleanup().await; - create_ip_rl.cleanup().await; - push_rl.cleanup().await; - sync_trigger_rl.cleanup().await; - peer_write_rl.cleanup().await; + // Sweep every per-IP/DID limiter (incl. the ipfs walk brake) + // so bounded maps shed stale keys instead of sitting at cap. + cleanup_state.sweep_rate_limiters().await; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs() as i64) diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 6c9c0bff..31843bbc 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -74,8 +74,10 @@ pub async fn pin_object( /// `..._fail_closed` filter on the full-scan path before calling. `repo_path` is /// still needed to read each object's bytes. The twin in `ipfs_pin.rs` mirrors /// this shape — change both in lockstep. Objects already recorded with a -/// `pinata_cid` are skipped. Returns `(sha_hex, cid)` pairs for each newly -/// pinned object. +/// `pinata_cid` are skipped. `repo_id` records the pin's provenance (#173). +/// Returns `(sha_hex, provider_cid)` pairs for each newly pinned object: the +/// provider CID is the Pinata gateway CID (used for branch→CID recording and +/// ref-update gossip), NOT the raw resolver-key CID stored in `pinned_cids.cid`. pub async fn pin_new_objects( client: &reqwest::Client, upload_url: &str, @@ -83,6 +85,7 @@ pub async fn pin_new_objects( repo_path: &std::path::Path, object_list: Vec, db: &crate::db::Db, + repo_id: &str, ) -> Vec<(String, String)> { if jwt.is_empty() { return vec![]; @@ -92,7 +95,15 @@ pub async fn pin_new_objects( for sha in object_list { match db.has_pinata_cid(&sha).await { - Ok(true) => continue, + Ok(true) => { + // F1 (#173 round 8): record this repo as an additional source for the + // already-pinned object (mirrors the ipfs_pin skip-branch insert) so the + // resolver can serve a shared object from any pin-path source. + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinata_cid"); @@ -111,9 +122,21 @@ pub async fn pin_new_objects( match pin_object(client, upload_url, jwt, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinata_cid(&sha, &cid).await { + // The resolver key (`pinned_cids.cid`) must be the locally-computed + // raw-content CID, never the provider CID: Pinata wraps the bytes in + // dag-pb/UnixFS, so its returned CID does not hash the raw content and + // must not become an alias `/ipfs/{cid}` serves raw git bytes for (#173). + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + if let Err(e) = db + .record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) + .await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); } + // F1 (#173 round 8): also record the first pinner in pin_repo_sources. + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } pinned.push((sha, cid)); } Ok(_) => {} diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index 22281691..14c11a9c 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -121,6 +121,28 @@ impl RateLimiter { true } + /// Non-consuming check: is this key ALREADY at its limit for the current window? + /// Unlike [`check`], it records nothing and never inserts a new key — used to shed + /// expensive preparatory work (e.g. the `/ipfs/{cid}` legacy scan's O(repos) DB + /// preload) BEFORE it runs, without perturbing the per-unit budget the consuming + /// `check` maintains (#173, F3). An unknown key or a disabled limiter is not + /// throttled. Prunes the key's expired timestamps as a side effect (keeps state + /// tidy) but adds none, so it cannot itself fill or grow the map. + pub(crate) async fn is_throttled(&self, key: &str) -> bool { + if self.max_requests == 0 { + return false; + } + let now = Instant::now(); + let mut state = self.state.lock().await; + if let Some(window) = state.get_mut(key) { + window + .timestamps + .retain(|t| now.duration_since(*t) < self.window); + return window.timestamps.len() >= self.max_requests; + } + false + } + pub async fn cleanup(&self) { let now = Instant::now(); let mut state = self.state.lock().await; @@ -130,6 +152,13 @@ impl RateLimiter { !w.timestamps.is_empty() }); } + + /// Number of distinct keys currently tracked. Test-only introspection so a + /// cross-module test can assert that a sweep actually evicted expired entries. + #[cfg(test)] + pub(crate) async fn tracked_keys(&self) -> usize { + self.state.lock().await.len() + } } /// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index deccf4c7..10dbbf32 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -65,6 +65,29 @@ pub struct AppState { /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. pub push_rate_limiter: RateLimiter, + /// Per-client-IP rate limiter for the `GET /ipfs/{cid}` full-history walk. + /// The route is anonymous and a valid tree CID (exposed by the public pins + /// index) makes each repeat request pay a fresh allowed-set walk (rev-list + + /// ls-tree per commit), memoized only per request — unbounded amplification + /// (INV-10). Braking the walk on the non-farmable source IP caps that cost + /// without touching cheap non-walk fetches. Keyed by `push_limiter_trust`. + pub ipfs_rate_limiter: RateLimiter, + /// Per-request ceiling on full-history reachability walks the CID resolver + /// may spawn (default `api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST`). A field, + /// not a bare const, so tests can shrink it to exercise the cap cheaply; + /// production keeps the const default. + pub ipfs_max_history_walks: u32, + /// Per-request ceiling on legacy (NULL-provenance) repo probes in the CID + /// resolver's scan fallback (default `api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST`). + /// Bounds the anonymous `acquire` + `cat-file` fan-out across the node (#173, + /// INV-10); a field for the same test-seam reason as `ipfs_max_history_walks`. + pub ipfs_max_legacy_probes: u32, + /// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` will buffer and + /// serve (default `api::ipfs::MAX_SERVED_OBJECT_BYTES`). The serve reads via a + /// blocking `git cat-file` and buffers the whole object; without a bound a large + /// public blob could exhaust memory or block a runtime worker (#173, F6, INV-10). + /// A field for the same test-seam reason as the sibling caps. + pub ipfs_max_served_object_bytes: u64, /// Which forwarded header (if any) the edge is trusted to set, for /// resolving the push limiter's client-IP key. See `GITLAWB_TRUSTED_PROXY`. /// Node-wide; also keys the two peer-sync limiters below. @@ -171,13 +194,6 @@ pub struct AppState { /// (`with_default_max_keys`, reject-before-insert) so a source-key farm cannot grow /// it (INV-15). pub git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency, - /// Per-client-IP rate limiter for `GET /ipfs/{cid}`. The route is publicly - /// reachable and each request can drive a full-history git walk, so it carries a - /// per-IP flood brake in addition to the concurrency cap above — a rate limit - /// bounds request *rate*, the semaphore bounds concurrent slow holds (different - /// axes). Keyed on the resolved client IP via `push_limiter_trust`. Layered on the - /// `/ipfs` route via `rate_limit_by_ip`. - pub ipfs_rate_limiter: RateLimiter, /// The `git` executable the served-git withheld-blob walk spawns. Production is /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's /// process-group teardown in handler tests without mutating the process-global @@ -192,6 +208,20 @@ impl AppState { self.shutdown_tx.subscribe() } + /// Sweep expired entries from every per-IP/DID rate limiter. Driven by the + /// periodic cleanup task so a bounded limiter's key map sheds stale entries + /// instead of sitting near its cap until an inline capacity sweep reclaims + /// them. Every limiter on the state is swept here; adding a new limiter means + /// adding it to this list. + pub(crate) async fn sweep_rate_limiters(&self) { + self.rate_limiter.cleanup().await; + self.create_ip_rate_limiter.cleanup().await; + self.push_rate_limiter.cleanup().await; + self.ipfs_rate_limiter.cleanup().await; + self.sync_trigger_rate_limiter.cleanup().await; + self.peer_write_rate_limiter.cleanup().await; + } + /// Trigger graceful shutdown. Idempotent — calling more than once /// has no effect. Returns `true` if this call was the one that /// flipped the signal. diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 92f41a8d..21fa58e4 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -78,6 +78,10 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), @@ -98,7 +102,6 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 16, ), - ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } @@ -2020,13 +2023,19 @@ mod tests { /// Seed a SHA-256 source repo (public/a.txt + secret/b.txt), bare-clone it /// into each `/tmp//.git` path, and return guards + oids. - /// SHA-256 object format is required: `get_by_cid` resolves a CID whose - /// multihash digest IS the git object id, which only matches in sha256 repos. + /// SHA-256 object format matches production (`--object-format=sha256`) so the + /// oids are 64-hex. A real CID digests the raw object CONTENT (not the git + /// oid), so tests build the request CID with `pin_cid_for` — mirroring the pin + /// path — and `get_by_cid` maps it back to the oid via `pinned_cids` (#173). struct CidFixture { _guards: Vec, secret_oid: String, public_oid: String, secret_tree_oid: String, + public_tree_oid: String, + root_tree_oid: String, + commit_oid: String, + tag_oid: String, } impl Drop for CidFixture { fn drop(&mut self) { @@ -2060,6 +2069,8 @@ mod tests { run(&["config", "user.name", "t"], &src); run(&["add", "."], &src); run(&["commit", "-qm", "seed"], &src); + // Annotated tag of the commit — exercises the "tags stay served" guard. + run(&["tag", "-a", "-m", "annotated", "v1", "HEAD"], &src); let oid = |rev: &str| { let out = Command::new("git") .args(["rev-parse", rev]) @@ -2072,6 +2083,10 @@ mod tests { let secret_oid = oid("HEAD:secret/b.txt"); let public_oid = oid("HEAD:public/a.txt"); let secret_tree_oid = oid("HEAD:secret"); + let public_tree_oid = oid("HEAD:public"); + let root_tree_oid = oid("HEAD^{tree}"); + let commit_oid = oid("HEAD"); + let tag_oid = oid("refs/tags/v1"); let mut guards = vec![src.clone()]; for name in bare_names { let bare = std::path::PathBuf::from("/tmp") @@ -2089,6 +2104,12 @@ mod tests { ], &src, ); + // `git clone --bare` does NOT copy the source repo's local identity, so + // fixtures that create objects directly in the bare repo (`commit-tree`, + // `git tag -a`) abort with "identity unknown" on a CI runner that has no + // ambient/global git identity. Set it explicitly so the suite is portable. + run(&["config", "user.email", "t@t"], &bare); + run(&["config", "user.name", "t"], &bare); } // One guard for the whole /tmp/ tree covers every bare clone. guards.push(std::path::PathBuf::from("/tmp").join(slug)); @@ -2097,300 +2118,3160 @@ mod tests { secret_oid, public_oid, secret_tree_oid, + public_tree_oid, + root_tree_oid, + commit_oid, + tag_oid, } } - /// CID whose sha2-256 multihash digest equals the given 64-hex git oid, so - /// `get_by_cid` decodes it back to that oid and `git cat-file`s it. - fn cid_for_oid(oid_hex: &str) -> String { - use gitlawb_core::cid::Cid; - let bytes = hex::decode(oid_hex).expect("hex oid"); - let arr: [u8; 32] = bytes.as_slice().try_into().expect("32-byte sha256 oid"); - Cid::from_sha256_bytes(&arr).to_string() + /// Record a pin exactly as the production pin path does — read the object's + /// raw bytes (`git cat-file `, no framing), CID them with + /// `Cid::from_git_object_bytes`, and store the `(oid, cid)` row — then return + /// the CID string the node advertises (`gl ipfs list`) and a client sends to + /// `GET /ipfs/{cid}`. Building the CID from the oid instead (the old + /// `cid_for_oid`) produced an identifier that never occurs in production and + /// made the gate assertions vacuous: a real pin CID digests the raw content, + /// not the git oid, so `get_by_cid` resolves it through `pinned_cids` (#173). + async fn pin_cid_for(bare_repo: &std::path::Path, oid: &str, db: &crate::db::Db) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + // Legacy-style pin (no provenance) so existing CID tests exercise the + // resolver's scan fallback; provenance-path tests pin via `pin_cid_for_repo`. + db.record_pinned_cid(oid, &cid, None) + .await + .expect("record pinned cid"); + cid } - fn cid_router(state: &AppState) -> Router { - Router::new() - .route( - "/ipfs/{cid}", - axum::routing::get(crate::api::ipfs::get_by_cid), - ) - .layer(axum::middleware::from_fn(crate::auth::optional_signature)) - .with_state(state.clone()) + /// Like [`pin_cid_for`] but records the pin's provenance (`repo_id`), so the + /// resolver resolves the CID straight to `repo_id` instead of scanning (#173). + #[allow(dead_code)] // used by the provenance-path resolver tests (P-U3) + async fn pin_cid_for_repo( + bare_repo: &std::path::Path, + oid: &str, + db: &crate::db::Db, + repo_id: &str, + ) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + db.record_pinned_cid(oid, &cid, Some(repo_id)) + .await + .expect("record pinned cid with provenance"); + cid } - async fn cid_parts(resp: axum::response::Response) -> (StatusCode, String) { - let st = resp.status(); - let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + + /// INV-7 upgrade path for the pin-provenance column (#173, jatmn round 2): a node + /// already past v11 gets `pinned_cids.repo_id` from the NEW v12 migration, and a + /// legacy pin recorded before the column existed survives with NULL provenance (so + /// it falls back to the repo scan). Simulate the pre-v12 node by dropping the + /// column and un-applying v12, seed a legacy row, then re-migrate. RED before the + /// v12 migration exists (the column is never re-added → the SELECT errors); GREEN + /// after. + #[sqlx::test] + async fn pinned_cids_repo_provenance_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v12 shape: drop the provenance column and forget v12 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS repo_id") + .execute(&pool) .await .unwrap(); - (st, String::from_utf8_lossy(&b).to_string()) - } - fn cid_anon(cid: &str) -> Request { - Request::builder() - .method(Method::GET) - .uri(format!("/ipfs/{cid}")) - .body(Body::empty()) - .unwrap() - } - fn cid_signed(kp: &gitlawb_core::identity::Keypair, cid: &str) -> Request { - let path = format!("/ipfs/{cid}"); - let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); - Request::builder() - .method(Method::GET) - .uri(&path) - .header("content-digest", s.content_digest) - .header("signature-input", s.signature_input) - .header("signature", s.signature) - .body(Body::empty()) - .unwrap() + sqlx::query("DELETE FROM schema_migrations WHERE version = 12") + .execute(&pool) + .await + .unwrap(); + + // A legacy pin recorded before provenance existed. + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind("legacyoid") + .bind("legacycid") + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + // Upgrade: re-run migrations → v12 re-adds the column. + state.db.run_migrations().await.expect("migrate to v12"); + + // The legacy pin survives with NULL provenance. + let legacy: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = 'legacyoid'") + .fetch_one(&pool) + .await + .expect("legacy pin row survives the upgrade"); + assert!( + legacy.is_none(), + "a pin recorded before v12 must keep NULL provenance (it falls back to the scan)" + ); + + // A new pin can carry provenance. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind("newoid") + .bind("newcid") + .bind("2026-01-01T00:00:00Z") + .bind("repo-abc") + .execute(&pool) + .await + .unwrap(); + let prov: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = 'newoid'") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + prov.as_deref(), + Some("repo-abc"), + "a pin recorded after v12 carries its source repo_id" + ); } - /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. - /// RED before U2 (the current handler serves the secret to anon). + /// #173: a pin records the repository it came from; `provenance_for_oid` reads it + /// back; a legacy pin (no repo) reads back None; and first-pinner-owns holds — a + /// second push of the same oid does NOT rewrite provenance (ON CONFLICT DO + /// NOTHING). This is what lets the resolver gate a CID against its ONE source repo. #[sqlx::test] - async fn ipfs_cid_gate_withholds_blob_from_unauthorized(pool: PgPool) { - use crate::db::VisibilityMode; - use gitlawb_core::identity::Keypair; - - let owner = Keypair::generate(); - let owner_did = owner.did().to_string(); - let reader = Keypair::generate(); - let reader_did = reader.did().to_string(); - let stranger = Keypair::generate(); - let slug = owner_did.replace([':', '/'], "_"); - let short = owner_did.split(':').next_back().unwrap().to_string(); + async fn record_pinned_cid_stores_and_reads_provenance(pool: PgPool) { let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let tree_cid = cid_for_oid(&fx.secret_tree_oid); - let public_cid = cid_for_oid(&fx.public_oid); - state .db - .create_repo(&seed_repo(&owner_did, "withhold")) + .record_pinned_cid("oidA", "cidA", Some("repo-xyz")) .await - .expect("seed repo"); - let rec = state + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("oidA") + .await + .unwrap() + .as_deref(), + Some("repo-xyz"), + "a provenanced pin reads back its source repo_id" + ); + + state .db - .get_repo(&owner_did, "withhold") + .record_pinned_cid("oidB", "cidB", None) .await - .unwrap() .unwrap(); + assert_eq!( + state.db.provenance_for_oid("oidB").await.unwrap(), + None, + "a legacy pin (no repo) has NULL provenance" + ); + + // First-pinner-owns: a later push of the same oid must not rewrite provenance. state .db - .set_visibility_rule( - &rec.id, - "/secret/**", - VisibilityMode::B, - std::slice::from_ref(&reader_did), - &owner_did, - ) + .record_pinned_cid("oidA", "cidA", Some("repo-OTHER")) .await - .expect("deny rule"); - - // anon → withheld blob: must 404, must not leak content. (RED on current handler.) - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; + .unwrap(); assert_eq!( - st, - StatusCode::NOT_FOUND, - "anon must not read the withheld blob" - ); - assert!( - !body.contains("TOP SECRET"), - "404 body must not leak the secret" + state + .db + .provenance_for_oid("oidA") + .await + .unwrap() + .as_deref(), + Some("repo-xyz"), + "ON CONFLICT DO NOTHING keeps the first repo's provenance" ); - // signed non-reader → 404. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&stranger, &secret_cid)) - .await - .unwrap(), - ) - .await; + // An unpinned oid has no provenance. assert_eq!( - st, - StatusCode::NOT_FOUND, - "non-reader must not read the withheld blob" + state.db.provenance_for_oid("never-pinned").await.unwrap(), + None ); - assert!(!body.contains("TOP SECRET")); - - // owner (signed) → 200 + secret bytes. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&owner, &secret_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "owner reads the withheld blob"); - assert!(body.contains("TOP SECRET"), "owner gets the content"); + } - // listed reader (signed) → 200. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&reader, &secret_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); - assert!(body.contains("TOP SECRET")); + /// #173 (provenance, happy path): a CID pinned with provenance resolves straight + /// to its ONE source repo and serves an authorized reader — no repo scan. + #[sqlx::test] + async fn ipfs_cid_provenance_serves_from_pinning_repo(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; - // KTD3: anon tree CID under /secret → 200 (trees/commits are not withheld). - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&tree_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "tree object is served to anon (KTD3)"); + let _fx = seed_cid_repos(&slug, &short, &["provserve"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provserve.git"); + let fx = &_fx; - // R3: public blob anon → 200 (non-withheld content not affected). - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&public_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "public blob stays served"); + // Build the repo FIRST so the pin can carry its id as provenance. + let repo = seed_repo(&owner_did, "provserve"); // public + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; - // R5: a genuine unknown CID also 404, uniform with the withheld 404. - let absent_cid = cid_for_oid(&"ab".repeat(32)); - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&absent_cid)) - .await - .unwrap(), - ) - .await; + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; assert_eq!( st, - StatusCode::NOT_FOUND, - "absent CID 404 (uniform with withheld)" + StatusCode::OK, + "a provenanced public CID serves its content" + ); + assert!( + body.contains("public bytes"), + "the pinning repo's object is served" ); - - // malformed CID → 400 (unchanged). - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon("not-a-cid")) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::BAD_REQUEST, "malformed CID still 400"); } - /// R4: the same object withheld in one repo but public in another is still - /// served from the public copy; the withholding repo is iterated first. + /// #173 (provenance, THE load-bearing one — #124 flip + bounded fan-out): a CID + /// pinned from a PRIVATE repo must gate against that pinning repo (404), NOT serve + /// from a byte-identical PUBLIC copy in another repo. Provenance is strictly more + /// restrictive than the old scan (which served the public copy). RED before the + /// rework (the scan serves the public copy → 200 + leaks the secret bytes); GREEN + /// after (provenance → the private repo → 404, no leak). #[sqlx::test] - async fn ipfs_cid_served_from_public_copy_when_withheld_elsewhere(pool: PgPool) { - use crate::db::VisibilityMode; - use chrono::Utc; + async fn ipfs_cid_provenance_private_denies_despite_public_copy(pool: PgPool) { use gitlawb_core::identity::Keypair; - let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); - let secret_cid = cid_for_oid(&fx.secret_oid); + let fx = seed_cid_repos(&slug, &short, &["privsrc", "pubcopy"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privsrc.git"); - // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). - let mut withhold = seed_repo(&owner_did, "withhold"); - withhold.updated_at = Utc::now(); + // Private source repo, built first so the pin carries its id as provenance. + let mut priv_repo = seed_repo(&owner_did, "privsrc"); + priv_repo.is_public = false; state .db - .create_repo(&withhold) + .create_repo(&priv_repo) .await - .expect("withhold repo"); + .expect("seed private repo"); + let cid = pin_cid_for_repo(&priv_bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + + // A PUBLIC repo holds the SAME object (the old scan would serve it). + let pub_repo = seed_repo(&owner_did, "pubcopy"); // public, no rule state .db - .set_visibility_rule( - &withhold.id, - "/secret/**", - VisibilityMode::B, - &[], - &owner_did, - ) + .create_repo(&pub_repo) .await - .expect("deny rule"); - - // Public copy, no rules, iterated AFTER. - let mut pubcopy = seed_repo(&owner_did, "pubcopy"); - pubcopy.updated_at = Utc::now() - chrono::Duration::seconds(60); - state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); + .expect("seed public copy"); - // anon: denied at the withholding repo (continue), served from the public copy. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; assert_eq!( st, - StatusCode::OK, - "served from the public copy despite the other deny" + StatusCode::NOT_FOUND, + "a provenanced private CID must 404, not serve from a public copy elsewhere (#124 flip)" ); assert!( - body.contains("TOP SECRET"), - "the public copy serves the content" + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" ); } - /// Repo-level "/" gate (KTD2a, first continue branch): a fully private repo - /// (is_public=false, no rules) denies anon before any per-blob check; the - /// owner still reads. The path-scoped tests pass the "/" gate and deny at the - /// per-blob stage, so this exercises the coarser repo-level deny separately. + /// #173 (jatmn round 8, F1 — load-bearing): a shared object first pinned from a + /// PRIVATE repo, then pushed again from a PUBLIC repo through the real pin path, + /// must serve by CID to an anonymous caller from the public source. First-pinner- + /// only provenance 404s it (only the private source is known); recording EVERY + /// pin-path source fixes it. The second push hits the already-pinned skip branch, + /// so this proves the skip-branch source insert fires (and does NOT re-pin: /add + /// expect(0)). RED before U1 (anon 404); GREEN after. #[sqlx::test] - async fn ipfs_cid_private_repo_denies_anon_at_repo_gate(pool: PgPool) { + async fn ipfs_cid_multi_source_serves_from_later_public_pinner(pool: PgPool) { use gitlawb_core::identity::Keypair; - let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["priv"]); - let blob_cid = cid_for_oid(&fx.public_oid); - - let mut rec = seed_repo(&owner_did, "priv"); - rec.is_public = false; - state.db.create_repo(&rec).await.expect("private repo"); + let fx = seed_cid_repos(&slug, &short, &["privfirst", "pubsecond"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privfirst.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubsecond.git"); + + // Private repo pins the object FIRST — it owns the first-pinner provenance. + let mut priv_repo = seed_repo(&owner_did, "privfirst"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private first-pinner"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // A PUBLIC repo pushes the SAME object through the real pin path. The object is + // already pinned, so this hits the already-pinned skip branch, which must record + // the public repo as an additional source without re-pinning (/add expect 0). + let pub_repo = seed_repo(&owner_did, "pubsecond"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public second-pinner"); + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &pub_bare, + vec![fx.public_oid.clone()], + &state.db, + &pub_repo.id, + ) + .await; + m.assert_async().await; // asserts /add was NOT called (already pinned) + + // Anonymous CID fetch: the private first source denies, the public second + // source serves → 200. Before F1 only the private source is known → 404. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a shared object must serve by CID from a later public pin-path source (F1)" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + + /// #173 (jatmn round 8, F1 — bound, R2): the per-object source set is capped at + /// `MAX_PIN_SOURCES` so an adversary pushing one object from many repos cannot make + /// resolution O(repos). Recording the same oid from `MAX_PIN_SOURCES + 3` distinct + /// repos leaves exactly `MAX_PIN_SOURCES` rows. + #[sqlx::test] + async fn ipfs_cid_pin_sources_capped_at_max(pool: PgPool) { + let state = test_state(pool).await; + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..(cap + 3) { + state + .db + .record_pin_source("capoid", &format!("repo-{i}")) + .await + .expect("record source"); + } + let sources = state.db.pin_sources_for_oid("capoid").await.unwrap(); + assert_eq!( + sources.len() as i64, + cap, + "the per-object source set is capped at MAX_PIN_SOURCES" + ); + } + + /// #173 (jatmn round 8, F1 — availability, grok-4.5 adversarial catch): the resolver's + /// per-object source cap must NEVER evict the first-pinner. A legacy public pin keeps + /// its source in `pinned_cids.repo_id` but not in `pin_repo_sources` (pre-v13 pins, or + /// a pin whose best-effort `record_pin_source` missed). If the cap `LIMIT` were applied + /// to the whole union with a lexicographic order, an attacker could push the same + /// object from `MAX_PIN_SOURCES` repos whose grindable ids sort before the public + /// source and evict it from the window — turning a public CID that served 200 into a + /// 404. This drives exactly that: a legacy public first-pinner plus `MAX_PIN_SOURCES` + /// lower-sorting attacker sources must STILL serve the public object. RED with a + /// whole-union LIMIT (the first-pinner is dropped → 404); GREEN once the first-pinner + /// is always included and the LIMIT caps only the additional sources. + #[sqlx::test] + async fn ipfs_cid_first_pinner_never_evicted_by_lower_sorting_sources(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["pubfirst"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubfirst.git"); + // Public repo whose id sorts AFTER every attacker id below. Legacy shape: the + // source lives in pinned_cids.repo_id only (pin_cid_for_repo records no + // pin_repo_sources row), exactly like a pin from before v13. + let mut pub_repo = seed_repo(&owner_did, "pubfirst"); // public, no rule + pub_repo.id = "zzzzzzzz-pubfirst".to_string(); + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public first-pinner"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &pub_repo.id).await; + + // Attacker fills the whole MAX_PIN_SOURCES window with lower-sorting source ids + // (non-existent repos — their mere presence would evict the first-pinner under a + // whole-union LIMIT). + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..cap { + state + .db + .record_pin_source(&fx.public_oid, &format!("00-attacker-{i:02}")) + .await + .expect("attacker source"); + } + + // The public first-pinner must still serve — never evicted by the cap window. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "the first-pinner public source must never be evicted by lower-sorting attacker sources (F1 availability)" + ); + assert!( + body.contains("public bytes"), + "the public object is served from the first-pinner" + ); + } + + /// INV-7 upgrade path for the F1 `pin_repo_sources` table (#173, jatmn round 8): a + /// node already past v12 gets the table from the NEW v13 migration. Simulate the + /// pre-v13 node by dropping the table and un-applying v13, then re-migrate and + /// assert a source row round-trips. RED before the v13 migration exists. + #[sqlx::test] + async fn pin_repo_sources_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + sqlx::query("DROP TABLE IF EXISTS pin_repo_sources") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 13") + .execute(&pool) + .await + .unwrap(); + state.db.run_migrations().await.expect("re-migrate"); + state + .db + .record_pin_source("upgradeoid", "repo-upg") + .await + .expect("record after re-migrate"); + assert_eq!( + state.db.pin_sources_for_oid("upgradeoid").await.unwrap(), + vec!["repo-upg".to_string()], + "the v13 pin_repo_sources table is present after upgrade" + ); + } + + /// #173 (jatmn round 8, F2 — load-bearing): a legacy `pinned_cids` row keyed on a + /// PROVIDER CID (Pinata/Kubo dag-pb — every release before this branch stored the + /// provider CID as the resolver key, not the raw-content CID) must NOT serve raw git + /// bytes that do not hash to the requested CID. `get_by_cid` recomputes the CID over + /// the served bytes and refuses to serve on mismatch. Seeded with a RAW SQL INSERT + /// because the current helpers store the raw CID, so a helper-seeded row is already + /// correct-shape and the RED assertion would be vacuous (INV-21). RED before U2 + /// (serves the git bytes → 200); GREEN after (not served, no bytes egress). + #[sqlx::test] + async fn ipfs_cid_legacy_provider_cid_row_not_served(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc"]); + let repo = seed_repo(&owner_did, "provsrc"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // A valid sha2-256 CID whose digest is NOT the object's raw-content digest — + // stands in for a Pinata/Kubo dag-pb provider CID (the legacy resolver key). + let provider_cid = gitlawb_core::cid::Cid::from_git_object_bytes( + b"a decoy object whose CID is not the served object's CID", + ) + .to_string(); + + // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers now store the + // raw CID and cannot reproduce this shape). The object itself is public+servable. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + // Requesting the provider CID resolves the row and passes the repo gate, but the + // served bytes hash to a DIFFERENT CID, so the integrity check must withhold them. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&provider_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st, + StatusCode::OK, + "a provider-CID legacy row must not serve raw git bytes (F2)" + ); + assert!( + !body.contains("public bytes"), + "the mismatched bytes must not egress" + ); + } + + /// #173 (jatmn round 8, F6 — INV-10 cost guard): the serve path buffers the object via + /// a blocking `cat-file`; an object larger than `ipfs_max_served_object_bytes` must be + /// WITHHELD (rejected by the size precheck, never buffered), with zero body bytes + /// egressed. Under the cap it serves unchanged. The oversize-reject counter guards it + /// both ways: a removed size precheck serves the object and leaves the counter at 0. + #[sqlx::test] + async fn ipfs_cid_f6_oversized_object_withheld(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["big"]); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("big.git"); + let repo = seed_repo(&owner_did, "big"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Cap below the object size ("public bytes\n" = 13 bytes) → withheld. + state.ipfs_max_served_object_bytes = 5; + crate::api::ipfs::reset_oversize_rejects(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_ne!( + st, + StatusCode::OK, + "an object over the size cap must not serve (F6)" + ); + assert!( + !body.contains("public bytes"), + "no object bytes egress for an over-cap object" + ); + assert_eq!( + crate::api::ipfs::oversize_rejects(), + 1, + "the oversized object was rejected by the size precheck" + ); + + // Control: raise the cap above the object size → serves unchanged. + state.ipfs_max_served_object_bytes = crate::api::ipfs::MAX_SERVED_OBJECT_BYTES; + crate::api::ipfs::reset_oversize_rejects(); + let (st2, body2) = + cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st2, + StatusCode::OK, + "under the cap the object serves normally" + ); + assert!( + body2.contains("public bytes"), + "the served body is the object's bytes" + ); + assert_eq!( + crate::api::ipfs::oversize_rejects(), + 0, + "no oversize reject under the cap" + ); + } + + /// #173 (provenance, INV-11): a quarantined pinning repo must 404 by CID even for + /// its own owner — quarantine hard-drops before the visibility gate on the + /// provenance path too. The owner-signed 404 is the load-bearing negative (a + /// visibility-only gate would Allow the owner). + #[sqlx::test] + async fn ipfs_cid_provenance_quarantined_repo_404_even_owner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["quarsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quarsrc.git"); + let repo = seed_repo(&owner_did, "quarsrc"); // public + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Baseline: before quarantine the provenanced CID serves (proves the path works). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "provenanced CID serves before quarantine" + ); + + state + .db + .set_repo_quarantine(&repo.id, true) + .await + .expect("quarantine"); + + for req in [cid_anon(&cid), cid_signed(&owner, &cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a quarantined pinning repo must 404 by CID (anon + owner)" + ); + assert!( + !body.contains("public bytes"), + "the 404 body must not leak quarantined content" + ); + } + } + + /// #173 (provenance, bounded — must NOT fall back to the scan): a CID whose + /// provenance points at a repo that no longer exists must 404 rather than scan + /// every repo and serve a byte-identical public copy. Falling back to the scan + /// would reopen the O(repos) anonymous fan-out the provenance rework closes. RED + /// before the rework (the scan serves the public copy → 200); GREEN after. + #[sqlx::test] + async fn ipfs_cid_provenance_missing_repo_404_no_scan_fallback(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["gonesrc", "pubcopy2"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("gonesrc.git"); + + // Pin with provenance = a repo_id that is never created (deleted/absent). + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, "nonexistent-repo-id").await; + + // A public repo holds the SAME object (the old scan would serve it). + let pub_repo = seed_repo(&owner_did, "pubcopy2"); + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public copy"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a provenance pointing at a missing repo must 404, not fall back to the scan" + ); + } + + /// #173 (provenance, path-scoped WALK gate): the #135/#173 per-object gates must + /// run on the NEW provenance path, not only the legacy scan. A provenanced pin from + /// a repo under a `/secret/**` rule runs `allowed_blob_set_for_caller` via the shared + /// gate: a withheld secret blob 404s to anon (no byte leak); the allowed reader gets + /// it. Exercises the walk gate on the provenance path in BOTH directions. + #[sqlx::test] + async fn ipfs_cid_provenance_path_scoped_walk_gates_withheld_blob(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["provwalk"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provwalk.git"); + let repo = seed_repo(&owner_did, "provwalk"); // public at "/" + state.db.create_repo(&repo).await.expect("seed repo"); + // /secret/** Mode B with the reader allowed → the secret blob walk gates by caller. + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + + // Anon: the walk denies the secret blob → 404, no leak. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a withheld secret blob 404s to anon on the provenance path (walk gate runs)" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld blob" + ); + + // Allowed reader: the walk includes the secret blob → 200 with content. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&reader, &cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "an allowed reader gets the secret blob via the provenance walk gate" + ); + assert!( + body.contains("TOP SECRET"), + "the allowed reader receives the content" + ); + } + + /// #173: the pinata pin path stores the locally-computed raw CID in the + /// resolver-key `cid` column and the provider CID in `pinata_cid`, and its ON + /// CONFLICT COALESCE fills a NULL provenance without overwriting an existing one + /// (first-pinner-owns). On conflict `cid` is left untouched so a prior local pin's + /// raw CID is never clobbered by a provider CID. + #[sqlx::test] + async fn record_pinata_cid_stores_and_coalesces_provenance(pool: PgPool) { + let state = test_state(pool).await; + + // A new row created via the pinata path carries provenance, and stores the + // raw CID in `cid` with the provider CID in `pinata_cid`. + state + .db + .record_pinata_cid("po1", "rawcid1", "pcid1", Some("repoA")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po1").await.unwrap().as_deref(), + Some("repoA") + ); + let po1 = state + .db + .list_pinned_cids() + .await + .unwrap() + .into_iter() + .find(|r| r.sha256_hex == "po1") + .expect("po1 row exists"); + assert_eq!(po1.cid, "rawcid1", "resolver-key cid is the raw CID"); + assert_eq!( + po1.pinata_cid.as_deref(), + Some("pcid1"), + "the provider CID is kept in pinata_cid" + ); + + // An existing NULL-provenance row: the pinata COALESCE fills it, and the + // prior local pin's `cid` is left untouched (not overwritten by the raw arg). + state + .db + .record_pinned_cid("po2", "localcid2", None) + .await + .unwrap(); + state + .db + .record_pinata_cid("po2", "rawcid2", "pcid2", Some("repoB")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po2").await.unwrap().as_deref(), + Some("repoB"), + "pinata fills a NULL provenance" + ); + let po2 = state + .db + .list_pinned_cids() + .await + .unwrap() + .into_iter() + .find(|r| r.sha256_hex == "po2") + .expect("po2 row exists"); + assert_eq!( + po2.cid, "localcid2", + "on conflict the prior local pin's cid is left untouched" + ); + + // An existing provenance: the pinata COALESCE must NOT overwrite it. + state + .db + .record_pinned_cid("po3", "cid3", Some("repoX")) + .await + .unwrap(); + state + .db + .record_pinata_cid("po3", "rawcid3", "pcid3", Some("repoY")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po3").await.unwrap().as_deref(), + Some("repoX"), + "pinata COALESCE keeps the first-pinner's provenance" + ); + } + + /// #173 (jatmn, F4, load-bearing security): a Pinata-first pin (no prior local pin) + /// must make the resolver key (`pinned_cids.cid`) the locally-computed raw CID, NOT + /// the provider CID. Pinata wraps the bytes in dag-pb/UnixFS, so its returned CID + /// does not hash the raw content; if it became the resolver key, `/ipfs/{provider_cid}` + /// would serve raw git bytes that do not hash to it, breaking raw content-addressing. + /// Assert `oids_for_cid(raw_cid)` finds the sha AND `oids_for_cid(provider_cid)` does NOT. + #[sqlx::test] + async fn record_pinata_cid_resolver_key_is_raw_not_provider(pool: PgPool) { + let state = test_state(pool).await; + + let bytes = b"raw git object content for pinata-first pin"; + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(bytes).to_string(); + // A distinct provider CID (a dag-pb wrapper CID Pinata would return). + let provider_cid = "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"; + assert_ne!( + raw_cid, provider_cid, + "the provider CID must differ from the raw CID for this test to be meaningful" + ); + + // Pinata-first: no prior local pin, so this INSERT creates the row. + state + .db + .record_pinata_cid("pfsha", &raw_cid, provider_cid, Some("repoP")) + .await + .unwrap(); + + // The raw CID resolves to the sha. + assert_eq!( + state.db.oids_for_cid(&raw_cid).await.unwrap(), + vec!["pfsha".to_string()], + "the locally-computed raw CID is the resolver key" + ); + // The provider (dag-pb) CID must NOT resolve raw bytes. + assert!( + state + .db + .oids_for_cid(provider_cid) + .await + .unwrap() + .is_empty(), + "the provider dag-pb CID must never resolve raw git bytes" + ); + } + + /// #173 (end-to-end pin wiring): `pin_new_objects` records the repo_id it is given + /// as the pin's provenance. Drives the real pin path against a mocked IPFS `/add` + /// endpoint (so `pin_git_object` succeeds) and asserts `provenance_for_oid` returns + /// the repo — closing the gap between the push handler's threading and the DB write. + #[sqlx::test] + async fn pin_new_objects_records_provenance(pool: PgPool) { + let state = test_state(pool).await; + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovtest"}"#) + .expect_at_least(1) + .create_async() + .await; + + let fx = seed_cid_repos("provpin_e2e", "ppe2e", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_e2e") + .join("pinsrc.git"); + + let pinned = crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![fx.public_oid.clone()], + &state.db, + "repoZ", + ) + .await; + assert!( + !pinned.is_empty(), + "the object was pinned via the real pin path" + ); + m.assert_async().await; + assert_eq!( + state + .db + .provenance_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some("repoZ"), + "pin_new_objects records the repo_id it was given as the pin's provenance" + ); + } + + /// #173 (jatmn, F2): a legacy pin with NULL provenance backfills its source + /// via `backfill_pin_provenance`, and the `AND repo_id IS NULL` guard preserves + /// first-pinner-owns (a non-NULL provenance is left untouched). + #[sqlx::test] + async fn backfill_pin_provenance_fills_null_keeps_existing(pool: PgPool) { + let state = test_state(pool).await; + + // A legacy pin: no provenance recorded. + state + .db + .record_pinned_cid("legacy_oid", "legacy_cid", None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("legacy_oid").await.unwrap(), + None, + "a legacy pin starts with NULL provenance" + ); + + // Backfill sets the NULL provenance. + state + .db + .backfill_pin_provenance("legacy_oid", "repo-src") + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("legacy_oid") + .await + .unwrap() + .as_deref(), + Some("repo-src"), + "backfill fills a NULL provenance from the known source" + ); + + // A pin that already has provenance: backfill must NOT overwrite it. + state + .db + .record_pinned_cid("owned_oid", "owned_cid", Some("repo-first")) + .await + .unwrap(); + state + .db + .backfill_pin_provenance("owned_oid", "repo-second") + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("owned_oid") + .await + .unwrap() + .as_deref(), + Some("repo-first"), + "the AND repo_id IS NULL guard keeps the first-pinner's provenance" + ); + } + + /// #173 (jatmn, F2, load-bearing): an object already pinned with NULL provenance + /// (a pre-provenance legacy pin) acquires its source when `pin_new_objects` sees + /// it again. The already-pinned skip path must backfill rather than leave the + /// object stuck on the O(repos) scan fallback — and it must NOT re-pin the bytes + /// (no IPFS `/add` call, the object is already on IPFS). + #[sqlx::test] + async fn pin_new_objects_backfills_legacy_null_provenance(pool: PgPool) { + let state = test_state(pool).await; + + let fx = seed_cid_repos("provpin_backfill", "ppbf", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_backfill") + .join("pinsrc.git"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(&bare, &fx.public_oid) + .expect("read object bytes") + .expect("object exists") + .1, + ) + .to_string(); + + // Legacy pin: the object is already recorded with NULL provenance. + state + .db + .record_pinned_cid(&fx.public_oid, &cid, None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid(&fx.public_oid).await.unwrap(), + None, + "the object starts as a legacy pin with NULL provenance" + ); + + // Mock IPFS `/add` and require it is NOT called: the already-pinned object + // must be backfilled, never re-pinned. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + + let pinned = crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![fx.public_oid.clone()], + &state.db, + "repoBF", + ) + .await; + + assert!( + pinned.is_empty(), + "an already-pinned object is not re-pinned (no bytes returned)" + ); + m.assert_async().await; // asserts /add was called 0 times + assert_eq!( + state + .db + .provenance_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some("repoBF"), + "pin_new_objects backfills the legacy pin's NULL provenance" + ); + } + + /// #173 (provenance-path throttle): a walk-requiring provenanced candidate whose + /// per-IP walk quota is spent returns 429 (the provenance arm's Throttled outcome, + /// then the fall-through). quota=1, keyed on XFF. The first reader request runs the + /// walk and spends the token; the second from the same IP is throttled → 429. + #[sqlx::test] + async fn ipfs_cid_provenance_walk_throttle_returns_429(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["provthrottle"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provthrottle.git"); + let repo = seed_repo(&owner_did, "provthrottle"); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + + // 1st reader request runs the walk (reader is allowed) and spends the token. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "1st provenance walk from the IP serves"); + + // 2nd request from the same IP: the walk is throttled → 429 (provenance path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "a throttled provenance walk returns 429" + ); + } + + /// #173 (multi-oid dispatch, mixed provenance + legacy): one CID mapping to a + /// provenanced-then-denied oid AND a legacy (NULL-provenance) oid must still resolve + /// to the legacy-servable copy — the provenance arm's skip does not abort the loop. + #[sqlx::test] + async fn ipfs_cid_mixed_provenance_and_legacy_serves_legacy(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["mixpriv", "mixpub"]); + + // Private repo holds secret_oid, pinned with provenance = itself (denies anon). + let mut priv_repo = seed_repo(&owner_did, "mixpriv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + // Public repo holds public_oid, legacy pin (NULL provenance -> scan serves it). + let pub_repo = seed_repo(&owner_did, "mixpub"); + state.db.create_repo(&pub_repo).await.expect("seed public"); + + // One REAL CID (the non-unique cid index) maps to BOTH oids: the public oid as a + // legacy (NULL) pin, and the secret oid provenanced to the private repo. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("mixpub.git"); + let shared_cid = pin_cid_for(&pub_bare, &fx.public_oid, &state.db).await; + state + .db + .record_pinned_cid(&fx.secret_oid, &shared_cid, Some(&priv_repo.id)) + .await + .unwrap(); + + // Anon: secret_oid (provenance -> private -> denied), public_oid (legacy -> scan + // -> public -> served). Resolves to the public copy regardless of oid order. + let resp = cid_router(&state) + .oneshot(cid_anon(&shared_cid)) + .await + .unwrap(); + let served = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a CID mixing a provenanced-denied oid and a legacy-servable oid resolves" + ); + assert_eq!( + served.as_deref(), + Some(fx.public_oid.as_str()), + "the served object is the legacy public oid" + ); + assert!( + body.contains("public bytes"), + "the public content is served" + ); + } + + // ---- #173 round 3: legacy (NULL-provenance) scan bound + 503-on-truncation ---- + // The provenance path targets one repo and is already bounded. These cover the + // legacy scan fallback, where an anonymous request could otherwise fan out to + // O(repos) `acquire` + `cat-file` probes (F1) and a walk-cap truncation could + // false-404 an object that may be readable (F2). The bound is a per-request probe + // BUDGET, not a per-IP brake: a walk-free public fetch stays un-rate-limited + // (ipfs_walk_rate_limited_per_source), while the expensive walk keeps its IP brake. + + /// T1 (F1): the probe budget gates BEFORE `acquire`/`cat-file`, so it genuinely + /// bounds the fan-out — a repo past the budget is never probed, even one that + /// WOULD serve. With the budget at 0, a PUBLIC legacy copy that would otherwise + /// serve 200 is not probed at all → 503 truncated (absence unproven). RED before + /// the budget check (the repo is probed and serves 200). + #[sqlx::test] + async fn ipfs_cid_legacy_probe_budget_gates_before_serving(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 0; // probe nothing → any legacy candidate truncates + + let fx = seed_cid_repos(&slug, &short, &["pubprobe"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubprobe.git"); + let repo = seed_repo(&owner_did, "pubprobe"); // public, no path rule → would serve + state.db.create_repo(&repo).await.expect("seed repo"); + // Legacy pin (NULL provenance) → resolver takes the scan fallback. + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "the probe budget gates before the probe: a servable copy past the budget is not reached → 503" + ); + } + + /// T7 (F1/F3 pre-limit): EVERY legacy probe is braked on the source IP from the + /// FIRST one, so a hostile caller cannot repeatedly force the whole-node `acquire` + /// fan-out across requests (each cold `acquire` is a Tigris round-trip, INV-10). + /// Since #173-F3 (jatmn) there is no free budget: a single-repo legacy scan is + /// itself charged. quota=1 keyed on XFF, one PUBLIC legacy copy that serves + /// walk-free (never touches the walk brake), so the second same-IP request can only + /// be shed by the probe brake: req1 serves and spends the token, req2 → 429. RED + /// before the probe brake (req2 serves 200). The cross-request bound this proves is + /// exactly the amplification F3 closes. + #[sqlx::test] + async fn ipfs_cid_legacy_fanout_braked_on_ip_past_free_budget(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["fanout"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("fanout.git"); + let repo = seed_repo(&owner_did, "fanout"); // public, no path rule → walk-free serve + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; // legacy pin + + let (st1, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st1, + StatusCode::OK, + "1st legacy fan-out probe from the IP serves" + ); + + let (st2, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st2, + StatusCode::TOO_MANY_REQUESTS, + "with no free budget, a repeat fan-out from the same IP is braked at the first probe" + ); + } + + /// F3 (jatmn, across-request amplification): the pre-fix free-probe budget was + /// PER REQUEST, so a caller could repeat a known NULL-provenance CID and force a + /// fresh batch of `acquire` + `cat-file` probes every request with zero limiter + /// contact, unbounded anonymous amplification against Tigris. Charging every + /// legacy probe from the first one makes those probes accumulate against the + /// per-IP `ipfs_rate_limiter` ACROSS requests. Four repos, none holding the CID, + /// so a full scan probes all four; the per-IP budget is sized to exactly ONE such + /// scan (4 tokens). req1 (a genuine absence) fully scans and 404s, spending the + /// budget; req2 from the SAME IP is shed at the first probe → 429 (it never + /// re-runs the four `acquire` probes). RED with the old free carve-out restored: + /// req2 re-scans un-braked and 404s again (the amplification stays open). This is + /// the load-bearing across-request bound F3 asks for. + #[sqlx::test] + async fn ipfs_cid_legacy_fanout_bounded_across_requests(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget = one full scan of the four seeded repos. A repeat scan from the same + // IP then finds it spent. Keyed on XFF so `oneshot` can choose the source IP. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(4, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let names = ["a0", "a1", "a2", "a3"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo → each probed repo misses, + // so req1 scans all four (spending the four-token budget) and 404s cleanly. + let bogus_oid = "0".repeat(64); + let cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-across-requests").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st1, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st1, + StatusCode::NOT_FOUND, + "1st scan completes under budget: a genuine absence is a definitive 404" + ); + + let (st2, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st2, + StatusCode::TOO_MANY_REQUESTS, + "2nd same-IP scan is shed at the first probe (429), not re-run un-braked: the across-request amplification is closed" + ); + } + + /// #173 (jatmn round 8, F3 — INV-10 cost guard): an already-throttled source's + /// legacy NULL-provenance request must be shed by the non-consuming admission peek + /// BEFORE the O(repos) `scan_ctx` preload runs — not after, where the per-probe + /// brake sits. The preload-query counter proves it both ways: 0 for the throttled + /// replay, 1 for an unthrottled source. RED if the peek is removed (the preload runs + /// while throttled → count 1). The two existing `_fanout_` tests confirm the per- + /// probe consuming charge is untouched (no double-charge, no under-charge). + #[sqlx::test] + async fn ipfs_cid_f3_throttled_source_skips_preload(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget 1, keyed on XFF so `oneshot` can choose the source IP. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let _fx = seed_cid_repos(&slug, &short, &["r0"]); + state + .db + .create_repo(&seed_repo(&owner_did, "r0")) + .await + .expect("seed repo"); + // A legacy pin absent from every repo → the scan probes and 404s (spending the + // one token on the first probe). + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"f3-absent").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("legacy pin"); + + // Req1 from 9.9.9.9 spends the one token (and runs the preload once). + let _ = cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(); + + // Measure the throttled replay: the peek must shed it before the preload runs. + crate::api::ipfs::reset_preload_queries(); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "an already-throttled legacy replay is 429" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "a throttled source must NOT run the O(repos) preload (F3): shed before scan_ctx" + ); + + // Control: an unthrottled source (a different IP) still runs the preload once — + // the peek must not over-block. + crate::api::ipfs::reset_preload_queries(); + let _ = cid_router(&state) + .oneshot(cid_anon_xff(&cid, "8.8.8.8")) + .await + .unwrap(); + assert_eq!( + crate::api::ipfs::preload_queries(), + 1, + "an unthrottled source runs the preload once (the peek must not over-block)" + ); + } + + /// T2 (F1): the legacy scan is bounded per request. With the probe ceiling shrunk + /// to 2 and 3 candidate repos none of which hold the object, the 3rd repo is never + /// probed and the search is reported truncated → 503, not an unbounded fan-out. + /// RED before the probe cap (all 3 probe, none serve, definitive 404). + #[sqlx::test] + async fn ipfs_cid_legacy_scan_probe_cap_truncates_to_503(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 2; + + let _fx = seed_cid_repos(&slug, &short, &["r0", "r1", "r2"]); + for n in ["r0", "r1", "r2"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo: each probed repo misses, + // so the cap (not a hit) decides the outcome. + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-t2").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "a scan truncated by the probe cap is a retryable 503, not a definitive 404" + ); + } + + /// T3 (F2): a walk-cap truncation must not false-404. Walk ceiling shrunk to 1; + /// two public repos each carry a path-scoped rule over the object and deny anon. + /// The 1st spends the single walk (deny), the 2nd is skipped at the cap — the + /// resolver did NOT prove the object unreadable everywhere, so 503, not 404. + /// RED before the walk-cap `truncated` flag (returns the opaque 404). + #[sqlx::test] + async fn ipfs_cid_legacy_walk_cap_truncates_to_503(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_history_walks = 1; + + let fx = seed_cid_repos(&slug, &short, &["wa", "wb"]); + for n in ["wa", "wb"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + } + // Legacy pin of the path-scoped secret blob (present in both repos, denies anon). + let bare_wa = std::path::PathBuf::from("/tmp").join(&slug).join("wa.git"); + let cid = pin_cid_for(&bare_wa, &fx.secret_oid, &state.db).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "the walk cap truncated the scan, so absence is unproven → 503, not a false 404" + ); + } + + /// T4 (must-not over-fire): a legacy CID genuinely absent from every repo on a + /// node UNDER the probe cap still returns the definitive 404 — the 503 fires only + /// on real truncation, never as a blanket replacement for not-found. + #[sqlx::test] + async fn ipfs_cid_legacy_true_absence_stays_404(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 8; // well above the single repo → no truncation + + let _fx = seed_cid_repos(&slug, &short, &["only"]); + let repo = seed_repo(&owner_did, "only"); + state.db.create_repo(&repo).await.expect("seed repo"); + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-t4").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a fully-scanned genuine absence is a definitive 404, not a 503" + ); + } + + /// T5 (provenance path untouched): the probe cap governs ONLY the legacy scan. + /// With the cap set to 0 (which would truncate any legacy probe immediately) a + /// PROVENANCED pin still resolves to its one repo and serves 200 — proving the + /// `legacy_scan=false` guard exempts the provenance path. RED if the guard were + /// dropped (provenance would truncate to 503). + #[sqlx::test] + async fn ipfs_cid_provenance_serves_despite_zero_probe_cap(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 0; // would truncate every LEGACY probe + + let fx = seed_cid_repos(&slug, &short, &["provonly"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provonly.git"); + let repo = seed_repo(&owner_did, "provonly"); // public, no path rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "the provenance path ignores the legacy probe cap and serves" + ); + } + + fn cid_router(state: &AppState) -> Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state.clone()) + } + async fn cid_parts(resp: axum::response::Response) -> (StatusCode, String) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, String::from_utf8_lossy(&b).to_string()) + } + /// Raw body bytes (NOT lossy-decoded). A git tree body stores each child oid + /// as 32 RAW bytes that `from_utf8_lossy` mangles to U+FFFD, so a hex + /// `contains` check on `cid_parts`'s String is vacuous. #135 deny tests must + /// witness the leak on these raw bytes. + async fn cid_bytes(resp: axum::response::Response) -> (StatusCode, Vec) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, b.to_vec()) + } + /// True if `needle` appears as a contiguous byte subsequence of `haystack`. + fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && haystack.windows(needle.len()).any(|w| w == needle) + } + fn cid_anon(cid: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap() + } + /// Anonymous CID request carrying `x-forwarded-for: ` — an anon caller with a + /// resolvable source IP, so the per-IP walk brake keys on it (the walk still + /// denies anon at a path rule). + fn cid_anon_xff(cid: &str, xff_ip: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } + fn cid_signed(kp: &gitlawb_core::identity::Keypair, cid: &str) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .body(Body::empty()) + .unwrap() + } + /// Signed CID request carrying `x-forwarded-for: `. Used by the walk + /// rate-limit test to key the per-IP limiter off a chosen source under + /// `TrustedProxy::XForwardedFor` (the request goes through `oneshot`, which + /// leaves no socket peer, so the header is the only key source). + fn cid_signed_xff( + kp: &gitlawb_core::identity::Keypair, + cid: &str, + xff_ip: &str, + ) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } + + /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. + /// RED before U2 (the current handler serves the secret to anon). + #[sqlx::test] + async fn ipfs_cid_gate_withholds_blob_from_unauthorized(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let stranger = Keypair::generate(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Request CIDs are the production pin CIDs (content-hash), recorded in + // pinned_cids so get_by_cid resolves each back to its oid (#173). + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &fx.tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "withhold")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "withhold") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("deny rule"); + + // anon → withheld blob: must 404, must not leak content. (RED on current handler.) + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "anon must not read the withheld blob" + ); + assert!( + !body.contains("TOP SECRET"), + "404 body must not leak the secret" + ); + + // signed non-reader → 404. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&stranger, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "non-reader must not read the withheld blob" + ); + assert!(!body.contains("TOP SECRET")); + + // owner (signed) → 200 + secret bytes. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "owner reads the withheld blob"); + assert!(body.contains("TOP SECRET"), "owner gets the content"); + + // listed reader (signed) → 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&reader, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); + assert!(body.contains("TOP SECRET")); + + // #135: anon tree CID under withheld /secret → 404. The 404 body is an opaque + // error string (never the object), so status is the load-bearing deny check; + // the real leak witness is the CONTRAST with the reader below, who DOES get a + // 200 carrying the child structure that anon is denied. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "withheld subtree tree must not be served to anon (#135)" + ); + + // Over-denial guard + positive leak witness: the listed reader (signed) DOES + // read the withheld subtree's tree, and its body carries the exact child + // structure anon was denied — the child filename plus the child oid as the 32 + // RAW bytes a git tree stores (witnessed on raw bytes, since cid_parts's lossy + // decode would mangle them). This proves b.txt / secret_raw are the real leak + // markers and that the anon 404 above actually withheld them. + let secret_raw = hex::decode(&fx.secret_oid).expect("hex oid"); + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_signed(&reader, &tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "listed reader reads the withheld subtree tree" + ); + assert!( + bytes_contain(&body, b"b.txt") && bytes_contain(&body, &secret_raw), + "reader's tree body carries the child filename and raw child oid" + ); + + // Root tree (path "/") stays served to anon who passes the "/" gate. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&root_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "root tree stays served (must-serve)"); + + // /public subtree tree stays served to anon (allowed path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public subtree tree stays served"); + + // Commit and annotated tag objects stay served (unchanged by #135). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "commit object stays served"); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tag_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "tag object stays served"); + + // R3: public blob anon → 200 (non-withheld content not affected). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public blob stays served"); + + // R5: a genuine unknown CID also 404, uniform with the withheld 404. A + // well-formed pin-style CID that was never recorded in pinned_cids, so the + // oid_for_cid resolve misses (the production not-found path). + let absent_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"never pinned to this node").to_string(); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&absent_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "absent CID 404 (uniform with withheld)" + ); + + // malformed CID → 400 (unchanged). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon("not-a-cid")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "malformed CID still 400"); + } + + /// R4: the same object withheld in one repo but public in another is still + /// served from the public copy; the withholding repo is iterated first. + #[sqlx::test] + async fn ipfs_cid_served_from_public_copy_when_withheld_elsewhere(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); + // Same content in both clones -> same oid/CID; read from either. + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + + // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). + let mut withhold = seed_repo(&owner_did, "withhold"); + withhold.updated_at = Utc::now(); + state + .db + .create_repo(&withhold) + .await + .expect("withhold repo"); + state + .db + .set_visibility_rule( + &withhold.id, + "/secret/**", + VisibilityMode::B, + &[], + &owner_did, + ) + .await + .expect("deny rule"); + + // Public copy, no rules, iterated AFTER. + let mut pubcopy = seed_repo(&owner_did, "pubcopy"); + pubcopy.updated_at = Utc::now() - chrono::Duration::seconds(60); + state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); + + // anon: denied at the withholding repo (continue), served from the public copy. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "served from the public copy despite the other deny" + ); + assert!( + body.contains("TOP SECRET"), + "the public copy serves the content" + ); + } + + /// Repo-level "/" gate (KTD2a, first continue branch): a fully private repo + /// (is_public=false, no rules) denies anon before any per-blob check; the + /// owner still reads. The path-scoped tests pass the "/" gate and deny at the + /// per-blob stage, so this exercises the coarser repo-level deny separately. + #[sqlx::test] + async fn ipfs_cid_private_repo_denies_anon_at_repo_gate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["priv"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("priv.git"); + let blob_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let mut rec = seed_repo(&owner_did, "priv"); + rec.is_public = false; + state.db.create_repo(&rec).await.expect("private repo"); + + // anon → repo-level deny → 404, no content leaked. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&blob_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "anon denied at a private repo's / gate" + ); + assert!(!body.contains("public bytes"), "404 must not leak content"); + + // owner-signed → 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &blob_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "owner reads their private repo's object" + ); + assert!(body.contains("public bytes"), "owner gets the content"); + } + + /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref + /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — + /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), + /// the handler skips the whole repo rather than serving. Asserts no leak of the + /// withheld blob AND that even the *public* blob in that repo is withheld — the + /// latter distinguishes fail-closed-skip from normal per-blob withholding and + /// would serve 200 if the error arm wrongly proceeded. + #[sqlx::test] + async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Recorded pins so get_by_cid resolves each CID to its oid and reaches the + // walk; the 404s below are then the fail-closed skip, not a table miss. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Force the withheld walk to fail closed: a ref pointing at a blob (not + // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` + // propagates as Err → the handler's `Ok(Err)` arm skips the repo. + std::fs::write( + bare.join("refs/heads/blobref"), + format!("{}\n", fx.secret_oid), + ) + .unwrap(); + + state + .db + .create_repo(&seed_repo(&owner_did, "withhold")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "withhold") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + // Withheld secret CID under a walk error → 404, no leak. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "walk error must not serve the withheld blob" + ); + assert!( + !body.contains("TOP SECRET"), + "walk-error 404 must not leak the secret" + ); + + // The PUBLIC blob in the same repo is also 404: the walk error fails closed + // by skipping the whole repo, not by serving. Without the fail-closed arm + // this would serve 200, so this assertion is the load-bearing discriminator. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "walk error fails closed: repo skipped, even the public blob is not served" + ); + } + + /// #173 review (F2): the commit/tag reachability walk must FAIL CLOSED on a git + /// error, exactly like the blob/tree walk. A ref pointing at a nonexistent object + /// makes `rev-list --all` fail, so `reachable_commit_tag_oids` returns Err, which + /// the handler's shared `Ok(Err) => continue` arm turns into a repo skip. The + /// load-bearing discriminator is that the PUBLIC commit is ALSO 404: if the arm + /// fail-OPENed (served on error) it would 200. Drives the commit/tag branch of + /// the shared fail-closed arm specifically (the sibling test covers blob/tree). + #[sqlx::test] + async fn ipfs_cid_commit_tag_walk_error_fails_closed(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["cterr"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("cterr.git"); + // A reachable commit CID — would serve 200 if the walk succeeded. + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + + // A ref to a NONEXISTENT object: `git rev-list --all` fails ("bad object"), + // so reachable_commit_tag_oids bails → the walk arm skips the repo. + std::fs::write( + bare.join("refs/heads/broken"), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n", + ) + .unwrap(); + + state + .db + .create_repo(&seed_repo(&owner_did, "cterr")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "cterr") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // Fail-closed: a walk error skips the repo, so even the otherwise-reachable + // public commit is 404 (not served). A fail-OPEN arm would 200 here. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a commit/tag walk error must fail closed (repo skipped), never serve" + ); + } + + /// #126: a dangling blob (written via `git hash-object -w`, never referenced + /// by any commit/tree) must 404 through `GET /ipfs/{cid}` under path-scoped + /// rules — for anon AND the owner. The pre-#126 deny-set was fail-open by + /// construction: dangling oids were absent from the reachable enumeration + /// and thus absent from the deny-set, so the handler served 200. The + /// allowed-set is fail-closed: dangling oids are absent from the reachable + /// allowed-set, so the handler 404s (per team memory: the owner shift to + /// 404 is the accepted fail-closed default — owners can still + /// `git cat-file` directly). + #[sqlx::test] + async fn ipfs_cid_dangling_blob_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + // Seed a normal repo with `secret/b.txt` reachable from HEAD, so the + // path-scoped rule has something to match — without this the rule has + // no anchor and we'd be testing nothing. + let _fx = seed_cid_repos(&slug, &short, &["dangling"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangling.git"); + + // Write a dangling blob: `git hash-object -w --stdin` adds it to the + // object DB but nothing references it, so the reachable walk never + // enumerates it. + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + let stdin = child.stdin.as_mut().expect("stdin"); + stdin.write_all(b"DANGLING SECRET\n").expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!( + out.status.success(), + "git hash-object: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + // Sanity: must be a 64-hex sha256 oid, since the repo is sha256-format. + assert_eq!( + dangling_oid.len(), + 64, + "expected sha256 oid: {dangling_oid}" + ); + // Record the pin so oid_for_cid resolves it — the 404 must then come from + // the allowed-set gate excluding the dangling oid, not from a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangling")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangling") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-blob allowed-set gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + // anon: the dangling blob is absent from the reachable allowed-set → + // 404, no leak. Pre-#126 (deny-set) would serve 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&dangling_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling blob must 404 under path-scoped rules" + ); + assert!( + !body.contains("DANGLING SECRET"), + "404 body must not leak the dangling content" + ); + + // owner (signed): same 404. The dangling blob has no path, so it's + // never visibility-checked → never in the allowed set, even for the + // owner. This is the accepted fail-closed shift documented in the PR. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &dangling_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "owner also 404s on dangling blobs under path-scoped rules (fail-closed default)" + ); + assert!(!body.contains("DANGLING SECRET")); + } + + /// #135: a DANGLING tree (in the ODB, referenced by no commit) 404s under + /// path-scoped rules for anon AND owner — the reachable-only allowed-tree-set + /// never enumerates it. Handler-level companion to the helper test + /// `allowed_tree_set_excludes_dangling_tree`, proving the `get_by_cid` tree arm + /// (memo insert + `!in_allowed` continue) fails closed on the dangling case. + #[sqlx::test] + async fn ipfs_cid_dangling_tree_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangtree"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangtree.git"); + + // Dangling tree via `git mktree`: a UNIQUE entry name so its oid is + // content-distinct from every reachable tree (a content-identical tree would + // dedup to a reachable oid — that is T2, not danglingness). + let mut child = std::process::Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("spawn git mktree"); + { + use std::io::Write; + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {}\tdangling-only-unreferenced.txt", + fx.secret_oid + ) + .unwrap(); + } + let out = child.wait_with_output().expect("mktree output"); + assert!( + out.status.success(), + "git mktree: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_tree_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + assert_eq!(dangling_tree_oid.len(), 64, "expected sha256 oid"); + // Record the pin so the 404 is the allowed-tree-set gate excluding the + // dangling tree, not a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangtree")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangtree") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for req in [cid_anon(&dangling_cid), cid_signed(&owner, &dangling_cid)] { + let (st, _) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling tree must 404 under path-scoped rules (anon + owner)" + ); + } + } + + /// #173 (F1): a QUARANTINED repo must not serve a pinned object by CID, to anon + /// OR to the mirror's own owner — quarantine is "hidden from serve/clone/listings, + /// owner included" (authorize_repo_read / feed_quarantined_mirror_withheld_from_owner). + /// The repo is PUBLIC with no path-scoped rule, so the "/" visibility gate ALLOWS + /// it and quarantine is the sole possible denier: RED before the fix (the loop + /// never checks quarantine → serves 200 + bytes), GREEN after the quarantine skip. + /// The owner-signed 404 is the load-bearing negative — a visibility-only gate + /// would Allow the owner and miss this. + #[sqlx::test] + async fn ipfs_cid_quarantined_repo_withheld_from_anon_and_owner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["quar"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quar.git"); + // Pin a ROOT-readable object (public/a.txt) — no path-scoped rule, so only + // quarantine can deny it. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "quar")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "quar") + .await + .unwrap() + .unwrap(); + + // Baseline: before quarantine the object serves 200 (proves the CID resolves + // and the object is otherwise servable, so the 404 below is quarantine's doing). + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "public root object serves before quarantine" + ); + assert!(body.contains("public bytes"), "baseline serves the content"); + + // Quarantine it. + state + .db + .set_repo_quarantine(&rec.id, true) + .await + .expect("quarantine"); + + // anon AND owner-signed must both 404 with no content leak. + for req in [cid_anon(&public_cid), cid_signed(&owner, &public_cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "quarantined repo must not serve by CID (anon + owner)" + ); + assert!( + !body.contains("public bytes"), + "404 body must not leak quarantined content" + ); + } + } + + /// #173 (F2): a DANGLING commit or annotated tag (in the ODB, referenced by no + /// ref) must 404 under path-scoped rules for anon AND owner. The resolver proved + /// reachability only for blobs/trees, so a dangling commit/tag fell through to + /// serve, leaking its message/metadata. RED before the fix (serves 200 + + /// sentinel), GREEN after (the reachable commit/tag set excludes them). The + /// reachable-commit/tag serve path is covered by + /// ipfs_cid_gate_withholds_blob_from_unauthorized (commit + annotated tag → 200). + #[sqlx::test] + async fn ipfs_cid_dangling_commit_and_tag_fail_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangct"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangct.git"); + + // Run a git plumbing command that reads from stdin and prints an oid. + let oid_from_stdin = |args: &[&str], input: &[u8]| -> String { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.as_mut().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().expect("git output"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Dangling commit: commit-tree with a sentinel message, NO ref update. + let dangling_commit_oid = oid_from_stdin( + &["commit-tree", &fx.root_tree_oid], + b"DANGLING COMMIT SECRET\n", + ); + assert_eq!(dangling_commit_oid.len(), 64, "expected sha256 commit oid"); + // Dangling annotated tag: mktag of the dangling commit, NO ref. + let tag_body = format!( + "object {dangling_commit_oid}\ntype commit\ntag dang\ntagger t 0 +0000\n\nDANGLING TAG SECRET\n" + ); + let dangling_tag_oid = oid_from_stdin(&["mktag"], tag_body.as_bytes()); + assert_eq!(dangling_tag_oid.len(), 64, "expected sha256 tag oid"); + + let commit_cid = pin_cid_for(&bare, &dangling_commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &dangling_tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangct")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangct") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-object gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for (cid, sentinel) in [ + (&commit_cid, "DANGLING COMMIT SECRET"), + (&tag_cid, "DANGLING TAG SECRET"), + ] { + for req in [cid_anon(cid), cid_signed(&owner, cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling commit/tag must 404 under path-scoped rules (anon + owner)" + ); + assert!( + !body.contains(sentinel), + "404 body must not leak the dangling message: {sentinel}" + ); + } + } + } + + /// #173 review (F2 hardening): a REACHABLE commit must still serve under a + /// path-scoped rule even when the repo carries a pushable non-commit ref (an + /// annotated tag of a tree, accepted by receive-pack). `reachable_commit_tag_oids` + /// must NOT route through `assert_all_refs_are_commits` (which bails on such a + /// ref and would fail-closed 404 every reachable commit/tag CID in the repo). + /// RED before the decoupling (the guard bails → 404), GREEN after. + #[sqlx::test] + async fn ipfs_cid_reachable_commit_served_despite_non_commit_ref(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["weirdref"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("weirdref.git"); + + // A pushable non-commit ref: an annotated tag pointing at a TREE. `git tag -a` + // in the bare repo creates refs/tags/treetag -> tag object -> tree, which + // peels to a non-commit and makes assert_all_refs_are_commits bail. + let out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &fx.root_tree_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&bare) + .output() + .expect("git tag -a"); + assert!( + out.status.success(), + "git tag -a: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Pin the REACHABLE root commit. + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "weirdref")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "weirdref") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // The reachable commit must still serve — the non-commit ref must not + // fail-closed the whole repo's commit/tag CID retrieval. + let resp = cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a reachable commit must serve despite a pushable non-commit ref in the repo" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.commit_oid.as_str()), + "the served object is the reachable root commit" + ); + } + + /// #173 review (F-F): an annotated tag pointing at a TREE is pushable through + /// receive-pack, and the TREE allowed-set path + /// (`allowed_tree_set_for_caller` -> `tree_paths` -> `reachable_commits`) runs + /// `assert_all_refs_are_commits`, which bails on that ref and fail-closes the + /// whole repo — 404-ing EVERY tree CID (root + public subtrees) for its owner + /// and readers, not just the offending tag. The tree allowed-set feeds ONLY the + /// CID gate (absence = fail-closed 404), so `tree_paths` uses the lenient + /// reachable-commit enumeration: commit-reachable trees still serve, while a + /// tree reachable only via such a tag stays excluded. `blob_paths` keeps the + /// strict guard (it also feeds serve/replication, where a miss under-withholds). + /// RED before the decoupling (whole-repo bail -> 404 on the root/public tree), + /// GREEN after; the withheld-subtree 404 is the load-bearing must-not. + #[sqlx::test] + async fn ipfs_cid_tree_served_despite_non_commit_ref(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["treeweird"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("treeweird.git"); + + // Pushable non-commit ref: an annotated tag pointing at the ROOT TREE. + let out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &fx.root_tree_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&bare) + .output() + .expect("git tag -a"); + assert!( + out.status.success(), + "git tag -a: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Pin the reachable root tree and public subtree (both at ALLOWED paths), + // plus the secret subtree (a DENIED path — the fail-closed negative). + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "treeweird")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "treeweird") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-object tree gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // Reachable trees at ALLOWED paths must still serve despite the tag-of-tree. + for (cid, want_oid, label) in [ + (&root_tree_cid, &fx.root_tree_oid, "root tree"), + (&public_tree_cid, &fx.public_tree_oid, "public subtree"), + ] { + let resp = cid_router(&state).oneshot(cid_anon(cid)).await.unwrap(); + let served = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "{label} CID must serve despite a pushable tag-of-tree in the repo" + ); + assert_eq!( + served.as_deref(), + Some(want_oid.as_str()), + "{label}: the served object is the reachable tree" + ); + } + + // Fail-closed preserved: the DENIED subtree's CID is still withheld — the + // lenient walk must not under-withhold a path the caller cannot read. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a withheld subtree's tree CID stays 404 (lenient walk must not under-withhold)" + ); + } + + /// #173 review (F2 hardening): the INNER tag object of a nested tag-of-a-tag is + /// reachable (via the outer ref tag) and pinnable, so its CID must serve under a + /// path rule. `reachable_commit_tag_oids` peels tag chains to include it. RED + /// before the peel loop (the inner tag is not a ref tip and rev-list dereferences + /// to the commit, so it is absent → 404), GREEN after. + #[sqlx::test] + async fn ipfs_cid_nested_tag_inner_object_served(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nested"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nested.git"); + + let git_stdin = |args: &[&str], input: &[u8]| -> String { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.as_mut().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().expect("git output"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Inner annotated tag of the reachable commit (no ref of its own). + let inner_body = format!( + "object {}\ntype commit\ntag inner\ntagger t 0 +0000\n\ninner\n", + fx.commit_oid + ); + let inner_tag_oid = git_stdin(&["mktag"], inner_body.as_bytes()); + // Outer annotated tag of the inner tag, then a ref to the outer tag. The + // inner tag is reachable only THROUGH the outer, not as a ref tip. + let outer_body = format!( + "object {inner_tag_oid}\ntype tag\ntag outer\ntagger t 0 +0000\n\nouter\n" + ); + let outer_tag_oid = git_stdin(&["mktag"], outer_body.as_bytes()); + let out = std::process::Command::new("git") + .args(["update-ref", "refs/tags/nested", &outer_tag_oid]) + .current_dir(&bare) + .output() + .expect("update-ref"); + assert!( + out.status.success(), + "update-ref: {}", + String::from_utf8_lossy(&out.stderr) + ); + + let inner_cid = pin_cid_for(&bare, &inner_tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "nested")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "nested") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&inner_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "the inner tag of a nested tag-of-a-tag is reachable and must serve" + ); + } + + /// #135: with NO path-scoped rule the per-object gate is skipped, so a tree CID + /// is served (the `"/"` gate is the whole story). Guards against over-gating + /// trees — the tree analog of the blob skip-walk branch. + #[sqlx::test] + async fn ipfs_cid_tree_served_when_no_path_scoped_rule(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nopathrule"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nopathrule.git"); + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // Public repo, no visibility rules → has_path_scoped_rule is false. + state + .db + .create_repo(&seed_repo(&owner_did, "nopathrule")) + .await + .expect("seed repo"); + + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "tree served to anon when no path-scoped rule exists" + ); + assert!( + bytes_contain(&body, b"b.txt"), + "served tree carries its child structure" + ); + } + + /// #173 (Fix 1): the pinned_cids lookup must use the canonical base32 CID, not + /// the raw request spelling. A pin is stored under `cid.to_string()` (canonical + /// base32); a request carrying the SAME CID re-encoded to a different multibase + /// (base58btc) parses and passes the sha2-256 check but, on the pre-fix handler, + /// misses the lookup key → false 404. Public repo, no path-scoped rule, so no + /// walk — this isolates the lookup-key canonicalization. + #[sqlx::test] + async fn ipfs_alt_encoding_cid_resolves(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["altenc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("altenc.git"); + // Canonical base32 CID as stored by the pin path. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Public repo, no visibility rules (no path-scoped walk). + state + .db + .create_repo(&seed_repo(&owner_did, "altenc")) + .await + .expect("seed repo"); + + // Re-encode the SAME CID to base58btc — a different, equally-valid spelling + // that is NOT the stored key. The `cid` crate re-exports `multibase`. + let alt = public_cid + .parse::>() + .unwrap() + .to_string_of_base(cid::multibase::Base::Base58Btc) + .unwrap(); + assert_ne!(alt, public_cid, "alt encoding must differ from canonical"); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&alt)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "alt-multibase spelling of a pinned CID must resolve (canonicalized lookup)" + ); + assert!( + body.contains("public bytes"), + "resolved object serves its content" + ); + } + + /// #173 (Fix 2a, db-level): `oids_for_cid` returns EVERY oid recorded under a + /// CID, not an arbitrary one. `record_pinned_cid` is unique on the git oid and + /// non-unique on cid, so two distinct oids can share one content-CID. Old + /// `oid_for_cid` did `LIMIT 1`; the new plural method must surface both. + #[sqlx::test] + async fn oids_for_cid_returns_all_duplicates(pool: PgPool) { + let state = test_state(pool).await; + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"shared content cid").to_string(); + let oid_a = "a".repeat(64); + let oid_b = "b".repeat(64); + state + .db + .record_pinned_cid(&oid_a, &cid, None) + .await + .unwrap(); + state + .db + .record_pinned_cid(&oid_b, &cid, None) + .await + .unwrap(); + + let mut oids = state.db.oids_for_cid(&cid).await.unwrap(); + oids.sort(); + assert_eq!( + oids, + vec![oid_a, oid_b], + "oids_for_cid must return every oid recorded under the shared CID" + ); + } + + /// #173 (Fix 2b, handler-level): when two oids collide on one CID and the + /// first-recorded is absent from every repo while the second is a readable + /// public object, the handler must try both and serve the readable one. The + /// pre-fix handler resolved a single oid (LIMIT 1 → first-inserted for equal + /// keys) and 404'd. Ordering caveat: this relies on `oids_for_cid` returning + /// the absent oid before the readable one (heap/insert order for equal keys); + /// if that ordering ever changes, `oids_for_cid_returns_all_duplicates` remains + /// the load-bearing, deterministic driver for Fix 2. + #[sqlx::test] + async fn ipfs_cid_collision_serves_readable_duplicate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["collision"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("collision.git"); + + // A GENUINE content collision: the shared CID is the readable object's REAL + // content CID, and a second (absent) oid is recorded under the SAME cid. The + // handler must try every oid and serve the one whose bytes hash to the CID. + // (F2, #173: the served bytes must match the requested content address, so the + // shared cid has to be the object's real cid — an arbitrary seed would now be + // withheld by the integrity check as an unverifiable provider-CID-style row.) + let (_ty, raw) = crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .unwrap(); + let shared_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + let absent_oid = "c".repeat(64); + state + .db + .record_pinned_cid(&absent_oid, &shared_cid, None) + .await + .expect("record absent oid first"); + state + .db + .record_pinned_cid(&fx.public_oid, &shared_cid, None) + .await + .expect("record readable oid second"); + + // Public repo, no rules → the readable public object is served if reached. + state + .db + .create_repo(&seed_repo(&owner_did, "collision")) + .await + .expect("seed repo"); - // anon → repo-level deny → 404, no content leaked. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&blob_cid)) + .oneshot(cid_anon(&shared_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "handler must try every oid under the CID and serve the readable duplicate" + ); + assert!( + body.contains("public bytes"), + "the readable duplicate's content is served" + ); + } + + /// #173 (Fix 3/F3, INV-10): the expensive legacy fan-out is rate-limited per + /// source IP. A valid tree CID makes the object-type pre-check pass, so each + /// repeat request pays a fresh walk (request-scoped memo only) — unbounded + /// amplification. Since #173-F3 (jatmn) the source charge sits on the LEGACY + /// PROBE (`acquire` + `cat-file`), which precedes the walk, so every legacy + /// candidate is charged to the non-farmable source IP from the first probe; a + /// second identical request from the same IP is shed with 429, but a targeted + /// PROVENANCE fetch (no scan) and a request from a different IP are unaffected. + /// The limiter is sized to admit one full scan of the two seeded repos (2 probes) + /// so the first request serves; the repeat then finds the bucket spent. + #[sqlx::test] + async fn ipfs_walk_rate_limited_per_source(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + + let mut state = test_state(pool).await; + // The scan probes both seeded repos (walklimit + walkpublic) per request, so + // size the per-IP budget to admit exactly one full scan (2 probes). A repeat + // scan from the same IP then finds the bucket spent. Keyed on the rightmost + // X-Forwarded-For hop so the test can choose a source IP under `oneshot`. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["walklimit"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walklimit.git"); + // The tree CID drives a path-scoped walk (the load-bearing amplification + // surface). The reader is allowed under /secret so the walk returns 200. + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // Oldest `updated_at` → `list_all_repos` (ORDER BY updated_at DESC) probes + // this serving repo LAST, so a scan deterministically charges the walk-free + // `walkpublic` miss first then this serve: exactly 2 probes per scan. + let mut walklimit = seed_repo(&owner_did, "walklimit"); + walklimit.updated_at = chrono::Utc::now() - chrono::Duration::seconds(60); + state.db.create_repo(&walklimit).await.expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "walklimit") + .await + .unwrap() + .unwrap(); + // Mode B path rule over /secret with the reader allowed → the reader's + // secret-tree fetch runs the allowed-tree walk and returns 200. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + + // The MUST-NOT object must be a genuinely CHEAP fetch: an object served + // from a repo with NO path-scoped rule takes the no-walk path, so the WALK + // brake never rate-limits it. It has to live in a repo that carries no path + // rule AND whose object graph does not overlap `walklimit` (a blob shared + // with the path-scoped repo would still walk there), so we seed a second bare + // repo with UNIQUE content. `acquire(owner, "walkpublic")` resolves to + // `/tmp//walkpublic.git`. This copy is PROVENANCED (`pin_cid_for_repo`) + // so it resolves straight to its repo and skips the legacy probe brake: the + // point here is the WALK brake, and post-#173-F3 a walk-free LEGACY fetch is + // itself source-charged at the probe, so a legacy pin would (correctly) be + // shed from the exhausted IP and no longer isolate the walk brake. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walkpublic.git"); + { + use std::process::Command; + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let src = std::env::temp_dir().join(format!("gl-cid-pub-{short}")); + let _ = std::fs::remove_dir_all(&src); + std::fs::create_dir_all(&src).unwrap(); + std::fs::write(src.join("cheap.txt"), b"cheap public bytes\n").unwrap(); + run(&["init", "-q", "--object-format=sha256"], &src); + run(&["config", "user.email", "t@t"], &src); + run(&["config", "user.name", "t"], &src); + run(&["add", "."], &src); + run(&["commit", "-qm", "cheap"], &src); + let _ = std::fs::remove_dir_all(&pub_bare); + run( + &[ + "clone", + "--bare", + "-q", + src.to_str().unwrap(), + pub_bare.to_str().unwrap(), + ], + &src, + ); + let _ = std::fs::remove_dir_all(&src); + } + let cheap_oid = { + use std::process::Command; + let out = Command::new("git") + .args(["rev-parse", "HEAD:cheap.txt"]) + .current_dir(&pub_bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse cheap.txt"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + // Public repo, NO visibility rules → the cheap object takes the no-walk path. + state + .db + .create_repo(&seed_repo(&owner_did, "walkpublic")) + .await + .expect("seed public repo"); + let pub_rec = state + .db + .get_repo(&owner_did, "walkpublic") + .await + .unwrap() + .unwrap(); + let public_cid = pin_cid_for_repo(&pub_bare, &cheap_oid, &state.db, &pub_rec.id).await; + + // 1st legacy scan from 1.2.3.4 → 200 (its two probes fit the budget; the + // walk ran, reader allowed). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "anon denied at a private repo's / gate" + StatusCode::OK, + "1st legacy scan from a source IP is served" ); - assert!(!body.contains("public bytes"), "404 must not leak content"); - // owner-signed → 200. + // 2nd identical scan from the SAME IP → 429 (per-IP probe budget spent). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "2nd legacy scan from the same source IP is shed with 429" + ); + + // MUST-NOT: a targeted PROVENANCE fetch (no scan, no probe brake) from the + // SAME limited IP, even after the 429, is served: the brake is on the legacy + // scan, not the route. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_signed(&owner, &blob_cid)) + .oneshot(cid_signed_xff(&reader, &public_cid, "1.2.3.4")) .await .unwrap(), ) @@ -2398,109 +5279,156 @@ mod tests { assert_eq!( st, StatusCode::OK, - "owner reads their private repo's object" + "a provenance (non-scan) fetch is never rate-limited, even from the exhausted IP" + ); + assert!( + body.contains("cheap public bytes"), + "the cheap fetch serves content" + ); + + // PER-SOURCE isolation: the same tree-CID scan from a DIFFERENT IP → 200. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "5.6.7.8")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "one source's exhaustion must not shed another source's walk" ); - assert!(body.contains("public bytes"), "owner gets the content"); } - /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref - /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — - /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), - /// the handler skips the whole repo rather than serving. Asserts no leak of the - /// withheld blob AND that even the *public* blob in that repo is withheld — the - /// latter distinguishes fail-closed-skip from normal per-blob withholding and - /// would serve 200 if the error arm wrongly proceeded. + /// #173 review (F-C): a SKIPPED legacy candidate (a walk-and-deny denier, OR a + /// probe-throttled repo since #173-F3) must not end the whole request: the scan + /// keeps going so a later walk-free copy still serves, and a spent probe budget is + /// a clean 429, never a false 404/503. Otherwise a public CID would 404/429 solely + /// because a newer path-scoped duplicate sorts ahead of an older no-rule copy under + /// `updated_at DESC`. Two same-oid legacy copies: a NEWER `/secret`-scoped denier + /// and an OLDER no-rule public copy. + /// + /// Two requests from the SAME IP, budget = 2 (one full scan of both copies): + /// req1 probes the denier (charged), its allowed-blob walk denies anon → skip and + /// keep scanning, then probes+serves the walk-free public copy → 200. That proves + /// the denier skip is non-fatal (`continue`, not `break`). req2 from the same IP + /// finds the probe budget spent, so the denier's probe throttles → skip-continue, + /// the public copy's probe throttles too → nothing servable → a clean 429 (not a + /// truncation 503 nor a false 404), proving the throttle is likewise non-fatal but + /// correctly shed. RED before `continue` (a `break` on the skipped denier 404s + /// req1 outright). #[sqlx::test] - async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { + async fn ipfs_walk_quota_skips_denier_and_serves_public_copy(pool: PgPool) { use crate::db::VisibilityMode; + use chrono::Utc; use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); - let state = test_state(pool).await; - - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let public_cid = cid_for_oid(&fx.public_oid); - - // Force the withheld walk to fail closed: a ref pointing at a blob (not - // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` - // propagates as Err → the handler's `Ok(Err)` arm skips the repo. - let bare = std::path::PathBuf::from("/tmp") + let mut state = test_state(pool).await; + // Budget = one full two-repo scan (2 probes), keyed on the rightmost XFF hop + // so `oneshot` can choose a source IP (no socket peer). A repeat scan from the + // same IP then finds the budget spent. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + // Identical secret-blob content in both bare clones → one CID resolves to + // `secret_oid` in each. A NEWER path-scoped denier (walk-and-deny anon) and an + // OLDER no-rule public copy (walk-free serve). + let fx = seed_cid_repos(&slug, &short, &["scopeddenier", "publiccopy"]); + let denier_bare = std::path::PathBuf::from("/tmp") .join(&slug) - .join("withhold.git"); - std::fs::write( - bare.join("refs/heads/blobref"), - format!("{}\n", fx.secret_oid), - ) - .unwrap(); - + .join("scopeddenier.git"); + let secret_cid = pin_cid_for(&denier_bare, &fx.secret_oid, &state.db).await; + + // Newer denier: public at "/", `/secret/**` Mode B empty readers → an anon + // blob fetch clears "/", runs the allowed-blob walk, is denied → continue. + let mut denier = seed_repo(&owner_did, "scopeddenier"); + denier.updated_at = Utc::now(); + state.db.create_repo(&denier).await.expect("seed denier"); state .db - .create_repo(&seed_repo(&owner_did, "withhold")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "withhold") + .set_visibility_rule(&denier.id, "/secret/**", VisibilityMode::B, &[], &owner_did) .await - .unwrap() - .unwrap(); + .expect("path rule"); + + // Older public copy — NO rule → the secret blob serves via the no-walk path. + let mut public = seed_repo(&owner_did, "publiccopy"); + public.updated_at = Utc::now() - chrono::Duration::seconds(60); state .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .create_repo(&public) .await - .expect("deny rule"); + .expect("seed public copy"); - // Withheld secret CID under a walk error → 404, no leak. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; + // req1 from 1.2.3.4: the denier is skipped (walk denies anon) and the scan + // keeps going to serve the older walk-free public copy. Both probes fit the + // budget, so this leaves the IP bucket spent. + let resp = cid_router(&state) + .oneshot(cid_anon_xff(&secret_cid, "1.2.3.4")) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error must not serve the withheld blob" + StatusCode::OK, + "a skipped walk-requiring denier must not end the scan: the later walk-free public copy still serves" ); - assert!( - !body.contains("TOP SECRET"), - "walk-error 404 must not leak the secret" + assert_eq!( + served_hash.as_deref(), + Some(fx.secret_oid.as_str()), + "the served object is the secret blob from the no-rule public copy" ); - // The PUBLIC blob in the same repo is also 404: the walk error fails closed - // by skipping the whole repo, not by serving. Without the fail-closed arm - // this would serve 200, so this assertion is the load-bearing discriminator. + // req2 from the SAME exhausted IP: every legacy probe is now throttled. The + // throttle is non-fatal (skip and keep scanning), but nothing is servable, so + // it resolves to a clean 429, not a truncation 503, not a false 404. let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&public_cid)) + .oneshot(cid_anon_xff(&secret_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error fails closed: repo skipped, even the public blob is not served" + StatusCode::TOO_MANY_REQUESTS, + "with the probe budget spent, the repeat legacy scan is shed with a clean 429" ); } - /// #126: a dangling blob (written via `git hash-object -w`, never referenced - /// by any commit/tree) must 404 through `GET /ipfs/{cid}` under path-scoped - /// rules — for anon AND the owner. The pre-#126 deny-set was fail-open by - /// construction: dangling oids were absent from the reachable enumeration - /// and thus absent from the deny-set, so the handler served 200. The - /// allowed-set is fail-closed: dangling oids are absent from the reachable - /// allowed-set, so the handler 404s (per team memory: the owner shift to - /// 404 is the accepted fail-closed default — owners can still - /// `git cat-file` directly). + /// INV-10 amplification bound: a single `GET /ipfs/{cid}` must not fan out an + /// unbounded number of full-history walks. The per-request `ipfs_rate_limiter` + /// check only brakes REPEAT requests (it fires once per request); within one + /// request the same object can exist under path-scoped rules in many repos, + /// each paying its own walk. `MAX_HISTORY_WALKS_PER_REQUEST` caps that fan-out. + /// + /// Load-bearing witness (#173, F4): a readable public copy (no path rule → + /// served via the no-walk path, exactly like + /// `ipfs_cid_served_from_public_copy_when_withheld_elsewhere`) is given the + /// OLDEST `updated_at` so `list_all_repos` (ORDER BY updated_at DESC) iterates it + /// LAST. Ahead of it sit `cap + 1` path-scoped deniers, each forcing an + /// allowed-blob walk that denies anon. The cap bounds SPAWNED walks to `cap`, but + /// hitting it must `continue` (skip only the walk-requiring denier), NOT `break` + /// the whole repo loop: the walk-free public copy needs no walk, so it is still + /// reached and served (200, `x-git-hash` = the blob oid). The old `break` + /// wrongly 404'd this publicly-readable content. Reverting `continue`→`break` + /// turns this 200 back into a 404: the RED proof that the loop keeps scanning for + /// a cheap readable copy after the cap. The `cap` walk ceiling still holds — only + /// `cap` walks are spawned across the deniers regardless (the amplification bound + /// is proven separately by `ipfs_walk_cap_still_serves_walk_free_candidate`). #[sqlx::test] - async fn ipfs_cid_dangling_blob_fails_closed_under_path_rules(pool: PgPool) { + async fn ipfs_walk_fanout_capped_per_request(pool: PgPool) { use crate::db::VisibilityMode; + use chrono::Utc; use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); @@ -2509,96 +5437,246 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - // Seed a normal repo with `secret/b.txt` reachable from HEAD, so the - // path-scoped rule has something to match — without this the rule has - // no anchor and we'd be testing nothing. - let _fx = seed_cid_repos(&slug, &short, &["dangling"]); - let bare = std::path::PathBuf::from("/tmp") - .join(&slug) - .join("dangling.git"); + let cap = crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize; - // Write a dangling blob: `git hash-object -w --stdin` adds it to the - // object DB but nothing references it, so the reachable walk never - // enumerates it. - let mut cmd = std::process::Command::new("git"); - cmd.args(["hash-object", "-w", "--stdin"]) - .current_dir(&bare) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()); - let mut child = cmd.spawn().expect("spawn git hash-object"); - { - use std::io::Write; - let stdin = child.stdin.as_mut().expect("stdin"); - stdin.write_all(b"DANGLING SECRET\n").expect("write stdin"); - } - let out = child.wait_with_output().expect("hash-object output"); - assert!( - out.status.success(), - "git hash-object: {}", - String::from_utf8_lossy(&out.stderr) - ); - let dangling_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); - // Sanity: must be a 64-hex sha256 oid, since the repo is sha256-format. - assert_eq!( - dangling_oid.len(), - 64, - "expected sha256 oid: {dangling_oid}" - ); - let dangling_cid = cid_for_oid(&dangling_oid); + // `cap + 1` deniers guarantee the fan-out crosses the ceiling before the + // readable copy (iterated last) is reached. All bare clones share identical + // content, so the one secret-BLOB CID resolves to `secret_oid` in every repo. + let denier_names: Vec = (0..=cap).map(|i| format!("denier{i}")).collect(); + let mut names: Vec<&str> = vec!["readable"]; + names.extend(denier_names.iter().map(|s| s.as_str())); + let fx = seed_cid_repos(&slug, &short, &names); + let readable_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("readable.git"); + // The secret BLOB CID drives the path-scoped allowed-blob walk in every + // denier (the amplification surface) and is served cheaply from the + // no-rule public copy — the proven serve path. + let secret_cid = pin_cid_for(&readable_bare, &fx.secret_oid, &state.db).await; + + // 1) Readable public copy — OLDEST updated_at → iterated LAST. Public with + // NO visibility rule, so the blob serves via the no-walk path. This is + // the copy an uncapped fan-out would eventually reach and serve. + let mut readable = seed_repo(&owner_did, "readable"); + readable.updated_at = Utc::now() - chrono::Duration::seconds(60); state .db - .create_repo(&seed_repo(&owner_did, "dangling")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "dangling") + .create_repo(&readable) + .await + .expect("seed readable copy"); + + // 2) cap+1 deniers with NEWER updated_at → iterated before the copy. Public + // at "/", but a `/secret/**` Mode B rule with an EMPTY reader list, so an + // anon blob fetch clears the "/" gate, runs the allowed-blob walk, and is + // denied (the secret blob is in no one's set) → continue. Each distinct + // repo.id is its own walk (the memo only dedups the same repo). + for name in &denier_names { + let mut denier = seed_repo(&owner_did, name); + denier.updated_at = Utc::now(); + state.db.create_repo(&denier).await.expect("seed denier"); + state + .db + .set_visibility_rule(&denier.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // Anon (no peer, no XFF → the IP brake is skipped, so the walk cap is the + // only thing in play). After the cap, `continue` skips only the + // walk-requiring deniers and keeps scanning, reaching the walk-free public + // copy (iterated last) → served 200. The served object is the secret blob + // from the no-rule public copy, which is legitimately public THERE. + let resp = cid_router(&state) + .oneshot(cid_anon(&secret_cid)) .await - .unwrap() .unwrap(); - // Path-scoped rule triggers the per-blob allowed-set gate (KTD4). + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "hitting the walk cap must skip only the walk-requiring candidate, not abandon the walk-free readable copy" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.secret_oid.as_str()), + "the served object is the blob from the no-rule public copy reached after the cap" + ); + } + + /// Multi-oid companion to `ipfs_walk_fanout_capped_per_request`: exercises the + /// outer oid loop and proves the per-request walk budget PERSISTS across oid + /// candidates, so a commit/tag candidate cannot re-open the fan-out. Since #173 + /// (F2) a `commit`/`tag` under a path-scoped rule is itself walk-gated (its + /// reachability is proven by a `rev-list` walk via `reachable_commit_tag_oids`), + /// so it is NOT walk-free — it draws from the same budget as the blob/tree walks. + /// + /// One CID → TWO oids (the non-unique cid index, #173): a withheld `/secret` + /// blob (walk-triggering, denied to anon in every denier) recorded FIRST so a + /// seq scan tries it first and burns the whole walk budget across the deniers; + /// the reachable root commit is second. Because the budget is already spent, the + /// commit candidate's reachability walk is also capped in every denier, so the + /// request 404s — proving commit/tag walks (F2) respect the fan-out ceiling and + /// cannot be used to bypass it (R6/F3). A reachable commit served with budget to + /// spare is covered by `ipfs_cid_gate_withholds_blob_from_unauthorized`. The + /// withheld blob must not leak. Since #173 F2 a scan the walk cap truncated + /// returns 503 (absence unproven), not the old opaque 404. + #[sqlx::test] + async fn ipfs_walk_commit_tag_candidate_respects_the_walk_cap(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let cap = crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize; + + // cap+1 path-scoped deniers, all carrying identical content (same oids). + let denier_names: Vec = (0..=cap).map(|i| format!("m{i}")).collect(); + let names: Vec<&str> = denier_names.iter().map(|s| s.as_str()).collect(); + let fx = seed_cid_repos(&slug, &short, &names); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("m0.git"); + + // ONE cid → TWO oids. The withheld blob is recorded first (seq scan lists it + // first → tried first → burns the budget); the reachable commit is second. + let multi_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; state .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .record_pinned_cid(&fx.commit_oid, &multi_cid, None) .await - .expect("deny rule"); + .expect("co-locate the commit oid under the same cid"); - // anon: the dangling blob is absent from the reachable allowed-set → - // 404, no leak. Pre-#126 (deny-set) would serve 200. + for name in &denier_names { + let mut d = seed_repo(&owner_did, name); + d.updated_at = Utc::now(); + state.db.create_repo(&d).await.expect("seed denier"); + state + .db + .set_visibility_rule(&d.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // Anon: the blob candidate is denied in every denier (a walk each, spending + // the budget); the commit candidate's reachability walk is then also capped + // in every denier — so no candidate is served AND the walk cap truncated the + // scan, leaving absence unproven → 503 (not the old false 404, #173 F2). + // Either way commit/tag walks respect the ceiling and cannot re-open the + // fan-out (R6/F3). The withheld blob must not leak in the body. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&dangling_cid)) + .oneshot(cid_anon(&multi_cid)) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "dangling blob must 404 under path-scoped rules" + StatusCode::SERVICE_UNAVAILABLE, + "a commit/tag reachability walk respects the per-request cap; a truncated scan is 503, not a false 404" ); assert!( - !body.contains("DANGLING SECRET"), - "404 body must not leak the dangling content" + !body.contains("TOP SECRET"), + "the withheld blob must not leak in the truncation response" ); + } - // owner (signed): same 404. The dangling blob has no path, so it's - // never visibility-checked → never in the allowed set, even for the - // owner. This is the accepted fail-closed shift documented in the PR. - let (st, body) = cid_parts( + /// #173 (F3, INV-15): the per-IP quota debits ONE token per expensive legacy + /// candidate, not once per request, so one IP cannot drive an unbounded fan-out. + /// With quota=1 and two path-scoped deniers holding one CID, a SINGLE request is + /// shed at 429: since #173-F3 (jatmn) each legacy PROBE (`acquire` + `cat-file`, + /// which precedes the walk) debits, so the first denier probes+walks+denies on + /// token 1 and the second denier's probe finds no token → 429. (Before F3 the + /// debit sat on the walk; the outcome is unchanged, the charge point moved earlier + /// to also bound walk-free probes.) Defeating the per-candidate debit let one IP + /// drive up to MAX_HISTORY_WALKS_PER_REQUEST × quota expensive ops/hour. + #[sqlx::test] + async fn ipfs_walk_quota_debited_per_walk(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + // Signed but NOT a reader → cleared at "/", denied at /secret → forces a walk. + let stranger = Keypair::generate(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["w0", "w1"]); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("w0.git"); + // The secret BLOB CID forces a path-scoped allowed-blob walk in each denier. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + + // Two path-scoped deniers (Mode B /secret, empty readers): each forces a + // walk that denies the signed stranger, so ONE request spawns two walks. + for name in ["w0", "w1"] { + let d = seed_repo(&owner_did, name); + state.db.create_repo(&d).await.expect("seed denier"); + state + .db + .set_visibility_rule(&d.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // ONE request, quota 1: walk 1 debits the token, walk 2 has none → 429. + let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_signed(&owner, &dangling_cid)) + .oneshot(cid_signed_xff(&stranger, &secret_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "owner also 404s on dangling blobs under path-scoped rules (fail-closed default)" + StatusCode::TOO_MANY_REQUESTS, + "the second full-history walk in one request must be shed with 429 (per-walk debit)" + ); + } + + /// The periodic cleanup task must sweep the ipfs walk limiter, not only its + /// five siblings. Drives `AppState::sweep_rate_limiters` — the exact method the + /// 300s loop calls — and asserts the ipfs limiter's expired entry is evicted. + /// Dropping `ipfs_rate_limiter.cleanup()` from that method leaves the entry in + /// place (`tracked_keys` stays 1): the RED proof that the sweep covers it. + #[sqlx::test] + async fn sweep_rate_limiters_includes_ipfs_limiter(pool: PgPool) { + let mut state = test_state(pool).await; + // Short window so a single recorded hit is already expired at sweep time. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_millis(50)); + + assert!( + state.ipfs_rate_limiter.check("1.2.3.4").await, + "record a hit on the ipfs limiter" + ); + assert_eq!( + state.ipfs_rate_limiter.tracked_keys().await, + 1, + "the source-IP key is tracked before the sweep" + ); + + // Expire the entry (still mapped — cleanup hasn't run), then sweep. + tokio::time::sleep(Duration::from_millis(60)).await; + state.sweep_rate_limiters().await; + + assert_eq!( + state.ipfs_rate_limiter.tracked_keys().await, + 0, + "the periodic sweep must evict the ipfs limiter's expired entries" ); - assert!(!body.contains("DANGLING SECRET")); } // --------------------------------------------------------------------------- diff --git a/crates/gitlawb-node/src/visibility.rs b/crates/gitlawb-node/src/visibility.rs index 56616872..a8d7ddba 100644 --- a/crates/gitlawb-node/src/visibility.rs +++ b/crates/gitlawb-node/src/visibility.rs @@ -437,6 +437,31 @@ mod tests { ); } + // #135 T1: a Mode-B rule on `/secret/**` must DENY the withheld directory's + // OWN path `/secret` (the `path == prefix` arm), not just strict descendants — + // otherwise get_by_cid's tree gate would serve the /secret tree object and leak + // its children. Pins parity with get_tree, which denies the /secret path. + #[test] + fn subtree_rule_denies_the_withheld_directory_itself() { + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", VisibilityMode::B, &[reader])]; + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/secret"), + Decision::Deny, + "anon denied at the withheld directory's OWN path /secret" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/public"), + Decision::Allow, + "anon allowed at a sibling path outside the withheld subtree" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, Some(reader), "/secret"), + Decision::Allow, + "listed reader allowed at the withheld directory (caller-aware)" + ); + } + // #153 regression: cross-method DID must still be denied even when the // trailing segment collides with a bare owner key. #[test] diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index d4cb64fc..fa7a3f3f 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -33,13 +33,16 @@ pub enum IpfsCmd { cid: String, #[arg(long, default_value = "https://node.gitlawb.com", env = "GITLAWB_NODE")] node: String, + /// Identity directory (default: ~/.gitlawb) + #[arg(long)] + dir: Option, }, } pub async fn run(args: IpfsArgs) -> Result<()> { match args.cmd { IpfsCmd::List { node, dir } => cmd_list(node, dir).await, - IpfsCmd::Get { cid, node } => cmd_get(cid, node).await, + IpfsCmd::Get { cid, node, dir } => cmd_get(cid, node, dir).await, } } @@ -86,11 +89,32 @@ async fn cmd_list(node: String, dir: Option) -> Result<()> { Ok(()) } -async fn cmd_get(cid: String, node: String) -> Result<()> { - let client = NodeClient::new(&node, None); - let path = format!("/ipfs/{cid}"); +async fn cmd_get(cid: String, node: String, dir: Option) -> Result<()> { + // #173 (F5): the resolver now serves path-scoped objects to authorized readers, + // so sign with an available identity like `gl ipfs list` — otherwise an owner or + // listed reader gets the opaque anonymous 404 for content they can read. + // `get_authed` signs when a keypair is present and falls back to unsigned. + // + // An explicit `--dir` is a request to use THAT identity: propagate a + // missing/corrupt-keystore error (like `list`) instead of silently sending an + // anonymous request the authorized reader would see as the node's opaque 404 + // (#173 review). Only the default (no `--dir`) keeps the best-effort unsigned + // fallback, so `get` stays usable for genuinely public content. + let keypair = match dir.as_deref() { + Some(dir) => Some(crate::identity::load_keypair_from_dir(Some(dir))?), + None => crate::identity::load_keypair_from_dir(None).ok(), + }; + let client = NodeClient::new(&node, keypair); + // #173 review (F1): the node now accepts equivalent multibase spellings, + // including base64 CIDs (prefix 'm'), whose alphabet contains '/', '+', '='. + // Interpolating the CID raw would make the client request (and sign) + // `/ipfs//`, which neither matches the single-segment Axum + // route nor points at the intended target. Percent-encode the CID as exactly + // one path segment so the signed and sent target agree and the server's + // `Path` extractor decodes it back to the original CID. + let path = format!("/ipfs/{}", encode_cid_segment(&cid)); let resp = client - .get(&path) + .get_authed(&path) .await .with_context(|| format!("failed to fetch CID {cid} from {node}"))?; @@ -119,6 +143,16 @@ async fn cmd_get(cid: String, node: String) -> Result<()> { Ok(()) } +/// Percent-encode a CID so it occupies exactly one path segment of `/ipfs/`. +/// `urlencoding::encode` escapes every byte outside the RFC 3986 unreserved set +/// (ALPHA / DIGIT / `-._~`), so the base64-CID characters that would otherwise +/// break the single-segment route — `/`, `+`, `=` — are all escaped, and the +/// server's `Path` extractor decodes the result back to the original CID (#173 +/// review, F1). +fn encode_cid_segment(cid: &str) -> String { + urlencoding::encode(cid).into_owned() +} + #[cfg(test)] mod tests { use super::*; @@ -235,4 +269,162 @@ mod tests { m.assert_async().await; } + + /// #173 (F5): `gl ipfs get` must SIGN with an available identity, like + /// `gl ipfs list`, so an owner/reader can retrieve a path-scoped object the node + /// now resolves by CID. RED before the fix: cmd_get ignores the identity dir and + /// sends an unsigned request, so the signature-matching mock is never hit + /// (cmd_get errors on the unmatched 501, and m.assert fails). GREEN after: the + /// signed request carries the RFC 9421 headers and is served 200. + #[tokio::test] + async fn test_cmd_get_signs_when_identity_present() { + let mut server = mockito::Server::new_async().await; + let keystore = seed_keystore(); + + let m = server + .mock("GET", "/ipfs/bafkreitestcid") + .match_header("signature", mockito::Matcher::Any) + .match_header("signature-input", mockito::Matcher::Any) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_header("x-git-hash", "abc123") + .with_body("object bytes") + .create_async() + .await; + + cmd_get( + "bafkreitestcid".to_string(), + server.url(), + Some(keystore.path().to_path_buf()), + ) + .await + .expect("signed get of a resolvable object should succeed"); + + m.assert_async().await; + } + + /// #173 (F5) must-not: a genuine anonymous denial must surface as an error, not + /// be masked as success. With no identity dir the request is unsigned; a 404 + /// from the node must produce an Err mentioning the status. + #[tokio::test] + async fn test_cmd_get_anonymous_denial_is_error() { + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreidenied") + .with_status(404) + .with_header("content-type", "text/plain") + .with_body("no git object found") + .create_async() + .await; + + let err = cmd_get("bafkreidenied".to_string(), server.url(), None) + .await + .expect_err("a 404 denial must be an error, not masked success"); + assert!( + err.to_string().contains("404"), + "error should mention the status, got: {err}" + ); + + m.assert_async().await; + } + + /// #173 (INV-8) must-not: the node's new 503 "search incomplete" (the legacy CID + /// scan hit its bound and could not prove absence) must surface as an actionable + /// Err naming the status, NOT be rendered as an empty/"not found" success — a + /// retryable outcome the caller has to see. Mirrors the 404 denial case for the + /// bounded-search response the resolver now emits. + #[tokio::test] + async fn test_cmd_get_search_incomplete_503_is_error() { + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreiincomplete") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(r#"{"error":"search_incomplete","message":"CID search incomplete — retry"}"#) + .create_async() + .await; + + let err = cmd_get("bafkreiincomplete".to_string(), server.url(), None) + .await + .expect_err("a 503 incomplete-search must be an error, not masked as not-found"); + assert!( + err.to_string().contains("503"), + "error should mention the status, got: {err}" + ); + + m.assert_async().await; + } + + /// #173 review (F1): a base64 CID (multibase prefix 'm') can contain '/', '+', + /// and '='. The client must percent-encode it into ONE path segment before + /// building and signing `/ipfs/`; otherwise the '/' splits the target so + /// it misses the single-segment Axum route and the signature covers the wrong + /// path. Assert the encoded segment carries no raw '/', '+', or '=', and that + /// it decodes back to the original CID (the server's `Path` extractor performs + /// that same decode). RED with the old raw `format!("/ipfs/{cid}")`: the + /// segment still contains '/'. + #[test] + fn test_encode_cid_segment_escapes_base64_alphabet() { + let cid = "mFoo/Bar+baz=="; + let encoded = encode_cid_segment(cid); + + assert!( + !encoded.contains('/'), + "encoded CID must be a single path segment (no raw '/'), got: {encoded}" + ); + assert!( + !encoded.contains('+'), + "encoded CID must escape '+', got: {encoded}" + ); + assert!( + !encoded.contains('='), + "encoded CID must escape '=', got: {encoded}" + ); + + let decoded = urlencoding::decode(&encoded).expect("encoded CID must decode"); + assert_eq!( + decoded, cid, + "encoding must round-trip back to the original CID" + ); + } + + /// #173 review: `gl ipfs get --dir ` must PROPAGATE a missing/corrupt + /// identity-load error like `gl ipfs list`, not silently fall back to an anonymous + /// request — otherwise an authorized reader pointing `--dir` at a broken keystore + /// gets the node's opaque 404 instead of the actionable key-load error. The + /// unsigned fallback is preserved only when NO `--dir` is given (covered by + /// `test_cmd_get_anonymous_denial_is_error`). RED before the fix (`.ok()` swallows + /// the error, an anonymous request is sent, and the `.expect(0)` mock is hit), + /// GREEN after. + #[tokio::test] + async fn test_cmd_get_explicit_dir_no_identity_errors_without_request() { + let mut server = mockito::Server::new_async().await; + // Empty keystore dir passed explicitly via --dir: no identity.pem present. + let empty = tempfile::TempDir::new().unwrap(); + + // The endpoint must never be hit when an explicit --dir fails to load. + let m = server + .mock("GET", "/ipfs/bafkreitestcid") + .expect(0) + .create_async() + .await; + + let err = cmd_get( + "bafkreitestcid".to_string(), + server.url(), + Some(empty.path().to_path_buf()), + ) + .await + .expect_err("an explicit --dir that fails to load must be an error"); + assert!( + err.to_string().contains("gl identity new") + || err.to_string().contains("no identity found") + || err.to_string().contains("failed to load keypair"), + "error should name the key-load failure, got: {err}" + ); + + m.assert_async().await; + } } From 8afc9eaf97988d9e677a8cc256d2307531d277b5 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 09:06:21 -0500 Subject: [PATCH 39/52] fix(node): close grok P1 findings on the #173/#174 IPFS integration P1-A: gate_and_serve held the /ipfs walk permit across a bare repo_store.acquire; wrap it in git_acquire_timeout_secs (mirroring #174's own handler) so a cold/hung Tigris acquire cannot pin the global walk slot. On expiry skip the repo and mark the search truncated (retryable 503, not a false 404). P1-B: the local-IPFS pin path recorded Kubo's provider Hash as pinned_cids.cid; for objects above the block size that is a dag-pb root that does not hash the raw content, so GET /ipfs/{cid} listed then 404'd them under the F2 integrity check. Record the locally-computed raw-content CID, mirroring the pinata twin. --- crates/gitlawb-node/src/api/ipfs.rs | 23 ++++++++++++++++++++--- crates/gitlawb-node/src/ipfs_pin.rs | 12 ++++++++++-- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index a764901b..fe16582b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -502,9 +502,26 @@ async fn gate_and_serve( } walk.probes += 1; } - let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { - Ok(p) => p, - Err(_) => return GateOutcome::Skip, + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this gate runs while + // the /ipfs walk permit is held (F5), so a hung or cold-Tigris acquire would otherwise + // pin the global walk slot for the whole request. On expiry skip the repo (a public + // copy may still serve) and mark the search truncated so a wholly-unserved request + // tails to a retryable 503, never a false 404 (reopened the #174 P1-2 stall vector on + // this path otherwise). + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let repo_path = match tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&repo.owner_did, &repo.name), + ) + .await + { + Ok(Ok(p)) => p, + Ok(Err(_)) => return GateOutcome::Skip, + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs gate; skipping repo"); + walk.truncated = true; + return GateOutcome::Skip; + } }; // Existence probe before any walk (random-CID spray must not trigger a walk on a diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index a4ed8e4d..bdc6e090 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -159,7 +159,15 @@ pub async fn pin_new_objects( // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinned_cid(&sha, &cid, Some(repo_id)).await { + // The resolver key (`pinned_cids.cid`) must be the locally-computed + // raw-content CID, never the provider Hash: Kubo returns a dag-pb/UnixFS + // root for objects above its block size, which does not hash the raw + // content, so `GET /ipfs/{provider_cid}` would resolve then fail the F2 + // integrity check (list-then-404). The serve path reads bytes from git and + // verifies them against the requested CID, so the raw CID is the correct + // key. Mirrors the pinata twin, which already records the raw CID. + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + if let Err(e) = db.record_pinned_cid(&sha, &raw_cid, Some(repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } // F1 (#173 round 8): also record the first pinner in pin_repo_sources so @@ -167,7 +175,7 @@ pub async fn pin_new_objects( if let Err(e) = db.record_pin_source(&sha, repo_id).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } - pinned.push((sha, cid)); + pinned.push((sha, raw_cid)); } Ok(_) => {} Err(e) => { From c767658b7b731daf00dff7572ddc287f28df911c Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 09:18:16 -0500 Subject: [PATCH 40/52] fix(node): bound the /ipfs cat-file probes under the walk permit; backfill pinata provenance P2-E: the object-type probe and the F6 size+read ran bare git cat-file inside spawn_blocking while the /ipfs walk permit was held, so a wedged cat-file could pin the global walk slot for the request's life. Bound both under git_service_timeout_secs; on timeout free the slot (truncated -> retryable 503) and skip the repo. P2-D: the pinata already-pinned branch now backfills NULL first-pinner provenance in lockstep with the ipfs_pin skip branch (consistency; the object was already resolvable via the pin_repo_sources union). --- crates/gitlawb-node/src/api/ipfs.rs | 101 ++++++++++++++++++---------- crates/gitlawb-node/src/pinata.rs | 16 +++++ 2 files changed, 82 insertions(+), 35 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index fe16582b..cb18d52a 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -530,17 +530,34 @@ async fn gate_and_serve( let obj_type = { let rp = repo_path.clone(); let sha = sha256_hex.to_string(); - match tokio::task::spawn_blocking(move || store::object_type(&rp, &sha)).await { - Ok(Ok(Some(t))) => t, - Ok(Ok(None)) => return GateOutcome::Skip, - Ok(Err(e)) => { + // Bound the blocking `git cat-file -t` under `git_service_timeout_secs`: this probe + // runs while the /ipfs walk permit is held, so a wedged cat-file (corrupt pack, NFS + // stall) would otherwise pin the global walk slot for the request's life. On timeout + // free the slot (mark truncated -> retryable 503) and skip the repo. spawn_blocking + // cannot be cancelled, so the child may linger on a blocking-pool thread, but it no + // longer holds the walk permit. + let probe_deadline = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + match tokio::time::timeout( + probe_deadline, + tokio::task::spawn_blocking(move || store::object_type(&rp, &sha)), + ) + .await + { + Ok(Ok(Ok(Some(t)))) => t, + Ok(Ok(Ok(None))) => return GateOutcome::Skip, + Ok(Ok(Err(e))) => { tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); return GateOutcome::Skip; } - Err(e) => { + Ok(Err(e)) => { tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); return GateOutcome::Skip; } + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "object-type probe timed out under the /ipfs walk permit; skipping repo"); + walk.truncated = true; + return GateOutcome::Skip; + } } }; @@ -670,30 +687,49 @@ async fn gate_and_serve( let read_sha = sha256_hex.to_string(); let read_type = obj_type.clone(); let want_cid = ctx.canonical_cid.to_string(); - let read = tokio::task::spawn_blocking(move || -> ServedRead { - match store::object_size(&read_repo, &read_sha) { - Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), - Ok(Some(_)) => {} - // git ran and reported no such object (or an unparseable size): genuine - // not-found for this candidate. - Ok(None) => return ServedRead::Gone, - // git itself failed to run: an infra failure, not a not-found. - Err(e) => return ServedRead::ReadErr(e.to_string()), + // Bound the blocking size+read+verify under `git_service_timeout_secs` (same rationale + // as the object-type probe): a hung cat-file must not pin the held /ipfs walk permit. + let read_deadline = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let read = tokio::time::timeout( + read_deadline, + tokio::task::spawn_blocking(move || -> ServedRead { + match store::object_size(&read_repo, &read_sha) { + Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(Some(_)) => {} + // git ran and reported no such object (or an unparseable size): genuine + // not-found for this candidate. + Ok(None) => return ServedRead::Gone, + // git itself failed to run: an infra failure, not a not-found. + Err(e) => return ServedRead::ReadErr(e.to_string()), + } + let content = match store::read_object_content(&read_repo, &read_sha, &read_type) { + Ok(c) => c, + Err(e) => return ServedRead::ReadErr(e.to_string()), + }; + let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); + if served != want_cid { + return ServedRead::Mismatch(served); + } + ServedRead::Ok(content) + }), + ) + .await; + let served_read = match read { + Ok(Ok(sr)) => sr, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); + walk.truncated = true; + return GateOutcome::Skip; } - let content = match store::read_object_content(&read_repo, &read_sha, &read_type) { - Ok(c) => c, - Err(e) => return ServedRead::ReadErr(e.to_string()), - }; - let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); - if served != want_cid { - return ServedRead::Mismatch(served); + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "object read timed out under the /ipfs walk permit; skipping repo"); + walk.truncated = true; + return GateOutcome::Skip; } - ServedRead::Ok(content) - }) - .await; - let content = match read { - Ok(ServedRead::Ok(c)) => c, - Ok(ServedRead::TooLarge(size)) => { + }; + let content = match served_read { + ServedRead::Ok(c) => c, + ServedRead::TooLarge(size) => { tracing::warn!( repo = %repo.name, size, max = max_bytes, "withholding object: exceeds the served-object size cap (F6)" @@ -702,15 +738,15 @@ async fn gate_and_serve( note_oversize_reject(); return GateOutcome::Skip; } - Ok(ServedRead::Mismatch(served)) => { + ServedRead::Mismatch(served) => { tracing::warn!( repo = %repo.name, requested = %ctx.canonical_cid, served = %served, "withholding object: served bytes do not hash to the requested CID (legacy provider-CID row?)" ); return GateOutcome::Skip; } - Ok(ServedRead::Gone) => return GateOutcome::Skip, - Ok(ServedRead::ReadErr(e)) => { + ServedRead::Gone => return GateOutcome::Skip, + ServedRead::ReadErr(e) => { // Infra failure (git spawn/IO), NOT a not-found: mark the search truncated so // a wholly-unserved request tails to a retryable 503, never a definitive 404 // for an authorized caller (INV-25 spirit — logging alone is not surfacing). @@ -718,11 +754,6 @@ async fn gate_and_serve( walk.truncated = true; return GateOutcome::Skip; } - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); - walk.truncated = true; - return GateOutcome::Skip; - } }; let mut resp_headers = HeaderMap::new(); resp_headers.insert( diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 31843bbc..4d6b9704 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -96,6 +96,22 @@ pub async fn pin_new_objects( for sha in object_list { match db.has_pinata_cid(&sha).await { Ok(true) => { + // Backfill NULL first-pinner provenance from a known source, in lockstep + // with the ipfs_pin skip branch: a pinata-only node otherwise leaves + // pre-provenance rows' `pinned_cids.repo_id` NULL forever (grok P2-D). The + // resolver still finds the object via the pin_repo_sources union below, so + // this is a consistency backfill, not a correctness fix. + match db.provenance_for_oid(&sha).await { + Ok(None) => { + if let Err(e) = db.backfill_pin_provenance(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); + } + } + Ok(Some(_)) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error reading pin provenance"); + } + } // F1 (#173 round 8): record this repo as an additional source for the // already-pinned object (mirrors the ipfs_pin skip-branch insert) so the // resolver can serve a shared object from any pin-path source. From 2eea354188cbeaf1e1298fe41457e2c39e656f05 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 09:27:56 -0500 Subject: [PATCH 41/52] fix(node): tie the /ipfs walk ceiling to MAX_PIN_SOURCES + 1 A provenanced object has a bounded source set (first-pinner + up to MAX_PIN_SOURCES additional). With the walk ceiling at 16 == MAX_PIN_SOURCES, an authorizing public source sorting after 16 path-scoped denials was never reached: the ceiling truncated the search and returned a false 503 for a readable object. Raise the ceiling to MAX_PIN_SOURCES + 1 so the whole bounded provenance set is always tried; the legacy scan stays bounded by MAX_LEGACY_PROBES_PER_REQUEST. --- crates/gitlawb-node/src/api/ipfs.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index cb18d52a..9cb663b2 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -50,7 +50,15 @@ use crate::visibility::{visibility_check, Decision}; /// serves on the first repo that grants them, so reaching it requires being /// denied by this many path-scoped repos first, which real traffic effectively /// never does. Tunable if that assumption stops holding. -pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = 16; +/// +/// Kept at `MAX_PIN_SOURCES + 1` so the ceiling can never truncate a request +/// BEFORE its whole bounded provenance source set (first-pinner + up to +/// `MAX_PIN_SOURCES` additional) has been tried: an authorizing public source that +/// sorts after `MAX_PIN_SOURCES` path-scoped denials must still be reached and +/// served, not falsely 503'd as a truncated search. The legacy scan's fan-out is +/// separately bounded by `MAX_LEGACY_PROBES_PER_REQUEST`, so widening this by one +/// does not loosen that path. +pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = crate::db::MAX_PIN_SOURCES as u32 + 1; /// Hard per-request ceiling on how many legacy (NULL-provenance) repositories /// the CID resolver's scan fallback may PROBE (`acquire` + `git cat-file -t`). From 94ca74426965f9bb1d72eade791fa08b3526e207 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 09:37:43 -0500 Subject: [PATCH 42/52] fix(node): align ipfs_pin return with pinata; add Retry-After to SearchIncomplete - ipfs_pin::pin_new_objects returns the provider Hash (not the raw resolver key), matching the pinata twin's contract; the DB cid stays the raw content CID. The return is logging-only here, so this is drift-avoidance, not a functional fix. - AppError::SearchIncomplete now carries Retry-After: 1 like Overloaded, so both retryable 503s from the /ipfs handler advertise the retry hint consistently. --- crates/gitlawb-node/src/error.rs | 12 ++++++++---- crates/gitlawb-node/src/ipfs_pin.rs | 6 +++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index 371f0cf6..c13b92b4 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -182,10 +182,14 @@ impl IntoResponse for AppError { })); let mut resp = (status, body).into_response(); - // Overloaded advertises when to retry. It rides the shared tail above for - // its body/status, so the header is attached here rather than in a bespoke - // early return — keeping the variant handled in exactly one place. - if matches!(self, AppError::Overloaded(_)) { + // Both retryable 503s advertise when to retry: Overloaded (capacity shed) and + // SearchIncomplete (a bounded CID search cut short by a cap — retry may complete + // it). They ride the shared tail above for body/status, so the header is attached + // here rather than in bespoke early returns, keeping each variant handled once. + if matches!( + self, + AppError::Overloaded(_) | AppError::SearchIncomplete(_) + ) { resp.headers_mut().insert( axum::http::header::RETRY_AFTER, axum::http::HeaderValue::from_static("1"), diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index bdc6e090..e70d8f7b 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -175,7 +175,11 @@ pub async fn pin_new_objects( if let Err(e) = db.record_pin_source(&sha, repo_id).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } - pinned.push((sha, raw_cid)); + // Return the provider Hash (not the resolver key), mirroring the pinata + // twin's contract: the DB `cid` is the raw resolver key (recorded above), + // the returned value is the provider CID. Here the return is consumed only + // for logging, but keeping the twins structurally identical avoids drift. + pinned.push((sha, cid)); } Ok(_) => {} Err(e) => { From d26cf6dd1d6150dad03c7c9b0e2fdfaf01914cc8 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 10:04:04 -0500 Subject: [PATCH 43/52] fix(node): close the /ipfs pin-source griefing hole via a bounded scan fallback An attacker who pins a public object from MAX_PIN_SOURCES repos before the legitimate public source registers fills pin_repo_sources (record_pin_source is first-N-wins and drops later sources silently); the buried public source is then never reached because a non-empty provenance set suppressed the legacy scan, so anon GET /ipfs/{cid} 404s forever for a public object (re-breaks F1). U1: db::pin_sources_at_cap reports whether pin_repo_sources is at MAX_PIN_SOURCES for an oid (the only observable signal that a servable source may have been dropped, since the write cap never overshoots). U2: get_by_cid falls back to the bounded legacy scan on a provenance miss when the set is empty OR at_cap. The scan gates every repo through the real per-caller gate, so it finds the buried public copy. A complete (non-full) set still fast-404s, so ordinary denials never fan out to O(repos) (INV-10/F3); the fallback honors the is_throttled peek. Regression ipfs_cid_buried_public_source_still_serves_via_scan_fallback: 404 before, 200 after; pin_sources_at_cap_flips_at_max covers the boundary. --- crates/gitlawb-node/src/api/ipfs.rs | 127 +++++++++++++----------- crates/gitlawb-node/src/db/mod.rs | 19 ++++ crates/gitlawb-node/src/test_support.rs | 123 +++++++++++++++++++++++ 3 files changed, 213 insertions(+), 56 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 9cb663b2..c8911bac 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -246,15 +246,77 @@ pub async fn get_by_cid( .pin_sources_for_oid(sha256_hex) .await .map_err(AppError::Internal)?; - if sources.is_empty() { - // F3 (#173, INV-10/INV-15): the legacy NULL-provenance scan builds an - // O(repos) preload (repos + rules + quarantine) BEFORE gate_and_serve's - // per-probe brake can bite, so a throttled source could still force O(repos) - // DB work on every replay. Peek the per-IP limiter WITHOUT consuming a token: - // an already-throttled source is shed here, before the preload runs. The + // Provenance fast-path: try each recorded source repo through the SAME gate + // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance + // pin. The first source that authorizes serves — no scan fan-out on the common + // path. + for repo_id in &sources { + let repo = match state + .db + .get_repo_by_id(repo_id) + .await + .map_err(AppError::Internal)? + { + Some(r) => r, + // A source repo is gone: skip it; a later source or the scan fallback + // below may still resolve. + None => continue, + }; + let quarantined = state + .db + .is_repo_quarantined(repo_id) + .await + .map_err(AppError::Internal)?; + let rules_map = state + .db + .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) + .await + .map_err(AppError::Internal)?; + let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); + match gate_and_serve( + &state, + &repo, + rules, + quarantined, + sha256_hex, + &rctx, + &mut walk, + false, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + GateOutcome::Throttled => { + throttled = true; + continue; + } + GateOutcome::Skip => continue, + } + } + + // Bounded legacy-scan fallback. Run it when the provenance set could not have + // served the caller AND may be INCOMPLETE: + // - empty -> a legacy NULL-provenance pin (recorded before provenance existed), or + // - at_cap -> `record_pin_source` stops inserting at MAX_PIN_SOURCES and drops + // later sources SILENTLY, so a full table may hide a servable source + // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the + // pin-source griefing hole). The scan gates every repo through the + // real per-caller gate, so it finds that copy. + // A non-empty, non-full set is COMPLETE (every recorded source was just tried), so + // skip the scan and let the tail 404 — ordinary denials never fan out to O(repos) + // (INV-10 / F3). The at_cap query runs only on a provenance MISS (we return above + // on Served), so it never costs the serve path. + let needs_scan = sources.is_empty() + || state + .db + .pin_sources_at_cap(sha256_hex) + .await + .map_err(AppError::Internal)?; + if needs_scan { + // F3 (#173, INV-10/INV-15): peek the per-IP limiter WITHOUT consuming a token + // so an already-throttled source is shed BEFORE the O(repos) preload; the // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is - // load-bearing for the across-request bound), so this adds no double-charge — - // a non-consuming peek plus the existing per-probe charge, never two charges. + // load-bearing for the across-request bound), so this adds no double-charge. if let Some(key) = crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) { @@ -263,9 +325,7 @@ pub async fn get_by_cid( continue; } } - // Legacy pin (recorded before provenance existed): fall back to the repo - // scan, gating each repo through the SAME gate. Load the scan context - // once, lazily. + // Load the scan context once, lazily (shared across oid candidates). if scan_ctx.is_none() { #[cfg(test)] bump_preload_queries(); @@ -309,51 +369,6 @@ pub async fn get_by_cid( GateOutcome::Skip => {} } } - } else { - for repo_id in &sources { - let repo = match state - .db - .get_repo_by_id(repo_id) - .await - .map_err(AppError::Internal)? - { - Some(r) => r, - // A source repo is gone: skip this source. Do NOT fall back to the - // scan (that would reopen the fan-out); a later source or oid - // candidate may still resolve. - None => continue, - }; - let quarantined = state - .db - .is_repo_quarantined(repo_id) - .await - .map_err(AppError::Internal)?; - let rules_map = state - .db - .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) - .await - .map_err(AppError::Internal)?; - let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); - match gate_and_serve( - &state, - &repo, - rules, - quarantined, - sha256_hex, - &rctx, - &mut walk, - false, // provenance path: bounded source set, no scan fan-out - ) - .await - { - GateOutcome::Served(resp) => return Ok(resp), - GateOutcome::Throttled => { - throttled = true; - continue; - } - GateOutcome::Skip => continue, - } - } } } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 6b0d394e..8d723ffd 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -2379,6 +2379,25 @@ impl Db { .collect()) } + /// Whether `pin_repo_sources` is at the `MAX_PIN_SOURCES` cap for this oid, i.e. + /// the provenance source set returned by [`Self::pin_sources_for_oid`] may be + /// INCOMPLETE. `record_pin_source` stops inserting at exactly `MAX_PIN_SOURCES` + /// rows and drops later sources silently, so a full table is the only observable + /// signal that a servable source (e.g. a later public pinner) may have been + /// dropped. `get_by_cid` uses this to decide whether a provenance miss should fall + /// back to the bounded legacy scan (which gates every repo through the real + /// visibility gate and so finds a dropped public source) rather than 404 — closing + /// the pin-source griefing hole where 16 attacker sources bury a public one. `>=` + /// (not `==`) is defensive against any future overshoot. + pub async fn pin_sources_at_cap(&self, sha256_hex: &str) -> Result { + let count: i64 = + sqlx::query_scalar("SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_one(&self.pool) + .await?; + Ok(count >= MAX_PIN_SOURCES) + } + pub async fn record_encrypted_blob( &self, repo_id: &str, diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 21fa58e4..45b2b807 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2450,6 +2450,129 @@ mod tests { ); } + /// U1 (grok round-4 P1): `pin_sources_at_cap` flips exactly at `MAX_PIN_SOURCES`. + /// It is the signal `get_by_cid` uses to decide a provenance miss may be hiding a + /// dropped servable source and must fall back to the bounded scan. + #[sqlx::test] + async fn pin_sources_at_cap_flips_at_max(pool: PgPool) { + let state = test_state(pool).await; + let cap = crate::db::MAX_PIN_SOURCES; + assert!( + !state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "an oid with no pin_repo_sources rows is not at cap" + ); + for i in 0..(cap - 1) { + state + .db + .record_pin_source("atcapoid", &format!("r-{i:02}")) + .await + .unwrap(); + } + assert!( + !state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "one below MAX_PIN_SOURCES is not at cap" + ); + state + .db + .record_pin_source("atcapoid", "r-last") + .await + .unwrap(); + assert!( + state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "exactly MAX_PIN_SOURCES rows is at cap" + ); + } + + /// U2 (grok round-4 P1, load-bearing): the pin-source GRIEFING hole. A private + /// first-pinner denies anon; an attacker fills the whole `MAX_PIN_SOURCES` source + /// window with deny-anon sources BEFORE a legitimate public repo pins the same + /// object, so the public repo's `record_pin_source` no-ops (cap full) and it is + /// buried — present in NO provenance record. The resolver's provenance set is then + /// {private + 16 attacker}, all deny anon. Because the set is at_cap (may hide a + /// dropped source), the handler falls back to the bounded legacy scan, which gates + /// every repo through the real gate and finds the buried PUBLIC copy → 200. + /// MUTATION (RED): remove the `at_cap` fallback edge in `get_by_cid` and the buried + /// public object 404s forever. + #[sqlx::test] + async fn ipfs_cid_buried_public_source_still_serves_via_scan_fallback(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["privfirst", "pubburied"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privfirst.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubburied.git"); + + // Private repo pins FIRST — owns the first-pinner provenance, denies anon. + let mut priv_repo = seed_repo(&owner_did, "privfirst"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private first-pinner"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // Attacker fills the ENTIRE MAX_PIN_SOURCES window with deny-anon (non-existent) + // sources BEFORE the public repo registers, so the cap is full. + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..cap { + state + .db + .record_pin_source(&fx.public_oid, &format!("00-attacker-{i:02}")) + .await + .expect("attacker source"); + } + + // A PUBLIC repo pushes the SAME object through the real pin path. Already pinned + // (skip branch), so it only tries record_pin_source — which NO-OPS because the + // cap is full. The public repo is thus buried: not the first-pinner, not in + // pin_repo_sources. + let pub_repo = seed_repo(&owner_did, "pubburied"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public buried source"); + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &pub_bare, + vec![fx.public_oid.clone()], + &state.db, + &pub_repo.id, + ) + .await; + m.assert_async().await; // /add NOT called (already pinned) + + // The buried public object must STILL serve: the provenance set is at_cap and + // all-deny, so the handler falls back to the bounded scan, which finds pubburied. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a public source buried by a full attacker source window must still serve via the bounded scan fallback (F1)" + ); + assert!( + body.contains("public bytes"), + "the served body is the buried public object's bytes" + ); + } + /// #173 (jatmn round 8, F1 — bound, R2): the per-object source set is capped at /// `MAX_PIN_SOURCES` so an adversary pushing one object from many repos cannot make /// resolution O(repos). Recording the same oid from `MAX_PIN_SOURCES + 3` distinct From 86828046eaa475795689c01969e63b930b7a0745 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 17:54:49 -0500 Subject: [PATCH 44/52] fix(node): hold read admission through the filtered-pack reaper The path-scoped upload-pack branch kept the read and per-caller permits as handler locals and called upload_pack_excluding with no AdmissionGuard, so a cancelled filtered clone released admission while KillGroupOnDrop was still reaping the git group, bypassing the read and per-caller concurrency caps. drive_git_child now returns the disarmed guard on success so one guard rides both build_filtered_pack stages (rev-list then pack-objects); the handler builds the guard from the permits as the plain path does. Adds a disconnect regression and an INV-22 gate row, both proven load-bearing. --- crates/gitlawb-node/src/api/repos.rs | 14 +- crates/gitlawb-node/src/git/smart_http.rs | 238 ++++++++++++++++++---- crates/gitlawb-node/tests/inv22_gates.rs | 16 ++ 3 files changed, 229 insertions(+), 39 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index cb6ed541..fb902f01 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -940,11 +940,15 @@ pub async fn git_upload_pack( smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - // upload_pack_excluding runs its own rev-list/pack-objects (both pass `None` - // admission internally); the walk's permits stay handler-locals held across - // this serve, as be0cdd6 established, and drop when the handler returns. - let _hold = (_permit, _caller_permit); - smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await + // Move both admission permits into the guard so they release only after the + // filtered serve's git group (rev-list then pack-objects) is reaped, on + // complete/timeout/disconnect — not the instant a disconnect drops this + // future. Without this, disconnect-spam on a path-scoped repo could hold PIDs + // past the concurrency cap while the permits were already freed (#174 P1-a, + // R2). The guard rides both stages inside upload_pack_excluding. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout, Some(admission)) + .await } } .map_err(|e| { diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 64d3cb7f..ec115f1e 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -70,8 +70,9 @@ pub async fn info_refs( .arg("--stateless-rpc") .arg("--advertise-refs") .arg(repo_path); - // No request body — advertise-refs does not read stdin. - let stdout = + // No request body — advertise-refs does not read stdin. Single stage: the returned + // guard is dropped here (advertise-refs holds admission for its own child only). + let (stdout, _admission) = drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; let content_type = format!("application/x-{service}-advertisement"); @@ -331,23 +332,34 @@ async fn run_git_service( .arg(service_to_command(service)) .arg("--stateless-rpc") .arg(repo_path); - drive_git_child(command, input, timeout, service, admission).await + // Single stage: drop the returned guard when this function returns (its permits are + // released once the child's group is reaped, per drive_git_child). + let (out, _admission) = drive_git_child(command, input, timeout, service, admission).await?; + Ok(out) } /// Drive a spawned git child under `timeout` with process-group teardown, returning -/// its stdout. Shared core for [`run_git_service`] and [`info_refs`]: the caller -/// passes a `Command` with its args set; this adds piped stdio and `process_group(0)`. -/// On the deadline the whole group is torn down and reaped before returning -/// [`GitServiceTimeout`]; on a dropped future (client disconnect) the -/// [`KillGroupOnDrop`] guard fires. `input` is written to the child's stdin (empty -/// for the advertise-refs path, which has no request body); `what` labels errors. +/// its stdout AND the disarmed admission guard on success. Shared core for +/// [`run_git_service`] and [`info_refs`]: the caller passes a `Command` with its args +/// set; this adds piped stdio and `process_group(0)`. On the deadline the whole group +/// is torn down and reaped before returning [`GitServiceTimeout`]; on a dropped future +/// (client disconnect) the [`KillGroupOnDrop`] guard fires. `input` is written to the +/// child's stdin (empty for the advertise-refs path, which has no request body); `what` +/// labels errors. +/// +/// On success the guard is RETURNED rather than dropped internally (#174 KTD3), so a +/// caller running two sequential git stages under one admission — `build_filtered_pack`'s +/// rev-list then pack-objects — can hand the same guard from the first stage to the +/// second and keep the permits held across both, releasing them only when the second +/// stage's process group is reaped. Callers that run a single stage just let the +/// returned guard drop. async fn drive_git_child( mut command: Command, input: Bytes, timeout: Duration, what: &str, admission: Option, -) -> Result> { +) -> Result<(Vec, Option)> { command .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -383,9 +395,9 @@ async fn drive_git_child( admission, }; // On non-unix there is no process-group teardown, so hold the admission guard here - // for the child's whole interaction; it drops when this function returns. + // for the child's whole interaction; it is returned to the caller below. #[cfg(not(unix))] - let _admission = admission; + let admission_holder = admission; let mut out = Vec::new(); let mut err = Vec::new(); @@ -419,15 +431,24 @@ async fn drive_git_child( }; let timed = tokio::time::timeout(timeout, interact).await; + // The disarmed admission guard, handed back to the caller on success so it can ride + // a following sequential git stage (#174 KTD3) instead of releasing between stages. + let admission_out: Option; let (write_result, status) = match timed { Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the // guard's drop would reap an already-reaped child / signal a reused pgid. - // Dropping the returned admission guard here releases the permits at the - // earliest provably-free point on the success path (the op finished). + // The guard is returned (not dropped) so a two-stage caller keeps the + // permits held across both stages; a single-stage caller drops it on return. #[cfg(unix)] - drop(group_guard.disarm()); + { + admission_out = group_guard.disarm(); + } + #[cfg(not(unix))] + { + admission_out = admission_holder; + } result? } Err(_elapsed) => { @@ -460,7 +481,7 @@ async fn drive_git_child( write_result.context("failed to write to git stdin")?; - Ok(out) + Ok((out, admission_out)) } fn service_to_command(service: &str) -> &str { @@ -498,14 +519,18 @@ async fn rev_list_keep( repo_path: &Path, withheld: &HashSet, timeout: Duration, -) -> Result> { + admission: Option, +) -> Result<(Vec, Option)> { let mut command = Command::new(git_bin); command .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path); - // The visibility-walk callers intentionally hold no admission permit (their - // admission is governed elsewhere), so pass `None` (#174 KTD2). - let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list", None).await?; + // First of `build_filtered_pack`'s two sequential stages: it holds the caller's + // admission for the rev-list child and hands the disarmed guard back so the same + // permits ride the pack-objects stage (#174 KTD3, R2). The inter-stage window holds + // no live git group, so nothing is reaped-late by carrying the guard between them. + let (stdout, admission) = + drive_git_child(command, Bytes::new(), timeout, "rev-list", admission).await?; let mut keep = Vec::new(); for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); @@ -514,7 +539,7 @@ async fn rev_list_keep( } keep.push(oid.to_string()); } - Ok(keep) + Ok((keep, admission)) } /// Build a packfile containing every object reachable from all refs EXCEPT the @@ -532,15 +557,21 @@ pub async fn build_filtered_pack( repo_path: &Path, withheld: &HashSet, timeout: Duration, + admission: Option, ) -> Result> { // One deadline spans both git stages so a slow rev-list eats into the pack // budget rather than granting each stage a fresh `timeout` (2x the permit hold). let deadline = Instant::now() + timeout; - let keep = rev_list_keep( + // The caller's admission rides both stages: rev-list holds it, hands it back, then + // pack-objects holds it until ITS process group is reaped on disconnect. That closes + // the path-scoped cap bypass — permits release on group reap, not the instant the + // request future drops (#174 R2, KTD3). The plain upload-pack path already does this. + let (keep, admission) = rev_list_keep( git_bin, repo_path, withheld, deadline.saturating_duration_since(Instant::now()), + admission, ) .await?; let mut data = keep.join("\n").into_bytes(); @@ -549,15 +580,15 @@ pub async fn build_filtered_pack( command .args(["pack-objects", "--stdout"]) .current_dir(repo_path); - drive_git_child( + let (out, _admission) = drive_git_child( command, Bytes::from(data), deadline.saturating_duration_since(Instant::now()), "pack-objects", - // Visibility-walk pack build: no admission permit here (#174 KTD2). - None, + admission, ) - .await + .await?; + Ok(out) } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -588,12 +619,15 @@ pub async fn upload_pack_excluding( request_body: Bytes, withheld: &HashSet, timeout: Duration, + admission: Option, ) -> Result { - // The rev-list enumeration runs blocking off the runtime; the streaming - // pack-objects stage is duration-bounded and its process group is reaped on + // Both git stages are duration-bounded and their process groups are reaped on // disconnect via drive_git_child (#174), so a hung build no longer pins its - // concurrency slot and a client disconnect no longer orphans the git child. - let pack = build_filtered_pack("git", repo_path, withheld, timeout).await?; + // concurrency slot and a client disconnect no longer orphans the git child. The + // caller's read + per-caller admission is threaded through both stages so the + // permits are held until the pack-objects group is reaped, matching the plain path + // (#174 R2). + let pack = build_filtered_pack("git", repo_path, withheld, timeout, admission).await?; // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -712,7 +746,7 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30)) + let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30), None) .await .unwrap(); let ids = pack_object_ids(&pack); @@ -776,7 +810,7 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30)) + let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30), None) .await .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); @@ -837,7 +871,7 @@ mod tests { axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30)) + upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30), None) .await .unwrap() } @@ -1359,7 +1393,7 @@ mod tests { for i in 0..FAKE_GIT_RETRY_ATTEMPTS { let result = tokio::time::timeout( Duration::from_secs(10), - build_filtered_pack(git_bin, repo_path, withheld, stage_timeout), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout, None), ) .await .expect( @@ -1655,6 +1689,142 @@ mod tests { ); } + // #174 U1 (R2, KTD3, RED-before/GREEN-after): the path-scoped filtered-pack serve + // must hold read + per-caller admission until its pack-objects process group is + // reaped on a client disconnect, exactly as the plain upload_pack path does. Before + // the fix build_filtered_pack took no AdmissionGuard and the handler's `_hold` + // permits dropped the instant the request future was dropped, so disconnect-spam on + // a path-scoped repo could hold PIDs past the concurrency cap while the permits were + // already free (#174 P1-a, on the filtered path the plain path had already closed). + // + // A real AdmissionGuard built from two owned semaphore permits rides + // rev-list -> pack-objects. We drive the future until pack-objects has forked its + // grandchild (the streaming pack-writer stand-in), assert the permits are still held + // mid-serve, then DROP the future (client disconnect) and assert the permits are + // released only AFTER the group is ESRCH-confirmed gone — never while it is alive. + // Goes RED if the guard is not threaded into the pack-objects stage (it would then + // drop after rev-list, freeing the permits mid-serve or on the bare future drop). + #[cfg(unix)] + #[tokio::test] + async fn filtered_pack_holds_admission_until_group_reaped_on_disconnect() { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("pids"); + // rev-list returns one oid fast; pack-objects forks a grandchild (the streaming + // writer stand-in), records leader+grandchild pids, then hangs so the future + // parks mid-serve with the guard owned by the pack-objects KillGroupOnDrop. The + // grandchild inherits (holds open) the stdout pipe, so drive_git_child's + // read_to_end blocks and the future stays pending until we drop it. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 &\nprintf '%s\\n%s\\n' \"$$\" \"$!\" > \"{}\"\nwait ;;\n *) exit 1 ;;\nesac\n", + pidfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let withheld = HashSet::new(); + + // Retry the fake-git spawn race like the sibling disconnect test; each attempt + // gets a FRESH semaphore so a dropped losing attempt can't skew the winning + // attempt's permit accounting. Keep the winning attempt's future PENDING so the + // drop below exercises the client-disconnect teardown. + let (fut, sem, leader, grandchild) = { + let mut attempt = 0u64; + loop { + attempt += 1; + let _ = std::fs::remove_file(&pidfile); + // Semaphore(4): two owned permits model the handler's global-read + + // per-caller admission, leaving 2 available while the op is in flight. + let sem = Arc::new(Semaphore::new(4)); + let g = sem.clone().try_acquire_owned().unwrap(); + let c = sem.clone().try_acquire_owned().unwrap(); + let admission = AdmissionGuard::new(g, Some(c)); + let mut fut = Box::pin(build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(60), + Some(admission), + )); + // Advance the future a slice at a time until the fake records its pids + // (i.e. pack-objects is running). `Ok(_)` means the future returned + // before the pidfile appeared (spawn error / early exit); stop polling + // then, since re-polling a completed future panics. + let mut pids = None; + for _ in 0..500 { + let finished = + tokio::time::timeout(Duration::from_millis(10), &mut fut).await.is_ok(); + if let Some(p) = read_two_pids(&pidfile) { + pids = Some(p); + break; + } + if finished { + break; + } + } + match pids { + Some((l, gch)) => break (fut, sem, l, gch), + None => { + // Transient spawn miss: drop the still-armed future so its guard + // reaps anything that spawned (and returns the permits), then + // back off before retrying. + drop(fut); + assert!( + attempt < FAKE_GIT_RETRY_ATTEMPTS, + "fake git failed to reach the pack-objects stage after \ + {FAKE_GIT_RETRY_ATTEMPTS} attempts (persistent failure, \ + not a transient parallel-runner miss)" + ); + tokio::time::sleep(Duration::from_millis( + FAKE_GIT_BACKOFF_STEP_MS * attempt, + )) + .await; + } + } + } + }; + let _cleanup = ReapOnPanic(vec![leader, grandchild]); + assert!(alive(grandchild), "grandchild must be running mid-serve"); + + // Mid-serve: the pack-objects stage owns the guard, so the two permits are still + // held. A build that dropped the guard after rev-list (unthreaded pack-objects + // stage) would have freed them here. + assert_eq!( + sem.available_permits(), + 2, + "admission permits must be held while the filtered serve is in flight" + ); + + // Client disconnect: drop the request future. The pack-objects KillGroupOnDrop + // must tear the group down AND hold the permits until the group is + // ESRCH-confirmed gone, releasing them only then. + drop(fut); + + let mut released_while_alive = false; + let mut released_after_reap = false; + for _ in 0..500 { + let released = sem.available_permits() == 4; + let group_alive = alive(grandchild); + if released && group_alive { + released_while_alive = true; + } + if released && !group_alive { + released_after_reap = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!( + !released_while_alive, + "admission permits were released while the git group was still alive — the \ + path-scoped concurrency-cap bypass (#174 P1-a) is open" + ); + assert!( + released_after_reap, + "admission permits must be released once the group is reaped on disconnect" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 7d3fd1c8..e750a48f 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -68,6 +68,22 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { withheld_recipients_gated, which acquires git_encrypt_semaphore" ); + // U1 / R2 (#173 round-10): the path-scoped filtered-pack serve must thread the + // caller's AdmissionGuard through BOTH git stages so read + per-caller admission is + // held until the pack-objects group is reaped on disconnect, closing the cap bypass + // the plain upload_pack path already fixed. Two load-bearing markers: rev-list hands + // the disarmed guard back (its tuple return type), and build_filtered_pack forwards + // that guard into the pack-objects stage (the `admission` arg after the + // "pack-objects" label). Reverting either — dropping the guard between stages, or + // passing `None` to pack-objects — trips this. + assert!( + smart_http.contains("Result<(Vec, Option)>") + && smart_http.contains("\"pack-objects\",\n admission,"), + "U1/R2 gate missing: build_filtered_pack must thread the AdmissionGuard through \ + rev-list -> pack-objects so the permits are held until the pack-objects group \ + is reaped on disconnect (the path-scoped half of #174 P1-a)" + ); + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere // but inside withheld_recipients_gated. A second call site (count > 1) is a new // detached git walk that skips the admission gate — exactly the class U5 closed. From 757edf560cda78f264765db410e4d09956cb9a60 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 18:34:09 -0500 Subject: [PATCH 45/52] fix(node): own /ipfs admission for the life of the walk get_by_cid held the global walk permit and the per-source permit as handler locals, so dropping the request future released both while the spawn_blocking probe, visibility walk, and object read kept running; a cancel-spam or timeout client could exceed MAX_CONCURRENT_IPFS_WALKS and the per-source cap, and the bare cat-file probe/read children had no teardown at all. Move the gated serve pipeline into a detached task that owns an AdmissionGuard, awaited by the handler, so admission releases only after the bounded work is gone; back the /ipfs probe and read with run_bounded_git twins for process-group teardown. Adds cancel-mid-walk, timeout-reap, and cancel-spam regressions plus two load-bearing INV-22 rows. Also reflows one U1 test line to satisfy cargo fmt. --- crates/gitlawb-node/src/api/ipfs.rs | 749 +++++++++++++--------- crates/gitlawb-node/src/git/smart_http.rs | 5 +- crates/gitlawb-node/src/git/store.rs | 184 +++++- crates/gitlawb-node/tests/inv22_gates.rs | 33 + 4 files changed, 661 insertions(+), 310 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index c8911bac..016e130d 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -165,7 +165,7 @@ pub async fn get_by_cid( // admits any `did:key` unthrottled, so a DID key would be free to mint around); a // `None` key (no trusted header, no peer) is bounded by the global pool only, // never the per-source sub-cap. - let _ipfs_walk_permit = state + let ipfs_walk_permit = state .git_ipfs_walk_semaphore .clone() .try_acquire_owned() @@ -174,7 +174,7 @@ pub async fn get_by_cid( AppError::Overloaded("ipfs service at capacity, retry shortly".into()) })?; let source_key = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust); - let _ipfs_caller_permit = match &source_key { + let ipfs_caller_permit = match &source_key { Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); AppError::Overloaded("ipfs service at capacity for this source, retry shortly".into()) @@ -182,219 +182,253 @@ pub async fn get_by_cid( None => None, }; - // Resolve the content-addressed CID to the object's git oid(s). A real pin - // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the - // git oid (git frames content with a `" \0"` header first), so we - // map it back through `pinned_cids` rather than treating the digest as an oid - // (#173). The cid index is non-unique, so one CID can map to several oids (a - // tree and a blob whose raw bytes collide, or content pinned under two oids); - // we try each candidate below rather than pick one arbitrarily and false-404 - // when the chosen one is withheld or absent while another is readable (#173). - // An empty result is an opaque 404, uniform with a genuine not-found and a - // visibility denial. - let oids = state - .db - .oids_for_cid(&canonical_cid) - .await - .map_err(AppError::Internal)?; - if oids.is_empty() { - return Err(AppError::RepoNotFound(format!( - "no git object found for CID {cid_str}" - ))); - } - let caller = auth.as_ref().map(|e| e.0 .0.as_str()); - let caller_owned = caller.map(|c| c.to_string()); - - // Per-request walk budget + memos + throttle flag, shared by the provenance path - // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and - // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. - let mut walk = WalkState { - walks: 0, - probes: 0, - truncated: false, - allowed_blob_memo: HashMap::new(), - allowed_tree_memo: HashMap::new(), - reachable_ct_memo: HashMap::new(), - }; - // Set when a walk-requiring candidate is skipped because the source IP's walk quota - // is spent (#173 review, F-C): the scan keeps going so a later walk-free copy still - // serves; only if nothing is servable is it turned into the 429. - let mut throttled = false; - let rctx = ResolveCtx { - caller, - caller_owned: &caller_owned, - headers: &headers, - peer, - cid_str: &cid_str, - canonical_cid: &canonical_cid, - }; + // Caller DID (owned) resolved before the spawn: the detached task below cannot + // borrow the handler's `auth` extension. + let caller_owned = auth.as_ref().map(|e| e.0 .0.as_str().to_string()); + + // #173 round-10 (R1, KTD1): move BOTH admission permits into an AdmissionGuard OWNED + // by a detached tokio task that runs the whole gated serve pipeline, and have the + // handler await its JoinHandle. Dropping the request future on a client disconnect + // drops the JoinHandle, which DETACHES the task (tokio does not abort a task when its + // handle drops) rather than cancelling it — so the pipeline runs to its bounded + // completion and releases admission only then, instead of the permits dropping the + // instant the handler future is torn down while a spawn_blocking git probe/walk/read + // is still alive (the disconnect-spam cap bypass this closes, the /ipfs half of #174 + // P1-a). Every git child on this pipeline is duration-bounded (the run_bounded_git + // probe/read twins + the bounded walk), so the detached task cannot hold admission + // past ~git_service_timeout_secs. The client key is already captured (`source_key`) + // before the spawn; the detached task has no request extractors. + let admission = + crate::git::smart_http::AdmissionGuard::new(ipfs_walk_permit, ipfs_caller_permit); + let serve: tokio::task::JoinHandle> = tokio::spawn(async move { + // The guard is held for the whole task and drops LAST — after the response is + // built — releasing admission exactly once when the pipeline is truly done. + let _admission = admission; - // Legacy scan context (repos + rules + quarantined ids), loaded LAZILY only when a - // legacy NULL-provenance pin is hit — the provenance path must never trigger the - // O(repos) load (that fan-out is exactly what provenance removes, #173 round 2). - let mut scan_ctx: Option = None; - - for sha256_hex in &oids { - // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). - // Resolve a PROVENANCED pin by trying each source repo (bounded to - // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no - // scan fan-out. A shared object first pinned from a private/quarantined repo - // still serves from a later PUBLIC source. Deterministic (ORDER BY on the - // union), so no ordering can turn an authorized copy into a 404. - let sources = state + // Resolve the content-addressed CID to the object's git oid(s). A real pin + // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the + // git oid (git frames content with a `" \0"` header first), so we + // map it back through `pinned_cids` rather than treating the digest as an oid + // (#173). The cid index is non-unique, so one CID can map to several oids (a + // tree and a blob whose raw bytes collide, or content pinned under two oids); + // we try each candidate below rather than pick one arbitrarily and false-404 + // when the chosen one is withheld or absent while another is readable (#173). + // An empty result is an opaque 404, uniform with a genuine not-found and a + // visibility denial. + let oids = state .db - .pin_sources_for_oid(sha256_hex) + .oids_for_cid(&canonical_cid) .await .map_err(AppError::Internal)?; - // Provenance fast-path: try each recorded source repo through the SAME gate - // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance - // pin. The first source that authorizes serves — no scan fan-out on the common - // path. - for repo_id in &sources { - let repo = match state - .db - .get_repo_by_id(repo_id) - .await - .map_err(AppError::Internal)? - { - Some(r) => r, - // A source repo is gone: skip it; a later source or the scan fallback - // below may still resolve. - None => continue, - }; - let quarantined = state - .db - .is_repo_quarantined(repo_id) - .await - .map_err(AppError::Internal)?; - let rules_map = state - .db - .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) - .await - .map_err(AppError::Internal)?; - let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); - match gate_and_serve( - &state, - &repo, - rules, - quarantined, - sha256_hex, - &rctx, - &mut walk, - false, - ) - .await - { - GateOutcome::Served(resp) => return Ok(resp), - GateOutcome::Throttled => { - throttled = true; - continue; - } - GateOutcome::Skip => continue, - } + if oids.is_empty() { + return Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))); } + let caller = caller_owned.as_deref(); + + // Per-request walk budget + memos + throttle flag, shared by the provenance path + // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and + // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. + let mut walk = WalkState { + walks: 0, + probes: 0, + truncated: false, + allowed_blob_memo: HashMap::new(), + allowed_tree_memo: HashMap::new(), + reachable_ct_memo: HashMap::new(), + }; + // Set when a walk-requiring candidate is skipped because the source IP's walk quota + // is spent (#173 review, F-C): the scan keeps going so a later walk-free copy still + // serves; only if nothing is servable is it turned into the 429. + let mut throttled = false; + let rctx = ResolveCtx { + caller, + caller_owned: &caller_owned, + headers: &headers, + peer, + cid_str: &cid_str, + canonical_cid: &canonical_cid, + }; + + // Legacy scan context (repos + rules + quarantined ids), loaded LAZILY only when a + // legacy NULL-provenance pin is hit — the provenance path must never trigger the + // O(repos) load (that fan-out is exactly what provenance removes, #173 round 2). + let mut scan_ctx: Option = None; - // Bounded legacy-scan fallback. Run it when the provenance set could not have - // served the caller AND may be INCOMPLETE: - // - empty -> a legacy NULL-provenance pin (recorded before provenance existed), or - // - at_cap -> `record_pin_source` stops inserting at MAX_PIN_SOURCES and drops - // later sources SILENTLY, so a full table may hide a servable source - // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the - // pin-source griefing hole). The scan gates every repo through the - // real per-caller gate, so it finds that copy. - // A non-empty, non-full set is COMPLETE (every recorded source was just tried), so - // skip the scan and let the tail 404 — ordinary denials never fan out to O(repos) - // (INV-10 / F3). The at_cap query runs only on a provenance MISS (we return above - // on Served), so it never costs the serve path. - let needs_scan = sources.is_empty() - || state + for sha256_hex in &oids { + // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). + // Resolve a PROVENANCED pin by trying each source repo (bounded to + // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no + // scan fan-out. A shared object first pinned from a private/quarantined repo + // still serves from a later PUBLIC source. Deterministic (ORDER BY on the + // union), so no ordering can turn an authorized copy into a 404. + let sources = state .db - .pin_sources_at_cap(sha256_hex) + .pin_sources_for_oid(sha256_hex) .await .map_err(AppError::Internal)?; - if needs_scan { - // F3 (#173, INV-10/INV-15): peek the per-IP limiter WITHOUT consuming a token - // so an already-throttled source is shed BEFORE the O(repos) preload; the - // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is - // load-bearing for the across-request bound), so this adds no double-charge. - if let Some(key) = - crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) - { - if state.ipfs_rate_limiter.is_throttled(&key).await { - throttled = true; - continue; - } - } - // Load the scan context once, lazily (shared across oid candidates). - if scan_ctx.is_none() { - #[cfg(test)] - bump_preload_queries(); - let repos = state + // Provenance fast-path: try each recorded source repo through the SAME gate + // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance + // pin. The first source that authorizes serves — no scan fan-out on the common + // path. + for repo_id in &sources { + let repo = match state .db - .list_all_repos() + .get_repo_by_id(repo_id) .await - .map_err(AppError::Internal)?; - let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = state + .map_err(AppError::Internal)? + { + Some(r) => r, + // A source repo is gone: skip it; a later source or the scan fallback + // below may still resolve. + None => continue, + }; + let quarantined = state .db - .list_visibility_rules_for_repos(&repo_ids) + .is_repo_quarantined(repo_id) .await .map_err(AppError::Internal)?; - let quarantined: HashSet = state + let rules_map = state .db - .list_quarantined_repos() + .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) .await - .map_err(AppError::Internal)? - .into_iter() - .map(|r| r.id) - .collect(); - scan_ctx = Some((repos, rules_by_repo, quarantined)); - } - let (repos, rules_by_repo, quarantined) = scan_ctx.as_ref().unwrap(); - for repo in repos { - let rules = rules_by_repo - .get(&repo.id) - .map(Vec::as_slice) - .unwrap_or(&[]); - let is_quar = quarantined.contains(&repo.id); + .map_err(AppError::Internal)?; + let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); match gate_and_serve( - &state, repo, rules, is_quar, sha256_hex, &rctx, &mut walk, true, + &state, + &repo, + rules, + quarantined, + sha256_hex, + &rctx, + &mut walk, + false, ) .await { GateOutcome::Served(resp) => return Ok(resp), - // A throttled walk-requiring candidate is skipped, not fatal: - // keep scanning for a later walk-free copy (#173 review, F-C). - GateOutcome::Throttled => throttled = true, - GateOutcome::Skip => {} + GateOutcome::Throttled => { + throttled = true; + continue; + } + GateOutcome::Skip => continue, + } + } + + // Bounded legacy-scan fallback. Run it when the provenance set could not have + // served the caller AND may be INCOMPLETE: + // - empty -> a legacy NULL-provenance pin (recorded before provenance existed), or + // - at_cap -> `record_pin_source` stops inserting at MAX_PIN_SOURCES and drops + // later sources SILENTLY, so a full table may hide a servable source + // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the + // pin-source griefing hole). The scan gates every repo through the + // real per-caller gate, so it finds that copy. + // A non-empty, non-full set is COMPLETE (every recorded source was just tried), so + // skip the scan and let the tail 404 — ordinary denials never fan out to O(repos) + // (INV-10 / F3). The at_cap query runs only on a provenance MISS (we return above + // on Served), so it never costs the serve path. + let needs_scan = sources.is_empty() + || state + .db + .pin_sources_at_cap(sha256_hex) + .await + .map_err(AppError::Internal)?; + if needs_scan { + // F3 (#173, INV-10/INV-15): peek the per-IP limiter WITHOUT consuming a token + // so an already-throttled source is shed BEFORE the O(repos) preload; the + // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is + // load-bearing for the across-request bound), so this adds no double-charge. + if let Some(key) = + crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) + { + if state.ipfs_rate_limiter.is_throttled(&key).await { + throttled = true; + continue; + } + } + // Load the scan context once, lazily (shared across oid candidates). + if scan_ctx.is_none() { + #[cfg(test)] + bump_preload_queries(); + let repos = state + .db + .list_all_repos() + .await + .map_err(AppError::Internal)?; + let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); + let rules_by_repo = state + .db + .list_visibility_rules_for_repos(&repo_ids) + .await + .map_err(AppError::Internal)?; + let quarantined: HashSet = state + .db + .list_quarantined_repos() + .await + .map_err(AppError::Internal)? + .into_iter() + .map(|r| r.id) + .collect(); + scan_ctx = Some((repos, rules_by_repo, quarantined)); + } + let (repos, rules_by_repo, quarantined) = scan_ctx.as_ref().unwrap(); + for repo in repos { + let rules = rules_by_repo + .get(&repo.id) + .map(Vec::as_slice) + .unwrap_or(&[]); + let is_quar = quarantined.contains(&repo.id); + match gate_and_serve( + &state, repo, rules, is_quar, sha256_hex, &rctx, &mut walk, true, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + // A throttled walk-requiring candidate is skipped, not fatal: + // keep scanning for a later walk-free copy (#173 review, F-C). + GateOutcome::Throttled => throttled = true, + GateOutcome::Skip => {} + } } } } - } - // Nothing served — three distinct tails, in precedence order: - // 1. The scan was cut short by a cap (legacy probe ceiling or walk ceiling), so - // the object was NOT proven absent/unreadable everywhere → 503, retryable, and - // explicitly NOT a definitive not-found (#173, F2). This outranks the throttle: - // an incomplete search must not masquerade as a clean rate-limit outcome, and - // it carries only the caller-supplied CID (no object/OID/metadata leak). - // 2. A walk-requiring candidate was skipped for a spent IP quota while the scan - // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). - // 3. A full scan under the caps found nothing readable → opaque 404, uniform with - // a genuine not-found and a visibility denial. - if walk.truncated { - return Err(AppError::SearchIncomplete(format!( - "CID {cid_str} search incomplete — retry" - ))); - } - if throttled { - return Err(AppError::TooManyRequests( - "ipfs retrieval rate limit exceeded — try again later".into(), - )); + // Nothing served — three distinct tails, in precedence order: + // 1. The scan was cut short by a cap (legacy probe ceiling or walk ceiling), so + // the object was NOT proven absent/unreadable everywhere → 503, retryable, and + // explicitly NOT a definitive not-found (#173, F2). This outranks the throttle: + // an incomplete search must not masquerade as a clean rate-limit outcome, and + // it carries only the caller-supplied CID (no object/OID/metadata leak). + // 2. A walk-requiring candidate was skipped for a spent IP quota while the scan + // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). + // 3. A full scan under the caps found nothing readable → opaque 404, uniform with + // a genuine not-found and a visibility denial. + if walk.truncated { + return Err(AppError::SearchIncomplete(format!( + "CID {cid_str} search incomplete — retry" + ))); + } + if throttled { + return Err(AppError::TooManyRequests( + "ipfs retrieval rate limit exceeded — try again later".into(), + )); + } + Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))) + }); + + // Await the detached serve task. Dropping THIS future (client disconnect) drops the + // JoinHandle and detaches the task, which keeps running and releases admission only + // when it completes (KTD1). A JoinError here is a panic in the pipeline (the task is + // never aborted), surfaced as a 500 rather than a silent hang. + match serve.await { + Ok(result) => result, + Err(join_err) => Err(AppError::Internal(anyhow::anyhow!( + "ipfs serve task failed: {join_err}" + ))), } - Err(AppError::RepoNotFound(format!( - "no git object found for CID {cid_str}" - ))) } /// Outcome of gating one repo for one candidate oid. @@ -553,32 +587,34 @@ async fn gate_and_serve( let obj_type = { let rp = repo_path.clone(); let sha = sha256_hex.to_string(); - // Bound the blocking `git cat-file -t` under `git_service_timeout_secs`: this probe - // runs while the /ipfs walk permit is held, so a wedged cat-file (corrupt pack, NFS - // stall) would otherwise pin the global walk slot for the request's life. On timeout - // free the slot (mark truncated -> retryable 503) and skip the repo. spawn_blocking - // cannot be cancelled, so the child may linger on a blocking-pool thread, but it no - // longer holds the walk permit. - let probe_deadline = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - match tokio::time::timeout( - probe_deadline, - tokio::task::spawn_blocking(move || store::object_type(&rp, &sha)), - ) + let git_bin = state.git_bin.clone(); + // Bound the probe CHILD itself (process-group teardown at `git_service_timeout_secs` + // via `object_type_bounded` -> `run_bounded_git`), not just an outer tokio timeout + // racing an uncancellable `spawn_blocking`: this probe runs while the /ipfs walk + // permit is held by the owning task, so a wedged cat-file (corrupt pack, NFS stall) + // must be REAPED at the deadline rather than left to linger and delay the task's + // completion — and thus admission release (#173 round-10, KTD2). No outer timeout, + // mirroring the bounded walk below; a `GitServiceTimeout` marks the search truncated + // (retryable 503). + let probe_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + match tokio::task::spawn_blocking(move || { + store::object_type_bounded(&git_bin, &rp, &sha, probe_timeout) + }) .await { - Ok(Ok(Ok(Some(t)))) => t, - Ok(Ok(Ok(None))) => return GateOutcome::Skip, - Ok(Ok(Err(e))) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); - return GateOutcome::Skip; - } + Ok(Ok(Some(t))) => t, + Ok(Ok(None)) => return GateOutcome::Skip, Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); + if e.is::() { + tracing::warn!(repo = %repo.name, "object-type probe timed out under the /ipfs walk permit; skipping repo"); + walk.truncated = true; + } else { + tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + } return GateOutcome::Skip; } - Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "object-type probe timed out under the /ipfs walk permit; skipping repo"); - walk.truncated = true; + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); return GateOutcome::Skip; } } @@ -710,45 +746,49 @@ async fn gate_and_serve( let read_sha = sha256_hex.to_string(); let read_type = obj_type.clone(); let want_cid = ctx.canonical_cid.to_string(); - // Bound the blocking size+read+verify under `git_service_timeout_secs` (same rationale - // as the object-type probe): a hung cat-file must not pin the held /ipfs walk permit. - let read_deadline = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let read = tokio::time::timeout( - read_deadline, - tokio::task::spawn_blocking(move || -> ServedRead { - match store::object_size(&read_repo, &read_sha) { - Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), - Ok(Some(_)) => {} - // git ran and reported no such object (or an unparseable size): genuine - // not-found for this candidate. - Ok(None) => return ServedRead::Gone, - // git itself failed to run: an infra failure, not a not-found. - Err(e) => return ServedRead::ReadErr(e.to_string()), - } - let content = match store::read_object_content(&read_repo, &read_sha, &read_type) { - Ok(c) => c, - Err(e) => return ServedRead::ReadErr(e.to_string()), - }; - let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); - if served != want_cid { - return ServedRead::Mismatch(served); - } - ServedRead::Ok(content) - }), - ) + let git_bin = state.git_bin.clone(); + // Bound the size+read CHILDREN themselves (process-group teardown at + // `git_service_timeout_secs` via the `*_bounded` twins), not an outer tokio timeout + // over an uncancellable `spawn_blocking`: a hung cat-file must be REAPED at the + // deadline rather than left to pin the held /ipfs walk permit (#173 round-10, KTD2). + // No outer timeout, mirroring the bounded walk; a `GitServiceTimeout` from either + // twin surfaces as `ServedRead::ReadErr` -> truncated (retryable 503). + let read_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let read = tokio::task::spawn_blocking(move || -> ServedRead { + match store::object_size_bounded(&git_bin, &read_repo, &read_sha, read_timeout) { + Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(Some(_)) => {} + // git ran and reported no such object (or an unparseable size): genuine + // not-found for this candidate. + Ok(None) => return ServedRead::Gone, + // git failed to run OR the bounded read timed out (GitServiceTimeout): an + // infra/timeout failure, not a not-found. + Err(e) => return ServedRead::ReadErr(e.to_string()), + } + let content = match store::read_object_content_bounded( + &git_bin, + &read_repo, + &read_sha, + &read_type, + read_timeout, + ) { + Ok(c) => c, + Err(e) => return ServedRead::ReadErr(e.to_string()), + }; + let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); + if served != want_cid { + return ServedRead::Mismatch(served); + } + ServedRead::Ok(content) + }) .await; let served_read = match read { - Ok(Ok(sr)) => sr, - Ok(Err(e)) => { + Ok(sr) => sr, + Err(e) => { tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); walk.truncated = true; return GateOutcome::Skip; } - Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "object read timed out under the /ipfs walk permit; skipping repo"); - walk.truncated = true; - return GateOutcome::Skip; - } }; let content = match served_read { ServedRead::Ok(c) => c, @@ -1063,38 +1103,33 @@ mod tests { drop(held); } - /// Retain-through-blocking (#174 F5, the load-bearing async property, on the - /// NEWLY-BOUNDED TREE path): the walk admission is held until the `spawn_blocking` - /// walk actually RETURNS, not when a tokio timeout fires. The requested CID - /// resolves to a TREE object under a path-scoped rule, so the gate runs - /// `allowed_tree_set_for_caller_bounded` — the walk this integration converts to - /// `run_bounded_git` — rather than the blob walk #174 already proved. With the - /// global pool at size 1, drive a request until its walk (a fake git that hangs on - /// `rev-list`) is in flight; the slot must stay held (`available_permits() == 0`) - /// and a replacement from a DIFFERENT source must shed 503 for as long as the - /// blocking walk runs — even though the request future is only `.await`ing the - /// blocking join. When the blocking walk ends the permit frees and a replacement - /// is admitted. The permit lives INSIDE the handler across the blocking `.await`; - /// move it out (drop before the walk) and the replacement would be admitted while - /// the walk still burns a blocking thread (the bug this guards). + /// Build the shared `/ipfs` TREE-walk fixture. A fake `git` whose `rev-list` records + /// its pid then sleeps ~6s (so the tree walk blocks deterministically inside + /// `run_bounded_git`) and whose `cat-file -t` answers "tree" (so the bounded + /// object-type probe, `object_type_bounded` on `state.git_bin`, routes into the + /// tree-gate arm); a real SHA-256 bare repo with a committed `src/` tree pinned WITH + /// provenance; and a path-scoped rule so the gate takes the tree-walk branch. Returns + /// the tempdir (keep it alive for the whole test), the state (the caller sets the walk + /// semaphores), the requested CID, and the rev-list pidfile path. #[cfg(unix)] - #[sqlx::test] - async fn get_by_cid_walk_permit_held_through_bounded_tree_walk(pool: sqlx::PgPool) { + async fn seed_tree_walk_fixture( + pool: sqlx::PgPool, + ) -> ( + tempfile::TempDir, + crate::state::AppState, + String, + std::path::PathBuf, + ) { use std::process::Command; let tmp = tempfile::TempDir::new().unwrap(); let revlist_pid = tmp.path().join("revlist.pid"); - // Fake git for the /ipfs TREE walk only (object_type/read_object_content use - // the real `git`, so the tree must genuinely exist below). `rev-parse` - // resolves (so the lenient enumeration appends HEAD) and `rev-list` records - // its pid then sleeps ~6s so the walk BLOCKS deterministically inside - // `run_bounded_git`. The sleep bounds the walk so a broken fix cannot wedge - // the suite. let body = format!( "#!/bin/sh\n\ case \"$1\" in\n\ for-each-ref) : ;;\n\ rev-parse) echo deadbeef ;;\n\ + cat-file) if [ \"$2\" = \"-t\" ]; then echo tree; fi ;;\n\ rev-list) echo $$ > \"{}\"; sleep 6 ;;\n\ *) : ;;\n\ esac\n\ @@ -1114,10 +1149,6 @@ mod tests { let repos_dir = tmp.path().join("repos"); std::fs::create_dir_all(&repos_dir).unwrap(); state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); - // Isolate the global walk pool at size 1; per-source cap permissive so only the - // held global permit can shed the replacement. - state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); - state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); state.git_bin = git_path.to_str().unwrap().to_string(); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; @@ -1129,10 +1160,6 @@ mod tests { .await .unwrap(); let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); - // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 - // bare repo there with a committed `src/` directory, so real - // `git cat-file -t ` classifies the requested object as a TREE and the - // handler routes into the tree-walk arm of the gate. let bare = state .repo_store .acquire(&rec.owner_did, &rec.name) @@ -1177,7 +1204,6 @@ mod tests { ], tmp.path(), ); - // The `src` directory's TREE oid — the object the request asks for. let tree_oid = { let out = Command::new("git") .args(["rev-parse", "HEAD:src"]) @@ -1187,8 +1213,6 @@ mod tests { assert!(out.status.success(), "rev-parse failed"); String::from_utf8_lossy(&out.stdout).trim().to_string() }; - // Precondition: real git classifies the object as a TREE (so the handler - // reaches the tree-walk arm, not the blob arm or an early `continue`). assert_eq!( crate::git::store::object_type(&bare, &tree_oid) .unwrap() @@ -1196,9 +1220,6 @@ mod tests { Some("tree"), "the seeded sha256 tree must exist so the handler reaches the tree walk" ); - // Pin the tree's content CID WITH provenance so the resolver targets this one - // repo (no legacy scan). A real pin CID digests the raw object content, not - // the git oid, so build it exactly as the pin path does (#173). let (_ty, raw) = crate::git::store::read_object(&bare, &tree_oid) .unwrap() .expect("tree object readable"); @@ -1208,8 +1229,6 @@ mod tests { .record_pinned_cid(&tree_oid, &cid, Some(&rec.id)) .await .unwrap(); - // A path-scoped rule so has_path_scoped_rule() is true (the tree-gate branch) - // without denying the "/" gate on the public repo. state .db .set_visibility_rule( @@ -1222,6 +1241,34 @@ mod tests { .await .unwrap(); + (tmp, state, cid, revlist_pid) + } + + /// Retain-through-blocking (#174 F5, the load-bearing async property, on the + /// NEWLY-BOUNDED TREE path): the walk admission is held until the `spawn_blocking` + /// walk actually RETURNS, not when a tokio timeout fires. The requested CID + /// resolves to a TREE object under a path-scoped rule, so the gate runs + /// `allowed_tree_set_for_caller_bounded` — the walk this integration converts to + /// `run_bounded_git` — rather than the blob walk #174 already proved. With the + /// global pool at size 1, drive a request until its walk (a fake git that hangs on + /// `rev-list`) is in flight; the slot must stay held (`available_permits() == 0`) + /// and a replacement from a DIFFERENT source must shed 503 for as long as the + /// blocking walk runs — even though the request future is only `.await`ing the + /// blocking join. When the blocking walk ends the permit frees and a replacement + /// is admitted. The permit lives INSIDE the handler across the blocking `.await`; + /// move it out (drop before the walk) and the replacement would be admitted while + /// the walk still burns a blocking thread (the bug this guards). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_permit_held_through_bounded_tree_walk(pool: sqlx::PgPool) { + let (tmp, mut state, cid, revlist_pid) = seed_tree_walk_fixture(pool).await; + // Isolate the global walk pool at size 1; per-source cap permissive so only the + // held global permit can shed the replacement. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + // Keep the fixture tempdir alive for the whole test (its Drop removes the repos). + let _tmp = tmp; + let sem = state.git_ipfs_walk_semaphore.clone(); assert_eq!( sem.available_permits(), @@ -1291,13 +1338,34 @@ mod tests { "a replacement must shed 503 while the prior request's blocking tree walk still runs" ); - // Drop the in-flight request; the detached blocking walk keeps running (a - // spawn_blocking cannot be cancelled), but the permit is a handler local, so - // dropping the future releases it once the blocking join is abandoned. Either - // way, kill the sleeping child so the slot frees promptly and poll for - // recovery — the point already proven above is that the slot stayed held for - // the duration of the blocking work. + // #173 round-10 (R1, KTD1): the NEW invariant. Drop the in-flight request. The + // gated serve pipeline runs in a DETACHED tokio task that OWNS the admission + // guard, so dropping the request future must NOT release admission — the task + // runs to its bounded completion first. Under the OLD handler-local permit the + // slot would free the instant the future dropped even though the blocking walk + // is still burning a thread (the cap bypass this closes). drop(fut); + // While the detached task is still inside the ~6s blocking tree walk, admission + // stays held. Poll a short window: a handler-local permit would already read 1. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + assert_eq!( + sem.available_permits(), + 0, + "dropping the request must NOT release admission while the detached walk \ + still runs (the spawned task owns the guard until its bounded work ends)" + ); + // A replacement from a different source STILL sheds 503 after the drop — the + // detached task holds the global slot, exactly as it did while the future lived. + let peer3: SocketAddr = "203.0.113.83:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer3)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a replacement must still shed 503 after the original request was dropped, \ + while its detached walk holds admission" + ); + // Tear the walk down; the detached task finishes and drops the guard exactly + // once, freeing the single slot back to 1 (never double-freed to 2). unsafe { libc::kill(pid, libc::SIGKILL); } @@ -1311,7 +1379,112 @@ mod tests { } assert!( freed, - "once the blocking walk ends the walk permit must free the global slot" + "once the detached walk tears down, the task drops the guard and frees the slot" + ); + assert_eq!( + sem.available_permits(), + 1, + "admission released exactly once — the single slot is back, not double-freed" + ); + } + + /// Amplification negative (#173 round-10, R1): sequential cancel-spam from ONE source + /// cannot hold more than the per-source cap of concurrent detached walk tasks. A + /// detached serve task holds its per-source permit until its bounded work finishes (up + /// to `git_service_timeout_secs`), so with a per-source cap of 1 a second request from + /// the SAME source sheds 503 even though the GLOBAL pool has room — the source cannot + /// amplify its concurrent walk children past the cap by dropping-and-retrying. (The + /// worst case: a detached task can occupy its global/per-source permit for one + /// bound-interval, so distributed cancel-spam can hold the global pool that long — the + /// accepted bounded-admission tradeoff, not a leak.) + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_cancel_spam_bounded_by_per_source_cap(pool: sqlx::PgPool) { + let (tmp, mut state, cid, revlist_pid) = seed_tree_walk_fixture(pool).await; + // Global pool has ample room (4); the per-source cap is 1. So any shed of a + // same-source replacement is the PER-SOURCE cap, never global exhaustion. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(4)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let _tmp = tmp; + + let sem = state.git_ipfs_walk_semaphore.clone(); + let per_caller = state.git_ipfs_walk_per_caller.clone(); + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + // Source S fires request 1; drive until its tree walk is in flight (the task now + // holds source S's single per-source permit). + let source_s: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(source_s))); + let mut walk_pid: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + } + let pid = walk_pid.expect("the fake git rev-list must have spawned"); + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Cancel-spam: drop request 1's future. Its detached task keeps running the walk + // and KEEPS holding source S's single per-source permit. + drop(fut); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + // A SECOND request from source S sheds 503. The global pool still has room (only 1 + // of 4 taken), so this is the per-source cap, not global exhaustion. + let resp = router.clone().oneshot(make_req(source_s)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a same-source cancel-spam replacement must shed 503 on the per-source cap \ + while the detached task still holds the source's permit" + ); + assert!( + sem.available_permits() >= 3, + "the shed was the per-source cap, not global exhaustion (global pool still has room)" + ); + assert_eq!( + per_caller.tracked_keys(), + 1, + "exactly one per-source permit is outstanding for the one source — no amplification" + ); + + // Tear the detached walk down; the task completes and releases source S's permit + // (tracked_keys returns to 0), so the source is no longer over the cap. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut released = false; + for _ in 0..400 { + if per_caller.tracked_keys() == 0 { + released = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + released, + "once the detached task tears down it releases source S's per-source permit" ); } diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index ec115f1e..50145652 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -1752,8 +1752,9 @@ mod tests { // then, since re-polling a completed future panics. let mut pids = None; for _ in 0..500 { - let finished = - tokio::time::timeout(Duration::from_millis(10), &mut fut).await.is_ok(); + let finished = tokio::time::timeout(Duration::from_millis(10), &mut fut) + .await + .is_ok(); if let Some(p) = read_two_pids(&pidfile) { pids = Some(p); break; diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 25f11246..a79d596a 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -290,26 +290,6 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> )) } -/// Object size in bytes (`git cat-file -s`) WITHOUT reading the content, so an -/// oversized object can be rejected before it is buffered into memory (#173, F6). -/// `None` if the object does not exist or the size is unparseable. -pub fn object_size(repo_path: &Path, sha256_hex: &str) -> Result> { - // allow-unbounded-git: cheap cat-file -s header read (no content), holds no served-git - // permit and cannot hang; exact twin of object_type above. Not an INV-22 lifecycle op. - let out = Command::new("git") - .args(["cat-file", "-s", sha256_hex]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file -s")?; - if !out.status.success() { - return Ok(None); - } - Ok(String::from_utf8_lossy(&out.stdout) - .trim() - .parse::() - .ok()) -} - /// Read an object's content if its type is already known. pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) -> Result> { let content_output = Command::new("git") @@ -326,6 +306,90 @@ pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) - Ok(content_output.stdout) } +/// Bounded twin of [`object_type`] for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2). Runs `git cat-file -t` under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so the child runs +/// in its own process group and a watchdog reaps it (SIGTERM -> grace -> SIGKILL) at +/// `timeout`. The bare [`object_type`] is a `spawn_blocking` `Command::output` that an +/// async timeout cannot cancel, so a wedged `cat-file` there pins the caller's held +/// /ipfs walk admission for the whole hang; this twin cannot. Semantics mirror +/// [`object_type`]: `Ok(Some(t))` for an existing object, `Ok(None)` when git exits +/// non-zero without timing out (the object is absent), and +/// `Err(`[`GitServiceTimeout`](crate::git::smart_http::GitServiceTimeout)`)` on the +/// deadline so the handler can mark the search truncated (retryable 503) rather than a +/// false not-found. Callers off the /ipfs path keep the bare helper. +pub fn object_type_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + match crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-t", sha256_hex], + repo_path, + b"", + deadline, + ) { + Ok(out) => Ok(Some(String::from_utf8_lossy(&out).trim().to_string())), + Err(e) if e.is::() => Err(e), + // A non-timeout failure is git reporting no such object (a non-zero exit), + // matching `object_type`'s `Ok(None)`. + Err(_) => Ok(None), + } +} + +/// Bounded `git cat-file -s` size read for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2): reads the object size WITHOUT its content (so an oversized object +/// is rejected before it is buffered, #173 F6), under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so a wedged size +/// read is reaped at `timeout` instead of pinning the held /ipfs walk admission. +/// `Ok(Some(n))` on success, `Ok(None)` when the object is absent (a non-timeout +/// non-zero exit), `Err(GitServiceTimeout)` on the deadline. +pub fn object_size_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + match crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-s", sha256_hex], + repo_path, + b"", + deadline, + ) { + Ok(out) => Ok(String::from_utf8_lossy(&out).trim().parse::().ok()), + Err(e) if e.is::() => Err(e), + Err(_) => Ok(None), + } +} + +/// Bounded twin of [`read_object_content`] for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2): `git cat-file ` under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so a wedged content +/// read is reaped at `timeout` instead of pinning the held /ipfs walk admission. Returns +/// the raw object bytes on success and an error (including `GitServiceTimeout` on the +/// deadline) otherwise, mirroring [`read_object_content`]. +pub fn read_object_content_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + obj_type: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", obj_type, sha256_hex], + repo_path, + b"", + deadline, + ) +} + /// Read a git object by its SHA-256 hex object ID. /// /// Returns `(object_type, content_bytes)` where `content_bytes` is the raw @@ -521,4 +585,84 @@ mod tests { "unchanged file must not appear: {names:?}" ); } + + /// #173 round-10 (KTD2): `object_type_bounded` reaps a wedged `cat-file` child at its + /// deadline instead of blocking on it to natural exit, so a hung probe cannot pin the + /// /ipfs walk admission the owning task holds. A fake `git` records its pid and sleeps + /// far past the 1s deadline; the `run_bounded_git` watchdog (SIGTERM -> grace -> + /// SIGKILL of the process group) must kill it well before that natural exit, and the + /// call must surface `GitServiceTimeout`. REVERT PROOF (RED): swap the twin's + /// `run_bounded_git` for the bare `Command::output()` and the wedged child stays alive + /// past the deadline — the mid-flight liveness poll below reads it still running. + #[cfg(unix)] + #[test] + fn object_type_bounded_reaps_wedged_child_at_deadline() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("catfile.pid"); + // `cat-file` records its own pid then sleeps 8s (>> the 1s deadline) so the probe + // is genuinely wedged; the watchdog is what must end it. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + cat-file) echo $$ > \"{}\"; sleep 8 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pidfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let repo = tmp.path().to_path_buf(); + let git = git_path.to_str().unwrap().to_string(); + + let alive = |pid: i32| unsafe { libc::kill(pid, 0) == 0 }; + + // The bounded probe blocks until the watchdog tears the child down, so run it on + // a worker thread and poll for the reap from here. + let handle = std::thread::spawn(move || { + super::object_type_bounded(&git, &repo, "deadbeef", Duration::from_secs(1)) + }); + + let mut pid = None; + for _ in 0..500 { + if let Some(p) = std::fs::read_to_string(&pidfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + pid = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid = pid.expect("the fake cat-file must have spawned and recorded its pid"); + + // Past the 1s deadline + SIGTERM grace but well before the 8s natural exit: the + // watchdog must already have reaped the wedged group. A bare, unbounded read would + // leave it running here — the load-bearing RED. + std::thread::sleep(Duration::from_secs(3)); + let reaped = !alive(pid); + // Defensive reap so a RED run leaks no orphan. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + reaped, + "object_type_bounded must reap the wedged cat-file child at the deadline, \ + not leave it running to its natural exit" + ); + + let res = handle.join().expect("probe thread joins"); + let err = res.expect_err("a deadline overrun must be an error, not a value"); + assert!( + err.is::(), + "a deadline overrun must surface GitServiceTimeout, got: {err:?}" + ); + } } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index e750a48f..5ec653fc 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -29,6 +29,7 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { let repos = src("api/repos.rs"); let smart_http = src("git/smart_http.rs"); let vis = src("git/visibility_pack.rs"); + let ipfs = src("api/ipfs.rs"); // U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the // child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise @@ -84,6 +85,38 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { is reaped on disconnect (the path-scoped half of #174 P1-a)" ); + // U2 / R1 (#173 round-10): the `GET /ipfs/{cid}` serve pipeline must run in a + // DETACHED tokio task that OWNS the AdmissionGuard built from BOTH walk permits, so a + // cancelled or timed-out request releases admission only after the spawned (bounded) + // work completes — not the instant the handler future drops (the /ipfs half of #174 + // P1-a). Two load-bearing markers: the guard is constructed from the two named + // permits, and the whole pipeline is moved into a `tokio::spawn`. Reverting to + // handler-local permits (dropping the guard/spawn) trips this. + assert!( + ipfs.contains("AdmissionGuard::new(ipfs_walk_permit, ipfs_caller_permit)") + && ipfs.contains( + "let serve: tokio::task::JoinHandle> = tokio::spawn(async move {" + ), + "U2/R1 gate missing: get_by_cid must move both /ipfs admission permits into an \ + AdmissionGuard owned by a detached tokio::spawn task so admission is released \ + only after the spawned serve work completes (the /ipfs half of #174 P1-a)" + ); + + // U2 / KTD2 (#173 round-10): the probe/read children on the /ipfs path must be the + // duration-bounded twins (process-group teardown via run_bounded_git), not the bare + // `store::object_type` / `read_object_content` (or an unbounded `cat-file -s`) a tokio + // timeout cannot cancel — otherwise a wedged cat-file lingers and pins the held + // admission past the deadline. Reverting any twin call site back to a bare read trips + // this. + assert!( + ipfs.contains("object_type_bounded(") + && ipfs.contains("object_size_bounded(") + && ipfs.contains("read_object_content_bounded("), + "U2/KTD2 gate missing: the /ipfs probe+read must call the run_bounded_git-backed \ + *_bounded twins so a wedged cat-file is reaped at the deadline, not left to pin \ + the held /ipfs walk admission" + ); + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere // but inside withheld_recipients_gated. A second call site (count > 1) is a new // detached git walk that skips the admission gate — exactly the class U5 closed. From 3351e09c082085bf33a0adfb382336d572fa0b0f Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 19:04:55 -0500 Subject: [PATCH 46/52] fix(node): requeue coalesced pushes instead of dropping them Per-repo coalescing dropped every push that arrived while a repo's encryption task was in flight, and no reconciliation ever processed it, so a withheld blob added by the coalesced push had its recovery copy permanently absent. The drop also silently skipped the coalesced push's local-IPFS pin work. EncryptInflight becomes a dirty-flag map; the detached task loops, and the atomic check-and-clear sits at the task tail, unconditional on the has_path_scoped_rule gate and walk success (a public or rules-free repo would otherwise exit before it ran). Each requeue re-reads repo state fresh and re-enumerates the pin half through the fail-closed full scan, never bare list_all_objects, so a coalesced rule change is honored and no withheld or dangling object leaks. At-most-one-task-per-repo is preserved. --- crates/gitlawb-node/src/api/repos.rs | 531 +++++++++++++++++++----- crates/gitlawb-node/src/state.rs | 124 ++++-- crates/gitlawb-node/src/test_support.rs | 351 ++++++++++++++++ 3 files changed, 857 insertions(+), 149 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index fb902f01..c9a2b3dd 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -790,6 +790,269 @@ async fn withheld_recipients_gated( .await } +/// Everything the detached post-push replication task needs that does not change +/// between requeue passes. Cloned once from `AppState` at the spawn site so the task +/// is self-contained (the handler keeps no copy). +struct PostPushReplication { + db: std::sync::Arc, + disk_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, + ipfs_api: String, + repo_id: String, + encrypt_sem: std::sync::Arc, + node_seed: [u8; 32], + node_did: String, + repo_name: String, + irys_url: String, + http_client: std::sync::Arc, +} + +/// The requeue enumeration for the pin half: a fail-closed FULL scan of the current +/// object DB under the REFRESHED rules. The coalesced push's ref tips are gone by the +/// time we requeue, so the delta path is unavailable; the whole-repo scan is the safe +/// superset. Never pins the bare `list_all_objects` output — that includes +/// dangling/withheld blobs — it feeds it as CANDIDATES to the same fail-closed filter +/// the push path's full-scan branch uses, which drops dangling and visibility-withheld +/// blobs before anything is pinned. +async fn requeue_full_scan_object_list( + disk_path: &std::path::Path, + git_bin: &str, + timeout: std::time::Duration, + rules: Vec, + is_public: bool, + owner_did: String, +) -> Vec { + let disk = disk_path.to_path_buf(); + let gb = git_bin.to_string(); + let candidates = tokio::task::spawn_blocking(move || { + crate::git::push_delta::list_all_objects(&disk, &gb, std::time::Instant::now() + timeout) + }) + .await + .ok() + .and_then(|r| { + r.map_err(|e| { + tracing::warn!(err = %e, "requeue full-scan enumeration failed; pinning nothing this pass") + }) + .ok() + }) + .unwrap_or_default(); + fail_closed_full_scan_objects( + disk_path.to_path_buf(), + rules, + is_public, + owner_did, + candidates, + git_bin.to_string(), + timeout, + ) + .await +} + +/// The detached post-push encryption + local-IPFS pin task, as a REQUEUE LOOP. +/// +/// Pass one uses the spawn-time captures (`first_*`) — the delta the push handler +/// already computed. At the TASK TAIL (unconditional on the encrypt gate below and on +/// walk success) it consults the coalescing dirty flag: if a push coalesced during the +/// window it re-reads repo state (rules, is_public, owner_did, withheld) FRESH from the +/// DB, re-enumerates the pin set fail-closed, and runs another pass — so the coalesced +/// push is covered before the task exits. Otherwise it releases the key and returns. +/// +/// The tail placement is load-bearing: the encrypt+anchor block only runs under a +/// path-scoped rule, so a check-and-clear placed inside that gate would never run for a +/// public/rules-free repo or a failed walk, dropping exactly the pin-half push this +/// requeue must cover. +#[allow(clippy::too_many_arguments)] +async fn run_post_push_replication( + mut guard: crate::state::EncryptInflightGuard, + ctx: PostPushReplication, + first_object_list: Vec, + first_rules: Option>, + first_is_public: bool, + first_owner_did: String, + first_withheld: std::collections::HashSet, +) { + let mut object_list = first_object_list; + let mut rules_opt = first_rules; + let mut is_public = first_is_public; + let mut owner_did = first_owner_did; + // The task only spawns when `withheld.is_some()`, so pass one always replicates. + let mut withheld: Option> = Some(first_withheld); + + loop { + if withheld.is_some() { + // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). + crate::ipfs_pin::pin_new_objects( + &ctx.ipfs_api, + &ctx.disk_path, + object_list.clone(), + &ctx.db, + &ctx.repo_id, + ) + .await; + + // Option B1: encrypt-then-pin the withheld blobs. No path-scoped rule can + // withhold a blob, so a rules-free repo has nothing to seal; skip. Mirrors + // the has_path_scoped_rule gate on the other two withheld-walk sites. + if let Some(rules) = rules_opt + .clone() + .filter(|r| visibility_pack::has_path_scoped_rule(r)) + { + let recip = withheld_recipients_gated( + ctx.encrypt_sem.clone(), + ctx.disk_path.clone(), + ctx.git_bin.clone(), + ctx.timeout, + rules, + is_public, + owner_did.clone(), + ) + .await; + if let Ok(Ok(recipients)) = recip { + let delta = crate::encrypted_pin::encrypt_and_pin( + &ctx.ipfs_api, + &ctx.disk_path, + &ctx.db, + &ctx.repo_id, + &ctx.node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the sealed blobs to + // Arweave. Best-effort; never fails the push. + if !delta.is_empty() && !ctx.irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&owner_did); + let repo_slug = format!("{owner_short}/{}", ctx.repo_name); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &owner_did, + node_did: &ctx.node_did, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &ctx.http_client, + &ctx.irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } + } + } + } + + // TASK TAIL — unconditional check-and-clear, atomic with the release decision. + if !guard.requeue_or_release() { + break; + } + + // A push coalesced during this pass. Re-read repo state FRESH (never the stale + // spawn-time captures) so a coalesced push that changed `.gitlawb` withholding + // is walked under the new policy, then re-enumerate the pin set fail-closed. + let (r_rules, r_is_public, r_owner) = match ctx.db.get_repo_by_id(&ctx.repo_id).await { + Ok(Some(rec)) => ( + ctx.db.list_visibility_rules(&ctx.repo_id).await.ok(), + rec.is_public, + rec.owner_did, + ), + Ok(None) => { + tracing::debug!(repo = %ctx.repo_id, "repo gone before requeue pass; releasing"); + (None, false, String::new()) + } + Err(e) => { + tracing::warn!(repo = %ctx.repo_id, err = %e, "requeue repo re-read failed; skipping this pass's work"); + (None, false, String::new()) + } + }; + let (_announce, r_withheld) = replication_withheld_set( + r_rules.clone(), + &r_owner, + r_is_public, + ctx.disk_path.clone(), + ctx.git_bin.clone(), + ctx.timeout, + ) + .await; + object_list = match &r_withheld { + Some(_) => { + requeue_full_scan_object_list( + &ctx.disk_path, + &ctx.git_bin, + ctx.timeout, + r_rules.clone().unwrap_or_default(), + r_is_public, + r_owner.clone(), + ) + .await + } + None => Vec::new(), + }; + rules_opt = r_rules; + is_public = r_is_public; + owner_did = r_owner; + withheld = r_withheld; + } +} + +/// Test-only entry point: build the `PostPushReplication` context from a test +/// `AppState` (with an overridable `ipfs_api` for a mock Kubo server and an explicit +/// `disk_path` for the fixture repo) and run the requeue loop. Keeps +/// `PostPushReplication` and `run_post_push_replication` private to this module. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub(crate) async fn run_post_push_replication_for_test( + state: &AppState, + guard: crate::state::EncryptInflightGuard, + disk_path: std::path::PathBuf, + repo_id: String, + ipfs_api: String, + is_public: bool, + owner_did: String, + object_list: Vec, + rules: Option>, + withheld: std::collections::HashSet, +) { + let ctx = PostPushReplication { + db: state.db.clone(), + disk_path: disk_path.clone(), + git_bin: state.git_bin.clone(), + timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + ipfs_api, + repo_id, + encrypt_sem: state.git_encrypt_semaphore.clone(), + node_seed: *state.node_keypair.to_seed(), + node_did: state.node_did.to_string(), + repo_name: String::new(), + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + }; + run_post_push_replication( + guard, + ctx, + object_list, + rules, + is_public, + owner_did, + withheld, + ) + .await; +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1434,128 +1697,44 @@ pub async fn git_receive_pack( // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). // Skipped entirely when the public cannot read the repo (withheld == None). // - // Coalesce per repo (#174 P2-2): this task parks on `git_encrypt_semaphore` - // (which DEFERS when the pool is full rather than dropping the recovery copy). To - // bound the OUTSTANDING parked-task set, only spawn if no encryption task for this - // repo is already in flight; otherwise skip — the pending/next walk over this - // repo's history already covers the newer push's objects, so the recovery copy is - // delayed, never lost. The guard removes the repo key when the task ends (success, - // error, or panic), so a later push is re-admitted (no permanent skip). - if withheld.is_some() { + // Coalesce per repo (#174 P2-2): only spawn a task if none is in flight for this + // repo; otherwise `try_begin` marks the repo DIRTY and the in-flight task requeues + // one more pass (re-reading fresh repo state) before it exits. This bounds the + // outstanding parked-task set to one per repo while covering every coalesced push — + // there is no reconciliation sweep, so a dropped job would be lost forever (#173 F3). + if let Some(withheld_set) = withheld.clone() { match state.encrypt_inflight.try_begin(&record.id) { None => { tracing::debug!( repo = %record.id, "post-push encryption task already in flight for this repo; coalescing \ - (the pending recovery-copy walk covers this push's objects)" + (the in-flight task will requeue one more pass to cover this push)" ); } Some(inflight_guard) => { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - let enc_git_bin = state.git_bin.clone(); - let enc_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let encrypt_sem = state.git_encrypt_semaphore.clone(); - tokio::spawn(async move { - // Held for the whole task; drop (on completion/error/panic) releases the - // repo's coalescing key so the next push for this repo can spawn again. - let _inflight_guard = inflight_guard; - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - &repo_id, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } - } - - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = - rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - // Bound the number of concurrent post-push encryption walks (#174 P1-e): - // acquire an admission permit before the full-history walk, deferring - // when the pool is full rather than shedding the recovery pin. - let recip = withheld_recipients_gated( - encrypt_sem.clone(), - repo_path_clone.clone(), - enc_git_bin.clone(), - enc_timeout, - rules, - is_public, - owner_did.clone(), - ) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( - &ipfs_api, - &repo_path_clone, - &db_clone, - &repo_id, - &node_seed, - &recipients, - ) - .await; - - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, - ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), - } - } - } - } - }); + let ctx = PostPushReplication { + db: state.db.clone(), + disk_path: disk_path.clone(), + git_bin: state.git_bin.clone(), + timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + ipfs_api: state.config.ipfs_api.clone(), + repo_id: record.id.clone(), + encrypt_sem: state.git_encrypt_semaphore.clone(), + node_seed: *state.node_keypair.to_seed(), + node_did: state.node_did.to_string(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + }; + tokio::spawn(run_post_push_replication( + inflight_guard, + ctx, + object_list.clone(), + rules_opt.clone(), + record.is_public, + record.owner_did.clone(), + withheld_set, + )); } } } @@ -4573,6 +4752,130 @@ mod tests { ); } + // ---- U3 (#173 F3): dirty-flag requeue seam (the mechanics behind the end-to-end + // requeue tests in test_support.rs) ---- + + /// A coalesced push MARKS the repo dirty (not just "skip"), and the in-flight task's + /// tail check-and-clear then runs ONE more pass before releasing: `requeue_or_release` + /// returns `true` (loop) while dirty, clearing the flag, then `false` (release) when + /// clean. This is the mechanism that makes a coalesced push requeued, not dropped. + #[test] + fn u3_try_begin_marks_dirty_then_requeue_loops_once_then_releases() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkU3DirtyAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/proj"; + + let mut guard = inflight.try_begin(repo).expect("first push admits"); + assert_eq!(inflight.dirty(repo), Some(false), "fresh task starts clean"); + + // A push coalesces during the in-flight window: marked dirty, no new task. + assert!( + inflight.try_begin(repo).is_none(), + "coalesced push does not spawn" + ); + assert_eq!( + inflight.dirty(repo), + Some(true), + "the coalesced push marked the repo dirty" + ); + + // Task tail, pass 1: dirty -> requeue (clear the flag, keep the key, loop). + assert!( + guard.requeue_or_release(), + "dirty repo requeues one more pass" + ); + assert_eq!( + inflight.dirty(repo), + Some(false), + "the dirty flag is cleared for the next pass" + ); + assert_eq!( + inflight.len(), + 1, + "the key is still held while the task loops" + ); + + // Task tail, pass 2: clean -> release (remove the key, exit). + assert!( + !guard.requeue_or_release(), + "a clean repo releases and exits" + ); + assert_eq!(inflight.len(), 0, "the key is removed on the clean release"); + assert_eq!( + inflight.dirty(repo), + None, + "no in-flight entry after release" + ); + } + + /// The check-and-clear is ATOMIC with the release decision, so no push lands in a + /// "checked clean but still present" gap (scenario 6, race gap). A push that arrives + /// BEFORE the tail check sets dirty -> the same pass requeues it. A push that arrives + /// AFTER a clean release finds an empty set -> `try_begin` spawns a fresh task. Both + /// directions covered; neither drops the push. + #[test] + fn u3_requeue_or_release_leaves_no_uncovered_gap() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkU3GapBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB/proj"; + + // Case A: push arrives before the tail check -> requeue covers it. + let mut guard = inflight.try_begin(repo).expect("admit"); + assert!( + inflight.try_begin(repo).is_none(), + "push arrives during the window" + ); + assert!( + guard.requeue_or_release(), + "the pre-check push is covered by a requeue" + ); + // Now clean: the task releases and exits. + assert!(!guard.requeue_or_release(), "clean -> release"); + assert_eq!(inflight.len(), 0); + + // Case B: a push arriving after release starts a brand-new task (not dropped). + let guard2 = inflight.try_begin(repo); + assert!( + guard2.is_some(), + "a post-release push spawns a fresh task, never dropped" + ); + } + + /// Drop is a PANIC BACKSTOP only. A task that panics before releasing still frees the + /// key (so a crashed walk never permanently locks the repo out); a task that releases + /// via `requeue_or_release` and then drops does not double-free. + #[test] + fn u3_drop_is_panic_backstop_for_unreleased_guard() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkU3PanicCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC/proj"; + + // Normal release then drop: key already gone, drop is a no-op. + let mut g = inflight.try_begin(repo).expect("admit"); + assert!(!g.requeue_or_release(), "clean release"); + assert_eq!(inflight.len(), 0); + drop(g); + assert_eq!( + inflight.len(), + 0, + "dropping an already-released guard does not resurrect a key" + ); + + // Panic before releasing: Drop-on-unwind frees the key. + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _g = inflight.try_begin(repo).expect("admit before panic"); + assert_eq!(inflight.len(), 1); + panic!("task panics before reaching its tail check-and-clear"); + })); + assert!(panicked.is_err(), "the simulated task panicked"); + assert_eq!( + inflight.len(), + 0, + "a panic before release still frees the key (backstop), so the repo is not locked out" + ); + assert!( + inflight.try_begin(repo).is_some(), + "the repo can be admitted again after a panic" + ); + } + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. /// Returns the count of tasks spawned (== the size of the unbounded outstanding set /// the fix prevents), used as the RED comparison in the bound test above. diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 10dbbf32..61bbec6a 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -253,24 +253,29 @@ impl AppState { /// repo spawn N parked tasks, each holding cloned object lists/rules/paths/keys — an /// unbounded outstanding set. /// -/// This tracks the repo keys with an in-flight encryption task. Before spawning, the -/// handler calls [`try_begin`](Self::try_begin): if a task for the repo is already -/// in-flight it returns `None` and the handler SKIPS spawning a duplicate (coalesce — -/// the newer push's objects are covered by the pending/next walk over the same repo's -/// history). This bounds the outstanding set to <=1 pending task per repo WITHOUT -/// dropping work: coalescing only delays a duplicate walk, it never sheds the recovery -/// copy (there is no reconciliation sweep, so a *dropped* job would be lost forever). +/// This tracks the repo keys with an in-flight encryption task, each carrying a +/// DIRTY flag. Before spawning, the handler calls [`try_begin`](Self::try_begin): if +/// a task for the repo is already in-flight it MARKS THE REPO DIRTY and returns `None` +/// so the handler SKIPS spawning a duplicate (coalesce). The in-flight task consults +/// the flag at its tail via [`requeue_or_release`](EncryptInflightGuard::requeue_or_release): +/// a set flag makes it run ONE MORE pass (re-reading repo state) before it exits, so a +/// push coalesced during the in-flight window is REQUEUED, never dropped. This bounds +/// the outstanding set to <=1 task per repo without losing work: there is no +/// reconciliation sweep, so a *dropped* job would be lost forever. /// -/// The returned [`EncryptInflightGuard`] is moved into the detached task and removes -/// the repo key on drop — on normal completion, error, OR panic (Drop runs on unwind) -/// — so one crashed walk can never permanently lock a repo out of future recovery -/// copies. +/// The returned [`EncryptInflightGuard`] is moved into the detached task. The normal +/// exit path is `requeue_or_release`, which removes the repo key ATOMICALLY with the +/// "clean" decision — no push can land in the gap between "checked clean" and "task +/// exits". `Drop` is a panic backstop only: if the task panics (or returns without +/// calling `requeue_or_release`) it still removes the key so a crashed walk can never +/// permanently lock a repo out of future recovery copies. #[derive(Clone, Default)] pub struct EncryptInflight { - // std::sync::Mutex: only ever held for O(1) HashSet insert/remove in a sync - // context (right before `tokio::spawn`, and in the guard's Drop) — never across - // an await, so a std Mutex is correct and cheaper than a tokio one. - repos: Arc>>, + // std::sync::Mutex: only ever held for O(1) HashMap ops in a sync context (before + // `tokio::spawn`, at the task tail, and in Drop) — never across an await, so a std + // Mutex is correct and cheaper than a tokio one. The value is the DIRTY flag: a + // coalesced push flips it true so the in-flight task requeues one more pass. + repos: Arc>>, } impl EncryptInflight { @@ -279,18 +284,24 @@ impl EncryptInflight { } /// Try to begin an encryption task for `repo_id`. Returns `Some(guard)` if no task - /// for the repo was in-flight (the caller should spawn), or `None` if one already - /// is (the caller should COALESCE — skip spawning a duplicate). The guard releases - /// the repo key on drop. + /// for the repo was in-flight (the caller should spawn). If one already is, MARKS + /// the repo dirty and returns `None` (the caller COALESCES — skips the duplicate + /// spawn; the in-flight task requeues one more pass to cover this push). pub fn try_begin(&self, repo_id: &str) -> Option { - let mut set = self.repos.lock().expect("encrypt_inflight mutex poisoned"); - if set.insert(repo_id.to_string()) { - Some(EncryptInflightGuard { - repos: Arc::clone(&self.repos), - repo_id: repo_id.to_string(), - }) - } else { - None + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(repo_id) { + Some(dirty) => { + *dirty = true; + None + } + None => { + map.insert(repo_id.to_string(), false); + Some(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + released: false, + }) + } } } @@ -309,23 +320,66 @@ impl EncryptInflight { pub fn is_empty(&self) -> bool { self.len() == 0 } + + /// Test-only: read the dirty flag for `repo_id` (`None` if no task is in-flight). + #[cfg(test)] + pub fn dirty(&self, repo_id: &str) -> Option { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .get(repo_id) + .copied() + } } -/// RAII guard removing a repo key from [`EncryptInflight`] when the detached -/// encryption task finishes (drop on completion, error, or panic-unwind). Move-only — -/// there is no reason to clone a guard, and cloning would double-remove. +/// RAII guard for one in-flight encryption task's repo key. The task drives it at its +/// tail with [`requeue_or_release`](Self::requeue_or_release); `Drop` is a panic +/// backstop. Move-only — cloning would double-release. pub struct EncryptInflightGuard { - repos: Arc>>, + repos: Arc>>, repo_id: String, + released: bool, +} + +impl EncryptInflightGuard { + /// TASK-TAIL check-and-clear, atomic with the release decision. If a push + /// coalesced since the last pass (dirty), clear the flag and return `true` (the + /// task loops one more pass). Otherwise remove the key and return `false` (the task + /// exits). Atomic under the mutex: a concurrent push either sets the flag BEFORE + /// this reads it (-> requeue covers it) or arrives AFTER the key is removed (-> a + /// fresh `try_begin` spawns a new task) — there is no window where the key is + /// present-but-clean while the task exits. + pub fn requeue_or_release(&mut self) -> bool { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(&self.repo_id) { + Some(dirty) if *dirty => { + *dirty = false; + true + } + _ => { + map.remove(&self.repo_id); + self.released = true; + false + } + } + } } impl Drop for EncryptInflightGuard { fn drop(&mut self) { - // A poisoned lock (a prior panic while holding it) still lets us take the inner - // set via into_inner-on-guard; but poisoning here is not expected because the - // only critical sections are the O(1) ops above. Remove best-effort. - if let Ok(mut set) = self.repos.lock() { - set.remove(&self.repo_id); + // Panic backstop ONLY. The normal exit is `requeue_or_release` (which removed + // the key atomically and set `released`), so this does nothing then. If the + // task panicked or returned without releasing, remove the key so one crashed + // walk never permanently locks the repo out. + // + // Accepted residual: a panic with the dirty flag still set drops the requeued + // work — Drop cannot loop — the same loss class as the on-panic behavior before + // this change. No reconciliation sweep re-derives it. + if self.released { + return; + } + if let Ok(mut map) = self.repos.lock() { + map.remove(&self.repo_id); } } } diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 45b2b807..20e325ac 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -7306,4 +7306,355 @@ mod tests { "result includes the deep cert matching the prefix" ); } + + // ---- U3 (#173 F3): coalesced post-push work is REQUEUED, not dropped ---- + // + // The seam mechanics (dirty flag, atomic check-and-clear, Drop backstop) are unit + // tested next to the #174 coalescing tests in `api/repos.rs`. These drive the whole + // detached task through a mock Kubo node and a real git repo, so the requeue's + // fresh re-read (encrypt half) and fail-closed full-scan enumeration (pin half) are + // proven end to end, with DB-observable effects. Each test models a push that + // coalesced during the in-flight window by (a) marking the repo dirty via a second + // `try_begin` and (b) making the repo/policy dynamic so the FIRST pass's spawn-time + // captures are stale — a static-state test would pass vacuously over the gap. + mod u3_requeue { + use super::*; + use crate::db::VisibilityMode; + use std::collections::HashSet; + use std::path::{Path, PathBuf}; + use std::process::Command; + + fn git(args: &[&str], dir: &Path) { + let ok = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(); + assert!(ok, "git {args:?} failed"); + } + fn oid(rev: &str, dir: &Path) -> String { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(dir) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}: {out:?}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + struct Repo { + _td: tempfile::TempDir, + path: PathBuf, + } + fn init_repo() -> Repo { + let td = tempfile::TempDir::new().unwrap(); + let path = td.path().to_path_buf(); + git(&["init", "-q"], &path); + git(&["config", "user.email", "t@t"], &path); + git(&["config", "user.name", "t"], &path); + Repo { _td: td, path } + } + /// Commit `content` at `rel`, return the blob oid. + fn commit(repo: &Path, rel: &str, content: &str) -> String { + let full = repo.join(rel); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(&full, content).unwrap(); + git(&["add", "."], repo); + git(&["commit", "-qm", rel], repo); + oid(&format!("HEAD:{rel}"), repo) + } + /// Write a loose, UNREACHABLE blob (dangling object). + fn write_dangling_blob(repo: &Path, content: &str) -> String { + let out = Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(repo) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + use std::io::Write; + out.stdin + .as_ref() + .unwrap() + .write_all(content.as_bytes()) + .unwrap(); + let o = out.wait_with_output().unwrap(); + assert!(o.status.success()); + String::from_utf8_lossy(&o.stdout).trim().to_string() + } + fn new_did() -> String { + Keypair::generate().did().to_string() + } + + /// SCENARIO 2 + 5 (pin half, TAIL-PLACEMENT guard). A coalesced push on a PUBLIC + /// repo with NO path-scoped rule must still requeue its pin half: the second + /// push's new object is pinned after the task. RED without the loop (stale spawn + /// object_list never lists obj2), and RED if the check-and-clear sits inside the + /// `has_path_scoped_rule` block (a rules-free repo would never reach it). + #[sqlx::test] + async fn u3_rules_free_public_repo_requeues_pin_half(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-pin"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + // The coalesced push B adds obj2 (present at requeue time, NOT in the stale + // push-A spawn object_list). + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Push A admits (guard); push B coalesces (marks dirty). + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces while A is in flight" + ); + + // Spawn-time (push A) captures are STALE: object_list lists only obj1, no rule. + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned on the first pass" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's new object is pinned by the REQUEUE full scan (RED \ + without the loop, or if the check-and-clear sits inside the encrypt gate)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits clean" + ); + } + + /// SCENARIO 1 + 3 (encrypt half, FRESH re-read). A coalesced push adds a + /// path-scoped rule withholding a blob. The task must re-read rules FRESH on + /// requeue and seal the newly-withheld blob's recovery copy. RED without the loop + /// (pass one's stale empty rule set seals nothing). + #[sqlx::test] + async fn u3_requeue_seals_blob_withheld_by_coalesced_rule_change(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-enc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let _pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + + // Coalesced push B changes .gitlawb: withhold /secret/** from anon, grant reader. + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + // Push A captures are STALE: no rule, empty withheld set (public repo). + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the coalesced push's newly-withheld blob is sealed after the REQUEUE re-read \ + (RED without the loop: pass one's stale empty rules seal nothing)" + ); + assert!(state.encrypt_inflight.is_empty(), "guard key released"); + } + + /// SCENARIO 4 (visibility-leak negative). The requeue full scan must feed + /// `list_all_objects` through the fail-closed filter, never pin it bare: a + /// withheld secret blob and a dangling blob must NOT land in the public pin set. + #[sqlx::test] + async fn u3_requeue_full_scan_does_not_publicly_pin_withheld_or_dangling(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-leak"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + // Coalesced push adds a new public object and a dangling blob. + let new_pub_oid = commit(&git_repo.path, "public/c.txt", "more public\n"); + let dangling_oid = write_dangling_blob(&git_repo.path, "orphan bytes\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let rules = state.db.list_visibility_rules(&repo.id).await.unwrap(); + let mut withheld = HashSet::new(); + withheld.insert(secret_oid.clone()); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![pub_oid.clone()], + Some(rules), + withheld, + ) + .await; + + assert!( + state.db.is_pinned(&new_pub_oid).await.unwrap(), + "the coalesced push's new PUBLIC object is pinned by the requeue" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "a WITHHELD blob is never publicly pinned by the requeue enumeration (leak guard)" + ); + assert!( + !state.db.is_pinned(&dangling_oid).await.unwrap(), + "a DANGLING blob is never publicly pinned by the requeue enumeration (leak guard)" + ); + // The withheld blob still gets its ENCRYPTED recovery copy (not a public pin). + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "withheld blob is sealed as an encrypted recovery copy, not pinned in the clear" + ); + } + + /// SCENARIO 8 (no-coalesce happy path). A single push with no coalesced follower + /// runs exactly one pass, pins its object, and releases the key. No requeue. + #[sqlx::test] + async fn u3_no_coalesce_single_pass_pins_and_releases(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-happy"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // No second try_begin: the repo is never marked dirty. + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push admits"); + assert_eq!( + state.encrypt_inflight.dirty(&repo.id), + Some(false), + "clean, no coalesce" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "the single push's object is pinned" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released after one pass" + ); + } + } } From c2fce67bae6f8ca67850de5a3b65581d2a2497d8 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 19:17:12 -0500 Subject: [PATCH 47/52] fix(node): wire GITLAWB_IPFS_MAX_REPOS_WALKED to the legacy-probe budget The knob was parsed and documented as the per-request /ipfs fan-out cap but never read in production: AppState seeded the legacy-probe budget from a fixed 256 constant, so setting the knob did nothing and an operator could still incur 256 acquire/probe operations per request. Seed ipfs_max_legacy_probes from the knob (default raised to 256 to preserve shipped behavior); leave the history-walk ceiling on its MAX_PIN_SOURCES+1 constant so a provenanced request is never truncated into a false 503. Updates the README and .env.example to the shipped behavior and adds cap-honored and plumbing tests. --- .env.example | 7 +-- README.md | 2 +- crates/gitlawb-node/src/config.rs | 57 +++++++++++++++++++++---- crates/gitlawb-node/src/main.rs | 5 ++- crates/gitlawb-node/src/state.rs | 12 ++++++ crates/gitlawb-node/src/test_support.rs | 49 +++++++++++++++++++++ 6 files changed, 118 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 149ea8b1..1c390a1c 100644 --- a/.env.example +++ b/.env.example @@ -168,9 +168,10 @@ GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 # per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). # Default 4. GITLAWB_IPFS_WALK_PER_SOURCE=4 -# Max repos walked per single /ipfs request, so one request cannot serialize a -# full-history walk over every repo carrying the CID. Default 64. -GITLAWB_IPFS_MAX_REPOS_WALKED=64 +# Max legacy (NULL-provenance) repos probed per single /ipfs request, bounding the +# scan-fallback fan-out (git cat-file per candidate repo) for an anonymous caller. A +# truncated scan sheds a retryable 503, never a false 404. Default 256. +GITLAWB_IPFS_MAX_REPOS_WALKED=256 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. GITLAWB_IPFS_RATE_LIMIT=600 diff --git a/README.md b/README.md index e9765876..a7e4425b 100644 --- a/README.md +++ b/README.md @@ -347,7 +347,7 @@ Important node settings: | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | -| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max repos walked per `/ipfs/{cid}` request, bounding one request's fan-out. Default 64. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max legacy (NULL-provenance) repos probed per `/ipfs/{cid}` request, bounding the scan-fallback fan-out. A truncated scan returns a retryable 503, not a false 404. Default 256. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index b3667024..a9b76067 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -372,17 +372,20 @@ pub struct Config { )] pub ipfs_walk_per_source: usize, - /// Upper bound on the number of candidate repos a single `/ipfs/{cid}` request - /// will walk before giving up (returning the opaque 404). The handler already - /// short-circuits the moment it serves the object, but a CID that is present in - /// (or path-gated out of) many repos could otherwise serialize one full-history - /// walk per repo inside a single held admission slot. Capping the count bounds - /// the worst-case work one request can pin its slot with. Must be between 1 and - /// 1_048_576. Default: 64. + /// Per-request ceiling on the number of legacy (NULL-provenance) repos the + /// `/ipfs/{cid}` resolver's scan fallback will PROBE (`acquire` + `git cat-file + /// -t`) before giving up. The provenance path targets one repo; the legacy scan, + /// absent this bound, fans one anonymous request out to O(repos) subprocess spawns + /// and cold-cache fetches for a CID enumerable from the public pins index (#173, + /// INV-10). A truncated scan surfaces as a retryable 503, never a false 404. Wired + /// into `AppState::ipfs_max_legacy_probes` at construction; the history-walk ceiling + /// stays constant (`MAX_HISTORY_WALKS_PER_REQUEST`) and is NOT governed by this knob + /// (a smaller value would falsely 503 a provenanced request). Must be between 1 and + /// 1_048_576. Default: 256. #[arg( long, env = "GITLAWB_IPFS_MAX_REPOS_WALKED", - default_value_t = 64, + default_value_t = crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST as usize, value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) )] pub ipfs_max_repos_walked: usize, @@ -534,7 +537,7 @@ mod tests { fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { assert_eq!( Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked, - 64 + 256 ); assert_eq!( Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"]) @@ -548,6 +551,42 @@ mod tests { ); } + /// The `GITLAWB_IPFS_MAX_REPOS_WALKED` knob must actually reach the legacy-probe + /// budget it advertises (R5, KTD5): production seeds `ipfs_max_legacy_probes` from + /// this helper, so the knob is a no-op unless the helper reflects it. RED while the + /// helper returns the hardcoded `MAX_LEGACY_PROBES_PER_REQUEST` (256 regardless of + /// the knob), GREEN once it reads the knob. + #[test] + fn ipfs_max_repos_walked_wires_the_legacy_probe_budget() { + use crate::state::AppState; + // Knob set to 1 → a one-probe legacy budget. + let one = Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1"]); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&one), + 1, + "the knob must control the legacy-probe budget, not be ignored" + ); + // Unset knob preserves the shipped 256-probe behaviour. + let default = Config::parse_from(["gitlawb-node"]); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&default), + 256, + "the default knob keeps the shipped 256-probe budget" + ); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&default), + crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + "the default budget equals the constant it replaced" + ); + // Ceiling guard: the knob never governs the history-walk ceiling, which must + // stay at MAX_PIN_SOURCES + 1 or a provenanced full source set false-503s. + assert!( + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST + >= crate::db::MAX_PIN_SOURCES as u32 + 1, + "the history-walk ceiling is independent of the repos-walked knob" + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 37190c76..0af288c2 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -375,7 +375,10 @@ async fn main() -> Result<()> { create_ip_rate_limiter, push_rate_limiter, ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, - ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + // The legacy-probe budget is operator-tunable via GITLAWB_IPFS_MAX_REPOS_WALKED + // (R5); the history-walk ceiling above stays constant (a smaller value false-503s + // a provenanced request). Default 256 preserves the shipped behaviour. + ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, sync_trigger_rate_limiter, diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 61bbec6a..bc57b2d9 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -242,6 +242,18 @@ impl AppState { pub fn is_shutting_down(&self) -> bool { *self.shutdown_tx.borrow() } + + /// Legacy-probe budget wired from the `GITLAWB_IPFS_MAX_REPOS_WALKED` operator + /// knob (R5, KTD5). The knob seeds `ipfs_max_legacy_probes` at construction so it + /// controls the per-request legacy (NULL-provenance) probe fan-out it advertises. + /// It deliberately does NOT feed `ipfs_max_history_walks`: that ceiling must stay + /// at `MAX_HISTORY_WALKS_PER_REQUEST` (`MAX_PIN_SOURCES + 1`) or a provenanced + /// request with a full source set is truncated into a false 503, and the knob's + /// range starts at 1. The knob is `usize`, the field `u32`; the range cap + /// (1_048_576) keeps the cast lossless. + pub(crate) fn ipfs_legacy_probe_budget(config: &crate::config::Config) -> u32 { + config.ipfs_max_repos_walked as u32 + } } /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 20e325ac..30f63d83 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -3674,6 +3674,55 @@ mod tests { ); } + /// T2b (R5, KTD5): the `GITLAWB_IPFS_MAX_REPOS_WALKED` knob drives the legacy-probe + /// budget end to end. With the knob at 1 (fed through the same production helper the + /// state seeding uses) and two candidate repos that miss, the first repo spends the + /// single probe and the second is skipped at the cap → truncated → 503. If the knob + /// budget were not honoured (unbounded), both would probe, both miss, and the request + /// would be a definitive 404. Proves the wired knob=1 → exactly one probe path. + #[sqlx::test] + async fn ipfs_cid_repos_walked_knob_caps_legacy_probes(pool: PgPool) { + use clap::Parser; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Seed the legacy-probe budget the way production does: from the operator knob. + let cfg = + crate::config::Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1"]); + state.ipfs_max_legacy_probes = AppState::ipfs_legacy_probe_budget(&cfg); + assert_eq!(state.ipfs_max_legacy_probes, 1, "knob=1 → one-probe budget"); + // The knob must not touch the history-walk ceiling (must stay MAX_PIN_SOURCES + 1). + assert_eq!( + state.ipfs_max_history_walks, + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + "the repos-walked knob leaves the history-walk ceiling untouched" + ); + + let _fx = seed_cid_repos(&slug, &short, &["k0", "k1"]); + for n in ["k0", "k1"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo: the cap, not a hit, decides. + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-knob").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "knob=1 caps the scan at one probe → incomplete search → retryable 503" + ); + } + /// T3 (F2): a walk-cap truncation must not false-404. Walk ceiling shrunk to 1; /// two public repos each carry a path-scoped rule over the object and deny anon. /// The 1st spends the single walk (deny), the 2nd is skipped at the cap — the From 0a1618f7fb81c8bac92721095035f5eb02b08c46 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 19:41:57 -0500 Subject: [PATCH 48/52] fix(node): split /ipfs work accounting off the route rate limiter The route middleware charged ipfs_rate_limiter once per request while the legacy scan and provenance walk charged the same bucket per probe/walk, so a one-probe request cost two tokens and GITLAWB_IPFS_RATE_LIMIT=1 admitted the request at the route then 429'd it at the pre-scan peek. Add a bounded ipfs_work_rate_limiter, move the in-scan, per-walk, and pre-scan charges onto it, and leave the route limiter as the pure once-per-request brake. The work-budget capacity derives from the route limit with a floor of one full legacy-probe budget per window (no new operator knob), so a default-config deep search never self-throttles. Adds it to the periodic sweep, updates all three AppState construction sites, and rewrites the contradictory limiter comments. --- crates/gitlawb-node/src/api/ipfs.rs | 36 +++-- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/config.rs | 83 ++++++++++- crates/gitlawb-node/src/main.rs | 9 ++ crates/gitlawb-node/src/state.rs | 46 +++++- crates/gitlawb-node/src/test_support.rs | 182 ++++++++++++++++++++++-- 6 files changed, 322 insertions(+), 35 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 016e130d..c5fa35bc 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -38,11 +38,12 @@ use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; /// Hard ceiling on the number of full-history reachability walks a single -/// `GET /ipfs/{cid}` request may spawn. The per-request `ipfs_rate_limiter` -/// check brakes *repeat* requests, but within one request the object can exist -/// under path-scoped rules in many repos, and each distinct repo pays its own -/// `spawn_blocking` walk (the memo only dedups the same repo). Without a ceiling -/// a single request fans out to O(repos) walks for one rate-limiter token — an +/// `GET /ipfs/{cid}` request may spawn. The route brake (`ipfs_rate_limiter`, charged +/// once per request by the middleware) caps request RATE, and the per-walk charge on +/// the separate `ipfs_work_rate_limiter` bounds the walk work across requests, but +/// within ONE request the object can exist under path-scoped rules in many repos, and +/// each distinct repo pays its own `spawn_blocking` walk (the memo only dedups the same +/// repo). Without a ceiling a single request fans out to O(repos) walks — an /// amplification sink (INV-10). Once this many walks have run, no further walk is /// spawned for the rest of the request: any remaining candidate that still needs /// a walk is skipped (and, with nothing else readable, the request falls through @@ -335,14 +336,18 @@ pub async fn get_by_cid( .await .map_err(AppError::Internal)?; if needs_scan { - // F3 (#173, INV-10/INV-15): peek the per-IP limiter WITHOUT consuming a token - // so an already-throttled source is shed BEFORE the O(repos) preload; the - // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is - // load-bearing for the across-request bound), so this adds no double-charge. + // F3 (#173, INV-10/INV-15): peek the per-IP WORK-budget limiter WITHOUT + // consuming a token so an already-throttled source is shed BEFORE the + // O(repos) preload; the consuming per-probe charge inside gate_and_serve is + // left UNCHANGED (it is load-bearing for the across-request bound), so this + // adds no double-charge. This peeks `ipfs_work_rate_limiter`, the SAME bucket + // the per-probe charge below debits — NOT the route limiter (`ipfs_rate_limiter`, + // charged once per request by the middleware): peeking the route bucket here + // would re-shed a request the route already admitted (R6, U5). if let Some(key) = crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) { - if state.ipfs_rate_limiter.is_throttled(&key).await { + if state.ipfs_work_rate_limiter.is_throttled(&key).await { throttled = true; continue; } @@ -536,9 +541,10 @@ async fn gate_and_serve( // reset each request, leaving a NULL-provenance CID open to unbounded ACROSS- // request amplification: N requests spending N x budget cold `acquire` calls // against Tigris with zero limiter contact (#173, F3, jatmn). Charging the first - // probe makes those requests accumulate against the per-IP `ipfs_rate_limiter`, - // closing that path. The per-request cap below stays as the second bound (a - // single request's ceiling). A spent quota is the same non-fatal Throttled as the + // probe makes those requests accumulate against the per-IP `ipfs_work_rate_limiter` + // (the resolver's WORK bucket, separate from the once-per-request route brake + // `ipfs_rate_limiter` — R6, U5), closing that path. The per-request cap below stays + // as the second bound (a single request's ceiling). A spent quota is the same non-fatal Throttled as the // walk brake: keep scanning for a walk-free copy, and only a wholly-unservable // request becomes the 429. No resolvable key (a test oneshot with no peer/header) // skips the brake, as the walk brake does. The provenance path targets one repo @@ -553,7 +559,7 @@ async fn gate_and_serve( if let Some(key) = crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) { - if !state.ipfs_rate_limiter.check(&key).await { + if !state.ipfs_work_rate_limiter.check(&key).await { return GateOutcome::Throttled; } } @@ -658,7 +664,7 @@ async fn gate_and_serve( if let Some(key) = crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) { - if !state.ipfs_rate_limiter.check(&key).await { + if !state.ipfs_work_rate_limiter.check(&key).await { return GateOutcome::Throttled; } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index c03eb2e5..96e6f5ea 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -517,6 +517,7 @@ mod tests { create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index a9b76067..5480beda 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -396,6 +396,11 @@ pub struct Config { /// concurrency cap above (a rate limit bounds request *rate*, the semaphore /// bounds concurrent slow holds — different axes). Keyed on the resolved client /// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600. + /// + /// This is the pure once-per-request ROUTE brake. The resolver's internal + /// per-probe/per-walk WORK budget is a SEPARATE bucket whose capacity is DERIVED + /// from this value (`AppState::ipfs_work_budget`), not a knob of its own; `0` here + /// disables that derived bucket too. #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] pub ipfs_rate_limit: usize, } @@ -581,12 +586,86 @@ mod tests { // Ceiling guard: the knob never governs the history-walk ceiling, which must // stay at MAX_PIN_SOURCES + 1 or a provenanced full source set false-503s. assert!( - crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST - >= crate::db::MAX_PIN_SOURCES as u32 + 1, + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST > crate::db::MAX_PIN_SOURCES as u32, "the history-walk ceiling is independent of the repos-walked knob" ); } + /// The `/ipfs` work-budget capacity is DERIVED from the route limit (R6, KTD6), with + /// a hard floor of one full legacy search per window (the effective + /// `ipfs_max_legacy_probes`). This guards the derived default so a single + /// default-config deep search never self-throttles mid-scan and recreates the F6 + /// admit-then-429 for a legitimate caller. A `RateLimiter` sized to the derived + /// budget must admit the whole probe budget back to back. + #[test] + fn ipfs_work_budget_derives_from_route_limit_and_clears_the_probe_floor() { + use crate::state::AppState; + + // Default config: derived work budget = max(route 600, probe budget 256) = 600, + // comfortably above the 256-probe floor. + let default = Config::parse_from(["gitlawb-node"]); + let budget = AppState::ipfs_work_budget(&default); + assert_eq!(budget, 600, "default derives max(route 600, probe 256)"); + assert!( + budget >= AppState::ipfs_legacy_probe_budget(&default) as usize, + "the work budget must clear one full legacy search per window" + ); + + // Tight route limit (1): the floor lifts the work budget to the probe budget + // (256), NOT down to 1 — a single deep search still completes its full scan. + let tight = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "1"]); + assert_eq!( + AppState::ipfs_work_budget(&tight), + 256, + "a tight route limit is floored at the 256-probe budget, not clamped to 1" + ); + + // Raised probe budget lifts the floor with it (the work budget tracks the + // effective probe budget, not the constant). + let raised = Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "10", + "--ipfs-max-repos-walked", + "1000", + ]); + assert_eq!( + AppState::ipfs_work_budget(&raised), + 1000, + "the floor tracks the operator-raised legacy-probe budget" + ); + + // 0 route limit disables the derived bucket too (a 0-capacity limiter admits all). + let disabled = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "0"]); + assert_eq!( + AppState::ipfs_work_budget(&disabled), + 0, + "route limit 0 disables the derived work bucket alongside the route brake" + ); + + // Behavioral floor: a limiter sized to the derived (tight-route) budget admits + // the whole probe budget back to back for one source, then sheds the next. + let budget = AppState::ipfs_work_budget(&tight); + let limiter = + crate::rate_limit::RateLimiter::new(budget, std::time::Duration::from_secs(3600)); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + for i in 0..budget { + assert!( + limiter.check("1.2.3.4").await, + "probe {i} of one full default-config scan must be admitted (no mid-scan throttle)" + ); + } + assert!( + !limiter.check("1.2.3.4").await, + "the probe past the derived budget is shed" + ); + }); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 0af288c2..1ed45929 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -437,6 +437,15 @@ async fn main() -> Result<()> { std::time::Duration::from_secs(3600), 200_000, ), + // Separate WORK-budget bucket for the resolver's per-probe/per-walk charges (R6). + // Its capacity is DERIVED from the route limit (no new knob) and floored at the + // legacy-probe budget, so one full default-config legacy scan never self-throttles + // mid-request while the route brake above stays the pure once-per-request cap. + ipfs_work_rate_limiter: rate_limit::RateLimiter::new_bounded( + AppState::ipfs_work_budget(&config), + std::time::Duration::from_secs(3600), + 200_000, + ), git_bin: "git".to_string(), }; if config.ipfs_rate_limit == 0 { diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index bc57b2d9..51618b78 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -65,13 +65,29 @@ pub struct AppState { /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. pub push_rate_limiter: RateLimiter, - /// Per-client-IP rate limiter for the `GET /ipfs/{cid}` full-history walk. - /// The route is anonymous and a valid tree CID (exposed by the public pins - /// index) makes each repeat request pay a fresh allowed-set walk (rev-list + - /// ls-tree per commit), memoized only per request — unbounded amplification - /// (INV-10). Braking the walk on the non-farmable source IP caps that cost - /// without touching cheap non-walk fetches. Keyed by `push_limiter_trust`. + /// Per-client-IP ROUTE brake for `GET /ipfs/{cid}`: charged ONCE per request by the + /// `rate_limit_by_ip` middleware (server.rs), never inside the handler. It bounds + /// request RATE (the "requests per hour" contract of `GITLAWB_IPFS_RATE_LIMIT`) on + /// the non-farmable source IP, so an anonymous flood of the public route is capped. + /// The per-probe/per-walk WORK accounting the resolver does WITHIN a request draws + /// from the SEPARATE `ipfs_work_rate_limiter` below — the two cannot share one bucket + /// or a single request that spends a route token and then its own probe token off the + /// same bucket is admitted at the route and falsely shed mid-request (#173 round-10, + /// R6). Keyed by `push_limiter_trust`. pub ipfs_rate_limiter: RateLimiter, + /// Per-client-IP WORK-budget limiter for the `GET /ipfs/{cid}` resolver's internal + /// fan-out: charged per legacy (NULL-provenance) PROBE (`acquire` + `cat-file`) and + /// per provenance-path WALK, and peeked non-consuming before the O(repos) legacy + /// preload. A legacy CID from the public pins index otherwise lets one request drive + /// O(repos) subprocess spawns and cold Tigris fetches, and repeat requests amplify + /// that across requests with zero limiter contact (INV-10, F3). Charging the work to + /// the non-farmable source IP bounds it. A bucket DISTINCT from the route brake above: + /// one request legitimately spends many work tokens (a full legacy scan is up to + /// `ipfs_max_legacy_probes` probes), so it must not double as the once-per-request + /// route bucket. Capacity is DERIVED from the route limit (`AppState::ipfs_work_budget`, + /// no separate operator knob), floored at the legacy-probe budget so a single default- + /// config deep search never self-throttles mid-scan. Keyed by `push_limiter_trust`. + pub ipfs_work_rate_limiter: RateLimiter, /// Per-request ceiling on full-history reachability walks the CID resolver /// may spawn (default `api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST`). A field, /// not a bare const, so tests can shrink it to exercise the cap cheaply; @@ -218,6 +234,7 @@ impl AppState { self.create_ip_rate_limiter.cleanup().await; self.push_rate_limiter.cleanup().await; self.ipfs_rate_limiter.cleanup().await; + self.ipfs_work_rate_limiter.cleanup().await; self.sync_trigger_rate_limiter.cleanup().await; self.peer_write_rate_limiter.cleanup().await; } @@ -254,6 +271,23 @@ impl AppState { pub(crate) fn ipfs_legacy_probe_budget(config: &crate::config::Config) -> u32 { config.ipfs_max_repos_walked as u32 } + + /// Work-budget capacity for [`ipfs_work_rate_limiter`](Self#structfield.ipfs_work_rate_limiter) + /// (R6, KTD6), DERIVED from the route limit rather than a new operator knob. The route + /// limiter (`ipfs_rate_limiter`) charges once per request; this separate bucket absorbs + /// the resolver's per-probe/per-walk work charges so both the route "requests per hour" + /// contract and the amplification bound hold. Floor: at least one full legacy search per + /// window — the effective `ipfs_max_legacy_probes` (the `GITLAWB_IPFS_MAX_REPOS_WALKED` + /// knob, U4) — so a single default-config deep search cannot self-throttle mid-scan and + /// recreate the F6 admit-then-429 for a legitimate caller. `GITLAWB_IPFS_RATE_LIMIT=0` + /// disables the route brake and this derived bucket alike (a 0-capacity limiter admits + /// everything). + pub(crate) fn ipfs_work_budget(config: &crate::config::Config) -> usize { + if config.ipfs_rate_limit == 0 { + return 0; + } + config.ipfs_rate_limit.max(config.ipfs_max_repos_walked) + } } /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 30f63d83..01363abf 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -79,6 +79,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, @@ -3285,7 +3286,8 @@ mod tests { let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["provthrottle"]); @@ -3455,7 +3457,8 @@ mod tests { let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["fanout"]); @@ -3498,7 +3501,7 @@ mod tests { /// fresh batch of `acquire` + `cat-file` probes every request with zero limiter /// contact, unbounded anonymous amplification against Tigris. Charging every /// legacy probe from the first one makes those probes accumulate against the - /// per-IP `ipfs_rate_limiter` ACROSS requests. Four repos, none holding the CID, + /// per-IP `ipfs_work_rate_limiter` ACROSS requests. Four repos, none holding the CID, /// so a full scan probes all four; the per-IP budget is sized to exactly ONE such /// scan (4 tokens). req1 (a genuine absence) fully scans and 404s, spending the /// budget; req2 from the SAME IP is shed at the first probe → 429 (it never @@ -3515,7 +3518,8 @@ mod tests { let mut state = test_state(pool).await; // Budget = one full scan of the four seeded repos. A repeat scan from the same // IP then finds it spent. Keyed on XFF so `oneshot` can choose the source IP. - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(4, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(4, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let names = ["a0", "a1", "a2", "a3"]; @@ -3578,7 +3582,8 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; // Budget 1, keyed on XFF so `oneshot` can choose the source IP. - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let _fx = seed_cid_repos(&slug, &short, &["r0"]); @@ -5298,7 +5303,8 @@ mod tests { // size the per-IP budget to admit exactly one full scan (2 probes). A repeat // scan from the same IP then finds the bucket spent. Keyed on the rightmost // X-Forwarded-For hop so the test can choose a source IP under `oneshot`. - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["walklimit"]); @@ -5504,7 +5510,8 @@ mod tests { // Budget = one full two-repo scan (2 probes), keyed on the rightmost XFF hop // so `oneshot` can choose a source IP (no socket peer). A repeat scan from the // same IP then finds the budget spent. - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; // Identical secret-blob content in both bare clones → one CID resolves to @@ -5578,10 +5585,11 @@ mod tests { } /// INV-10 amplification bound: a single `GET /ipfs/{cid}` must not fan out an - /// unbounded number of full-history walks. The per-request `ipfs_rate_limiter` - /// check only brakes REPEAT requests (it fires once per request); within one - /// request the same object can exist under path-scoped rules in many repos, - /// each paying its own walk. `MAX_HISTORY_WALKS_PER_REQUEST` caps that fan-out. + /// unbounded number of full-history walks. The route brake (`ipfs_rate_limiter`) + /// fires once per request and the per-walk `ipfs_work_rate_limiter` charge bounds + /// walk work across requests, but within ONE request the same object can exist under + /// path-scoped rules in many repos, each paying its own walk. + /// `MAX_HISTORY_WALKS_PER_REQUEST` caps that fan-out. /// /// Load-bearing witness (#173, F4): a readable public copy (no path rule → /// served via the no-walk path, exactly like @@ -5784,7 +5792,8 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["w0", "w1"]); @@ -5851,6 +5860,155 @@ mod tests { ); } + /// U5 (R6, KTD6), the observed defect: the `/ipfs` route rate limit and the + /// resolver's per-probe WORK budget are SEPARATE buckets, so a single request with + /// one probe COMPLETES even at route limit = 1. Through the production router the + /// `rate_limit_by_ip` middleware charges `ipfs_rate_limiter` once (its 1-slot bucket + /// is now full); the handler's legacy pre-scan peek and per-probe charge then draw + /// from `ipfs_work_rate_limiter`, a different bucket, so the walk-free public copy + /// still serves 200. RED before the split (both charges on `ipfs_rate_limiter`): the + /// middleware fills the one slot, the pre-scan peek reads it throttled, nothing is + /// servable → 429 on the FIRST request. Trust None so the middleware and the handler + /// resolve the same `ConnectInfo` peer IP. + #[sqlx::test] + async fn ipfs_route_limit_1_still_serves_one_probe(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(600, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Public, no-rule legacy pin (NULL provenance) → the resolver takes the scan + // fallback and serves walk-free (exactly one probe). + let fx = seed_cid_repos(&slug, &short, &["routeone"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("routeone.git"); + let repo = seed_repo(&owner_did, "routeone"); + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let router = crate::server::build_router(state); + let peer: std::net::SocketAddr = "203.0.113.7:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "a single /ipfs request with one probe must serve even at route limit = 1 \ + (the route brake and the resolver's work budget are separate buckets)" + ); + } + + /// U5 (R6): the two buckets are independent — the WORK budget can be exhausted + /// (429) WITHOUT draining the ROUTE bucket. Through the production router, route + /// generous (5) but work tight (1): one request drives two legacy probes, so the + /// second probe finds the work bucket spent → 429 (the route middleware admitted it). + /// The route bucket, charged once by the middleware, still has room afterward — the + /// work charges never touched it, so it admits four more direct checks. + #[sqlx::test] + async fn ipfs_work_exhaustion_leaves_route_bucket_intact(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // A legacy pin absent from every repo so the scan probes both seeded repos: two + // probes, work budget 1 → the second probe is shed → 429. + let names = ["we0", "we1"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + state + .db + .create_repo(&seed_repo(&owner_did, n)) + .await + .expect("seed repo"); + } + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"work-exhaustion").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("legacy pin"); + + let route_bucket = state.ipfs_rate_limiter.clone(); + let peer_ip = "203.0.113.8"; + let peer: std::net::SocketAddr = format!("{peer_ip}:5000").parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a request whose probes exceed the work budget is shed 429 (work bucket), \ + not blocked at the route (route bucket generous)" + ); + // The route bucket recorded only the single request the middleware charged; the + // work charges did not drain it. Sized 5, one used by the request → four left. + for i in 0..4 { + assert!( + route_bucket.check(peer_ip).await, + "route check {i} must still admit — work charges never drained the route bucket" + ); + } + } + + /// U5 (R6): the periodic cleanup task sweeps the NEW work-budget limiter too, not + /// only the route limiter and its siblings. Mirrors + /// `sweep_rate_limiters_includes_ipfs_limiter`: drive `sweep_rate_limiters` and + /// assert the work limiter's expired entry is evicted. Dropping the + /// `ipfs_work_rate_limiter.cleanup()` call from that method leaves the entry in place + /// (`tracked_keys` stays 1): the RED proof the sweep covers it. + #[sqlx::test] + async fn sweep_rate_limiters_includes_ipfs_work_limiter(pool: PgPool) { + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(5, Duration::from_millis(50)); + + assert!( + state.ipfs_work_rate_limiter.check("1.2.3.4").await, + "record a hit on the work limiter" + ); + assert_eq!( + state.ipfs_work_rate_limiter.tracked_keys().await, + 1, + "the source-IP key is tracked before the sweep" + ); + + tokio::time::sleep(Duration::from_millis(60)).await; + state.sweep_rate_limiters().await; + + assert_eq!( + state.ipfs_work_rate_limiter.tracked_keys().await, + 0, + "the periodic sweep must evict the work limiter's expired entries" + ); + } + // --------------------------------------------------------------------------- // Issue #120 — repo-scoped read surfaces visibility gate // --------------------------------------------------------------------------- From b774c435a63691c65473082208b771a790eeab9e Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 19:50:23 -0500 Subject: [PATCH 49/52] fix(node): retry pin-source and pinned-CID recording on transient errors The three warn-only record sites in the detached post-push task dropped a transient DB failure silently, leaving a permanently incomplete pin-source set that makes the resolver 404 a valid public copy. Wrap them in a bounded retry (inserts are idempotent via ON CONFLICT DO NOTHING); on exhaustion the warn still fires so behavior degrades to today's. Documents the residual crash/outage window that only a reconciliation sweep retires. --- crates/gitlawb-node/src/ipfs_pin.rs | 135 +++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index e70d8f7b..1a37005d 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -9,6 +9,44 @@ use anyhow::Result; use gitlawb_core::cid::Cid; +use std::time::Duration; + +/// Attempts (including the first) for a transient DB-record retry. +const PIN_RECORD_ATTEMPTS: u32 = 3; +/// Backoff between DB-record retry attempts. +const PIN_RECORD_BACKOFF: Duration = Duration::from_millis(50); + +/// Run an idempotent DB-record operation with a bounded retry so a sub-second +/// transient error does not silently leave the pin-source set permanently +/// incomplete. The resolver treats a nonempty below-cap source set as complete, +/// so a dropped `record_pin_source`/`record_pinned_cid` makes `GET /ipfs/{cid}` +/// 404 a valid public copy. Every wrapped insert is idempotent (`ON CONFLICT DO +/// NOTHING` / provenance-preserving upsert), so re-running is safe. On exhausted +/// attempts the last error is returned and the caller keeps its warn — behavior +/// degrades to the pre-retry state, not worse. Process death mid-retry or a DB +/// outage outlasting the backoff horizon leaves the same residual hole (no +/// persisted marker to reconcile from at startup), retired only by a future +/// reconciliation sweep. Runs inside the already-detached post-push task, so the +/// backoff adds no push latency. +async fn retry_db_record(mut op: F) -> Result<()> +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut attempt = 1; + loop { + match op().await { + Ok(()) => return Ok(()), + Err(e) => { + if attempt >= PIN_RECORD_ATTEMPTS { + return Err(e); + } + tokio::time::sleep(PIN_RECORD_BACKOFF).await; + attempt += 1; + } + } + } +} /// Pin a single git object to the local IPFS/Kubo node. /// @@ -134,7 +172,7 @@ pub async fn pin_new_objects( // and without it `GET /ipfs/{cid}` only ever knows the first pinner, so a // shared object first pinned from a private/quarantined repo 404s even // when this repo would serve it. Bounded per object (MAX_PIN_SOURCES). - if let Err(e) = db.record_pin_source(&sha, repo_id).await { + if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } continue; @@ -167,12 +205,14 @@ pub async fn pin_new_objects( // verifies them against the requested CID, so the raw CID is the correct // key. Mirrors the pinata twin, which already records the raw CID. let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); - if let Err(e) = db.record_pinned_cid(&sha, &raw_cid, Some(repo_id)).await { + if let Err(e) = + retry_db_record(|| db.record_pinned_cid(&sha, &raw_cid, Some(repo_id))).await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } // F1 (#173 round 8): also record the first pinner in pin_repo_sources so // every source (first and subsequent) is tried uniformly by the resolver. - if let Err(e) = db.record_pin_source(&sha, repo_id).await { + if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } // Return the provider Hash (not the resolver key), mirroring the pinata @@ -190,3 +230,92 @@ pub async fn pin_new_objects( pinned } + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + // The retry helper is the load-bearing unit: it converts a sub-second + // transient DB error at the three warn-only record sites into a landed row, + // instead of a permanently incomplete pin-source set. These drive the helper + // directly against a controlled closure (the record sites take a concrete + // `&Db` over a `PgPool`, so a failing-first wrapper cannot slot in without + // changing signatures — see U6 seam note). + + #[tokio::test] + async fn retry_lands_after_transient_failures() { + let calls = Cell::new(0u32); + let result = retry_db_record(|| { + let n = calls.get() + 1; + calls.set(n); + async move { + if n < PIN_RECORD_ATTEMPTS { + Err(anyhow::anyhow!("transient failure on attempt {n}")) + } else { + Ok(()) + } + } + }) + .await; + + assert!( + result.is_ok(), + "retry lands the row after transient failures" + ); + assert_eq!( + calls.get(), + PIN_RECORD_ATTEMPTS, + "op is retried until it succeeds" + ); + } + + #[tokio::test] + async fn retry_returns_last_err_after_exhaustion() { + let calls = Cell::new(0u32); + let result = retry_db_record(|| { + let n = calls.get() + 1; + calls.set(n); + async move { Err::<(), _>(anyhow::anyhow!("attempt {n} failed")) } + }) + .await; + + let err = result.expect_err("all attempts fail so the last error surfaces"); + assert_eq!( + calls.get(), + PIN_RECORD_ATTEMPTS, + "attempts are bounded to the cap" + ); + assert_eq!( + err.to_string(), + "attempt 3 failed", + "the LAST error is returned, not the first" + ); + } + + // Happy path against a real DB: a single-attempt success lands the row, and a + // redundant call is idempotent (`ON CONFLICT DO NOTHING`), so the source set + // holds exactly one row. + #[sqlx::test] + async fn retry_records_pin_source_once(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let sha = "a".repeat(64); + let repo_id = "repo-retry-1"; + + retry_db_record(|| db.record_pin_source(&sha, repo_id)) + .await + .expect("happy-path record succeeds in one attempt"); + retry_db_record(|| db.record_pin_source(&sha, repo_id)) + .await + .expect("a redundant record is idempotent"); + + let sources = db.pin_sources_for_oid(&sha).await.unwrap(); + assert_eq!( + sources, + vec![repo_id.to_string()], + "exactly one source row lands under ON CONFLICT DO NOTHING" + ); + } +} From 45e269b2ae5bcc328204f6179c9987bcfa0a7911 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 20:11:42 -0500 Subject: [PATCH 50/52] fix(node): opportunistically repair legacy provider-CID pins Before this PR the pin path stored the provider CID (Kubo dag-pb / Pinata) in pinned_cids.cid, which the new resolver recomputes-and-404s, while /api/v1/ipfs/pins still advertises the stale value. Repair a legacy row in the already-pinned skip branch: gate on a cheap stored-CID codec check (only a non-raw-codec CID reads bytes, so raw rows keep the DB-only skip cost), recompute the raw CID, and rewrite it while stashing the old value in a new v14 legacy_provider_cid column (distinct from pinata_cid, which gates the Pinata pin-skip). Rows whose bytes are gone stay withheld. Documents that the deferred one-shot sweep, not this opportunistic path, fully retires the window. --- README.md | 2 + crates/gitlawb-core/src/cid.rs | 45 ++++ crates/gitlawb-node/src/db/mod.rs | 57 ++++ crates/gitlawb-node/src/ipfs_pin.rs | 76 ++++++ crates/gitlawb-node/src/test_support.rs | 329 ++++++++++++++++++++++++ 5 files changed, 509 insertions(+) diff --git a/README.md b/README.md index a7e4425b..80f71cb7 100644 --- a/README.md +++ b/README.md @@ -355,6 +355,8 @@ Important node settings: Production note: change the default Postgres password before exposing a node publicly. +Legacy-pin window: releases before the CID-resolver work stored the provider CID (Kubo dag-pb / Pinata) as a pinned object's resolver key. The `/ipfs/{cid}` resolver now recomputes the raw-content CID from the object bytes and refuses to serve a key that does not match, so `GET /api/v1/ipfs/pins` can still advertise an unrepaired legacy CID that 404s. Such a row is repaired opportunistically the next time a push carries the object again (its key is rewritten to the raw CID, the old value kept in `legacy_provider_cid`), but git negotiation omits objects the node already has, so most legacy rows never re-enter a push delta. A deferred one-shot startup sweep, not this opportunistic path, is what fully retires the advertise-then-404 window. Rows whose object bytes are gone stay withheld. + --- ## Optional node staking diff --git a/crates/gitlawb-core/src/cid.rs b/crates/gitlawb-core/src/cid.rs index b7993cc4..2071d478 100644 --- a/crates/gitlawb-core/src/cid.rs +++ b/crates/gitlawb-core/src/cid.rs @@ -64,6 +64,19 @@ impl Cid { } } +/// True when `s` parses as a CIDv1 with the raw codec — the exact shape +/// [`Cid::from_git_object_bytes`] produces and the `/ipfs` resolver looks up. +/// A legacy provider CID (Kubo dag-pb, Pinata CIDv0) parses to a different +/// version or codec and returns `false`, marking it an opportunistic-repair +/// candidate. Decidable from the string alone (no object bytes), so the pin path +/// can gate the byte-read/recompute cost on it and leave non-legacy rows at the +/// existing DB-only skip cost. An unparseable string is non-canonical (`false`). +pub fn is_raw_cidv1(s: &str) -> bool { + s.parse::>() + .map(|c| c.version() == cid::Version::V1 && c.codec() == RAW) + .unwrap_or(false) +} + impl fmt::Display for Cid { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) @@ -177,6 +190,38 @@ mod tests { assert!(result.is_err()); } + #[test] + fn is_raw_cidv1_classifies_codec_from_string() { + // The canonical resolver key: CIDv1 + raw codec → not a repair candidate. + let raw = Cid::from_git_object_bytes(b"blob 5\0hello"); + assert!( + is_raw_cidv1(raw.as_str()), + "from_git_object_bytes output is CIDv1/raw" + ); + + // A CIDv0 (Pinata dag-pb legacy shape) → repair candidate. + assert!( + !is_raw_cidv1("QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"), + "a CIDv0 dag-pb value is a legacy-repair candidate" + ); + + // A CIDv1 with the dag-pb codec (the Kubo above-block-size root) over the + // same multihash → still a repair candidate (codec, not just version). + let parsed = raw.as_str().parse::>().unwrap(); + const DAG_PB: u64 = 0x70; + let dagpb = CidGeneric::<64>::new_v1(DAG_PB, *parsed.hash()).to_string(); + assert!( + !is_raw_cidv1(&dagpb), + "a CIDv1 dag-pb value is a legacy-repair candidate" + ); + + // Garbage is non-canonical. + assert!( + !is_raw_cidv1("not-a-cid"), + "an unparseable string is non-canonical" + ); + } + #[test] fn sha256_hex_of_empty_input_is_well_known() { // SHA-256("") is a fixed constant; verifies the hasher is wired correctly. diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 8d723ffd..775e7c17 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -921,6 +921,22 @@ const MIGRATIONS: &[Migration] = &[ "CREATE INDEX IF NOT EXISTS idx_pin_repo_sources_sha ON pin_repo_sources(sha256_hex)", ], }, + Migration { + version: 14, + name: "pinned_cids_legacy_provider_cid", + stmts: &[ + // R8 (#173, jatmn round 10): the opportunistic legacy provider-CID repair + // rewrites `pinned_cids.cid` from a stored PROVIDER CID (Kubo dag-pb / + // Pinata CIDv0) to the raw-content resolver key and stashes the OLD value + // here, so the rewrite is auditable and the row's legacy origin survives. + // Distinct from `pinata_cid` on purpose: `has_pinata_cid` gates the Pinata + // pin-skip, so parking a Kubo-legacy CID there would make Pinata forever + // skip re-pinning that object. NEW versioned migration (never appended to an + // applied block, INV-7) so a node past v13 actually gets the column. + // Nullable: only a repaired row sets it. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS legacy_provider_cid TEXT", + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -2287,6 +2303,47 @@ impl Db { Ok(()) } + /// The resolver key currently stored for a pinned object (`pinned_cids.cid`), + /// or `None` for an unpinned oid. The opportunistic legacy-repair path reads + /// it to decide candidacy from the codec of the string alone (no object bytes) + /// before it recomputes anything. + pub async fn cid_for_oid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get::("cid"))) + } + + /// Rewrite a legacy provider-CID row to the raw-content resolver key, stashing + /// the old provider value in `legacy_provider_cid` (#173 R8, KTD8). Before this + /// branch the pin path stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in + /// `cid`; the `/ipfs` resolver recomputes the raw CID and 404s a mismatched key + /// even though `list_pinned_cids` still advertises it. The `WHERE cid = + /// $old_provider_cid` guard makes a concurrent double-repair a no-op (the second + /// writer sees the already-rewritten key and matches nothing) and never touches + /// a row keyed on a different value. Stashed in `legacy_provider_cid`, NOT + /// `pinata_cid`: the latter gates the Pinata pin-skip (`has_pinata_cid`), so a + /// Kubo-legacy CID parked there would make Pinata permanently skip the object. + pub async fn repair_legacy_provider_cid( + &self, + sha256_hex: &str, + raw_cid: &str, + old_provider_cid: &str, + ) -> Result<()> { + sqlx::query( + "UPDATE pinned_cids + SET cid = $2, legacy_provider_cid = $3 + WHERE sha256_hex = $1 AND cid = $3", + ) + .bind(sha256_hex) + .bind(raw_cid) + .bind(old_provider_cid) + .execute(&self.pool) + .await?; + Ok(()) + } + /// The repository a pinned object was recorded from (`pinned_cids.repo_id`), /// or `None` for a legacy pin (recorded before provenance existed) or an /// unpinned oid. `GET /ipfs/{cid}` uses this to gate+serve the ONE source diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 1a37005d..3cf5610c 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -48,6 +48,74 @@ where } } +/// Opportunistically repair a legacy provider-CID row on the already-pinned skip +/// path (#173 R8, KTD8). Releases before this branch stored the PROVIDER CID +/// (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`; the `/ipfs` resolver +/// recomputes the raw CID from object bytes and 404s any row whose key does not +/// match, yet `list_pinned_cids` still advertises the stored key — so a client +/// gets a CID the resolver deliberately withholds. When a re-push carries the +/// object again, rewrite the key to the raw CID and stash the old provider value +/// in `legacy_provider_cid`. +/// +/// COST GATE: candidacy is decided from the stored key's codec alone — a +/// CIDv1/raw key is already the resolver key and reads NO bytes, keeping the +/// steady-state skip cost DB-only. Only a legacy-codec row reads the object to +/// recompute. A row whose bytes are gone stays withheld (no destructive rewrite). +async fn repair_legacy_provider_cid( + repo_path: &std::path::Path, + sha: &str, + db: &crate::db::Db, +) -> Result<()> { + let stored = match db.cid_for_oid(sha).await? { + Some(c) => c, + None => return Ok(()), + }; + // Cost gate: a canonical raw CIDv1 key is already correct — never read bytes. + if gitlawb_core::cid::is_raw_cidv1(&stored) { + return Ok(()); + } + // Legacy-codec row: read the object to recompute. Counted so a test can prove + // the gate above spares non-legacy rows this read. + #[cfg(test)] + note_legacy_repair_read(); + let data = match crate::git::store::read_object(repo_path, sha)? { + Some((_ty, bytes)) => bytes, + // Bytes gone: the row stays withheld, never destructively rewritten. + None => return Ok(()), + }; + let raw = Cid::from_git_object_bytes(&data).to_string(); + if raw == stored { + return Ok(()); + } + db.repair_legacy_provider_cid(sha, &raw, &stored).await +} + +// Test-only cost-gate counter (R8, U7): how many times the opportunistic repair +// read an object's bytes on the skip path. The codec gate must spare a CIDv1/raw +// row this read; the counter is the both-ways guard (removing the gate reads the +// raw row and increments it). Same thread_local discipline as the serve-path +// oversize counter — the pin tests await `pin_new_objects` on a current-thread +// runtime, so the increment and the assertion share one thread. +#[cfg(test)] +thread_local! { + static LEGACY_REPAIR_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_legacy_repair_reads() { + LEGACY_REPAIR_READS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn legacy_repair_reads() -> usize { + LEGACY_REPAIR_READS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_legacy_repair_read() { + LEGACY_REPAIR_READS.with(|c| c.set(c.get() + 1)); +} + /// Pin a single git object to the local IPFS/Kubo node. /// /// - `ipfs_api`: base URL of the Kubo HTTP API, e.g. `http://127.0.0.1:5001`. @@ -175,6 +243,14 @@ pub async fn pin_new_objects( if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } + // R8 (#173 round 10): opportunistically repair a legacy provider-CID + // row (Kubo dag-pb / Pinata) to the raw-content resolver key on this + // re-push. Cost-gated on the stored key's codec — a non-legacy row + // reads no bytes. Warn-only: a failure leaves the row as-is for a + // later re-push or the deferred one-shot sweep. + if let Err(e) = repair_legacy_provider_cid(repo_path, &sha, db).await { + tracing::warn!(sha = %sha, err = %e, "failed to repair legacy provider CID"); + } continue; } Ok(false) => {} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 01363abf..06b216c1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -3271,6 +3271,335 @@ mod tests { ); } + /// Build a legacy provider CID (CIDv1 dag-pb — the Kubo above-block-size root + /// shape, and codec-equivalent to the Pinata CIDv0 legacy key for the cost + /// gate) over the object's own multihash. Non-raw codec, so `is_raw_cidv1` + /// flags it a repair candidate, and a different string from the raw key, so a + /// repair rewrites it. The existing `ipfs_cid_legacy_provider_cid_row_not_served` + /// fixture seeds a raw-codec decoy (an integrity negative the cost gate treats + /// as non-legacy on purpose); this produces the genuine dag-pb legacy shape the + /// repair path targets. Uses only the `cid` crate (already a node dep). + fn legacy_dagpb_cid(raw_cid: &str) -> String { + const DAG_PB: u64 = 0x70; + let parsed = raw_cid + .parse::>() + .expect("the raw CID parses"); + cid::CidGeneric::<64>::new_v1(DAG_PB, *parsed.hash()).to_string() + } + + /// #173 R8 (jatmn round 10, U7 — load-bearing): a legacy row keyed on a PROVIDER + /// CID (Kubo dag-pb / Pinata) is opportunistically rewritten to the raw-content + /// key on a re-push whose pack carries the object, stashing the old value in + /// `legacy_provider_cid`. The advertised key 404s while the row is legacy (the + /// resolver recomputes the raw CID and the stored key does not match) and serves + /// after repair. RED before the skip-branch repair lands (the raw key 404s post + /// pin). Also asserts the repair leaves `pinata_cid` NULL (scenario 3) and that + /// the retired provider CID still refuses to serve (scenario 6, integrity). + #[sqlx::test] + async fn ipfs_cid_legacy_provider_cid_repaired_on_repush(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provsrc.git"); + let repo = seed_repo(&owner_did, "provsrc"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // The canonical raw key the resolver accepts once the row is repaired. + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .unwrap() + .1, + ) + .to_string(); + // The key stored today: a genuine legacy dag-pb provider CID. + let provider_cid = legacy_dagpb_cid(&raw_cid); + assert_ne!( + provider_cid, raw_cid, + "the provider CID differs from the raw resolver key" + ); + + // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers store the + // raw CID). The object itself is public and servable. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + // RED baseline: the raw key a correct client sends 404s while the row is legacy. + let (st_before, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_before, + StatusCode::OK, + "the raw key 404s while the row is keyed on the provider CID" + ); + + // Re-push carries the object again: `pin_new_objects` hits the already-pinned + // skip branch and repairs the row. The `/add` mock must NOT fire — the object + // is already on IPFS, never re-pinned. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![fx.public_oid.clone()], + &state.db, + &repo.id, + ) + .await; + m.assert_async().await; + + // GREEN: the key is repaired to the raw CID and the old value is stashed. + let (stored_cid, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&fx.public_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored_cid, raw_cid, + "the key is repaired to the raw-content CID" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed in legacy_provider_cid" + ); + + // The advertised (raw) key now serves 200. + let (st_after, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st_after, + StatusCode::OK, + "the repaired raw key serves after the re-push" + ); + assert!(body.contains("public bytes"), "the object's bytes serve"); + + // Scenario 3: repair never wrote `pinata_cid`, so the Pinata pin-skip gate + // (`has_pinata_cid`) is untouched and Pinata still pins the object. + assert!( + !state.db.has_pinata_cid(&fx.public_oid).await.unwrap(), + "repair leaves pinata_cid NULL" + ); + + // Scenario 6 (integrity negative): the retired provider CID still 404s — no + // serve-path alias for a CID the bytes do not hash to. + let (st_old, body_old) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&provider_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_old, + StatusCode::OK, + "the retired provider CID must not serve after repair" + ); + assert!( + !body_old.contains("public bytes"), + "no bytes egress under the retired provider CID" + ); + } + + /// #173 R8 (U7 cost gate): a well-formed CIDv1/raw already-pinned row triggers NO + /// object read on the skip path — the codec check decides candidacy from the + /// stored string alone, so a non-legacy row keeps the DB-only skip cost. Also + /// covers the small-object equivalence: a small legacy object Kubo pins under the + /// raw key (raw-leaves) is already CIDv1/raw and needs no repair. The read counter + /// is the both-ways guard: removing the codec gate reads the raw row and trips it. + #[sqlx::test] + async fn ipfs_cid_repair_codec_gate_skips_raw_row(pool: PgPool) { + let state = test_state(pool).await; + let fx = seed_cid_repos("codecgate", "cg", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("codecgate") + .join("pinsrc.git"); + + // A correct raw-CID row (steady state), recorded via the production helper. + let raw_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + assert!( + gitlawb_core::cid::is_raw_cidv1(&raw_cid), + "the helper records a CIDv1/raw key" + ); + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"x"}"#) + .expect(0) + .create_async() + .await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![fx.public_oid.clone()], + &state.db, + "repoCG", + ) + .await; + m.assert_async().await; + + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a CIDv1/raw row triggers no object read on the skip path (cost gate)" + ); + assert_eq!( + state + .db + .cid_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some(raw_cid.as_str()), + "the raw row is left as-is" + ); + } + + /// #173 R8 (U7): a legacy row whose object bytes are gone stays withheld — the + /// repair never destructively rewrites it, so the row is preserved for a future + /// re-push or the deferred one-shot sweep. + #[sqlx::test] + async fn ipfs_cid_repair_unrepairable_row_stays_withheld(pool: PgPool) { + let state = test_state(pool.clone()).await; + let _fx = seed_cid_repos("unrep", "ur", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("unrep") + .join("pinsrc.git"); + + // A legacy dag-pb row for an oid whose bytes are NOT in this bare repo. + let phantom_oid = "b".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(&phantom_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + let mut server = mockito::Server::new_async().await; + server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"x"}"#) + .expect(0) + .create_async() + .await; + + // Skip-branch runs (is_pinned true) but read_object returns None (bytes gone), + // so the repair returns without touching the row. + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![phantom_oid.clone()], + &state.db, + "repoUR", + ) + .await; + + let (stored, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&phantom_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored, provider_cid, + "an unrepairable row keeps its provider CID (no destructive rewrite)" + ); + assert_eq!( + stashed, None, + "no legacy_provider_cid is stashed when the bytes are gone" + ); + } + + /// #173 R8 (U7, INV-7 upgrade path): a node already at the prior-max schema (v13) + /// gets `pinned_cids.legacy_provider_cid` from the NEW v14 migration. Simulate the + /// pre-v14 node by dropping the column and un-applying v14, then re-migrate and + /// assert a repair round-trips through the column. RED before the v14 migration + /// exists (the column is never re-added → the repair UPDATE errors). + #[sqlx::test] + async fn pinned_cids_legacy_provider_cid_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v14 shape: drop the column and forget v14 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS legacy_provider_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 14") + .execute(&pool) + .await + .unwrap(); + + // Upgrade: re-run migrations → v14 re-adds the column. + state.db.run_migrations().await.expect("migrate to v14"); + + // A repair round-trips through the v14 column. + state + .db + .record_pinned_cid("upg_oid", "QmProviderLegacy", None) + .await + .unwrap(); + state + .db + .repair_legacy_provider_cid("upg_oid", "bRawContentKey", "QmProviderLegacy") + .await + .unwrap(); + let (cid, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = 'upg_oid'", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(cid, "bRawContentKey", "v14 lets the repair rewrite the key"); + assert_eq!( + stashed.as_deref(), + Some("QmProviderLegacy"), + "the v14 legacy_provider_cid column is present after upgrade" + ); + } + /// #173 (provenance-path throttle): a walk-requiring provenanced candidate whose /// per-IP walk quota is spent returns 429 (the provenance arm's Throttled outcome, /// then the fall-through). quota=1, keyed on XFF. The first reader request runs the From fba3c6fda3a70f3cf7b9a1b6260cbbe263540ba5 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 20:27:35 -0500 Subject: [PATCH 51/52] fix(review): share one deadline across the /ipfs size+content reads Code review found each candidate's size and content reads were each granted a full git_service_timeout, so a single served candidate could hold the /ipfs walk permit for 2x the timeout. Share one deadline across both stages (mirrors build_filtered_pack) so the read holds admission for one timeout total, and correct the admission-bound comment: the worst case is O(candidates x timeout) bounded by the walk concurrency cap, not a single deadline. --- crates/gitlawb-node/src/api/ipfs.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index c5fa35bc..624a1b27 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -196,9 +196,13 @@ pub async fn get_by_cid( // instant the handler future is torn down while a spawn_blocking git probe/walk/read // is still alive (the disconnect-spam cap bypass this closes, the /ipfs half of #174 // P1-a). Every git child on this pipeline is duration-bounded (the run_bounded_git - // probe/read twins + the bounded walk), so the detached task cannot hold admission - // past ~git_service_timeout_secs. The client key is already captured (`source_key`) - // before the spawn; the detached task has no request extractors. + // probe/read twins + the bounded walk), and each candidate's size+content read shares + // one deadline; so admission is bounded per candidate. The legacy scan can still + // iterate up to `ipfs_max_legacy_probes` candidates serially, so the worst-case hold + // is O(candidates x git_service_timeout_secs), NOT a single timeout — what actually + // bounds cancel-spam is the walk concurrency cap (global + per-source), not the + // per-child deadline. The client key is already captured (`source_key`) before the + // spawn; the detached task has no request extractors. let admission = crate::git::smart_http::AdmissionGuard::new(ipfs_walk_permit, ipfs_caller_permit); let serve: tokio::task::JoinHandle> = tokio::spawn(async move { @@ -759,9 +763,14 @@ async fn gate_and_serve( // deadline rather than left to pin the held /ipfs walk permit (#173 round-10, KTD2). // No outer timeout, mirroring the bounded walk; a `GitServiceTimeout` from either // twin surfaces as `ServedRead::ReadErr` -> truncated (retryable 503). - let read_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + // ONE deadline spans the size and content reads, so a single served candidate + // holds the /ipfs walk permit for at most `git_service_timeout_secs` total, not + // one full timeout per stage (mirrors `build_filtered_pack`'s shared deadline). + let read_deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.git_service_timeout_secs); let read = tokio::task::spawn_blocking(move || -> ServedRead { - match store::object_size_bounded(&git_bin, &read_repo, &read_sha, read_timeout) { + let size_budget = read_deadline.saturating_duration_since(std::time::Instant::now()); + match store::object_size_bounded(&git_bin, &read_repo, &read_sha, size_budget) { Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), Ok(Some(_)) => {} // git ran and reported no such object (or an unparseable size): genuine @@ -771,12 +780,13 @@ async fn gate_and_serve( // infra/timeout failure, not a not-found. Err(e) => return ServedRead::ReadErr(e.to_string()), } + let content_budget = read_deadline.saturating_duration_since(std::time::Instant::now()); let content = match store::read_object_content_bounded( &git_bin, &read_repo, &read_sha, &read_type, - read_timeout, + content_budget, ) { Ok(c) => c, Err(e) => return ServedRead::ReadErr(e.to_string()), From 62a99270a935324956dcb99abbc620e3ed19750b Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 21:06:10 -0500 Subject: [PATCH 52/52] fix(node): bound the post-push pin and repair reads so a wedge cannot pin the coalescing key A cross-model adversarial pass found that run_post_push_replication holds the per-repo coalescing key until requeue_or_release, but pin_new_objects reached it only after two unbounded git reads: the U7 legacy-repair read and the pre-existing pin read, both plain Command::output with no teardown. A wedged cat-file on a stuck backend hung the task forever, so the key was held until process death and later pushes only marked it dirty without spawning a replacement (the same class this PR closed on the /ipfs serve path). Add read_object_bounded (the bounded twins under one shared deadline) and use it at both sites; a timeout skips that object and lets the task proceed to requeue_or_release. Adds a wedge-reap regression proven load-bearing two ways. --- crates/gitlawb-node/src/api/repos.rs | 2 + crates/gitlawb-node/src/git/store.rs | 36 +++++++ crates/gitlawb-node/src/ipfs_pin.rs | 45 +++++--- crates/gitlawb-node/src/test_support.rs | 131 ++++++++++++++++++++++++ 4 files changed, 201 insertions(+), 13 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index c9a2b3dd..08075e4e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -885,6 +885,8 @@ async fn run_post_push_replication( crate::ipfs_pin::pin_new_objects( &ctx.ipfs_api, &ctx.disk_path, + &ctx.git_bin, + ctx.timeout, object_list.clone(), &ctx.db, &ctx.repo_id, diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index a79d596a..899394ee 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -407,6 +407,42 @@ pub fn read_object(repo_path: &Path, sha256_hex: &str) -> Result Result)>> { + let deadline = std::time::Instant::now() + timeout; + let obj_type = match object_type_bounded( + git_bin, + repo_path, + sha256_hex, + deadline.saturating_duration_since(std::time::Instant::now()), + )? { + Some(t) => t, + None => return Ok(None), + }; + let content = read_object_content_bounded( + git_bin, + repo_path, + sha256_hex, + &obj_type, + deadline.saturating_duration_since(std::time::Instant::now()), + )?; + Ok(Some((obj_type, content))) +} + /// Get the diff between two branches: changes on source_branch not in target_branch. pub fn branch_diff(repo_path: &Path, target_branch: &str, source_branch: &str) -> Result { let output = Command::new("git") diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3cf5610c..3d2f13df 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -63,6 +63,8 @@ where /// recompute. A row whose bytes are gone stays withheld (no destructive rewrite). async fn repair_legacy_provider_cid( repo_path: &std::path::Path, + git_bin: &str, + git_timeout: Duration, sha: &str, db: &crate::db::Db, ) -> Result<()> { @@ -78,10 +80,18 @@ async fn repair_legacy_provider_cid( // the gate above spares non-legacy rows this read. #[cfg(test)] note_legacy_repair_read(); - let data = match crate::git::store::read_object(repo_path, sha)? { - Some((_ty, bytes)) => bytes, + let data = match crate::git::store::read_object_bounded(git_bin, repo_path, sha, git_timeout) { + Ok(Some((_ty, bytes))) => bytes, // Bytes gone: the row stays withheld, never destructively rewritten. - None => return Ok(()), + Ok(None) => return Ok(()), + // A wedged/D-state `git cat-file` (timeout/infra): the repair is opportunistic + // and best-effort, so skip it and return Ok so the pin task PROCEEDS to + // requeue_or_release rather than hanging the coalescing key until process death + // (grok F2, #173). A later re-push or the deferred sweep retries the repair. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "skipping legacy provider-CID repair: bounded object read failed"); + return Ok(()); + } }; let raw = Cid::from_git_object_bytes(&data).to_string(); if raw == stored { @@ -204,6 +214,8 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { pub async fn pin_new_objects( ipfs_api: &str, repo_path: &std::path::Path, + git_bin: &str, + git_timeout: Duration, object_list: Vec, db: &crate::db::Db, repo_id: &str, @@ -248,7 +260,9 @@ pub async fn pin_new_objects( // re-push. Cost-gated on the stored key's codec — a non-legacy row // reads no bytes. Warn-only: a failure leaves the row as-is for a // later re-push or the deferred one-shot sweep. - if let Err(e) = repair_legacy_provider_cid(repo_path, &sha, db).await { + if let Err(e) = + repair_legacy_provider_cid(repo_path, git_bin, git_timeout, &sha, db).await + { tracing::warn!(sha = %sha, err = %e, "failed to repair legacy provider CID"); } continue; @@ -260,15 +274,20 @@ pub async fn pin_new_objects( } } - // Read raw object content - let data = match crate::git::store::read_object(repo_path, &sha) { - Ok(Some((_obj_type, bytes))) => bytes, - Ok(None) => continue, - Err(e) => { - tracing::warn!(sha = %sha, err = %e, "failed to read git object for pinning"); - continue; - } - }; + // Read raw object content under a bounded read so a wedged/D-state `git + // cat-file` (stuck NFS/Tigris backend) is reaped at `git_timeout` instead of + // hanging pin_new_objects forever — which would pin the post-push coalescing + // key until process death (grok F2, #173). On Err the object is simply not + // pinned this pass; a later pass/push retries. + let data = + match crate::git::store::read_object_bounded(git_bin, repo_path, &sha, git_timeout) { + Ok(Some((_obj_type, bytes))) => bytes, + Ok(None) => continue, + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to read git object for pinning"); + continue; + } + }; // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data).await { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 06b216c1..0d0ae2e8 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2430,6 +2430,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &pub_bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, &pub_repo.id, @@ -2553,6 +2555,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &pub_bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, &pub_repo.id, @@ -3120,6 +3124,8 @@ mod tests { let pinned = crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, "repoZ", @@ -3142,6 +3148,123 @@ mod tests { ); } + /// #173 (grok F2): the post-push pin read is BOUNDED, so a wedged/D-state + /// `git cat-file` (stuck NFS/Tigris backend) is reaped at `git_timeout` and + /// `pin_new_objects` RETURNS — reaching `requeue_or_release` in production — + /// instead of hanging forever and pinning the per-repo coalescing key until + /// process death. A fake `git` whose `cat-file` records its pid then sleeps far + /// past a SHORT 1s timeout stands in for the wedged backend; the `run_bounded_git` + /// watchdog (SIGTERM -> grace -> SIGKILL of the process group) must reap it well + /// before its 8s natural exit, and the call must return with nothing pinned. + /// + /// REVERT PROOF (RED): swap `read_object_bounded` back to the bare + /// `store::read_object` at the pin read and the wedged child is STILL RUNNING at + /// the mid-flight liveness poll below (unbounded `Command::output` cannot be + /// reaped at the deadline) — the reap assertion fails. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_reaps_wedged_read_at_deadline(pool: PgPool) { + use std::time::Duration; + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Fake `git`: `cat-file` records its own pid then sleeps 8s (>> the 1s + // deadline) so the read is genuinely wedged; the watchdog is what must end it. + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("catfile.pid"); + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + cat-file) echo $$ > \"{}\"; sleep 8 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pidfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let repo = tmp.path().to_path_buf(); + let git = git_path.to_str().unwrap().to_string(); + // A never-pinned OID so the call reaches the object-read stage (not the + // already-pinned skip path). + let oid = "f".repeat(64); + // Non-empty ipfs_api so `pin_new_objects` does not early-return; the wedged + // read is reaped and the OID skipped before any `/add`, so this URL is unused. + let ipfs_api = "http://127.0.0.1:1".to_string(); + + // `pin_new_objects` must run on THIS runtime so its `is_pinned` DB call keeps + // the sqlx pool on its home runtime. The bounded read is a synchronous blocking + // call, so the reap poll runs on a separate OS thread (independent of tokio): it + // captures the wedged child's pid, waits past the deadline, records whether it + // was reaped, then SIGKILLs defensively so even a true infinite hang cannot leak + // an orphan or stall the awaited call. + let pidfile_poll = pidfile.clone(); + let poll = std::thread::spawn(move || -> (Option, bool) { + let alive = |pid: i32| unsafe { libc::kill(pid, 0) == 0 }; + let mut pid = None; + for _ in 0..500 { + if let Some(p) = std::fs::read_to_string(&pidfile_poll) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + pid = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid = match pid { + Some(p) => p, + None => return (None, false), + }; + // Past the 1s deadline + SIGTERM grace but well before the 8s natural exit: + // the bounded read must already have reaped the wedged group. The unbounded + // `store::read_object` leaves it running here — the load-bearing RED. + std::thread::sleep(Duration::from_secs(3)); + let reaped = !alive(pid); + unsafe { + libc::kill(pid, libc::SIGKILL); + } + (Some(pid), reaped) + }); + + // The call must RETURN — reaching `requeue_or_release` in production — rather + // than hang on the 8s sleep. The poll thread's defensive SIGKILL guarantees the + // read completes even in the unbounded RED case, so this observes a bounded + // return either way; the reap assertion below is what separates RED from GREEN. + let pinned = tokio::time::timeout( + Duration::from_secs(6), + crate::ipfs_pin::pin_new_objects( + &ipfs_api, + &repo, + &git, + Duration::from_secs(1), + vec![oid], + &db, + "repoWedge", + ), + ) + .await + .expect("pin_new_objects must return within the bound, not hang on the wedged read"); + + let (pid, reaped) = poll.join().expect("poll thread joins"); + pid.expect("the fake cat-file must have spawned and recorded its pid"); + assert!( + reaped, + "the post-push pin read must reap the wedged cat-file child at the deadline, \ + not leave it running (which would pin the coalescing key until process death)" + ); + assert!( + pinned.is_empty(), + "a wedged read pins nothing this pass; a later pass/push retries" + ); + } + /// #173 (jatmn, F2): a legacy pin with NULL provenance backfills its source /// via `backfill_pin_provenance`, and the `AND repo_id IS NULL` guard preserves /// first-pinner-owns (a non-NULL provenance is left untouched). @@ -3248,6 +3371,8 @@ mod tests { let pinned = crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, "repoBF", @@ -3367,6 +3492,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, &repo.id, @@ -3468,6 +3595,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, "repoCG", @@ -3530,6 +3659,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![phantom_oid.clone()], &state.db, "repoUR",