diff --git a/.env.example b/.env.example index bbd9a342..1c390a1c 100644 --- a/.env.example +++ b/.env.example @@ -109,15 +109,73 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Max seconds a served git upload-pack / receive-pack (clone / push) may run # before it is aborted with a 504. Bounds a hung git that would otherwise pin a -# worker and, on push, the repo write lock. Does NOT cover the info/refs -# advertisement or the withheld-blob path, which remain unbounded. Default 600. +# worker and, on push, the repo write lock. Also bounds both info/refs +# advertisements, the withheld-blob pack build, and the push-side candidate +# discovery (rev-list / cat-file), all reaped via process-group teardown (#174). +# Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max seconds the storage-ACQUISITION phase of a served git op may run before the +# request is shed with a 503, separate from the git-run timeout above. A +# concurrency permit is taken before this phase and GITLAWB_GIT_SERVICE_TIMEOUT_SECS +# only starts once git spawns, so without this a stalled backend (a hung Tigris +# HEAD/GET, or a hung pg advisory-lock iteration on push) pins the permit and drains +# the pool until every later request 503s. On expiry the permit is released +# (fail-closed). Kept separate because acquisition and git execution are distinct +# cost centers. Must be positive; set very large to effectively disable. Default 30. +GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS=30 + +# Max concurrent git READ ops (upload-pack + the upload-pack info/refs +# advertisement) served at once, a global pool separate from the push pool below. +# The anon receive-pack info/refs advertisement has its OWN pool (see below), not +# this one. Over-cap sheds a clean 503 + Retry-After. Anonymous reads draw from +# here, so pair it with GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one +# caller cannot monopolize the pool. Default 128. +GITLAWB_MAX_CONCURRENT_GIT_OPS=128 + +# Max concurrent git-receive-pack (push) POST operations, in a pool separate from +# the read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an +# authenticated push at admission. The anon receive-pack info/refs advertisement +# runs in a SEPARATE pool of the same size (disjoint from this one), so an +# advertisement flood cannot shed a push either. Over-cap sheds a 503 + +# Retry-After. Default 32. +GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 + +# Max concurrent read ops (upload-pack + the upload-pack info/refs advertisement) +# a single caller may hold, so one caller cannot monopolize the read pool. Keyed +# on the resolved SOURCE IP, never the DID: a signature does not move a caller off +# this cap. The source-IP key is only as granular as GITLAWB_TRUSTED_PROXY below: +# left unset, a node behind an edge/NAT keys all callers on the edge IP and this +# collapses to one global cap. Set GITLAWB_TRUSTED_PROXY for per-client keying; a +# high-fanout caller (CI behind one NAT) then needs the operator to raise this. +# Default 16. +GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 + # ── Push rate limiting (git-receive-pack flood brake) ───────────────────── # Max receive-pack requests (info/refs advertisement + push POST) per client # IP per hour. 0 disables. Default 600. GITLAWB_PUSH_RATE_LIMIT=600 +# ── /ipfs/{cid} visibility-walk admission (#174) ────────────────────────── +# GET /ipfs/{cid} runs a per-repo full-history git walk in a blocking thread to +# decide whether the caller may read a path-scoped blob. It is publicly reachable, +# so it is bounded to keep a permissionless caller from fanning out unbounded +# concurrent walks and exhausting blocking-pool threads + PIDs. +# Max concurrent /ipfs walks across all callers (a pool of its own, disjoint from +# the served-git pools). Over-cap sheds a 503. Default 32. +GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 +# Max concurrent /ipfs walks a single SOURCE IP may hold (keyed like the git +# per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). +# Default 4. +GITLAWB_IPFS_WALK_PER_SOURCE=4 +# Max legacy (NULL-provenance) repos probed per single /ipfs request, bounding the +# scan-fallback fan-out (git cat-file per candidate repo) for an anonymous caller. A +# truncated scan sheds a retryable 503, never a false 404. Default 256. +GITLAWB_IPFS_MAX_REPOS_WALKED=256 +# Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct +# from the concurrency caps above). 0 disables. Default 600. +GITLAWB_IPFS_RATE_LIMIT=600 + # ── Creation rate limiting (repo/agent/issue/PR flood brake) ────────────── # Max creation requests (POST /api/v1/repos, /api/register, fork, issues, # pulls) per client IP per hour, in addition to the per-DID limit. The per-DID diff --git a/Cargo.lock b/Cargo.lock index d12daa43..24beb9ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "git-remote-gitlawb" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "gitlawb-core", @@ -3312,7 +3312,7 @@ dependencies = [ [[package]] name = "gitlawb-attest" -version = "0.5.0" +version = "0.5.1" dependencies = [ "base64", "ed25519-dalek", @@ -3329,7 +3329,7 @@ dependencies = [ [[package]] name = "gitlawb-core" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "base64", @@ -3356,7 +3356,7 @@ dependencies = [ [[package]] name = "gitlawb-node" -version = "0.5.0" +version = "0.5.1" dependencies = [ "alloy", "anyhow", @@ -3412,7 +3412,7 @@ dependencies = [ [[package]] name = "gl" -version = "0.5.0" +version = "0.5.1" dependencies = [ "alloy", "anyhow", @@ -3824,6 +3824,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "tracing", ] diff --git a/README.md b/README.md index 57ce2885..80f71cb7 100644 --- a/README.md +++ b/README.md @@ -343,13 +343,20 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack/receive-pack may run before it is aborted (504). Default 600. Does not bound `info/refs` or the withheld-blob path. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | +| `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | +| `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | +| `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max legacy (NULL-provenance) repos probed per `/ipfs/{cid}` request, bounding the scan-fallback fan-out. A truncated scan returns a retryable 503, not a false 404. Default 256. | +| `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | Production note: change the default Postgres password before exposing a node publicly. +Legacy-pin window: releases before the CID-resolver work stored the provider CID (Kubo dag-pb / Pinata) as a pinned object's resolver key. The `/ipfs/{cid}` resolver now recomputes the raw-content CID from the object bytes and refuses to serve a key that does not match, so `GET /api/v1/ipfs/pins` can still advertise an unrepaired legacy CID that 404s. Such a row is repaired opportunistically the next time a push carries the object again (its key is rewritten to the raw CID, the old value kept in `legacy_provider_cid`), but git negotiation omits objects the node already has, so most legacy rows never re-enter a push delta. A deferred one-shot startup sweep, not this opportunistic path, is what fully retires the advertise-then-404 window. Rows whose object bytes are gone stay withheld. + --- ## Optional node staking diff --git a/crates/gitlawb-core/src/cid.rs b/crates/gitlawb-core/src/cid.rs index b7993cc4..2071d478 100644 --- a/crates/gitlawb-core/src/cid.rs +++ b/crates/gitlawb-core/src/cid.rs @@ -64,6 +64,19 @@ impl Cid { } } +/// True when `s` parses as a CIDv1 with the raw codec — the exact shape +/// [`Cid::from_git_object_bytes`] produces and the `/ipfs` resolver looks up. +/// A legacy provider CID (Kubo dag-pb, Pinata CIDv0) parses to a different +/// version or codec and returns `false`, marking it an opportunistic-repair +/// candidate. Decidable from the string alone (no object bytes), so the pin path +/// can gate the byte-read/recompute cost on it and leave non-legacy rows at the +/// existing DB-only skip cost. An unparseable string is non-canonical (`false`). +pub fn is_raw_cidv1(s: &str) -> bool { + s.parse::>() + .map(|c| c.version() == cid::Version::V1 && c.codec() == RAW) + .unwrap_or(false) +} + impl fmt::Display for Cid { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) @@ -177,6 +190,38 @@ mod tests { assert!(result.is_err()); } + #[test] + fn is_raw_cidv1_classifies_codec_from_string() { + // The canonical resolver key: CIDv1 + raw codec → not a repair candidate. + let raw = Cid::from_git_object_bytes(b"blob 5\0hello"); + assert!( + is_raw_cidv1(raw.as_str()), + "from_git_object_bytes output is CIDv1/raw" + ); + + // A CIDv0 (Pinata dag-pb legacy shape) → repair candidate. + assert!( + !is_raw_cidv1("QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"), + "a CIDv0 dag-pb value is a legacy-repair candidate" + ); + + // A CIDv1 with the dag-pb codec (the Kubo above-block-size root) over the + // same multihash → still a repair candidate (codec, not just version). + let parsed = raw.as_str().parse::>().unwrap(); + const DAG_PB: u64 = 0x70; + let dagpb = CidGeneric::<64>::new_v1(DAG_PB, *parsed.hash()).to_string(); + assert!( + !is_raw_cidv1(&dagpb), + "a CIDv1 dag-pb value is a legacy-repair candidate" + ); + + // Garbage is non-canonical. + assert!( + !is_raw_cidv1("not-a-cid"), + "an unparseable string is non-canonical" + ); + } + #[test] fn sha256_hex_of_empty_input_is_well_known() { // SHA-256("") is a fixed constant; verifies the hasher is wired correctly. diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f3de7570..624a1b27 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -1,14 +1,17 @@ //! GET /ipfs/{cid} — content-addressed retrieval of git objects by CIDv1. //! -//! Every git object stored on this node is addressable by its IPFS CIDv1. +//! Every git object pinned on this node is addressable by its IPFS CIDv1. //! The CID is computed as: //! //! CIDv1(codec=raw, multihash=sha2-256(content_bytes)) //! //! where `content_bytes` is the raw object content as returned by -//! `git cat-file ` (i.e. without the git framing header). -//! This is consistent with how `gitlawb_core::cid::Cid::from_git_object_bytes` -//! computes CIDs when objects are pushed. +//! `git cat-file ` (i.e. without the git framing header) — the +//! same bytes `gitlawb_core::cid::Cid::from_git_object_bytes` hashes when the +//! object is pinned. That digest is NOT the object's git oid: git frames the +//! content with a `" \0"` header before hashing, so `sha2-256(content)` +//! and the git oid differ. The handler therefore maps the CID back to its oid via +//! the `pinned_cids` table rather than treating the digest as an oid (#173). //! //! Serving is access-controlled: an object is returned only from a repo row the //! requesting caller is permitted to read (per-caller path-scoped visibility, @@ -27,14 +30,75 @@ use std::str::FromStr; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; use crate::git::store; -use crate::git::visibility_pack::{allowed_blob_set_for_caller, has_path_scoped_rule}; +use crate::git::visibility_pack::{ + allowed_blob_set_for_caller_bounded, allowed_tree_set_for_caller_bounded, has_path_scoped_rule, + reachable_commit_tag_oids_bounded, +}; use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; +/// Hard ceiling on the number of full-history reachability walks a single +/// `GET /ipfs/{cid}` request may spawn. The route brake (`ipfs_rate_limiter`, charged +/// once per request by the middleware) caps request RATE, and the per-walk charge on +/// the separate `ipfs_work_rate_limiter` bounds the walk work across requests, but +/// within ONE request the object can exist under path-scoped rules in many repos, and +/// each distinct repo pays its own `spawn_blocking` walk (the memo only dedups the same +/// repo). Without a ceiling a single request fans out to O(repos) walks — an +/// amplification sink (INV-10). Once this many walks have run, no further walk is +/// spawned for the rest of the request: any remaining candidate that still needs +/// a walk is skipped (and, with nothing else readable, the request falls through +/// to the opaque 404). The bound is deliberately generous: a legitimate caller +/// serves on the first repo that grants them, so reaching it requires being +/// denied by this many path-scoped repos first, which real traffic effectively +/// never does. Tunable if that assumption stops holding. +/// +/// Kept at `MAX_PIN_SOURCES + 1` so the ceiling can never truncate a request +/// BEFORE its whole bounded provenance source set (first-pinner + up to +/// `MAX_PIN_SOURCES` additional) has been tried: an authorizing public source that +/// sorts after `MAX_PIN_SOURCES` path-scoped denials must still be reached and +/// served, not falsely 503'd as a truncated search. The legacy scan's fan-out is +/// separately bounded by `MAX_LEGACY_PROBES_PER_REQUEST`, so widening this by one +/// does not loosen that path. +pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = crate::db::MAX_PIN_SOURCES as u32 + 1; + +/// Hard per-request ceiling on how many legacy (NULL-provenance) repositories +/// the CID resolver's scan fallback may PROBE (`acquire` + `git cat-file -t`). +/// The provenance path targets one repo; the legacy scan, absent this bound, +/// fans one anonymous request out to O(repos) subprocess spawns and cold-cache +/// Tigris fetches for a CID enumerable from the public pins index (#173 round 3, +/// F1, INV-10). Deliberately generous: a normal node has far fewer repos than +/// this, so a genuine miss still completes the whole scan and returns a truthful +/// 404; only a node larger than the cap truncates, and a truncated search +/// surfaces as a retryable 503 (never a false "absent"). Legacy pins are a +/// shrinking set — each re-pin backfills provenance — so this fallback is a +/// transitional path, not the steady state. Tunable via `AppState`. +pub(crate) const MAX_LEGACY_PROBES_PER_REQUEST: u32 = 256; + +/// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` buffers and serves +/// (#173 round 8, F6, INV-10). The serve reads via a blocking `git cat-file` and +/// buffers the whole object; unbounded, a large public blob (enumerable from the pins +/// index) could exhaust memory or block a runtime worker. A content-addressed serve +/// must verify the whole object hashes to the requested CID before any byte egresses +/// (F2), so it cannot stream — it buffers up to this cap and withholds anything larger +/// (raise the cap if a class of legitimate objects legitimately exceeds it; never +/// stream unverified). 32 MiB is generous for git blobs/trees/commits. Tunable via +/// `AppState` for the test seam, like the sibling caps. +pub(crate) const MAX_SERVED_OBJECT_BYTES: u64 = 32 * 1024 * 1024; + +/// Lazily-loaded context for the legacy (NULL-provenance) scan fallback in +/// `get_by_cid`: all repos, their visibility rules keyed by repo id, and the set of +/// quarantined repo ids. Loaded once per request only if a legacy pin is hit. +type LegacyScanCtx = ( + Vec, + HashMap>, + HashSet, +); + /// GET /ipfs/{cid} /// -/// Search all repos on the node for a git object whose SHA-256 hash matches -/// the given CIDv1, returning its raw content if the caller may read it. +/// Resolve the CIDv1 to its git oid via the `pinned_cids` table, then search all +/// repos on the node for that object, returning its raw content if the caller may +/// read it. /// /// Visibility (#110, #126): the object is served only from a repo row the /// caller passes. For each iterated row we gate against that row's OWN rules @@ -43,13 +107,16 @@ use crate::visibility::{visibility_check, Decision}; /// row than the one read (KTD2a). We check object existence via /// `store::object_type` *before* the expensive reachability walk so random-CID /// spray cannot trigger full-history git walks on repos that don't carry the -/// object. When the row carries path-scoped rules (KTD4) the served object -/// must be either a non-blob (trees/commits are structural; KTD3) OR a blob -/// in the caller's *reachable* allowed-set (`allowed_blob_set_for_caller`). -/// The reachable allowed-set excludes dangling blobs — a blob written via -/// `git hash-object -w` and never committed has no path to gate, so it is -/// fail-closed 404'd under path-scoped rules (#126). Denial and genuine -/// not-found both fall through to an opaque 404. +/// object. When the row carries path-scoped rules (KTD4) the served object is +/// gated by type: a `blob`/`tree` must be in the caller's *reachable* allowed-set +/// (`allowed_blob_set_for_caller` / `allowed_tree_set_for_caller`), and a +/// `commit`/`tag` must be in the repo's *reachable* commit/tag set +/// (`reachable_commit_tag_oids`, #173). A withheld subtree's tree object is denied +/// here exactly as `get_tree` denies its path, so its child names and oids cannot +/// leak by CID (#135). All these sets exclude dangling objects — a blob, tree, +/// commit, or tag written via plumbing and never referenced has no reachable path, +/// so it is fail-closed 404'd under path-scoped rules (#126, #173). Denial and +/// genuine not-found both fall through to an opaque 404. /// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked @@ -57,9 +124,12 @@ use crate::visibility::{visibility_check, Decision}; pub async fn get_by_cid( Path(cid_str): Path, State(state): State, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: HeaderMap, auth: Option>, ) -> Result { - // 1. Decode the CID and extract the SHA-256 digest + // 1. Decode and validate the CID (uniform 400 on a malformed / non-sha2-256 + // CID, before any DB or git work). let cid = CidGeneric::<64>::from_str(&cid_str) .map_err(|e| AppError::BadRequest(format!("invalid CID: {e}")))?; @@ -72,143 +142,712 @@ pub async fn get_by_cid( )); } - let sha256_hex = hex::encode(mh.digest()); - let caller = auth.as_ref().map(|e| e.0 .0.as_str()); - let caller_owned = caller.map(|c| c.to_string()); + // Canonicalize the CID for the pinned_cids lookup. Pins are stored under the + // canonical base32 `cid.to_string()`, but a client may send any equivalent + // multibase spelling (base58/base64) of the same CID; those parse and pass + // the sha2-256 check yet miss the canonical key, so they must be normalized + // before the DB lookup (#173). Response headers and error messages still echo + // the original `cid_str` the client sent. + let canonical_cid = cid.to_string(); - // 2. Search all repos for an object with this SHA-256 - let repos = state - .db - .list_all_repos() - .await - .map_err(AppError::Internal)?; + // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds + // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with + // no served-git admission of its own; a permissionless caller could otherwise fan + // out concurrent walks past every git pool, exhausting the blocking pool + PIDs. + // Acquire the global permit (and, for a resolvable source, the per-source + // sub-permit) ONCE here and hold BOTH for the whole request — across every + // `spawn_blocking` walk below — so the slot reflects real blocking-thread + // occupancy (a tokio walk-timeout cannot free it while the blocking work still runs) + // and one request cannot open more than its share of concurrent walks. Holding a + // slot across a walk is only safe because every walk child is duration-bounded + // (`*_bounded` + `run_bounded_git` teardown), so a hung git cannot pin the slot + // past `git_service_timeout_secs`. On unavailability shed a clean 503. The + // per-source key is the resolved source IP (`client_key`), never the DID (`/ipfs` + // admits any `did:key` unthrottled, so a DID key would be free to mint around); a + // `None` key (no trusted header, no peer) is bounded by the global pool only, + // never the per-source sub-cap. + let ipfs_walk_permit = state + .git_ipfs_walk_semaphore + .clone() + .try_acquire_owned() + .map_err(|_| { + tracing::warn!("/ipfs walk concurrency cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity, retry shortly".into()) + })?; + let source_key = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust); + let ipfs_caller_permit = match &source_key { + Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { + tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity for this source, retry shortly".into()) + })?), + None => None, + }; - // Fetch every repo's visibility rules in one query rather than one per row - // (the gate runs each row against its OWN rules — KTD2a). A row absent from - // the map has no rules. - let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = state - .db - .list_visibility_rules_for_repos(&repo_ids) - .await - .map_err(AppError::Internal)?; + // Caller DID (owned) resolved before the spawn: the detached task below cannot + // borrow the handler's `auth` extension. + let caller_owned = auth.as_ref().map(|e| e.0 .0.as_str().to_string()); - // Request-scoped memo of the per-repo allowed-blob set (KTD1, #126). The - // caller is constant for one request, so `repo.id` alone is a safe, - // sufficient key — never a coarse caller "class", which - // `visibility_check`'s exact full-DID reader match would make unsafe. - // - // We flipped from a deny-set (`withheld_blob_oids`) to an allowed-set - // (`allowed_blob_set_for_caller`) so dangling blobs — never enumerated by - // the reachable walk — fail closed instead of slipping through an empty - // deny entry (#126). - let mut allowed_memo: HashMap> = HashMap::new(); - - for repo in &repos { - // Repo-level read gate against THIS row's own rules (KTD2a). - let rules: &[crate::db::VisibilityRule] = rules_by_repo - .get(&repo.id) - .map(Vec::as_slice) - .unwrap_or(&[]); - if visibility_check(rules, repo.is_public, &repo.owner_did, caller, "/") == Decision::Deny { - continue; - } - - let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { - Ok(p) => p, - Err(_) => continue, - }; + // #173 round-10 (R1, KTD1): move BOTH admission permits into an AdmissionGuard OWNED + // by a detached tokio task that runs the whole gated serve pipeline, and have the + // handler await its JoinHandle. Dropping the request future on a client disconnect + // drops the JoinHandle, which DETACHES the task (tokio does not abort a task when its + // handle drops) rather than cancelling it — so the pipeline runs to its bounded + // completion and releases admission only then, instead of the permits dropping the + // instant the handler future is torn down while a spawn_blocking git probe/walk/read + // is still alive (the disconnect-spam cap bypass this closes, the /ipfs half of #174 + // P1-a). Every git child on this pipeline is duration-bounded (the run_bounded_git + // probe/read twins + the bounded walk), and each candidate's size+content read shares + // one deadline; so admission is bounded per candidate. The legacy scan can still + // iterate up to `ipfs_max_legacy_probes` candidates serially, so the worst-case hold + // is O(candidates x git_service_timeout_secs), NOT a single timeout — what actually + // bounds cancel-spam is the walk concurrency cap (global + per-source), not the + // per-child deadline. The client key is already captured (`source_key`) before the + // spawn; the detached task has no request extractors. + let admission = + crate::git::smart_http::AdmissionGuard::new(ipfs_walk_permit, ipfs_caller_permit); + let serve: tokio::task::JoinHandle> = tokio::spawn(async move { + // The guard is held for the whole task and drops LAST — after the response is + // built — releasing admission exactly once when the pipeline is truly done. + let _admission = admission; - // Check whether the object exists in this repo before any expensive - // reachability walk. This prevents random-CID spray from triggering - // full-history git walks on repos that don't carry the object. - let obj_type = match store::object_type(&repo_path, &sha256_hex) { - Ok(Some(t)) => t, - Ok(None) => continue, - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); - continue; - } + // Resolve the content-addressed CID to the object's git oid(s). A real pin + // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the + // git oid (git frames content with a `" \0"` header first), so we + // map it back through `pinned_cids` rather than treating the digest as an oid + // (#173). The cid index is non-unique, so one CID can map to several oids (a + // tree and a blob whose raw bytes collide, or content pinned under two oids); + // we try each candidate below rather than pick one arbitrarily and false-404 + // when the chosen one is withheld or absent while another is readable (#173). + // An empty result is an opaque 404, uniform with a genuine not-found and a + // visibility denial. + let oids = state + .db + .oids_for_cid(&canonical_cid) + .await + .map_err(AppError::Internal)?; + if oids.is_empty() { + return Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))); + } + let caller = caller_owned.as_deref(); + + // Per-request walk budget + memos + throttle flag, shared by the provenance path + // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and + // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. + let mut walk = WalkState { + walks: 0, + probes: 0, + truncated: false, + allowed_blob_memo: HashMap::new(), + allowed_tree_memo: HashMap::new(), + reachable_ct_memo: HashMap::new(), + }; + // Set when a walk-requiring candidate is skipped because the source IP's walk quota + // is spent (#173 review, F-C): the scan keeps going so a later walk-free copy still + // serves; only if nothing is servable is it turned into the 429. + let mut throttled = false; + let rctx = ResolveCtx { + caller, + caller_owned: &caller_owned, + headers: &headers, + peer, + cid_str: &cid_str, + canonical_cid: &canonical_cid, }; - // Per-blob gating only applies when a path-scoped rule exists (KTD4). - // Without any path-scoped rule, the "/" gate above is the whole story. - // Trees/commits are always served under path-scoped rules (KTD3). - let path_scoped = has_path_scoped_rule(rules); - if path_scoped && obj_type == "blob" { - if !allowed_memo.contains_key(&repo.id) { - let rp = repo_path.clone(); - let r = rules.to_vec(); - let is_public = repo.is_public; - let owner = repo.owner_did.clone(); - let caller_for_walk = caller_owned.clone(); - // Full-history walk shells out to git — keep it off the async runtime. - let walk = tokio::task::spawn_blocking(move || { - allowed_blob_set_for_caller( - &rp, - &r, - is_public, - &owner, - caller_for_walk.as_deref(), - ) - }) - .await; - // Fail closed on EITHER a task panic (JoinError) or a walk error: - // we cannot prove the caller may read here, so skip this repo and - // let a public copy (if any) serve. Never serve on an unproven gate. - let set = match walk { - Ok(Ok(set)) => set, - Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed; skipping repo"); + // Legacy scan context (repos + rules + quarantined ids), loaded LAZILY only when a + // legacy NULL-provenance pin is hit — the provenance path must never trigger the + // O(repos) load (that fan-out is exactly what provenance removes, #173 round 2). + let mut scan_ctx: Option = None; + + for sha256_hex in &oids { + // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). + // Resolve a PROVENANCED pin by trying each source repo (bounded to + // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no + // scan fan-out. A shared object first pinned from a private/quarantined repo + // still serves from a later PUBLIC source. Deterministic (ORDER BY on the + // union), so no ordering can turn an authorized copy into a 404. + let sources = state + .db + .pin_sources_for_oid(sha256_hex) + .await + .map_err(AppError::Internal)?; + // Provenance fast-path: try each recorded source repo through the SAME gate + // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance + // pin. The first source that authorizes serves — no scan fan-out on the common + // path. + for repo_id in &sources { + let repo = match state + .db + .get_repo_by_id(repo_id) + .await + .map_err(AppError::Internal)? + { + Some(r) => r, + // A source repo is gone: skip it; a later source or the scan fallback + // below may still resolve. + None => continue, + }; + let quarantined = state + .db + .is_repo_quarantined(repo_id) + .await + .map_err(AppError::Internal)?; + let rules_map = state + .db + .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) + .await + .map_err(AppError::Internal)?; + let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); + match gate_and_serve( + &state, + &repo, + rules, + quarantined, + sha256_hex, + &rctx, + &mut walk, + false, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + GateOutcome::Throttled => { + throttled = true; continue; } - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked; skipping repo"); + GateOutcome::Skip => continue, + } + } + + // Bounded legacy-scan fallback. Run it when the provenance set could not have + // served the caller AND may be INCOMPLETE: + // - empty -> a legacy NULL-provenance pin (recorded before provenance existed), or + // - at_cap -> `record_pin_source` stops inserting at MAX_PIN_SOURCES and drops + // later sources SILENTLY, so a full table may hide a servable source + // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the + // pin-source griefing hole). The scan gates every repo through the + // real per-caller gate, so it finds that copy. + // A non-empty, non-full set is COMPLETE (every recorded source was just tried), so + // skip the scan and let the tail 404 — ordinary denials never fan out to O(repos) + // (INV-10 / F3). The at_cap query runs only on a provenance MISS (we return above + // on Served), so it never costs the serve path. + let needs_scan = sources.is_empty() + || state + .db + .pin_sources_at_cap(sha256_hex) + .await + .map_err(AppError::Internal)?; + if needs_scan { + // F3 (#173, INV-10/INV-15): peek the per-IP WORK-budget limiter WITHOUT + // consuming a token so an already-throttled source is shed BEFORE the + // O(repos) preload; the consuming per-probe charge inside gate_and_serve is + // left UNCHANGED (it is load-bearing for the across-request bound), so this + // adds no double-charge. This peeks `ipfs_work_rate_limiter`, the SAME bucket + // the per-probe charge below debits — NOT the route limiter (`ipfs_rate_limiter`, + // charged once per request by the middleware): peeking the route bucket here + // would re-shed a request the route already admitted (R6, U5). + if let Some(key) = + crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) + { + if state.ipfs_work_rate_limiter.is_throttled(&key).await { + throttled = true; continue; } - }; - allowed_memo.insert(repo.id.clone(), set); + } + // Load the scan context once, lazily (shared across oid candidates). + if scan_ctx.is_none() { + #[cfg(test)] + bump_preload_queries(); + let repos = state + .db + .list_all_repos() + .await + .map_err(AppError::Internal)?; + let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); + let rules_by_repo = state + .db + .list_visibility_rules_for_repos(&repo_ids) + .await + .map_err(AppError::Internal)?; + let quarantined: HashSet = state + .db + .list_quarantined_repos() + .await + .map_err(AppError::Internal)? + .into_iter() + .map(|r| r.id) + .collect(); + scan_ctx = Some((repos, rules_by_repo, quarantined)); + } + let (repos, rules_by_repo, quarantined) = scan_ctx.as_ref().unwrap(); + for repo in repos { + let rules = rules_by_repo + .get(&repo.id) + .map(Vec::as_slice) + .unwrap_or(&[]); + let is_quar = quarantined.contains(&repo.id); + match gate_and_serve( + &state, repo, rules, is_quar, sha256_hex, &rctx, &mut walk, true, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + // A throttled walk-requiring candidate is skipped, not fatal: + // keep scanning for a later walk-free copy (#173 review, F-C). + GateOutcome::Throttled => throttled = true, + GateOutcome::Skip => {} + } + } } - let in_allowed = allowed_memo - .get(&repo.id) - .is_some_and(|set| set.contains(&sha256_hex)); - if !in_allowed { - continue; + } + + // Nothing served — three distinct tails, in precedence order: + // 1. The scan was cut short by a cap (legacy probe ceiling or walk ceiling), so + // the object was NOT proven absent/unreadable everywhere → 503, retryable, and + // explicitly NOT a definitive not-found (#173, F2). This outranks the throttle: + // an incomplete search must not masquerade as a clean rate-limit outcome, and + // it carries only the caller-supplied CID (no object/OID/metadata leak). + // 2. A walk-requiring candidate was skipped for a spent IP quota while the scan + // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). + // 3. A full scan under the caps found nothing readable → opaque 404, uniform with + // a genuine not-found and a visibility denial. + if walk.truncated { + return Err(AppError::SearchIncomplete(format!( + "CID {cid_str} search incomplete — retry" + ))); + } + if throttled { + return Err(AppError::TooManyRequests( + "ipfs retrieval rate limit exceeded — try again later".into(), + )); + } + Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))) + }); + + // Await the detached serve task. Dropping THIS future (client disconnect) drops the + // JoinHandle and detaches the task, which keeps running and releases admission only + // when it completes (KTD1). A JoinError here is a panic in the pipeline (the task is + // never aborted), surfaced as a 500 rather than a silent hang. + match serve.await { + Ok(result) => result, + Err(join_err) => Err(AppError::Internal(anyhow::anyhow!( + "ipfs serve task failed: {join_err}" + ))), + } +} + +/// Outcome of gating one repo for one candidate oid. +enum GateOutcome { + /// The object passed the gate; serve this response. + Served(Response), + /// This repo does not serve the object (absent, denied, quarantined, walk-capped, + /// or a walk error) — try the next candidate. + Skip, + /// A walk-requiring candidate hit the per-IP walk quota; skip it but let the caller + /// record the throttle so a later walk-free copy can still serve. + Throttled, +} + +/// Outcome of the bounded, off-worker object read for one gated candidate (F6, #173). +enum ServedRead { + /// Verified: the object's bytes hash to the requested CID; serve them. + Ok(Vec), + /// The bytes do not hash to the requested CID (a legacy provider-CID row); withhold. + Mismatch(String), + /// The object exceeds the served-object size cap; withhold rather than buffer it. + TooLarge(u64), + /// The object is genuinely absent (git reported it does not exist); try the next + /// candidate. Distinct from `ReadErr` so an infra failure is never silently rendered + /// as a clean not-found. + Gone, + /// A git subprocess failed to run (spawn/IO error, not a "no such object"). Logged at + /// the handler layer and skipped — an infra failure must surface as an error, not a + /// silent 404 for an authorized caller (INV-25 spirit, #173). + ReadErr(String), +} + +/// Immutable per-request context threaded into the gate. +struct ResolveCtx<'a> { + caller: Option<&'a str>, + caller_owned: &'a Option, + headers: &'a HeaderMap, + peer: Option, + cid_str: &'a str, + /// Canonical base32 form of the requested CID (`cid.to_string()`), used by the + /// serve-side integrity check to confirm the served bytes actually hash to the + /// requested content address (F2, #173). Compared against the recomputed CID, NOT + /// `cid_str` — a client may send an equivalent non-canonical multibase spelling. + canonical_cid: &'a str, +} + +/// Per-request walk budget + memos, shared across the provenance path and the legacy +/// scan so the fan-out ceiling and per-repo memoization span the whole request. +struct WalkState { + walks: u32, + /// Count of legacy (NULL-provenance) repos actually probed this request, so the + /// scan can stop at `ipfs_max_legacy_probes` instead of fanning out to O(repos) + /// `acquire` + `cat-file` (#173, F1, INV-10). Only the legacy path bumps it. + probes: u32, + /// Set when any cap (the legacy probe ceiling or the walk ceiling) cut the scan + /// short. A truncated scan did NOT prove the object absent/unreadable everywhere, + /// so the tail returns a retryable 503 rather than a definitive 404 (#173, F2). + truncated: bool, + allowed_blob_memo: HashMap>, + allowed_tree_memo: HashMap>, + reachable_ct_memo: HashMap>, +} + +/// Gate ONE repo for ONE candidate oid and, if the caller may read it, serve it. The +/// SINGLE gate both the provenance path and the legacy scan call, so INV-11 (quarantine +/// hard-drops before visibility), INV-2 (the repo's own "/" gate), and the per-object +/// reachability walk hold identically on both paths (KTD5). Never re-resolves via +/// `authorize_repo_read`, whose fuzzy match could authorize a different physical row +/// than the one read (KTD2a). +// The per-repo gate genuinely needs the row, its rules, its quarantine bit, the oid, +// the request context, the shared walk budget, and whether this is the fan-out-bounded +// legacy scan; bundling them buys nothing over the existing threshold. +#[allow(clippy::too_many_arguments)] +async fn gate_and_serve( + state: &AppState, + repo: &crate::db::RepoRecord, + rules: &[crate::db::VisibilityRule], + quarantined: bool, + sha256_hex: &str, + ctx: &ResolveCtx<'_>, + walk: &mut WalkState, + // True only for the legacy NULL-provenance scan, which iterates every repo. The + // provenance path targets one repo (no fan-out) and passes false, so it does not + // consume the per-request probe budget below. + legacy_scan: bool, +) -> GateOutcome { + // Quarantine gate (INV-11): a quarantined mirror is hidden from every reader, owner + // included, BEFORE any visibility check — so an owner whom visibility would Allow + // still 404s. + if quarantined { + return GateOutcome::Skip; + } + // Repo-level "/" read gate against THIS row's own rules (INV-2, KTD2a). + if visibility_check(rules, repo.is_public, &repo.owner_did, ctx.caller, "/") == Decision::Deny { + return GateOutcome::Skip; + } + // Legacy-scan fan-out control (#173, F1/F3, INV-10). The legacy path probes every + // root-visible repo, and the probe below (`acquire` — a possible cold-cache + // Tigris fetch — plus a `git cat-file -t` subprocess) is the expensive part. + // Cap it per request BEFORE that work runs, so an anonymous caller wielding a + // CID from the public pins index cannot amplify one request into O(repos) + // subprocesses. A legacy scan is inherently fan-out (unlike a targeted + // provenance fetch), so EVERY legacy probe is charged to the source IP from the + // first one, not just the ones past a free budget. A per-request-only budget + // reset each request, leaving a NULL-provenance CID open to unbounded ACROSS- + // request amplification: N requests spending N x budget cold `acquire` calls + // against Tigris with zero limiter contact (#173, F3, jatmn). Charging the first + // probe makes those requests accumulate against the per-IP `ipfs_work_rate_limiter` + // (the resolver's WORK bucket, separate from the once-per-request route brake + // `ipfs_rate_limiter` — R6, U5), closing that path. The per-request cap below stays + // as the second bound (a single request's ceiling). A spent quota is the same non-fatal Throttled as the + // walk brake: keep scanning for a walk-free copy, and only a wholly-unservable + // request becomes the 429. No resolvable key (a test oneshot with no peer/header) + // skips the brake, as the walk brake does. The provenance path targets one repo + // (no fan-out) and is exempt (`legacy_scan == false`). + if legacy_scan { + if walk.probes >= state.ipfs_max_legacy_probes { + // Budget spent: stop probing and mark the scan truncated so the tail + // reports an incomplete search (503), not a false 404 (#173, F2). + walk.truncated = true; + return GateOutcome::Skip; + } + if let Some(key) = + crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) + { + if !state.ipfs_work_rate_limiter.check(&key).await { + return GateOutcome::Throttled; } } + walk.probes += 1; + } + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this gate runs while + // the /ipfs walk permit is held (F5), so a hung or cold-Tigris acquire would otherwise + // pin the global walk slot for the whole request. On expiry skip the repo (a public + // copy may still serve) and mark the search truncated so a wholly-unserved request + // tails to a retryable 503, never a false 404 (reopened the #174 P1-2 stall vector on + // this path otherwise). + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let repo_path = match tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&repo.owner_did, &repo.name), + ) + .await + { + Ok(Ok(p)) => p, + Ok(Err(_)) => return GateOutcome::Skip, + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs gate; skipping repo"); + walk.truncated = true; + return GateOutcome::Skip; + } + }; - // Now that we've passed the gate, read the content. - let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { - Ok(c) => c, + // Existence probe before any walk (random-CID spray must not trigger a walk on a + // repo that lacks the object). Off the async runtime — it shells out to + // `git cat-file -t`. Fail closed (skip) on a task panic. + let obj_type = { + let rp = repo_path.clone(); + let sha = sha256_hex.to_string(); + let git_bin = state.git_bin.clone(); + // Bound the probe CHILD itself (process-group teardown at `git_service_timeout_secs` + // via `object_type_bounded` -> `run_bounded_git`), not just an outer tokio timeout + // racing an uncancellable `spawn_blocking`: this probe runs while the /ipfs walk + // permit is held by the owning task, so a wedged cat-file (corrupt pack, NFS stall) + // must be REAPED at the deadline rather than left to linger and delay the task's + // completion — and thus admission release (#173 round-10, KTD2). No outer timeout, + // mirroring the bounded walk below; a `GitServiceTimeout` marks the search truncated + // (retryable 503). + let probe_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + match tokio::task::spawn_blocking(move || { + store::object_type_bounded(&git_bin, &rp, &sha, probe_timeout) + }) + .await + { + Ok(Ok(Some(t))) => t, + Ok(Ok(None)) => return GateOutcome::Skip, + Ok(Err(e)) => { + if e.is::() { + tracing::warn!(repo = %repo.name, "object-type probe timed out under the /ipfs walk permit; skipping repo"); + walk.truncated = true; + } else { + tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + } + return GateOutcome::Skip; + } Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); - continue; + tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); + return GateOutcome::Skip; } - }; + } + }; - // 3. Return the content with IPFS-style headers - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_static("content-type"), - HeaderValue::from_static("application/octet-stream"), - ); - headers.insert( - HeaderName::from_static("x-content-cid"), - HeaderValue::from_str(&cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), - ); - headers.insert( - HeaderName::from_static("x-git-hash"), - HeaderValue::from_str(&sha256_hex) - .unwrap_or_else(|_| HeaderValue::from_static("invalid")), - ); + // Per-object gating applies only under a path-scoped rule (KTD4); otherwise the "/" + // gate above is the whole story. A blob is gated on the caller's allowed-blob set, a + // tree on the allowed-tree set (#135), a commit/tag on the repo's reachable + // commit/tag set (#173) — each a full-history walk sharing the per-request cap and + // per-walk IP quota. + let path_scoped = has_path_scoped_rule(rules); + let gated = path_scoped && matches!(obj_type.as_str(), "blob" | "tree" | "commit" | "tag"); + if gated { + let already = match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.contains_key(&repo.id), + "tree" => walk.allowed_tree_memo.contains_key(&repo.id), + "commit" | "tag" => walk.reachable_ct_memo.contains_key(&repo.id), + other => unreachable!("gated admits only blob/tree/commit/tag, got {other}"), + }; + if !already { + // Per-request fan-out ceiling (INV-10): once this many walks have run, skip + // THIS walk-requiring candidate and keep scanning (a later walk-free copy + // must still serve). `walks` is bumped only inside this block, so walk-free + // candidates never consume budget. + if walk.walks >= state.ipfs_max_history_walks { + // The walk ceiling truncated the search: a later repo (possibly one that + // authorizes this caller) is left unwalked, so absence is unproven — + // record it so the tail returns 503, not a false 404 (#173, F2). + walk.truncated = true; + return GateOutcome::Skip; + } + // Brake each spawned walk on the source IP (#173, F3, INV-15), BEFORE + // spending walk budget: a throttled candidate neither walks nor consumes + // budget and must not end the request — skip it and keep scanning + // (#173 review, F-C). No key (a test oneshot with no peer/header) skips the + // brake, as the other IP brakes do. On the LEGACY path the probe brake + // above already charged THIS candidate to the source (#173, F3, jatmn), so + // the walk brake must not double-charge it: only the provenance path + // (`legacy_scan == false`, no probe toll) charges here. + if !legacy_scan { + if let Some(key) = + crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) + { + if !state.ipfs_work_rate_limiter.check(&key).await { + return GateOutcome::Throttled; + } + } + } + walk.walks += 1; - return Ok((StatusCode::OK, headers, content).into_response()); + let rp = repo_path.clone(); + let r = rules.to_vec(); + let is_public = repo.is_public; + let owner = repo.owner_did.clone(); + let caller_for_walk = ctx.caller_owned.clone(); + let kind = obj_type.clone(); + // Every walk is the DURATION-BOUNDED twin (`run_bounded_git` teardown under + // `git_service_timeout_secs`): the handler holds its /ipfs walk permit + // across this spawn_blocking, and a held permit is only safe if no walk + // child can outlive the deadline (#174 F5). + let git_bin = state.git_bin.clone(); + let walk_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let result = tokio::task::spawn_blocking(move || match kind.as_str() { + "blob" => allowed_blob_set_for_caller_bounded( + &rp, + &git_bin, + walk_timeout, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ), + "tree" => allowed_tree_set_for_caller_bounded( + &rp, + &git_bin, + walk_timeout, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ), + "commit" | "tag" => reachable_commit_tag_oids_bounded(&rp, &git_bin, walk_timeout), + other => unreachable!("gated admits only blob/tree/commit/tag, got {other}"), + }) + .await; + // Fail closed on a walk error or task panic: we cannot prove readability, so + // skip rather than serve on an unproven gate. + let set = match result { + Ok(Ok(set)) => set, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk failed; skipping repo"); + return GateOutcome::Skip; + } + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk task panicked; skipping repo"); + return GateOutcome::Skip; + } + }; + match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.insert(repo.id.clone(), set), + "tree" => walk.allowed_tree_memo.insert(repo.id.clone(), set), + _ => walk.reachable_ct_memo.insert(repo.id.clone(), set), + }; + } + let in_set = match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.get(&repo.id), + "tree" => walk.allowed_tree_memo.get(&repo.id), + _ => walk.reachable_ct_memo.get(&repo.id), + } + .is_some_and(|set| set.contains(sha256_hex)); + if !in_set { + return GateOutcome::Skip; + } } - // Not found in any repo - Err(AppError::RepoNotFound(format!( - "no git object found for CID {cid_str}" - ))) + // Passed the gate — bound the object, read it OFF the async worker, and verify the + // content address, all before any byte egresses. F6 (#173): read_object_content runs a + // blocking `git cat-file` and buffers the whole object; called directly on the Axum + // worker (the type-probe and walk are already off-worker) it blocks a runtime thread, + // and unbounded it can exhaust memory for a large public blob (enumerable from the pins + // index). Precheck the SIZE and run size + read + verify inside spawn_blocking. A + // content-addressed serve cannot verify a STREAMED body (the digest is known only after + // the last byte, by which point the prefix has already egressed), so we never stream: + // buffer-verify-then-serve up to the cap and withhold anything larger. F2's integrity + // check moves in here too, so no unverified bytes are ever assembled into a response. + let max_bytes = state.ipfs_max_served_object_bytes; + let read_repo = repo_path.clone(); + let read_sha = sha256_hex.to_string(); + let read_type = obj_type.clone(); + let want_cid = ctx.canonical_cid.to_string(); + let git_bin = state.git_bin.clone(); + // Bound the size+read CHILDREN themselves (process-group teardown at + // `git_service_timeout_secs` via the `*_bounded` twins), not an outer tokio timeout + // over an uncancellable `spawn_blocking`: a hung cat-file must be REAPED at the + // deadline rather than left to pin the held /ipfs walk permit (#173 round-10, KTD2). + // No outer timeout, mirroring the bounded walk; a `GitServiceTimeout` from either + // twin surfaces as `ServedRead::ReadErr` -> truncated (retryable 503). + // ONE deadline spans the size and content reads, so a single served candidate + // holds the /ipfs walk permit for at most `git_service_timeout_secs` total, not + // one full timeout per stage (mirrors `build_filtered_pack`'s shared deadline). + let read_deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let read = tokio::task::spawn_blocking(move || -> ServedRead { + let size_budget = read_deadline.saturating_duration_since(std::time::Instant::now()); + match store::object_size_bounded(&git_bin, &read_repo, &read_sha, size_budget) { + Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(Some(_)) => {} + // git ran and reported no such object (or an unparseable size): genuine + // not-found for this candidate. + Ok(None) => return ServedRead::Gone, + // git failed to run OR the bounded read timed out (GitServiceTimeout): an + // infra/timeout failure, not a not-found. + Err(e) => return ServedRead::ReadErr(e.to_string()), + } + let content_budget = read_deadline.saturating_duration_since(std::time::Instant::now()); + let content = match store::read_object_content_bounded( + &git_bin, + &read_repo, + &read_sha, + &read_type, + content_budget, + ) { + Ok(c) => c, + Err(e) => return ServedRead::ReadErr(e.to_string()), + }; + let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); + if served != want_cid { + return ServedRead::Mismatch(served); + } + ServedRead::Ok(content) + }) + .await; + let served_read = match read { + Ok(sr) => sr, + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); + walk.truncated = true; + return GateOutcome::Skip; + } + }; + let content = match served_read { + ServedRead::Ok(c) => c, + ServedRead::TooLarge(size) => { + tracing::warn!( + repo = %repo.name, size, max = max_bytes, + "withholding object: exceeds the served-object size cap (F6)" + ); + #[cfg(test)] + note_oversize_reject(); + return GateOutcome::Skip; + } + ServedRead::Mismatch(served) => { + tracing::warn!( + repo = %repo.name, requested = %ctx.canonical_cid, served = %served, + "withholding object: served bytes do not hash to the requested CID (legacy provider-CID row?)" + ); + return GateOutcome::Skip; + } + ServedRead::Gone => return GateOutcome::Skip, + ServedRead::ReadErr(e) => { + // Infra failure (git spawn/IO), NOT a not-found: mark the search truncated so + // a wholly-unserved request tails to a retryable 503, never a definitive 404 + // for an authorized caller (INV-25 spirit — logging alone is not surfacing). + tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); + walk.truncated = true; + return GateOutcome::Skip; + } + }; + let mut resp_headers = HeaderMap::new(); + resp_headers.insert( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/octet-stream"), + ); + resp_headers.insert( + HeaderName::from_static("x-content-cid"), + HeaderValue::from_str(ctx.cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + resp_headers.insert( + HeaderName::from_static("x-git-hash"), + HeaderValue::from_str(sha256_hex).unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + GateOutcome::Served((StatusCode::OK, resp_headers, content).into_response()) } /// GET /api/v1/ipfs/pins @@ -228,3 +867,695 @@ pub async fn list_pins(State(state): State) -> Result = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_preload_queries() { + PRELOAD_QUERIES.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn preload_queries() -> usize { + PRELOAD_QUERIES.with(|c| c.get()) +} + +#[cfg(test)] +fn bump_preload_queries() { + PRELOAD_QUERIES.with(|c| c.set(c.get() + 1)); +} + +// Test-only INV-10 cost counter (F6, U6/U7): how many times the serve path withheld an +// object because it exceeded `ipfs_max_served_object_bytes`. The bounded read must reject +// an oversized object rather than buffer it on the worker; the counter is the both-ways +// guard (a removed size precheck stops incrementing it and serves the oversized object). +// Set from the match arm after `spawn_blocking` resolves, i.e. on the test's runtime +// thread, so the thread-local is read on the same thread it is written. +#[cfg(test)] +thread_local! { + static OVERSIZE_REJECTS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_oversize_rejects() { + OVERSIZE_REJECTS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn oversize_rejects() -> usize { + OVERSIZE_REJECTS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_oversize_reject() { + OVERSIZE_REJECTS.with(|c| c.set(c.get() + 1)); +} + +#[cfg(test)] +mod tests { + //! #174 P1-3 (U3): the public `GET /ipfs/{cid}` walk carries bounded CONCURRENCY + //! admission (a global pool + per-source sub-cap) held through the `spawn_blocking` + //! walk, plus a per-IP route rate limit. These are handler-layer proofs: mount the + //! real handler/router, drive one request, assert the exact 503 shed, then name the + //! mutation that turns each RED. The per-source key resolves an IP only (`Some(ip)` + //! vs `None`), never a DID — both arms are driven so neither is vacuous. The + //! CID-resolution / visibility-gate behavior of the handler itself is covered by the + //! `#[sqlx::test]` suite in `test_support.rs`. + + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use axum::Router; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + /// A router mounting the real `get_by_cid` on `/ipfs/{cid}` with `optional_signature`, + /// matching production wiring for the extractors (`PeerAddr` reads `ConnectInfo`). + fn ipfs_router(state: crate::state::AppState) -> Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state) + } + + /// A syntactically valid CIDv1(raw, sha2-256) string the handler decodes past its + /// CID/hash-code validation, so the request reaches the walk admission (not a 400). + fn valid_cid() -> String { + gitlawb_core::cid::Cid::from_git_object_bytes(b"blob 5\0hello") + .as_str() + .to_string() + } + + fn get_cid(cid: &str, peer: Option) -> Request { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + if let Some(p) = peer { + req.extensions_mut().insert(ConnectInfo(p)); + } + req + } + + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` + /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID + /// validation), so a lazy DB-free state suffices — exactly like the served-git shed + /// tests. MUTATION (RED): delete the `git_ipfs_walk_semaphore` acquire in + /// `get_by_cid` and the request no longer sheds here (it falls through to the DB / + /// walk and returns something other than 503). + #[tokio::test] + async fn get_by_cid_sheds_with_503_when_walk_pool_exhausted() { + let mut state = crate::test_support::test_state_lazy(); + // Global /ipfs walk pool exhausted; per-source cap permissive so only the global + // pool can shed. Route rate limit is applied as a layer in production, not here. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.9:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted /ipfs walk pool must shed the request with 503" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// Per-source sub-cap, the `Some(ip)` arm: with per-source = 1 and the source pinned + /// at its single slot, a request from THAT source sheds 503 (global pool has room), + /// while a request from a DIFFERENT source is NOT shed by the cap (it proceeds past + /// admission). Pinning proves the `PeerAddr`/`HeaderMap` extractors resolved the key + /// — an inert `None` key would never shed on the per-source cap. MUTATION (RED): + /// delete the `git_ipfs_walk_per_caller` acquire and the capped source no longer + /// sheds. + #[tokio::test] + async fn get_by_cid_per_source_cap_sheds_same_source_admits_other() { + let mut state = crate::test_support::test_state_lazy(); + // Global pool has room; the per-source cap is 1. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(8)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let capped: SocketAddr = "203.0.113.20:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.21:5000".parse().unwrap(); + + // Pin the capped source at its single walk slot. + let _slot = state + .git_ipfs_walk_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first walk slot for the capped source IP"); + + let cid = valid_cid(); + // The capped source sheds on the per-source cap even with global capacity free. + let resp = ipfs_router(state.clone()) + .oneshot(get_cid(&cid, Some(capped))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its per-source /ipfs walk cap must shed 503 with global capacity free" + ); + + // A DIFFERENT source is NOT shed by the per-source cap: it clears admission and + // proceeds (then errors on the lazy DB, which is not a 503). + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(other))) + .await + .unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must not be shed by the per-source cap" + ); + } + + /// The `None`-key arm: a request with no resolvable source key (no trusted-proxy + /// header, no `ConnectInfo`) is bounded by the GLOBAL pool only, never the per-source + /// sub-cap. With the global pool exhausted it still sheds 503 (the counterpart to the + /// `Some(ip)` arm above, so neither arm is vacuous). + #[tokio::test] + async fn get_by_cid_none_key_arm_sheds_on_global_pool() { + let mut state = crate::test_support::test_state_lazy(); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + // Per-source cap permissive so only the global pool can shed. + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // No ConnectInfo + no trusted header -> client_key resolves None. + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), None)) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL /ipfs walk pool" + ); + } + + /// Map self-bound (INV-15): the `/ipfs` per-source map is a `PerCallerConcurrency` + /// built via `with_default_max_keys`, so a distinct-source-key flood cannot grow it + /// past the cap and a rejected key never allocates (reject-before-insert). Mirrors + /// `per_caller_concurrency_map_is_self_bounding_and_reject_before_insert` for the + /// pool U3 adds. + #[tokio::test] + async fn ipfs_walk_per_caller_map_is_self_bounding_and_reject_before_insert() { + let lim = crate::rate_limit::PerCallerConcurrency::new(4, 3); + // Acquire+drop a flood of distinct keys — the map self-empties (a key is removed + // the instant its in-flight count hits zero). + for i in 0..50 { + let _p = lim.try_acquire(&format!("src{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "an acquire+drop flood of distinct sources leaves the /ipfs map empty" + ); + // Reject-before-insert: hold max_keys distinct sources, then a new one sheds + // without growing the map. + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct sources held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new source key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + + /// Build the shared `/ipfs` TREE-walk fixture. A fake `git` whose `rev-list` records + /// its pid then sleeps ~6s (so the tree walk blocks deterministically inside + /// `run_bounded_git`) and whose `cat-file -t` answers "tree" (so the bounded + /// object-type probe, `object_type_bounded` on `state.git_bin`, routes into the + /// tree-gate arm); a real SHA-256 bare repo with a committed `src/` tree pinned WITH + /// provenance; and a path-scoped rule so the gate takes the tree-walk branch. Returns + /// the tempdir (keep it alive for the whole test), the state (the caller sets the walk + /// semaphores), the requested CID, and the rev-list pidfile path. + #[cfg(unix)] + async fn seed_tree_walk_fixture( + pool: sqlx::PgPool, + ) -> ( + tempfile::TempDir, + crate::state::AppState, + String, + std::path::PathBuf, + ) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + cat-file) if [ \"$2\" = \"-t\" ]; then echo tree; fi ;;\n\ + rev-list) echo $$ > \"{}\"; sleep 6 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6ipfstree"; + let name = "iptree"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let work = tmp.path().join("work"); + std::fs::create_dir_all(work.join("src")).unwrap(); + std::fs::write( + work.join("src/secret.txt"), + b"ipfs tree walk retain proof\n", + ) + .unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + let tree_oid = { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + assert!(out.status.success(), "rev-parse failed"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + assert_eq!( + crate::git::store::object_type(&bare, &tree_oid) + .unwrap() + .as_deref(), + Some("tree"), + "the seeded sha256 tree must exist so the handler reaches the tree walk" + ); + let (_ty, raw) = crate::git::store::read_object(&bare, &tree_oid) + .unwrap() + .expect("tree object readable"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + state + .db + .record_pinned_cid(&tree_oid, &cid, Some(&rec.id)) + .await + .unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF5IpfsTreeReaderAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + (tmp, state, cid, revlist_pid) + } + + /// Retain-through-blocking (#174 F5, the load-bearing async property, on the + /// NEWLY-BOUNDED TREE path): the walk admission is held until the `spawn_blocking` + /// walk actually RETURNS, not when a tokio timeout fires. The requested CID + /// resolves to a TREE object under a path-scoped rule, so the gate runs + /// `allowed_tree_set_for_caller_bounded` — the walk this integration converts to + /// `run_bounded_git` — rather than the blob walk #174 already proved. With the + /// global pool at size 1, drive a request until its walk (a fake git that hangs on + /// `rev-list`) is in flight; the slot must stay held (`available_permits() == 0`) + /// and a replacement from a DIFFERENT source must shed 503 for as long as the + /// blocking walk runs — even though the request future is only `.await`ing the + /// blocking join. When the blocking walk ends the permit frees and a replacement + /// is admitted. The permit lives INSIDE the handler across the blocking `.await`; + /// move it out (drop before the walk) and the replacement would be admitted while + /// the walk still burns a blocking thread (the bug this guards). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_permit_held_through_bounded_tree_walk(pool: sqlx::PgPool) { + let (tmp, mut state, cid, revlist_pid) = seed_tree_walk_fixture(pool).await; + // Isolate the global walk pool at size 1; per-source cap permissive so only the + // held global permit can shed the replacement. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + // Keep the fixture tempdir alive for the whole test (its Drop removes the repos). + let _tmp = tmp; + + let sem = state.git_ipfs_walk_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one walk slot before the request" + ); + + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until the fake git's rev-list records its pid — the TREE walk is now in + // the blocking pool and the request future is `.await`ing its join, holding the + // walk permit. Stop polling the instant the future completes (re-polling a + // completed oneshot panics). + let mut walk_pid: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let pid = walk_pid + .unwrap_or_else(|| panic!("the fake git rev-list must have spawned; early: {early:?}")); + // Reap the sleeping child on drop so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Load-bearing: while the blocking TREE walk runs, the slot is HELD and a + // replacement from a DIFFERENT source sheds 503 — proving the permit is + // retained across the spawn_blocking join, not freed by a tokio timeout. + assert_eq!( + sem.available_permits(), + 0, + "the walk slot must be held while the spawn_blocking tree walk runs" + ); + let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a replacement must shed 503 while the prior request's blocking tree walk still runs" + ); + + // #173 round-10 (R1, KTD1): the NEW invariant. Drop the in-flight request. The + // gated serve pipeline runs in a DETACHED tokio task that OWNS the admission + // guard, so dropping the request future must NOT release admission — the task + // runs to its bounded completion first. Under the OLD handler-local permit the + // slot would free the instant the future dropped even though the blocking walk + // is still burning a thread (the cap bypass this closes). + drop(fut); + // While the detached task is still inside the ~6s blocking tree walk, admission + // stays held. Poll a short window: a handler-local permit would already read 1. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + assert_eq!( + sem.available_permits(), + 0, + "dropping the request must NOT release admission while the detached walk \ + still runs (the spawned task owns the guard until its bounded work ends)" + ); + // A replacement from a different source STILL sheds 503 after the drop — the + // detached task holds the global slot, exactly as it did while the future lived. + let peer3: SocketAddr = "203.0.113.83:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer3)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a replacement must still shed 503 after the original request was dropped, \ + while its detached walk holds admission" + ); + // Tear the walk down; the detached task finishes and drops the guard exactly + // once, freeing the single slot back to 1 (never double-freed to 2). + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the detached walk tears down, the task drops the guard and frees the slot" + ); + assert_eq!( + sem.available_permits(), + 1, + "admission released exactly once — the single slot is back, not double-freed" + ); + } + + /// Amplification negative (#173 round-10, R1): sequential cancel-spam from ONE source + /// cannot hold more than the per-source cap of concurrent detached walk tasks. A + /// detached serve task holds its per-source permit until its bounded work finishes (up + /// to `git_service_timeout_secs`), so with a per-source cap of 1 a second request from + /// the SAME source sheds 503 even though the GLOBAL pool has room — the source cannot + /// amplify its concurrent walk children past the cap by dropping-and-retrying. (The + /// worst case: a detached task can occupy its global/per-source permit for one + /// bound-interval, so distributed cancel-spam can hold the global pool that long — the + /// accepted bounded-admission tradeoff, not a leak.) + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_cancel_spam_bounded_by_per_source_cap(pool: sqlx::PgPool) { + let (tmp, mut state, cid, revlist_pid) = seed_tree_walk_fixture(pool).await; + // Global pool has ample room (4); the per-source cap is 1. So any shed of a + // same-source replacement is the PER-SOURCE cap, never global exhaustion. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(4)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let _tmp = tmp; + + let sem = state.git_ipfs_walk_semaphore.clone(); + let per_caller = state.git_ipfs_walk_per_caller.clone(); + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + // Source S fires request 1; drive until its tree walk is in flight (the task now + // holds source S's single per-source permit). + let source_s: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(source_s))); + let mut walk_pid: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + } + let pid = walk_pid.expect("the fake git rev-list must have spawned"); + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Cancel-spam: drop request 1's future. Its detached task keeps running the walk + // and KEEPS holding source S's single per-source permit. + drop(fut); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + // A SECOND request from source S sheds 503. The global pool still has room (only 1 + // of 4 taken), so this is the per-source cap, not global exhaustion. + let resp = router.clone().oneshot(make_req(source_s)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a same-source cancel-spam replacement must shed 503 on the per-source cap \ + while the detached task still holds the source's permit" + ); + assert!( + sem.available_permits() >= 3, + "the shed was the per-source cap, not global exhaustion (global pool still has room)" + ); + assert_eq!( + per_caller.tracked_keys(), + 1, + "exactly one per-source permit is outstanding for the one source — no amplification" + ); + + // Tear the detached walk down; the task completes and releases source S's permit + // (tracked_keys returns to 0), so the source is no longer over the cap. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut released = false; + for _ in 0..400 { + if per_caller.tracked_keys() == 0 { + released = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + released, + "once the detached task tears down it releases source S's per-source permit" + ); + } + + /// Route rate limit is WIRED (not a silent no-op): the production `build_router` + /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP + /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does + /// nothing, so this proves the extension is attached. Drive it through the real + /// router with a tight limiter (1/hr): the second request from the same IP is 429. + /// MUTATION (RED): drop the `axum::Extension(ipfs_limiter)` layer in `server.rs` and + /// the second request is no longer braked (it reaches the handler, 404, not 429). + #[sqlx::test] + async fn ipfs_route_ip_rate_limit_is_attached(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + // Tight per-IP /ipfs bucket so the second request from one IP trips 429. + state.ipfs_rate_limiter = + crate::rate_limit::RateLimiter::new(1, std::time::Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = crate::server::build_router(state); + let cid = valid_cid(); + let make = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + let peer: SocketAddr = "203.0.113.99:5000".parse().unwrap(); + + // First request from this IP passes the brake and reaches the handler (404 — no + // such object anywhere), debiting the single-slot bucket. + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "the first /ipfs request from an IP must pass the rate brake" + ); + // Second request from the SAME IP is braked with 429 — proving the limiter + // extension is attached (a bare no-op layer would let it through to 404). + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "an exhausted per-IP /ipfs bucket must brake with 429 — the IpRateLimiter \ + extension must be attached to the route" + ); + // A DIFFERENT IP still has its own budget (independent bucket). + let other: SocketAddr = "203.0.113.100:5000".parse().unwrap(); + let resp = router.oneshot(make(other)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a different IP must not be braked by another IP's exhausted bucket" + ); + } +} diff --git a/crates/gitlawb-node/src/api/mod.rs b/crates/gitlawb-node/src/api/mod.rs index 10d76daf..86594c03 100644 --- a/crates/gitlawb-node/src/api/mod.rs +++ b/crates/gitlawb-node/src/api/mod.rs @@ -200,9 +200,11 @@ mod authz_guard { (issues, "create_issue", "authorize_repo_read("), (bounties, "create_bounty", "authorize_repo_read("), (repos, "fork_repo", "authorize_repo_read("), - // get_by_cid gates each iterated repo row directly via visibility_check - // (KTD2a: it must NOT route through authorize_repo_read's fuzzy re-resolve). - (ipfs, "get_by_cid", "visibility_check("), + // get_by_cid resolves each candidate (provenance path + legacy scan) through + // the shared `gate_and_serve` (#173 round 2); the gate markers themselves are + // asserted below. This row proves the delegation is real — the gate is + // actually reached, not dead code. + (ipfs, "get_by_cid", "gate_and_serve("), // Bucket C — signer-self: the acting DID is matched/bound to auth.0 (tasks, "create_task", "did_matches("), (tasks, "claim_task", "did_matches("), @@ -236,6 +238,22 @@ mod authz_guard { "visibility::require_owner must use did_matches for DID-safe owner matching" ); + // The CID read surface (#173) enforces its gate inside the shared + // `gate_and_serve`, which BOTH the provenance path and the legacy scan call, so + // the markers must live there (the get_by_cid row above only proves delegation). + // The repo's own "/" visibility check (KTD2a — never authorize_repo_read's fuzzy + // re-resolve) and the quarantine hard-drop BEFORE visibility (INV-11) are both + // load-bearing: removing either re-opens a leak on the provenance path. + let gate_body = fn_body(ipfs, "gate_and_serve"); + assert!( + gate_body.contains("visibility_check("), + "gate_and_serve must gate the CID read surface via visibility_check (KTD2a)" + ); + assert!( + gate_body.contains("if quarantined"), + "gate_and_serve must hard-drop a quarantined repo before the visibility gate (INV-11)" + ); + for (src, func, marker) in rows { let body = fn_body(src, func); assert!( diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b9cbc352..08075e4e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -44,6 +44,8 @@ async fn replication_withheld_set( owner_did: &str, is_public: bool, disk_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, ) -> (bool, Option>) { let announce = match &rules { Some(rules) => crate::visibility::listable_at_root(rules, is_public, owner_did, None), @@ -65,8 +67,8 @@ async fn replication_withheld_set( Some(rules) => { let owner_did = owner_did.to_string(); tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_oids( - &disk_path, &rules, is_public, &owner_did, None, + crate::git::visibility_pack::withheld_blob_oids_bounded( + &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, ) }) .await @@ -106,12 +108,14 @@ async fn fail_closed_full_scan_objects( is_public: bool, owner_did: String, candidates: Vec, + git_bin: String, + timeout: std::time::Duration, ) -> Vec { tokio::task::spawn_blocking(move || -> anyhow::Result> { - let allowed = crate::git::visibility_pack::replicable_blob_set( - &disk_path, &rules, is_public, &owner_did, + let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( + &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path)?; + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, std::time::Instant::now() + timeout)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( candidates, &allowed, &all_blobs, )) @@ -508,6 +512,43 @@ pub async fn git_info_refs( auth: Option>, ) -> Result { let name = smart_http_repo_name(&repo)?; + let service = query + .service + .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; + // Reject an unsupported service BEFORE taking a read slot or doing any DB/Tigris + // work (#174 P2-1). git_info_refs otherwise treats everything that is not + // git-receive-pack as a read op, so an unauthenticated `?service=anything` to a + // public repo would consume a read permit and the visibility/Tigris work before + // validate_service rejected it downstream in smart_http. + if service != "git-upload-pack" && service != "git-receive-pack" { + return Err(AppError::BadRequest(format!( + "unsupported git service: {service}" + ))); + } + // #62 cheap pre-DB load shed: if the pool this service draws from is already + // saturated, shed with 503 before any DB/disk work. Best-effort (holds no + // permit); the authoritative hold is `git_permit` below, after the per-source + // cap. Restores the shed-before-DB property the reordered held acquire alone + // would drop, while the reorder still prevents one source from occupying global + // slots during the DB/visibility window. + { + // The receive-pack advertisement peeks its DEDICATED advert pool, not the + // write pool the authenticated POST uses (#174) — matching the held acquire + // below, so the pre-DB shed and the authoritative hold agree on the pool. + let pool = if service == "git-receive-pack" { + &state.git_push_advert_semaphore + } else { + &state.git_read_semaphore + }; + if pool.available_permits() == 0 { + tracing::warn!( + "served-git concurrency cap reached; shedding request with 503 (pre-DB)" + ); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } + } tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state .db @@ -521,9 +562,6 @@ pub async fn git_info_refs( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } - let service = query - .service - .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; tracing::debug!(service = %service, repo = %name, "info/refs service"); // Enforce read visibility on the ref advertisement, for BOTH services. The @@ -563,30 +601,458 @@ pub async fn git_info_refs( } } - // For receive-pack (push), download the latest from Tigris so the client - // sees the same refs that acquire_write() will operate on. - let disk_path = if service == "git-receive-pack" { - state - .repo_store - .acquire_fresh(&record.owner_did, &record.name) - .await + // Per-source concurrency sub-cap (#174), keyed on the resolved source IP and + // acquired AFTER the visibility + push-rate gates (KTD7) so a denied or + // rate-limited request never consumes a slot; held for the whole op. The + // upload-pack advertisement is bounded on the read pool (git_read_per_caller). + // The receive-pack advertisement draws from the write pool, so it is bounded per + // source (git_push_advert_per_caller) instead: without this, an anonymous + // multi-source flood of push-handshake advertisements could hold the write pool's + // slots across acquire_fresh and shed authenticated pushes, since the per-IP push + // rate limiter caps rate, not concurrency (#174 review fix). + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = if service == "git-receive-pack" { + acquire_read_caller_permit( + &state.git_push_advert_per_caller, + caller_key.as_deref(), + name, + "receive-pack advert", + )? } else { - state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - } - .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); - AppError::Git(e.to_string()) - })?; + acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "info/refs", + )? + }; + + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire_fresh/git so it bounds the fresh Tigris acquire + // and git exec (INV-10). The receive-pack advertisement is phase one of a push, + // but it is ANON-reachable, so it draws from the dedicated advert pool + // (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses: + // an advert flood can at worst exhaust the advert pool, never a permit a push + // POST needs at admission (#174 U2). A clone flood on the read pool likewise + // can't touch either. The upload-pack advertisement stays on the read pool with + // its per-caller sub-cap. + let _permit = if service == "git-receive-pack" { + git_permit(&state.git_push_advert_semaphore)? + } else { + git_permit(&state.git_read_semaphore)? + }; - smart_http::info_refs(&disk_path, &service) + // For receive-pack (push), download the latest from Tigris so the client + // sees the same refs that acquire_write() will operate on. + // + // Bound the acquire under `git_acquire_timeout_secs`: the concurrency permit is + // already held above, and `git_service_timeout_secs` only starts once git spawns, + // so an un-deadlined acquire (a hung Tigris HEAD/GET here) pins the permit until + // the pool drains (#174 P1-2). On expiry the handler-local `_permit`/`_caller_permit` + // drop on the early return (the AdmissionGuard is not built until after acquire), + // so the shed frees the slot; return a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let acquire_fut = async { + if service == "git-receive-pack" { + state + .repo_store + .acquire_fresh(&record.owner_did, &record.name) + .await + } else { + state + .repo_store + .acquire(&record.owner_did, &record.name) + .await + } + }; + let disk_path = tokio::time::timeout(acquire_deadline, acquire_fut) .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, service = %service, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed"); + tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); AppError::Git(e.to_string()) + })?; + + // Move the admission permits into the guard so they release only after the spawned + // git process group is confirmed reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper is still tearing + // the group down (#174 P1-a). The handler keeps no copy: `_permit`/`_caller_permit` + // are moved in, so admission tracks the real process lifetime. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + smart_http::info_refs("git", &service, &disk_path, git_timeout, Some(admission)) + .await + .map_err(|e| { + let app = git_service_app_error(&e); + match &app { + AppError::Timeout(_) => { + tracing::warn!(repo = %name, service = %service, "info/refs advertisement timed out") + } + _ => { + tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed") + } + } + app + }) +} + +/// Acquire a permit from the served-git concurrency semaphore, or shed the +/// request with a 503 + Retry-After when every slot is in use. Bind the returned +/// permit to a named local so it is held for the whole git op (it releases on +/// drop); a bare `_` would release it immediately. +fn git_permit( + sem: &std::sync::Arc, +) -> Result { + sem.clone().try_acquire_owned().map_err(|_| { + // Surface the shed so operators can see the cap engaging, mirroring the + // receive-pack rate-limit warn above. A silent 503 makes a saturated or + // misconfigured cap look like a client problem instead of a capacity one. + tracing::warn!("served-git concurrency cap reached; shedding request with 503"); + AppError::Overloaded("git service at capacity, retry shortly".into()) + }) +} + +/// Resolve the per-caller key for the read sub-cap (#174): always the resolved +/// source IP (`client_key`), never the signed DID. Public read routes accept any +/// valid `did:key` via `optional_signature` with no admission step, so keying on +/// the DID would let one host mint disposable DIDs to multiply its per-source +/// budget; the push path already throttles on the resolved source IP for exactly +/// this DID-farm reason (`rate_limit.rs`, `IpRateLimiter`). `None` when no key +/// resolves (no trusted header and no peer): such a request is bounded by the +/// global read pool only, never a 500. The per-source-IP key is only as granular +/// as `trust`; see the `max_concurrent_reads_per_caller` config doc. +fn read_caller_key( + headers: &axum::http::HeaderMap, + peer: Option, + trust: crate::rate_limit::TrustedProxy, +) -> Option { + crate::rate_limit::client_key(headers, peer, trust) +} + +/// Acquire the per-caller read sub-cap permit (#174), or shed with a 503. `key` is +/// `None` when no caller key resolves — that request is bounded by the global read +/// pool only and is never shed here (returns `Ok(None)`). `handler` labels the shed +/// log line. Shared by both read handlers so the two acquire sites cannot drift. +fn acquire_read_caller_permit( + limiter: &crate::rate_limit::PerCallerConcurrency, + key: Option<&str>, + repo: &str, + handler: &str, +) -> Result> { + match key { + Some(k) => match limiter.try_acquire(k) { + Some(p) => Ok(Some(p)), + None => { + tracing::warn!(repo = %repo, caller = %k, handler, "per-caller cap reached; shedding with 503"); + Err(AppError::Overloaded( + "git service at capacity for this caller, retry shortly".into(), + )) + } + }, + None => Ok(None), + } +} + +/// Acquire an encryption-walk admission permit, then run the bounded withheld-blob +/// recipients walk. Blocks (defers) when `git_encrypt_semaphore` is full rather than +/// shedding — the walk is background so added latency is fine, and dropping it would +/// lose the withheld-blob recovery copy (#174 P1-e). Bounds the number of concurrent +/// post-push encryption walks so N fast completed pushes cannot spawn N concurrent +/// full-history git walks. Mirrors the original `spawn_blocking(...).await` return +/// shape so the caller's `Ok(Ok(recipients))` match is unchanged. +async fn withheld_recipients_gated( + encrypt_sem: std::sync::Arc, + repo_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, + rules: Vec, + is_public: bool, + owner_did: String, +) -> std::result::Result< + anyhow::Result>>, + tokio::task::JoinError, +> { + let _permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tokio::task::spawn_blocking(move || { + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &repo_path, &git_bin, timeout, &rules, is_public, &owner_did, + ) + }) + .await +} + +/// Everything the detached post-push replication task needs that does not change +/// between requeue passes. Cloned once from `AppState` at the spawn site so the task +/// is self-contained (the handler keeps no copy). +struct PostPushReplication { + db: std::sync::Arc, + disk_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, + ipfs_api: String, + repo_id: String, + encrypt_sem: std::sync::Arc, + node_seed: [u8; 32], + node_did: String, + repo_name: String, + irys_url: String, + http_client: std::sync::Arc, +} + +/// The requeue enumeration for the pin half: a fail-closed FULL scan of the current +/// object DB under the REFRESHED rules. The coalesced push's ref tips are gone by the +/// time we requeue, so the delta path is unavailable; the whole-repo scan is the safe +/// superset. Never pins the bare `list_all_objects` output — that includes +/// dangling/withheld blobs — it feeds it as CANDIDATES to the same fail-closed filter +/// the push path's full-scan branch uses, which drops dangling and visibility-withheld +/// blobs before anything is pinned. +async fn requeue_full_scan_object_list( + disk_path: &std::path::Path, + git_bin: &str, + timeout: std::time::Duration, + rules: Vec, + is_public: bool, + owner_did: String, +) -> Vec { + let disk = disk_path.to_path_buf(); + let gb = git_bin.to_string(); + let candidates = tokio::task::spawn_blocking(move || { + crate::git::push_delta::list_all_objects(&disk, &gb, std::time::Instant::now() + timeout) + }) + .await + .ok() + .and_then(|r| { + r.map_err(|e| { + tracing::warn!(err = %e, "requeue full-scan enumeration failed; pinning nothing this pass") }) + .ok() + }) + .unwrap_or_default(); + fail_closed_full_scan_objects( + disk_path.to_path_buf(), + rules, + is_public, + owner_did, + candidates, + git_bin.to_string(), + timeout, + ) + .await +} + +/// The detached post-push encryption + local-IPFS pin task, as a REQUEUE LOOP. +/// +/// Pass one uses the spawn-time captures (`first_*`) — the delta the push handler +/// already computed. At the TASK TAIL (unconditional on the encrypt gate below and on +/// walk success) it consults the coalescing dirty flag: if a push coalesced during the +/// window it re-reads repo state (rules, is_public, owner_did, withheld) FRESH from the +/// DB, re-enumerates the pin set fail-closed, and runs another pass — so the coalesced +/// push is covered before the task exits. Otherwise it releases the key and returns. +/// +/// The tail placement is load-bearing: the encrypt+anchor block only runs under a +/// path-scoped rule, so a check-and-clear placed inside that gate would never run for a +/// public/rules-free repo or a failed walk, dropping exactly the pin-half push this +/// requeue must cover. +#[allow(clippy::too_many_arguments)] +async fn run_post_push_replication( + mut guard: crate::state::EncryptInflightGuard, + ctx: PostPushReplication, + first_object_list: Vec, + first_rules: Option>, + first_is_public: bool, + first_owner_did: String, + first_withheld: std::collections::HashSet, +) { + let mut object_list = first_object_list; + let mut rules_opt = first_rules; + let mut is_public = first_is_public; + let mut owner_did = first_owner_did; + // The task only spawns when `withheld.is_some()`, so pass one always replicates. + let mut withheld: Option> = Some(first_withheld); + + loop { + if withheld.is_some() { + // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). + crate::ipfs_pin::pin_new_objects( + &ctx.ipfs_api, + &ctx.disk_path, + &ctx.git_bin, + ctx.timeout, + object_list.clone(), + &ctx.db, + &ctx.repo_id, + ) + .await; + + // Option B1: encrypt-then-pin the withheld blobs. No path-scoped rule can + // withhold a blob, so a rules-free repo has nothing to seal; skip. Mirrors + // the has_path_scoped_rule gate on the other two withheld-walk sites. + if let Some(rules) = rules_opt + .clone() + .filter(|r| visibility_pack::has_path_scoped_rule(r)) + { + let recip = withheld_recipients_gated( + ctx.encrypt_sem.clone(), + ctx.disk_path.clone(), + ctx.git_bin.clone(), + ctx.timeout, + rules, + is_public, + owner_did.clone(), + ) + .await; + if let Ok(Ok(recipients)) = recip { + let delta = crate::encrypted_pin::encrypt_and_pin( + &ctx.ipfs_api, + &ctx.disk_path, + &ctx.db, + &ctx.repo_id, + &ctx.node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the sealed blobs to + // Arweave. Best-effort; never fails the push. + if !delta.is_empty() && !ctx.irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&owner_did); + let repo_slug = format!("{owner_short}/{}", ctx.repo_name); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &owner_did, + node_did: &ctx.node_did, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &ctx.http_client, + &ctx.irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } + } + } + } + + // TASK TAIL — unconditional check-and-clear, atomic with the release decision. + if !guard.requeue_or_release() { + break; + } + + // A push coalesced during this pass. Re-read repo state FRESH (never the stale + // spawn-time captures) so a coalesced push that changed `.gitlawb` withholding + // is walked under the new policy, then re-enumerate the pin set fail-closed. + let (r_rules, r_is_public, r_owner) = match ctx.db.get_repo_by_id(&ctx.repo_id).await { + Ok(Some(rec)) => ( + ctx.db.list_visibility_rules(&ctx.repo_id).await.ok(), + rec.is_public, + rec.owner_did, + ), + Ok(None) => { + tracing::debug!(repo = %ctx.repo_id, "repo gone before requeue pass; releasing"); + (None, false, String::new()) + } + Err(e) => { + tracing::warn!(repo = %ctx.repo_id, err = %e, "requeue repo re-read failed; skipping this pass's work"); + (None, false, String::new()) + } + }; + let (_announce, r_withheld) = replication_withheld_set( + r_rules.clone(), + &r_owner, + r_is_public, + ctx.disk_path.clone(), + ctx.git_bin.clone(), + ctx.timeout, + ) + .await; + object_list = match &r_withheld { + Some(_) => { + requeue_full_scan_object_list( + &ctx.disk_path, + &ctx.git_bin, + ctx.timeout, + r_rules.clone().unwrap_or_default(), + r_is_public, + r_owner.clone(), + ) + .await + } + None => Vec::new(), + }; + rules_opt = r_rules; + is_public = r_is_public; + owner_did = r_owner; + withheld = r_withheld; + } +} + +/// Test-only entry point: build the `PostPushReplication` context from a test +/// `AppState` (with an overridable `ipfs_api` for a mock Kubo server and an explicit +/// `disk_path` for the fixture repo) and run the requeue loop. Keeps +/// `PostPushReplication` and `run_post_push_replication` private to this module. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub(crate) async fn run_post_push_replication_for_test( + state: &AppState, + guard: crate::state::EncryptInflightGuard, + disk_path: std::path::PathBuf, + repo_id: String, + ipfs_api: String, + is_public: bool, + owner_did: String, + object_list: Vec, + rules: Option>, + withheld: std::collections::HashSet, +) { + let ctx = PostPushReplication { + db: state.db.clone(), + disk_path: disk_path.clone(), + git_bin: state.git_bin.clone(), + timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + ipfs_api, + repo_id, + encrypt_sem: state.git_encrypt_semaphore.clone(), + node_seed: *state.node_keypair.to_seed(), + node_did: state.node_did.to_string(), + repo_name: String::new(), + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + }; + run_post_push_replication( + guard, + ctx, + object_list, + rules, + is_public, + owner_did, + withheld, + ) + .await; } /// Map an error from a `smart_http` git service call to the right `AppError`: @@ -614,8 +1080,19 @@ pub async fn git_upload_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, auth: Option>, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { + // #62 cheap pre-DB load shed (see git_info_refs): shed before DB when the read + // pool is saturated; the authoritative hold is `git_permit` below, after the + // per-source cap. + if state.git_read_semaphore.available_permits() == 0 { + tracing::warn!("served-git concurrency cap reached; shedding request with 503 (pre-DB)"); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } let name = smart_http_repo_name(&repo)?; let record = state .db @@ -636,11 +1113,41 @@ pub async fn git_upload_pack( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } - let disk_path = state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; + // Per-caller read sub-cap (#174): after the visibility gate (KTD7) so a + // visibility-denied caller never consumes a scarce read slot. Keyed on the + // resolved source IP (never the signed DID, #174 U1); no resolvable key -> + // global read pool only. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "upload-pack", + )?; + + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire/git so it bounds the Tigris acquire and git + // exec (INV-10). + let _permit = git_permit(&state.git_read_semaphore)?; + + // Bound the acquire under `git_acquire_timeout_secs` so a hung Tigris HEAD/GET + // cannot pin the read permit indefinitely (#174 P1-2). The permit is a handler + // local here (moved into the AdmissionGuard only below, once git is spawned), so + // the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let disk_path = tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| AppError::Git(e.to_string()))?; let body_len = body.len(); // No path-scoped rule can withhold an individual blob, and the whole-repo @@ -648,35 +1155,65 @@ pub async fn git_upload_pack( // withheld walk and serve the pack directly. let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); let resp = if !visibility_pack::has_path_scoped_rule(&rules) { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // Plain (non-path-scoped) serve: move both admission permits into the guard so + // they release only after the spawned git group is reaped, on + // complete/timeout/disconnect — not the instant a disconnect drops this future + // (#174 P1-a). The handler keeps no copy. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { - // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep - // that off the async worker thread. - let withheld = { + // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep that + // off the async worker thread. Move BOTH admission permits INTO the blocking + // task so they are held for the walk's real duration: spawn_blocking cannot be + // cancelled, so on a client disconnect the handler future drops but the walk + // keeps running — and now so do its permits, released only when the walk + // finishes rather than the instant the future drops (#174 P1-b). On success the + // task hands the permits back so the serve phase below keeps them; on a + // dropped future the returned tuple (with the permits) is discarded only when + // the blocking task completes, so admission tracks the real git work. + let (withheld, _permit, _caller_permit) = { let path = disk_path.clone(); let rules = rules.clone(); let owner_did = record.owner_did.clone(); let caller_owned = caller.map(str::to_string); let is_public = record.is_public; + let git_bin = state.git_bin.clone(); tokio::task::spawn_blocking(move || { - visibility_pack::withheld_blob_oids( + let withheld = visibility_pack::withheld_blob_oids_bounded( &path, + &git_bin, + git_timeout, &rules, is_public, &owner_did, caller_owned.as_deref(), - ) + ); + (withheld, _permit, _caller_permit) }) .await .map_err(|e| AppError::Git(e.to_string()))? - .map_err(|e| AppError::Git(e.to_string()))? }; + // A walk that hit its deadline carries GitServiceTimeout; map it to 504 like + // the smart_http paths, not a generic 500 (#174 U3). + let withheld = withheld.map_err(|e| git_service_app_error(&e))?; if withheld.is_empty() { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // No blobs to withhold: serve the plain pack, moving the permits returned by + // the walk into the guard so admission tracks the served git group's reap + // (the walk already held them per be0cdd6; this hands them to the serve). + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - smart_http::upload_pack_excluding(&disk_path, body, &withheld).await + // Move both admission permits into the guard so they release only after the + // filtered serve's git group (rev-list then pack-objects) is reaped, on + // complete/timeout/disconnect — not the instant a disconnect drops this + // future. Without this, disconnect-spam on a path-scoped repo could hold PIDs + // past the concurrency cap while the permits were already freed (#174 P1-a, + // R2). The guard rides both stages inside upload_pack_excluding. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout, Some(admission)) + .await } } .map_err(|e| { @@ -851,9 +1388,32 @@ pub async fn git_receive_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, Extension(auth): Extension, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { let name = smart_http_repo_name(&repo)?; + // Per-source write sub-cap (#174 P1-d): before the global write permit so one + // source IP cannot occupy the whole write pool via many slow authenticated pushes + // and 503 every other source. Owner enforcement defaults off, so any valid did:key + // is accepted (auth != authz), and the 600/hour push limiter bounds arrival RATE, + // not in-flight concurrency — so without this a single host minting disposable DIDs + // saturates the pool. Keyed on the resolved source IP, NEVER the signed DID (a DID + // farm defeats a DID key); no resolvable key -> global write pool only. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_write_per_caller, + caller_key.as_deref(), + name, + "receive-pack", + )?; + // Shed with a 503 before spawning git when the concurrency cap is saturated. + // Pushes draw from the dedicated WRITE pool, separate from reads, so a flood of + // anonymous reads cannot shed an authenticated push (#174). Taken after the + // per-source cap above so one source cannot occupy global slots it would be + // sub-cap-denied for; still before the Tigris acquire_write, bounding concurrent + // fresh acquires (INV-10); held for the whole op. + let _permit = git_permit(&state.git_write_semaphore)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state .db @@ -924,19 +1484,49 @@ pub async fn git_receive_pack( } tracing::debug!(repo = %name, "acquiring write lock"); - let guard = state - .repo_store - .acquire_write(&record.owner_did, &record.name) - .await - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - })?; + // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's + // advisory-lock loop already caps at ~60s, but its per-iteration + // `pg_try_advisory_lock().fetch_one(&pool)` can block indefinitely on a hung / + // exhausted Postgres pool (so the 60-count never advances) — and the write permit + // is held the whole time, draining the pool (#174 P1-2). The outer + // `tokio::time::timeout` cancels a mid-sleep/mid-`fetch_one` future, so it bounds + // both the loop and a hung iteration without any repo_store.rs change (KTD3). The + // permit is a handler local here (moved into the AdmissionGuard only after this), + // so the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let guard = tokio::time::timeout( + acquire_deadline, + state + .repo_store + .acquire_write(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "acquire_write timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| { + tracing::error!(repo = %name, err = %e, "acquire_write failed"); + AppError::Git(e.to_string()) + })?; let disk_path = guard.path().to_path_buf(); tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let receive_result = smart_http::receive_pack(&disk_path, body, git_timeout).await; + // Move both admission permits into the guard so they release only after the spawned + // receive-pack process group is reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). + // The handler keeps no copy. This is independent of the write-lock `guard.release` + // below: admission tracks the git process lifetime, the write lock tracks the repo. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + let receive_result = smart_http::receive_pack( + &state.git_bin, + &disk_path, + body, + git_timeout, + Some(admission), + ) + .await; // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push @@ -1057,6 +1647,8 @@ pub async fn git_receive_pack( &record.owner_did, record.is_public, disk_path.clone(), + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await; @@ -1082,6 +1674,8 @@ pub async fn git_receive_pack( disk_path.clone(), new_tips, old_tips, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await; if pin_set.full_scan { @@ -1091,6 +1685,8 @@ pub async fn git_receive_pack( record.is_public, record.owner_did.clone(), pin_set.candidates, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await } else { @@ -1102,106 +1698,68 @@ pub async fn git_receive_pack( // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). // Skipped entirely when the public cannot read the repo (withheld == None). - if withheld.is_some() { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - tokio::spawn(async move { - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } + // + // Coalesce per repo (#174 P2-2): only spawn a task if none is in flight for this + // repo; otherwise `try_begin` marks the repo DIRTY and the in-flight task requeues + // one more pass (re-reading fresh repo state) before it exits. This bounds the + // outstanding parked-task set to one per repo while covering every coalesced push — + // there is no reconciliation sweep, so a dropped job would be lost forever (#173 F3). + if let Some(withheld_set) = withheld.clone() { + match state.encrypt_inflight.try_begin(&record.id) { + None => { + tracing::debug!( + repo = %record.id, + "post-push encryption task already in flight for this repo; coalescing \ + (the in-flight task will requeue one more pass to cover this push)" + ); } - - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - let p = repo_path_clone.clone(); - let owner = owner_did.clone(); - let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients( - &p, &rules, is_public, &owner, - ) - }) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( - &ipfs_api, - &repo_path_clone, - &db_clone, - &repo_id, - &node_seed, - &recipients, - ) - .await; - - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, - ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), - } - } - } + Some(inflight_guard) => { + let ctx = PostPushReplication { + db: state.db.clone(), + disk_path: disk_path.clone(), + git_bin: state.git_bin.clone(), + timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + ipfs_api: state.config.ipfs_api.clone(), + repo_id: record.id.clone(), + encrypt_sem: state.git_encrypt_semaphore.clone(), + node_seed: *state.node_keypair.to_seed(), + node_did: state.node_did.to_string(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + }; + tokio::spawn(run_post_push_replication( + inflight_guard, + ctx, + object_list.clone(), + rules_opt.clone(), + record.is_public, + record.owner_did.clone(), + withheld_set, + )); } - }); + } } - // Pin new git objects to Pinata, then record branch→CID and gossip + // Pin new git objects to Pinata, then record branch→CID and gossip. + // + // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought + // under the per-repo encryption coalescing above. Two reasons: (1) it does not + // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` + // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter + // set that is the P2-2 residual; it runs to completion under the HTTP client's + // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does + // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription + // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's + // ref_updates. Coalescing it against an in-flight task for the same repo would DROP + // a later push's ref-update announcements (a correctness regression), not merely + // delay a duplicate. So it is scoped out with rationale, not brought under the bound. { let pinata_jwt = state.config.pinata_jwt.clone(); let pinata_upload_url = state.config.pinata_upload_url.clone(); let repo_path_clone = disk_path.clone(); let db_clone = state.db.clone(); + let repo_id = record.id.clone(); let http_client = Arc::clone(&state.http_client); let node_did_str = state.node_did.to_string(); let repo_slug = format!( @@ -1232,6 +1790,7 @@ pub async fn git_receive_pack( &repo_path_clone, object_list_pinata, &db_clone, + &repo_id, ) .await } else { @@ -1862,6 +2421,17 @@ mod tests { assert!(matches!(git_service_app_error(&other), AppError::Git(_))); } + #[test] + fn git_permit_sheds_at_capacity_and_releases() { + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let p1 = git_permit(&sem).expect("first acquire succeeds"); + // At capacity the next request is shed with Overloaded (-> 503), not queued. + assert!(matches!(git_permit(&sem), Err(AppError::Overloaded(_)))); + // Releasing the permit frees the slot for the next request. + drop(p1); + assert!(git_permit(&sem).is_ok()); + } + fn repo_owned_by(owner_did: &str) -> crate::db::RepoRecord { let now = chrono::Utc::now(); crate::db::RepoRecord { @@ -1892,16 +2462,39 @@ mod tests { let dummy = std::path::PathBuf::from("/nonexistent"); // Private: no rules at all. - let (announce, _) = replication_withheld_set(None, OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + None, + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (no rules) must not announce"); // Private: empty rule set, is_public=false → still not listable at root. - let (announce, _) = - replication_withheld_set(Some(vec![]), OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + Some(vec![]), + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (empty rules) must not announce"); // Public: empty rule set, is_public=true → listable at root, announces. - let (announce, _) = replication_withheld_set(Some(vec![]), OWNER_DID, true, dummy).await; + let (announce, _) = replication_withheld_set( + Some(vec![]), + OWNER_DID, + true, + dummy, + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(announce, "public repo must announce"); } @@ -1993,6 +2586,150 @@ mod tests { } } + #[cfg(unix)] + fn write_fake_git(dir: &std::path::Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 (write-pool twin, vetted by execution not reasoning): the receive-pack + /// post-push replication walk is bounded. Drive `replication_withheld_set` with an + /// injected fake git that hangs on `rev-list` and a short budget: it must RETURN + /// within the budget (so `git_receive_pack` releases the write permit it holds + /// across this await, rather than pinning it for the hang) AND fail closed + /// (announce suppressed) because the walk could not be vetted. Proves this path + /// funnels through the bounded `blob_paths`, on the write-permit-holding side. + #[cfg(unix)] + #[tokio::test] + async fn replication_walk_is_bounded_and_fails_closed_on_a_hung_git() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + // Public root (announceable) + a path-scoped rule, so the walk actually runs + // rather than taking the has_path_scoped_rule short-circuit. + let rules = Some(vec![vis_rule("/secret/**", &[])]); + + let result = tokio::time::timeout( + Duration::from_secs(10), + replication_withheld_set( + rules, + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin, + Duration::from_millis(200), + ), + ) + .await + .expect( + "replication_withheld_set must return within the budget; a hung walk must \ + not pin the write permit git_receive_pack holds across it", + ); + assert_eq!( + result, + (false, None), + "a walk that could not be vetted must suppress the announce (fail closed)" + ); + } + + /// #174 (serve-path 504, vetted by execution): a hung withheld-blob walk on the + /// upload-pack POST maps to 504, not a generic 500. Real repo dir on disk (so + /// acquire's fast path returns it) + a path-scoped rule (so the walk runs) + + /// an injected fake git that hangs on rev-list. The handler must return 504, + /// proving git_upload_pack routes the walk's GitServiceTimeout through + /// git_service_app_error end to end. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_hung_withheld_walk_returns_504(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let fake = write_fake_git(tmp.path(), body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 1; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6srv504", "sv", "/tmp/z6srv504-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6srv504", "sv").await.unwrap().unwrap(); + // Path-scoped rule so has_path_scoped_rule() is true and the walk runs; the + // public root still lets an anonymous caller past the "/" gate. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + // acquire()'s fast path returns the local path when it exists on disk. + let disk = std::path::Path::new("/tmp/z6srv504/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.91:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6srv504/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6srv504"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "a hung withheld-blob walk must surface as 504, not a generic 500" + ); + } + + /// #174 (F2 sizing edge, vetted by execution): the receive-pack advertisement + /// per-source cap is derived in main.rs as `(max_concurrent_git_pushes / 8).max(1)`, + /// so it is never 0 even at the minimum write-pool size (1). A 0 cap would make + /// PerCallerConcurrency shed EVERY receive-pack advertisement and break all pushes. + #[test] + fn advert_per_caller_cap_sizing_is_never_zero() { + let cap = |pushes: usize| (pushes / 8).max(1); + for pushes in [1usize, 4, 8, 32, 256] { + assert!( + cap(pushes) >= 1, + "advert cap must be >= 1 for pushes={pushes}" + ); + } + assert_eq!(cap(1), 1, "minimum write pool must derive cap 1, not 0"); + assert_eq!( + cap(32), + 4, + "default write pool 32 derives cap 4 (~8 source IPs to fill)" + ); + // A cap of 1 admits one and sheds the second from the same source. + let lim = crate::rate_limit::PerCallerConcurrency::new(cap(1), 100); + let _held = lim.try_acquire("src").expect("first advert admitted"); + assert!( + lim.try_acquire("src").is_none(), + "second advert from the same source is shed" + ); + } + #[test] fn fork_owner_full_did_with_path_rule_allowed() { // Owner reads everything (implicit reader), so nothing is withheld. @@ -2579,6 +3316,1700 @@ mod tests { ); } + /// #174 P2-1: an unsupported `?service=` must be rejected with 400 BEFORE taking a + /// read slot or doing DB/Tigris work. Isolate it: exhaust the read pool so a read + /// op WOULD shed 503 at the pre-DB check — a garbage service must still return 400 + /// (validation runs first), proving `?service=anything` cannot consume the read + /// pool. Removing the validation makes this 503 (RED). + #[sqlx::test] + async fn info_refs_rejects_unsupported_service_before_the_read_slot(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state + .db + .upsert_mirror_repo("z6svcowner", "svc", "/tmp/svc", None, false) + .await + .unwrap(); + // Exhaust the read pool: a read op would shed 503 at the pre-DB check. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); + + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.90:7000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6svcowner/svc/info/refs?service=git-explode") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "an unsupported ?service= must be 400 before the read-pool shed, not 503" + ); + } + + /// #174 (jatmn P1): the anon-reachable receive-pack advertisement + /// (`GET info/refs?service=git-receive-pack`) draws from a DEDICATED advert pool + /// (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses. + /// Proven at the handler by saturating each pool to zero and checking who shares + /// it (INV-10, across the auth boundary). The load-bearing pair: + /// * advert pool at 0 -> the advert SHEDS 503 (it is bound to that pool); + /// * write pool at 0 -> the advert SURVIVES (it can NOT consume a permit the + /// authenticated POST needs — the reservation jatmn asked for). + /// Revert the branch to `git_write_semaphore` and BOTH flip: the advert-pool-0 + /// case stops shedding and the write-pool-0 case starts shedding (the exact + /// anon-sheds-authed-push starvation). + #[sqlx::test] + async fn receive_pack_advertisement_draws_from_dedicated_advert_pool(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + // Build a fresh state with the three pools sized independently, then drive one + // info/refs advertisement for `service` and return its handler status. + async fn advert_status( + pool: &sqlx::PgPool, + read_permits: usize, + write_permits: usize, + advert_permits: usize, + service: &str, + ) -> StatusCode { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_read_semaphore = Arc::new(Semaphore::new(read_permits)); + state.git_write_semaphore = Arc::new(Semaphore::new(write_permits)); + state.git_push_advert_semaphore = Arc::new(Semaphore::new(advert_permits)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6wpadv", "wp", "/tmp/wp-nonexistent", None, false) + .await + .unwrap(); + let peer: SocketAddr = "203.0.113.61:6000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/z6wpadv/wp/info/refs?service={service}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + router.oneshot(req).await.unwrap().status() + } + + // Advert pool saturated (read + write free): the receive-pack advert SHEDS, + // proving it is bound to the dedicated advert pool. + assert_eq!( + advert_status(&pool, 8, 8, 0, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement draws from the dedicated advert pool: a saturated advert pool sheds it 503" + ); + // WRITE pool saturated (advert + read free): the advert SURVIVES. This is the + // reservation — an advert flood can never occupy a permit the authenticated + // push POST relies on at admission. + assert_ne!( + advert_status(&pool, 8, 0, 8, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must NOT draw from the write pool: a saturated write pool must not shed it" + ); + // Read pool saturated (advert + write free): the advert SURVIVES (never on the read pool). + assert_ne!( + advert_status(&pool, 0, 8, 8, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must not draw from the read pool" + ); + // Read pool saturated: the upload-pack advertisement still SHEDS (unchanged). + assert_eq!( + advert_status(&pool, 0, 8, 8, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement stays on the read pool: a saturated read pool sheds it 503" + ); + // Write + advert pools saturated, read free: the upload-pack advertisement is + // UNAFFECTED, proving reads never touch either write-side pool. + assert_ne!( + advert_status(&pool, 8, 0, 0, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement never touches the write or advert pool" + ); + } + + /// #174 U2: the receive-pack advertisement is a write-path op, so it must not be + /// shed by the READ per-caller sub-cap even when the caller's source IP has + /// exhausted its read budget (e.g. concurrent clones from the same host). Fill + /// the IP's read per-caller slot, then the receive-pack advertisement from that + /// same IP must still get through. Restore the unconditional read-cap acquire on + /// the receive-pack branch and this goes 503. + #[sqlx::test] + async fn receive_pack_advertisement_ignores_read_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6wpc", "wp", "/tmp/wp-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.71:6000".parse().unwrap(); + // Exhaust the source IP's single READ per-caller slot, as concurrent clones + // from the same host would. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("fill the IP's read per-caller slot"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6wpc/wp/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must not be shed by the read per-caller cap: it is a write-path op" + ); + } + + /// #174 (review fix): the anon-reachable receive-pack advertisement draws from + /// the write pool, so it is bounded per source by `git_push_advert_per_caller` to + /// stop one source from monopolizing the write pool and shedding authenticated + /// pushes. Fill one source IP's advert slot; its next receive-pack advertisement + /// sheds 503, while a different source and the upload-pack advertisement are + /// unaffected. Remove the advert-cap acquisition and the same-source assertion + /// goes green-not-503. + #[sqlx::test] + async fn receive_pack_advertisement_capped_per_source(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6advcap", "ac", "/tmp/ac-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:6000".parse().unwrap(); + // Fill this source IP's single receive-pack-advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + // Same source: the receive-pack advertisement sheds 503 (advert cap full). + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its receive-pack advertisement cap must shed 503, so it cannot monopolize the write pool" + ); + + // A DIFFERENT source keeps its own advert budget -> not shed. + let other: SocketAddr = "203.0.113.82:6000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must keep its own receive-pack advertisement budget" + ); + + // The upload-pack advertisement is NOT bounded by the receive-pack advert cap. + let router3 = crate::server::build_router(state); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "the upload-pack advertisement must not be shed by the receive-pack advert cap" + ); + } + + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that + /// is already at its concurrency budget on the upload-pack advertisement, while + /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and + /// the same-caller assertion goes green-not-503 — this is the info_refs half of + /// the two-handler mutation probe. + #[sqlx::test] + async fn info_refs_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcadv", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.31:5000".parse().unwrap(); + // Fill this caller's single read slot (a clone shares the Arc-backed map). + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + // Same caller (IP) at its cap -> shed 503 before the git/Tigris work. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed the advertisement with 503" + ); + + // A DIFFERENT caller (IP) has its own budget -> not shed by the per-caller cap. + let other: SocketAddr = "203.0.113.32:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 SC2 (upload_pack probe): the same per-caller shed on the POST + /// upload-pack path. Remove the sub-cap from `git_upload_pack` and this goes + /// green-not-503 — the upload_pack half of the two-handler mutation probe. + #[sqlx::test] + async fn upload_pack_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcupl", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.41:5000".parse().unwrap(); + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed upload-pack with 503" + ); + + let other: SocketAddr = "203.0.113.42:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 (review fix): the per-source caller cap is an independent brake that + /// sheds a capped source even when the global pool has free capacity — the + /// sub-cap is not a mere pre-filter for pool exhaustion. Proven by leaving the + /// global read pool with capacity (so the pre-DB early shed passes) AND + /// pre-holding the source's upload-pack read sub-cap: the request reaches the + /// caller cap and sheds there, so its 503 body reads "for this caller". Remove + /// the `acquire_read_caller_permit` call and the capped source falls through to + /// the git op instead of shedding with "for this caller" — this is the + /// caller-cap acquire probe for the info/refs upload-pack branch. + #[sqlx::test] + async fn info_refs_upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordir", "oi", "/tmp/oi-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.91:5000".parse().unwrap(); + // Pin this source at its single upload-pack read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordir/oi/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the receive-pack + /// advertisement branch of info/refs — its per-source cap + /// (`git_push_advert_per_caller`) sheds a capped source even when the global + /// write pool has capacity. Leave the global write pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's advert slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". The push rate limiter is left permissive so the + /// request reaches the caller cap. + #[sqlx::test] + async fn info_refs_receive_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has free capacity (early shed passes); source pre-held at + // its advert sub-cap so it sheds on the caller cap, not the global pool. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + // Permissive push rate limiter so the advertisement passes the rate gate and + // reaches the per-source concurrency cap. + state.push_rate_limiter = crate::rate_limit::RateLimiter::new(100, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordrp", "or", "/tmp/or-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.92:5000".parse().unwrap(); + // Pin this source at its single receive-pack advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordrp/or/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its advert sub-cap must shed 503 even with global write pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source advert cap is an independent brake: with global write capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the POST upload-pack + /// handler — its per-source read cap sheds a capped source even when the global + /// read pool has capacity. Leave the global read pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's read slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". + #[sqlx::test] + async fn upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordup", "ou", "/tmp/ou-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.93:5000".parse().unwrap(); + // Pin this source at its single read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6ordup/ou/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 U3 (P1-b, RED-before/GREEN-after): a client disconnect during the + /// path-scoped withheld-blob walk must NOT release the read admission while the + /// uncancellable `spawn_blocking` walk is still running. The handler takes the + /// global read permit, enters the walk (a fake git hangs on rev-list), then the + /// request future is dropped mid-walk. With both permits moved into the blocking + /// task the global slot stays occupied until the walk finishes; on the pre-fix code + /// the handler-local permits drop on future-drop and the slot frees instantly (RED), + /// letting disconnect-spam exceed the cap while real git work keeps running. + #[sqlx::test] + async fn upload_pack_permit_held_through_walk_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git: resolve refs fast, hang on rev-list (recording its pid first). The + // ~6s sleep bounds the walk so a broken fix cannot wedge the suite. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > \"{}\" ; sleep 6 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Root the repo store at this test's TempDir so the bare repo is isolated per + // run (the default for_testing store uses a fixed /tmp path that would collide + // across runs). + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up3rd"; + let name = "up3"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the walk. + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + // A path-scoped rule so has_path_scoped_rule() is true (the walk path) without + // denying the "/" gate for the public repo. + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.77:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let mut fut = Box::pin(router.oneshot(req)); + // Drive until the walk's rev-list starts (its pidfile appears) — i.e. the + // request is inside the spawn_blocking walk, holding the global read permit. + let mut in_walk = false; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if revlist_pid.exists() { + in_walk = true; + break; + } + } + assert!( + in_walk, + "the walk's rev-list must start (request reached the spawn_blocking walk)" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the walk runs" + ); + + // Client disconnect: drop the request future mid-walk. + drop(fut); + + // Load-bearing: the slot must STAY held while the uncancellable walk runs. On + // the pre-fix code the handler-local permits drop here and the slot frees at + // once (RED). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be held until the spawn_blocking walk \ + finishes, not released the instant the future drops (P1-b)" + ); + + // Cleanup: let the walk finish so the slot releases and no blocking task leaks. + for _ in 0..400 { + if sem.available_permits() == 1 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + unsafe { + libc::kill(p, libc::SIGKILL); + } + } + } + + /// #174 U1 (P1-a, plain-spawn residual, RED-before/GREEN-after): on the PLAIN + /// (non-path-scoped) upload-pack path a client disconnect must NOT release the + /// global read admission while the detached process-group reaper is still tearing + /// down a git group that ignores SIGTERM. The `be0cdd6` fix moved permits into the + /// path-scoped `spawn_blocking` walk; this closes the residual plain path, where the + /// permits were handler-locals that dropped the instant the future was dropped. + /// + /// Isolate the GLOBAL pool: read pool = 1, per-source cap + rate limiter permissive, + /// so the only thing that can shed a replacement is the leaked global permit. Drive + /// the handler until git spawns, disconnect, then assert the global slot stays held + /// (`available_permits() == 0`) AND a replacement sheds 503 while the group is alive; + /// after the reaper SIGKILLs+reaps the group the slot frees and a replacement is no + /// longer shed by the global cap. On the pre-fix code the handler-local permit drops + /// on future-drop and the slot frees at once (RED). + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_plain_permit_held_through_group_reap_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Fake git for the plain upload-pack path (invoked as `git upload-pack + // --stateless-rpc `). It forks a descendant that TRAPS SIGTERM, records its + // pid, and loops ~20s, then `wait`s — so on disconnect the group leader dies on + // the reaper's SIGTERM but the descendant survives until the reaper escalates to + // SIGKILL, keeping the group alive (ESRCH not reached) across the observation + // window. Bounded so a broken fix leaks no permanent orphan. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + upload-pack)\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + descfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global read pool: size 1; per-source cap + rate limiter permissive + // so only the leaked global permit can shed the replacement. + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up1st"; + let name = "up1"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the + // spawn. No path-scoped rule -> the PLAIN serve branch (this test's target). + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until git spawns (the descendant records its pid) — the request is + // inside the plain serve, holding the global read permit. Stop polling the + // instant the future completes (re-polling a completed oneshot panics); read the + // descfile first so a spawn that recorded its pid then returned is still caught. + let mut spawned: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + spawned = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let desc = spawned + .unwrap_or_else(|| panic!("the fake git must have spawned; early finish: {early:?}")); + // Kill the descendant regardless of outcome so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(desc); + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "descendant should be running before the disconnect" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the git op runs" + ); + + // Client disconnect: drop the request future. The detached reaper now owns the + // AdmissionGuard and will not drop it until the group is ESRCH-confirmed reaped. + drop(fut); + + // Load-bearing: the slot must STAY held while the SIGTERM-ignoring group is still + // alive. On the pre-fix code the handler-local permit drops here and the slot + // frees at once (RED). Check quickly (before the reaper's ~2s SIGKILL escalation). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be HELD until the process group is \ + reaped, not released the instant the future drops (P1-a)" + ); + // A replacement request from a DIFFERENT source must shed 503 — the only pool + // that can shed it is the leaked global permit (per-source cap is permissive). + let peer2: SocketAddr = "203.0.113.72:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "while the prior group is still alive the held global permit must shed a \ + replacement with 503" + ); + + // After the reaper SIGKILLs + reaps the group the AdmissionGuard drops and the + // slot frees. Poll for recovery. + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the reaper confirms the group gone the admission guard must drop and \ + free the global slot" + ); + // A replacement is now no longer shed by the global cap (it proceeds past + // admission; it then fails downstream on the fake git, which is not a 503). + let peer3: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let resp = router.oneshot(make_req(peer3)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "after the group is reaped the freed slot must admit a replacement" + ); + } + + /// #174 U1 (P1-a): the `None`-key arm — a request with no resolvable source key + /// (no trusted-proxy header, no peer) is bounded by the GLOBAL read pool only, never + /// a per-source cap. With the global read pool exhausted such a request still sheds + /// 503, proving the plain path admits/sheds on the global pool for the `None` arm + /// (the counterpart to the `Some(ip)` arm above). Complements the resolver-arm rule: + /// neither arm is vacuous. + #[tokio::test] + async fn upload_pack_plain_none_key_arm_sheds_on_global_pool() { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use axum::Router; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state_lazy(); + // Global read pool exhausted; per-source cap permissive so only the global pool + // can shed. No ConnectInfo + no trusted header -> read_caller_key resolves None. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + // No ConnectInfo extension and no XFF header: the caller key is None. + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL read pool" + ); + } + + /// #174 U4 (P1-d, RED-before/GREEN-after): the authenticated receive-pack POST + /// carries a per-source WRITE sub-cap so one source IP cannot monopolize the write + /// pool with many slow pushes (owner enforcement defaults off, so disposable DIDs + /// are free). Global write pool has capacity; the source is pre-held at its single + /// write slot. A push from THAT source sheds (Overloaded/503) — which also proves + /// the PeerAddr+HeaderMap extractors resolve a key (without them the key is None and + /// the cap is inert, never shedding). A push from a DIFFERENT source is NOT shed by + /// the cap. Called directly so the test needs no signed request; the handler is + /// where the cap lives. Remove the `git_write_per_caller` acquire and the capped + /// source no longer sheds (RED). + #[sqlx::test] + async fn receive_pack_per_source_write_cap_sheds_capped_source_not_others(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has capacity; the per-source cap is 1. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6rp4wr", "rp4", "/tmp/rp4-nonexistent", None, false) + .await + .unwrap(); + + let did = "did:key:z6MkReceivePackWriteCapProofDidAAAAAAAAAA"; + let capped: SocketAddr = "203.0.113.44:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.45:5000".parse().unwrap(); + + // Pin the capped source at its single write slot. + let _slot = state + .git_write_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first write slot for the capped source IP"); + + // A push from the capped source must shed on the per-source write cap even with + // global write capacity free. The shed also proves the source-IP key resolved + // via the extractors (an inert None key would fall through to Ok(None)). + let capped_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(capped)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + matches!(capped_result, Err(AppError::Overloaded(_))), + "a source at its per-source write cap must shed (Overloaded/503) with global \ + pool capacity free; got {capped_result:?}" + ); + + // A push from a DIFFERENT source must NOT be shed by the per-source cap — it + // proceeds past admission (and fails later on the nonexistent repo, which is not + // an Overloaded error). + let other_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(other)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(other_result, Err(AppError::Overloaded(_))), + "a different source must not be shed by the per-source write cap while the \ + capped source holds its slot; got {other_result:?}" + ); + } + + /// #174 U2 (P1-2, RED-before/GREEN-after): the storage-acquisition phase is bounded + /// by `git_acquire_timeout_secs`, so a stalled backend releases the admission permit + /// and sheds a 503 instead of pinning the pool. The permit is taken BEFORE + /// `acquire_write`, whose advisory-lock loop can spin ~60s (and whose per-iteration + /// `pg_try_advisory_lock` can block indefinitely on a hung pool), so without the + /// `tokio::time::timeout` wrapper the permit is held far past the deadline. + /// + /// Real stall (no `RepoStore` trait to fake): hold the SAME session-level advisory + /// lock `acquire_write` derives (`advisory_lock_key(owner_slug, repo_name)`, where + /// `owner_slug = owner_did.replace([':','/'], "_")`) on a second pooled connection, + /// so the handler's `pg_try_advisory_lock` returns false every iteration and the loop + /// must retry against the deadline. `git_acquire_timeout_secs = 2`; the request must + /// return 503 (Overloaded) at ~2s (NOT ~59s), and the write permit must be released + /// (`available_permits()` recovers to full once the shed returns). Covers R2. + /// + /// Load-bearing / mutation: remove the `tokio::time::timeout` wrapper on + /// `acquire_write` and the loop runs to ~59s with the permit held the whole time — + /// the `< DEADLINE_CEILING` timing assertion goes RED (observed ~59s) and the permit + /// stays pinned past the deadline. Restore to return GREEN. + #[sqlx::test] + async fn receive_pack_acquire_deadline_sheds_and_releases_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + // Reproduce acquire_write's session-level advisory-lock key exactly so the + // second-connection lock collides with the handler's pg_try_advisory_lock + // (repo_store.rs: advisory_lock_key over owner_slug then repo_name). + fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + owner_slug.hash(&mut hasher); + repo_name.hash(&mut hasher); + hasher.finish() as i64 + } + + let owner = "z6acqdead"; + let name = "acq1"; + // owner_slug as local_path() computes it from the record's owner_did. The + // mirror row stores the short owner as owner_did, so slug == owner (no ':'/'/'). + let owner_slug = owner.replace([':', '/'], "_"); + let lock_key = advisory_lock_key(&owner_slug, name); + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Isolate the write pool at size 1 so available_permits() cleanly reports + // held (0) vs released (1). Per-source cap + trust permissive so only the + // write pool / acquire path can gate. + state.git_write_semaphore = Arc::new(Semaphore::new(1)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Short acquire deadline: the fix must shed here, well before acquire_write's + // ~59s advisory-lock loop would bail on its own. + const ACQUIRE_TIMEOUT_SECS: u64 = 2; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = ACQUIRE_TIMEOUT_SECS; + // Keep the git-service timeout large so the deadline under test is the acquire + // one, not git execution (which is never reached on the stalled path anyway). + cfg.git_service_timeout_secs = 600; + state.config = std::sync::Arc::new(cfg); + + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6acqdead-acq1", None, false) + .await + .unwrap(); + + // Hold the advisory lock on a dedicated pooled connection (a distinct session), + // so the handler's pg_try_advisory_lock($lock_key) returns false every iteration + // and acquire_write's real loop must retry against the deadline. Released when + // this connection drops at end of test. + let mut lock_conn = pool + .acquire() + .await + .expect("second connection for the lock"); + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("hold the advisory lock on the second connection"); + + let did = "did:key:z6MkAcquireDeadlineProofDidAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + + let sem = state.git_write_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one write slot before the request" + ); + + // Drive the authenticated push in the background so we can observe the permit is + // held while acquire_write stalls, then that it is released on the shed. + let state_for_task = state.clone(); + let start = std::time::Instant::now(); + let handle = tokio::spawn(async move { + git_receive_pack( + State(state_for_task), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + + // The handler takes the write permit BEFORE acquire_write, so once it is stalled + // in the advisory-lock loop the pool reports 0 available. Wait for that to prove + // the permit is genuinely held during the stall (and the request really reached + // acquire_write, not an earlier reject). + let mut held = false; + for _ in 0..200 { + if sem.available_permits() == 0 { + held = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + held, + "the write permit must be held while acquire_write stalls on the advisory lock" + ); + + // The bounded acquire deadline must shed with 503 (Overloaded), NOT wait out the + // ~59s advisory-lock loop. Ceiling is comfortably above the 2s deadline + task + // scheduling but far below 59s, so a RED run (no wrapper -> ~59s) fails here. + const DEADLINE_CEILING: std::time::Duration = std::time::Duration::from_secs(20); + let result = tokio::time::timeout( + DEADLINE_CEILING + std::time::Duration::from_secs(10), + handle, + ) + .await + .expect("the handler must return within the ceiling — a hang means the acquire deadline is missing (RED)") + .expect("the receive-pack task must not panic"); + let elapsed = start.elapsed(); + + assert!( + matches!(result, Err(AppError::Overloaded(_))), + "a stalled acquire_write must shed with Overloaded/503 at the acquire deadline; \ + got {result:?}" + ); + assert!( + elapsed < DEADLINE_CEILING, + "the shed must land at ~{ACQUIRE_TIMEOUT_SECS}s (the acquire deadline), not ~59s \ + (the advisory-lock loop). Observed {elapsed:?}; without the timeout wrapper this \ + is ~59s (RED)" + ); + + // Permit release on expiry: the Overloaded return drops the handler-local permit, + // so the isolated write pool must recover to full. A leaked permit here means the + // pool drains under a stalled backend (the #174 P1-2 bug). + let mut freed = false; + for _ in 0..200 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + freed, + "on the acquire-deadline shed the write permit must be released; the pool did \ + not recover to full (permit leaked)" + ); + + // Follow-up admits once the contended lock is released: release the second-conn + // lock, then a fresh push proceeds PAST admission (it fails later on the + // nonexistent on-disk repo, which is NOT an Overloaded/503). Proves the freed + // slot is usable, not merely counted. + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("release the advisory lock"); + let followup = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.62:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(followup, Err(AppError::Overloaded(_))), + "once the lock frees, a follow-up push must admit past the (recovered) write \ + pool and acquire; got {followup:?}" + ); + } + + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a + /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn + /// unbounded concurrent full-history walks. With the pool exhausted the gated walk + /// must DEFER (block on admission) and NOT run its rev-list; on the pre-fix code + /// (no acquire) the walk runs regardless of the pool (RED). It defers rather than + /// sheds — releasing the permit lets the SAME walk run and pin (durability stays + /// fail-closed). Exercises the gating seam directly; the detached push task calls + /// this exact helper. + #[tokio::test] + async fn encrypt_walk_defers_when_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("revlist.ran"); + // Fake git records when rev-list runs (the walk's first git call). + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo ran > \"{}\" ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let git_bin = git_path.to_str().unwrap().to_string(); + let owner = "did:key:z6MkEncWalkOwnerAAAAAAAAAAAAAAAAAAAAAAAA".to_string(); + + // Exhaust the pool: hold its only permit so a gated walk must defer. + let sem = Arc::new(Semaphore::new(1)); + let held = sem.clone().acquire_owned().await.unwrap(); + + // Blocked: the gated walk must NOT complete or run rev-list while exhausted. + let blocked = tokio::time::timeout( + Duration::from_millis(500), + withheld_recipients_gated( + sem.clone(), + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + Vec::new(), + true, + owner.clone(), + ), + ) + .await; + assert!( + blocked.is_err(), + "the encryption walk must defer (block on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the walk's rev-list must not run while its admission permit is unavailable (P1-e)" + ); + + // Release admission: the SAME walk now runs (defer, not shed) — rev-list fires. + drop(held); + let ran = withheld_recipients_gated( + sem, + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(5), + Vec::new(), + true, + owner, + ) + .await; + assert!( + ran.is_ok(), + "with a permit the walk runs and joins: {ran:?}" + ); + assert!( + marker.exists(), + "once admission is available the deferred walk runs its rev-list" + ); + } + + // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- + // + // The residual jatmn found is not the WALK (bounded by `git_encrypt_semaphore`, + // proven by `encrypt_walk_defers_when_pool_exhausted` above) but the OUTER + // `tokio::spawn` + its parked `acquire_owned().await` waiters: N rapid pushes to a + // repo spawn N tasks that each park holding cloned object lists/rules/keys — an + // unbounded outstanding set. U4 bounds it by coalescing per repo: before spawning, + // if a task for the repo is in flight, skip the duplicate. Crucially this DEFERS a + // duplicate walk (the newer push's objects are covered by the pending one) and does + // NOT shed — there is no reconciliation sweep, so a dropped job would permanently + // lose the withheld-blob recovery copy (`2a54c15`'s fail-closed durability stance). + // + // These drive the coalescing seam (`EncryptInflight`) that the detached spawn at + // `repos.rs` consults directly (the try_begin gate on the in-flight set, guarded by + // `withheld.is_some()`). Observing `encrypt_and_pin`'s IPFS effect end-to-end needs a live IPFS node + // (`pin_git_object` hits the API), so the durability property is proven at this + // layer: a coalesced repo's key is released when its task ends, so a later push for + // that repo is processed once — NOT permanently skipped, which is exactly what a + // coalesce->shed mutation would break by dropping the job with no sweep to recover it. + + /// Bounded outstanding set under saturation (R4). Simulate K rapid path-scoped + /// pushes to the SAME repo while the encrypt pool is saturated (every spawned task + /// would park, so none has finished and removed its key): the first `try_begin` + /// admits (spawns), the rest coalesce (skip). The in-flight set holds at 1, not K. + /// + /// MUTATION (RED): removing the coalescing check makes every push spawn — modeled by + /// `simulate_without_coalescing`, which reaches K. If the coalesced count equaled the + /// un-coalesced one the gate would be a no-op; the strict inequality proves it bites. + #[test] + fn u4_outstanding_encrypt_set_is_bounded_to_one_per_repo_under_saturation() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkRepoOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA/proj"; + const K: usize = 32; + + // Hold every admitted guard so the tasks are "still in flight" (the saturated + // case: all parked on acquire_owned().await, none finished, none removed a key). + let mut admitted = Vec::new(); + let mut coalesced = 0usize; + for _ in 0..K { + match inflight.try_begin(repo) { + Some(g) => admitted.push(g), + None => coalesced += 1, + } + } + + assert_eq!( + admitted.len(), + 1, + "exactly ONE detached task may spawn per repo while one is in flight — the \ + outstanding set is bounded to 1, not K parked waiters" + ); + assert_eq!( + coalesced, + K - 1, + "the other K-1 rapid pushes to the same repo coalesce (skip spawning)" + ); + assert_eq!( + inflight.len(), + 1, + "the in-flight set holds at most one entry per repo under saturation" + ); + + let no_coalesce = simulate_without_coalescing(K); + assert_eq!( + no_coalesce, K, + "sanity: without the coalescing check all K pushes spawn (the unbounded set \ + the fix prevents) — proves the bound above is not vacuously 1" + ); + assert!( + admitted.len() < no_coalesce, + "coalesced set ({}) must be strictly smaller than the un-coalesced one ({})", + admitted.len(), + no_coalesce + ); + } + + /// Coalescing is PER-REPO: distinct repos are never coalesced against each other, so + /// one repo in flight cannot starve a second repo's recovery copy. + #[test] + fn u4_distinct_repos_each_admit_one_encrypt_task() { + let inflight = crate::state::EncryptInflight::new(); + let a = inflight.try_begin("owner/repo-a"); + let b = inflight.try_begin("owner/repo-b"); + let c = inflight.try_begin("owner/repo-c"); + assert!( + a.is_some() && b.is_some() && c.is_some(), + "three distinct repos each admit their own encryption task" + ); + assert_eq!(inflight.len(), 3, "one in-flight entry per distinct repo"); + } + + /// NO LOST RECOVERY COPY — the security guard (R4/R6). Coalescing must DELAY a + /// duplicate walk, never permanently drop a repo's recovery copy. Observable + /// property: once an in-flight task ENDS (its guard drops — completion, error, or + /// panic-unwind) the repo key is released, so the NEXT push for that repo is admitted + /// and processed again. A coalesce->shed mutation would drop the job AND never + /// re-admit — with no reconciliation sweep the copy is lost forever. Here re-admission + /// survives normal completion AND a panic, so no permanent skip / no leaked key. + #[test] + fn u4_coalesced_repo_is_reprocessed_after_task_ends_not_permanently_skipped() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkDurableRepoBBBBBBBBBBBBBBBBBBBBBBBBB/repo"; + + // Push #1 admits and "spawns". A concurrent push #2 (task #1 still in flight) + // coalesces — no duplicate spawn. + let guard1 = inflight.try_begin(repo).expect("first push admits"); + assert!( + inflight.try_begin(repo).is_none(), + "while task #1 is in flight, push #2 to the same repo coalesces" + ); + + // Task #1 finishes (encrypt_and_pin ran or errored): guard drops, key released. + drop(guard1); + assert_eq!( + inflight.len(), + 0, + "when the in-flight task ends its repo key is released — the set does not leak" + ); + + // A LATER push for the SAME repo is admitted again (processed, not skipped + // forever). This is what coalesce->shed breaks: shed drops the job and no sweep + // re-derives the missing copy, so the recovery copy is permanently lost. + let guard2 = inflight.try_begin(repo).expect( + "a later push for a coalesced repo MUST be re-admitted — durability: the \ + deferred recovery copy is produced eventually, never dropped", + ); + drop(guard2); + assert_eq!(inflight.len(), 0); + + // Durability across PANIC: a task that panics mid-walk must still release its key + // (Drop runs on unwind), so one crashed walk never permanently locks a repo out + // of future recovery copies. + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _g = inflight.try_begin(repo).expect("admit before the panic"); + assert_eq!(inflight.len(), 1); + panic!("simulate the detached encryption task panicking mid-walk"); + })); + assert!(panicked.is_err(), "the simulated task panicked"); + assert_eq!( + inflight.len(), + 0, + "a panicked encryption task still releases its repo key (Drop on unwind) — no \ + permanent leak that would block every future recovery copy for the repo" + ); + assert!( + inflight.try_begin(repo).is_some(), + "after a panicked task the repo can still be admitted — durability preserved" + ); + } + + /// Degenerate state: the first push on a cold/empty in-flight set always admits + /// (never a false coalesce on an empty set). + #[test] + fn u4_first_push_on_a_cold_set_always_admits() { + let inflight = crate::state::EncryptInflight::new(); + assert!(inflight.is_empty(), "cold set is empty"); + assert!( + inflight.try_begin("owner/first").is_some(), + "the first push on a cold in-flight set must admit (never falsely coalesce)" + ); + } + + // ---- U3 (#173 F3): dirty-flag requeue seam (the mechanics behind the end-to-end + // requeue tests in test_support.rs) ---- + + /// A coalesced push MARKS the repo dirty (not just "skip"), and the in-flight task's + /// tail check-and-clear then runs ONE more pass before releasing: `requeue_or_release` + /// returns `true` (loop) while dirty, clearing the flag, then `false` (release) when + /// clean. This is the mechanism that makes a coalesced push requeued, not dropped. + #[test] + fn u3_try_begin_marks_dirty_then_requeue_loops_once_then_releases() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkU3DirtyAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/proj"; + + let mut guard = inflight.try_begin(repo).expect("first push admits"); + assert_eq!(inflight.dirty(repo), Some(false), "fresh task starts clean"); + + // A push coalesces during the in-flight window: marked dirty, no new task. + assert!( + inflight.try_begin(repo).is_none(), + "coalesced push does not spawn" + ); + assert_eq!( + inflight.dirty(repo), + Some(true), + "the coalesced push marked the repo dirty" + ); + + // Task tail, pass 1: dirty -> requeue (clear the flag, keep the key, loop). + assert!( + guard.requeue_or_release(), + "dirty repo requeues one more pass" + ); + assert_eq!( + inflight.dirty(repo), + Some(false), + "the dirty flag is cleared for the next pass" + ); + assert_eq!( + inflight.len(), + 1, + "the key is still held while the task loops" + ); + + // Task tail, pass 2: clean -> release (remove the key, exit). + assert!( + !guard.requeue_or_release(), + "a clean repo releases and exits" + ); + assert_eq!(inflight.len(), 0, "the key is removed on the clean release"); + assert_eq!( + inflight.dirty(repo), + None, + "no in-flight entry after release" + ); + } + + /// The check-and-clear is ATOMIC with the release decision, so no push lands in a + /// "checked clean but still present" gap (scenario 6, race gap). A push that arrives + /// BEFORE the tail check sets dirty -> the same pass requeues it. A push that arrives + /// AFTER a clean release finds an empty set -> `try_begin` spawns a fresh task. Both + /// directions covered; neither drops the push. + #[test] + fn u3_requeue_or_release_leaves_no_uncovered_gap() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkU3GapBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB/proj"; + + // Case A: push arrives before the tail check -> requeue covers it. + let mut guard = inflight.try_begin(repo).expect("admit"); + assert!( + inflight.try_begin(repo).is_none(), + "push arrives during the window" + ); + assert!( + guard.requeue_or_release(), + "the pre-check push is covered by a requeue" + ); + // Now clean: the task releases and exits. + assert!(!guard.requeue_or_release(), "clean -> release"); + assert_eq!(inflight.len(), 0); + + // Case B: a push arriving after release starts a brand-new task (not dropped). + let guard2 = inflight.try_begin(repo); + assert!( + guard2.is_some(), + "a post-release push spawns a fresh task, never dropped" + ); + } + + /// Drop is a PANIC BACKSTOP only. A task that panics before releasing still frees the + /// key (so a crashed walk never permanently locks the repo out); a task that releases + /// via `requeue_or_release` and then drops does not double-free. + #[test] + fn u3_drop_is_panic_backstop_for_unreleased_guard() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkU3PanicCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC/proj"; + + // Normal release then drop: key already gone, drop is a no-op. + let mut g = inflight.try_begin(repo).expect("admit"); + assert!(!g.requeue_or_release(), "clean release"); + assert_eq!(inflight.len(), 0); + drop(g); + assert_eq!( + inflight.len(), + 0, + "dropping an already-released guard does not resurrect a key" + ); + + // Panic before releasing: Drop-on-unwind frees the key. + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _g = inflight.try_begin(repo).expect("admit before panic"); + assert_eq!(inflight.len(), 1); + panic!("task panics before reaching its tail check-and-clear"); + })); + assert!(panicked.is_err(), "the simulated task panicked"); + assert_eq!( + inflight.len(), + 0, + "a panic before release still frees the key (backstop), so the repo is not locked out" + ); + assert!( + inflight.try_begin(repo).is_some(), + "the repo can be admitted again after a panic" + ); + } + + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. + /// Returns the count of tasks spawned (== the size of the unbounded outstanding set + /// the fix prevents), used as the RED comparison in the bound test above. + fn simulate_without_coalescing(pushes: usize) -> usize { + (0..pushes).count() + } + + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the + /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot + /// multiply its budget. Fill the source IP's single read slot, then drive two + /// requests signed under DIFFERENT DIDs from that SAME IP: both must shed 503 + /// (keyed by the saturated IP, not their own free DID slots). A signed request + /// from a DIFFERENT source IP keeps its own budget. Revert `read_caller_key` to + /// prefer the DID and the same-IP assertions go green-not-503 (each fresh DID + /// gets a free slot) -- the farm-defeat mutation probe. + #[sqlx::test] + async fn info_refs_per_caller_cap_keys_on_ip_not_did(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcip", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let did_a = "did:key:z6MkPerCallerKeyingProofDidAAAAAAAAAAAAAAAA"; + let did_b = "did:key:z6MkPerCallerKeyingProofDidBBBBBBBBBBBBBBBB"; + let peer: SocketAddr = "203.0.113.51:5000".parse().unwrap(); + + // Fill the SOURCE IP's single read slot; both DIDs' own slots stay free. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this source IP"); + + // Signed as DID_A from `peer`: keyed by the saturated source IP -> shed 503. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller must be keyed by its source IP, not its DID: the saturated IP must shed it 503" + ); + + // Same IP, a DIFFERENT DID: still keyed by the same saturated IP -> also shed. + // The farm defeat: minting a fresh DID buys no fresh per-source budget. + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(peer)); + req2.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_b.to_string())); + assert_eq!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a second DID from the same source IP must also shed 503: a DID farm cannot multiply the per-source budget" + ); + + // A signed caller from a DIFFERENT source IP keeps its own budget -> not shed. + let other: SocketAddr = "203.0.113.52:5000".parse().unwrap(); + let router3 = crate::server::build_router(state.clone()); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(other)); + req3.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller from a different source IP must keep its own per-source budget" + ); + } + + /// #174 SC2 (None-key): a request with no resolvable caller key (no ConnectInfo, + /// no trusted header) must NOT be shed by the per-caller cap even when another + /// caller's budget is full — it is bounded by the global read pool only. A None + /// key never keys into the map, so it never 503s from the per-caller sub-cap. + #[sqlx::test] + async fn info_refs_none_key_bypasses_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcnone", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + // Saturate an unrelated caller's budget; the None-key request must be + // unaffected because it never keys into the per-caller map. + let _slot = state + .git_read_per_caller + .try_acquire("203.0.113.99") + .expect("hold an unrelated caller's slot"); + + // No ConnectInfo inserted -> PeerAddr is None -> no per-caller key. + let router = crate::server::build_router(state.clone()); + let req = Request::builder() + .method(Method::GET) + .uri("/z6pcnone/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a request with no resolvable caller key must not be shed by the per-caller cap" + ); + } + /// Repo creation must be throttled by the per-IP creation limiter BEFORE /// signature verification — otherwise a DID farm (one throwaway did:key per /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 720fb3ae..96e6f5ea 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -516,10 +516,28 @@ mod tests { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..5480beda 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -172,9 +172,9 @@ pub struct Config { /// its process group torn down, in seconds. Bounds a git that neither /// finishes nor disconnects. Must be positive; set it very large to /// effectively disable the bound. Default: 600s (10 min), generous for large - /// clones. Does not cover the ref advertisement (`info/refs`) or the - /// withheld-blob fetch path (`upload_pack_excluding`, a blocking - /// `spawn_blocking` a tokio timeout cannot cancel); both remain unbounded. + /// clones. Also bounds the ref advertisement (`info/refs`) and the withheld-blob + /// pack build (`upload_pack_excluding`'s pack-objects stage), which now share the + /// same timeout + process-group teardown (#174). #[arg( long, env = "GITLAWB_GIT_SERVICE_TIMEOUT_SECS", @@ -183,6 +183,27 @@ pub struct Config { )] pub git_service_timeout_secs: u64, + /// Maximum wall-clock time the storage-acquisition phase of a served git + /// operation may run before the request is shed with a 503, in seconds. This + /// bounds `RepoStore::{acquire,acquire_fresh,acquire_write}` — the Tigris + /// HEAD/GET on a read/advert acquire and the advisory-lock retry loop (incl. a + /// per-iteration `pg_try_advisory_lock` that can block on a hung Postgres pool) + /// on a write acquire. A concurrency permit is taken BEFORE this phase, and + /// `git_service_timeout_secs` only starts once git spawns, so without this the + /// acquire phase is unbounded: a stalled backend pins the permit and drains the + /// pool until every later request 503s. On expiry the permit is released and a + /// bounded 503 + Retry-After is returned (fail-closed). Kept separate from + /// `git_service_timeout_secs` because acquisition and git execution are distinct + /// cost centers — one shared budget would let a slow acquire starve git. Must be + /// positive; set it very large to effectively disable the bound. Default: 30s. + #[arg( + long, + env = "GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS", + default_value_t = 30, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub git_acquire_timeout_secs: u64, + /// Maximum connections in the PostgreSQL pool. This is a cap, not a floor /// (connections open lazily). Size against the database server's /// max_connections, remembering admin tooling opens its own pool. @@ -234,6 +255,154 @@ pub struct Config { value_parser = clap::value_parser!(u64).range(1..) )] pub db_retry_max_secs: u64, + + /// Maximum number of served git operations (upload-pack / receive-pack / + /// info-refs) allowed to run concurrently. Beyond this the node sheds the + /// request with a clean 503 + Retry-After instead of spawning another git + /// subprocess and risking PID/thread exhaustion. Portable backstop: the + /// compose `pids_limit` is not present on Fly, whose connection-concurrency + /// cap is a different axis (500 connections each fan out to git + + /// pack-objects + threads). Size below the process budget with headroom. + /// + /// This is the READ pool (`git_read_semaphore`): upload-pack and the UPLOAD-PACK + /// `info/refs` advertisement only. The authenticated push POST draws from a + /// separate write pool (`max_concurrent_git_pushes`) that anonymous reads can + /// never reach, and each read caller is additionally bounded by + /// `max_concurrent_reads_per_caller`, so an anonymous flood cannot shed the actual + /// push nor monopolize reads (#174). The anon-reachable RECEIVE-PACK `info/refs` + /// advertisement draws from its OWN dedicated pool (sized like the write pool but + /// disjoint), so an advertisement flood can never occupy a permit the + /// authenticated push POST needs at admission (#174). + /// + /// A permit is held for the whole op. Every git subprocess that STREAMS is + /// duration-bounded and reaps its process group on disconnect: upload-pack, + /// receive-pack, and both info/refs advertisements run under + /// `git_service_timeout_secs` with `process_group(0)` teardown, and the + /// withheld-blob (`upload_pack_excluding`) pack-objects stage plus the push-side + /// candidate-discovery children (`rev-list` / `cat-file`) now run under the same + /// bounded runner with process-group teardown, so a stuck git child no longer + /// holds its slot indefinitely (#174 closed the duration/cancellation gaps this + /// comment previously tracked). + /// + /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value + /// well under tokio's `Semaphore` permit limit so an oversized value is a + /// clean CLI error rather than a boot-time panic. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_OPS", + default_value_t = 128, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_ops: usize, + + /// Maximum number of concurrent `git-receive-pack` (push) operations. The + /// authenticated push POST draws from this dedicated pool, separate from + /// `max_concurrent_git_ops` (reads), so a flood of anonymous reads cannot shed an + /// authenticated push at admission (#174). The anon-reachable receive-pack + /// `info/refs` advertisement runs in a SEPARATE pool of the same size (derived + /// from this knob), disjoint from this one, so an advertisement flood cannot + /// occupy a POST's slot either (#174). Beyond this a push sheds a clean 503 + + /// Retry-After. + /// + /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value + /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI + /// error rather than a boot-time panic). + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_pushes: usize, + + /// Maximum concurrent read operations (`upload-pack` and the upload-pack + /// `info/refs` advertisement) a single caller may hold at once, so one caller + /// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are + /// keyed on the RESOLVED SOURCE IP, never the DID — a signature does not move a + /// caller off this cap, so an authenticated client cannot mint DIDs to escape it. + /// IMPORTANT: the source-IP key is only as granular as `GITLAWB_TRUSTED_PROXY`. + /// Left unset (the default), a node behind an edge/NAT keys all callers on the + /// edge IP, so this cap collapses to a single global cap rather than per-client. + /// Set `GITLAWB_TRUSTED_PROXY` to key on the real client; a high-fanout caller (a + /// CI fleet behind one NAT) then needs the operator to raise this. Over-cap for a + /// caller sheds a clean 503 + Retry-After. + /// + /// Default: 16. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_READS_PER_CALLER", + default_value_t = 16, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_reads_per_caller: usize, + + /// Maximum number of concurrent `GET /ipfs/{cid}` requests that may run their + /// visibility walk at once. The publicly-reachable `/ipfs/{cid}` route runs + /// `allowed_blob_set_for_caller_bounded` in `spawn_blocking` — a full-history + /// git walk (up to `git_service_timeout_secs`) — for each candidate repo. It + /// draws from THIS pool, not any served-git pool: a distinct public cost center + /// on a distinct surface, so sharing a git pool would let anonymous /ipfs + /// traffic shed authenticated git ops (the auth-boundary trap). A permit is + /// held for the whole request (across the repo loop) so it reflects real + /// blocking-thread occupancy, not merely the tokio wait. Beyond this the request + /// sheds a clean 503 + Retry-After. Must be between 1 and 1_048_576; the ceiling + /// keeps the value under tokio's `Semaphore` permit limit so an oversized value + /// is a clean CLI error rather than a boot-time panic. Default: 32. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_IPFS_WALKS", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_ipfs_walks: usize, + + /// Maximum concurrent `/ipfs/{cid}` walk requests a single source may hold at + /// once, so one source cannot monopolize `max_concurrent_ipfs_walks` (#174). + /// Callers are keyed on the RESOLVED SOURCE IP (`client_key`/`GITLAWB_TRUSTED_PROXY`), + /// never the DID — `/ipfs` accepts any `did:key` via `optional_signature` with no + /// admission step, so keying on the DID would let one host mint disposable DIDs to + /// multiply its budget. A request with no resolvable key (no trusted header, no + /// peer) is bounded by the global pool only, never this sub-cap. Over-cap sheds a + /// clean 503 + Retry-After. Must be between 1 and 1_048_576. Default: 4. + #[arg( + long, + env = "GITLAWB_IPFS_WALK_PER_SOURCE", + default_value_t = 4, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_walk_per_source: usize, + + /// Per-request ceiling on the number of legacy (NULL-provenance) repos the + /// `/ipfs/{cid}` resolver's scan fallback will PROBE (`acquire` + `git cat-file + /// -t`) before giving up. The provenance path targets one repo; the legacy scan, + /// absent this bound, fans one anonymous request out to O(repos) subprocess spawns + /// and cold-cache fetches for a CID enumerable from the public pins index (#173, + /// INV-10). A truncated scan surfaces as a retryable 503, never a false 404. Wired + /// into `AppState::ipfs_max_legacy_probes` at construction; the history-walk ceiling + /// stays constant (`MAX_HISTORY_WALKS_PER_REQUEST`) and is NOT governed by this knob + /// (a smaller value would falsely 503 a provenanced request). Must be between 1 and + /// 1_048_576. Default: 256. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_REPOS_WALKED", + default_value_t = crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST as usize, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_repos_walked: usize, + + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The + /// route is publicly reachable (`optional_signature`) and each request can drive + /// a full-history git walk, so it carries a per-IP flood brake in addition to the + /// concurrency cap above (a rate limit bounds request *rate*, the semaphore + /// bounds concurrent slow holds — different axes). Keyed on the resolved client + /// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600. + /// + /// This is the pure once-per-request ROUTE brake. The resolver's internal + /// per-probe/per-walk WORK budget is a SEPARATE bucket whose capacity is DERIVED + /// from this value (`AppState::ipfs_work_budget`), not a knob of its own; `0` here + /// disables that derived bucket too. + #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] + pub ipfs_rate_limit: usize, } impl Config { @@ -272,4 +441,261 @@ mod tests { Config::try_parse_from(["gitlawb-node", "--git-service-timeout-secs", "0"]).is_err() ); } + + #[test] + fn max_concurrent_git_ops_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_ops, + 128 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "8"]) + .max_concurrent_git_ops, + 8 + ); + // 0 permits would shed every served-git request with a 503; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "0"]).is_err()); + // Above the ceiling would panic tokio's Semaphore::new at boot (permits > + // usize::MAX >> 3); clap must reject it as a clean CLI error instead. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048577"]) + .is_err() + ); + // The ceiling itself is accepted. + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048576"]) + .max_concurrent_git_ops, + 1_048_576 + ); + } + + #[test] + fn max_concurrent_git_pushes_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_pushes, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "8"]) + .max_concurrent_git_pushes, + 8 + ); + // 0 permits would shed every push with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048576"]) + .max_concurrent_git_pushes, + 1_048_576 + ); + } + + #[test] + fn max_concurrent_ipfs_walks_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_ipfs_walks, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "4"]) + .max_concurrent_ipfs_walks, + 4 + ); + // 0 permits would shed every /ipfs walk with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048576"]) + .max_concurrent_ipfs_walks, + 1_048_576 + ); + } + + #[test] + fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() { + assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-walk-per-source", "2"]) + .ipfs_walk_per_source, + 2 + ); + // 0 would shed every /ipfs walk from a keyed source; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "1048577"]).is_err() + ); + } + + #[test] + fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked, + 256 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"]) + .ipfs_max_repos_walked, + 8 + ); + // 0 would walk no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1048577"]).is_err() + ); + } + + /// The `GITLAWB_IPFS_MAX_REPOS_WALKED` knob must actually reach the legacy-probe + /// budget it advertises (R5, KTD5): production seeds `ipfs_max_legacy_probes` from + /// this helper, so the knob is a no-op unless the helper reflects it. RED while the + /// helper returns the hardcoded `MAX_LEGACY_PROBES_PER_REQUEST` (256 regardless of + /// the knob), GREEN once it reads the knob. + #[test] + fn ipfs_max_repos_walked_wires_the_legacy_probe_budget() { + use crate::state::AppState; + // Knob set to 1 → a one-probe legacy budget. + let one = Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1"]); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&one), + 1, + "the knob must control the legacy-probe budget, not be ignored" + ); + // Unset knob preserves the shipped 256-probe behaviour. + let default = Config::parse_from(["gitlawb-node"]); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&default), + 256, + "the default knob keeps the shipped 256-probe budget" + ); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&default), + crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + "the default budget equals the constant it replaced" + ); + // Ceiling guard: the knob never governs the history-walk ceiling, which must + // stay at MAX_PIN_SOURCES + 1 or a provenanced full source set false-503s. + assert!( + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST > crate::db::MAX_PIN_SOURCES as u32, + "the history-walk ceiling is independent of the repos-walked knob" + ); + } + + /// The `/ipfs` work-budget capacity is DERIVED from the route limit (R6, KTD6), with + /// a hard floor of one full legacy search per window (the effective + /// `ipfs_max_legacy_probes`). This guards the derived default so a single + /// default-config deep search never self-throttles mid-scan and recreates the F6 + /// admit-then-429 for a legitimate caller. A `RateLimiter` sized to the derived + /// budget must admit the whole probe budget back to back. + #[test] + fn ipfs_work_budget_derives_from_route_limit_and_clears_the_probe_floor() { + use crate::state::AppState; + + // Default config: derived work budget = max(route 600, probe budget 256) = 600, + // comfortably above the 256-probe floor. + let default = Config::parse_from(["gitlawb-node"]); + let budget = AppState::ipfs_work_budget(&default); + assert_eq!(budget, 600, "default derives max(route 600, probe 256)"); + assert!( + budget >= AppState::ipfs_legacy_probe_budget(&default) as usize, + "the work budget must clear one full legacy search per window" + ); + + // Tight route limit (1): the floor lifts the work budget to the probe budget + // (256), NOT down to 1 — a single deep search still completes its full scan. + let tight = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "1"]); + assert_eq!( + AppState::ipfs_work_budget(&tight), + 256, + "a tight route limit is floored at the 256-probe budget, not clamped to 1" + ); + + // Raised probe budget lifts the floor with it (the work budget tracks the + // effective probe budget, not the constant). + let raised = Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "10", + "--ipfs-max-repos-walked", + "1000", + ]); + assert_eq!( + AppState::ipfs_work_budget(&raised), + 1000, + "the floor tracks the operator-raised legacy-probe budget" + ); + + // 0 route limit disables the derived bucket too (a 0-capacity limiter admits all). + let disabled = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "0"]); + assert_eq!( + AppState::ipfs_work_budget(&disabled), + 0, + "route limit 0 disables the derived work bucket alongside the route brake" + ); + + // Behavioral floor: a limiter sized to the derived (tight-route) budget admits + // the whole probe budget back to back for one source, then sheds the next. + let budget = AppState::ipfs_work_budget(&tight); + let limiter = + crate::rate_limit::RateLimiter::new(budget, std::time::Duration::from_secs(3600)); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + for i in 0..budget { + assert!( + limiter.check("1.2.3.4").await, + "probe {i} of one full default-config scan must be admitted (no mid-scan throttle)" + ); + } + assert!( + !limiter.check("1.2.3.4").await, + "the probe past the derived budget is shed" + ); + }); + } + + #[test] + fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_reads_per_caller, + 16 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "4"]) + .max_concurrent_reads_per_caller, + 4 + ); + // 0 would shed every read from a keyed caller; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "0"]) + .is_err() + ); + assert!(Config::try_parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048577" + ]) + .is_err()); + assert_eq!( + Config::parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048576" + ]) + .max_concurrent_reads_per_caller, + 1_048_576 + ); + } } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 5324a4b9..775e7c17 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -872,8 +872,79 @@ const MIGRATIONS: &[Migration] = &[ "CREATE UNIQUE INDEX IF NOT EXISTS idx_ref_certs_repo_ref ON ref_certificates(repo_id, ref_name)", ], }, + Migration { + version: 11, + name: "pinned_cids_cid_index", + stmts: &[ + // GET /ipfs/{cid} resolves an incoming CID -> git oid via pinned_cids.cid + // (#173); index it so the per-request lookup is not a table scan. This is + // a NEW versioned migration (not appended to the applied v1 bundle) so a + // node already past v1 actually gets the index. Non-unique on purpose: cid + // is a function of raw content, so a UNIQUE index could reject a legitimate + // record_pinned_cid insert, and colliding rows serve byte-identical content. + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_cid ON pinned_cids(cid)", + ], + }, + Migration { + version: 12, + name: "pinned_cids_repo_provenance", + stmts: &[ + // Record the repository a pin came from so GET /ipfs/{cid} resolves a + // provenanced pin straight to its ONE source repo instead of scanning every + // repo (#173, jatmn round 2 — bounds the anonymous fan-out and removes the + // updated_at-ordering false-404). NEW versioned migration (never appended to + // the applied v1 pinned_cids table) so a node past v1 gets the column. + // Nullable: pins recorded before this migration have no provenance and fall + // back to the legacy repo scan; new pins carry repo_id and resolve to one + // repo. Indexed for the resolver's oid -> repo_id lookup. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS repo_id TEXT", + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_repo_id ON pinned_cids(repo_id)", + ], + }, + Migration { + version: 13, + name: "pin_repo_sources", + stmts: &[ + // F1 (#173, jatmn round 8): a shared object (a blob/tree/commit common to + // forks and mirrors) can be pinned from more than one repo. `pinned_cids` + // keeps only the FIRST pinner's `repo_id`, so a shared object first pinned + // from a private/quarantined repo 404s by CID even when a later PUBLIC repo + // also pinned it. Record EVERY pin-path source so `GET /ipfs/{cid}` can try + // each. NEW versioned migration (never appended to an applied block, INV-7). + // Bounded per object at insert time (MAX_PIN_SOURCES) so an adversary pushing + // one object from N repos cannot make resolution O(repos) (R2, INV-10). + "CREATE TABLE IF NOT EXISTS pin_repo_sources ( + sha256_hex TEXT NOT NULL, + repo_id TEXT NOT NULL, + PRIMARY KEY (sha256_hex, repo_id) + )", + "CREATE INDEX IF NOT EXISTS idx_pin_repo_sources_sha ON pin_repo_sources(sha256_hex)", + ], + }, + Migration { + version: 14, + name: "pinned_cids_legacy_provider_cid", + stmts: &[ + // R8 (#173, jatmn round 10): the opportunistic legacy provider-CID repair + // rewrites `pinned_cids.cid` from a stored PROVIDER CID (Kubo dag-pb / + // Pinata CIDv0) to the raw-content resolver key and stashes the OLD value + // here, so the rewrite is auditable and the row's legacy origin survives. + // Distinct from `pinata_cid` on purpose: `has_pinata_cid` gates the Pinata + // pin-skip, so parking a Kubo-legacy CID there would make Pinata forever + // skip re-pinning that object. NEW versioned migration (never appended to an + // applied block, INV-7) so a node past v13 actually gets the column. + // Nullable: only a repaired row sets it. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS legacy_provider_cid TEXT", + ], + }, ]; +/// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). +/// Bounds both the resolver's per-OID source loop and the `pin_repo_sources` growth, +/// so an adversary re-pushing one object from many repos cannot make resolution +/// O(repos) (R2, INV-10). +pub const MAX_PIN_SOURCES: i64 = 16; + // ── Repos ───────────────────────────────────────────────────────────────────── pub(crate) fn normalize_owner_key(did: &str) -> &str { @@ -1046,6 +1117,22 @@ impl Db { Ok(row.map(row_to_repo)) } + /// Fetch a repo by its stable `id`. Used by the `/ipfs/{cid}` provenance path, + /// which resolves a pin straight to its ONE source repo (#173) instead of + /// scanning `list_all_repos`. `id` is exact, so unlike `get_repo`'s fuzzy + /// owner/name match there is no mirror-vs-canonical disambiguation. + pub async fn get_repo_by_id(&self, id: &str) -> Result> { + let row = sqlx::query( + "SELECT id, name, owner_did, description, is_public, default_branch, + created_at, updated_at, disk_path, forked_from, machine_id + FROM repos WHERE id = $1 LIMIT 1", + ) + .bind(id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(row_to_repo)) + } + #[allow(dead_code)] pub async fn list_repos(&self, owner_did: &str) -> Result> { let rows = sqlx::query( @@ -2166,20 +2253,208 @@ impl Db { Ok(row.get::("cnt") > 0) } - pub async fn record_pinned_cid(&self, sha256_hex: &str, cid: &str) -> Result<()> { + /// Every git oid a pinned CID maps to (`pinned_cids.cid` -> `sha256_hex`). + /// `GET /ipfs/{cid}` resolves the content-addressed CID a client sends back to + /// the object's git oid this way: a real pin CID digests the raw object + /// content, not the git oid, so the digest cannot be `git cat-file`d directly + /// (#173). The index is unique on the git oid but NON-unique on cid, so two + /// distinct oids can share one content-CID (a tree and a blob whose raw bytes + /// collide, or byte-identical content pinned under two oids). Returning every + /// candidate lets the handler try each rather than pick one arbitrarily and + /// false-404 when the chosen one is withheld or absent while another is + /// readable (#173). Empty when the CID was never pinned on this node. + pub async fn oids_for_cid(&self, cid: &str) -> Result> { + let rows = sqlx::query("SELECT sha256_hex FROM pinned_cids WHERE cid = $1") + .bind(cid) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("sha256_hex")) + .collect()) + } + + /// Record a pinned object's CID and the repository it was pinned from + /// (`repo_id`, #173). On conflict the `COALESCE` backfills a NULL provenance + /// from a known source while keeping first-pinner-owns: an existing non-NULL + /// `repo_id` is never rewritten by a later push of the same oid, but a legacy + /// pin (or a pin recorded before provenance existed) whose `repo_id` is NULL + /// gets it filled the next time the object is re-pinned with a known source. + /// `cid`/`pinned_at` are left untouched on conflict. `repo_id` is `None` only + /// for a legacy pin with no known source; those fall back to the resolver's scan. + pub async fn record_pinned_cid( + &self, + sha256_hex: &str, + cid: &str, + repo_id: Option<&str>, + ) -> Result<()> { sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) - VALUES ($1, $2, $3) - ON CONFLICT(sha256_hex) DO NOTHING", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT(sha256_hex) DO UPDATE SET + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) .bind(cid) .bind(Utc::now().to_rfc3339()) + .bind(repo_id) .execute(&self.pool) .await?; Ok(()) } + /// The resolver key currently stored for a pinned object (`pinned_cids.cid`), + /// or `None` for an unpinned oid. The opportunistic legacy-repair path reads + /// it to decide candidacy from the codec of the string alone (no object bytes) + /// before it recomputes anything. + pub async fn cid_for_oid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get::("cid"))) + } + + /// Rewrite a legacy provider-CID row to the raw-content resolver key, stashing + /// the old provider value in `legacy_provider_cid` (#173 R8, KTD8). Before this + /// branch the pin path stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in + /// `cid`; the `/ipfs` resolver recomputes the raw CID and 404s a mismatched key + /// even though `list_pinned_cids` still advertises it. The `WHERE cid = + /// $old_provider_cid` guard makes a concurrent double-repair a no-op (the second + /// writer sees the already-rewritten key and matches nothing) and never touches + /// a row keyed on a different value. Stashed in `legacy_provider_cid`, NOT + /// `pinata_cid`: the latter gates the Pinata pin-skip (`has_pinata_cid`), so a + /// Kubo-legacy CID parked there would make Pinata permanently skip the object. + pub async fn repair_legacy_provider_cid( + &self, + sha256_hex: &str, + raw_cid: &str, + old_provider_cid: &str, + ) -> Result<()> { + sqlx::query( + "UPDATE pinned_cids + SET cid = $2, legacy_provider_cid = $3 + WHERE sha256_hex = $1 AND cid = $3", + ) + .bind(sha256_hex) + .bind(raw_cid) + .bind(old_provider_cid) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// The repository a pinned object was recorded from (`pinned_cids.repo_id`), + /// or `None` for a legacy pin (recorded before provenance existed) or an + /// unpinned oid. `GET /ipfs/{cid}` uses this to gate+serve the ONE source + /// repo instead of scanning every repo (#173). + pub async fn provenance_for_oid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT repo_id FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.and_then(|r| r.get::, _>("repo_id"))) + } + + /// Backfill the source repo on an already-pinned object whose provenance is + /// NULL (a legacy pin recorded before provenance existed, #173, jatmn). The + /// `AND repo_id IS NULL` guard keeps first-pinner-owns: an existing non-NULL + /// provenance is left untouched. Touches only `repo_id` and never re-pins the + /// object's bytes, so it is safe to call on the already-pinned skip path. + pub async fn backfill_pin_provenance(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + sqlx::query( + "UPDATE pinned_cids SET repo_id = $2 WHERE sha256_hex = $1 AND repo_id IS NULL", + ) + .bind(sha256_hex) + .bind(repo_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Record a repository as a source for a pinned object (F1, #173 jatmn round 8), + /// bounded to about `MAX_PIN_SOURCES` distinct repos per object. The count guard + /// lives inside the INSERT (a single statement), which suppresses a re-push of the + /// SAME `(oid, repo)` via `ON CONFLICT DO NOTHING`. It does NOT hard-serialize + /// concurrent inserts of DIFFERENT repos for the same object: under Postgres READ + /// COMMITTED each concurrent writer's count subquery reads a snapshot that omits the + /// others' uncommitted rows, so N concurrent pushers can each see `count < cap` and + /// overshoot by up to N-1 rows. The overshoot is a small constant (bounded by + /// concurrent-pusher count, never O(repos)), and the RESOLVER read side + /// (`pin_sources_for_oid`) caps the ADDITIONAL sources at `MAX_PIN_SOURCES` (always + /// keeping the first-pinner), so the INV-10 bound on serve-time work holds at + /// `O(MAX_PIN_SOURCES + 1)` regardless of a table overshoot. + pub async fn record_pin_source(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Every source repository recorded for a pinned object (F1, #173 jatmn round 8): + /// the union of the first-pinner `pinned_cids.repo_id` and the `pin_repo_sources` + /// rows, deduped and ordered for a deterministic resolver walk. + /// + /// The first-pinner (a single row by `pinned_cids`' PK on `sha256_hex`) is ALWAYS + /// included; the `LIMIT MAX_PIN_SOURCES` caps only the ADDITIONAL `pin_repo_sources` + /// rows. This keeps the resolver's per-source work a bounded `O(MAX_PIN_SOURCES + 1)` + /// ceiling (INV-10) while never letting the cap evict the original source. A prior + /// version applied the `LIMIT` to the whole UNION with a lexicographic `ORDER BY`, + /// which let an attacker 404 a legacy public CID (first-pinner in `pinned_cids` but + /// not yet in `pin_repo_sources`) by pushing the same object from `MAX_PIN_SOURCES` + /// repos whose grindable ids sort before it, evicting the public source from the + /// window. Empty for a legacy pin with no known source (it falls back to the repo + /// scan) or an unpinned oid. + pub async fn pin_sources_for_oid(&self, sha256_hex: &str) -> Result> { + let rows = sqlx::query( + "SELECT repo_id FROM pinned_cids + WHERE sha256_hex = $1 AND repo_id IS NOT NULL + UNION + SELECT repo_id FROM ( + SELECT repo_id FROM pin_repo_sources + WHERE sha256_hex = $1 + ORDER BY repo_id + LIMIT $2 + ) capped + ORDER BY repo_id", + ) + .bind(sha256_hex) + .bind(MAX_PIN_SOURCES) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("repo_id")) + .collect()) + } + + /// Whether `pin_repo_sources` is at the `MAX_PIN_SOURCES` cap for this oid, i.e. + /// the provenance source set returned by [`Self::pin_sources_for_oid`] may be + /// INCOMPLETE. `record_pin_source` stops inserting at exactly `MAX_PIN_SOURCES` + /// rows and drops later sources silently, so a full table is the only observable + /// signal that a servable source (e.g. a later public pinner) may have been + /// dropped. `get_by_cid` uses this to decide whether a provenance miss should fall + /// back to the bounded legacy scan (which gates every repo through the real + /// visibility gate and so finds a dropped public source) rather than 404 — closing + /// the pin-source griefing hole where 16 attacker sources bury a public one. `>=` + /// (not `==`) is defensive against any future overshoot. + pub async fn pin_sources_at_cap(&self, sha256_hex: &str) -> Result { + let count: i64 = + sqlx::query_scalar("SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_one(&self.pool) + .await?; + Ok(count >= MAX_PIN_SOURCES) + } + pub async fn record_encrypted_blob( &self, repo_id: &str, @@ -2279,18 +2554,34 @@ impl Db { } /// Record the Pinata CID for a git object. - /// Inserts the row if it doesn't exist (objects pinned directly to Pinata - /// without a prior local IPFS pin get cid = pinata_cid). - pub async fn record_pinata_cid(&self, sha256_hex: &str, pinata_cid: &str) -> Result<()> { + /// + /// `raw_cid` is the locally-computed raw-content CID (`Cid::from_git_object_bytes`, + /// CIDv1/raw/sha2-256), the resolver key `GET /ipfs/{cid}` looks up; `pinata_cid` + /// is the provider CID Pinata returned (a dag-pb/UnixFS CID for gateway retrieval). + /// Inserts the row if it doesn't exist (an object pinned directly to Pinata with + /// no prior local IPFS pin gets `cid = raw_cid`, never the provider CID — a dag-pb + /// provider CID must never become an alias that serves raw bytes that do not hash + /// to it, #173). On conflict `cid` is left untouched: a prior local pin already + /// stored the correct raw CID, and the COALESCE backfills a NULL provenance from a + /// known source while keeping first-pinner-owns. + pub async fn record_pinata_cid( + &self, + sha256_hex: &str, + raw_cid: &str, + pinata_cid: &str, + repo_id: Option<&str>, + ) -> Result<()> { sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) - VALUES ($1, $2, $3, $4) - ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid, repo_id) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid, + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) - .bind(pinata_cid) // fallback local cid if row is new + .bind(raw_cid) // resolver-key cid: locally-computed raw CID, never the provider CID .bind(Utc::now().to_rfc3339()) .bind(pinata_cid) + .bind(repo_id) .execute(&self.pool) .await?; Ok(()) @@ -5037,6 +5328,51 @@ mod ref_certificate_tests { ); } + /// INV-7: upgrade-path test — an existing node already past v1 must still get + /// the `pinned_cids.cid` index. It ships as its OWN v11 migration (not appended + /// to the applied v1 bundle), so dropping the index + its `schema_migrations` + /// row and re-running migrations must recreate it, exercising the real code + /// path rather than hand-copying the SQL. + #[sqlx::test] + async fn v11_pinned_cids_cid_index_applies_on_upgrade(pool: PgPool) { + async fn index_exists(pool: &PgPool) -> bool { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_indexes WHERE indexname = 'idx_pinned_cids_cid')", + ) + .fetch_one(pool) + .await + .unwrap() + } + + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "fresh migration chain creates the index" + ); + + // Simulate a node at v10 (pre-v11): drop the index and its migration record. + sqlx::query("DROP INDEX IF EXISTS idx_pinned_cids_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 11") + .execute(&pool) + .await + .unwrap(); + assert!( + !index_exists(&pool).await, + "precondition: index and its migration record removed" + ); + + // Re-run migrations: v11 re-applies and recreates the index on the upgrade. + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "v11 must recreate idx_pinned_cids_cid on an upgrading node" + ); + } + /// INV-7: upgrade-path test — seed a database at v9 with duplicate /// ref_certificates, then let the real v10 migration fire via /// run_migrations(). This exercises the migration code path rather than diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index bdd5aec9..c13b92b4 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -41,12 +41,18 @@ pub enum AppError { #[error("incomplete: {0}")] Incomplete(String), + #[error("search incomplete: {0}")] + SearchIncomplete(String), + #[error("git error: {0}")] Git(String), #[error("git service timed out: {0}")] Timeout(String), + #[error("server overloaded: {0}")] + Overloaded(String), + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -138,6 +144,15 @@ impl IntoResponse for AppError { AppError::Incomplete(msg) => { (StatusCode::UNPROCESSABLE_ENTITY, "incomplete", msg.clone()) } + // A bounded search that could not complete (the CID resolver hit its + // legacy-probe or walk ceiling), distinct from the 404 that asserts a + // definitive not-found: absence was NOT proven, so the caller should + // retry rather than treat it as gone (#173, F2). 503, retryable. + AppError::SearchIncomplete(msg) => ( + StatusCode::SERVICE_UNAVAILABLE, + "search_incomplete", + msg.clone(), + ), AppError::Git(msg) => (StatusCode::INTERNAL_SERVER_ERROR, "git_error", msg.clone()), // 504, distinct from the 500 git_error and from the read-gate's 404 / // the auth 401, so the client can tell a deadline from a failure. @@ -147,6 +162,12 @@ impl IntoResponse for AppError { DB_UNAVAILABLE_CODE, DB_UNAVAILABLE_MESSAGE.into(), ), + // 503 with a Retry-After (attached after this match — the shared tail + // can't carry per-variant headers). This is the single place Overloaded + // becomes a response, so it can never ship a 503 without the retry hint. + AppError::Overloaded(msg) => { + (StatusCode::SERVICE_UNAVAILABLE, "overloaded", msg.clone()) + } AppError::Db(e) => (StatusCode::INTERNAL_SERVER_ERROR, "db_error", e.to_string()), AppError::Internal(e) => ( StatusCode::INTERNAL_SERVER_ERROR, @@ -160,7 +181,21 @@ impl IntoResponse for AppError { "message": message, })); - (status, body).into_response() + let mut resp = (status, body).into_response(); + // Both retryable 503s advertise when to retry: Overloaded (capacity shed) and + // SearchIncomplete (a bounded CID search cut short by a cap — retry may complete + // it). They ride the shared tail above for body/status, so the header is attached + // here rather than in bespoke early returns, keeping each variant handled once. + if matches!( + self, + AppError::Overloaded(_) | AppError::SearchIncomplete(_) + ) { + resp.headers_mut().insert( + axum::http::header::RETRY_AFTER, + axum::http::HeaderValue::from_static("1"), + ); + } + resp } } @@ -182,4 +217,14 @@ mod tests { StatusCode::INTERNAL_SERVER_ERROR ); } + + #[test] + fn overloaded_maps_to_503_with_retry_after() { + let resp = AppError::Overloaded("x".into()).into_response(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1" + ); + } } diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 7ab00816..962c9c65 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -29,9 +29,9 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::time::{Duration, Instant}; -use anyhow::{bail, Context, Result}; +use anyhow::Result; /// Env var that forces the push path to always full-scan, bypassing the delta /// optimization (KTD7 kill-switch). Reuses the already-tested fallback branch @@ -64,11 +64,21 @@ pub enum PinCandidates { /// Fail-closed: any condition where the introduced set cannot be safely /// determined returns [`PinCandidates::FullScanRequired`] rather than a partial /// set, so the caller full-scans instead of silently under-pinning. -pub fn resolve_push_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> PinCandidates { - // KTD7 kill-switch: force the (already-tested) full-scan fallback. The env - // read is split out from the pure logic so the resolver stays unit-testable - // without touching process-global state. - resolve_push_delta_inner(repo_path, new_tips, old_tips, force_full_scan()) +pub fn resolve_push_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> PinCandidates { + resolve_push_delta_inner( + repo_path, + new_tips, + old_tips, + force_full_scan(), + git_bin, + deadline, + ) } /// Whether the force-full-scan kill-switch env var is set. @@ -83,6 +93,8 @@ fn resolve_push_delta_inner( new_tips: &[&str], old_tips: &[&str], force_full_scan: bool, + git_bin: &str, + deadline: Instant, ) -> PinCandidates { if force_full_scan { tracing::debug!("{FORCE_FULL_SCAN_ENV} set — forcing full scan"); @@ -102,7 +114,7 @@ fn resolve_push_delta_inner( // commit). Bare `cat-file -t` returns `tag` for an annotated tag, and // `for-each-ref %(*objecttype)` peels only one level — neither is correct. for tip in new_tips { - match peeled_object_type(repo_path, tip) { + match peeled_object_type(repo_path, tip, git_bin, deadline) { Some(t) if t == "commit" => {} other => { tracing::debug!( @@ -115,7 +127,7 @@ fn resolve_push_delta_inner( } } - match rev_list_delta(repo_path, new_tips, old_tips) { + match rev_list_delta(repo_path, new_tips, old_tips, git_bin, deadline) { Ok(oids) => PinCandidates::Delta(oids), Err(e) => { tracing::debug!(err = %e, "push-delta rev-list failed — forcing full scan"); @@ -126,41 +138,43 @@ fn resolve_push_delta_inner( /// Return the fully-peeled object type of `sha` (e.g. `commit`, `tree`, /// `blob`), or `None` if the object is missing/unpeelable or git errored. -fn peeled_object_type(repo_path: &Path, sha: &str) -> Option { - let output = Command::new("git") - .args(["cat-file", "-t", &format!("{sha}^{{}}")]) - .current_dir(repo_path) - .output() - .ok()?; - if !output.status.success() { - return None; - } - Some(String::from_utf8_lossy(&output.stdout).trim().to_string()) +fn peeled_object_type( + repo_path: &Path, + sha: &str, + git_bin: &str, + deadline: Instant, +) -> Option { + let peel = format!("{sha}^{{}}"); + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-t", &peel], + repo_path, + b"", + deadline, + ) + .ok()?; + Some(String::from_utf8_lossy(&out).trim().to_string()) } /// Run `git rev-list --objects --no-object-names --not ` and return /// the bare OID set. Decides on `status.success()` *before* parsing stdout, so /// a walk that prints a valid prefix then errors mid-walk is discarded. -fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Result> { +fn rev_list_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> Result> { let mut args: Vec<&str> = vec!["rev-list", "--objects", "--no-object-names"]; args.extend_from_slice(new_tips); if !old_tips.is_empty() { args.push("--not"); args.extend_from_slice(old_tips); } - - let output = Command::new("git") - .args(&args) - .current_dir(repo_path) - .output() - .context("failed to run git rev-list for push delta")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git rev-list failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + let out = + crate::git::visibility_pack::run_bounded_git(git_bin, &args, repo_path, b"", deadline)?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -175,23 +189,19 @@ fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Res /// reconciliation sweep relies on. It returns *all* objects (including /// unreachable/dangling ones), which is what the sweep needs to catch /// stragglers — do not swap it for a reachability walk. -pub fn list_all_objects(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -203,23 +213,23 @@ pub fn list_all_objects(repo_path: &Path) -> Result> { /// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed /// filter needs to tell blobs (content, withholdable) from commits/trees /// (structural, never withheld) without typing the candidate list itself. -pub fn list_all_objects_with_type(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects_with_type( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname) %(objecttype)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .filter_map(|l| { @@ -235,8 +245,12 @@ pub fn list_all_objects_with_type(repo_path: &Path) -> Result Result> { - Ok(list_all_objects_with_type(repo_path)? +pub fn all_blob_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + Ok(list_all_objects_with_type(repo_path, git_bin, deadline)? .into_iter() .filter(|(_, ty)| ty == "blob") .map(|(oid, _)| oid) @@ -272,18 +286,22 @@ pub async fn resolve_candidates_for_push( repo_path: PathBuf, new_tips: Vec, old_tips: Vec, + git_bin: String, + timeout: Duration, ) -> PinCandidateSet { tokio::task::spawn_blocking(move || { + // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). + let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); let old_refs: Vec<&str> = old_tips.iter().map(String::as_str).collect(); - match resolve_push_delta(&repo_path, &new_refs, &old_refs) { + match resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) { PinCandidates::Delta(objs) => { tracing::info!(delta = objs.len(), repo = %repo_path.display(), "pin candidate set from push delta"); PinCandidateSet { candidates: objs, full_scan: false } } PinCandidates::FullScanRequired => { tracing::warn!(repo = %repo_path.display(), "pin delta unavailable (non-commit tip, git error, or force-full-scan) — full-scan fallback"); - match list_all_objects(&repo_path) { + match list_all_objects(&repo_path, &git_bin, deadline) { Ok(objs) => PinCandidateSet { candidates: objs, full_scan: true }, Err(e) => { tracing::warn!(repo = %repo_path.display(), err = %e, "full-scan fallback failed; pinning nothing this push (reconciliation sweep backstops)"); @@ -304,8 +322,13 @@ pub async fn resolve_candidates_for_push( mod tests { use super::*; use std::collections::HashSet; + use std::process::Command; use tempfile::TempDir; + fn td() -> std::time::Instant { + std::time::Instant::now() + std::time::Duration::from_secs(600) + } + /// Minimal git helper for building test repos. struct Repo { _td: TempDir, @@ -363,9 +386,10 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c2], &[&c1])) - .into_iter() - .collect(); + let got: HashSet = + delta(resolve_push_delta(&repo.path, &[&c2], &[&c1], "git", td())) + .into_iter() + .collect(); // The new blob b.txt and commit c2 are in the delta; the old blob a.txt // and commit c1 are not. let new_blob = repo.rev("HEAD:b.txt"); @@ -383,7 +407,7 @@ mod tests { // genuinely new objects, never fewer. let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[], "git", td())) .into_iter() .collect(); assert!(got.contains(&c1)); @@ -399,9 +423,15 @@ mod tests { // Rewrite history: reset to base, commit a different file. repo.git(&["reset", "-q", "--hard", &base]); let new_tip = repo.commit_file("c.txt", "three\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&new_tip], &[&old_tip])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&new_tip], + &[&old_tip], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&new_tip), "new tip in delta"); assert!(got.contains(&repo.rev(&format!("{new_tip}:c.txt")))); // No error; force-push computes new-minus-old cleanly. @@ -413,7 +443,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); // All updates were deletions => new_tips empty after the caller strips zeros. assert_eq!( - resolve_push_delta(&repo.path, &[], &[ZERO]), + resolve_push_delta(&repo.path, &[], &[ZERO], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -424,7 +454,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); assert_eq!( - resolve_push_delta(&repo.path, &[&blob], &[]), + resolve_push_delta(&repo.path, &[&blob], &[], "git", td()), PinCandidates::FullScanRequired, "a blob tip must force full scan (rev-list would exit 0 and walk it)" ); @@ -436,7 +466,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let tree = repo.rev("HEAD^{tree}"); assert_eq!( - resolve_push_delta(&repo.path, &[&tree], &[]), + resolve_push_delta(&repo.path, &[&tree], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -449,7 +479,7 @@ mod tests { repo.git(&["tag", "-a", "treetag", "-m", "x", &tree]); let tag = repo.rev("treetag"); assert_eq!( - resolve_push_delta(&repo.path, &[&tag], &[]), + resolve_push_delta(&repo.path, &[&tag], &[], "git", td()), PinCandidates::FullScanRequired, "annotated tag peeling to a tree must force full scan" ); @@ -465,7 +495,7 @@ mod tests { repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); assert_eq!( - resolve_push_delta(&repo.path, &[&t2], &[]), + resolve_push_delta(&repo.path, &[&t2], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -480,7 +510,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); repo.git(&["tag", "-a", "rel", "-m", "release", &c1]); let tag = repo.rev("rel"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[], "git", td())) .into_iter() .collect(); assert!( @@ -501,7 +531,7 @@ mod tests { let t1 = repo.rev("t1"); repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[], "git", td())) .into_iter() .collect(); assert!( @@ -516,9 +546,14 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let set = - resolve_candidates_for_push(repo.path.clone(), vec![c2.clone()], vec![c1.clone()]) - .await; + let set = resolve_candidates_for_push( + repo.path.clone(), + vec![c2.clone()], + vec![c1.clone()], + "git".to_string(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!set.full_scan, "happy-path delta is not a full scan"); let got: HashSet = set.candidates.into_iter().collect(); let new_blob = repo.rev("HEAD:b.txt"); @@ -536,8 +571,18 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); - let all: HashSet = list_all_objects(&repo.path).unwrap().into_iter().collect(); - let set = resolve_candidates_for_push(repo.path.clone(), vec![blob], vec![]).await; + let all: HashSet = list_all_objects(&repo.path, "git", td()) + .unwrap() + .into_iter() + .collect(); + let set = resolve_candidates_for_push( + repo.path.clone(), + vec![blob], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + ) + .await; assert!(set.full_scan, "non-commit tip is signalled as a full scan"); let got: HashSet = set.candidates.into_iter().collect(); assert_eq!(got, all, "non-commit tip falls back to full repo scan"); @@ -549,7 +594,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let bogus = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; assert_eq!( - resolve_push_delta(&repo.path, &[bogus], &[]), + resolve_push_delta(&repo.path, &[bogus], &[], "git", td()), PinCandidates::FullScanRequired, "a missing/corrupt tip OID must force full scan" ); @@ -564,7 +609,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); let old_tree = repo.rev(&format!("{c1}^{{tree}}")); - let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree]); + let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree], "git", td()); match result { PinCandidates::FullScanRequired => {} // safe: caller full-scans PinCandidates::Delta(objs) => { @@ -583,10 +628,15 @@ mod tests { // branch2 from base advances independently repo.git(&["checkout", "-q", "-b", "branch2", &base]); let b2 = repo.commit_file("c.txt", "three\n"); - let got: HashSet = - delta(resolve_push_delta(&repo.path, &[&b1, &b2], &[&base, &base])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&b1, &b2], + &[&base, &base], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&b1), "branch1 new commit"); assert!(got.contains(&b2), "branch2 new commit"); } @@ -595,7 +645,7 @@ mod tests { fn empty_repo_no_tips_yields_empty_delta() { let repo = Repo::new(); assert_eq!( - resolve_push_delta(&repo.path, &[], &[]), + resolve_push_delta(&repo.path, &[], &[], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -609,12 +659,12 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); assert_eq!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], true), + resolve_push_delta_inner(&repo.path, &[&c1], &[], true, "git", td()), PinCandidates::FullScanRequired ); // And with the flag off, the same push yields a Delta. assert!(matches!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], false), + resolve_push_delta_inner(&repo.path, &[&c1], &[], false, "git", td()), PinCandidates::Delta(_) )); } @@ -624,7 +674,7 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); repo.commit_file("b.txt", "two\n"); - let all = list_all_objects(&repo.path).unwrap(); + let all = list_all_objects(&repo.path, "git", td()).unwrap(); // 2 commits + 2 trees + 2 blobs = 6 objects. assert_eq!(all.len(), 6, "got: {all:?}"); } @@ -642,7 +692,7 @@ mod tests { std::fs::write(repo.path.join("orphan.bin"), b"dangling\n").unwrap(); let dangling = repo.git(&["hash-object", "-w", "orphan.bin"]); - let blobs = all_blob_oids(&repo.path).unwrap(); + let blobs = all_blob_oids(&repo.path, "git", td()).unwrap(); assert!(blobs.contains(&reachable_blob), "reachable blob present"); assert!( blobs.contains(&dangling), @@ -652,7 +702,7 @@ mod tests { assert!(!blobs.contains(&tree), "tree is not a blob"); // The typed lister tags each object; spot-check the dangling blob's type. - let typed = list_all_objects_with_type(&repo.path).unwrap(); + let typed = list_all_objects_with_type(&repo.path, "git", td()).unwrap(); assert!( typed .iter() @@ -660,4 +710,96 @@ mod tests { "dangling object is typed as a blob" ); } + + #[cfg(unix)] + #[test] + fn list_all_objects_times_out_on_a_hung_git_instead_of_blocking() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git: hang on cat-file (bounded to 30s so a broken test can't wedge). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + + let (tx, rx) = std::sync::mpsc::channel(); + let path = dir.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(list_all_objects( + &path, + &git_bin, + Instant::now() + Duration::from_millis(150), + )); + }); + let res = rx.recv_timeout(Duration::from_secs(10)).expect( + "list_all_objects must return within the watchdog budget, not block on a hung git", + ); + assert!( + res.is_err(), + "a hung git must make list_all_objects error out, not hang" + ); + } + + /// #174 finding 2: the DELTA-path children — `peeled_object_type` (cat-file), + /// `rev_list_delta` (rev-list) — and `list_all_objects_with_type` (cat-file) are + /// the remaining candidate-discovery execs (the common push path). Each must + /// return within the watchdog budget on a hung git rather than block and pin the + /// write permit. Revert any one to a bare `Command::output()` and its arm blocks + /// past the recv budget (RED). + #[cfg(unix)] + #[test] + fn delta_path_exec_fns_time_out_on_a_hung_git() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git hangs on BOTH cat-file and rev-list (bounded 30s so a broken + // test can't leak a permanent orphan or wedge the suite). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file|rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + let path = dir.path().to_path_buf(); + + // Run `f` on a thread and require it to RETURN (not block) within 10s. + fn returns_within(f: impl FnOnce() -> T + Send + 'static) -> T { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(Duration::from_secs(10)).expect( + "a delta-path exec fn must return within the watchdog budget, not block on a hung git", + ) + } + let dl = || Instant::now() + Duration::from_millis(150); + let sha = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || peeled_object_type(&p, sha, &g, d)).is_none(), + "peeled_object_type must time out to None on a hung cat-file" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || rev_list_delta(&p, &[sha], &[], &g, d)).is_err(), + "rev_list_delta must error out on a hung rev-list" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || list_all_objects_with_type(&p, &g, d)).is_err(), + "list_all_objects_with_type must error out on a hung cat-file" + ); + } } diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index eeb35b99..50145652 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -6,29 +6,74 @@ use bytes::Bytes; use std::collections::HashSet; use std::path::Path; use std::process::Stdio; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; +/// Owns the served-git admission permits (global + per-source) for the lifetime of +/// the work they admitted, so admission is released only when that work is truly +/// done — not the instant the handler future drops on a client disconnect. +/// +/// A move-only wrapper: no methods beyond construction and `Drop`. The handler +/// MOVEs its permits in and keeps no copy (a retained copy would drop early and +/// release admission the moment the future is dropped, defeating the guard). It is +/// threaded into `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the +/// detached reaper on disconnect, so both permits drop only after the process group +/// is confirmed reaped (`kill(-pgid,0)==ESRCH`) rather than while the group is still +/// alive holding PIDs past the concurrency cap (#174 P1-a, plain-spawn residual). +/// +/// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by +/// moving its permits into the `spawn_blocking`; this generalizes it to the plain +/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths. +pub struct AdmissionGuard { + // Boxed so the concrete permit types stay private to the caller (repos.rs) — the + // guard only needs to hold them and drop them, never inspect them. `Send + + // 'static` so the guard can move into the detached reaper task. + _global: Option>, + _caller: Option>, +} + +impl AdmissionGuard { + /// Take ownership of the global permit and an optional per-caller permit. Both are + /// erased to `Box` — the guard's only job is to hold them until it drops. + pub fn new(global: impl Send + 'static, caller: Option) -> Self { + Self { + _global: Some(Box::new(global)), + _caller: caller.map(|c| Box::new(c) as Box), + } + } +} + /// Handle `GET /:owner/:repo/info/refs?service=git-upload-pack` /// or `?service=git-receive-pack` /// /// This is the ref advertisement — the first step of a clone or push. -pub async fn info_refs(repo_path: &Path, service: &str) -> Result { +/// +/// `git_bin` is injectable purely so the process-group teardown can be driven by a +/// fake `git` in tests (production passes `"git"`). `timeout` bounds the whole child +/// interaction: previously the advertisement ran a bare `Command::output()` with no +/// deadline and no teardown, so a hung git pinned its concurrency slot indefinitely +/// and a client disconnect orphaned the child (#174). It now shares +/// [`drive_git_child`]'s timeout + `process_group(0)` + [`KillGroupOnDrop`] teardown. +pub async fn info_refs( + git_bin: &str, + service: &str, + repo_path: &Path, + timeout: Duration, + admission: Option, +) -> Result { validate_service(service)?; - let output = Command::new("git") + let mut command = Command::new(git_bin); + command .arg(service_to_command(service)) .arg("--stateless-rpc") .arg("--advertise-refs") - .arg(repo_path) - .output() - .await?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git {service} --advertise-refs failed: {stderr}"); - } + .arg(repo_path); + // No request body — advertise-refs does not read stdin. Single stage: the returned + // guard is dropped here (advertise-refs holds admission for its own child only). + let (stdout, _admission) = + drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; let content_type = format!("application/x-{service}-advertisement"); @@ -38,7 +83,7 @@ pub async fn info_refs(repo_path: &Path, service: &str) -> Result { let mut body = Vec::new(); body.extend_from_slice(&pkt_service); body.extend_from_slice(flush); - body.extend_from_slice(&output.stdout); + body.extend_from_slice(&stdout); Ok(Response::builder() .status(StatusCode::OK) @@ -53,12 +98,21 @@ pub async fn info_refs(repo_path: &Path, service: &str) -> Result { /// Serves pack data for a clone or fetch. This is stateless — the entire /// negotiation happens in a single request/response. pub async fn upload_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-upload-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-upload-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -72,12 +126,21 @@ pub async fn upload_pack( /// Accepts a push. The caller MUST verify HTTP Signature auth before /// calling this function. pub async fn receive_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-receive-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-receive-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -100,24 +163,84 @@ pub async fn receive_pack( /// `wait_with_output()` returns, so a request that completed cleanly never signals. #[cfg(unix)] struct KillGroupOnDrop { + // Holds the child + its pgid while armed. The interaction drives the child through + // `child_mut()`; the success/timeout paths call `disarm` once they have reaped it. + // On drop — a client disconnect that drops the whole request future — the guard + // launches a detached reaper that OWNS the child and runs the full + // SIGTERM -> grace -> SIGKILL -> reap sequence (`reap_group_on_timeout`), as strong + // as the timeout path, so a SIGTERM-ignoring group member is SIGKILLed and reaped + // rather than left running until EPIPE to accumulate past the concurrency cap + // (#174 P1-c). Owning the tokio `Child` in the reaper (rather than a raw + // `waitpid(-pgid)`) keeps a single reaper of the leader, so it never races tokio's + // own SIGCHLD-driven orphan reaper. + child: Option, pgid: Option, + // The global + per-source admission permits for this op, if any. On the plain + // (non-path-scoped) spawn paths repos.rs moves its permits in here so admission is + // released only when the group is confirmed reaped, on every exit — complete, + // timeout, or client-disconnect (#174 P1-a). The rev-list/pack-objects + // visibility-walk callers hold no admission and pass `None`. + admission: Option, } #[cfg(unix)] impl KillGroupOnDrop { - fn disarm(&mut self) { + /// The child, while armed. The interaction drives stdin/stdout/`wait` through this. + fn child_mut(&mut self) -> &mut tokio::process::Child { + self.child.as_mut().expect("child present while armed") + } + + /// Disarm on the success/timeout path: the body has already reaped the child (its + /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear + /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard so the + /// caller drops it at the earliest provably-free point (the group is already reaped + /// on both callers of this) rather than at end-of-bookkeeping. + fn disarm(&mut self) -> Option { self.pgid = None; + self.child = None; + self.admission.take() } } #[cfg(unix)] impl Drop for KillGroupOnDrop { fn drop(&mut self) { - if let Some(pgid) = self.pgid { - // SAFETY: kill(2) takes only integer arguments and borrows no Rust - // memory. Signalling a stale group just returns ESRCH, which we ignore. - unsafe { - libc::kill(-pgid, libc::SIGTERM); + // Move any admission guard out first so it travels with the reaper (or drops + // here on the no-child / no-runtime fallback), never before the group is gone. + let admission = self.admission.take(); + let (Some(mut child), Some(pgid)) = (self.child.take(), self.pgid) else { + // Nothing to reap (already disarmed); dropping `admission` here is correct — + // the group is already gone. + return; + }; + // A sync Drop cannot await, so launch a detached reaper that owns the child and + // runs the same teardown as the timeout path (TERM -> grace -> SIGKILL -> reap + // the whole group). Owning the tokio `Child` means this task is the sole reaper + // of the leader, so it cannot race tokio's orphan reaper. Prefer the current + // runtime handle; if dropped outside a runtime, fall back to a best-effort + // synchronous SIGTERM so the group is at least signalled. + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + handle.spawn(async move { + reap_group_on_timeout(&mut child).await; + // Release admission only now: the group is ESRCH-confirmed gone (or + // hit the ~4s D-state hard cap, past which nothing in userspace can + // free the PIDs anyway). Holding the permits until here is what + // stops disconnect-spam from admitting replacements while the prior + // group is still alive (#174 P1-a). `admission` is moved in and + // dropped when this closure ends. + drop(admission); + }); + } + Err(_) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { + libc::kill(-pgid, libc::SIGTERM); + } + // No runtime to await the reap; drop admission best-effort after the + // synchronous signal (the fallback path is not on the hot request path). + drop(admission); } } } @@ -202,12 +325,42 @@ async fn run_git_service( repo_path: &Path, input: Bytes, timeout: Duration, + admission: Option, ) -> Result> { let mut command = Command::new(git_bin); command .arg(service_to_command(service)) .arg("--stateless-rpc") - .arg(repo_path) + .arg(repo_path); + // Single stage: drop the returned guard when this function returns (its permits are + // released once the child's group is reaped, per drive_git_child). + let (out, _admission) = drive_git_child(command, input, timeout, service, admission).await?; + Ok(out) +} + +/// Drive a spawned git child under `timeout` with process-group teardown, returning +/// its stdout AND the disarmed admission guard on success. Shared core for +/// [`run_git_service`] and [`info_refs`]: the caller passes a `Command` with its args +/// set; this adds piped stdio and `process_group(0)`. On the deadline the whole group +/// is torn down and reaped before returning [`GitServiceTimeout`]; on a dropped future +/// (client disconnect) the [`KillGroupOnDrop`] guard fires. `input` is written to the +/// child's stdin (empty for the advertise-refs path, which has no request body); `what` +/// labels errors. +/// +/// On success the guard is RETURNED rather than dropped internally (#174 KTD3), so a +/// caller running two sequential git stages under one admission — `build_filtered_pack`'s +/// rev-list then pack-objects — can hand the same guard from the first stage to the +/// second and keep the permits held across both, releasing them only when the second +/// stage's process group is reaped. Callers that run a single stage just let the +/// returned guard drop. +async fn drive_git_child( + mut command: Command, + input: Bytes, + timeout: Duration, + what: &str, + admission: Option, +) -> Result<(Vec, Option)> { + command .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -219,20 +372,33 @@ async fn run_git_service( let mut child = command.spawn()?; - // Arm the group-kill guard for the lifetime of the request. With - // process_group(0) the child is its own group leader, so pgid == its pid. - // This fires on a client disconnect (the whole future is dropped mid-request). + // Own the pipes so the child stays reap-able: wait_with_output would consume it, + // but on a timeout we must actively reap the group first (see + // reap_group_on_timeout), and on a disconnect the guard's detached reaper does. + // Take the pipes before the child moves into the guard below. + let mut stdin = child.stdin.take(); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + + // Arm the group-kill guard for the lifetime of the request. With process_group(0) + // the child is its own group leader, so pgid == its pid. On a client disconnect + // (the whole future is dropped mid-request) the guard's Drop launches a detached + // reaper that OWNS the child and runs the full TERM/grace/SIGKILL/reap — as strong + // as the timeout path below (#174 P1-c). The guard owns the child; the interaction + // drives it through `child_mut()`. + #[cfg(unix)] + let pgid = child.id().map(|id| id as i32); #[cfg(unix)] let mut group_guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid, + admission, }; + // On non-unix there is no process-group teardown, so hold the admission guard here + // for the child's whole interaction; it is returned to the caller below. + #[cfg(not(unix))] + let admission_holder = admission; - // Own the pipes so `child` stays reap-able after a timeout: wait_with_output - // would consume it, but on timeout we must actively reap the group before - // returning (see reap_group_on_timeout). - let mut stdin = child.stdin.take(); - let mut stdout = child.stdout.take().context("git stdout was not piped")?; - let mut stderr = child.stderr.take().context("git stderr was not piped")?; let mut out = Vec::new(); let mut err = Vec::new(); @@ -249,11 +415,15 @@ async fn run_git_service( None => Ok(()), } }; + #[cfg(unix)] + let child_ref = group_guard.child_mut(); + #[cfg(not(unix))] + let child_ref = &mut child; let (write_result, r_out, r_err, status) = tokio::join!( write, stdout.read_to_end(&mut out), stderr.read_to_end(&mut err), - child.wait(), + child_ref.wait(), ); r_out?; r_err?; @@ -261,23 +431,35 @@ async fn run_git_service( }; let timed = tokio::time::timeout(timeout, interact).await; + // The disarmed admission guard, handed back to the caller on success so it can ride + // a following sequential git stage (#174 KTD3) instead of releasing between stages. + let admission_out: Option; let (write_result, status) = match timed { Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the - // guard's drop would fire SIGTERM on the reaped, possibly-reused pgid. + // guard's drop would reap an already-reaped child / signal a reused pgid. + // The guard is returned (not dropped) so a two-stage caller keeps the + // permits held across both stages; a single-stage caller drops it on return. #[cfg(unix)] - group_guard.disarm(); + { + admission_out = group_guard.disarm(); + } + #[cfg(not(unix))] + { + admission_out = admission_holder; + } result? } Err(_elapsed) => { // Timeout: tear the whole group down and reap it before returning so a // caller releasing a write lock can't race a still-live git. Then disarm - // the (now redundant) guard so its drop can't hit a reused pgid. + // the (now redundant) guard so its drop can't hit a reused pgid. The + // returned admission guard drops here, AFTER the group is confirmed reaped. #[cfg(unix)] { - reap_group_on_timeout(&mut child).await; - group_guard.disarm(); + reap_group_on_timeout(group_guard.child_mut()).await; + drop(group_guard.disarm()); } #[cfg(not(unix))] { @@ -294,12 +476,12 @@ async fn run_git_service( // the real cause. if !status.success() { let stderr = String::from_utf8_lossy(&err); - bail!("{service} failed: {stderr}"); + bail!("{what} failed: {stderr}"); } write_result.context("failed to write to git stdin")?; - Ok(out) + Ok((out, admission_out)) } fn service_to_command(service: &str) -> &str { @@ -324,62 +506,89 @@ fn pkt_line(data: &str) -> Vec { format!("{len:04x}{data}").into_bytes() } -/// Build a packfile containing every object reachable from all refs EXCEPT the -/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; -/// only the named blobs are dropped. -pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Result> { - // All reachable objects as "oid [path]" lines. - let rev = std::process::Command::new("git") +/// Run `rev-list --objects --all` under `timeout` and return the reachable object +/// ids minus the withheld blobs. Runs under [`drive_git_child`] (async, with the +/// `tokio::time::timeout` + `process_group(0)` + [`KillGroupOnDrop`] teardown), so a +/// hung/slow enumeration is duration-bounded and reaped on client disconnect just +/// like the pack-objects stage. A bare blocking `Command::output()` inside a +/// `spawn_blocking` is uncancellable, so a hung rev-list would pin the endpoint's +/// concurrency permit for the whole hang (#174). `git_bin` is injectable for the +/// same fake-git testing reason as `run_git_service`. +async fn rev_list_keep( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, + timeout: Duration, + admission: Option, +) -> Result<(Vec, Option)> { + let mut command = Command::new(git_bin); + command .args(["rev-list", "--objects", "--all"]) - .current_dir(repo_path) - .output()?; - if !rev.status.success() { - bail!( - "git rev-list failed: {}", - String::from_utf8_lossy(&rev.stderr) - ); - } + .current_dir(repo_path); + // First of `build_filtered_pack`'s two sequential stages: it holds the caller's + // admission for the rev-list child and hands the disarmed guard back so the same + // permits ride the pack-objects stage (#174 KTD3, R2). The inter-stage window holds + // no live git group, so nothing is reaped-late by carrying the guard between them. + let (stdout, admission) = + drive_git_child(command, Bytes::new(), timeout, "rev-list", admission).await?; let mut keep = Vec::new(); - for line in String::from_utf8_lossy(&rev.stdout).lines() { + for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); if oid.is_empty() || withheld.contains(oid) { continue; } keep.push(oid.to_string()); } - let mut child = std::process::Command::new("git") + Ok((keep, admission)) +} + +/// Build a packfile containing every object reachable from all refs EXCEPT the +/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; +/// only the named blobs are dropped. +/// +/// Both git stages — the `rev-list` enumeration and the streaming `pack-objects` +/// build — run under [`drive_git_child`] sharing one deadline, so a hung/slow git at +/// either stage is duration-bounded and its process group is reaped on client +/// disconnect. An outer `tokio::time::timeout` around a `spawn_blocking` cannot +/// cancel the blocking thread, so neither stage may live off the async side +/// (#174, KTD5). +pub async fn build_filtered_pack( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, + timeout: Duration, + admission: Option, +) -> Result> { + // One deadline spans both git stages so a slow rev-list eats into the pack + // budget rather than granting each stage a fresh `timeout` (2x the permit hold). + let deadline = Instant::now() + timeout; + // The caller's admission rides both stages: rev-list holds it, hands it back, then + // pack-objects holds it until ITS process group is reaped on disconnect. That closes + // the path-scoped cap bypass — permits release on group reap, not the instant the + // request future drops (#174 R2, KTD3). The plain upload-pack path already does this. + let (keep, admission) = rev_list_keep( + git_bin, + repo_path, + withheld, + deadline.saturating_duration_since(Instant::now()), + admission, + ) + .await?; + let mut data = keep.join("\n").into_bytes(); + data.push(b'\n'); + let mut command = Command::new(git_bin); + command .args(["pack-objects", "--stdout"]) - .current_dir(repo_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - // Feed the object ids on stdin, but always reap the child afterward even if - // the write fails or stdin is missing, so an error can't drop the Child - // unwaited and leak a zombie (#53). - let write_result: std::io::Result<()> = { - use std::io::Write as _; - match child.stdin.take() { - Some(mut stdin) => { - let mut data = keep.join("\n").into_bytes(); - data.push(b'\n'); - stdin.write_all(&data) - } - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git pack-objects stdin unavailable", - )), - } - }; - let out = child.wait_with_output()?; - write_result.context("failed to write object ids to git pack-objects stdin")?; - if !out.status.success() { - bail!( - "git pack-objects failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - } - Ok(out.stdout) + .current_dir(repo_path); + let (out, _admission) = drive_git_child( + command, + Bytes::from(data), + deadline.saturating_duration_since(Instant::now()), + "pack-objects", + admission, + ) + .await?; + Ok(out) } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -409,17 +618,16 @@ pub async fn upload_pack_excluding( repo_path: &Path, request_body: Bytes, withheld: &HashSet, + timeout: Duration, + admission: Option, ) -> Result { - // build_filtered_pack shells out to git (rev-list, pack-objects) with - // blocking std::process I/O; run it off the async worker so a large repo's - // pack build does not stall the tokio runtime. - let pack = { - let repo_path = repo_path.to_path_buf(); - let withheld = withheld.clone(); - tokio::task::spawn_blocking(move || build_filtered_pack(&repo_path, &withheld)) - .await - .context("filtered-pack build task panicked")?? - }; + // Both git stages are duration-bounded and their process groups are reaped on + // disconnect via drive_git_child (#174), so a hung build no longer pins its + // concurrency slot and a client disconnect no longer orphans the git child. The + // caller's read + per-caller admission is threaded through both stages so the + // permits are held until the pack-objects group is reaped, matching the plain path + // (#174 R2). + let pack = build_filtered_pack("git", repo_path, withheld, timeout, admission).await?; // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -538,7 +746,9 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack(&bare, &withheld).unwrap(); + let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30), None) + .await + .unwrap(); let ids = pack_object_ids(&pack); assert!(ids.contains(&public), "public blob must be in the pack"); assert!( @@ -600,7 +810,9 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld).await.unwrap(); + let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30), None) + .await + .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); let ids = pack_object_ids(&extract_pack(&body)); assert!( @@ -650,14 +862,16 @@ mod tests { axum::extract::Query(q): axum::extract::Query>, ) -> Response { let service = q.get("service").cloned().unwrap_or_default(); - info_refs(&st.repo, &service).await.unwrap() + info_refs("git", &service, &st.repo, Duration::from_secs(30), None) + .await + .unwrap() } async fn pack_handler( axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld) + upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30), None) .await .unwrap() } @@ -920,23 +1134,37 @@ mod tests { #[cfg(unix)] #[tokio::test] async fn kill_group_guard_terminates_child_on_drop() { - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; { - let _guard = KillGroupOnDrop { pgid }; - } // guard drops here -> SIGTERM to the group + // The guard owns the child; on drop its detached reaper TERM/grace/KILL/ + // reaps the group (a plain sleep dies on the first SIGTERM). + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + admission: None, + }; + } - use std::os::unix::process::ExitStatusExt; - let status = child.wait().await.unwrap(); - assert_eq!( - status.signal(), - Some(libc::SIGTERM), - "child must be terminated by SIGTERM via its process group" + let mut gone = false; + for _ in 0..300 { + if !alive(pid) { + gone = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + gone, + "child must be terminated and reaped via its process group on guard drop" ); } @@ -954,9 +1182,10 @@ mod tests { .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; - // Read the backgrounded grandchild's pid from the first stdout line. + // Read the backgrounded grandchild's pid from the first stdout line, before + // the child moves into the guard. let mut stdout = child.stdout.take().unwrap(); let mut buf = Vec::new(); loop { @@ -971,20 +1200,26 @@ mod tests { assert!(alive(grandchild), "grandchild should be running"); { - let _guard = KillGroupOnDrop { pgid }; - } // group SIGTERM reaches sh AND the sleep grandchild - - let _ = child.wait().await; // reap sh + // The guard owns the child; on drop the detached reaper group-kills sh AND + // the sleep grandchild. + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + admission: None, + }; + } - // The grandchild reparents to init and is reaped; poll until it's gone. let mut gone = false; - for _ in 0..200 { + for _ in 0..300 { if !alive(grandchild) { gone = true; break; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } + unsafe { + libc::kill(grandchild, libc::SIGKILL); + } assert!( gone, "grandchild must be terminated by the group signal (#53)" @@ -996,27 +1231,32 @@ mod tests { async fn kill_group_guard_disarmed_does_not_kill() { // A request that completed cleanly disarms the guard; dropping it must not // signal anything. - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); + let pid = child.id().unwrap() as i32; { let mut guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid: Some(pid), + admission: None, }; guard.disarm(); - } // disarmed -> no kill + } // disarmed -> no reaper, no kill - assert!( - child.try_wait().unwrap().is_none(), - "disarmed guard must not kill the child" - ); + // Give any erroneously-spawned reaper a chance to run, then assert alive. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!(alive(pid), "disarmed guard must not kill the child"); - // Clean up the still-running child. - let _ = child.kill().await; - let _ = child.wait().await; + // Clean up the still-running child (disarm dropped the handle, so reap by pid). + unsafe { + libc::kill(pid, libc::SIGKILL); + let mut status = 0; + libc::waitpid(pid, &mut status, 0); + } } // ── #62 PR1: end-to-end teardown wiring through run_git_service ───────── @@ -1120,6 +1360,61 @@ mod tests { ); } + /// True when `err` is the transient ETXTBSY exec race a freshly-written fake + /// `git` hits under fork-storm load — a concurrent worker forked while this + /// file's write fd was still open, so `execve` sees it as busy (the same race + /// `fake_git_run_with_pids` retries). Deliberately narrow: only this raw-OS + /// error is retried, so a wrong error *type* (e.g. a missing async bound + /// surfacing as anything other than `GitServiceTimeout`) still fails loudly at + /// the caller's assertion instead of being swallowed. `spawn()?` propagates the + /// `io::Error` with no context, so it survives as the anyhow root. + #[cfg(unix)] + fn is_transient_exec_race(err: &anyhow::Error) -> bool { + // ETXTBSY == 26 on Linux and the BSDs/macOS; std has no stable ErrorKind. + err.downcast_ref::() + .and_then(std::io::Error::raw_os_error) + == Some(26) + } + + /// Drive `build_filtered_pack` under a per-attempt watchdog, retrying ONLY the + /// transient ETXTBSY exec race and returning the terminal error for the caller + /// to classify. Every attempt keeps its own outer `tokio::time::timeout`, so a + /// MISSING async bound — the regression the `build_filtered_pack_times_out_*` + /// tests guard — still trips the watchdog loudly on every attempt: the retry + /// can only absorb a fast exec failure, never a hang (a hang never returns, so + /// it can never reach the retry decision). + #[cfg(unix)] + async fn build_filtered_pack_or_exec_race_retry( + git_bin: &str, + repo_path: &std::path::Path, + withheld: &HashSet, + stage_timeout: Duration, + ) -> anyhow::Error { + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout, None), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the git stage \ + must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung git stage must return an error, not hang"); + if is_transient_exec_race(&err) { + // Fresh-fake-git ETXTBSY: back off (growing) so a bursty fork-pressure + // spike subsides before retrying, per fake_git_run_with_pids. + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + return err; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + // Dropping the request future mid-flight (client disconnect) must SIGTERM the // whole group so git AND its pack-objects grandchild die together. Goes RED // if `process_group(0)` or the guard-arming is removed: without its own @@ -1154,6 +1449,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, )); // Advance the future a slice at a time until the fake records its @@ -1225,6 +1521,311 @@ mod tests { ); } + /// #174 U2 (P1-c, RED-before/GREEN-after): on a client disconnect the teardown must + /// be as strong as the timeout path — a group member that IGNORES SIGTERM is still + /// SIGKILLed and reaped, not left running to accumulate past the concurrency cap. + /// The old `KillGroupOnDrop::drop` sent a lone SIGTERM with no escalation or reap, + /// so a SIGTERM-ignoring descendant survived the disconnect (RED). The fix launches + /// a detached reaper owning the child that runs the full TERM/grace/SIGKILL/reap. + /// This is distinct from the well-behaved-grandchild test above, whose `sleep` dies + /// on the first SIGTERM. + #[cfg(unix)] + #[tokio::test] + async fn run_git_service_sigkills_a_sigterm_ignoring_child_on_disconnect() { + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Leader (dies on the group SIGTERM) spawns a descendant that traps SIGTERM, + // records its own pid, and loops ~30s (bounded so a RED run leaks no permanent + // orphan; the assertion fires well before then). + let body = format!( + "#!/bin/sh\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n", + descfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut fut = Box::pin(run_git_service( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Bytes::new(), + Duration::from_secs(60), + None, + )); + // Drive the future a slice at a time until the descendant records its pid. + let mut desc: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let _cleanup = ReapOnPanic(vec![desc]); + assert!(alive(desc), "descendant should be running before the drop"); + + // Client disconnect: drop the request future. The guard's detached reaper must + // escalate SIGTERM -> SIGKILL and reap the SIGTERM-ignoring descendant. + drop(fut); + + let mut gone = false; + for _ in 0..500 { + if !alive(desc) { + gone = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + // Kill regardless so a RED run leaks no orphan. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed and reaped on \ + disconnect, not left running (old KillGroupOnDrop sent SIGTERM only)" + ); + } + + // #174 (SC3): the info/refs advertisement is now duration-bounded — previously + // it ran a bare `Command::output()` with no deadline, so a hung git pinned its + // concurrency slot forever. A hung fake git must abort with GitServiceTimeout + // (which the handler maps to 504). The outer watchdog turns a missing timeout + // into a loud failure instead of hanging the suite. info_refs shares + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped above. + #[cfg(unix)] + #[tokio::test] + async fn info_refs_times_out_a_hung_advertisement() { + let tmp = tempfile::TempDir::new().unwrap(); + // A fake git that hangs forever instead of advertising refs. + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\nsleep 300\n"); + + let result = tokio::time::timeout( + Duration::from_secs(10), + info_refs( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Duration::from_millis(200), + None, + ), + ) + .await + .expect("info_refs must return within the watchdog — its own timeout must fire"); + let err = result.expect_err("a hung advertisement must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung advertisement must abort with GitServiceTimeout (-> 504), got: {err}" + ); + } + + // #174 (SC3, KTD5): the filtered-pack build's streaming pack-objects stage is + // duration-bounded on the ASYNC side. The old build ran the whole thing in a + // spawn_blocking, so an outer tokio timeout could not cancel the blocking thread + // and a disconnect orphaned the git child. A fake git that returns objects fast + // on rev-list but hangs on pack-objects must now abort with GitServiceTimeout; + // the watchdog turns a missing bound into a loud failure. The stage runs under + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_pack_objects() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list returns one oid fast; pack-objects hangs forever. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ) + .await; + assert!( + err.downcast_ref::().is_some(), + "a hung pack-objects must abort with GitServiceTimeout, got: {err}" + ); + } + + // #174 (SC3, KTD5): the filtered-pack build's rev-list ENUMERATION stage is now + // duration-bounded on the ASYNC side too. It previously ran inside a + // spawn_blocking via a bare `Command::output()`, which an outer tokio timeout + // cannot cancel, so a hung/slow rev-list pinned the endpoint's concurrency permit + // for the whole hang. A fake git that hangs on rev-list must now abort with + // GitServiceTimeout; the watchdog turns a missing bound into a loud failure. + // rev-list runs under drive_git_child, so its disconnect/group-teardown is the + // same code proven by run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_rev_list() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list hangs forever; pack-objects would return fast if it were reached. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 300 ;;\n pack-objects) printf '' ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ) + .await; + assert!( + err.downcast_ref::().is_some(), + "a hung rev-list must abort with GitServiceTimeout, got: {err}" + ); + } + + // #174 U1 (R2, KTD3, RED-before/GREEN-after): the path-scoped filtered-pack serve + // must hold read + per-caller admission until its pack-objects process group is + // reaped on a client disconnect, exactly as the plain upload_pack path does. Before + // the fix build_filtered_pack took no AdmissionGuard and the handler's `_hold` + // permits dropped the instant the request future was dropped, so disconnect-spam on + // a path-scoped repo could hold PIDs past the concurrency cap while the permits were + // already free (#174 P1-a, on the filtered path the plain path had already closed). + // + // A real AdmissionGuard built from two owned semaphore permits rides + // rev-list -> pack-objects. We drive the future until pack-objects has forked its + // grandchild (the streaming pack-writer stand-in), assert the permits are still held + // mid-serve, then DROP the future (client disconnect) and assert the permits are + // released only AFTER the group is ESRCH-confirmed gone — never while it is alive. + // Goes RED if the guard is not threaded into the pack-objects stage (it would then + // drop after rev-list, freeing the permits mid-serve or on the bare future drop). + #[cfg(unix)] + #[tokio::test] + async fn filtered_pack_holds_admission_until_group_reaped_on_disconnect() { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("pids"); + // rev-list returns one oid fast; pack-objects forks a grandchild (the streaming + // writer stand-in), records leader+grandchild pids, then hangs so the future + // parks mid-serve with the guard owned by the pack-objects KillGroupOnDrop. The + // grandchild inherits (holds open) the stdout pipe, so drive_git_child's + // read_to_end blocks and the future stays pending until we drop it. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 &\nprintf '%s\\n%s\\n' \"$$\" \"$!\" > \"{}\"\nwait ;;\n *) exit 1 ;;\nesac\n", + pidfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let withheld = HashSet::new(); + + // Retry the fake-git spawn race like the sibling disconnect test; each attempt + // gets a FRESH semaphore so a dropped losing attempt can't skew the winning + // attempt's permit accounting. Keep the winning attempt's future PENDING so the + // drop below exercises the client-disconnect teardown. + let (fut, sem, leader, grandchild) = { + let mut attempt = 0u64; + loop { + attempt += 1; + let _ = std::fs::remove_file(&pidfile); + // Semaphore(4): two owned permits model the handler's global-read + + // per-caller admission, leaving 2 available while the op is in flight. + let sem = Arc::new(Semaphore::new(4)); + let g = sem.clone().try_acquire_owned().unwrap(); + let c = sem.clone().try_acquire_owned().unwrap(); + let admission = AdmissionGuard::new(g, Some(c)); + let mut fut = Box::pin(build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(60), + Some(admission), + )); + // Advance the future a slice at a time until the fake records its pids + // (i.e. pack-objects is running). `Ok(_)` means the future returned + // before the pidfile appeared (spawn error / early exit); stop polling + // then, since re-polling a completed future panics. + let mut pids = None; + for _ in 0..500 { + let finished = tokio::time::timeout(Duration::from_millis(10), &mut fut) + .await + .is_ok(); + if let Some(p) = read_two_pids(&pidfile) { + pids = Some(p); + break; + } + if finished { + break; + } + } + match pids { + Some((l, gch)) => break (fut, sem, l, gch), + None => { + // Transient spawn miss: drop the still-armed future so its guard + // reaps anything that spawned (and returns the permits), then + // back off before retrying. + drop(fut); + assert!( + attempt < FAKE_GIT_RETRY_ATTEMPTS, + "fake git failed to reach the pack-objects stage after \ + {FAKE_GIT_RETRY_ATTEMPTS} attempts (persistent failure, \ + not a transient parallel-runner miss)" + ); + tokio::time::sleep(Duration::from_millis( + FAKE_GIT_BACKOFF_STEP_MS * attempt, + )) + .await; + } + } + } + }; + let _cleanup = ReapOnPanic(vec![leader, grandchild]); + assert!(alive(grandchild), "grandchild must be running mid-serve"); + + // Mid-serve: the pack-objects stage owns the guard, so the two permits are still + // held. A build that dropped the guard after rev-list (unthreaded pack-objects + // stage) would have freed them here. + assert_eq!( + sem.available_permits(), + 2, + "admission permits must be held while the filtered serve is in flight" + ); + + // Client disconnect: drop the request future. The pack-objects KillGroupOnDrop + // must tear the group down AND hold the permits until the group is + // ESRCH-confirmed gone, releasing them only then. + drop(fut); + + let mut released_while_alive = false; + let mut released_after_reap = false; + for _ in 0..500 { + let released = sem.available_permits() == 4; + let group_alive = alive(grandchild); + if released && group_alive { + released_while_alive = true; + } + if released && !group_alive { + released_after_reap = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!( + !released_while_alive, + "admission permits were released while the git group was still alive — the \ + path-scoped concurrency-cap bypass (#174 P1-a) is open" + ); + assert!( + released_after_reap, + "admission permits must be released once the group is reaped on disconnect" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the @@ -1257,6 +1858,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1301,6 +1903,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1351,6 +1954,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1425,6 +2029,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1462,6 +2067,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_millis(200), + None, ), ) .await; @@ -1502,6 +2108,7 @@ mod tests { tmp.path(), big, Duration::from_secs(60), + None, ) .await; diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 229ee695..899394ee 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -306,6 +306,90 @@ pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) - Ok(content_output.stdout) } +/// Bounded twin of [`object_type`] for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2). Runs `git cat-file -t` under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so the child runs +/// in its own process group and a watchdog reaps it (SIGTERM -> grace -> SIGKILL) at +/// `timeout`. The bare [`object_type`] is a `spawn_blocking` `Command::output` that an +/// async timeout cannot cancel, so a wedged `cat-file` there pins the caller's held +/// /ipfs walk admission for the whole hang; this twin cannot. Semantics mirror +/// [`object_type`]: `Ok(Some(t))` for an existing object, `Ok(None)` when git exits +/// non-zero without timing out (the object is absent), and +/// `Err(`[`GitServiceTimeout`](crate::git::smart_http::GitServiceTimeout)`)` on the +/// deadline so the handler can mark the search truncated (retryable 503) rather than a +/// false not-found. Callers off the /ipfs path keep the bare helper. +pub fn object_type_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + match crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-t", sha256_hex], + repo_path, + b"", + deadline, + ) { + Ok(out) => Ok(Some(String::from_utf8_lossy(&out).trim().to_string())), + Err(e) if e.is::() => Err(e), + // A non-timeout failure is git reporting no such object (a non-zero exit), + // matching `object_type`'s `Ok(None)`. + Err(_) => Ok(None), + } +} + +/// Bounded `git cat-file -s` size read for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2): reads the object size WITHOUT its content (so an oversized object +/// is rejected before it is buffered, #173 F6), under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so a wedged size +/// read is reaped at `timeout` instead of pinning the held /ipfs walk admission. +/// `Ok(Some(n))` on success, `Ok(None)` when the object is absent (a non-timeout +/// non-zero exit), `Err(GitServiceTimeout)` on the deadline. +pub fn object_size_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + match crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-s", sha256_hex], + repo_path, + b"", + deadline, + ) { + Ok(out) => Ok(String::from_utf8_lossy(&out).trim().parse::().ok()), + Err(e) if e.is::() => Err(e), + Err(_) => Ok(None), + } +} + +/// Bounded twin of [`read_object_content`] for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2): `git cat-file ` under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so a wedged content +/// read is reaped at `timeout` instead of pinning the held /ipfs walk admission. Returns +/// the raw object bytes on success and an error (including `GitServiceTimeout` on the +/// deadline) otherwise, mirroring [`read_object_content`]. +pub fn read_object_content_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + obj_type: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", obj_type, sha256_hex], + repo_path, + b"", + deadline, + ) +} + /// Read a git object by its SHA-256 hex object ID. /// /// Returns `(object_type, content_bytes)` where `content_bytes` is the raw @@ -323,6 +407,42 @@ pub fn read_object(repo_path: &Path, sha256_hex: &str) -> Result Result)>> { + let deadline = std::time::Instant::now() + timeout; + let obj_type = match object_type_bounded( + git_bin, + repo_path, + sha256_hex, + deadline.saturating_duration_since(std::time::Instant::now()), + )? { + Some(t) => t, + None => return Ok(None), + }; + let content = read_object_content_bounded( + git_bin, + repo_path, + sha256_hex, + &obj_type, + deadline.saturating_duration_since(std::time::Instant::now()), + )?; + Ok(Some((obj_type, content))) +} + /// Get the diff between two branches: changes on source_branch not in target_branch. pub fn branch_diff(repo_path: &Path, target_branch: &str, source_branch: &str) -> Result { let output = Command::new("git") @@ -501,4 +621,84 @@ mod tests { "unchanged file must not appear: {names:?}" ); } + + /// #173 round-10 (KTD2): `object_type_bounded` reaps a wedged `cat-file` child at its + /// deadline instead of blocking on it to natural exit, so a hung probe cannot pin the + /// /ipfs walk admission the owning task holds. A fake `git` records its pid and sleeps + /// far past the 1s deadline; the `run_bounded_git` watchdog (SIGTERM -> grace -> + /// SIGKILL of the process group) must kill it well before that natural exit, and the + /// call must surface `GitServiceTimeout`. REVERT PROOF (RED): swap the twin's + /// `run_bounded_git` for the bare `Command::output()` and the wedged child stays alive + /// past the deadline — the mid-flight liveness poll below reads it still running. + #[cfg(unix)] + #[test] + fn object_type_bounded_reaps_wedged_child_at_deadline() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("catfile.pid"); + // `cat-file` records its own pid then sleeps 8s (>> the 1s deadline) so the probe + // is genuinely wedged; the watchdog is what must end it. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + cat-file) echo $$ > \"{}\"; sleep 8 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pidfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let repo = tmp.path().to_path_buf(); + let git = git_path.to_str().unwrap().to_string(); + + let alive = |pid: i32| unsafe { libc::kill(pid, 0) == 0 }; + + // The bounded probe blocks until the watchdog tears the child down, so run it on + // a worker thread and poll for the reap from here. + let handle = std::thread::spawn(move || { + super::object_type_bounded(&git, &repo, "deadbeef", Duration::from_secs(1)) + }); + + let mut pid = None; + for _ in 0..500 { + if let Some(p) = std::fs::read_to_string(&pidfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + pid = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid = pid.expect("the fake cat-file must have spawned and recorded its pid"); + + // Past the 1s deadline + SIGTERM grace but well before the 8s natural exit: the + // watchdog must already have reaped the wedged group. A bare, unbounded read would + // leave it running here — the load-bearing RED. + std::thread::sleep(Duration::from_secs(3)); + let reaped = !alive(pid); + // Defensive reap so a RED run leaks no orphan. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + reaped, + "object_type_bounded must reap the wedged cat-file child at the deadline, \ + not leave it running to its natural exit" + ); + + let res = handle.join().expect("probe thread joins"); + let err = res.expect_err("a deadline overrun must be an error, not a value"); + assert!( + err.is::(), + "a deadline overrun must surface GitServiceTimeout, got: {err:?}" + ); + } } diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index cb70e39c..d463360c 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -4,11 +4,290 @@ //! content is held back. use crate::db::VisibilityRule; -use crate::git::store; use crate::visibility::{visibility_check, Decision}; use anyhow::{Context, Result}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::Path; +use std::process::Stdio; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +/// Fixed budget bounding the whole withheld-blob classification walk (#174 U3). +/// The walk is fast for a real repo; this bound exists to reap a hung or +/// pathologically slow git child so it cannot pin a served-git permit (the read +/// permit on the upload-pack serve path, the write permit on the receive-pack +/// post-push replication path) past the deadline. Every caller funnels through +/// `blob_paths`, so bounding here bounds both paths at one seam. Production callers +/// pass the operator-configured `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` instead; this +/// fixed budget only backs the `git_bin`-less test wrappers. +#[cfg(test)] +const WALK_TIMEOUT: Duration = Duration::from_secs(600); + +/// How long the process-group watchdog waits after SIGTERM before escalating to +/// SIGKILL, giving a well-behaved git child time to clean up its `*.lock` files. Only +/// paid on a timeout (already the exceptional path). +#[cfg(unix)] +const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); + +/// Run one git child under a shared `deadline` with process-group teardown, +/// BLOCKING, and return its stdout. The child runs in its own process group; a +/// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, +/// the whole group if the deadline passes before the child is reaped, so a hung or +/// slow git can pin neither a served-git permit nor a blocking thread past the +/// deadline (jatmn's "retain admission until they are reaped"). This is the +/// blocking-side counterpart of `smart_http::drive_git_child`, needed because the +/// walk's callers run it inside `spawn_blocking`, which an async timeout cannot +/// cancel. Returns [`crate::git::smart_http::GitServiceTimeout`] on the deadline so +/// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can +/// drive the teardown in tests without mutating the process-global PATH; +/// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). +/// Returns true if `pid` (a process-group leader we spawned) has terminated, WITHOUT +/// reaping it. `waitid(..., WNOWAIT)` reports the exit state but leaves the child +/// waitable, so the caller's later `child.wait()` still collects the status and the +/// pid/pgid stays live until then — which is what keeps the watchdog's `kill(-pgid)` +/// teardown from ever racing a recycled pgid. Used to distinguish "the child actually +/// exited" from "the child merely closed stdout" after the drain returns (#174 P1-a). +#[cfg(unix)] +fn child_terminated_without_reaping(pid: i32) -> bool { + // SAFETY: waitid writes only into the zeroed siginfo and borrows no Rust memory; + // WNOWAIT leaves the child unreaped, WNOHANG makes the probe non-blocking. + let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; + let rc = unsafe { + libc::waitid( + libc::P_PID, + pid as libc::id_t, + &mut info, + libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, + ) + }; + // rc == 0 with si_pid == 0 means "no state change yet" (still running); a non-zero + // si_pid means the child has entered a waitable, exited state. EINTR/other errors + // (rc != 0) are treated as "not yet terminated" and the caller re-polls. + rc == 0 && unsafe { info.si_pid() } != 0 +} + +#[cfg(unix)] +pub(crate) fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + use std::io::{Read, Write}; + use std::os::unix::process::CommandExt; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + // With process_group(0) the child leads its own group, so pgid == its pid. + let pgid = child.id() as i32; + + // Watchdog: on the deadline, tear the WHOLE process group down — SIGTERM, a grace + // for a well-behaved child to clean up its `*.lock` files, then an UNCONDITIONAL + // SIGKILL of the group. It never stands down on leader-reap alone: a group member + // that ignores SIGTERM while the leader exits cleanly would otherwise escape the + // SIGKILL and keep running past the deadline (finding 3, #174). The main thread + // defers reaping the leader until this thread returns (see below), so the leader's + // pid is still unreaped while every `kill(-pgid)` fires and the pgid cannot have + // been recycled — which is why this no longer needs the old `reaped` short-circuit. + // Kept off the main thread because the main thread's stdout drain is exactly what + // blocks until a hung child is torn down. + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + // Fixed grace: because the main thread defers the leader's reap, a + // fully-exited group still shows a zombie leader here, so polling for + // ESRCH cannot detect early completion — just wait the grace, then + // SIGKILL. On a group of only zombies the SIGKILL is a harmless no-op; + // on a SIGTERM-ignoring member it is what actually kills it. + std::thread::sleep(WATCHDOG_TERM_GRACE); + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + // Brief settle so the SIGKILL is delivered before the main thread + // reaps the leader and frees the pgid. A wedged (D-state) member + // survives even SIGKILL — the documented residual, as in smart_http. + std::thread::sleep(Duration::from_millis(20)); + if unsafe { libc::kill(-pgid, 0) } == 0 { + tracing::warn!( + pgid, + "withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)" + ); + } + true + } + } + }); + + // Feed stdin on a writer thread and drain stderr on a reader thread so the main + // thread can drain stdout concurrently; writing all of stdin (or draining one + // pipe) before the others can deadlock once a pipe buffer fills. + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut out = Vec::new(); + // Blocking drain, unblocked by the child closing stdout on exit. The watchdog's + // SIGTERM/SIGKILL is what makes a hung child exit; a git wedged in uninterruptible + // (D-state) I/O survives even SIGKILL, so this drain and the wait below can block + // until the kernel returns, pinning the walk thread and its permit. That residual + // is unreachable in userspace (no signal reaps a D-state process) and matches the + // async `reap_group_on_timeout`, which likewise only warns and gives up there. + let read_result = stdout.read_to_end(&mut out); + // The drain has returned, but that only means all stdout write ends are closed — + // NOT that the child has exited. A group member, or the leader itself, can close + // stdout and keep running; standing the watchdog down on the drain alone (as the + // old code did) would then let `child.wait()` block forever on that live child, + // past the deadline, pinning the walk thread and its permit (finding P1-a, #174). + // So stand the watchdog down only once the child has ACTUALLY terminated, detected + // WITHOUT reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its + // pgid un-recycled until the watchdog finishes and we join it below. Past the + // deadline the watchdog owns the teardown, so we stop polling and let it run the + // full SIGTERM -> grace -> SIGKILL; joining it before `child.wait()` keeps every + // `kill(-pgid)` firing while the pid is still unreaped and guarantees a + // stdout-closing-then-hanging member has been SIGKILLed rather than left running. + loop { + if child_terminated_without_reaping(pgid) { + let _ = done_tx.send(()); + break; + } + if Instant::now() >= deadline { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + let killed = watchdog.join().unwrap_or(false); + let status = child.wait().context("git wait failed")?; + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + read_result.context("failed to read git stdout")?; + // The watchdog runs off a wall clock that can race a child finishing right at the + // deadline. A child that exited on its own (success) is not a timeout even if the + // watchdog fired late; only a child that did not exit successfully is a genuine + // timeout, which keeps a walk completing at its budget from a spurious 504. + if killed && !status.success() { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} + +/// Non-Unix fallback for [`run_bounded_git`]. Windows and other non-Unix targets +/// have no process-group teardown (`process_group(0)` / `kill(-pgid)` are Unix-only), +/// so this bounds a single child on its own: threads feed stdin and drain stderr +/// while the main thread drains stdout, and a watchdog thread kills the child at the +/// deadline (which closes stdout and unblocks the drain). The child is shared with +/// the watchdog behind a mutex that the main thread does NOT hold while draining, so +/// the watchdog can always acquire it to kill. Best-effort — it reaps only the direct +/// child, not a descendant group — which is why the hardened, group-aware path above +/// is gated to Unix, the only target the served node actually runs on (the Windows +/// release binary is best-effort / `continue-on-error` in CI). Kept in lockstep with +/// the Unix version's signature and result semantics so every caller compiles on all +/// targets (#174). +#[cfg(not(unix))] +pub(crate) fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + use std::io::{Read, Write}; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + + // Share the child with the watchdog. The main thread drains stdout WITHOUT + // holding this lock, so the watchdog can always acquire it to kill on timeout; + // killing closes stdout and unblocks the drain below. + let child = std::sync::Arc::new(std::sync::Mutex::new(child)); + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = { + let child = child.clone(); + std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + if let Ok(mut c) = child.lock() { + let _ = c.kill(); + } + true + } + } + }) + }; + + let mut out = Vec::new(); + let read_result = stdout.read_to_end(&mut out); + // The drain has returned (child exited or was killed), so taking the lock here + // cannot deadlock against the watchdog. + let status = child + .lock() + .expect("git child mutex poisoned") + .wait() + .context("git wait failed")?; + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + read_result.context("failed to read git stdout")?; + if killed && !status.success() { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} /// Fail closed unless every ref ultimately resolves to a commit (a ref pointing /// directly at a blob or tree, or an annotated tag — even a nested one — of such @@ -21,19 +300,15 @@ use std::path::Path; /// Full peeling is why this is not `for-each-ref %(*objecttype)`, which /// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a- /// commit as a non-commit. -fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { - let refs = std::process::Command::new("git") - .args(["for-each-ref", "--format=%(refname)"]) - .current_dir(repo_path) - .output() - .context("git for-each-ref failed")?; - if !refs.status.success() { - anyhow::bail!( - "git for-each-ref failed: {}", - String::from_utf8_lossy(&refs.stderr) - ); - } - let refs_stdout = String::from_utf8_lossy(&refs.stdout); +fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<()> { + let refs_out = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(refname)"], + repo_path, + b"", + deadline, + )?; + let refs_stdout = String::from_utf8_lossy(&refs_out); let refnames: Vec<&str> = refs_stdout .lines() .map(str::trim) @@ -43,59 +318,25 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { return Ok(()); } - // Peel every ref in one `git cat-file --batch-check` pass: one - // `^{}` query per line, one output line per input line, in order. - // The stdin write runs on a separate thread so this thread can drain stdout - // concurrently. cat-file echoes the full query on a ` missing` line, - // so output scales with refname length (not a fixed size per ref); writing - // all of stdin before reading any stdout would deadlock both pipes once the - // child's stdout buffer fills. Dropping `stdin` at the end of the closure - // sends EOF. + // Peel every ref in one `git cat-file --batch-check` pass: one `^{}` + // query per line, one output line per input line, in order. cat-file echoes the + // full query on a ` missing` line, so output scales with refname length; + // run_bounded_git drains stdout concurrently with the stdin write, so the pipe + // cannot deadlock, and the whole peel is bounded by the shared walk deadline. let queries = refnames .iter() .map(|r| format!("{r}^{{}}")) .collect::>() .join("\n"); - use std::io::Write; - let mut child = std::process::Command::new("git") - .args(["cat-file", "--batch-check=%(objecttype)"]) - .current_dir(repo_path) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .context("failed to spawn git cat-file")?; - // Feed stdin on a writer thread so this thread can drain stdout via - // wait_with_output concurrently; a None handle (the pipe vanished) becomes a - // broken-pipe write error. wait_with_output reaps the child unconditionally - // before any error is surfaced, so no path drops it unwaited (#53), and the - // writer is joined only after the drain so the join cannot deadlock. - let writer = child - .stdin - .take() - .map(|mut stdin| std::thread::spawn(move || stdin.write_all(queries.as_bytes()))); - let peel_result = child.wait_with_output(); - let write_result = match writer { - Some(handle) => handle - .join() - .map_err(|_| anyhow::anyhow!("git cat-file stdin writer thread panicked"))?, - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git cat-file stdin unavailable", - )), - }; - // Surface a write error only if the process didn't already fail with a - // clearer status. - let peel = peel_result.context("git cat-file failed")?; - if !peel.status.success() { - anyhow::bail!( - "git cat-file --batch-check failed: {}", - String::from_utf8_lossy(&peel.stderr) - ); - } - write_result.context("failed to write to git cat-file stdin")?; - - let peel_stdout = String::from_utf8_lossy(&peel.stdout); + let peel_out = run_bounded_git( + git_bin, + &["cat-file", "--batch-check=%(objecttype)"], + repo_path, + queries.as_bytes(), + deadline, + )?; + + let peel_stdout = String::from_utf8_lossy(&peel_out); let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect(); // A short read means at least one ref went unclassified — fail closed. if types.len() != refnames.len() { @@ -147,47 +388,46 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { /// Fails closed: if commit enumeration or any tree walk fails, returns an error so /// the caller aborts the serve/pin rather than producing a partial (under-withheld) /// set. -fn blob_paths(repo_path: &Path) -> Result> { - assert_all_refs_are_commits(repo_path)?; +fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result> { + // One deadline spans the whole walk (the ref check, the HEAD probe, rev-list, + // and every per-commit ls-tree), so a slow or hung walk is bounded as a unit + // rather than granting each git child a fresh timeout. + let deadline = Instant::now() + timeout; + assert_all_refs_are_commits(repo_path, git_bin, deadline)?; // Enumerate every reachable commit, not just ref tips. `--all` walks all refs; // append HEAD so a detached HEAD (reachable by rev-list/upload-pack but in no // ref) is still classified. When HEAD does not resolve (unborn branch on an - // empty repo) `--all` alone yields nothing, which is correct — no objects exist. - let head = store::head_commit(repo_path).context("resolve HEAD failed")?; + // empty repo) `--all` alone yields nothing, which is correct: no objects exist. + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), replacing the previously unbounded `store::head_commit` child. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); let mut rev_args = vec!["rev-list", "--all"]; - if head.is_some() { + if head_resolves { rev_args.push("HEAD"); } - let commits = std::process::Command::new("git") - .args(&rev_args) - .current_dir(repo_path) - .output() - .context("git rev-list --all failed")?; - if !commits.status.success() { - anyhow::bail!( - "git rev-list --all failed: {}", - String::from_utf8_lossy(&commits.stderr) - ); - } - let commits_stdout = String::from_utf8_lossy(&commits.stdout); + let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let commits_stdout = String::from_utf8_lossy(&commits_out); let mut out: HashSet<(String, String)> = HashSet::new(); for commit in commits_stdout.lines() { let commit = commit.trim(); if commit.is_empty() { continue; } - let listing = std::process::Command::new("git") - .args(["ls-tree", "-rz", commit]) - .current_dir(repo_path) - .output() - .context("git ls-tree -rz failed")?; - if !listing.status.success() { - anyhow::bail!( - "git ls-tree -rz {commit} failed: {}", - String::from_utf8_lossy(&listing.stderr) - ); - } + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rz", commit], + repo_path, + b"", + deadline, + )?; // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes // "secret/caf\303\251.txt"), and that quoted literal would not match a @@ -198,7 +438,7 @@ fn blob_paths(repo_path: &Path) -> Result> { // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string // would no longer match its deny rule — the same under-withholding class, one // layer down. Fail closed instead so the caller aborts rather than leaks. - let Ok(listing_stdout) = std::str::from_utf8(&listing.stdout) else { + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { anyhow::bail!( "git ls-tree -rz {commit} returned a non-UTF-8 path; \ refusing to produce a partial (under-withheld) set" @@ -229,6 +469,7 @@ fn blob_paths(repo_path: &Path) -> Result> { /// /// The whole-repo "/" gate is handled by the caller before this function runs: /// if "/" denies, the caller gets a 404 and never reaches the filtered serve. +#[cfg(test)] pub fn withheld_blob_oids( repo_path: &Path, rules: &[VisibilityRule], @@ -236,7 +477,33 @@ pub fn withheld_blob_oids( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + withheld_blob_oids_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`withheld_blob_oids`] with an injectable `git_bin` and walk `timeout`. Served +/// handlers call this with the operator-configured git binary and +/// `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`, so the whole walk is bounded by the same +/// budget as the other served-git ops and a fake `git` can drive its teardown in +/// tests. The `git_bin`-less wrapper above keeps the fixed [`WALK_TIMEOUT`] for the +/// classification tests that run against real git. +pub fn withheld_blob_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; Ok(withheld_from_pairs( &pairs, rules, is_public, owner_did, caller, )) @@ -304,6 +571,7 @@ pub fn replicable_objects(all: Vec, withheld: &HashSet) -> Vec Result> { + allowed_blob_set_for_caller_bounded( + repo_path, git_bin, timeout, rules, is_public, owner_did, None, + ) +} + /// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The /// caller-aware generalization of `replicable_blob_set` (which is the anonymous /// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against @@ -325,6 +608,7 @@ pub fn replicable_blob_set( /// elsewhere (its content is readable to this caller elsewhere). Trees and /// commits are NOT included here; the caller decides per object type whether /// the allow-set applies (it does not for trees/commits — KTD3). +#[cfg(test)] pub fn allowed_blob_set_for_caller( repo_path: &Path, rules: &[VisibilityRule], @@ -332,7 +616,29 @@ pub fn allowed_blob_set_for_caller( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + allowed_blob_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_blob_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` gate. +pub fn allowed_blob_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { @@ -342,6 +648,477 @@ pub fn allowed_blob_set_for_caller( Ok(allowed) } +/// The reachable-commit enumeration for the LENIENT walks (the `/ipfs/{cid}` tree +/// gate and the commit/tag reachability set): bounded `git rev-list --all [HEAD]` +/// under the caller's shared `deadline`, deliberately WITHOUT +/// `assert_all_refs_are_commits`. That guard fail-closes a repo's whole walk when +/// any ref peels to a non-commit (an annotated tag of a tree is pushable through +/// receive-pack), which would 404 every reachable tree/commit/tag CID here for a +/// legitimate reader. `rev-list --all` skips such refs cleanly, so the commit set +/// stays complete; an object reachable only via such a ref is simply excluded — +/// correctly fail-closed. Fails closed on a rev-list error. +/// +/// Safe ONLY for a caller whose output feeds a fail-closed allow-list where absence +/// = withhold: a tolerant walk there over-withholds, never leaks. NOT safe for a +/// serve/replication filter, where a missed reachable object under-withholds — +/// those go through `blob_paths`, which runs the guard first. +fn reachable_commit_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), matching `blob_paths`. When HEAD does not resolve (unborn + // branch on an empty repo) `--all` alone yields nothing, which is correct. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); + let mut rev_args = vec!["rev-list", "--all"]; + if head_resolves { + rev_args.push("HEAD"); + } + let out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + Ok(String::from_utf8_lossy(&out) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect()) +} + +/// Every `(oid, "/repo/relative/path", kind)` triple reachable from the given +/// `commits` — the shared ls-tree seam the tree walk filters (`kind == "tree"`). +/// One bounded `git ls-tree -rzt` per commit under the caller's shared `deadline`: +/// `-rzt` is byte-identical to `-rz` for blob records and additionally emits the +/// tree object for each directory at its own path. `kind` is git's object-type +/// string ("blob", "tree", or "commit" for a gitlink). The commit's ROOT tree is +/// not emitted by `ls-tree` (it lists entries *under* a tree); `tree_paths` adds +/// it. Triples are de-duplicated across commits and paths carry a leading "/" to +/// match the glob form of visibility rules ("/secret/**"). +/// +/// Fails closed: if any tree walk fails — or a path is not valid UTF-8 — it +/// returns an error so the caller aborts rather than producing a partial +/// (under-withheld) set. +fn object_paths( + repo_path: &Path, + git_bin: &str, + commits: &[String], + deadline: Instant, +) -> Result> { + let mut out: HashSet<(String, String, String)> = HashSet::new(); + for commit in commits { + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rzt", commit], + repo_path, + b"", + deadline, + )?; + // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` + // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes + // "secret/caf\303\251.txt"), and that quoted literal would not match a + // visibility rule like "/secret/**", under-withholding the object. The TAB + // field separator survives `-z`, so the per-record parse is unchanged. + // + // Parse strictly: a lossy decode would replace an invalid byte in a denied + // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string + // would no longer match its deny rule — the same under-withholding class, one + // layer down. Fail closed instead so the caller aborts rather than leaks. + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { + anyhow::bail!( + "git ls-tree -rzt {commit} returned a non-UTF-8 path; \ + refusing to produce a partial (under-withheld) set" + ); + }; + for record in listing_stdout.split('\0') { + // " \t" + let Some((meta, path)) = record.split_once('\t') else { + continue; + }; + let mut parts = meta.split_whitespace(); + let _mode = parts.next(); + let kind = parts.next(); + let oid = parts.next(); + if let (Some(kind), Some(oid)) = (kind, oid) { + out.insert((oid.to_string(), format!("/{path}"), kind.to_string())); + } + } + } + Ok(out) +} + +/// Root tree oid of every reachable commit, at "/". `ls-tree` never emits a commit's +/// own root tree (it lists entries *under* a tree), so it is added explicitly here. +/// Resolved in ONE bounded `git log --no-walk --format=%T --stdin` pass over the +/// shared commit set — not a per-commit `rev-parse` — so a tree-set walk costs the +/// same subprocess order as the blob walk. The commit oids go on STDIN, not argv: a +/// long history has tens of thousands of reachable commits, and passing them all as +/// arguments overflows ARG_MAX so `git log` fails to spawn — which the caller treats +/// as a walk error and fail-closed 404s an authorized reader of a reachable/root +/// tree (#173 P2). `run_bounded_git` drains stdout concurrently with the stdin +/// write, so a large history cannot deadlock the pipes. A commit whose root tree git +/// cannot resolve fails the pass (bail), failing closed. +fn root_tree_pairs( + repo_path: &Path, + git_bin: &str, + commits: &[String], + deadline: Instant, +) -> Result> { + if commits.is_empty() { + return Ok(HashSet::new()); + } + let mut buf = String::with_capacity(commits.len() * 65); + for c in commits { + buf.push_str(c); + buf.push('\n'); + } + let out = run_bounded_git( + git_bin, + &["log", "--no-walk=unsorted", "--format=%T", "--stdin"], + repo_path, + buf.as_bytes(), + deadline, + )?; + let mut set = HashSet::new(); + for line in String::from_utf8_lossy(&out).lines() { + let oid = line.trim(); + if !oid.is_empty() { + set.insert((oid.to_string(), "/".to_string())); + } + } + Ok(set) +} + +/// Every `(tree_oid, "/path")` pair reachable in `repo_path`: the `kind == "tree"` +/// slice of [`object_paths`] (subtree trees at their directory paths) PLUS every +/// reachable commit's root tree at "/" (see [`root_tree_pairs`]). Computes the +/// reachable-commit set ONCE (leniently — see [`reachable_commit_oids`]; the tree +/// allowed-set feeds ONLY the `/ipfs/{cid}` tree gate, where absence = fail-closed +/// 404) and drives both the ls-tree walk and the root-tree pass from it, so the two +/// cannot diverge and neither re-enumerates. The tree analog of [`blob_paths`], +/// bounded by the same shared `deadline`. +fn tree_paths( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let commits = reachable_commit_oids(repo_path, git_bin, deadline)?; + let mut out: HashSet<(String, String)> = object_paths(repo_path, git_bin, &commits, deadline)? + .into_iter() + .filter(|(_, _, kind)| kind == "tree") + .map(|(oid, path, _)| (oid, path)) + .collect(); + out.extend(root_tree_pairs(repo_path, git_bin, &commits, deadline)?); + Ok(out) +} + +/// The OIDs from a `(oid, "/path")` listing that visibility ALLOWS `caller` at some +/// path — the shared inner loop of the blob and tree allowed-sets. An oid reachable +/// at an allowed path is kept even when also reachable at a denied one. +fn allowed_set_from_pairs<'a>( + pairs: impl IntoIterator, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> HashSet { + pairs + .into_iter() + .filter(|(_, path)| { + visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow + }) + .map(|(oid, _)| oid.clone()) + .collect() +} + +/// Reachable tree OIDs that visibility ALLOWS `caller` at some path — the tree +/// analog of [`allowed_blob_set_for_caller`]. `GET /ipfs/{cid}` gates tree objects +/// with this so the CID surface matches `get_tree`: a tree reachable only at a +/// withheld path is absent from the set and 404'd; the root tree ("/") and any tree +/// on the path to an allowed subtree are present. Fails closed on a +/// dangling/unreachable tree (never enumerated by the reachable walk, so never in +/// the set — the #126 geometry, for trees). A tree reachable at an allowed path is +/// included even when also reachable at a withheld one (its structure is visible to +/// this caller elsewhere). +#[cfg(test)] +pub fn allowed_tree_set_for_caller( + repo_path: &Path, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + allowed_tree_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_tree_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` tree gate. One deadline spans the whole walk (the HEAD +/// probe, rev-list, every per-commit ls-tree, and the root-tree pass), matching +/// `blob_paths`, so a slow or hung walk is bounded as a unit while the handler holds +/// its /ipfs walk permit (#174 F5). +pub fn allowed_tree_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let deadline = Instant::now() + timeout; + Ok(allowed_set_from_pairs( + &tree_paths(repo_path, git_bin, deadline)?, + rules, + is_public, + owner_did, + caller, + )) +} + +/// Object bound for the annotated-tag reachability walk (#173, jatmn tag fan-out). +/// A path-scoped pinned-CID request drives this walk while holding one per-request +/// and one per-IP walk slot, so the total tag work must be finite regardless of how +/// many tag refs the repo has. 8192 is far past any real repo's annotated-tag count +/// (the Linux kernel has a few hundred), yet finite: a repo beyond it fails closed +/// (Err), matching this function's fail-closed-on-any-git-error contract, rather than +/// truncating silently (which would under-withhold a still-reachable tag object). +const MAX_TAG_OBJECTS: usize = 8192; + +/// Walk the annotated-tag chains rooted at `seeds`, inserting every tag object they +/// pass through into `set`. A tag whose target is itself a tag (tag-of-a-tag) +/// discovers the inner tag, which is walked in a later round. +/// +/// #173 (jatmn): the tag inspection is BATCHED, not one process per tag. Each round +/// feeds every not-yet-inspected tag oid to a SINGLE `git cat-file --batch` child on +/// stdin and reads back framed ` \n\n` records, so the +/// number of child processes is bounded by the tag-chain DEPTH (rounds), not the tag +/// COUNT. Oids go on stdin, never argv, so a large tag set cannot overflow ARG_MAX. +/// The child runs through [`run_bounded_git`], which drains stdout concurrently with +/// the stdin write (subsuming #173's F4 writer-thread drain — a round large enough to +/// fill both pipes cannot deadlock) and tears the child down at `deadline`, so a hung +/// cat-file cannot pin the caller's /ipfs walk permit (#174 F5). Total tag objects +/// inspected are capped at `max_tag_objects`; exceeding it is an error (fail closed), +/// not a silent truncation. Takes the bound as a parameter so a test can drive a tiny +/// value while the caller passes the real `MAX_TAG_OBJECTS`. +fn walk_tag_chain( + repo_path: &Path, + git_bin: &str, + seeds: Vec, + set: &mut HashSet, + max_tag_objects: usize, + deadline: Instant, +) -> Result<()> { + // Tag oids known but not yet inspected. Seeds may repeat / already be present; + // the `set.insert` gate below is what actually dedups and terminates cycles. + let mut pending: Vec = seeds; + let mut inspected: usize = 0; + + while !pending.is_empty() { + // Inspect only oids new to `set`; a re-seen oid was already walked. + let round: Vec = pending + .drain(..) + .filter(|oid| set.insert(oid.clone())) + .collect(); + if round.is_empty() { + break; + } + inspected += round.len(); + if inspected > max_tag_objects { + anyhow::bail!( + "annotated-tag walk exceeded the object bound ({max_tag_objects}); refusing to serve" + ); + } + + // One bounded child for the whole round: feed all oids on stdin, read the + // framed records from the returned stdout. + let mut buf = String::with_capacity(round.len() * 65); + for oid in &round { + buf.push_str(oid); + buf.push('\n'); + } + let stdout = run_bounded_git( + git_bin, + &["cat-file", "--batch"], + repo_path, + buf.as_bytes(), + deadline, + )?; + + // Parse one record per requested oid: ` \n\n`. + // A ` missing\n` record has no size/body and is anomalous here (every + // oid came from a ref tip or a prior tag body), so fail closed. + let mut i = 0usize; + for _ in 0..round.len() { + let hdr_end = stdout[i..] + .iter() + .position(|&b| b == b'\n') + .map(|p| i + p) + .context("git cat-file --batch: truncated record header")?; + let header = std::str::from_utf8(&stdout[i..hdr_end]) + .context("git cat-file --batch: non-utf8 record header")?; + i = hdr_end + 1; + let mut fields = header.split(' '); + let _oid = fields.next().unwrap_or(""); + let ty = fields.next().unwrap_or(""); + if ty == "missing" || fields.clone().next().is_none() { + anyhow::bail!("git cat-file --batch: object {header:?} missing or malformed"); + } + let size: usize = fields + .next() + .unwrap_or("") + .parse() + .context("git cat-file --batch: bad record size")?; + let body_end = i + .checked_add(size) + .filter(|&e| e <= stdout.len()) + .context("git cat-file --batch: truncated record body")?; + // Only a tag object can point at an inner tag; walk its header. + if ty == "tag" { + let body = std::str::from_utf8(&stdout[i..body_end]) + .context("git cat-file --batch: non-utf8 tag body")?; + let mut target = None; + let mut is_tag = false; + for line in body.lines() { + if let Some(oid) = line.strip_prefix("object ") { + target = Some(oid.trim().to_string()); + } else if line == "type tag" { + is_tag = true; + } else if line.is_empty() { + break; // end of header + } + } + if is_tag { + if let Some(t) = target { + pending.push(t); + } + } + } + // Skip body plus its trailing newline to the next record. + i = body_end + 1; + } + } + Ok(()) +} + +/// The reachable-commit/tag gate set for the `/ipfs/{cid}` resolver (#173, F2): +/// every reachable commit oid UNION every reachable annotated-tag OBJECT oid. A +/// DANGLING commit/tag (referenced by no ref, directly or via a tag chain) is in +/// neither part, so the resolver denies it under a path-scoped rule instead of +/// leaking its message; a reachable one still serves. +#[cfg(test)] +pub fn reachable_commit_tag_oids(repo_path: &Path) -> Result> { + reachable_commit_tag_oids_bounded(repo_path, "git", WALK_TIMEOUT) +} + +/// [`reachable_commit_tag_oids`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` commit/tag gate. One deadline spans the whole walk. +/// +/// Reachable commits come from bounded `git rev-list --all` (+ HEAD for the +/// detached case). Unlike the blob allowed-set, this does NOT run +/// `assert_all_refs_are_commits`: that guard fail-closes a repo's whole walk when +/// any ref peels to a non-commit (an annotated tag of a tree is pushable through +/// receive-pack), which would 404 every reachable commit/tag CID here for a +/// legitimate reader. The guard exists to stop blob/tree UNDER-withholding; it is +/// unnecessary for reachability, since a dangling object is absent from +/// `rev-list --all` and the ref walk below regardless of odd refs — so dropping it +/// recovers availability without admitting any dangling object (no leak). +/// +/// Reachable tag OBJECTS: `rev-list --all` dereferences annotated tags to commits, +/// so the tag objects are absent from it. Collect them by walking every ref tip and +/// peeling each tag's chain, so a nested tag-of-a-tag's INNER tag object (reachable +/// and pinnable, but not itself a ref tip) is included too. Fails closed on any git +/// error. +pub fn reachable_commit_tag_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, +) -> Result> { + let deadline = Instant::now() + timeout; + // Reachable commits — no ref-commit assertion (see docstring). The HEAD probe + // doubles as the seed source for the tag-valued detached HEAD below: + // `rev-parse --verify HEAD` returns the tag oid UNPEELED when HEAD names a tag + // object. Failing to resolve HEAD (unborn/absent) is not fatal — there is + // simply no HEAD to walk or seed. + let head_oid: Option = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .ok() + .map(|out| String::from_utf8_lossy(&out).trim().to_string()) + .filter(|s| !s.is_empty()); + let mut rev_args = vec!["rev-list", "--all"]; + if head_oid.is_some() { + rev_args.push("HEAD"); + } + let rev = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let mut set: HashSet = String::from_utf8_lossy(&rev) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect(); + + // Ref tips that are annotated tag objects seed the tag-chain walk. + let refs = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(objectname) %(objecttype)"], + repo_path, + b"", + deadline, + )?; + let mut worklist: Vec = Vec::new(); + for line in String::from_utf8_lossy(&refs).lines() { + let mut it = line.split_whitespace(); + if let (Some(oid), Some("tag")) = (it.next(), it.next()) { + worklist.push(oid.to_string()); + } + } + // A detached/direct HEAD may name an annotated tag object with no ref at that tag + // (#173 review, finding 3): `rev-list --all HEAD` above peels it to its commit and + // `for-each-ref` has no tag row, so the tag OBJECT would be omitted and its pinned + // CID would 404 for an authorized reader. Seed a tag-valued HEAD into the tag-chain + // walk; a `commit` HEAD adds nothing. A cat-file failure here only skips the seed + // (over-withholds that one tag — fail-closed), matching the original's tolerance. + if let Some(head_oid) = head_oid { + if let Ok(ty) = run_bounded_git( + git_bin, + &["cat-file", "-t", &head_oid], + repo_path, + b"", + deadline, + ) { + if String::from_utf8_lossy(&ty).trim() == "tag" { + worklist.push(head_oid); + } + } + } + // Peel every tag object's chain into `set`, adding each tag object it passes + // through. Bounded and batched (#173, jatmn tag fan-out): see `walk_tag_chain`. + walk_tag_chain( + repo_path, + git_bin, + worklist, + &mut set, + MAX_TAG_OBJECTS, + deadline, + )?; + Ok(set) +} + /// Objects safe to replicate, failing closed on blobs (#99). A candidate /// replicates iff it is NOT a blob (`all_blob_oids` — commits and trees are /// structural, never content-withheld) OR it is in `allowed_blobs` (reachable @@ -365,14 +1142,28 @@ pub fn replicable_objects_fail_closed( /// owner plus any reader DID that `visibility_check` Allows at some path the /// blob appears at. Least-privilege: a reader of one private subtree is not a /// recipient of a blob that only lives in another. +#[cfg(test)] pub fn withheld_blob_recipients( repo_path: &Path, rules: &[VisibilityRule], is_public: bool, owner_did: &str, +) -> Result>> { + withheld_blob_recipients_bounded(repo_path, "git", WALK_TIMEOUT, rules, is_public, owner_did) +} + +/// [`withheld_blob_recipients`] with an injectable `git_bin` and walk `timeout`, for +/// the receive-pack encrypt-then-pin path. +pub fn withheld_blob_recipients_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, ) -> Result>> { // One history walk feeds both the withheld set and the recipient mapping. - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, git_bin, timeout)?; let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None); if withheld.is_empty() { return Ok(HashMap::new()); @@ -402,44 +1193,324 @@ pub fn withheld_blob_recipients( #[cfg(test)] mod tests { use super::*; - use crate::db::VisibilityMode; - use chrono::Utc; - use std::process::Command; - use tempfile::TempDir; - fn rule(path_glob: &str, readers: &[&str]) -> VisibilityRule { - VisibilityRule { - id: "x".into(), - repo_id: "r1".into(), - path_glob: path_glob.into(), - mode: VisibilityMode::B, - reader_dids: readers.iter().map(|s| s.to_string()).collect(), - created_by: "did:key:zOwner".into(), - created_at: Utc::now(), + /// Write an executable fake `git` shell script into `dir` and return its path, + /// so a test can drive the walk's process-group teardown without a real git and + /// without mutating the process-global PATH (the crate's only injection seam). + #[cfg(unix)] + fn write_fake_git(dir: &Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 U3: the withheld-blob walk is bounded at the shared `blob_paths` seam, so + /// a hung git child cannot pin the caller's permit past the deadline. A fake git + /// that hangs on `rev-list` must make `blob_paths` return `GitServiceTimeout` + /// within the watchdog budget (not block for the child's lifetime), and the + /// child's process group must be reaped (its recorded leader PID gone). Every + /// caller (upload-pack serve, receive-pack replication) funnels through + /// `blob_paths`, so this seam-level proof covers both permit pools. Neutralize + /// the watchdog SIGTERM and this hangs past the recv budget (RED). + #[cfg(unix)] + #[test] + fn blob_paths_times_out_and_reaps_a_hung_walk() { + use std::time::Duration; + let tmp = TempDir::new().unwrap(); + // Fast on every stage except rev-list, which records its own (group-leader) + // PID and then hangs. `sleep 30` bounds the worst case if the watchdog is + // ever broken, so a regression cannot wedge the suite for 300s. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > revlist.pid ; sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + + // Run the walk on a thread with a short budget; the recv_timeout succeeding + // is itself proof the walk did not block on the hung child. + let (tx, rx) = mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(blob_paths(&path, &git_bin, Duration::from_millis(200))); + }); + let result = rx.recv_timeout(Duration::from_secs(10)).expect( + "blob_paths must return within the watchdog budget, not hang on a stuck git child", + ); + let err = result.expect_err("a hung rev-list must abort the walk with an error"); + assert!( + err.downcast_ref::() + .is_some(), + "a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err}" + ); + + // The recorded process-group leader must be gone: the watchdog reaps the + // whole group before blob_paths returns, so no orphaned git lingers. + let pid: i32 = std::fs::read_to_string(tmp.path().join("revlist.pid")) + .expect("the fake git must have recorded its rev-list PID") + .trim() + .parse() + .expect("recorded PID must parse"); + let mut gone = false; + for _ in 0..200 { + // SAFETY: kill(2) with signal 0 only probes existence; ESRCH (-1) means + // the process is gone. Borrows no Rust memory. + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); } + assert!( + gone, + "the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts" + ); } - const OWNER: &str = "did:key:zOwner"; + /// #174 (F1 status-gate, vetted by execution): a child that exits SUCCESSFULLY is + /// never reported as a timeout even when the watchdog fires, so a walk finishing + /// right at its deadline is not a spurious 504. The fake only exits when signalled + /// and exits 0 on SIGTERM, so with a deadline already elapsed the watchdog always + /// reaches its kill path (killed == true) yet the child's status is success. + /// Drop the `!status.success()` guard and this returns GitServiceTimeout (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_success_at_the_deadline_is_not_a_timeout() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap 'exit 0' TERM\nsleep 30 &\nwait\n"; + let git_bin = write_fake_git(tmp.path(), body); + let out = run_bounded_git( + &git_bin, + &["rev-list"], + tmp.path(), + b"", + Instant::now() + Duration::from_millis(100), + ); + assert!( + out.is_ok(), + "a child that exited successfully must not be reported as a timeout even if the watchdog fired: {out:?}" + ); + } - /// Build a bare repo with public/a.txt and secret/b.txt at one commit. - /// Returns (tempdir, bare_path, secret_blob_oid, public_blob_oid). - fn fixture() -> (TempDir, std::path::PathBuf, String, String) { - let td = TempDir::new().unwrap(); - let work = td.path().join("work"); - let bare = td.path().join("bare.git"); - let run = |args: &[&str], dir: &Path| { - let ok = Command::new("git") - .args(args) - .current_dir(dir) - .status() - .unwrap() - .success(); - assert!(ok, "git {args:?} failed"); - }; - std::fs::create_dir_all(work.join("public")).unwrap(); - std::fs::create_dir_all(work.join("secret")).unwrap(); - std::fs::write(work.join("public/a.txt"), b"public bytes\n").unwrap(); - std::fs::write(work.join("secret/b.txt"), b"TOP SECRET\n").unwrap(); + /// #174 (F3, vetted by execution): a child that IGNORES SIGTERM is still reaped + /// via the watchdog's SIGKILL escalation, so it cannot pin the walk thread or its + /// permit. The fake traps SIGTERM and keeps sleeping; run_bounded_git must still + /// return (via SIGKILL at the grace step) with a timeout error and the group must + /// be gone. (A truly uninterruptible D-state child, which no signal can reap, is + /// the documented residual this teardown, like the async twin, cannot cover.) + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_sigterm_ignoring_child_via_sigkill() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nwhile true; do sleep 1; done\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return via SIGKILL even for a SIGTERM-ignoring child"); + assert!( + out.is_err(), + "a SIGTERM-ignoring child killed by SIGKILL is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running" + ); + } + + /// #174 finding 3 (jatmn/CodeRabbit): a group MEMBER that ignores SIGTERM must + /// still be SIGKILLed even when the group LEADER exits cleanly on SIGTERM. The + /// leader traps SIGTERM to exit 0, but first spawns a descendant (`sh -c`, so its + /// `$$` is its OWN pid — a `( )` subshell's `$$` is the parent's) that ignores + /// SIGTERM and closes its inherited stdout/stderr. When the watchdog SIGTERMs the + /// group, the leader exits, its stdout closes, the main drain unblocks, and the + /// leader is reaped — the exact window a `reaped`-gated watchdog stands down in, + /// before escalating to SIGKILL. The descendant must be dead when run_bounded_git + /// returns; a teardown that stands down on leader-reap leaves it running (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // Both loops are bounded (~30s) so a broken teardown cannot leak a permanent + // orphan or wedge the suite; the assertion fires well before then. + let body = "#!/bin/sh\n\ +case \"$1\" in\n\ + rev-list)\n\ + sh -c 'trap \"\" TERM; echo $$ > desc.pid; exec 1>&- 2>&-; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + trap 'exit 0' TERM\n\ + i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n\ + *) : ;;\n\ +esac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let _ = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return within the watchdog budget"); + + // Wait for the descendant to record its OWN pid, then assert it is gone. + let desc_pid_path = tmp.path().join("desc.pid"); + let mut desc: Option = None; + for _ in 0..200 { + if let Some(p) = std::fs::read_to_string(&desc_pid_path) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(desc, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(desc, libc::SIGKILL) }; + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned" + ); + } + + /// #174 U1 (P1-a, RED-before/GREEN-after): the group LEADER closes its own + /// stdout/stderr BEFORE the deadline and then keeps running. On the pre-fix code + /// the stdout drain returns EOF early, `done_tx.send` stands the watchdog down + /// before it ever fires (`recv` gets `Ok` -> `false`, no kill), and `child.wait()` + /// then blocks on the still-alive leader — pinning the walk thread and its read/ + /// write permit past the deadline, bypassing GITLAWB_GIT_SERVICE_TIMEOUT_SECS. + /// This is distinct from the descendant case above: there the leader sleeps until + /// the deadline so the watchdog DOES time out; here the drain-EOF races ahead of + /// the deadline. The fix keeps the watchdog armed until the child is actually + /// reaped, so the deadline SIGTERM still fires and the call returns within budget. + /// A pre-fix build blocks on `child.wait()` past the recv budget (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // rev-list records its (leader) pid, closes stdout+stderr so the drain EOFs + // immediately, then sleeps without trapping TERM. `sleep 30` bounds the worst + // case so a RED run cannot wedge the suite; the recv budget fires first. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > leader.pid; exec 1>&- 2>&-; sleep 30 ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx.recv_timeout(Duration::from_secs(10)).expect( + "run_bounded_git must return within the watchdog budget when the leader closes stdout then hangs, not block on child.wait()", + ); + assert!( + out.is_err(), + "a leader killed at the deadline (no TERM trap) is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("leader.pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(pid, libc::SIGKILL) }; + assert!( + gone, + "the hung leader (pid {pid}) must be killed and reaped at the deadline, not left running" + ); + } + + use crate::db::VisibilityMode; + use chrono::Utc; + use std::process::Command; + use tempfile::TempDir; + + fn rule(path_glob: &str, readers: &[&str]) -> VisibilityRule { + VisibilityRule { + id: "x".into(), + repo_id: "r1".into(), + path_glob: path_glob.into(), + mode: VisibilityMode::B, + reader_dids: readers.iter().map(|s| s.to_string()).collect(), + created_by: "did:key:zOwner".into(), + created_at: Utc::now(), + } + } + + const OWNER: &str = "did:key:zOwner"; + + /// Build a bare repo with public/a.txt and secret/b.txt at one commit. + /// Returns (tempdir, bare_path, secret_blob_oid, public_blob_oid). + fn fixture() -> (TempDir, std::path::PathBuf, String, String) { + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| { + let ok = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(); + assert!(ok, "git {args:?} failed"); + }; + std::fs::create_dir_all(work.join("public")).unwrap(); + std::fs::create_dir_all(work.join("secret")).unwrap(); + std::fs::write(work.join("public/a.txt"), b"public bytes\n").unwrap(); + std::fs::write(work.join("secret/b.txt"), b"TOP SECRET\n").unwrap(); run(&["init", "-q"], &work); run(&["config", "user.email", "t@t"], &work); run(&["config", "user.name", "t"], &work); @@ -468,6 +1539,614 @@ mod tests { (td, bare, secret, public) } + /// #173 (jatmn round 8, F4 — load-bearing): a repo with enough annotated tags that + /// one `cat-file --batch` round fills BOTH pipes (stdin > 64 KiB of oids while the + /// child blocks on a full stdout) must not deadlock. The old order wrote the whole + /// round to stdin before draining stdout and hung indefinitely, stranding a blocking- + /// pool thread; `run_bounded_git`'s concurrent writer/drain completes. Driven with a + /// completion timeout: GREEN finishes in well under a second, RED (old order) hangs + /// and the recv_timeout fires. ~3000 tags is well past the ~2030-oid deadlock + /// threshold (41 bytes/oid, 64 KiB pipes) and under MAX_TAG_OBJECTS (8192). + /// Bulk-created via one fast-import stream so the fixture cost is one git process, + /// not 3000 `git tag -a` spawns. + #[test] + fn walk_tag_chain_large_batch_does_not_deadlock() { + use std::io::Write; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(&work).unwrap(); + std::fs::write(work.join("f.txt"), b"x\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + let head = { + let out = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Bulk-create ~3000 annotated tags via one fast-import stream. + const N: usize = 3000; + let mut stream = String::new(); + for i in 0..N { + let msg = format!("annotated tag {i}\n"); + stream.push_str(&format!("tag t{i}\n")); + stream.push_str(&format!("from {head}\n")); + stream.push_str("tagger t 1700000000 +0000\n"); + stream.push_str(&format!("data {}\n", msg.len())); + stream.push_str(&msg); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&work) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + + // Drive the walk on a worker thread with a completion timeout. The old + // write-all-before-drain order hangs here; the fix completes near-instantly. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(reachable_commit_tag_oids(&bare).map(|s| s.len())); + }); + match rx.recv_timeout(std::time::Duration::from_secs(20)) { + Ok(Ok(n)) => assert!( + n >= N, + "the walk must resolve every annotated tag object (got {n}, expected >= {N})" + ), + Ok(Err(e)) => panic!("walk errored: {e}"), + Err(_) => panic!("walk_tag_chain deadlocked on a large tag batch (F4 regression)"), + } + } + + /// #173 review (finding 3): an annotated tag reachable ONLY through a tag-valued + /// detached HEAD (raw HEAD naming a tag object, with no ref at that tag) must still + /// enter `reachable_commit_tag_oids`. `rev-list --all HEAD` peels such a HEAD to its + /// commit and `for-each-ref` has no tag row, so without a HEAD tag-seed the tag + /// OBJECT is omitted and its pinned CID would 404 for an authorized reader. RED + /// before the HEAD tag-seed (the tag oid is absent); GREEN after. + #[test] + fn reachable_commit_tag_oids_includes_tag_valued_detached_head() { + use std::io::Write; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::create_dir_all(&work).unwrap(); + std::fs::write(work.join("a.txt"), b"hi\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "seed"], &work); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + let commit = run(&["rev-parse", "HEAD"], &bare); + + // An annotated tag OBJECT in the bare ODB, with NO ref pointing at it. + let tag_body = format!( + "object {commit}\ntype commit\ntag htag\ntagger t 0 +0000\n\nHEAD-only tag\n" + ); + let tag_oid = { + let mut child = Command::new("git") + .args(["hash-object", "-t", "tag", "-w", "--stdin"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .as_mut() + .unwrap() + .write_all(tag_body.as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + assert_eq!(run(&["cat-file", "-t", &tag_oid], &bare), "tag"); + // Raw-write HEAD directly to the tag object (the only way this state arises; + // update-ref / checkout both refuse a non-commit HEAD). + std::fs::write(bare.join("HEAD"), format!("{tag_oid}\n")).unwrap(); + + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!( + set.contains(&tag_oid), + "a tag reachable only via a tag-valued detached HEAD must be in the reachable set" + ); + assert!( + set.contains(&commit), + "the commit the HEAD tag peels to stays reachable (no regression)" + ); + } + + /// #173: `reachable_commit_tag_oids` on an empty repo (unborn HEAD) must return an + /// empty set, not error — exercising the `rev-parse HEAD` fail branch of the + /// detached-HEAD tag seed (there is simply no HEAD to seed). + #[test] + fn reachable_commit_tag_oids_handles_unborn_head() { + let td = TempDir::new().unwrap(); + let bare = td.path().join("empty.git"); + let ok = Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success(); + assert!(ok, "git init --bare failed"); + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!( + set.is_empty(), + "an empty repo (unborn HEAD) yields an empty reachable set with no error" + ); + } + + #[test] + fn object_paths_emits_trees_and_blob_paths_is_the_blob_slice() { + let (_td, bare, secret_oid, public_oid) = fixture(); + let deadline = Instant::now() + WALK_TIMEOUT; + // The lenient enumeration; on this clean fixture it matches the strict one. + let commits = reachable_commit_oids(&bare, "git", deadline).unwrap(); + let objs = object_paths(&bare, "git", &commits, deadline).unwrap(); + + // Blob records survive the `-rzt` change, at their paths (unchanged). + assert!(objs.contains(&(secret_oid.clone(), "/secret/b.txt".into(), "blob".into()))); + assert!(objs.contains(&(public_oid.clone(), "/public/a.txt".into(), "blob".into()))); + + // The #135 addition: subtree tree objects at their directory paths. + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/secret"), + "the /secret subtree tree must be emitted at its dir path" + ); + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/public"), + "the /public subtree tree must be emitted at its dir path" + ); + + // blob_paths must equal the blob slice of object_paths exactly — compared as + // SETS (both walks dedup via HashSet; the collected order is nondeterministic). + let bp: HashSet<(String, String)> = blob_paths(&bare, "git", WALK_TIMEOUT) + .unwrap() + .into_iter() + .collect(); + let bp_from_obj: HashSet<(String, String)> = objs + .iter() + .filter(|(_, _, k)| k == "blob") + .map(|(o, p, _)| (o.clone(), p.clone())) + .collect(); + assert_eq!( + bp, bp_from_obj, + "blob_paths output must be byte-identical to object_paths' blob slice" + ); + } + + #[test] + fn allowed_tree_set_gates_withheld_subtree_tree() { + let (_td, bare, _s, _p) = fixture(); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let secret_tree = oid("HEAD:secret"); + let public_tree = oid("HEAD:public"); + let root_tree = oid("HEAD^{tree}"); + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", &[reader])]; + + // anon: the withheld /secret tree is excluded; root ("/") and /public are in. + let anon = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, None).unwrap(); + assert!( + !anon.contains(&secret_tree), + "withheld /secret subtree tree excluded for anon" + ); + assert!(anon.contains(&root_tree), "root tree included (path /)"); + assert!(anon.contains(&public_tree), "/public subtree tree included"); + + // listed reader: sees the /secret tree (caller-aware, not a blanket deny). + let rd = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(reader)).unwrap(); + assert!( + rd.contains(&secret_tree), + "listed reader sees the /secret tree" + ); + + // owner: sees every reachable tree. + let ow = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(OWNER)).unwrap(); + assert!( + ow.contains(&secret_tree) && ow.contains(&public_tree) && ow.contains(&root_tree), + "owner sees all reachable trees" + ); + } + + #[test] + fn allowed_tree_set_excludes_dangling_tree() { + use std::io::Write; + let (_td, bare, secret_oid, _p) = fixture(); + // A DANGLING tree: written to the ODB but referenced by no commit. Uses a + // UNIQUE entry name so its oid is content-distinct from every reachable tree + // (a content-identical tree would dedup to a reachable oid — that is T2, not + // danglingness). The reachable-only walk never enumerates it -> fail closed. + let mut child = Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {secret_oid}\tdangling-only-unreferenced.txt" + ) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "git mktree"); + let dangling = String::from_utf8_lossy(&out.stdout).trim().to_string(); + + let rules = [rule("/secret/**", &[])]; + for caller in [None, Some(OWNER)] { + let set = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, caller).unwrap(); + assert!( + !set.contains(&dangling), + "dangling tree must never be in the reachable allowed-set (caller={caller:?})" + ); + } + } + + #[test] + fn allowed_tree_set_includes_tree_shared_across_allowed_and_denied_paths() { + // T2 (content-dedup): the SAME tree oid reachable at both an allowed and a + // withheld path is INCLUDED for anon (allowed-wins) — its structure is + // visible to the caller at the allowed path. Mirrors the blob analog + // `same_blob_at_allowed_and_denied_path_is_not_withheld`. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(work.join("pub/sub")).unwrap(); + std::fs::create_dir_all(work.join("sec/sub")).unwrap(); + std::fs::write(work.join("pub/sub/f.txt"), b"same bytes\n").unwrap(); + std::fs::write(work.join("sec/sub/f.txt"), b"same bytes\n").unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + run(&["add", "."]); + run(&["commit", "-qm", "seed"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let pub_sub = oid("HEAD:pub/sub"); + let sec_sub = oid("HEAD:sec/sub"); + assert_eq!(pub_sub, sec_sub, "identical content dedups to one tree oid"); + + // Withhold /sec from anon; the shared oid is still reachable at /pub/sub. + let rules = [rule("/sec/**", &[])]; + let anon = allowed_tree_set_for_caller(&work, &rules, true, OWNER, None).unwrap(); + assert!( + anon.contains(&pub_sub), + "a tree reachable at an allowed path is included even when also at a withheld path" + ); + } + + #[test] + fn allowed_tree_set_includes_root_trees_of_all_reachable_commits() { + // The batched root-tree pass (root_tree_pairs) must return EVERY reachable + // commit's root tree, not just HEAD's — two commits with distinct root trees + // both land in the set. Guards the git-log-over-N-commits root derivation. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::write(work.join("a.txt"), b"one\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c1"]); + let root1 = oid("HEAD^{tree}"); + std::fs::write(work.join("b.txt"), b"two\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c2"]); + let root2 = oid("HEAD^{tree}"); + assert_ne!(root1, root2, "the two commits have distinct root trees"); + + // Public repo, no rules: every reachable tree is allowed for anon. + let set = allowed_tree_set_for_caller(&work, &[], true, OWNER, None).unwrap(); + assert!( + set.contains(&root1) && set.contains(&root2), + "root trees of BOTH reachable commits are in the set (batched root pass)" + ); + } + + #[test] + fn root_tree_pairs_returns_every_root_tree_at_scale() { + // Parity + liveness at scale for root_tree_pairs (#173 P2): feed every + // reachable commit oid to `git log --format=%T --stdin` and collect each + // commit's root tree. With N commits that is ~N*41 bytes of oids in and + // ~N*41 bytes of %T out — past the ~64 KiB pipe buffer in both directions — + // so this exercises the large-bidirectional-IO path the 2-commit test above + // cannot, and asserts parity: every distinct root tree comes back. + // + // NOTE: this is NOT a deadlock guard. `git log --stdin` reads its whole + // revision list to EOF before emitting any %T, so the naive "write all of + // stdin, then drain stdout" form does not deadlock at any scale for this + // invocation. `run_bounded_git`'s concurrent writer/drain is cheap defensive + // isolation, not load-bearing, and this test does not claim otherwise. The + // 30s watchdog is a general liveness bound so a future regression that + // genuinely hangs fails fast here rather than stalling the suite. + const N: usize = 2500; + let td = TempDir::new().unwrap(); + let bare = td.path().join("many.git"); + assert!(Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success()); + + // fast-import a linear chain of N commits, each adding a distinct file so + // every root tree is distinct (dedup cannot shrink the output). One + // subprocess, ~1s — far cheaper than N `git commit` spawns. + let mut stream = String::new(); + for i in 0..N { + let (b, cm) = (2 * i + 1, 2 * i + 2); + let content = format!("v{i}"); + let msg = format!("c{i}"); + stream.push_str(&format!( + "blob\nmark :{b}\ndata {}\n{content}\n", + content.len() + )); + stream.push_str(&format!( + "commit refs/heads/main\nmark :{cm}\ncommitter t 0 +0000\ndata {}\n{msg}\n", + msg.len() + )); + if i > 0 { + stream.push_str(&format!("from :{}\n", 2 * (i - 1) + 2)); + } + stream.push_str(&format!("M 100644 :{b} f{i}\n\n")); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + { + use std::io::Write; + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + } + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + let commits = reachable_commit_oids(&bare, "git", Instant::now() + WALK_TIMEOUT).unwrap(); + assert_eq!(commits.len(), N, "all {N} commits reachable"); + + // Call root_tree_pairs directly (private, same module) under a liveness + // watchdog, then assert it returned every distinct root tree. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send( + root_tree_pairs(&bare, "git", &commits, Instant::now() + WALK_TIMEOUT) + .map(|s| s.len()), + ); + }); + match rx.recv_timeout(std::time::Duration::from_secs(30)) { + Ok(Ok(len)) => assert_eq!(len, N, "every distinct root tree returned"), + Ok(Err(e)) => panic!("root_tree_pairs errored: {e}"), + Err(_) => panic!("root_tree_pairs did not return within 30s"), + } + } + + /// #173 (jatmn tag fan-out): the batched `git cat-file --batch` tag walk must + /// return the SAME reachable set as the old per-tag `cat-file tag` loop — every + /// commit, the outer tag object, AND the inner tag object of a tag-of-a-tag chain + /// (the inner tag is reachable but is not itself a ref tip, so it is only found by + /// peeling the outer tag's target). Behavior-preservation proof for the rewrite. + #[test] + fn reachable_commit_tag_oids_includes_nested_tag_objects() { + let (_td, bare, _secret, _public) = fixture(); + let run = |args: &[&str]| -> String { + let out = Command::new("git") + .args(args) + .current_dir(&bare) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + // v1 -> commit, v2 -> v1 (tag-of-a-tag), plus a couple of sibling tags so the + // round batches more than one oid. Capture v1's oid, then DELETE the v1 ref so + // the inner tag object survives in the ODB but is NOT a ref tip: it is then + // reachable ONLY by peeling v2's target chain. That makes the peel load-bearing + // (breaking the inner-tag enqueue drops v1 from the set), unlike leaving v1 as + // its own ref where `for-each-ref` would seed it directly. + run(&["tag", "-a", "-m", "inner", "v1", "HEAD"]); + run(&["tag", "-a", "-m", "outer", "v2", "v1"]); + run(&["tag", "-a", "-m", "s1", "s1", "HEAD"]); + run(&["tag", "-a", "-m", "s2", "s2", "HEAD"]); + let commit = run(&["rev-parse", "HEAD"]); + let v1 = run(&["rev-parse", "v1"]); + let v2 = run(&["rev-parse", "v2"]); + let s1 = run(&["rev-parse", "s1"]); + let s2 = run(&["rev-parse", "s2"]); + run(&["tag", "-d", "v1"]); + + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!(set.contains(&commit), "the commit must be reachable"); + assert!( + set.contains(&v2), + "the outer tag object (ref tip) must be present" + ); + assert!( + set.contains(&v1), + "the INNER tag object of a tag-of-a-tag must be present (peeled from v2, no ref)" + ); + assert!(set.contains(&s1), "sibling tag s1 must be present"); + assert!(set.contains(&s2), "sibling tag s2 must be present"); + } + + /// #173 (jatmn tag fan-out): the object bound is load-bearing. A repo whose tag + /// count exceeds the bound must FAIL CLOSED (Err), not return a truncated set that + /// would under-withhold a still-reachable tag. Drives `walk_tag_chain` with a tiny + /// injected bound (the public fn uses the real `MAX_TAG_OBJECTS`); with the bound + /// check removed this would collect all tags and return Ok. + #[test] + fn walk_tag_chain_fails_closed_over_object_bound() { + let (_td, bare, _secret, _public) = fixture(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&bare) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let mut seeds = Vec::new(); + for n in 0..5 { + let name = format!("t{n}"); + run(&["tag", "-a", "-m", &name, &name, "HEAD"]); + let oid = Command::new("git") + .args(["rev-parse", &name]) + .current_dir(&bare) + .output() + .unwrap(); + seeds.push(String::from_utf8_lossy(&oid.stdout).trim().to_string()); + } + + // Within a generous bound: the walk succeeds and collects the tags. + let mut ok_set = HashSet::new(); + walk_tag_chain( + &bare, + "git", + seeds.clone(), + &mut ok_set, + 8192, + Instant::now() + WALK_TIMEOUT, + ) + .unwrap(); + assert!( + seeds.iter().all(|s| ok_set.contains(s)), + "all 5 tags collected under a generous bound" + ); + + // Under a bound of 2 with 5 tags: fail closed (Err), not a partial set. + let mut small_set = HashSet::new(); + let result = walk_tag_chain( + &bare, + "git", + seeds, + &mut small_set, + 2, + Instant::now() + WALK_TIMEOUT, + ); + assert!( + result.is_err(), + "a tag count exceeding the object bound must fail closed (Err), not truncate" + ); + } + #[test] fn anonymous_caller_withholds_only_private_blob() { let (_td, bare, secret_oid, public_oid) = fixture(); @@ -735,7 +2414,12 @@ mod tests { String::from_utf8_lossy(&out.stdout).trim().to_string() }; - let all_blobs = crate::git::push_delta::all_blob_oids(&work).unwrap(); + let all_blobs = crate::git::push_delta::all_blob_oids( + &work, + "git", + std::time::Instant::now() + std::time::Duration::from_secs(600), + ) + .unwrap(); assert!( all_blobs.contains(&dangling_oid), "precondition: the dangling blob is in the all-objects universe" diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3b346190..3d2f13df 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -9,6 +9,122 @@ use anyhow::Result; use gitlawb_core::cid::Cid; +use std::time::Duration; + +/// Attempts (including the first) for a transient DB-record retry. +const PIN_RECORD_ATTEMPTS: u32 = 3; +/// Backoff between DB-record retry attempts. +const PIN_RECORD_BACKOFF: Duration = Duration::from_millis(50); + +/// Run an idempotent DB-record operation with a bounded retry so a sub-second +/// transient error does not silently leave the pin-source set permanently +/// incomplete. The resolver treats a nonempty below-cap source set as complete, +/// so a dropped `record_pin_source`/`record_pinned_cid` makes `GET /ipfs/{cid}` +/// 404 a valid public copy. Every wrapped insert is idempotent (`ON CONFLICT DO +/// NOTHING` / provenance-preserving upsert), so re-running is safe. On exhausted +/// attempts the last error is returned and the caller keeps its warn — behavior +/// degrades to the pre-retry state, not worse. Process death mid-retry or a DB +/// outage outlasting the backoff horizon leaves the same residual hole (no +/// persisted marker to reconcile from at startup), retired only by a future +/// reconciliation sweep. Runs inside the already-detached post-push task, so the +/// backoff adds no push latency. +async fn retry_db_record(mut op: F) -> Result<()> +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut attempt = 1; + loop { + match op().await { + Ok(()) => return Ok(()), + Err(e) => { + if attempt >= PIN_RECORD_ATTEMPTS { + return Err(e); + } + tokio::time::sleep(PIN_RECORD_BACKOFF).await; + attempt += 1; + } + } + } +} + +/// Opportunistically repair a legacy provider-CID row on the already-pinned skip +/// path (#173 R8, KTD8). Releases before this branch stored the PROVIDER CID +/// (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`; the `/ipfs` resolver +/// recomputes the raw CID from object bytes and 404s any row whose key does not +/// match, yet `list_pinned_cids` still advertises the stored key — so a client +/// gets a CID the resolver deliberately withholds. When a re-push carries the +/// object again, rewrite the key to the raw CID and stash the old provider value +/// in `legacy_provider_cid`. +/// +/// COST GATE: candidacy is decided from the stored key's codec alone — a +/// CIDv1/raw key is already the resolver key and reads NO bytes, keeping the +/// steady-state skip cost DB-only. Only a legacy-codec row reads the object to +/// recompute. A row whose bytes are gone stays withheld (no destructive rewrite). +async fn repair_legacy_provider_cid( + repo_path: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + sha: &str, + db: &crate::db::Db, +) -> Result<()> { + let stored = match db.cid_for_oid(sha).await? { + Some(c) => c, + None => return Ok(()), + }; + // Cost gate: a canonical raw CIDv1 key is already correct — never read bytes. + if gitlawb_core::cid::is_raw_cidv1(&stored) { + return Ok(()); + } + // Legacy-codec row: read the object to recompute. Counted so a test can prove + // the gate above spares non-legacy rows this read. + #[cfg(test)] + note_legacy_repair_read(); + let data = match crate::git::store::read_object_bounded(git_bin, repo_path, sha, git_timeout) { + Ok(Some((_ty, bytes))) => bytes, + // Bytes gone: the row stays withheld, never destructively rewritten. + Ok(None) => return Ok(()), + // A wedged/D-state `git cat-file` (timeout/infra): the repair is opportunistic + // and best-effort, so skip it and return Ok so the pin task PROCEEDS to + // requeue_or_release rather than hanging the coalescing key until process death + // (grok F2, #173). A later re-push or the deferred sweep retries the repair. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "skipping legacy provider-CID repair: bounded object read failed"); + return Ok(()); + } + }; + let raw = Cid::from_git_object_bytes(&data).to_string(); + if raw == stored { + return Ok(()); + } + db.repair_legacy_provider_cid(sha, &raw, &stored).await +} + +// Test-only cost-gate counter (R8, U7): how many times the opportunistic repair +// read an object's bytes on the skip path. The codec gate must spare a CIDv1/raw +// row this read; the counter is the both-ways guard (removing the gate reads the +// raw row and increments it). Same thread_local discipline as the serve-path +// oversize counter — the pin tests await `pin_new_objects` on a current-thread +// runtime, so the increment and the assertion share one thread. +#[cfg(test)] +thread_local! { + static LEGACY_REPAIR_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_legacy_repair_reads() { + LEGACY_REPAIR_READS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn legacy_repair_reads() -> usize { + LEGACY_REPAIR_READS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_legacy_repair_read() { + LEGACY_REPAIR_READS.with(|c| c.set(c.get() + 1)); +} /// Pin a single git object to the local IPFS/Kubo node. /// @@ -91,14 +207,18 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { /// `..._fail_closed` filter on the full-scan path before calling, so this /// function never sees a withheld blob. `repo_path` is still needed to read each /// object's bytes. The twin in `pinata.rs` mirrors this shape — change both in -/// lockstep. +/// lockstep. `repo_id` records the pin's provenance so `GET /ipfs/{cid}` resolves +/// straight to this repo instead of scanning every repo (#173). /// /// Returns a list of `(sha256_hex, cid)` pairs for objects pinned this call. pub async fn pin_new_objects( ipfs_api: &str, repo_path: &std::path::Path, + git_bin: &str, + git_timeout: Duration, object_list: Vec, db: &crate::db::Db, + repo_id: &str, ) -> Vec<(String, String)> { if ipfs_api.is_empty() { return vec![]; @@ -107,9 +227,46 @@ pub async fn pin_new_objects( let mut pinned = Vec::new(); for sha in object_list { - // Skip if already pinned + // Skip if already pinned — but first backfill provenance if the existing + // pin has none. A legacy pin (recorded before repo_id existed, #173, jatmn) + // is skipped here before record_pinned_cid ever runs, so its NULL provenance + // would never resolve to one repo and known CIDs keep hitting the scan. The + // backfill only sets repo_id (AND repo_id IS NULL guard preserves + // first-pinner-owns) and never re-pins the bytes — the object is already on IPFS. match db.is_pinned(&sha).await { - Ok(true) => continue, + Ok(true) => { + match db.provenance_for_oid(&sha).await { + Ok(None) => { + if let Err(e) = db.backfill_pin_provenance(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); + } + } + Ok(Some(_)) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error reading pin provenance"); + } + } + // F1 (#173 round 8): record this repo as an ADDITIONAL source for the + // already-pinned object. This is the load-bearing skip-branch insert — + // a later repo pushing a shared object hits this path (already pinned), + // and without it `GET /ipfs/{cid}` only ever knows the first pinner, so a + // shared object first pinned from a private/quarantined repo 404s even + // when this repo would serve it. Bounded per object (MAX_PIN_SOURCES). + if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } + // R8 (#173 round 10): opportunistically repair a legacy provider-CID + // row (Kubo dag-pb / Pinata) to the raw-content resolver key on this + // re-push. Cost-gated on the stored key's codec — a non-legacy row + // reads no bytes. Warn-only: a failure leaves the row as-is for a + // later re-push or the deferred one-shot sweep. + if let Err(e) = + repair_legacy_provider_cid(repo_path, git_bin, git_timeout, &sha, db).await + { + tracing::warn!(sha = %sha, err = %e, "failed to repair legacy provider CID"); + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinned status"); @@ -117,22 +274,46 @@ pub async fn pin_new_objects( } } - // Read raw object content - let data = match crate::git::store::read_object(repo_path, &sha) { - Ok(Some((_obj_type, bytes))) => bytes, - Ok(None) => continue, - Err(e) => { - tracing::warn!(sha = %sha, err = %e, "failed to read git object for pinning"); - continue; - } - }; + // Read raw object content under a bounded read so a wedged/D-state `git + // cat-file` (stuck NFS/Tigris backend) is reaped at `git_timeout` instead of + // hanging pin_new_objects forever — which would pin the post-push coalescing + // key until process death (grok F2, #173). On Err the object is simply not + // pinned this pass; a later pass/push retries. + let data = + match crate::git::store::read_object_bounded(git_bin, repo_path, &sha, git_timeout) { + Ok(Some((_obj_type, bytes))) => bytes, + Ok(None) => continue, + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to read git object for pinning"); + continue; + } + }; // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinned_cid(&sha, &cid).await { + // The resolver key (`pinned_cids.cid`) must be the locally-computed + // raw-content CID, never the provider Hash: Kubo returns a dag-pb/UnixFS + // root for objects above its block size, which does not hash the raw + // content, so `GET /ipfs/{provider_cid}` would resolve then fail the F2 + // integrity check (list-then-404). The serve path reads bytes from git and + // verifies them against the requested CID, so the raw CID is the correct + // key. Mirrors the pinata twin, which already records the raw CID. + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + if let Err(e) = + retry_db_record(|| db.record_pinned_cid(&sha, &raw_cid, Some(repo_id))).await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } + // F1 (#173 round 8): also record the first pinner in pin_repo_sources so + // every source (first and subsequent) is tried uniformly by the resolver. + if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } + // Return the provider Hash (not the resolver key), mirroring the pinata + // twin's contract: the DB `cid` is the raw resolver key (recorded above), + // the returned value is the provider CID. Here the return is consumed only + // for logging, but keeping the twins structurally identical avoids drift. pinned.push((sha, cid)); } Ok(_) => {} @@ -144,3 +325,92 @@ pub async fn pin_new_objects( pinned } + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + // The retry helper is the load-bearing unit: it converts a sub-second + // transient DB error at the three warn-only record sites into a landed row, + // instead of a permanently incomplete pin-source set. These drive the helper + // directly against a controlled closure (the record sites take a concrete + // `&Db` over a `PgPool`, so a failing-first wrapper cannot slot in without + // changing signatures — see U6 seam note). + + #[tokio::test] + async fn retry_lands_after_transient_failures() { + let calls = Cell::new(0u32); + let result = retry_db_record(|| { + let n = calls.get() + 1; + calls.set(n); + async move { + if n < PIN_RECORD_ATTEMPTS { + Err(anyhow::anyhow!("transient failure on attempt {n}")) + } else { + Ok(()) + } + } + }) + .await; + + assert!( + result.is_ok(), + "retry lands the row after transient failures" + ); + assert_eq!( + calls.get(), + PIN_RECORD_ATTEMPTS, + "op is retried until it succeeds" + ); + } + + #[tokio::test] + async fn retry_returns_last_err_after_exhaustion() { + let calls = Cell::new(0u32); + let result = retry_db_record(|| { + let n = calls.get() + 1; + calls.set(n); + async move { Err::<(), _>(anyhow::anyhow!("attempt {n} failed")) } + }) + .await; + + let err = result.expect_err("all attempts fail so the last error surfaces"); + assert_eq!( + calls.get(), + PIN_RECORD_ATTEMPTS, + "attempts are bounded to the cap" + ); + assert_eq!( + err.to_string(), + "attempt 3 failed", + "the LAST error is returned, not the first" + ); + } + + // Happy path against a real DB: a single-attempt success lands the row, and a + // redundant call is idempotent (`ON CONFLICT DO NOTHING`), so the source set + // holds exactly one row. + #[sqlx::test] + async fn retry_records_pin_source_once(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let sha = "a".repeat(64); + let repo_id = "repo-retry-1"; + + retry_db_record(|| db.record_pin_source(&sha, repo_id)) + .await + .expect("happy-path record succeeds in one attempt"); + retry_db_record(|| db.record_pin_source(&sha, repo_id)) + .await + .expect("a redundant record is idempotent"); + + let sources = db.pin_sources_for_oid(&sha).await.unwrap(); + assert_eq!( + sources, + vec![repo_id.to_string()], + "exactly one source row lands under ON CONFLICT DO NOTHING" + ); + } +} diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..1ed45929 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -374,11 +374,83 @@ async fn main() -> Result<()> { rate_limiter, create_ip_rate_limiter, push_rate_limiter, + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + // The legacy-probe budget is operator-tunable via GITLAWB_IPFS_MAX_REPOS_WALKED + // (R5); the history-walk ceiling above stays constant (a smaller value false-503s + // a provenanced request). Default 256 preserves the shipped behaviour. + ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, sync_trigger_rate_limiter, peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), + // Anon receive-pack advertisements get their OWN pool, same size as the + // write pool but disjoint, so filling it (which takes many source IPs, each + // capped by git_push_advert_per_caller) never occupies a permit the + // authenticated POST needs (#174). + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), + // Bounds concurrent detached post-push encryption walks, sized from the push + // pool (no separate knob — Q1): completed pushes cannot outnumber active + // encryption walks past this (#174 P1-e). + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), + // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher + // cannot grow the outstanding parked-waiter set past one task per repo (#174 + // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a + // sized pool. + encrypt_inflight: crate::state::EncryptInflight::new(), + git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.max_concurrent_reads_per_caller, + ), + // Per-source cap on the receive-pack advertisement, sized to an eighth of the + // write pool (min 1): a single source can hold at most this many write-pool + // slots via the anon advertisement, so saturating the pool takes ~8 distinct + // source IPs, each also rate-limited (#174). + git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + (config.max_concurrent_git_pushes / 8).max(1), + ), + // Per-source cap on the authenticated receive-pack POST, sized like the advert + // cap: one source IP can hold at most this many write-pool slots, so + // monopolizing the pool takes ~8 distinct source IPs, each also rate-limited + // (#174 P1-d). + git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + (config.max_concurrent_git_pushes / 8).max(1), + ), + // Bounds concurrent /ipfs visibility walks — a distinct public cost center, so + // its own pool + per-source sub-cap + per-IP rate limiter, never a git pool + // (#174 P1-3). The per-source map is bounded (reject-before-insert, INV-15). + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_ipfs_walks, + )), + git_ipfs_walk_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.ipfs_walk_per_source, + ), + ipfs_rate_limiter: rate_limit::RateLimiter::new_bounded( + config.ipfs_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ), + // Separate WORK-budget bucket for the resolver's per-probe/per-walk charges (R6). + // Its capacity is DERIVED from the route limit (no new knob) and floored at the + // legacy-probe budget, so one full default-config legacy scan never self-throttles + // mid-request while the route brake above stays the pure once-per-request cap. + ipfs_work_rate_limiter: rate_limit::RateLimiter::new_bounded( + AppState::ipfs_work_budget(&config), + std::time::Duration::from_secs(3600), + 200_000, + ), + git_bin: "git".to_string(), }; + if config.ipfs_rate_limit == 0 { + tracing::warn!("GITLAWB_IPFS_RATE_LIMIT=0 — per-IP /ipfs rate limiting disabled"); + } // Periodic peer-count poll for the metrics gauge. If p2p is disabled // we still set the gauge to 0 so dashboards don't show "no data". @@ -408,22 +480,16 @@ async fn main() -> Result<()> { // Periodic cleanup of expired rate limit entries + consumed-proof ledger { - let rl = state.rate_limiter.clone(); - let create_ip_rl = state.create_ip_rate_limiter.clone(); - let push_rl = state.push_rate_limiter.clone(); - let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); - let peer_write_rl = state.peer_write_rate_limiter.clone(); + let cleanup_state = state.clone(); let db = state.db.clone(); let mut shutdown_rx = state.subscribe_shutdown(); tokio::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - rl.cleanup().await; - create_ip_rl.cleanup().await; - push_rl.cleanup().await; - sync_trigger_rl.cleanup().await; - peer_write_rl.cleanup().await; + // Sweep every per-IP/DID limiter (incl. the ipfs walk brake) + // so bounded maps shed stale keys instead of sitting at cap. + cleanup_state.sweep_rate_limiters().await; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs() as i64) diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 6c9c0bff..4d6b9704 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -74,8 +74,10 @@ pub async fn pin_object( /// `..._fail_closed` filter on the full-scan path before calling. `repo_path` is /// still needed to read each object's bytes. The twin in `ipfs_pin.rs` mirrors /// this shape — change both in lockstep. Objects already recorded with a -/// `pinata_cid` are skipped. Returns `(sha_hex, cid)` pairs for each newly -/// pinned object. +/// `pinata_cid` are skipped. `repo_id` records the pin's provenance (#173). +/// Returns `(sha_hex, provider_cid)` pairs for each newly pinned object: the +/// provider CID is the Pinata gateway CID (used for branch→CID recording and +/// ref-update gossip), NOT the raw resolver-key CID stored in `pinned_cids.cid`. pub async fn pin_new_objects( client: &reqwest::Client, upload_url: &str, @@ -83,6 +85,7 @@ pub async fn pin_new_objects( repo_path: &std::path::Path, object_list: Vec, db: &crate::db::Db, + repo_id: &str, ) -> Vec<(String, String)> { if jwt.is_empty() { return vec![]; @@ -92,7 +95,31 @@ pub async fn pin_new_objects( for sha in object_list { match db.has_pinata_cid(&sha).await { - Ok(true) => continue, + Ok(true) => { + // Backfill NULL first-pinner provenance from a known source, in lockstep + // with the ipfs_pin skip branch: a pinata-only node otherwise leaves + // pre-provenance rows' `pinned_cids.repo_id` NULL forever (grok P2-D). The + // resolver still finds the object via the pin_repo_sources union below, so + // this is a consistency backfill, not a correctness fix. + match db.provenance_for_oid(&sha).await { + Ok(None) => { + if let Err(e) = db.backfill_pin_provenance(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); + } + } + Ok(Some(_)) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error reading pin provenance"); + } + } + // F1 (#173 round 8): record this repo as an additional source for the + // already-pinned object (mirrors the ipfs_pin skip-branch insert) so the + // resolver can serve a shared object from any pin-path source. + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinata_cid"); @@ -111,9 +138,21 @@ pub async fn pin_new_objects( match pin_object(client, upload_url, jwt, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinata_cid(&sha, &cid).await { + // The resolver key (`pinned_cids.cid`) must be the locally-computed + // raw-content CID, never the provider CID: Pinata wraps the bytes in + // dag-pb/UnixFS, so its returned CID does not hash the raw content and + // must not become an alias `/ipfs/{cid}` serves raw git bytes for (#173). + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + if let Err(e) = db + .record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) + .await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); } + // F1 (#173 round 8): also record the first pinner in pin_repo_sources. + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } pinned.push((sha, cid)); } Ok(_) => {} diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d40e2691..14c11a9c 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -121,6 +121,28 @@ impl RateLimiter { true } + /// Non-consuming check: is this key ALREADY at its limit for the current window? + /// Unlike [`check`], it records nothing and never inserts a new key — used to shed + /// expensive preparatory work (e.g. the `/ipfs/{cid}` legacy scan's O(repos) DB + /// preload) BEFORE it runs, without perturbing the per-unit budget the consuming + /// `check` maintains (#173, F3). An unknown key or a disabled limiter is not + /// throttled. Prunes the key's expired timestamps as a side effect (keeps state + /// tidy) but adds none, so it cannot itself fill or grow the map. + pub(crate) async fn is_throttled(&self, key: &str) -> bool { + if self.max_requests == 0 { + return false; + } + let now = Instant::now(); + let mut state = self.state.lock().await; + if let Some(window) = state.get_mut(key) { + window + .timestamps + .retain(|t| now.duration_since(*t) < self.window); + return window.timestamps.len() >= self.max_requests; + } + false + } + pub async fn cleanup(&self) { let now = Instant::now(); let mut state = self.state.lock().await; @@ -130,6 +152,103 @@ impl RateLimiter { !w.timestamps.is_empty() }); } + + /// Number of distinct keys currently tracked. Test-only introspection so a + /// cross-module test can assert that a sweep actually evicted expired entries. + #[cfg(test)] + pub(crate) async fn tracked_keys(&self) -> usize { + self.state.lock().await.len() + } +} + +/// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which +/// caps request RATE. Each caller key may hold at most `per_caller` in-flight +/// permits at once; beyond that [`try_acquire`](Self::try_acquire) returns `None` +/// and the caller sheds. Used to stop one caller (a single anonymous source-IP or +/// DID) monopolizing the served-git read pool (#174). +/// +/// The key map is self-bounding: a key is removed the instant its in-flight count +/// reaches zero, so it never holds more keys than there are concurrently-active +/// callers (itself bounded by the read semaphore). A `max_keys` reject-before-insert +/// backstop guarantees a key farm can never grow the map even if that invariant +/// weakened — a NEW key at the cap is rejected WITHOUT allocating an entry (INV-15). +/// +/// Uses a `std::sync::Mutex` (not the file's `tokio::sync::Mutex`) because the +/// permit's `Drop` must release synchronously; the critical section holds no await. +#[derive(Clone)] +pub struct PerCallerConcurrency { + state: Arc>>, + per_caller: usize, + max_keys: usize, +} + +/// RAII permit from [`PerCallerConcurrency::try_acquire`]. On drop it decrements +/// the caller's in-flight count and removes the key when it reaches zero. +pub struct PerCallerPermit { + state: Arc>>, + key: String, +} + +impl PerCallerConcurrency { + pub fn new(per_caller: usize, max_keys: usize) -> Self { + Self { + state: Arc::new(std::sync::Mutex::new(HashMap::new())), + per_caller: per_caller.max(1), + max_keys: max_keys.max(1), + } + } + + /// Convenience constructor with the default key bound. + pub fn with_default_max_keys(per_caller: usize) -> Self { + Self::new(per_caller, DEFAULT_MAX_KEYS) + } + + /// `Some(permit)` when the caller is under its cap and the map has room; + /// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is + /// rejected without allocating. + pub fn try_acquire(&self, key: &str) -> Option { + // Recover from a poisoned lock rather than panicking: the critical section + // is pure counter arithmetic and cannot itself panic, so a poisoned mutex + // would only ever come from an unrelated abort, and a slightly-off count + // self-heals as permits drop. A panic here would instead brick the limiter + // for every caller (each subsequent lock re-panics). + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); + match map.get_mut(key) { + Some(count) => { + if *count >= self.per_caller { + return None; + } + *count += 1; + } + None => { + if map.len() >= self.max_keys { + return None; + } + map.insert(key.to_string(), 1); + } + } + Some(PerCallerPermit { + state: self.state.clone(), + key: key.to_string(), + }) + } + + #[cfg(test)] + pub fn tracked_keys(&self) -> usize { + self.state.lock().unwrap().len() + } +} + +impl Drop for PerCallerPermit { + fn drop(&mut self) { + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(count) = map.get_mut(&self.key) { + *count -= 1; + if *count == 0 { + map.remove(&self.key); + } + } + } } pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { @@ -288,6 +407,63 @@ pub async fn rate_limit_by_ip(request: Request, next: Next) -> Response { mod tests { use super::*; + #[test] + fn per_caller_concurrency_caps_one_caller_and_frees_on_drop() { + let lim = PerCallerConcurrency::new(2, 100); + let p1 = lim.try_acquire("did:key:zA").expect("first under cap"); + let p2 = lim.try_acquire("did:key:zA").expect("second under cap"); + assert!( + lim.try_acquire("did:key:zA").is_none(), + "a third in-flight op for the same caller sheds (over the per-caller cap)" + ); + // A DIFFERENT caller is unaffected — the cap is per-caller, not global. + let _other = lim + .try_acquire("did:key:zB") + .expect("a different caller has its own budget"); + drop(p1); + assert!( + lim.try_acquire("did:key:zA").is_some(), + "freeing one in-flight slot lets the same caller back in" + ); + drop(p2); + } + + #[test] + fn per_caller_concurrency_map_is_self_bounding_and_reject_before_insert() { + // Self-bounding: acquire+drop many distinct keys — the map never grows + // because a key is removed the instant its in-flight count hits zero. + let lim = PerCallerConcurrency::new(4, 3); + for i in 0..50 { + let _p = lim.try_acquire(&format!("k{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "keys with zero in-flight ops are removed, so an acquire+drop flood leaves the map empty" + ); + // Reject-before-insert: HOLD max_keys distinct keys, then a new key sheds + // WITHOUT growing the map past the cap (INV-15 — a rejected request never + // allocates an entry). + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct callers held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected new key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + #[tokio::test] async fn allows_within_limit() { let limiter = RateLimiter::new(3, Duration::from_secs(60)); diff --git a/crates/gitlawb-node/src/server.rs b/crates/gitlawb-node/src/server.rs index f4c0d3e3..de61fcbe 100644 --- a/crates/gitlawb-node/src/server.rs +++ b/crates/gitlawb-node/src/server.rs @@ -214,9 +214,20 @@ pub fn build_router(state: AppState) -> Router { // identity and can apply per-repo visibility (#110); anonymous callers stay // anonymous and still read genuinely public content. `/api/v1/ipfs/pins` // stays unsigned — gating the pin index is tracked separately (#121). + // `/ipfs/{cid}` also carries a per-IP flood brake: it is anon-reachable and each + // request can drive a full-history git walk, so the per-IP rate limiter is the + // outermost layer (rejects a flood before the walk-admission work), mirroring the + // push/create routers. The extension MUST be attached or rate_limit_by_ip is a + // silent no-op. `/api/v1/ipfs/pins` (no walk) is merged in unbraked, as before. + let ipfs_limiter = rate_limit::IpRateLimiter { + limiter: state.ipfs_rate_limiter.clone(), + trust: state.push_limiter_trust, + }; let ipfs_routes = Router::new() .route("/ipfs/{cid}", get(ipfs::get_by_cid)) .layer(middleware::from_fn(auth::optional_signature)) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(ipfs_limiter)) .merge(Router::new().route("/api/v1/ipfs/pins", get(ipfs::list_pins))); // ── Arweave permanent anchors ────────────────────────────────────────── diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 6a84b3c2..51618b78 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -65,6 +65,45 @@ pub struct AppState { /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. pub push_rate_limiter: RateLimiter, + /// Per-client-IP ROUTE brake for `GET /ipfs/{cid}`: charged ONCE per request by the + /// `rate_limit_by_ip` middleware (server.rs), never inside the handler. It bounds + /// request RATE (the "requests per hour" contract of `GITLAWB_IPFS_RATE_LIMIT`) on + /// the non-farmable source IP, so an anonymous flood of the public route is capped. + /// The per-probe/per-walk WORK accounting the resolver does WITHIN a request draws + /// from the SEPARATE `ipfs_work_rate_limiter` below — the two cannot share one bucket + /// or a single request that spends a route token and then its own probe token off the + /// same bucket is admitted at the route and falsely shed mid-request (#173 round-10, + /// R6). Keyed by `push_limiter_trust`. + pub ipfs_rate_limiter: RateLimiter, + /// Per-client-IP WORK-budget limiter for the `GET /ipfs/{cid}` resolver's internal + /// fan-out: charged per legacy (NULL-provenance) PROBE (`acquire` + `cat-file`) and + /// per provenance-path WALK, and peeked non-consuming before the O(repos) legacy + /// preload. A legacy CID from the public pins index otherwise lets one request drive + /// O(repos) subprocess spawns and cold Tigris fetches, and repeat requests amplify + /// that across requests with zero limiter contact (INV-10, F3). Charging the work to + /// the non-farmable source IP bounds it. A bucket DISTINCT from the route brake above: + /// one request legitimately spends many work tokens (a full legacy scan is up to + /// `ipfs_max_legacy_probes` probes), so it must not double as the once-per-request + /// route bucket. Capacity is DERIVED from the route limit (`AppState::ipfs_work_budget`, + /// no separate operator knob), floored at the legacy-probe budget so a single default- + /// config deep search never self-throttles mid-scan. Keyed by `push_limiter_trust`. + pub ipfs_work_rate_limiter: RateLimiter, + /// Per-request ceiling on full-history reachability walks the CID resolver + /// may spawn (default `api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST`). A field, + /// not a bare const, so tests can shrink it to exercise the cap cheaply; + /// production keeps the const default. + pub ipfs_max_history_walks: u32, + /// Per-request ceiling on legacy (NULL-provenance) repo probes in the CID + /// resolver's scan fallback (default `api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST`). + /// Bounds the anonymous `acquire` + `cat-file` fan-out across the node (#173, + /// INV-10); a field for the same test-seam reason as `ipfs_max_history_walks`. + pub ipfs_max_legacy_probes: u32, + /// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` will buffer and + /// serve (default `api::ipfs::MAX_SERVED_OBJECT_BYTES`). The serve reads via a + /// blocking `git cat-file` and buffers the whole object; without a bound a large + /// public blob could exhaust memory or block a runtime worker (#173, F6, INV-10). + /// A field for the same test-seam reason as the sibling caps. + pub ipfs_max_served_object_bytes: u64, /// Which forwarded header (if any) the edge is trusted to set, for /// resolving the push limiter's client-IP key. See `GITLAWB_TRUSTED_PROXY`. /// Node-wide; also keys the two peer-sync limiters below. @@ -88,6 +127,94 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, + /// Bounds concurrent served git READ operations (upload-pack + both info/refs + /// advertisements). A read handler acquires a permit before spawning git and + /// holds it for the op; when none are free the request is shed with a 503. + /// Writes draw from `git_write_semaphore` so a read flood cannot shed an + /// authenticated push at admission (#174). + pub git_read_semaphore: Arc, + /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate + /// from `git_read_semaphore` so an anonymous READ flood can never shed an + /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from + /// by the `git-receive-pack` POST (owner-gated) ONLY. The anon-reachable + /// receive-pack `info/refs` advertisement draws from the SEPARATE + /// `git_push_advert_semaphore` below, never this pool, so a multi-source flood + /// of push-handshake advertisements can never occupy a permit an authenticated + /// POST needs at admission (#174). + pub git_write_semaphore: Arc, + /// Bounds concurrent anon-reachable `git-receive-pack` `info/refs` + /// advertisements — a pool SEPARATE from `git_write_semaphore` so adverts (which + /// hold a permit across `acquire_fresh` + `info/refs`) can never consume a slot + /// the authenticated POST relies on. A per-source flood can at worst exhaust this + /// advert pool (each source also capped by `git_push_advert_per_caller` and the + /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). + pub git_push_advert_semaphore: Arc, + /// Bounds concurrent post-push encrypt-then-pin history walks. Each successful + /// path-scoped push releases its handler write permit and then runs a DETACHED + /// full-history walk (`withheld_blob_recipients_bounded`) to seal withheld blobs; + /// without a cap, N fast pushes spawn N concurrent full-history git walks past + /// `max_concurrent_git_pushes` (which only bounds the in-handler phase). The walk + /// acquires a permit here and DEFERS (blocks) when the pool is full rather than + /// shedding — the work is background and dropping it would lose the recovery copy + /// (#174 P1-e). A pool of its own, not `git_write_semaphore`: a long background + /// walk must not hold a foreground write slot, and a handler already holding a + /// write permit that needed a second would self-deadlock at pool size 1. + pub git_encrypt_semaphore: Arc, + /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing + /// (#174 P2-2). `git_encrypt_semaphore` caps *active* walks; this caps how many + /// detached tasks *spawn and park* on that semaphore's `acquire_owned().await`. + /// Before spawning a per-push encryption task, the receive-pack handler consults + /// this set: if the repo already has a task in flight it coalesces (skips the + /// duplicate spawn) rather than parking a new unbounded waiter. Coalescing only + /// delays a duplicate walk — it never drops the withheld-blob recovery copy, which + /// `2a54c15` deliberately kept fail-closed (there is no reconciliation sweep to + /// re-derive a dropped copy). See [`EncryptInflight`]. + pub encrypt_inflight: EncryptInflight, + /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the + /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` + /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` + /// (#174). Applied by `git_upload_pack` and the upload-pack `info/refs` + /// advertisement. + pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the anon-reachable receive-pack `info/refs` + /// advertisement: each source IP may hold at most a small share of the write + /// pool, so a multi-source flood of push-handshake advertisements cannot + /// saturate `git_write_semaphore` and shed authenticated pushes (#174). Sized as + /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many + /// distinct source IPs (each also braked by the per-IP push rate limiter). + pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the authenticated `git-receive-pack` POST: + /// each source IP may hold at most a small share of `git_write_semaphore`, so one + /// host minting disposable `did:key` identities cannot open enough slow pushes to + /// monopolize the write pool and 503 every other source's push (#174 P1-d). Keyed + /// on the resolved source IP (never the DID — a DID farm defeats a DID key). Sized + /// like `git_push_advert_per_caller`, a fraction of `max_concurrent_git_pushes`. + pub git_write_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Bounds concurrent `GET /ipfs/{cid}` visibility-walk requests. The public + /// `/ipfs/{cid}` route runs `allowed_blob_set_for_caller_bounded` in + /// `spawn_blocking` (a full-history git walk) with NO served-git admission of its + /// own; without this a permissionless caller fans out concurrent walks past every + /// git pool, exhausting the blocking pool + PIDs (#174 P1-3). A request acquires a + /// permit before the repo loop and holds it for the whole request (across every + /// `spawn_blocking` walk), so the slot reflects real thread occupancy — a tokio + /// walk-timeout cannot free it while the blocking work still runs. A pool of its + /// own (`max_concurrent_ipfs_walks`), NOT a git pool: distinct cost center + public + /// surface, so anonymous /ipfs traffic can never shed an authenticated git op. + pub git_ipfs_walk_semaphore: Arc, + /// Per-source concurrency sub-cap on the `/ipfs/{cid}` walk pool: each source + /// (keyed on the resolved source IP, never the DID — `/ipfs` admits any `did:key` + /// unthrottled, so a DID key would be free to mint around) may hold at most + /// `ipfs_walk_per_source` in-flight walk slots, so one source cannot monopolize + /// `git_ipfs_walk_semaphore` (#174 P1-3). A request with no resolvable key is + /// bounded by the global pool only, never this sub-cap. The key map is bounded + /// (`with_default_max_keys`, reject-before-insert) so a source-key farm cannot grow + /// it (INV-15). + pub git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency, + /// The `git` executable the served-git withheld-blob walk spawns. Production is + /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's + /// process-group teardown in handler tests without mutating the process-global + /// PATH (#174). + pub git_bin: String, } impl AppState { @@ -97,6 +224,21 @@ impl AppState { self.shutdown_tx.subscribe() } + /// Sweep expired entries from every per-IP/DID rate limiter. Driven by the + /// periodic cleanup task so a bounded limiter's key map sheds stale entries + /// instead of sitting near its cap until an inline capacity sweep reclaims + /// them. Every limiter on the state is swept here; adding a new limiter means + /// adding it to this list. + pub(crate) async fn sweep_rate_limiters(&self) { + self.rate_limiter.cleanup().await; + self.create_ip_rate_limiter.cleanup().await; + self.push_rate_limiter.cleanup().await; + self.ipfs_rate_limiter.cleanup().await; + self.ipfs_work_rate_limiter.cleanup().await; + self.sync_trigger_rate_limiter.cleanup().await; + self.peer_write_rate_limiter.cleanup().await; + } + /// Trigger graceful shutdown. Idempotent — calling more than once /// has no effect. Returns `true` if this call was the one that /// flipped the signal. @@ -117,4 +259,173 @@ impl AppState { pub fn is_shutting_down(&self) -> bool { *self.shutdown_tx.borrow() } + + /// Legacy-probe budget wired from the `GITLAWB_IPFS_MAX_REPOS_WALKED` operator + /// knob (R5, KTD5). The knob seeds `ipfs_max_legacy_probes` at construction so it + /// controls the per-request legacy (NULL-provenance) probe fan-out it advertises. + /// It deliberately does NOT feed `ipfs_max_history_walks`: that ceiling must stay + /// at `MAX_HISTORY_WALKS_PER_REQUEST` (`MAX_PIN_SOURCES + 1`) or a provenanced + /// request with a full source set is truncated into a false 503, and the knob's + /// range starts at 1. The knob is `usize`, the field `u32`; the range cap + /// (1_048_576) keeps the cast lossless. + pub(crate) fn ipfs_legacy_probe_budget(config: &crate::config::Config) -> u32 { + config.ipfs_max_repos_walked as u32 + } + + /// Work-budget capacity for [`ipfs_work_rate_limiter`](Self#structfield.ipfs_work_rate_limiter) + /// (R6, KTD6), DERIVED from the route limit rather than a new operator knob. The route + /// limiter (`ipfs_rate_limiter`) charges once per request; this separate bucket absorbs + /// the resolver's per-probe/per-walk work charges so both the route "requests per hour" + /// contract and the amplification bound hold. Floor: at least one full legacy search per + /// window — the effective `ipfs_max_legacy_probes` (the `GITLAWB_IPFS_MAX_REPOS_WALKED` + /// knob, U4) — so a single default-config deep search cannot self-throttle mid-scan and + /// recreate the F6 admit-then-429 for a legitimate caller. `GITLAWB_IPFS_RATE_LIMIT=0` + /// disables the route brake and this derived bucket alike (a 0-capacity limiter admits + /// everything). + pub(crate) fn ipfs_work_budget(config: &crate::config::Config) -> usize { + if config.ipfs_rate_limit == 0 { + return 0; + } + config.ipfs_rate_limit.max(config.ipfs_max_repos_walked) + } +} + +/// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing +/// (#174 P2-2). Each successful path-scoped push `tokio::spawn`s a DETACHED task that +/// parks on `git_encrypt_semaphore.acquire_owned().await` (which DEFERS when the pool +/// is full rather than shedding — `2a54c15` kept it fail-closed so the withheld-blob +/// recovery copy is never dropped). The semaphore caps *active* walks, but nothing +/// capped how many detached tasks *spawn and park* on that await: N rapid pushes to a +/// repo spawn N parked tasks, each holding cloned object lists/rules/paths/keys — an +/// unbounded outstanding set. +/// +/// This tracks the repo keys with an in-flight encryption task, each carrying a +/// DIRTY flag. Before spawning, the handler calls [`try_begin`](Self::try_begin): if +/// a task for the repo is already in-flight it MARKS THE REPO DIRTY and returns `None` +/// so the handler SKIPS spawning a duplicate (coalesce). The in-flight task consults +/// the flag at its tail via [`requeue_or_release`](EncryptInflightGuard::requeue_or_release): +/// a set flag makes it run ONE MORE pass (re-reading repo state) before it exits, so a +/// push coalesced during the in-flight window is REQUEUED, never dropped. This bounds +/// the outstanding set to <=1 task per repo without losing work: there is no +/// reconciliation sweep, so a *dropped* job would be lost forever. +/// +/// The returned [`EncryptInflightGuard`] is moved into the detached task. The normal +/// exit path is `requeue_or_release`, which removes the repo key ATOMICALLY with the +/// "clean" decision — no push can land in the gap between "checked clean" and "task +/// exits". `Drop` is a panic backstop only: if the task panics (or returns without +/// calling `requeue_or_release`) it still removes the key so a crashed walk can never +/// permanently lock a repo out of future recovery copies. +#[derive(Clone, Default)] +pub struct EncryptInflight { + // std::sync::Mutex: only ever held for O(1) HashMap ops in a sync context (before + // `tokio::spawn`, at the task tail, and in Drop) — never across an await, so a std + // Mutex is correct and cheaper than a tokio one. The value is the DIRTY flag: a + // coalesced push flips it true so the in-flight task requeues one more pass. + repos: Arc>>, +} + +impl EncryptInflight { + pub fn new() -> Self { + Self::default() + } + + /// Try to begin an encryption task for `repo_id`. Returns `Some(guard)` if no task + /// for the repo was in-flight (the caller should spawn). If one already is, MARKS + /// the repo dirty and returns `None` (the caller COALESCES — skips the duplicate + /// spawn; the in-flight task requeues one more pass to cover this push). + pub fn try_begin(&self, repo_id: &str) -> Option { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(repo_id) { + Some(dirty) => { + *dirty = true; + None + } + None => { + map.insert(repo_id.to_string(), false); + Some(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + released: false, + }) + } + } + } + + /// Number of repos with an in-flight encryption task. Test/metrics observability; + /// the bound under saturation is `len() <= number of distinct repos`, i.e. at most + /// one task per repo. + #[allow(dead_code)] + pub fn len(&self) -> usize { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .len() + } + + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Test-only: read the dirty flag for `repo_id` (`None` if no task is in-flight). + #[cfg(test)] + pub fn dirty(&self, repo_id: &str) -> Option { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .get(repo_id) + .copied() + } +} + +/// RAII guard for one in-flight encryption task's repo key. The task drives it at its +/// tail with [`requeue_or_release`](Self::requeue_or_release); `Drop` is a panic +/// backstop. Move-only — cloning would double-release. +pub struct EncryptInflightGuard { + repos: Arc>>, + repo_id: String, + released: bool, +} + +impl EncryptInflightGuard { + /// TASK-TAIL check-and-clear, atomic with the release decision. If a push + /// coalesced since the last pass (dirty), clear the flag and return `true` (the + /// task loops one more pass). Otherwise remove the key and return `false` (the task + /// exits). Atomic under the mutex: a concurrent push either sets the flag BEFORE + /// this reads it (-> requeue covers it) or arrives AFTER the key is removed (-> a + /// fresh `try_begin` spawns a new task) — there is no window where the key is + /// present-but-clean while the task exits. + pub fn requeue_or_release(&mut self) -> bool { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(&self.repo_id) { + Some(dirty) if *dirty => { + *dirty = false; + true + } + _ => { + map.remove(&self.repo_id); + self.released = true; + false + } + } + } +} + +impl Drop for EncryptInflightGuard { + fn drop(&mut self) { + // Panic backstop ONLY. The normal exit is `requeue_or_release` (which removed + // the key atomically and set `released`), so this does nothing then. If the + // task panicked or returned without releasing, remove the key so one crashed + // walk never permanently locks the repo out. + // + // Accepted residual: a panic with the dirty flag still set drops the requeued + // work — Drop cannot loop — the same loss class as the on-panic behavior before + // this change. No reconciliation sweep re-derives it. + if self.released { + return; + } + if let Ok(mut map) = self.repos.lock() { + map.remove(&self.repo_id); + } + } } diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 4fe52f64..0d0ae2e8 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -78,10 +78,32 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + // Generous — no test drives the handler-level shed (git_permit is unit-tested). + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 8, + ), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + // Generous — a test that drives the /ipfs walk shed overrides these directly. + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 16, + ), + git_bin: "git".to_string(), } } @@ -187,6 +209,199 @@ mod tests { ); } + /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer before the + /// DB. The held `git_permit` acquire now sits after the per-source cap, so the + /// shed-before-DB property is carried by an explicit `available_permits() == 0` + /// early check at the top of the handler (the held permit remains the + /// authoritative bound further down). DB-free: an exhausted semaphore sheds + /// before any DB/disk access, so a lazy state works. Remove the early-shed block + /// from git_info_refs and this goes red (the request falls through to the DB and + /// returns something other than 503). + #[tokio::test] + async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-upload-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed info/refs with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack carries the same + /// explicit `available_permits() == 0` early-shed check at the top, so an + /// exhausted semaphore must shed it with a 503 before any DB/disk work. + /// Anonymous-reachable, so no auth injection is needed. Remove the early-shed + /// block from git_upload_pack and this goes red. + #[tokio::test] + async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed git-upload-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed + /// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174), + /// so an exhausted advert pool sheds the advert with 503 before any DB/disk work + /// — while the write pool (reserved for authenticated POSTs) is left free here. + /// Flip the pool selection back to the write pool, or remove the early-shed + /// block, and this goes red. + #[tokio::test] + async fn git_info_refs_receive_pack_sheds_with_503_when_advert_pool_exhausted() { + let mut state = test_state_lazy(); + state.git_push_advert_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-receive-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted ADVERT pool must shed the receive-pack advertisement with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling for the push path: git-receive-pack requires an + /// AuthenticatedDid extension (production: require_signature injects it), so the + /// request carries one via signed_request_as — without it the Extension + /// extractor 500s before the handler body reaches git_permit. The permit is the + /// first statement, so an exhausted semaphore still sheds 503 before any DB + /// work. Remove the permit line from git_receive_pack and this goes red. + #[tokio::test] + async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVSHEDOWNERAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted write pool must shed git-receive-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// #174 (SC1, load-bearing): a saturated READ pool must NOT shed an + /// authenticated push — the write pool is a separate budget. Read pool at zero, + /// write pool with capacity: the push proceeds PAST admission (it then errors on + /// the placeholder DB, but crucially it is not a 503). Route git-receive-pack + /// back to the read pool and this goes red — that is the isolation proof. + #[tokio::test] + async fn git_receive_pack_not_shed_by_exhausted_read_pool() { + let mut state = test_state_lazy(); + // Read pool exhausted as if a flood of anonymous clones held every slot. + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + // Write pool keeps its default capacity from test_state_lazy. + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVCROSSBOUNDARYAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted READ pool must not shed a push — the write pool is a separate budget (#174)" + ); + } + /// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner /// before any git work (so no on-disk repo is needed for the rejection). #[sqlx::test] @@ -1809,13 +2024,19 @@ mod tests { /// Seed a SHA-256 source repo (public/a.txt + secret/b.txt), bare-clone it /// into each `/tmp//.git` path, and return guards + oids. - /// SHA-256 object format is required: `get_by_cid` resolves a CID whose - /// multihash digest IS the git object id, which only matches in sha256 repos. + /// SHA-256 object format matches production (`--object-format=sha256`) so the + /// oids are 64-hex. A real CID digests the raw object CONTENT (not the git + /// oid), so tests build the request CID with `pin_cid_for` — mirroring the pin + /// path — and `get_by_cid` maps it back to the oid via `pinned_cids` (#173). struct CidFixture { _guards: Vec, secret_oid: String, public_oid: String, secret_tree_oid: String, + public_tree_oid: String, + root_tree_oid: String, + commit_oid: String, + tag_oid: String, } impl Drop for CidFixture { fn drop(&mut self) { @@ -1849,6 +2070,8 @@ mod tests { run(&["config", "user.name", "t"], &src); run(&["add", "."], &src); run(&["commit", "-qm", "seed"], &src); + // Annotated tag of the commit — exercises the "tags stay served" guard. + run(&["tag", "-a", "-m", "annotated", "v1", "HEAD"], &src); let oid = |rev: &str| { let out = Command::new("git") .args(["rev-parse", rev]) @@ -1861,6 +2084,10 @@ mod tests { let secret_oid = oid("HEAD:secret/b.txt"); let public_oid = oid("HEAD:public/a.txt"); let secret_tree_oid = oid("HEAD:secret"); + let public_tree_oid = oid("HEAD:public"); + let root_tree_oid = oid("HEAD^{tree}"); + let commit_oid = oid("HEAD"); + let tag_oid = oid("refs/tags/v1"); let mut guards = vec![src.clone()]; for name in bare_names { let bare = std::path::PathBuf::from("/tmp") @@ -1878,6 +2105,12 @@ mod tests { ], &src, ); + // `git clone --bare` does NOT copy the source repo's local identity, so + // fixtures that create objects directly in the bare repo (`commit-tree`, + // `git tag -a`) abort with "identity unknown" on a CI runner that has no + // ambient/global git identity. Set it explicitly so the suite is portable. + run(&["config", "user.email", "t@t"], &bare); + run(&["config", "user.name", "t"], &bare); } // One guard for the whole /tmp/ tree covers every bare clone. guards.push(std::path::PathBuf::from("/tmp").join(slug)); @@ -1886,203 +2119,3845 @@ mod tests { secret_oid, public_oid, secret_tree_oid, + public_tree_oid, + root_tree_oid, + commit_oid, + tag_oid, } } - /// CID whose sha2-256 multihash digest equals the given 64-hex git oid, so - /// `get_by_cid` decodes it back to that oid and `git cat-file`s it. - fn cid_for_oid(oid_hex: &str) -> String { - use gitlawb_core::cid::Cid; - let bytes = hex::decode(oid_hex).expect("hex oid"); - let arr: [u8; 32] = bytes.as_slice().try_into().expect("32-byte sha256 oid"); - Cid::from_sha256_bytes(&arr).to_string() + /// Record a pin exactly as the production pin path does — read the object's + /// raw bytes (`git cat-file `, no framing), CID them with + /// `Cid::from_git_object_bytes`, and store the `(oid, cid)` row — then return + /// the CID string the node advertises (`gl ipfs list`) and a client sends to + /// `GET /ipfs/{cid}`. Building the CID from the oid instead (the old + /// `cid_for_oid`) produced an identifier that never occurs in production and + /// made the gate assertions vacuous: a real pin CID digests the raw content, + /// not the git oid, so `get_by_cid` resolves it through `pinned_cids` (#173). + async fn pin_cid_for(bare_repo: &std::path::Path, oid: &str, db: &crate::db::Db) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + // Legacy-style pin (no provenance) so existing CID tests exercise the + // resolver's scan fallback; provenance-path tests pin via `pin_cid_for_repo`. + db.record_pinned_cid(oid, &cid, None) + .await + .expect("record pinned cid"); + cid } - fn cid_router(state: &AppState) -> Router { - Router::new() - .route( - "/ipfs/{cid}", - axum::routing::get(crate::api::ipfs::get_by_cid), - ) - .layer(axum::middleware::from_fn(crate::auth::optional_signature)) - .with_state(state.clone()) + /// Like [`pin_cid_for`] but records the pin's provenance (`repo_id`), so the + /// resolver resolves the CID straight to `repo_id` instead of scanning (#173). + #[allow(dead_code)] // used by the provenance-path resolver tests (P-U3) + async fn pin_cid_for_repo( + bare_repo: &std::path::Path, + oid: &str, + db: &crate::db::Db, + repo_id: &str, + ) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + db.record_pinned_cid(oid, &cid, Some(repo_id)) + .await + .expect("record pinned cid with provenance"); + cid } - async fn cid_parts(resp: axum::response::Response) -> (StatusCode, String) { - let st = resp.status(); - let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + + /// INV-7 upgrade path for the pin-provenance column (#173, jatmn round 2): a node + /// already past v11 gets `pinned_cids.repo_id` from the NEW v12 migration, and a + /// legacy pin recorded before the column existed survives with NULL provenance (so + /// it falls back to the repo scan). Simulate the pre-v12 node by dropping the + /// column and un-applying v12, seed a legacy row, then re-migrate. RED before the + /// v12 migration exists (the column is never re-added → the SELECT errors); GREEN + /// after. + #[sqlx::test] + async fn pinned_cids_repo_provenance_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v12 shape: drop the provenance column and forget v12 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS repo_id") + .execute(&pool) .await .unwrap(); - (st, String::from_utf8_lossy(&b).to_string()) - } - fn cid_anon(cid: &str) -> Request { - Request::builder() - .method(Method::GET) - .uri(format!("/ipfs/{cid}")) - .body(Body::empty()) - .unwrap() + sqlx::query("DELETE FROM schema_migrations WHERE version = 12") + .execute(&pool) + .await + .unwrap(); + + // A legacy pin recorded before provenance existed. + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind("legacyoid") + .bind("legacycid") + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + // Upgrade: re-run migrations → v12 re-adds the column. + state.db.run_migrations().await.expect("migrate to v12"); + + // The legacy pin survives with NULL provenance. + let legacy: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = 'legacyoid'") + .fetch_one(&pool) + .await + .expect("legacy pin row survives the upgrade"); + assert!( + legacy.is_none(), + "a pin recorded before v12 must keep NULL provenance (it falls back to the scan)" + ); + + // A new pin can carry provenance. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind("newoid") + .bind("newcid") + .bind("2026-01-01T00:00:00Z") + .bind("repo-abc") + .execute(&pool) + .await + .unwrap(); + let prov: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = 'newoid'") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + prov.as_deref(), + Some("repo-abc"), + "a pin recorded after v12 carries its source repo_id" + ); } - fn cid_signed(kp: &gitlawb_core::identity::Keypair, cid: &str) -> Request { - let path = format!("/ipfs/{cid}"); - let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); - Request::builder() - .method(Method::GET) - .uri(&path) - .header("content-digest", s.content_digest) - .header("signature-input", s.signature_input) - .header("signature", s.signature) - .body(Body::empty()) - .unwrap() + + /// #173: a pin records the repository it came from; `provenance_for_oid` reads it + /// back; a legacy pin (no repo) reads back None; and first-pinner-owns holds — a + /// second push of the same oid does NOT rewrite provenance (ON CONFLICT DO + /// NOTHING). This is what lets the resolver gate a CID against its ONE source repo. + #[sqlx::test] + async fn record_pinned_cid_stores_and_reads_provenance(pool: PgPool) { + let state = test_state(pool).await; + + state + .db + .record_pinned_cid("oidA", "cidA", Some("repo-xyz")) + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("oidA") + .await + .unwrap() + .as_deref(), + Some("repo-xyz"), + "a provenanced pin reads back its source repo_id" + ); + + state + .db + .record_pinned_cid("oidB", "cidB", None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("oidB").await.unwrap(), + None, + "a legacy pin (no repo) has NULL provenance" + ); + + // First-pinner-owns: a later push of the same oid must not rewrite provenance. + state + .db + .record_pinned_cid("oidA", "cidA", Some("repo-OTHER")) + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("oidA") + .await + .unwrap() + .as_deref(), + Some("repo-xyz"), + "ON CONFLICT DO NOTHING keeps the first repo's provenance" + ); + + // An unpinned oid has no provenance. + assert_eq!( + state.db.provenance_for_oid("never-pinned").await.unwrap(), + None + ); } - /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. - /// RED before U2 (the current handler serves the secret to anon). + /// #173 (provenance, happy path): a CID pinned with provenance resolves straight + /// to its ONE source repo and serves an authorized reader — no repo scan. #[sqlx::test] - async fn ipfs_cid_gate_withholds_blob_from_unauthorized(pool: PgPool) { - use crate::db::VisibilityMode; + async fn ipfs_cid_provenance_serves_from_pinning_repo(pool: PgPool) { use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let _fx = seed_cid_repos(&slug, &short, &["provserve"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provserve.git"); + let fx = &_fx; + + // Build the repo FIRST so the pin can carry its id as provenance. + let repo = seed_repo(&owner_did, "provserve"); // public + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a provenanced public CID serves its content" + ); + assert!( + body.contains("public bytes"), + "the pinning repo's object is served" + ); + } + /// #173 (provenance, THE load-bearing one — #124 flip + bounded fan-out): a CID + /// pinned from a PRIVATE repo must gate against that pinning repo (404), NOT serve + /// from a byte-identical PUBLIC copy in another repo. Provenance is strictly more + /// restrictive than the old scan (which served the public copy). RED before the + /// rework (the scan serves the public copy → 200 + leaks the secret bytes); GREEN + /// after (provenance → the private repo → 404, no leak). + #[sqlx::test] + async fn ipfs_cid_provenance_private_denies_despite_public_copy(pool: PgPool) { + use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); let owner_did = owner.did().to_string(); - let reader = Keypair::generate(); - let reader_did = reader.did().to_string(); - let stranger = Keypair::generate(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let tree_cid = cid_for_oid(&fx.secret_tree_oid); - let public_cid = cid_for_oid(&fx.public_oid); + let fx = seed_cid_repos(&slug, &short, &["privsrc", "pubcopy"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privsrc.git"); + // Private source repo, built first so the pin carries its id as provenance. + let mut priv_repo = seed_repo(&owner_did, "privsrc"); + priv_repo.is_public = false; state .db - .create_repo(&seed_repo(&owner_did, "withhold")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "withhold") + .create_repo(&priv_repo) .await - .unwrap() - .unwrap(); + .expect("seed private repo"); + let cid = pin_cid_for_repo(&priv_bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + + // A PUBLIC repo holds the SAME object (the old scan would serve it). + let pub_repo = seed_repo(&owner_did, "pubcopy"); // public, no rule state .db - .set_visibility_rule( - &rec.id, - "/secret/**", - VisibilityMode::B, - std::slice::from_ref(&reader_did), - &owner_did, - ) + .create_repo(&pub_repo) .await - .expect("deny rule"); + .expect("seed public copy"); - // anon → withheld blob: must 404, must not leak content. (RED on current handler.) - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; assert_eq!( st, StatusCode::NOT_FOUND, - "anon must not read the withheld blob" + "a provenanced private CID must 404, not serve from a public copy elsewhere (#124 flip)" ); assert!( !body.contains("TOP SECRET"), - "404 body must not leak the secret" + "the 404 body must not leak the withheld object" ); + } - // signed non-reader → 404. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&stranger, &secret_cid)) - .await - .unwrap(), + /// #173 (jatmn round 8, F1 — load-bearing): a shared object first pinned from a + /// PRIVATE repo, then pushed again from a PUBLIC repo through the real pin path, + /// must serve by CID to an anonymous caller from the public source. First-pinner- + /// only provenance 404s it (only the private source is known); recording EVERY + /// pin-path source fixes it. The second push hits the already-pinned skip branch, + /// so this proves the skip-branch source insert fires (and does NOT re-pin: /add + /// expect(0)). RED before U1 (anon 404); GREEN after. + #[sqlx::test] + async fn ipfs_cid_multi_source_serves_from_later_public_pinner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["privfirst", "pubsecond"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privfirst.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubsecond.git"); + + // Private repo pins the object FIRST — it owns the first-pinner provenance. + let mut priv_repo = seed_repo(&owner_did, "privfirst"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private first-pinner"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // A PUBLIC repo pushes the SAME object through the real pin path. The object is + // already pinned, so this hits the already-pinned skip branch, which must record + // the public repo as an additional source without re-pinning (/add expect 0). + let pub_repo = seed_repo(&owner_did, "pubsecond"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public second-pinner"); + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &pub_bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + &pub_repo.id, + ) + .await; + m.assert_async().await; // asserts /add was NOT called (already pinned) + + // Anonymous CID fetch: the private first source denies, the public second + // source serves → 200. Before F1 only the private source is known → 404. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a shared object must serve by CID from a later public pin-path source (F1)" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + + /// U1 (grok round-4 P1): `pin_sources_at_cap` flips exactly at `MAX_PIN_SOURCES`. + /// It is the signal `get_by_cid` uses to decide a provenance miss may be hiding a + /// dropped servable source and must fall back to the bounded scan. + #[sqlx::test] + async fn pin_sources_at_cap_flips_at_max(pool: PgPool) { + let state = test_state(pool).await; + let cap = crate::db::MAX_PIN_SOURCES; + assert!( + !state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "an oid with no pin_repo_sources rows is not at cap" + ); + for i in 0..(cap - 1) { + state + .db + .record_pin_source("atcapoid", &format!("r-{i:02}")) + .await + .unwrap(); + } + assert!( + !state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "one below MAX_PIN_SOURCES is not at cap" + ); + state + .db + .record_pin_source("atcapoid", "r-last") + .await + .unwrap(); + assert!( + state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "exactly MAX_PIN_SOURCES rows is at cap" + ); + } + + /// U2 (grok round-4 P1, load-bearing): the pin-source GRIEFING hole. A private + /// first-pinner denies anon; an attacker fills the whole `MAX_PIN_SOURCES` source + /// window with deny-anon sources BEFORE a legitimate public repo pins the same + /// object, so the public repo's `record_pin_source` no-ops (cap full) and it is + /// buried — present in NO provenance record. The resolver's provenance set is then + /// {private + 16 attacker}, all deny anon. Because the set is at_cap (may hide a + /// dropped source), the handler falls back to the bounded legacy scan, which gates + /// every repo through the real gate and finds the buried PUBLIC copy → 200. + /// MUTATION (RED): remove the `at_cap` fallback edge in `get_by_cid` and the buried + /// public object 404s forever. + #[sqlx::test] + async fn ipfs_cid_buried_public_source_still_serves_via_scan_fallback(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["privfirst", "pubburied"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privfirst.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubburied.git"); + + // Private repo pins FIRST — owns the first-pinner provenance, denies anon. + let mut priv_repo = seed_repo(&owner_did, "privfirst"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private first-pinner"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // Attacker fills the ENTIRE MAX_PIN_SOURCES window with deny-anon (non-existent) + // sources BEFORE the public repo registers, so the cap is full. + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..cap { + state + .db + .record_pin_source(&fx.public_oid, &format!("00-attacker-{i:02}")) + .await + .expect("attacker source"); + } + + // A PUBLIC repo pushes the SAME object through the real pin path. Already pinned + // (skip branch), so it only tries record_pin_source — which NO-OPS because the + // cap is full. The public repo is thus buried: not the first-pinner, not in + // pin_repo_sources. + let pub_repo = seed_repo(&owner_did, "pubburied"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public buried source"); + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &pub_bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + &pub_repo.id, + ) + .await; + m.assert_async().await; // /add NOT called (already pinned) + + // The buried public object must STILL serve: the provenance set is at_cap and + // all-deny, so the handler falls back to the bounded scan, which finds pubburied. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a public source buried by a full attacker source window must still serve via the bounded scan fallback (F1)" + ); + assert!( + body.contains("public bytes"), + "the served body is the buried public object's bytes" + ); + } + + /// #173 (jatmn round 8, F1 — bound, R2): the per-object source set is capped at + /// `MAX_PIN_SOURCES` so an adversary pushing one object from many repos cannot make + /// resolution O(repos). Recording the same oid from `MAX_PIN_SOURCES + 3` distinct + /// repos leaves exactly `MAX_PIN_SOURCES` rows. + #[sqlx::test] + async fn ipfs_cid_pin_sources_capped_at_max(pool: PgPool) { + let state = test_state(pool).await; + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..(cap + 3) { + state + .db + .record_pin_source("capoid", &format!("repo-{i}")) + .await + .expect("record source"); + } + let sources = state.db.pin_sources_for_oid("capoid").await.unwrap(); + assert_eq!( + sources.len() as i64, + cap, + "the per-object source set is capped at MAX_PIN_SOURCES" + ); + } + + /// #173 (jatmn round 8, F1 — availability, grok-4.5 adversarial catch): the resolver's + /// per-object source cap must NEVER evict the first-pinner. A legacy public pin keeps + /// its source in `pinned_cids.repo_id` but not in `pin_repo_sources` (pre-v13 pins, or + /// a pin whose best-effort `record_pin_source` missed). If the cap `LIMIT` were applied + /// to the whole union with a lexicographic order, an attacker could push the same + /// object from `MAX_PIN_SOURCES` repos whose grindable ids sort before the public + /// source and evict it from the window — turning a public CID that served 200 into a + /// 404. This drives exactly that: a legacy public first-pinner plus `MAX_PIN_SOURCES` + /// lower-sorting attacker sources must STILL serve the public object. RED with a + /// whole-union LIMIT (the first-pinner is dropped → 404); GREEN once the first-pinner + /// is always included and the LIMIT caps only the additional sources. + #[sqlx::test] + async fn ipfs_cid_first_pinner_never_evicted_by_lower_sorting_sources(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["pubfirst"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubfirst.git"); + // Public repo whose id sorts AFTER every attacker id below. Legacy shape: the + // source lives in pinned_cids.repo_id only (pin_cid_for_repo records no + // pin_repo_sources row), exactly like a pin from before v13. + let mut pub_repo = seed_repo(&owner_did, "pubfirst"); // public, no rule + pub_repo.id = "zzzzzzzz-pubfirst".to_string(); + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public first-pinner"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &pub_repo.id).await; + + // Attacker fills the whole MAX_PIN_SOURCES window with lower-sorting source ids + // (non-existent repos — their mere presence would evict the first-pinner under a + // whole-union LIMIT). + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..cap { + state + .db + .record_pin_source(&fx.public_oid, &format!("00-attacker-{i:02}")) + .await + .expect("attacker source"); + } + + // The public first-pinner must still serve — never evicted by the cap window. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "the first-pinner public source must never be evicted by lower-sorting attacker sources (F1 availability)" + ); + assert!( + body.contains("public bytes"), + "the public object is served from the first-pinner" + ); + } + + /// INV-7 upgrade path for the F1 `pin_repo_sources` table (#173, jatmn round 8): a + /// node already past v12 gets the table from the NEW v13 migration. Simulate the + /// pre-v13 node by dropping the table and un-applying v13, then re-migrate and + /// assert a source row round-trips. RED before the v13 migration exists. + #[sqlx::test] + async fn pin_repo_sources_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + sqlx::query("DROP TABLE IF EXISTS pin_repo_sources") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 13") + .execute(&pool) + .await + .unwrap(); + state.db.run_migrations().await.expect("re-migrate"); + state + .db + .record_pin_source("upgradeoid", "repo-upg") + .await + .expect("record after re-migrate"); + assert_eq!( + state.db.pin_sources_for_oid("upgradeoid").await.unwrap(), + vec!["repo-upg".to_string()], + "the v13 pin_repo_sources table is present after upgrade" + ); + } + + /// #173 (jatmn round 8, F2 — load-bearing): a legacy `pinned_cids` row keyed on a + /// PROVIDER CID (Pinata/Kubo dag-pb — every release before this branch stored the + /// provider CID as the resolver key, not the raw-content CID) must NOT serve raw git + /// bytes that do not hash to the requested CID. `get_by_cid` recomputes the CID over + /// the served bytes and refuses to serve on mismatch. Seeded with a RAW SQL INSERT + /// because the current helpers store the raw CID, so a helper-seeded row is already + /// correct-shape and the RED assertion would be vacuous (INV-21). RED before U2 + /// (serves the git bytes → 200); GREEN after (not served, no bytes egress). + #[sqlx::test] + async fn ipfs_cid_legacy_provider_cid_row_not_served(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc"]); + let repo = seed_repo(&owner_did, "provsrc"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // A valid sha2-256 CID whose digest is NOT the object's raw-content digest — + // stands in for a Pinata/Kubo dag-pb provider CID (the legacy resolver key). + let provider_cid = gitlawb_core::cid::Cid::from_git_object_bytes( + b"a decoy object whose CID is not the served object's CID", + ) + .to_string(); + + // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers now store the + // raw CID and cannot reproduce this shape). The object itself is public+servable. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + // Requesting the provider CID resolves the row and passes the repo gate, but the + // served bytes hash to a DIFFERENT CID, so the integrity check must withhold them. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&provider_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st, + StatusCode::OK, + "a provider-CID legacy row must not serve raw git bytes (F2)" + ); + assert!( + !body.contains("public bytes"), + "the mismatched bytes must not egress" + ); + } + + /// #173 (jatmn round 8, F6 — INV-10 cost guard): the serve path buffers the object via + /// a blocking `cat-file`; an object larger than `ipfs_max_served_object_bytes` must be + /// WITHHELD (rejected by the size precheck, never buffered), with zero body bytes + /// egressed. Under the cap it serves unchanged. The oversize-reject counter guards it + /// both ways: a removed size precheck serves the object and leaves the counter at 0. + #[sqlx::test] + async fn ipfs_cid_f6_oversized_object_withheld(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["big"]); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("big.git"); + let repo = seed_repo(&owner_did, "big"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Cap below the object size ("public bytes\n" = 13 bytes) → withheld. + state.ipfs_max_served_object_bytes = 5; + crate::api::ipfs::reset_oversize_rejects(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_ne!( + st, + StatusCode::OK, + "an object over the size cap must not serve (F6)" + ); + assert!( + !body.contains("public bytes"), + "no object bytes egress for an over-cap object" + ); + assert_eq!( + crate::api::ipfs::oversize_rejects(), + 1, + "the oversized object was rejected by the size precheck" + ); + + // Control: raise the cap above the object size → serves unchanged. + state.ipfs_max_served_object_bytes = crate::api::ipfs::MAX_SERVED_OBJECT_BYTES; + crate::api::ipfs::reset_oversize_rejects(); + let (st2, body2) = + cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st2, + StatusCode::OK, + "under the cap the object serves normally" + ); + assert!( + body2.contains("public bytes"), + "the served body is the object's bytes" + ); + assert_eq!( + crate::api::ipfs::oversize_rejects(), + 0, + "no oversize reject under the cap" + ); + } + + /// #173 (provenance, INV-11): a quarantined pinning repo must 404 by CID even for + /// its own owner — quarantine hard-drops before the visibility gate on the + /// provenance path too. The owner-signed 404 is the load-bearing negative (a + /// visibility-only gate would Allow the owner). + #[sqlx::test] + async fn ipfs_cid_provenance_quarantined_repo_404_even_owner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["quarsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quarsrc.git"); + let repo = seed_repo(&owner_did, "quarsrc"); // public + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Baseline: before quarantine the provenanced CID serves (proves the path works). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "provenanced CID serves before quarantine" + ); + + state + .db + .set_repo_quarantine(&repo.id, true) + .await + .expect("quarantine"); + + for req in [cid_anon(&cid), cid_signed(&owner, &cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a quarantined pinning repo must 404 by CID (anon + owner)" + ); + assert!( + !body.contains("public bytes"), + "the 404 body must not leak quarantined content" + ); + } + } + + /// #173 (provenance, bounded — must NOT fall back to the scan): a CID whose + /// provenance points at a repo that no longer exists must 404 rather than scan + /// every repo and serve a byte-identical public copy. Falling back to the scan + /// would reopen the O(repos) anonymous fan-out the provenance rework closes. RED + /// before the rework (the scan serves the public copy → 200); GREEN after. + #[sqlx::test] + async fn ipfs_cid_provenance_missing_repo_404_no_scan_fallback(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["gonesrc", "pubcopy2"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("gonesrc.git"); + + // Pin with provenance = a repo_id that is never created (deleted/absent). + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, "nonexistent-repo-id").await; + + // A public repo holds the SAME object (the old scan would serve it). + let pub_repo = seed_repo(&owner_did, "pubcopy2"); + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public copy"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a provenance pointing at a missing repo must 404, not fall back to the scan" + ); + } + + /// #173 (provenance, path-scoped WALK gate): the #135/#173 per-object gates must + /// run on the NEW provenance path, not only the legacy scan. A provenanced pin from + /// a repo under a `/secret/**` rule runs `allowed_blob_set_for_caller` via the shared + /// gate: a withheld secret blob 404s to anon (no byte leak); the allowed reader gets + /// it. Exercises the walk gate on the provenance path in BOTH directions. + #[sqlx::test] + async fn ipfs_cid_provenance_path_scoped_walk_gates_withheld_blob(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["provwalk"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provwalk.git"); + let repo = seed_repo(&owner_did, "provwalk"); // public at "/" + state.db.create_repo(&repo).await.expect("seed repo"); + // /secret/** Mode B with the reader allowed → the secret blob walk gates by caller. + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + + // Anon: the walk denies the secret blob → 404, no leak. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a withheld secret blob 404s to anon on the provenance path (walk gate runs)" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld blob" + ); + + // Allowed reader: the walk includes the secret blob → 200 with content. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&reader, &cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "an allowed reader gets the secret blob via the provenance walk gate" + ); + assert!( + body.contains("TOP SECRET"), + "the allowed reader receives the content" + ); + } + + /// #173: the pinata pin path stores the locally-computed raw CID in the + /// resolver-key `cid` column and the provider CID in `pinata_cid`, and its ON + /// CONFLICT COALESCE fills a NULL provenance without overwriting an existing one + /// (first-pinner-owns). On conflict `cid` is left untouched so a prior local pin's + /// raw CID is never clobbered by a provider CID. + #[sqlx::test] + async fn record_pinata_cid_stores_and_coalesces_provenance(pool: PgPool) { + let state = test_state(pool).await; + + // A new row created via the pinata path carries provenance, and stores the + // raw CID in `cid` with the provider CID in `pinata_cid`. + state + .db + .record_pinata_cid("po1", "rawcid1", "pcid1", Some("repoA")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po1").await.unwrap().as_deref(), + Some("repoA") + ); + let po1 = state + .db + .list_pinned_cids() + .await + .unwrap() + .into_iter() + .find(|r| r.sha256_hex == "po1") + .expect("po1 row exists"); + assert_eq!(po1.cid, "rawcid1", "resolver-key cid is the raw CID"); + assert_eq!( + po1.pinata_cid.as_deref(), + Some("pcid1"), + "the provider CID is kept in pinata_cid" + ); + + // An existing NULL-provenance row: the pinata COALESCE fills it, and the + // prior local pin's `cid` is left untouched (not overwritten by the raw arg). + state + .db + .record_pinned_cid("po2", "localcid2", None) + .await + .unwrap(); + state + .db + .record_pinata_cid("po2", "rawcid2", "pcid2", Some("repoB")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po2").await.unwrap().as_deref(), + Some("repoB"), + "pinata fills a NULL provenance" + ); + let po2 = state + .db + .list_pinned_cids() + .await + .unwrap() + .into_iter() + .find(|r| r.sha256_hex == "po2") + .expect("po2 row exists"); + assert_eq!( + po2.cid, "localcid2", + "on conflict the prior local pin's cid is left untouched" + ); + + // An existing provenance: the pinata COALESCE must NOT overwrite it. + state + .db + .record_pinned_cid("po3", "cid3", Some("repoX")) + .await + .unwrap(); + state + .db + .record_pinata_cid("po3", "rawcid3", "pcid3", Some("repoY")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po3").await.unwrap().as_deref(), + Some("repoX"), + "pinata COALESCE keeps the first-pinner's provenance" + ); + } + + /// #173 (jatmn, F4, load-bearing security): a Pinata-first pin (no prior local pin) + /// must make the resolver key (`pinned_cids.cid`) the locally-computed raw CID, NOT + /// the provider CID. Pinata wraps the bytes in dag-pb/UnixFS, so its returned CID + /// does not hash the raw content; if it became the resolver key, `/ipfs/{provider_cid}` + /// would serve raw git bytes that do not hash to it, breaking raw content-addressing. + /// Assert `oids_for_cid(raw_cid)` finds the sha AND `oids_for_cid(provider_cid)` does NOT. + #[sqlx::test] + async fn record_pinata_cid_resolver_key_is_raw_not_provider(pool: PgPool) { + let state = test_state(pool).await; + + let bytes = b"raw git object content for pinata-first pin"; + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(bytes).to_string(); + // A distinct provider CID (a dag-pb wrapper CID Pinata would return). + let provider_cid = "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"; + assert_ne!( + raw_cid, provider_cid, + "the provider CID must differ from the raw CID for this test to be meaningful" + ); + + // Pinata-first: no prior local pin, so this INSERT creates the row. + state + .db + .record_pinata_cid("pfsha", &raw_cid, provider_cid, Some("repoP")) + .await + .unwrap(); + + // The raw CID resolves to the sha. + assert_eq!( + state.db.oids_for_cid(&raw_cid).await.unwrap(), + vec!["pfsha".to_string()], + "the locally-computed raw CID is the resolver key" + ); + // The provider (dag-pb) CID must NOT resolve raw bytes. + assert!( + state + .db + .oids_for_cid(provider_cid) + .await + .unwrap() + .is_empty(), + "the provider dag-pb CID must never resolve raw git bytes" + ); + } + + /// #173 (end-to-end pin wiring): `pin_new_objects` records the repo_id it is given + /// as the pin's provenance. Drives the real pin path against a mocked IPFS `/add` + /// endpoint (so `pin_git_object` succeeds) and asserts `provenance_for_oid` returns + /// the repo — closing the gap between the push handler's threading and the DB write. + #[sqlx::test] + async fn pin_new_objects_records_provenance(pool: PgPool) { + let state = test_state(pool).await; + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovtest"}"#) + .expect_at_least(1) + .create_async() + .await; + + let fx = seed_cid_repos("provpin_e2e", "ppe2e", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_e2e") + .join("pinsrc.git"); + + let pinned = crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + "repoZ", + ) + .await; + assert!( + !pinned.is_empty(), + "the object was pinned via the real pin path" + ); + m.assert_async().await; + assert_eq!( + state + .db + .provenance_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some("repoZ"), + "pin_new_objects records the repo_id it was given as the pin's provenance" + ); + } + + /// #173 (grok F2): the post-push pin read is BOUNDED, so a wedged/D-state + /// `git cat-file` (stuck NFS/Tigris backend) is reaped at `git_timeout` and + /// `pin_new_objects` RETURNS — reaching `requeue_or_release` in production — + /// instead of hanging forever and pinning the per-repo coalescing key until + /// process death. A fake `git` whose `cat-file` records its pid then sleeps far + /// past a SHORT 1s timeout stands in for the wedged backend; the `run_bounded_git` + /// watchdog (SIGTERM -> grace -> SIGKILL of the process group) must reap it well + /// before its 8s natural exit, and the call must return with nothing pinned. + /// + /// REVERT PROOF (RED): swap `read_object_bounded` back to the bare + /// `store::read_object` at the pin read and the wedged child is STILL RUNNING at + /// the mid-flight liveness poll below (unbounded `Command::output` cannot be + /// reaped at the deadline) — the reap assertion fails. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_reaps_wedged_read_at_deadline(pool: PgPool) { + use std::time::Duration; + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Fake `git`: `cat-file` records its own pid then sleeps 8s (>> the 1s + // deadline) so the read is genuinely wedged; the watchdog is what must end it. + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("catfile.pid"); + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + cat-file) echo $$ > \"{}\"; sleep 8 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pidfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let repo = tmp.path().to_path_buf(); + let git = git_path.to_str().unwrap().to_string(); + // A never-pinned OID so the call reaches the object-read stage (not the + // already-pinned skip path). + let oid = "f".repeat(64); + // Non-empty ipfs_api so `pin_new_objects` does not early-return; the wedged + // read is reaped and the OID skipped before any `/add`, so this URL is unused. + let ipfs_api = "http://127.0.0.1:1".to_string(); + + // `pin_new_objects` must run on THIS runtime so its `is_pinned` DB call keeps + // the sqlx pool on its home runtime. The bounded read is a synchronous blocking + // call, so the reap poll runs on a separate OS thread (independent of tokio): it + // captures the wedged child's pid, waits past the deadline, records whether it + // was reaped, then SIGKILLs defensively so even a true infinite hang cannot leak + // an orphan or stall the awaited call. + let pidfile_poll = pidfile.clone(); + let poll = std::thread::spawn(move || -> (Option, bool) { + let alive = |pid: i32| unsafe { libc::kill(pid, 0) == 0 }; + let mut pid = None; + for _ in 0..500 { + if let Some(p) = std::fs::read_to_string(&pidfile_poll) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + pid = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid = match pid { + Some(p) => p, + None => return (None, false), + }; + // Past the 1s deadline + SIGTERM grace but well before the 8s natural exit: + // the bounded read must already have reaped the wedged group. The unbounded + // `store::read_object` leaves it running here — the load-bearing RED. + std::thread::sleep(Duration::from_secs(3)); + let reaped = !alive(pid); + unsafe { + libc::kill(pid, libc::SIGKILL); + } + (Some(pid), reaped) + }); + + // The call must RETURN — reaching `requeue_or_release` in production — rather + // than hang on the 8s sleep. The poll thread's defensive SIGKILL guarantees the + // read completes even in the unbounded RED case, so this observes a bounded + // return either way; the reap assertion below is what separates RED from GREEN. + let pinned = tokio::time::timeout( + Duration::from_secs(6), + crate::ipfs_pin::pin_new_objects( + &ipfs_api, + &repo, + &git, + Duration::from_secs(1), + vec![oid], + &db, + "repoWedge", + ), + ) + .await + .expect("pin_new_objects must return within the bound, not hang on the wedged read"); + + let (pid, reaped) = poll.join().expect("poll thread joins"); + pid.expect("the fake cat-file must have spawned and recorded its pid"); + assert!( + reaped, + "the post-push pin read must reap the wedged cat-file child at the deadline, \ + not leave it running (which would pin the coalescing key until process death)" + ); + assert!( + pinned.is_empty(), + "a wedged read pins nothing this pass; a later pass/push retries" + ); + } + + /// #173 (jatmn, F2): a legacy pin with NULL provenance backfills its source + /// via `backfill_pin_provenance`, and the `AND repo_id IS NULL` guard preserves + /// first-pinner-owns (a non-NULL provenance is left untouched). + #[sqlx::test] + async fn backfill_pin_provenance_fills_null_keeps_existing(pool: PgPool) { + let state = test_state(pool).await; + + // A legacy pin: no provenance recorded. + state + .db + .record_pinned_cid("legacy_oid", "legacy_cid", None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("legacy_oid").await.unwrap(), + None, + "a legacy pin starts with NULL provenance" + ); + + // Backfill sets the NULL provenance. + state + .db + .backfill_pin_provenance("legacy_oid", "repo-src") + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("legacy_oid") + .await + .unwrap() + .as_deref(), + Some("repo-src"), + "backfill fills a NULL provenance from the known source" + ); + + // A pin that already has provenance: backfill must NOT overwrite it. + state + .db + .record_pinned_cid("owned_oid", "owned_cid", Some("repo-first")) + .await + .unwrap(); + state + .db + .backfill_pin_provenance("owned_oid", "repo-second") + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("owned_oid") + .await + .unwrap() + .as_deref(), + Some("repo-first"), + "the AND repo_id IS NULL guard keeps the first-pinner's provenance" + ); + } + + /// #173 (jatmn, F2, load-bearing): an object already pinned with NULL provenance + /// (a pre-provenance legacy pin) acquires its source when `pin_new_objects` sees + /// it again. The already-pinned skip path must backfill rather than leave the + /// object stuck on the O(repos) scan fallback — and it must NOT re-pin the bytes + /// (no IPFS `/add` call, the object is already on IPFS). + #[sqlx::test] + async fn pin_new_objects_backfills_legacy_null_provenance(pool: PgPool) { + let state = test_state(pool).await; + + let fx = seed_cid_repos("provpin_backfill", "ppbf", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_backfill") + .join("pinsrc.git"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(&bare, &fx.public_oid) + .expect("read object bytes") + .expect("object exists") + .1, + ) + .to_string(); + + // Legacy pin: the object is already recorded with NULL provenance. + state + .db + .record_pinned_cid(&fx.public_oid, &cid, None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid(&fx.public_oid).await.unwrap(), + None, + "the object starts as a legacy pin with NULL provenance" + ); + + // Mock IPFS `/add` and require it is NOT called: the already-pinned object + // must be backfilled, never re-pinned. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + + let pinned = crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + "repoBF", + ) + .await; + + assert!( + pinned.is_empty(), + "an already-pinned object is not re-pinned (no bytes returned)" + ); + m.assert_async().await; // asserts /add was called 0 times + assert_eq!( + state + .db + .provenance_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some("repoBF"), + "pin_new_objects backfills the legacy pin's NULL provenance" + ); + } + + /// Build a legacy provider CID (CIDv1 dag-pb — the Kubo above-block-size root + /// shape, and codec-equivalent to the Pinata CIDv0 legacy key for the cost + /// gate) over the object's own multihash. Non-raw codec, so `is_raw_cidv1` + /// flags it a repair candidate, and a different string from the raw key, so a + /// repair rewrites it. The existing `ipfs_cid_legacy_provider_cid_row_not_served` + /// fixture seeds a raw-codec decoy (an integrity negative the cost gate treats + /// as non-legacy on purpose); this produces the genuine dag-pb legacy shape the + /// repair path targets. Uses only the `cid` crate (already a node dep). + fn legacy_dagpb_cid(raw_cid: &str) -> String { + const DAG_PB: u64 = 0x70; + let parsed = raw_cid + .parse::>() + .expect("the raw CID parses"); + cid::CidGeneric::<64>::new_v1(DAG_PB, *parsed.hash()).to_string() + } + + /// #173 R8 (jatmn round 10, U7 — load-bearing): a legacy row keyed on a PROVIDER + /// CID (Kubo dag-pb / Pinata) is opportunistically rewritten to the raw-content + /// key on a re-push whose pack carries the object, stashing the old value in + /// `legacy_provider_cid`. The advertised key 404s while the row is legacy (the + /// resolver recomputes the raw CID and the stored key does not match) and serves + /// after repair. RED before the skip-branch repair lands (the raw key 404s post + /// pin). Also asserts the repair leaves `pinata_cid` NULL (scenario 3) and that + /// the retired provider CID still refuses to serve (scenario 6, integrity). + #[sqlx::test] + async fn ipfs_cid_legacy_provider_cid_repaired_on_repush(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provsrc.git"); + let repo = seed_repo(&owner_did, "provsrc"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // The canonical raw key the resolver accepts once the row is repaired. + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .unwrap() + .1, + ) + .to_string(); + // The key stored today: a genuine legacy dag-pb provider CID. + let provider_cid = legacy_dagpb_cid(&raw_cid); + assert_ne!( + provider_cid, raw_cid, + "the provider CID differs from the raw resolver key" + ); + + // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers store the + // raw CID). The object itself is public and servable. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + // RED baseline: the raw key a correct client sends 404s while the row is legacy. + let (st_before, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_before, + StatusCode::OK, + "the raw key 404s while the row is keyed on the provider CID" + ); + + // Re-push carries the object again: `pin_new_objects` hits the already-pinned + // skip branch and repairs the row. The `/add` mock must NOT fire — the object + // is already on IPFS, never re-pinned. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + &repo.id, + ) + .await; + m.assert_async().await; + + // GREEN: the key is repaired to the raw CID and the old value is stashed. + let (stored_cid, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&fx.public_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored_cid, raw_cid, + "the key is repaired to the raw-content CID" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed in legacy_provider_cid" + ); + + // The advertised (raw) key now serves 200. + let (st_after, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st_after, + StatusCode::OK, + "the repaired raw key serves after the re-push" + ); + assert!(body.contains("public bytes"), "the object's bytes serve"); + + // Scenario 3: repair never wrote `pinata_cid`, so the Pinata pin-skip gate + // (`has_pinata_cid`) is untouched and Pinata still pins the object. + assert!( + !state.db.has_pinata_cid(&fx.public_oid).await.unwrap(), + "repair leaves pinata_cid NULL" + ); + + // Scenario 6 (integrity negative): the retired provider CID still 404s — no + // serve-path alias for a CID the bytes do not hash to. + let (st_old, body_old) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&provider_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_old, + StatusCode::OK, + "the retired provider CID must not serve after repair" + ); + assert!( + !body_old.contains("public bytes"), + "no bytes egress under the retired provider CID" + ); + } + + /// #173 R8 (U7 cost gate): a well-formed CIDv1/raw already-pinned row triggers NO + /// object read on the skip path — the codec check decides candidacy from the + /// stored string alone, so a non-legacy row keeps the DB-only skip cost. Also + /// covers the small-object equivalence: a small legacy object Kubo pins under the + /// raw key (raw-leaves) is already CIDv1/raw and needs no repair. The read counter + /// is the both-ways guard: removing the codec gate reads the raw row and trips it. + #[sqlx::test] + async fn ipfs_cid_repair_codec_gate_skips_raw_row(pool: PgPool) { + let state = test_state(pool).await; + let fx = seed_cid_repos("codecgate", "cg", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("codecgate") + .join("pinsrc.git"); + + // A correct raw-CID row (steady state), recorded via the production helper. + let raw_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + assert!( + gitlawb_core::cid::is_raw_cidv1(&raw_cid), + "the helper records a CIDv1/raw key" + ); + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"x"}"#) + .expect(0) + .create_async() + .await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + "repoCG", + ) + .await; + m.assert_async().await; + + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a CIDv1/raw row triggers no object read on the skip path (cost gate)" + ); + assert_eq!( + state + .db + .cid_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some(raw_cid.as_str()), + "the raw row is left as-is" + ); + } + + /// #173 R8 (U7): a legacy row whose object bytes are gone stays withheld — the + /// repair never destructively rewrites it, so the row is preserved for a future + /// re-push or the deferred one-shot sweep. + #[sqlx::test] + async fn ipfs_cid_repair_unrepairable_row_stays_withheld(pool: PgPool) { + let state = test_state(pool.clone()).await; + let _fx = seed_cid_repos("unrep", "ur", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("unrep") + .join("pinsrc.git"); + + // A legacy dag-pb row for an oid whose bytes are NOT in this bare repo. + let phantom_oid = "b".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(&phantom_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + let mut server = mockito::Server::new_async().await; + server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"x"}"#) + .expect(0) + .create_async() + .await; + + // Skip-branch runs (is_pinned true) but read_object returns None (bytes gone), + // so the repair returns without touching the row. + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![phantom_oid.clone()], + &state.db, + "repoUR", + ) + .await; + + let (stored, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&phantom_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored, provider_cid, + "an unrepairable row keeps its provider CID (no destructive rewrite)" + ); + assert_eq!( + stashed, None, + "no legacy_provider_cid is stashed when the bytes are gone" + ); + } + + /// #173 R8 (U7, INV-7 upgrade path): a node already at the prior-max schema (v13) + /// gets `pinned_cids.legacy_provider_cid` from the NEW v14 migration. Simulate the + /// pre-v14 node by dropping the column and un-applying v14, then re-migrate and + /// assert a repair round-trips through the column. RED before the v14 migration + /// exists (the column is never re-added → the repair UPDATE errors). + #[sqlx::test] + async fn pinned_cids_legacy_provider_cid_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v14 shape: drop the column and forget v14 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS legacy_provider_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 14") + .execute(&pool) + .await + .unwrap(); + + // Upgrade: re-run migrations → v14 re-adds the column. + state.db.run_migrations().await.expect("migrate to v14"); + + // A repair round-trips through the v14 column. + state + .db + .record_pinned_cid("upg_oid", "QmProviderLegacy", None) + .await + .unwrap(); + state + .db + .repair_legacy_provider_cid("upg_oid", "bRawContentKey", "QmProviderLegacy") + .await + .unwrap(); + let (cid, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = 'upg_oid'", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(cid, "bRawContentKey", "v14 lets the repair rewrite the key"); + assert_eq!( + stashed.as_deref(), + Some("QmProviderLegacy"), + "the v14 legacy_provider_cid column is present after upgrade" + ); + } + + /// #173 (provenance-path throttle): a walk-requiring provenanced candidate whose + /// per-IP walk quota is spent returns 429 (the provenance arm's Throttled outcome, + /// then the fall-through). quota=1, keyed on XFF. The first reader request runs the + /// walk and spends the token; the second from the same IP is throttled → 429. + #[sqlx::test] + async fn ipfs_cid_provenance_walk_throttle_returns_429(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["provthrottle"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provthrottle.git"); + let repo = seed_repo(&owner_did, "provthrottle"); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + + // 1st reader request runs the walk (reader is allowed) and spends the token. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "1st provenance walk from the IP serves"); + + // 2nd request from the same IP: the walk is throttled → 429 (provenance path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "a throttled provenance walk returns 429" + ); + } + + /// #173 (multi-oid dispatch, mixed provenance + legacy): one CID mapping to a + /// provenanced-then-denied oid AND a legacy (NULL-provenance) oid must still resolve + /// to the legacy-servable copy — the provenance arm's skip does not abort the loop. + #[sqlx::test] + async fn ipfs_cid_mixed_provenance_and_legacy_serves_legacy(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["mixpriv", "mixpub"]); + + // Private repo holds secret_oid, pinned with provenance = itself (denies anon). + let mut priv_repo = seed_repo(&owner_did, "mixpriv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + // Public repo holds public_oid, legacy pin (NULL provenance -> scan serves it). + let pub_repo = seed_repo(&owner_did, "mixpub"); + state.db.create_repo(&pub_repo).await.expect("seed public"); + + // One REAL CID (the non-unique cid index) maps to BOTH oids: the public oid as a + // legacy (NULL) pin, and the secret oid provenanced to the private repo. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("mixpub.git"); + let shared_cid = pin_cid_for(&pub_bare, &fx.public_oid, &state.db).await; + state + .db + .record_pinned_cid(&fx.secret_oid, &shared_cid, Some(&priv_repo.id)) + .await + .unwrap(); + + // Anon: secret_oid (provenance -> private -> denied), public_oid (legacy -> scan + // -> public -> served). Resolves to the public copy regardless of oid order. + let resp = cid_router(&state) + .oneshot(cid_anon(&shared_cid)) + .await + .unwrap(); + let served = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a CID mixing a provenanced-denied oid and a legacy-servable oid resolves" + ); + assert_eq!( + served.as_deref(), + Some(fx.public_oid.as_str()), + "the served object is the legacy public oid" + ); + assert!( + body.contains("public bytes"), + "the public content is served" + ); + } + + // ---- #173 round 3: legacy (NULL-provenance) scan bound + 503-on-truncation ---- + // The provenance path targets one repo and is already bounded. These cover the + // legacy scan fallback, where an anonymous request could otherwise fan out to + // O(repos) `acquire` + `cat-file` probes (F1) and a walk-cap truncation could + // false-404 an object that may be readable (F2). The bound is a per-request probe + // BUDGET, not a per-IP brake: a walk-free public fetch stays un-rate-limited + // (ipfs_walk_rate_limited_per_source), while the expensive walk keeps its IP brake. + + /// T1 (F1): the probe budget gates BEFORE `acquire`/`cat-file`, so it genuinely + /// bounds the fan-out — a repo past the budget is never probed, even one that + /// WOULD serve. With the budget at 0, a PUBLIC legacy copy that would otherwise + /// serve 200 is not probed at all → 503 truncated (absence unproven). RED before + /// the budget check (the repo is probed and serves 200). + #[sqlx::test] + async fn ipfs_cid_legacy_probe_budget_gates_before_serving(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 0; // probe nothing → any legacy candidate truncates + + let fx = seed_cid_repos(&slug, &short, &["pubprobe"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubprobe.git"); + let repo = seed_repo(&owner_did, "pubprobe"); // public, no path rule → would serve + state.db.create_repo(&repo).await.expect("seed repo"); + // Legacy pin (NULL provenance) → resolver takes the scan fallback. + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "the probe budget gates before the probe: a servable copy past the budget is not reached → 503" + ); + } + + /// T7 (F1/F3 pre-limit): EVERY legacy probe is braked on the source IP from the + /// FIRST one, so a hostile caller cannot repeatedly force the whole-node `acquire` + /// fan-out across requests (each cold `acquire` is a Tigris round-trip, INV-10). + /// Since #173-F3 (jatmn) there is no free budget: a single-repo legacy scan is + /// itself charged. quota=1 keyed on XFF, one PUBLIC legacy copy that serves + /// walk-free (never touches the walk brake), so the second same-IP request can only + /// be shed by the probe brake: req1 serves and spends the token, req2 → 429. RED + /// before the probe brake (req2 serves 200). The cross-request bound this proves is + /// exactly the amplification F3 closes. + #[sqlx::test] + async fn ipfs_cid_legacy_fanout_braked_on_ip_past_free_budget(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["fanout"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("fanout.git"); + let repo = seed_repo(&owner_did, "fanout"); // public, no path rule → walk-free serve + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; // legacy pin + + let (st1, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st1, + StatusCode::OK, + "1st legacy fan-out probe from the IP serves" + ); + + let (st2, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st2, + StatusCode::TOO_MANY_REQUESTS, + "with no free budget, a repeat fan-out from the same IP is braked at the first probe" + ); + } + + /// F3 (jatmn, across-request amplification): the pre-fix free-probe budget was + /// PER REQUEST, so a caller could repeat a known NULL-provenance CID and force a + /// fresh batch of `acquire` + `cat-file` probes every request with zero limiter + /// contact, unbounded anonymous amplification against Tigris. Charging every + /// legacy probe from the first one makes those probes accumulate against the + /// per-IP `ipfs_work_rate_limiter` ACROSS requests. Four repos, none holding the CID, + /// so a full scan probes all four; the per-IP budget is sized to exactly ONE such + /// scan (4 tokens). req1 (a genuine absence) fully scans and 404s, spending the + /// budget; req2 from the SAME IP is shed at the first probe → 429 (it never + /// re-runs the four `acquire` probes). RED with the old free carve-out restored: + /// req2 re-scans un-braked and 404s again (the amplification stays open). This is + /// the load-bearing across-request bound F3 asks for. + #[sqlx::test] + async fn ipfs_cid_legacy_fanout_bounded_across_requests(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget = one full scan of the four seeded repos. A repeat scan from the same + // IP then finds it spent. Keyed on XFF so `oneshot` can choose the source IP. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(4, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let names = ["a0", "a1", "a2", "a3"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo → each probed repo misses, + // so req1 scans all four (spending the four-token budget) and 404s cleanly. + let bogus_oid = "0".repeat(64); + let cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-across-requests").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st1, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st1, + StatusCode::NOT_FOUND, + "1st scan completes under budget: a genuine absence is a definitive 404" + ); + + let (st2, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st2, + StatusCode::TOO_MANY_REQUESTS, + "2nd same-IP scan is shed at the first probe (429), not re-run un-braked: the across-request amplification is closed" + ); + } + + /// #173 (jatmn round 8, F3 — INV-10 cost guard): an already-throttled source's + /// legacy NULL-provenance request must be shed by the non-consuming admission peek + /// BEFORE the O(repos) `scan_ctx` preload runs — not after, where the per-probe + /// brake sits. The preload-query counter proves it both ways: 0 for the throttled + /// replay, 1 for an unthrottled source. RED if the peek is removed (the preload runs + /// while throttled → count 1). The two existing `_fanout_` tests confirm the per- + /// probe consuming charge is untouched (no double-charge, no under-charge). + #[sqlx::test] + async fn ipfs_cid_f3_throttled_source_skips_preload(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget 1, keyed on XFF so `oneshot` can choose the source IP. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let _fx = seed_cid_repos(&slug, &short, &["r0"]); + state + .db + .create_repo(&seed_repo(&owner_did, "r0")) + .await + .expect("seed repo"); + // A legacy pin absent from every repo → the scan probes and 404s (spending the + // one token on the first probe). + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"f3-absent").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("legacy pin"); + + // Req1 from 9.9.9.9 spends the one token (and runs the preload once). + let _ = cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(); + + // Measure the throttled replay: the peek must shed it before the preload runs. + crate::api::ipfs::reset_preload_queries(); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "an already-throttled legacy replay is 429" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "a throttled source must NOT run the O(repos) preload (F3): shed before scan_ctx" + ); + + // Control: an unthrottled source (a different IP) still runs the preload once — + // the peek must not over-block. + crate::api::ipfs::reset_preload_queries(); + let _ = cid_router(&state) + .oneshot(cid_anon_xff(&cid, "8.8.8.8")) + .await + .unwrap(); + assert_eq!( + crate::api::ipfs::preload_queries(), + 1, + "an unthrottled source runs the preload once (the peek must not over-block)" + ); + } + + /// T2 (F1): the legacy scan is bounded per request. With the probe ceiling shrunk + /// to 2 and 3 candidate repos none of which hold the object, the 3rd repo is never + /// probed and the search is reported truncated → 503, not an unbounded fan-out. + /// RED before the probe cap (all 3 probe, none serve, definitive 404). + #[sqlx::test] + async fn ipfs_cid_legacy_scan_probe_cap_truncates_to_503(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 2; + + let _fx = seed_cid_repos(&slug, &short, &["r0", "r1", "r2"]); + for n in ["r0", "r1", "r2"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo: each probed repo misses, + // so the cap (not a hit) decides the outcome. + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-t2").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "a scan truncated by the probe cap is a retryable 503, not a definitive 404" + ); + } + + /// T2b (R5, KTD5): the `GITLAWB_IPFS_MAX_REPOS_WALKED` knob drives the legacy-probe + /// budget end to end. With the knob at 1 (fed through the same production helper the + /// state seeding uses) and two candidate repos that miss, the first repo spends the + /// single probe and the second is skipped at the cap → truncated → 503. If the knob + /// budget were not honoured (unbounded), both would probe, both miss, and the request + /// would be a definitive 404. Proves the wired knob=1 → exactly one probe path. + #[sqlx::test] + async fn ipfs_cid_repos_walked_knob_caps_legacy_probes(pool: PgPool) { + use clap::Parser; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Seed the legacy-probe budget the way production does: from the operator knob. + let cfg = + crate::config::Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1"]); + state.ipfs_max_legacy_probes = AppState::ipfs_legacy_probe_budget(&cfg); + assert_eq!(state.ipfs_max_legacy_probes, 1, "knob=1 → one-probe budget"); + // The knob must not touch the history-walk ceiling (must stay MAX_PIN_SOURCES + 1). + assert_eq!( + state.ipfs_max_history_walks, + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + "the repos-walked knob leaves the history-walk ceiling untouched" + ); + + let _fx = seed_cid_repos(&slug, &short, &["k0", "k1"]); + for n in ["k0", "k1"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo: the cap, not a hit, decides. + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-knob").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "knob=1 caps the scan at one probe → incomplete search → retryable 503" + ); + } + + /// T3 (F2): a walk-cap truncation must not false-404. Walk ceiling shrunk to 1; + /// two public repos each carry a path-scoped rule over the object and deny anon. + /// The 1st spends the single walk (deny), the 2nd is skipped at the cap — the + /// resolver did NOT prove the object unreadable everywhere, so 503, not 404. + /// RED before the walk-cap `truncated` flag (returns the opaque 404). + #[sqlx::test] + async fn ipfs_cid_legacy_walk_cap_truncates_to_503(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_history_walks = 1; + + let fx = seed_cid_repos(&slug, &short, &["wa", "wb"]); + for n in ["wa", "wb"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + } + // Legacy pin of the path-scoped secret blob (present in both repos, denies anon). + let bare_wa = std::path::PathBuf::from("/tmp").join(&slug).join("wa.git"); + let cid = pin_cid_for(&bare_wa, &fx.secret_oid, &state.db).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "the walk cap truncated the scan, so absence is unproven → 503, not a false 404" + ); + } + + /// T4 (must-not over-fire): a legacy CID genuinely absent from every repo on a + /// node UNDER the probe cap still returns the definitive 404 — the 503 fires only + /// on real truncation, never as a blanket replacement for not-found. + #[sqlx::test] + async fn ipfs_cid_legacy_true_absence_stays_404(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 8; // well above the single repo → no truncation + + let _fx = seed_cid_repos(&slug, &short, &["only"]); + let repo = seed_repo(&owner_did, "only"); + state.db.create_repo(&repo).await.expect("seed repo"); + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-t4").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a fully-scanned genuine absence is a definitive 404, not a 503" + ); + } + + /// T5 (provenance path untouched): the probe cap governs ONLY the legacy scan. + /// With the cap set to 0 (which would truncate any legacy probe immediately) a + /// PROVENANCED pin still resolves to its one repo and serves 200 — proving the + /// `legacy_scan=false` guard exempts the provenance path. RED if the guard were + /// dropped (provenance would truncate to 503). + #[sqlx::test] + async fn ipfs_cid_provenance_serves_despite_zero_probe_cap(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 0; // would truncate every LEGACY probe + + let fx = seed_cid_repos(&slug, &short, &["provonly"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provonly.git"); + let repo = seed_repo(&owner_did, "provonly"); // public, no path rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "the provenance path ignores the legacy probe cap and serves" + ); + } + + fn cid_router(state: &AppState) -> Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state.clone()) + } + async fn cid_parts(resp: axum::response::Response) -> (StatusCode, String) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, String::from_utf8_lossy(&b).to_string()) + } + /// Raw body bytes (NOT lossy-decoded). A git tree body stores each child oid + /// as 32 RAW bytes that `from_utf8_lossy` mangles to U+FFFD, so a hex + /// `contains` check on `cid_parts`'s String is vacuous. #135 deny tests must + /// witness the leak on these raw bytes. + async fn cid_bytes(resp: axum::response::Response) -> (StatusCode, Vec) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, b.to_vec()) + } + /// True if `needle` appears as a contiguous byte subsequence of `haystack`. + fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && haystack.windows(needle.len()).any(|w| w == needle) + } + fn cid_anon(cid: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap() + } + /// Anonymous CID request carrying `x-forwarded-for: ` — an anon caller with a + /// resolvable source IP, so the per-IP walk brake keys on it (the walk still + /// denies anon at a path rule). + fn cid_anon_xff(cid: &str, xff_ip: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } + fn cid_signed(kp: &gitlawb_core::identity::Keypair, cid: &str) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .body(Body::empty()) + .unwrap() + } + /// Signed CID request carrying `x-forwarded-for: `. Used by the walk + /// rate-limit test to key the per-IP limiter off a chosen source under + /// `TrustedProxy::XForwardedFor` (the request goes through `oneshot`, which + /// leaves no socket peer, so the header is the only key source). + fn cid_signed_xff( + kp: &gitlawb_core::identity::Keypair, + cid: &str, + xff_ip: &str, + ) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } + + /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. + /// RED before U2 (the current handler serves the secret to anon). + #[sqlx::test] + async fn ipfs_cid_gate_withholds_blob_from_unauthorized(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let stranger = Keypair::generate(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Request CIDs are the production pin CIDs (content-hash), recorded in + // pinned_cids so get_by_cid resolves each back to its oid (#173). + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &fx.tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "withhold")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "withhold") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("deny rule"); + + // anon → withheld blob: must 404, must not leak content. (RED on current handler.) + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "anon must not read the withheld blob" + ); + assert!( + !body.contains("TOP SECRET"), + "404 body must not leak the secret" + ); + + // signed non-reader → 404. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&stranger, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "non-reader must not read the withheld blob" + ); + assert!(!body.contains("TOP SECRET")); + + // owner (signed) → 200 + secret bytes. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "owner reads the withheld blob"); + assert!(body.contains("TOP SECRET"), "owner gets the content"); + + // listed reader (signed) → 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&reader, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); + assert!(body.contains("TOP SECRET")); + + // #135: anon tree CID under withheld /secret → 404. The 404 body is an opaque + // error string (never the object), so status is the load-bearing deny check; + // the real leak witness is the CONTRAST with the reader below, who DOES get a + // 200 carrying the child structure that anon is denied. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "withheld subtree tree must not be served to anon (#135)" + ); + + // Over-denial guard + positive leak witness: the listed reader (signed) DOES + // read the withheld subtree's tree, and its body carries the exact child + // structure anon was denied — the child filename plus the child oid as the 32 + // RAW bytes a git tree stores (witnessed on raw bytes, since cid_parts's lossy + // decode would mangle them). This proves b.txt / secret_raw are the real leak + // markers and that the anon 404 above actually withheld them. + let secret_raw = hex::decode(&fx.secret_oid).expect("hex oid"); + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_signed(&reader, &tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "listed reader reads the withheld subtree tree" + ); + assert!( + bytes_contain(&body, b"b.txt") && bytes_contain(&body, &secret_raw), + "reader's tree body carries the child filename and raw child oid" + ); + + // Root tree (path "/") stays served to anon who passes the "/" gate. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&root_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "root tree stays served (must-serve)"); + + // /public subtree tree stays served to anon (allowed path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public subtree tree stays served"); + + // Commit and annotated tag objects stay served (unchanged by #135). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "commit object stays served"); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tag_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "tag object stays served"); + + // R3: public blob anon → 200 (non-withheld content not affected). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public blob stays served"); + + // R5: a genuine unknown CID also 404, uniform with the withheld 404. A + // well-formed pin-style CID that was never recorded in pinned_cids, so the + // oid_for_cid resolve misses (the production not-found path). + let absent_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"never pinned to this node").to_string(); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&absent_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "absent CID 404 (uniform with withheld)" + ); + + // malformed CID → 400 (unchanged). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon("not-a-cid")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "malformed CID still 400"); + } + + /// R4: the same object withheld in one repo but public in another is still + /// served from the public copy; the withholding repo is iterated first. + #[sqlx::test] + async fn ipfs_cid_served_from_public_copy_when_withheld_elsewhere(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); + // Same content in both clones -> same oid/CID; read from either. + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + + // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). + let mut withhold = seed_repo(&owner_did, "withhold"); + withhold.updated_at = Utc::now(); + state + .db + .create_repo(&withhold) + .await + .expect("withhold repo"); + state + .db + .set_visibility_rule( + &withhold.id, + "/secret/**", + VisibilityMode::B, + &[], + &owner_did, + ) + .await + .expect("deny rule"); + + // Public copy, no rules, iterated AFTER. + let mut pubcopy = seed_repo(&owner_did, "pubcopy"); + pubcopy.updated_at = Utc::now() - chrono::Duration::seconds(60); + state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); + + // anon: denied at the withholding repo (continue), served from the public copy. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "served from the public copy despite the other deny" + ); + assert!( + body.contains("TOP SECRET"), + "the public copy serves the content" + ); + } + + /// Repo-level "/" gate (KTD2a, first continue branch): a fully private repo + /// (is_public=false, no rules) denies anon before any per-blob check; the + /// owner still reads. The path-scoped tests pass the "/" gate and deny at the + /// per-blob stage, so this exercises the coarser repo-level deny separately. + #[sqlx::test] + async fn ipfs_cid_private_repo_denies_anon_at_repo_gate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["priv"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("priv.git"); + let blob_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let mut rec = seed_repo(&owner_did, "priv"); + rec.is_public = false; + state.db.create_repo(&rec).await.expect("private repo"); + + // anon → repo-level deny → 404, no content leaked. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&blob_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "anon denied at a private repo's / gate" + ); + assert!(!body.contains("public bytes"), "404 must not leak content"); + + // owner-signed → 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &blob_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "owner reads their private repo's object" + ); + assert!(body.contains("public bytes"), "owner gets the content"); + } + + /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref + /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — + /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), + /// the handler skips the whole repo rather than serving. Asserts no leak of the + /// withheld blob AND that even the *public* blob in that repo is withheld — the + /// latter distinguishes fail-closed-skip from normal per-blob withholding and + /// would serve 200 if the error arm wrongly proceeded. + #[sqlx::test] + async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Recorded pins so get_by_cid resolves each CID to its oid and reaches the + // walk; the 404s below are then the fail-closed skip, not a table miss. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Force the withheld walk to fail closed: a ref pointing at a blob (not + // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` + // propagates as Err → the handler's `Ok(Err)` arm skips the repo. + std::fs::write( + bare.join("refs/heads/blobref"), + format!("{}\n", fx.secret_oid), + ) + .unwrap(); + + state + .db + .create_repo(&seed_repo(&owner_did, "withhold")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "withhold") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + // Withheld secret CID under a walk error → 404, no leak. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "walk error must not serve the withheld blob" + ); + assert!( + !body.contains("TOP SECRET"), + "walk-error 404 must not leak the secret" + ); + + // The PUBLIC blob in the same repo is also 404: the walk error fails closed + // by skipping the whole repo, not by serving. Without the fail-closed arm + // this would serve 200, so this assertion is the load-bearing discriminator. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "walk error fails closed: repo skipped, even the public blob is not served" + ); + } + + /// #173 review (F2): the commit/tag reachability walk must FAIL CLOSED on a git + /// error, exactly like the blob/tree walk. A ref pointing at a nonexistent object + /// makes `rev-list --all` fail, so `reachable_commit_tag_oids` returns Err, which + /// the handler's shared `Ok(Err) => continue` arm turns into a repo skip. The + /// load-bearing discriminator is that the PUBLIC commit is ALSO 404: if the arm + /// fail-OPENed (served on error) it would 200. Drives the commit/tag branch of + /// the shared fail-closed arm specifically (the sibling test covers blob/tree). + #[sqlx::test] + async fn ipfs_cid_commit_tag_walk_error_fails_closed(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["cterr"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("cterr.git"); + // A reachable commit CID — would serve 200 if the walk succeeded. + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + + // A ref to a NONEXISTENT object: `git rev-list --all` fails ("bad object"), + // so reachable_commit_tag_oids bails → the walk arm skips the repo. + std::fs::write( + bare.join("refs/heads/broken"), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n", + ) + .unwrap(); + + state + .db + .create_repo(&seed_repo(&owner_did, "cterr")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "cterr") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // Fail-closed: a walk error skips the repo, so even the otherwise-reachable + // public commit is 404 (not served). A fail-OPEN arm would 200 here. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a commit/tag walk error must fail closed (repo skipped), never serve" + ); + } + + /// #126: a dangling blob (written via `git hash-object -w`, never referenced + /// by any commit/tree) must 404 through `GET /ipfs/{cid}` under path-scoped + /// rules — for anon AND the owner. The pre-#126 deny-set was fail-open by + /// construction: dangling oids were absent from the reachable enumeration + /// and thus absent from the deny-set, so the handler served 200. The + /// allowed-set is fail-closed: dangling oids are absent from the reachable + /// allowed-set, so the handler 404s (per team memory: the owner shift to + /// 404 is the accepted fail-closed default — owners can still + /// `git cat-file` directly). + #[sqlx::test] + async fn ipfs_cid_dangling_blob_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + // Seed a normal repo with `secret/b.txt` reachable from HEAD, so the + // path-scoped rule has something to match — without this the rule has + // no anchor and we'd be testing nothing. + let _fx = seed_cid_repos(&slug, &short, &["dangling"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangling.git"); + + // Write a dangling blob: `git hash-object -w --stdin` adds it to the + // object DB but nothing references it, so the reachable walk never + // enumerates it. + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + let stdin = child.stdin.as_mut().expect("stdin"); + stdin.write_all(b"DANGLING SECRET\n").expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!( + out.status.success(), + "git hash-object: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + // Sanity: must be a 64-hex sha256 oid, since the repo is sha256-format. + assert_eq!( + dangling_oid.len(), + 64, + "expected sha256 oid: {dangling_oid}" + ); + // Record the pin so oid_for_cid resolves it — the 404 must then come from + // the allowed-set gate excluding the dangling oid, not from a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangling")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangling") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-blob allowed-set gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + // anon: the dangling blob is absent from the reachable allowed-set → + // 404, no leak. Pre-#126 (deny-set) would serve 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&dangling_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling blob must 404 under path-scoped rules" + ); + assert!( + !body.contains("DANGLING SECRET"), + "404 body must not leak the dangling content" + ); + + // owner (signed): same 404. The dangling blob has no path, so it's + // never visibility-checked → never in the allowed set, even for the + // owner. This is the accepted fail-closed shift documented in the PR. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &dangling_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "owner also 404s on dangling blobs under path-scoped rules (fail-closed default)" + ); + assert!(!body.contains("DANGLING SECRET")); + } + + /// #135: a DANGLING tree (in the ODB, referenced by no commit) 404s under + /// path-scoped rules for anon AND owner — the reachable-only allowed-tree-set + /// never enumerates it. Handler-level companion to the helper test + /// `allowed_tree_set_excludes_dangling_tree`, proving the `get_by_cid` tree arm + /// (memo insert + `!in_allowed` continue) fails closed on the dangling case. + #[sqlx::test] + async fn ipfs_cid_dangling_tree_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangtree"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangtree.git"); + + // Dangling tree via `git mktree`: a UNIQUE entry name so its oid is + // content-distinct from every reachable tree (a content-identical tree would + // dedup to a reachable oid — that is T2, not danglingness). + let mut child = std::process::Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("spawn git mktree"); + { + use std::io::Write; + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {}\tdangling-only-unreferenced.txt", + fx.secret_oid + ) + .unwrap(); + } + let out = child.wait_with_output().expect("mktree output"); + assert!( + out.status.success(), + "git mktree: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_tree_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + assert_eq!(dangling_tree_oid.len(), 64, "expected sha256 oid"); + // Record the pin so the 404 is the allowed-tree-set gate excluding the + // dangling tree, not a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangtree")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangtree") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for req in [cid_anon(&dangling_cid), cid_signed(&owner, &dangling_cid)] { + let (st, _) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling tree must 404 under path-scoped rules (anon + owner)" + ); + } + } + + /// #173 (F1): a QUARANTINED repo must not serve a pinned object by CID, to anon + /// OR to the mirror's own owner — quarantine is "hidden from serve/clone/listings, + /// owner included" (authorize_repo_read / feed_quarantined_mirror_withheld_from_owner). + /// The repo is PUBLIC with no path-scoped rule, so the "/" visibility gate ALLOWS + /// it and quarantine is the sole possible denier: RED before the fix (the loop + /// never checks quarantine → serves 200 + bytes), GREEN after the quarantine skip. + /// The owner-signed 404 is the load-bearing negative — a visibility-only gate + /// would Allow the owner and miss this. + #[sqlx::test] + async fn ipfs_cid_quarantined_repo_withheld_from_anon_and_owner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["quar"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quar.git"); + // Pin a ROOT-readable object (public/a.txt) — no path-scoped rule, so only + // quarantine can deny it. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "quar")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "quar") + .await + .unwrap() + .unwrap(); + + // Baseline: before quarantine the object serves 200 (proves the CID resolves + // and the object is otherwise servable, so the 404 below is quarantine's doing). + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "public root object serves before quarantine" + ); + assert!(body.contains("public bytes"), "baseline serves the content"); + + // Quarantine it. + state + .db + .set_repo_quarantine(&rec.id, true) + .await + .expect("quarantine"); + + // anon AND owner-signed must both 404 with no content leak. + for req in [cid_anon(&public_cid), cid_signed(&owner, &public_cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "quarantined repo must not serve by CID (anon + owner)" + ); + assert!( + !body.contains("public bytes"), + "404 body must not leak quarantined content" + ); + } + } + + /// #173 (F2): a DANGLING commit or annotated tag (in the ODB, referenced by no + /// ref) must 404 under path-scoped rules for anon AND owner. The resolver proved + /// reachability only for blobs/trees, so a dangling commit/tag fell through to + /// serve, leaking its message/metadata. RED before the fix (serves 200 + + /// sentinel), GREEN after (the reachable commit/tag set excludes them). The + /// reachable-commit/tag serve path is covered by + /// ipfs_cid_gate_withholds_blob_from_unauthorized (commit + annotated tag → 200). + #[sqlx::test] + async fn ipfs_cid_dangling_commit_and_tag_fail_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangct"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangct.git"); + + // Run a git plumbing command that reads from stdin and prints an oid. + let oid_from_stdin = |args: &[&str], input: &[u8]| -> String { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.as_mut().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().expect("git output"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Dangling commit: commit-tree with a sentinel message, NO ref update. + let dangling_commit_oid = oid_from_stdin( + &["commit-tree", &fx.root_tree_oid], + b"DANGLING COMMIT SECRET\n", + ); + assert_eq!(dangling_commit_oid.len(), 64, "expected sha256 commit oid"); + // Dangling annotated tag: mktag of the dangling commit, NO ref. + let tag_body = format!( + "object {dangling_commit_oid}\ntype commit\ntag dang\ntagger t 0 +0000\n\nDANGLING TAG SECRET\n" + ); + let dangling_tag_oid = oid_from_stdin(&["mktag"], tag_body.as_bytes()); + assert_eq!(dangling_tag_oid.len(), 64, "expected sha256 tag oid"); + + let commit_cid = pin_cid_for(&bare, &dangling_commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &dangling_tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangct")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangct") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-object gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for (cid, sentinel) in [ + (&commit_cid, "DANGLING COMMIT SECRET"), + (&tag_cid, "DANGLING TAG SECRET"), + ] { + for req in [cid_anon(cid), cid_signed(&owner, cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling commit/tag must 404 under path-scoped rules (anon + owner)" + ); + assert!( + !body.contains(sentinel), + "404 body must not leak the dangling message: {sentinel}" + ); + } + } + } + + /// #173 review (F2 hardening): a REACHABLE commit must still serve under a + /// path-scoped rule even when the repo carries a pushable non-commit ref (an + /// annotated tag of a tree, accepted by receive-pack). `reachable_commit_tag_oids` + /// must NOT route through `assert_all_refs_are_commits` (which bails on such a + /// ref and would fail-closed 404 every reachable commit/tag CID in the repo). + /// RED before the decoupling (the guard bails → 404), GREEN after. + #[sqlx::test] + async fn ipfs_cid_reachable_commit_served_despite_non_commit_ref(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["weirdref"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("weirdref.git"); + + // A pushable non-commit ref: an annotated tag pointing at a TREE. `git tag -a` + // in the bare repo creates refs/tags/treetag -> tag object -> tree, which + // peels to a non-commit and makes assert_all_refs_are_commits bail. + let out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &fx.root_tree_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&bare) + .output() + .expect("git tag -a"); + assert!( + out.status.success(), + "git tag -a: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Pin the REACHABLE root commit. + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "weirdref")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "weirdref") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // The reachable commit must still serve — the non-commit ref must not + // fail-closed the whole repo's commit/tag CID retrieval. + let resp = cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a reachable commit must serve despite a pushable non-commit ref in the repo" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.commit_oid.as_str()), + "the served object is the reachable root commit" + ); + } + + /// #173 review (F-F): an annotated tag pointing at a TREE is pushable through + /// receive-pack, and the TREE allowed-set path + /// (`allowed_tree_set_for_caller` -> `tree_paths` -> `reachable_commits`) runs + /// `assert_all_refs_are_commits`, which bails on that ref and fail-closes the + /// whole repo — 404-ing EVERY tree CID (root + public subtrees) for its owner + /// and readers, not just the offending tag. The tree allowed-set feeds ONLY the + /// CID gate (absence = fail-closed 404), so `tree_paths` uses the lenient + /// reachable-commit enumeration: commit-reachable trees still serve, while a + /// tree reachable only via such a tag stays excluded. `blob_paths` keeps the + /// strict guard (it also feeds serve/replication, where a miss under-withholds). + /// RED before the decoupling (whole-repo bail -> 404 on the root/public tree), + /// GREEN after; the withheld-subtree 404 is the load-bearing must-not. + #[sqlx::test] + async fn ipfs_cid_tree_served_despite_non_commit_ref(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["treeweird"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("treeweird.git"); + + // Pushable non-commit ref: an annotated tag pointing at the ROOT TREE. + let out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &fx.root_tree_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&bare) + .output() + .expect("git tag -a"); + assert!( + out.status.success(), + "git tag -a: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Pin the reachable root tree and public subtree (both at ALLOWED paths), + // plus the secret subtree (a DENIED path — the fail-closed negative). + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "treeweird")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "treeweird") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-object tree gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // Reachable trees at ALLOWED paths must still serve despite the tag-of-tree. + for (cid, want_oid, label) in [ + (&root_tree_cid, &fx.root_tree_oid, "root tree"), + (&public_tree_cid, &fx.public_tree_oid, "public subtree"), + ] { + let resp = cid_router(&state).oneshot(cid_anon(cid)).await.unwrap(); + let served = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "{label} CID must serve despite a pushable tag-of-tree in the repo" + ); + assert_eq!( + served.as_deref(), + Some(want_oid.as_str()), + "{label}: the served object is the reachable tree" + ); + } + + // Fail-closed preserved: the DENIED subtree's CID is still withheld — the + // lenient walk must not under-withhold a path the caller cannot read. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_tree_cid)) + .await + .unwrap(), ) .await; assert_eq!( st, StatusCode::NOT_FOUND, - "non-reader must not read the withheld blob" + "a withheld subtree's tree CID stays 404 (lenient walk must not under-withhold)" ); - assert!(!body.contains("TOP SECRET")); + } + + /// #173 review (F2 hardening): the INNER tag object of a nested tag-of-a-tag is + /// reachable (via the outer ref tag) and pinnable, so its CID must serve under a + /// path rule. `reachable_commit_tag_oids` peels tag chains to include it. RED + /// before the peel loop (the inner tag is not a ref tip and rev-list dereferences + /// to the commit, so it is absent → 404), GREEN after. + #[sqlx::test] + async fn ipfs_cid_nested_tag_inner_object_served(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nested"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nested.git"); + + let git_stdin = |args: &[&str], input: &[u8]| -> String { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.as_mut().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().expect("git output"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Inner annotated tag of the reachable commit (no ref of its own). + let inner_body = format!( + "object {}\ntype commit\ntag inner\ntagger t 0 +0000\n\ninner\n", + fx.commit_oid + ); + let inner_tag_oid = git_stdin(&["mktag"], inner_body.as_bytes()); + // Outer annotated tag of the inner tag, then a ref to the outer tag. The + // inner tag is reachable only THROUGH the outer, not as a ref tip. + let outer_body = format!( + "object {inner_tag_oid}\ntype tag\ntag outer\ntagger t 0 +0000\n\nouter\n" + ); + let outer_tag_oid = git_stdin(&["mktag"], outer_body.as_bytes()); + let out = std::process::Command::new("git") + .args(["update-ref", "refs/tags/nested", &outer_tag_oid]) + .current_dir(&bare) + .output() + .expect("update-ref"); + assert!( + out.status.success(), + "update-ref: {}", + String::from_utf8_lossy(&out.stderr) + ); + + let inner_cid = pin_cid_for(&bare, &inner_tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "nested")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "nested") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&inner_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "the inner tag of a nested tag-of-a-tag is reachable and must serve" + ); + } + + /// #135: with NO path-scoped rule the per-object gate is skipped, so a tree CID + /// is served (the `"/"` gate is the whole story). Guards against over-gating + /// trees — the tree analog of the blob skip-walk branch. + #[sqlx::test] + async fn ipfs_cid_tree_served_when_no_path_scoped_rule(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nopathrule"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nopathrule.git"); + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // Public repo, no visibility rules → has_path_scoped_rule is false. + state + .db + .create_repo(&seed_repo(&owner_did, "nopathrule")) + .await + .expect("seed repo"); + + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "tree served to anon when no path-scoped rule exists" + ); + assert!( + bytes_contain(&body, b"b.txt"), + "served tree carries its child structure" + ); + } + + /// #173 (Fix 1): the pinned_cids lookup must use the canonical base32 CID, not + /// the raw request spelling. A pin is stored under `cid.to_string()` (canonical + /// base32); a request carrying the SAME CID re-encoded to a different multibase + /// (base58btc) parses and passes the sha2-256 check but, on the pre-fix handler, + /// misses the lookup key → false 404. Public repo, no path-scoped rule, so no + /// walk — this isolates the lookup-key canonicalization. + #[sqlx::test] + async fn ipfs_alt_encoding_cid_resolves(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["altenc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("altenc.git"); + // Canonical base32 CID as stored by the pin path. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Public repo, no visibility rules (no path-scoped walk). + state + .db + .create_repo(&seed_repo(&owner_did, "altenc")) + .await + .expect("seed repo"); + + // Re-encode the SAME CID to base58btc — a different, equally-valid spelling + // that is NOT the stored key. The `cid` crate re-exports `multibase`. + let alt = public_cid + .parse::>() + .unwrap() + .to_string_of_base(cid::multibase::Base::Base58Btc) + .unwrap(); + assert_ne!(alt, public_cid, "alt encoding must differ from canonical"); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&alt)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "alt-multibase spelling of a pinned CID must resolve (canonicalized lookup)" + ); + assert!( + body.contains("public bytes"), + "resolved object serves its content" + ); + } + + /// #173 (Fix 2a, db-level): `oids_for_cid` returns EVERY oid recorded under a + /// CID, not an arbitrary one. `record_pinned_cid` is unique on the git oid and + /// non-unique on cid, so two distinct oids can share one content-CID. Old + /// `oid_for_cid` did `LIMIT 1`; the new plural method must surface both. + #[sqlx::test] + async fn oids_for_cid_returns_all_duplicates(pool: PgPool) { + let state = test_state(pool).await; + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"shared content cid").to_string(); + let oid_a = "a".repeat(64); + let oid_b = "b".repeat(64); + state + .db + .record_pinned_cid(&oid_a, &cid, None) + .await + .unwrap(); + state + .db + .record_pinned_cid(&oid_b, &cid, None) + .await + .unwrap(); + + let mut oids = state.db.oids_for_cid(&cid).await.unwrap(); + oids.sort(); + assert_eq!( + oids, + vec![oid_a, oid_b], + "oids_for_cid must return every oid recorded under the shared CID" + ); + } - // owner (signed) → 200 + secret bytes. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&owner, &secret_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "owner reads the withheld blob"); - assert!(body.contains("TOP SECRET"), "owner gets the content"); + /// #173 (Fix 2b, handler-level): when two oids collide on one CID and the + /// first-recorded is absent from every repo while the second is a readable + /// public object, the handler must try both and serve the readable one. The + /// pre-fix handler resolved a single oid (LIMIT 1 → first-inserted for equal + /// keys) and 404'd. Ordering caveat: this relies on `oids_for_cid` returning + /// the absent oid before the readable one (heap/insert order for equal keys); + /// if that ordering ever changes, `oids_for_cid_returns_all_duplicates` remains + /// the load-bearing, deterministic driver for Fix 2. + #[sqlx::test] + async fn ipfs_cid_collision_serves_readable_duplicate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["collision"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("collision.git"); + + // A GENUINE content collision: the shared CID is the readable object's REAL + // content CID, and a second (absent) oid is recorded under the SAME cid. The + // handler must try every oid and serve the one whose bytes hash to the CID. + // (F2, #173: the served bytes must match the requested content address, so the + // shared cid has to be the object's real cid — an arbitrary seed would now be + // withheld by the integrity check as an unverifiable provider-CID-style row.) + let (_ty, raw) = crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .unwrap(); + let shared_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + let absent_oid = "c".repeat(64); + state + .db + .record_pinned_cid(&absent_oid, &shared_cid, None) + .await + .expect("record absent oid first"); + state + .db + .record_pinned_cid(&fx.public_oid, &shared_cid, None) + .await + .expect("record readable oid second"); + + // Public repo, no rules → the readable public object is served if reached. + state + .db + .create_repo(&seed_repo(&owner_did, "collision")) + .await + .expect("seed repo"); - // listed reader (signed) → 200. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_signed(&reader, &secret_cid)) + .oneshot(cid_anon(&shared_cid)) .await .unwrap(), ) .await; - assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); - assert!(body.contains("TOP SECRET")); + assert_eq!( + st, + StatusCode::OK, + "handler must try every oid under the CID and serve the readable duplicate" + ); + assert!( + body.contains("public bytes"), + "the readable duplicate's content is served" + ); + } + + /// #173 (Fix 3/F3, INV-10): the expensive legacy fan-out is rate-limited per + /// source IP. A valid tree CID makes the object-type pre-check pass, so each + /// repeat request pays a fresh walk (request-scoped memo only) — unbounded + /// amplification. Since #173-F3 (jatmn) the source charge sits on the LEGACY + /// PROBE (`acquire` + `cat-file`), which precedes the walk, so every legacy + /// candidate is charged to the non-farmable source IP from the first probe; a + /// second identical request from the same IP is shed with 429, but a targeted + /// PROVENANCE fetch (no scan) and a request from a different IP are unaffected. + /// The limiter is sized to admit one full scan of the two seeded repos (2 probes) + /// so the first request serves; the repeat then finds the bucket spent. + #[sqlx::test] + async fn ipfs_walk_rate_limited_per_source(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + + let mut state = test_state(pool).await; + // The scan probes both seeded repos (walklimit + walkpublic) per request, so + // size the per-IP budget to admit exactly one full scan (2 probes). A repeat + // scan from the same IP then finds the bucket spent. Keyed on the rightmost + // X-Forwarded-For hop so the test can choose a source IP under `oneshot`. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["walklimit"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walklimit.git"); + // The tree CID drives a path-scoped walk (the load-bearing amplification + // surface). The reader is allowed under /secret so the walk returns 200. + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // Oldest `updated_at` → `list_all_repos` (ORDER BY updated_at DESC) probes + // this serving repo LAST, so a scan deterministically charges the walk-free + // `walkpublic` miss first then this serve: exactly 2 probes per scan. + let mut walklimit = seed_repo(&owner_did, "walklimit"); + walklimit.updated_at = chrono::Utc::now() - chrono::Duration::seconds(60); + state.db.create_repo(&walklimit).await.expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "walklimit") + .await + .unwrap() + .unwrap(); + // Mode B path rule over /secret with the reader allowed → the reader's + // secret-tree fetch runs the allowed-tree walk and returns 200. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + + // The MUST-NOT object must be a genuinely CHEAP fetch: an object served + // from a repo with NO path-scoped rule takes the no-walk path, so the WALK + // brake never rate-limits it. It has to live in a repo that carries no path + // rule AND whose object graph does not overlap `walklimit` (a blob shared + // with the path-scoped repo would still walk there), so we seed a second bare + // repo with UNIQUE content. `acquire(owner, "walkpublic")` resolves to + // `/tmp//walkpublic.git`. This copy is PROVENANCED (`pin_cid_for_repo`) + // so it resolves straight to its repo and skips the legacy probe brake: the + // point here is the WALK brake, and post-#173-F3 a walk-free LEGACY fetch is + // itself source-charged at the probe, so a legacy pin would (correctly) be + // shed from the exhausted IP and no longer isolate the walk brake. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walkpublic.git"); + { + use std::process::Command; + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let src = std::env::temp_dir().join(format!("gl-cid-pub-{short}")); + let _ = std::fs::remove_dir_all(&src); + std::fs::create_dir_all(&src).unwrap(); + std::fs::write(src.join("cheap.txt"), b"cheap public bytes\n").unwrap(); + run(&["init", "-q", "--object-format=sha256"], &src); + run(&["config", "user.email", "t@t"], &src); + run(&["config", "user.name", "t"], &src); + run(&["add", "."], &src); + run(&["commit", "-qm", "cheap"], &src); + let _ = std::fs::remove_dir_all(&pub_bare); + run( + &[ + "clone", + "--bare", + "-q", + src.to_str().unwrap(), + pub_bare.to_str().unwrap(), + ], + &src, + ); + let _ = std::fs::remove_dir_all(&src); + } + let cheap_oid = { + use std::process::Command; + let out = Command::new("git") + .args(["rev-parse", "HEAD:cheap.txt"]) + .current_dir(&pub_bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse cheap.txt"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + // Public repo, NO visibility rules → the cheap object takes the no-walk path. + state + .db + .create_repo(&seed_repo(&owner_did, "walkpublic")) + .await + .expect("seed public repo"); + let pub_rec = state + .db + .get_repo(&owner_did, "walkpublic") + .await + .unwrap() + .unwrap(); + let public_cid = pin_cid_for_repo(&pub_bare, &cheap_oid, &state.db, &pub_rec.id).await; - // KTD3: anon tree CID under /secret → 200 (trees/commits are not withheld). + // 1st legacy scan from 1.2.3.4 → 200 (its two probes fit the budget; the + // walk ran, reader allowed). let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&tree_cid)) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) .await .unwrap(), ) .await; - assert_eq!(st, StatusCode::OK, "tree object is served to anon (KTD3)"); + assert_eq!( + st, + StatusCode::OK, + "1st legacy scan from a source IP is served" + ); - // R3: public blob anon → 200 (non-withheld content not affected). + // 2nd identical scan from the SAME IP → 429 (per-IP probe budget spent). let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&public_cid)) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) .await .unwrap(), ) .await; - assert_eq!(st, StatusCode::OK, "public blob stays served"); + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "2nd legacy scan from the same source IP is shed with 429" + ); - // R5: a genuine unknown CID also 404, uniform with the withheld 404. - let absent_cid = cid_for_oid(&"ab".repeat(32)); - let (st, _) = cid_parts( + // MUST-NOT: a targeted PROVENANCE fetch (no scan, no probe brake) from the + // SAME limited IP, even after the 429, is served: the brake is on the legacy + // scan, not the route. + let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&absent_cid)) + .oneshot(cid_signed_xff(&reader, &public_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "absent CID 404 (uniform with withheld)" + StatusCode::OK, + "a provenance (non-scan) fetch is never rate-limited, even from the exhausted IP" + ); + assert!( + body.contains("cheap public bytes"), + "the cheap fetch serves content" ); - // malformed CID → 400 (unchanged). + // PER-SOURCE isolation: the same tree-CID scan from a DIFFERENT IP → 200. let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon("not-a-cid")) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "5.6.7.8")) .await .unwrap(), ) .await; - assert_eq!(st, StatusCode::BAD_REQUEST, "malformed CID still 400"); + assert_eq!( + st, + StatusCode::OK, + "one source's exhaustion must not shed another source's walk" + ); } - /// R4: the same object withheld in one repo but public in another is still - /// served from the public copy; the withholding repo is iterated first. + /// #173 review (F-C): a SKIPPED legacy candidate (a walk-and-deny denier, OR a + /// probe-throttled repo since #173-F3) must not end the whole request: the scan + /// keeps going so a later walk-free copy still serves, and a spent probe budget is + /// a clean 429, never a false 404/503. Otherwise a public CID would 404/429 solely + /// because a newer path-scoped duplicate sorts ahead of an older no-rule copy under + /// `updated_at DESC`. Two same-oid legacy copies: a NEWER `/secret`-scoped denier + /// and an OLDER no-rule public copy. + /// + /// Two requests from the SAME IP, budget = 2 (one full scan of both copies): + /// req1 probes the denier (charged), its allowed-blob walk denies anon → skip and + /// keep scanning, then probes+serves the walk-free public copy → 200. That proves + /// the denier skip is non-fatal (`continue`, not `break`). req2 from the same IP + /// finds the probe budget spent, so the denier's probe throttles → skip-continue, + /// the public copy's probe throttles too → nothing servable → a clean 429 (not a + /// truncation 503 nor a false 404), proving the throttle is likewise non-fatal but + /// correctly shed. RED before `continue` (a `break` on the skipped denier 404s + /// req1 outright). #[sqlx::test] - async fn ipfs_cid_served_from_public_copy_when_withheld_elsewhere(pool: PgPool) { + async fn ipfs_walk_quota_skips_denier_and_serves_public_copy(pool: PgPool) { use crate::db::VisibilityMode; use chrono::Utc; use gitlawb_core::identity::Keypair; @@ -2091,61 +5966,109 @@ mod tests { let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); - let state = test_state(pool).await; - - let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - - // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). - let mut withhold = seed_repo(&owner_did, "withhold"); - withhold.updated_at = Utc::now(); + let mut state = test_state(pool).await; + // Budget = one full two-repo scan (2 probes), keyed on the rightmost XFF hop + // so `oneshot` can choose a source IP (no socket peer). A repeat scan from the + // same IP then finds the budget spent. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + // Identical secret-blob content in both bare clones → one CID resolves to + // `secret_oid` in each. A NEWER path-scoped denier (walk-and-deny anon) and an + // OLDER no-rule public copy (walk-free serve). + let fx = seed_cid_repos(&slug, &short, &["scopeddenier", "publiccopy"]); + let denier_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("scopeddenier.git"); + let secret_cid = pin_cid_for(&denier_bare, &fx.secret_oid, &state.db).await; + + // Newer denier: public at "/", `/secret/**` Mode B empty readers → an anon + // blob fetch clears "/", runs the allowed-blob walk, is denied → continue. + let mut denier = seed_repo(&owner_did, "scopeddenier"); + denier.updated_at = Utc::now(); + state.db.create_repo(&denier).await.expect("seed denier"); state .db - .create_repo(&withhold) + .set_visibility_rule(&denier.id, "/secret/**", VisibilityMode::B, &[], &owner_did) .await - .expect("withhold repo"); + .expect("path rule"); + + // Older public copy — NO rule → the secret blob serves via the no-walk path. + let mut public = seed_repo(&owner_did, "publiccopy"); + public.updated_at = Utc::now() - chrono::Duration::seconds(60); state .db - .set_visibility_rule( - &withhold.id, - "/secret/**", - VisibilityMode::B, - &[], - &owner_did, - ) + .create_repo(&public) .await - .expect("deny rule"); + .expect("seed public copy"); - // Public copy, no rules, iterated AFTER. - let mut pubcopy = seed_repo(&owner_did, "pubcopy"); - pubcopy.updated_at = Utc::now() - chrono::Duration::seconds(60); - state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); + // req1 from 1.2.3.4: the denier is skipped (walk denies anon) and the scan + // keeps going to serve the older walk-free public copy. Both probes fit the + // budget, so this leaves the IP bucket spent. + let resp = cid_router(&state) + .oneshot(cid_anon_xff(&secret_cid, "1.2.3.4")) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a skipped walk-requiring denier must not end the scan: the later walk-free public copy still serves" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.secret_oid.as_str()), + "the served object is the secret blob from the no-rule public copy" + ); - // anon: denied at the withholding repo (continue), served from the public copy. - let (st, body) = cid_parts( + // req2 from the SAME exhausted IP: every legacy probe is now throttled. The + // throttle is non-fatal (skip and keep scanning), but nothing is servable, so + // it resolves to a clean 429, not a truncation 503, not a false 404. + let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&secret_cid)) + .oneshot(cid_anon_xff(&secret_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::OK, - "served from the public copy despite the other deny" - ); - assert!( - body.contains("TOP SECRET"), - "the public copy serves the content" + StatusCode::TOO_MANY_REQUESTS, + "with the probe budget spent, the repeat legacy scan is shed with a clean 429" ); } - /// Repo-level "/" gate (KTD2a, first continue branch): a fully private repo - /// (is_public=false, no rules) denies anon before any per-blob check; the - /// owner still reads. The path-scoped tests pass the "/" gate and deny at the - /// per-blob stage, so this exercises the coarser repo-level deny separately. + /// INV-10 amplification bound: a single `GET /ipfs/{cid}` must not fan out an + /// unbounded number of full-history walks. The route brake (`ipfs_rate_limiter`) + /// fires once per request and the per-walk `ipfs_work_rate_limiter` charge bounds + /// walk work across requests, but within ONE request the same object can exist under + /// path-scoped rules in many repos, each paying its own walk. + /// `MAX_HISTORY_WALKS_PER_REQUEST` caps that fan-out. + /// + /// Load-bearing witness (#173, F4): a readable public copy (no path rule → + /// served via the no-walk path, exactly like + /// `ipfs_cid_served_from_public_copy_when_withheld_elsewhere`) is given the + /// OLDEST `updated_at` so `list_all_repos` (ORDER BY updated_at DESC) iterates it + /// LAST. Ahead of it sit `cap + 1` path-scoped deniers, each forcing an + /// allowed-blob walk that denies anon. The cap bounds SPAWNED walks to `cap`, but + /// hitting it must `continue` (skip only the walk-requiring denier), NOT `break` + /// the whole repo loop: the walk-free public copy needs no walk, so it is still + /// reached and served (200, `x-git-hash` = the blob oid). The old `break` + /// wrongly 404'd this publicly-readable content. Reverting `continue`→`break` + /// turns this 200 back into a 404: the RED proof that the loop keeps scanning for + /// a cheap readable copy after the cap. The `cap` walk ceiling still holds — only + /// `cap` walks are spawned across the deniers regardless (the amplification bound + /// is proven separately by `ipfs_walk_cap_still_serves_walk_free_candidate`). #[sqlx::test] - async fn ipfs_cid_private_repo_denies_anon_at_repo_gate(pool: PgPool) { + async fn ipfs_walk_fanout_capped_per_request(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); @@ -2154,54 +6077,99 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["priv"]); - let blob_cid = cid_for_oid(&fx.public_oid); + let cap = crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize; - let mut rec = seed_repo(&owner_did, "priv"); - rec.is_public = false; - state.db.create_repo(&rec).await.expect("private repo"); + // `cap + 1` deniers guarantee the fan-out crosses the ceiling before the + // readable copy (iterated last) is reached. All bare clones share identical + // content, so the one secret-BLOB CID resolves to `secret_oid` in every repo. + let denier_names: Vec = (0..=cap).map(|i| format!("denier{i}")).collect(); + let mut names: Vec<&str> = vec!["readable"]; + names.extend(denier_names.iter().map(|s| s.as_str())); + let fx = seed_cid_repos(&slug, &short, &names); - // anon → repo-level deny → 404, no content leaked. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&blob_cid)) + let readable_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("readable.git"); + // The secret BLOB CID drives the path-scoped allowed-blob walk in every + // denier (the amplification surface) and is served cheaply from the + // no-rule public copy — the proven serve path. + let secret_cid = pin_cid_for(&readable_bare, &fx.secret_oid, &state.db).await; + + // 1) Readable public copy — OLDEST updated_at → iterated LAST. Public with + // NO visibility rule, so the blob serves via the no-walk path. This is + // the copy an uncapped fan-out would eventually reach and serve. + let mut readable = seed_repo(&owner_did, "readable"); + readable.updated_at = Utc::now() - chrono::Duration::seconds(60); + state + .db + .create_repo(&readable) + .await + .expect("seed readable copy"); + + // 2) cap+1 deniers with NEWER updated_at → iterated before the copy. Public + // at "/", but a `/secret/**` Mode B rule with an EMPTY reader list, so an + // anon blob fetch clears the "/" gate, runs the allowed-blob walk, and is + // denied (the secret blob is in no one's set) → continue. Each distinct + // repo.id is its own walk (the memo only dedups the same repo). + for name in &denier_names { + let mut denier = seed_repo(&owner_did, name); + denier.updated_at = Utc::now(); + state.db.create_repo(&denier).await.expect("seed denier"); + state + .db + .set_visibility_rule(&denier.id, "/secret/**", VisibilityMode::B, &[], &owner_did) .await - .unwrap(), - ) - .await; - assert_eq!( - st, - StatusCode::NOT_FOUND, - "anon denied at a private repo's / gate" - ); - assert!(!body.contains("public bytes"), "404 must not leak content"); + .expect("path rule"); + } - // owner-signed → 200. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&owner, &blob_cid)) - .await - .unwrap(), - ) - .await; + // Anon (no peer, no XFF → the IP brake is skipped, so the walk cap is the + // only thing in play). After the cap, `continue` skips only the + // walk-requiring deniers and keeps scanning, reaching the walk-free public + // copy (iterated last) → served 200. The served object is the secret blob + // from the no-rule public copy, which is legitimately public THERE. + let resp = cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; assert_eq!( st, StatusCode::OK, - "owner reads their private repo's object" + "hitting the walk cap must skip only the walk-requiring candidate, not abandon the walk-free readable copy" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.secret_oid.as_str()), + "the served object is the blob from the no-rule public copy reached after the cap" ); - assert!(body.contains("public bytes"), "owner gets the content"); } - /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref - /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — - /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), - /// the handler skips the whole repo rather than serving. Asserts no leak of the - /// withheld blob AND that even the *public* blob in that repo is withheld — the - /// latter distinguishes fail-closed-skip from normal per-blob withholding and - /// would serve 200 if the error arm wrongly proceeded. + /// Multi-oid companion to `ipfs_walk_fanout_capped_per_request`: exercises the + /// outer oid loop and proves the per-request walk budget PERSISTS across oid + /// candidates, so a commit/tag candidate cannot re-open the fan-out. Since #173 + /// (F2) a `commit`/`tag` under a path-scoped rule is itself walk-gated (its + /// reachability is proven by a `rev-list` walk via `reachable_commit_tag_oids`), + /// so it is NOT walk-free — it draws from the same budget as the blob/tree walks. + /// + /// One CID → TWO oids (the non-unique cid index, #173): a withheld `/secret` + /// blob (walk-triggering, denied to anon in every denier) recorded FIRST so a + /// seq scan tries it first and burns the whole walk budget across the deniers; + /// the reachable root commit is second. Because the budget is already spent, the + /// commit candidate's reachability walk is also capped in every denier, so the + /// request 404s — proving commit/tag walks (F2) respect the fan-out ceiling and + /// cannot be used to bypass it (R6/F3). A reachable commit served with budget to + /// spare is covered by `ipfs_cid_gate_withholds_blob_from_unauthorized`. The + /// withheld blob must not leak. Since #173 F2 a scan the walk cap truncated + /// returns 503 (absence unproven), not the old opaque 404. #[sqlx::test] - async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { + async fn ipfs_walk_commit_tag_candidate_respects_the_walk_cap(pool: PgPool) { use crate::db::VisibilityMode; + use chrono::Utc; use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); @@ -2210,184 +6178,295 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let public_cid = cid_for_oid(&fx.public_oid); - - // Force the withheld walk to fail closed: a ref pointing at a blob (not - // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` - // propagates as Err → the handler's `Ok(Err)` arm skips the repo. - let bare = std::path::PathBuf::from("/tmp") - .join(&slug) - .join("withhold.git"); - std::fs::write( - bare.join("refs/heads/blobref"), - format!("{}\n", fx.secret_oid), - ) - .unwrap(); + let cap = crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize; + // cap+1 path-scoped deniers, all carrying identical content (same oids). + let denier_names: Vec = (0..=cap).map(|i| format!("m{i}")).collect(); + let names: Vec<&str> = denier_names.iter().map(|s| s.as_str()).collect(); + let fx = seed_cid_repos(&slug, &short, &names); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("m0.git"); + + // ONE cid → TWO oids. The withheld blob is recorded first (seq scan lists it + // first → tried first → burns the budget); the reachable commit is second. + let multi_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; state .db - .create_repo(&seed_repo(&owner_did, "withhold")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "withhold") - .await - .unwrap() - .unwrap(); - state - .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .record_pinned_cid(&fx.commit_oid, &multi_cid, None) .await - .expect("deny rule"); + .expect("co-locate the commit oid under the same cid"); - // Withheld secret CID under a walk error → 404, no leak. + for name in &denier_names { + let mut d = seed_repo(&owner_did, name); + d.updated_at = Utc::now(); + state.db.create_repo(&d).await.expect("seed denier"); + state + .db + .set_visibility_rule(&d.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // Anon: the blob candidate is denied in every denier (a walk each, spending + // the budget); the commit candidate's reachability walk is then also capped + // in every denier — so no candidate is served AND the walk cap truncated the + // scan, leaving absence unproven → 503 (not the old false 404, #173 F2). + // Either way commit/tag walks respect the ceiling and cannot re-open the + // fan-out (R6/F3). The withheld blob must not leak in the body. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&secret_cid)) + .oneshot(cid_anon(&multi_cid)) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error must not serve the withheld blob" + StatusCode::SERVICE_UNAVAILABLE, + "a commit/tag reachability walk respects the per-request cap; a truncated scan is 503, not a false 404" ); assert!( !body.contains("TOP SECRET"), - "walk-error 404 must not leak the secret" + "the withheld blob must not leak in the truncation response" ); + } - // The PUBLIC blob in the same repo is also 404: the walk error fails closed - // by skipping the whole repo, not by serving. Without the fail-closed arm - // this would serve 200, so this assertion is the load-bearing discriminator. + /// #173 (F3, INV-15): the per-IP quota debits ONE token per expensive legacy + /// candidate, not once per request, so one IP cannot drive an unbounded fan-out. + /// With quota=1 and two path-scoped deniers holding one CID, a SINGLE request is + /// shed at 429: since #173-F3 (jatmn) each legacy PROBE (`acquire` + `cat-file`, + /// which precedes the walk) debits, so the first denier probes+walks+denies on + /// token 1 and the second denier's probe finds no token → 429. (Before F3 the + /// debit sat on the walk; the outcome is unchanged, the charge point moved earlier + /// to also bound walk-free probes.) Defeating the per-candidate debit let one IP + /// drive up to MAX_HISTORY_WALKS_PER_REQUEST × quota expensive ops/hour. + #[sqlx::test] + async fn ipfs_walk_quota_debited_per_walk(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + // Signed but NOT a reader → cleared at "/", denied at /secret → forces a walk. + let stranger = Keypair::generate(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["w0", "w1"]); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("w0.git"); + // The secret BLOB CID forces a path-scoped allowed-blob walk in each denier. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + + // Two path-scoped deniers (Mode B /secret, empty readers): each forces a + // walk that denies the signed stranger, so ONE request spawns two walks. + for name in ["w0", "w1"] { + let d = seed_repo(&owner_did, name); + state.db.create_repo(&d).await.expect("seed denier"); + state + .db + .set_visibility_rule(&d.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // ONE request, quota 1: walk 1 debits the token, walk 2 has none → 429. let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&public_cid)) + .oneshot(cid_signed_xff(&stranger, &secret_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error fails closed: repo skipped, even the public blob is not served" + StatusCode::TOO_MANY_REQUESTS, + "the second full-history walk in one request must be shed with 429 (per-walk debit)" ); } - /// #126: a dangling blob (written via `git hash-object -w`, never referenced - /// by any commit/tree) must 404 through `GET /ipfs/{cid}` under path-scoped - /// rules — for anon AND the owner. The pre-#126 deny-set was fail-open by - /// construction: dangling oids were absent from the reachable enumeration - /// and thus absent from the deny-set, so the handler served 200. The - /// allowed-set is fail-closed: dangling oids are absent from the reachable - /// allowed-set, so the handler 404s (per team memory: the owner shift to - /// 404 is the accepted fail-closed default — owners can still - /// `git cat-file` directly). + /// The periodic cleanup task must sweep the ipfs walk limiter, not only its + /// five siblings. Drives `AppState::sweep_rate_limiters` — the exact method the + /// 300s loop calls — and asserts the ipfs limiter's expired entry is evicted. + /// Dropping `ipfs_rate_limiter.cleanup()` from that method leaves the entry in + /// place (`tracked_keys` stays 1): the RED proof that the sweep covers it. #[sqlx::test] - async fn ipfs_cid_dangling_blob_fails_closed_under_path_rules(pool: PgPool) { - use crate::db::VisibilityMode; - use gitlawb_core::identity::Keypair; + async fn sweep_rate_limiters_includes_ipfs_limiter(pool: PgPool) { + let mut state = test_state(pool).await; + // Short window so a single recorded hit is already expired at sweep time. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_millis(50)); + + assert!( + state.ipfs_rate_limiter.check("1.2.3.4").await, + "record a hit on the ipfs limiter" + ); + assert_eq!( + state.ipfs_rate_limiter.tracked_keys().await, + 1, + "the source-IP key is tracked before the sweep" + ); + + // Expire the entry (still mapped — cleanup hasn't run), then sweep. + tokio::time::sleep(Duration::from_millis(60)).await; + state.sweep_rate_limiters().await; + + assert_eq!( + state.ipfs_rate_limiter.tracked_keys().await, + 0, + "the periodic sweep must evict the ipfs limiter's expired entries" + ); + } + /// U5 (R6, KTD6), the observed defect: the `/ipfs` route rate limit and the + /// resolver's per-probe WORK budget are SEPARATE buckets, so a single request with + /// one probe COMPLETES even at route limit = 1. Through the production router the + /// `rate_limit_by_ip` middleware charges `ipfs_rate_limiter` once (its 1-slot bucket + /// is now full); the handler's legacy pre-scan peek and per-probe charge then draw + /// from `ipfs_work_rate_limiter`, a different bucket, so the walk-free public copy + /// still serves 200. RED before the split (both charges on `ipfs_rate_limiter`): the + /// middleware fills the one slot, the pre-scan peek reads it throttled, nothing is + /// servable → 429 on the FIRST request. Trust None so the middleware and the handler + /// resolve the same `ConnectInfo` peer IP. + #[sqlx::test] + async fn ipfs_route_limit_1_still_serves_one_probe(pool: PgPool) { + use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); - let state = test_state(pool).await; - - // Seed a normal repo with `secret/b.txt` reachable from HEAD, so the - // path-scoped rule has something to match — without this the rule has - // no anchor and we'd be testing nothing. - let _fx = seed_cid_repos(&slug, &short, &["dangling"]); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(600, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Public, no-rule legacy pin (NULL provenance) → the resolver takes the scan + // fallback and serves walk-free (exactly one probe). + let fx = seed_cid_repos(&slug, &short, &["routeone"]); let bare = std::path::PathBuf::from("/tmp") .join(&slug) - .join("dangling.git"); + .join("routeone.git"); + let repo = seed_repo(&owner_did, "routeone"); + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; - // Write a dangling blob: `git hash-object -w --stdin` adds it to the - // object DB but nothing references it, so the reachable walk never - // enumerates it. - let mut cmd = std::process::Command::new("git"); - cmd.args(["hash-object", "-w", "--stdin"]) - .current_dir(&bare) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()); - let mut child = cmd.spawn().expect("spawn git hash-object"); - { - use std::io::Write; - let stdin = child.stdin.as_mut().expect("stdin"); - stdin.write_all(b"DANGLING SECRET\n").expect("write stdin"); - } - let out = child.wait_with_output().expect("hash-object output"); - assert!( - out.status.success(), - "git hash-object: {}", - String::from_utf8_lossy(&out.stderr) - ); - let dangling_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); - // Sanity: must be a 64-hex sha256 oid, since the repo is sha256-format. + let router = crate::server::build_router(state); + let peer: std::net::SocketAddr = "203.0.113.7:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); assert_eq!( - dangling_oid.len(), - 64, - "expected sha256 oid: {dangling_oid}" + resp.status(), + StatusCode::OK, + "a single /ipfs request with one probe must serve even at route limit = 1 \ + (the route brake and the resolver's work budget are separate buckets)" ); - let dangling_cid = cid_for_oid(&dangling_oid); + } + /// U5 (R6): the two buckets are independent — the WORK budget can be exhausted + /// (429) WITHOUT draining the ROUTE bucket. Through the production router, route + /// generous (5) but work tight (1): one request drives two legacy probes, so the + /// second probe finds the work bucket spent → 429 (the route middleware admitted it). + /// The route bucket, charged once by the middleware, still has room afterward — the + /// work charges never touched it, so it admits four more direct checks. + #[sqlx::test] + async fn ipfs_work_exhaustion_leaves_route_bucket_intact(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // A legacy pin absent from every repo so the scan probes both seeded repos: two + // probes, work budget 1 → the second probe is shed → 429. + let names = ["we0", "we1"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + state + .db + .create_repo(&seed_repo(&owner_did, n)) + .await + .expect("seed repo"); + } + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"work-exhaustion").to_string(); state .db - .create_repo(&seed_repo(&owner_did, "dangling")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "dangling") - .await - .unwrap() - .unwrap(); - // Path-scoped rule triggers the per-blob allowed-set gate (KTD4). - state - .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .record_pinned_cid(&bogus_oid, &cid, None) .await - .expect("deny rule"); + .expect("legacy pin"); - // anon: the dangling blob is absent from the reachable allowed-set → - // 404, no leak. Pre-#126 (deny-set) would serve 200. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&dangling_cid)) - .await - .unwrap(), - ) - .await; + let route_bucket = state.ipfs_rate_limiter.clone(); + let peer_ip = "203.0.113.8"; + let peer: std::net::SocketAddr = format!("{peer_ip}:5000").parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); assert_eq!( - st, - StatusCode::NOT_FOUND, - "dangling blob must 404 under path-scoped rules" + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a request whose probes exceed the work budget is shed 429 (work bucket), \ + not blocked at the route (route bucket generous)" ); + // The route bucket recorded only the single request the middleware charged; the + // work charges did not drain it. Sized 5, one used by the request → four left. + for i in 0..4 { + assert!( + route_bucket.check(peer_ip).await, + "route check {i} must still admit — work charges never drained the route bucket" + ); + } + } + + /// U5 (R6): the periodic cleanup task sweeps the NEW work-budget limiter too, not + /// only the route limiter and its siblings. Mirrors + /// `sweep_rate_limiters_includes_ipfs_limiter`: drive `sweep_rate_limiters` and + /// assert the work limiter's expired entry is evicted. Dropping the + /// `ipfs_work_rate_limiter.cleanup()` call from that method leaves the entry in place + /// (`tracked_keys` stays 1): the RED proof the sweep covers it. + #[sqlx::test] + async fn sweep_rate_limiters_includes_ipfs_work_limiter(pool: PgPool) { + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(5, Duration::from_millis(50)); + assert!( - !body.contains("DANGLING SECRET"), - "404 body must not leak the dangling content" + state.ipfs_work_rate_limiter.check("1.2.3.4").await, + "record a hit on the work limiter" + ); + assert_eq!( + state.ipfs_work_rate_limiter.tracked_keys().await, + 1, + "the source-IP key is tracked before the sweep" ); - // owner (signed): same 404. The dangling blob has no path, so it's - // never visibility-checked → never in the allowed set, even for the - // owner. This is the accepted fail-closed shift documented in the PR. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&owner, &dangling_cid)) - .await - .unwrap(), - ) - .await; + tokio::time::sleep(Duration::from_millis(60)).await; + state.sweep_rate_limiters().await; + assert_eq!( - st, - StatusCode::NOT_FOUND, - "owner also 404s on dangling blobs under path-scoped rules (fail-closed default)" + state.ipfs_work_rate_limiter.tracked_keys().await, + 0, + "the periodic sweep must evict the work limiter's expired entries" ); - assert!(!body.contains("DANGLING SECRET")); } // --------------------------------------------------------------------------- @@ -3894,4 +7973,355 @@ mod tests { "result includes the deep cert matching the prefix" ); } + + // ---- U3 (#173 F3): coalesced post-push work is REQUEUED, not dropped ---- + // + // The seam mechanics (dirty flag, atomic check-and-clear, Drop backstop) are unit + // tested next to the #174 coalescing tests in `api/repos.rs`. These drive the whole + // detached task through a mock Kubo node and a real git repo, so the requeue's + // fresh re-read (encrypt half) and fail-closed full-scan enumeration (pin half) are + // proven end to end, with DB-observable effects. Each test models a push that + // coalesced during the in-flight window by (a) marking the repo dirty via a second + // `try_begin` and (b) making the repo/policy dynamic so the FIRST pass's spawn-time + // captures are stale — a static-state test would pass vacuously over the gap. + mod u3_requeue { + use super::*; + use crate::db::VisibilityMode; + use std::collections::HashSet; + use std::path::{Path, PathBuf}; + use std::process::Command; + + fn git(args: &[&str], dir: &Path) { + let ok = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(); + assert!(ok, "git {args:?} failed"); + } + fn oid(rev: &str, dir: &Path) -> String { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(dir) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}: {out:?}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + struct Repo { + _td: tempfile::TempDir, + path: PathBuf, + } + fn init_repo() -> Repo { + let td = tempfile::TempDir::new().unwrap(); + let path = td.path().to_path_buf(); + git(&["init", "-q"], &path); + git(&["config", "user.email", "t@t"], &path); + git(&["config", "user.name", "t"], &path); + Repo { _td: td, path } + } + /// Commit `content` at `rel`, return the blob oid. + fn commit(repo: &Path, rel: &str, content: &str) -> String { + let full = repo.join(rel); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(&full, content).unwrap(); + git(&["add", "."], repo); + git(&["commit", "-qm", rel], repo); + oid(&format!("HEAD:{rel}"), repo) + } + /// Write a loose, UNREACHABLE blob (dangling object). + fn write_dangling_blob(repo: &Path, content: &str) -> String { + let out = Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(repo) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + use std::io::Write; + out.stdin + .as_ref() + .unwrap() + .write_all(content.as_bytes()) + .unwrap(); + let o = out.wait_with_output().unwrap(); + assert!(o.status.success()); + String::from_utf8_lossy(&o.stdout).trim().to_string() + } + fn new_did() -> String { + Keypair::generate().did().to_string() + } + + /// SCENARIO 2 + 5 (pin half, TAIL-PLACEMENT guard). A coalesced push on a PUBLIC + /// repo with NO path-scoped rule must still requeue its pin half: the second + /// push's new object is pinned after the task. RED without the loop (stale spawn + /// object_list never lists obj2), and RED if the check-and-clear sits inside the + /// `has_path_scoped_rule` block (a rules-free repo would never reach it). + #[sqlx::test] + async fn u3_rules_free_public_repo_requeues_pin_half(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-pin"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + // The coalesced push B adds obj2 (present at requeue time, NOT in the stale + // push-A spawn object_list). + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Push A admits (guard); push B coalesces (marks dirty). + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces while A is in flight" + ); + + // Spawn-time (push A) captures are STALE: object_list lists only obj1, no rule. + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned on the first pass" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's new object is pinned by the REQUEUE full scan (RED \ + without the loop, or if the check-and-clear sits inside the encrypt gate)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits clean" + ); + } + + /// SCENARIO 1 + 3 (encrypt half, FRESH re-read). A coalesced push adds a + /// path-scoped rule withholding a blob. The task must re-read rules FRESH on + /// requeue and seal the newly-withheld blob's recovery copy. RED without the loop + /// (pass one's stale empty rule set seals nothing). + #[sqlx::test] + async fn u3_requeue_seals_blob_withheld_by_coalesced_rule_change(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-enc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let _pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + + // Coalesced push B changes .gitlawb: withhold /secret/** from anon, grant reader. + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + // Push A captures are STALE: no rule, empty withheld set (public repo). + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the coalesced push's newly-withheld blob is sealed after the REQUEUE re-read \ + (RED without the loop: pass one's stale empty rules seal nothing)" + ); + assert!(state.encrypt_inflight.is_empty(), "guard key released"); + } + + /// SCENARIO 4 (visibility-leak negative). The requeue full scan must feed + /// `list_all_objects` through the fail-closed filter, never pin it bare: a + /// withheld secret blob and a dangling blob must NOT land in the public pin set. + #[sqlx::test] + async fn u3_requeue_full_scan_does_not_publicly_pin_withheld_or_dangling(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-leak"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + // Coalesced push adds a new public object and a dangling blob. + let new_pub_oid = commit(&git_repo.path, "public/c.txt", "more public\n"); + let dangling_oid = write_dangling_blob(&git_repo.path, "orphan bytes\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let rules = state.db.list_visibility_rules(&repo.id).await.unwrap(); + let mut withheld = HashSet::new(); + withheld.insert(secret_oid.clone()); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![pub_oid.clone()], + Some(rules), + withheld, + ) + .await; + + assert!( + state.db.is_pinned(&new_pub_oid).await.unwrap(), + "the coalesced push's new PUBLIC object is pinned by the requeue" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "a WITHHELD blob is never publicly pinned by the requeue enumeration (leak guard)" + ); + assert!( + !state.db.is_pinned(&dangling_oid).await.unwrap(), + "a DANGLING blob is never publicly pinned by the requeue enumeration (leak guard)" + ); + // The withheld blob still gets its ENCRYPTED recovery copy (not a public pin). + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "withheld blob is sealed as an encrypted recovery copy, not pinned in the clear" + ); + } + + /// SCENARIO 8 (no-coalesce happy path). A single push with no coalesced follower + /// runs exactly one pass, pins its object, and releases the key. No requeue. + #[sqlx::test] + async fn u3_no_coalesce_single_pass_pins_and_releases(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-happy"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // No second try_begin: the repo is never marked dirty. + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push admits"); + assert_eq!( + state.encrypt_inflight.dirty(&repo.id), + Some(false), + "clean, no coalesce" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "the single push's object is pinned" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released after one pass" + ); + } + } } diff --git a/crates/gitlawb-node/src/visibility.rs b/crates/gitlawb-node/src/visibility.rs index 56616872..a8d7ddba 100644 --- a/crates/gitlawb-node/src/visibility.rs +++ b/crates/gitlawb-node/src/visibility.rs @@ -437,6 +437,31 @@ mod tests { ); } + // #135 T1: a Mode-B rule on `/secret/**` must DENY the withheld directory's + // OWN path `/secret` (the `path == prefix` arm), not just strict descendants — + // otherwise get_by_cid's tree gate would serve the /secret tree object and leak + // its children. Pins parity with get_tree, which denies the /secret path. + #[test] + fn subtree_rule_denies_the_withheld_directory_itself() { + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", VisibilityMode::B, &[reader])]; + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/secret"), + Decision::Deny, + "anon denied at the withheld directory's OWN path /secret" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/public"), + Decision::Allow, + "anon allowed at a sibling path outside the withheld subtree" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, Some(reader), "/secret"), + Decision::Allow, + "listed reader allowed at the withheld directory (caller-aware)" + ); + } + // #153 regression: cross-method DID must still be denied even when the // trailing segment collides with a bare owner key. #[test] diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs new file mode 100644 index 00000000..5ec653fc --- /dev/null +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -0,0 +1,139 @@ +//! #174 U6 — INV-22 completeness guard (rung-raising). +//! +//! INV-22: a permit held per op recovers only if every path that holds it is also +//! duration-bounded and reaps the process group before releasing admission, and every +//! detached git/blocking task carries its own admission. PR #174 fixed five paths +//! (U1-U5) that violated this. Each fix has a per-unit RED/GREEN regression; together +//! those form the five-revert matrix. This guard adds the missing piece: a source-scan +//! tripwire that fails when a NEW site reintroduces the class, or when one of the five +//! gates is removed. +//! +//! It lives in `tests/` (a separate crate) on purpose: a guard that scanned the same +//! file it lives in would match its own identifier literals and pass vacuously. Here +//! the scanned `src/` files never contain this file's literals, so each check is +//! load-bearing — reverting the named gate turns the assertion red. +//! +//! These are deliberately coarse structural checks, not a parser. They cannot prove a +//! gate is *correct* (the per-unit tests do that); they prove a gate is *present and +//! not bypassed*, which is what stops the class from silently regressing. + +use std::path::Path; + +fn src(rel: &str) -> String { + let p = Path::new(env!("CARGO_MANIFEST_DIR")).join("src").join(rel); + std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())) +} + +#[test] +fn inv22_concurrency_gates_present_and_not_bypassed() { + let repos = src("api/repos.rs"); + let smart_http = src("git/smart_http.rs"); + let vis = src("git/visibility_pack.rs"); + let ipfs = src("api/ipfs.rs"); + + // U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the + // child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise + // a child that closes stdout then hangs pins the permit past the deadline. The + // probe is defined and called, so >= 2 occurrences; reverting the fix removes both. + assert!( + vis.matches("child_terminated_without_reaping").count() >= 2, + "U1/P1-a gate missing: run_bounded_git must confirm child exit via \ + child_terminated_without_reaping before signalling the watchdog" + ); + + // U2 / P1-c: on client disconnect KillGroupOnDrop must launch a detached reaper + // that runs the full TERM/grace/KILL/reap, not a lone SIGTERM. The reaper is spawned + // via a runtime handle in Drop; `Handle::try_current` is unique to that launch (the + // timeout path already has an async context and never calls it). + assert!( + smart_http.contains("Handle::try_current"), + "U2/P1-c gate missing: KillGroupOnDrop::drop must launch the full \ + TERM/grace/KILL reaper on disconnect (Handle::try_current), not a lone SIGTERM" + ); + + // U4 / P1-d: git_receive_pack must acquire the per-source write sub-cap before the + // global write permit. The acquire reads `state.git_write_per_caller`; comments name + // the field without the `state.` prefix, so this targets the real acquire site. + assert!( + repos.contains("state.git_write_per_caller"), + "U4/P1-d gate missing: git_receive_pack must acquire the per-source write cap \ + (state.git_write_per_caller) before the global write permit" + ); + + // U5 / P1-e: the detached post-push encryption walk must run through the + // admission-gated helper, which is wired to the shared encrypt pool. + assert!( + repos.contains("fn withheld_recipients_gated") + && repos.contains("state.git_encrypt_semaphore"), + "U5/P1-e gate missing: the encryption walk must run through \ + withheld_recipients_gated, which acquires git_encrypt_semaphore" + ); + + // U1 / R2 (#173 round-10): the path-scoped filtered-pack serve must thread the + // caller's AdmissionGuard through BOTH git stages so read + per-caller admission is + // held until the pack-objects group is reaped on disconnect, closing the cap bypass + // the plain upload_pack path already fixed. Two load-bearing markers: rev-list hands + // the disarmed guard back (its tuple return type), and build_filtered_pack forwards + // that guard into the pack-objects stage (the `admission` arg after the + // "pack-objects" label). Reverting either — dropping the guard between stages, or + // passing `None` to pack-objects — trips this. + assert!( + smart_http.contains("Result<(Vec, Option)>") + && smart_http.contains("\"pack-objects\",\n admission,"), + "U1/R2 gate missing: build_filtered_pack must thread the AdmissionGuard through \ + rev-list -> pack-objects so the permits are held until the pack-objects group \ + is reaped on disconnect (the path-scoped half of #174 P1-a)" + ); + + // U2 / R1 (#173 round-10): the `GET /ipfs/{cid}` serve pipeline must run in a + // DETACHED tokio task that OWNS the AdmissionGuard built from BOTH walk permits, so a + // cancelled or timed-out request releases admission only after the spawned (bounded) + // work completes — not the instant the handler future drops (the /ipfs half of #174 + // P1-a). Two load-bearing markers: the guard is constructed from the two named + // permits, and the whole pipeline is moved into a `tokio::spawn`. Reverting to + // handler-local permits (dropping the guard/spawn) trips this. + assert!( + ipfs.contains("AdmissionGuard::new(ipfs_walk_permit, ipfs_caller_permit)") + && ipfs.contains( + "let serve: tokio::task::JoinHandle> = tokio::spawn(async move {" + ), + "U2/R1 gate missing: get_by_cid must move both /ipfs admission permits into an \ + AdmissionGuard owned by a detached tokio::spawn task so admission is released \ + only after the spawned serve work completes (the /ipfs half of #174 P1-a)" + ); + + // U2 / KTD2 (#173 round-10): the probe/read children on the /ipfs path must be the + // duration-bounded twins (process-group teardown via run_bounded_git), not the bare + // `store::object_type` / `read_object_content` (or an unbounded `cat-file -s`) a tokio + // timeout cannot cancel — otherwise a wedged cat-file lingers and pins the held + // admission past the deadline. Reverting any twin call site back to a bare read trips + // this. + assert!( + ipfs.contains("object_type_bounded(") + && ipfs.contains("object_size_bounded(") + && ipfs.contains("read_object_content_bounded("), + "U2/KTD2 gate missing: the /ipfs probe+read must call the run_bounded_git-backed \ + *_bounded twins so a wedged cat-file is reaped at the deadline, not left to pin \ + the held /ipfs walk admission" + ); + + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere + // but inside withheld_recipients_gated. A second call site (count > 1) is a new + // detached git walk that skips the admission gate — exactly the class U5 closed. + assert_eq!( + repos.matches("withheld_blob_recipients_bounded").count(), + 1, + "P1-e bypass: the bounded recipients walk must be invoked only inside \ + withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap" + ); + + // U4 / P2-2: the detached post-push encryption task must be gated by the per-repo + // coalescing set (`encrypt_inflight.try_begin`) so the OUTSTANDING parked-task set is + // bounded to <=1 per repo. Removing the gate lets N rapid pushes spawn N parked + // waiters (the unbounded set U4 closed); the semaphore only caps active walks. + assert!( + repos.contains("state.encrypt_inflight.try_begin"), + "U4/P2-2 gate missing: the detached post-push encryption spawn must consult \ + encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" + ); +} diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index d4cb64fc..fa7a3f3f 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -33,13 +33,16 @@ pub enum IpfsCmd { cid: String, #[arg(long, default_value = "https://node.gitlawb.com", env = "GITLAWB_NODE")] node: String, + /// Identity directory (default: ~/.gitlawb) + #[arg(long)] + dir: Option, }, } pub async fn run(args: IpfsArgs) -> Result<()> { match args.cmd { IpfsCmd::List { node, dir } => cmd_list(node, dir).await, - IpfsCmd::Get { cid, node } => cmd_get(cid, node).await, + IpfsCmd::Get { cid, node, dir } => cmd_get(cid, node, dir).await, } } @@ -86,11 +89,32 @@ async fn cmd_list(node: String, dir: Option) -> Result<()> { Ok(()) } -async fn cmd_get(cid: String, node: String) -> Result<()> { - let client = NodeClient::new(&node, None); - let path = format!("/ipfs/{cid}"); +async fn cmd_get(cid: String, node: String, dir: Option) -> Result<()> { + // #173 (F5): the resolver now serves path-scoped objects to authorized readers, + // so sign with an available identity like `gl ipfs list` — otherwise an owner or + // listed reader gets the opaque anonymous 404 for content they can read. + // `get_authed` signs when a keypair is present and falls back to unsigned. + // + // An explicit `--dir` is a request to use THAT identity: propagate a + // missing/corrupt-keystore error (like `list`) instead of silently sending an + // anonymous request the authorized reader would see as the node's opaque 404 + // (#173 review). Only the default (no `--dir`) keeps the best-effort unsigned + // fallback, so `get` stays usable for genuinely public content. + let keypair = match dir.as_deref() { + Some(dir) => Some(crate::identity::load_keypair_from_dir(Some(dir))?), + None => crate::identity::load_keypair_from_dir(None).ok(), + }; + let client = NodeClient::new(&node, keypair); + // #173 review (F1): the node now accepts equivalent multibase spellings, + // including base64 CIDs (prefix 'm'), whose alphabet contains '/', '+', '='. + // Interpolating the CID raw would make the client request (and sign) + // `/ipfs//`, which neither matches the single-segment Axum + // route nor points at the intended target. Percent-encode the CID as exactly + // one path segment so the signed and sent target agree and the server's + // `Path` extractor decodes it back to the original CID. + let path = format!("/ipfs/{}", encode_cid_segment(&cid)); let resp = client - .get(&path) + .get_authed(&path) .await .with_context(|| format!("failed to fetch CID {cid} from {node}"))?; @@ -119,6 +143,16 @@ async fn cmd_get(cid: String, node: String) -> Result<()> { Ok(()) } +/// Percent-encode a CID so it occupies exactly one path segment of `/ipfs/`. +/// `urlencoding::encode` escapes every byte outside the RFC 3986 unreserved set +/// (ALPHA / DIGIT / `-._~`), so the base64-CID characters that would otherwise +/// break the single-segment route — `/`, `+`, `=` — are all escaped, and the +/// server's `Path` extractor decodes the result back to the original CID (#173 +/// review, F1). +fn encode_cid_segment(cid: &str) -> String { + urlencoding::encode(cid).into_owned() +} + #[cfg(test)] mod tests { use super::*; @@ -235,4 +269,162 @@ mod tests { m.assert_async().await; } + + /// #173 (F5): `gl ipfs get` must SIGN with an available identity, like + /// `gl ipfs list`, so an owner/reader can retrieve a path-scoped object the node + /// now resolves by CID. RED before the fix: cmd_get ignores the identity dir and + /// sends an unsigned request, so the signature-matching mock is never hit + /// (cmd_get errors on the unmatched 501, and m.assert fails). GREEN after: the + /// signed request carries the RFC 9421 headers and is served 200. + #[tokio::test] + async fn test_cmd_get_signs_when_identity_present() { + let mut server = mockito::Server::new_async().await; + let keystore = seed_keystore(); + + let m = server + .mock("GET", "/ipfs/bafkreitestcid") + .match_header("signature", mockito::Matcher::Any) + .match_header("signature-input", mockito::Matcher::Any) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_header("x-git-hash", "abc123") + .with_body("object bytes") + .create_async() + .await; + + cmd_get( + "bafkreitestcid".to_string(), + server.url(), + Some(keystore.path().to_path_buf()), + ) + .await + .expect("signed get of a resolvable object should succeed"); + + m.assert_async().await; + } + + /// #173 (F5) must-not: a genuine anonymous denial must surface as an error, not + /// be masked as success. With no identity dir the request is unsigned; a 404 + /// from the node must produce an Err mentioning the status. + #[tokio::test] + async fn test_cmd_get_anonymous_denial_is_error() { + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreidenied") + .with_status(404) + .with_header("content-type", "text/plain") + .with_body("no git object found") + .create_async() + .await; + + let err = cmd_get("bafkreidenied".to_string(), server.url(), None) + .await + .expect_err("a 404 denial must be an error, not masked success"); + assert!( + err.to_string().contains("404"), + "error should mention the status, got: {err}" + ); + + m.assert_async().await; + } + + /// #173 (INV-8) must-not: the node's new 503 "search incomplete" (the legacy CID + /// scan hit its bound and could not prove absence) must surface as an actionable + /// Err naming the status, NOT be rendered as an empty/"not found" success — a + /// retryable outcome the caller has to see. Mirrors the 404 denial case for the + /// bounded-search response the resolver now emits. + #[tokio::test] + async fn test_cmd_get_search_incomplete_503_is_error() { + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreiincomplete") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(r#"{"error":"search_incomplete","message":"CID search incomplete — retry"}"#) + .create_async() + .await; + + let err = cmd_get("bafkreiincomplete".to_string(), server.url(), None) + .await + .expect_err("a 503 incomplete-search must be an error, not masked as not-found"); + assert!( + err.to_string().contains("503"), + "error should mention the status, got: {err}" + ); + + m.assert_async().await; + } + + /// #173 review (F1): a base64 CID (multibase prefix 'm') can contain '/', '+', + /// and '='. The client must percent-encode it into ONE path segment before + /// building and signing `/ipfs/`; otherwise the '/' splits the target so + /// it misses the single-segment Axum route and the signature covers the wrong + /// path. Assert the encoded segment carries no raw '/', '+', or '=', and that + /// it decodes back to the original CID (the server's `Path` extractor performs + /// that same decode). RED with the old raw `format!("/ipfs/{cid}")`: the + /// segment still contains '/'. + #[test] + fn test_encode_cid_segment_escapes_base64_alphabet() { + let cid = "mFoo/Bar+baz=="; + let encoded = encode_cid_segment(cid); + + assert!( + !encoded.contains('/'), + "encoded CID must be a single path segment (no raw '/'), got: {encoded}" + ); + assert!( + !encoded.contains('+'), + "encoded CID must escape '+', got: {encoded}" + ); + assert!( + !encoded.contains('='), + "encoded CID must escape '=', got: {encoded}" + ); + + let decoded = urlencoding::decode(&encoded).expect("encoded CID must decode"); + assert_eq!( + decoded, cid, + "encoding must round-trip back to the original CID" + ); + } + + /// #173 review: `gl ipfs get --dir ` must PROPAGATE a missing/corrupt + /// identity-load error like `gl ipfs list`, not silently fall back to an anonymous + /// request — otherwise an authorized reader pointing `--dir` at a broken keystore + /// gets the node's opaque 404 instead of the actionable key-load error. The + /// unsigned fallback is preserved only when NO `--dir` is given (covered by + /// `test_cmd_get_anonymous_denial_is_error`). RED before the fix (`.ok()` swallows + /// the error, an anonymous request is sent, and the `.expect(0)` mock is hit), + /// GREEN after. + #[tokio::test] + async fn test_cmd_get_explicit_dir_no_identity_errors_without_request() { + let mut server = mockito::Server::new_async().await; + // Empty keystore dir passed explicitly via --dir: no identity.pem present. + let empty = tempfile::TempDir::new().unwrap(); + + // The endpoint must never be hit when an explicit --dir fails to load. + let m = server + .mock("GET", "/ipfs/bafkreitestcid") + .expect(0) + .create_async() + .await; + + let err = cmd_get( + "bafkreitestcid".to_string(), + server.url(), + Some(empty.path().to_path_buf()), + ) + .await + .expect_err("an explicit --dir that fails to load must be an error"); + assert!( + err.to_string().contains("gl identity new") + || err.to_string().contains("no identity found") + || err.to_string().contains("failed to load keypair"), + "error should name the key-load failure, got: {err}" + ); + + m.assert_async().await; + } }