feat(storage): storage-agnostic BlobStore layer + push-latency quick wins

.env.example

@@ -17,6 +17,32 @@ GITLAWB_PORT=7545
 # ── Storage ───────────────────────────────────────────────────────────────
 GITLAWB_REPOS_DIR=/data/repos
 
+# ── Object storage (durable repo archives) ────────────────────────────────
+# Backend for whole-repo archives: s3 | fs | ipfs. Empty = auto-detect:
+# s3 when a bucket is set, else fs when GITLAWB_STORAGE_FS_DIR is set, else
+# local-only (repos live only on this node's disk). `ipfs` is never
+# auto-selected — setting GITLAWB_IPFS_API alone keeps its pinning-only
+# meaning; opt in explicitly with GITLAWB_STORAGE_BACKEND=ipfs. NOTE: the
+# ipfs backend stores archives in the Kubo daemon's LOCAL MFS namespace, so
+# every node must point at the SAME Kubo instance.
+GITLAWB_STORAGE_BACKEND=
+# Bucket for the s3 backend (Tigris, R2, AWS S3, MinIO, B2).
+# GITLAWB_TIGRIS_BUCKET is honored as a legacy alias.
+GITLAWB_S3_BUCKET=
+# Endpoint URL override for the s3 backend (R2/MinIO). On Tigris/Fly the
+# endpoint arrives via AWS_ENDPOINT_URL_S3 — leave empty.
+GITLAWB_S3_ENDPOINT=
+# Force path-style addressing (required by MinIO and some S3-compatibles).
+GITLAWB_S3_FORCE_PATH_STYLE=false
+# Directory for the fs (local filesystem) backend.
+GITLAWB_STORAGE_FS_DIR=
+# Ack pushes before the durable upload finishes (write-back). Lower latency;
+# opt-in durability tradeoff — see --help for the full semantics.
+GITLAWB_ASYNC_UPLOAD=false
+# Dedicated DB pool for per-repo write locks; a push pins one connection for
+# its lifetime, so this bounds per-node push concurrency.
+GITLAWB_ADVISORY_LOCK_POOL_SIZE=16
+
 # PostgreSQL connection URL. Required.
 # When using the bundled docker-compose, this is wired automatically.
 DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb

README.md

@@ -344,7 +344,14 @@ Important node settings:
 | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. |
 | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. |
 | `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack/receive-pack may run before it is aborted (504). Default 600. Does not bound `info/refs` or the withheld-blob path. |
-| `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. |
+| `GITLAWB_STORAGE_BACKEND` | Object-storage backend for repo archives: `s3`, `fs`, or `ipfs`. Empty = auto-detect (`s3` if a bucket is set, else `fs` if a dir is set, else local-only). `ipfs` is never auto-selected and requires all nodes to share one Kubo instance (MFS is daemon-local). |
+| `GITLAWB_S3_BUCKET` | Bucket for the `s3` backend (Tigris, R2, AWS S3, MinIO, B2). |
+| `GITLAWB_S3_ENDPOINT` | Endpoint URL override for the `s3` backend (R2/MinIO; empty on Tigris/Fly). |
+| `GITLAWB_S3_FORCE_PATH_STYLE` | Force path-style S3 addressing (MinIO and some S3-compatibles). |
+| `GITLAWB_STORAGE_FS_DIR` | Directory for the `fs` (local filesystem) backend. |
+| `GITLAWB_ASYNC_UPLOAD` | Ack pushes before the durable storage upload (write-back). Lower latency, opt-in durability tradeoff. Default `false`. |
+| `GITLAWB_ADVISORY_LOCK_POOL_SIZE` | Dedicated DB pool for per-repo write locks; bounds per-node push concurrency. Default 16. |
+| `GITLAWB_TIGRIS_BUCKET` | Legacy alias for `GITLAWB_S3_BUCKET` (selects the `s3` backend). |
 | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. |
 | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. |
 

crates/gitlawb-node/Cargo.toml

@@ -33,9 +33,15 @@ sqlx = { version = "0.8", features = ["postgres", "runtime-tokio-rustls", "chron
 clap = { version = "4", features = ["derive", "env"] }
 bytes = "1"
 libc = "0.2"
+async-trait = "0.1"
 cid = { workspace = true }
 hex = { workspace = true }
 sha2 = { workspace = true }
+# Content etags for repo archives: S3 single-part ETags are the body MD5, so a
+# client-side MD5 lets the uploader record its *intended* etag before the PUT
+# (crash-recovery provenance for the pending-upload marker). Not used for
+# anything security-sensitive.
+md-5 = "0.10"
 hmac = { workspace = true }
 http-body-util = "0.1"
 tokio-util = { version = "0.7", features = ["io"] }
@@ -57,7 +63,7 @@ aws-sdk-s3 = { version = "1", default-features = false, features = ["sigv4a", "d
 aws-config = { version = "1", features = ["behavior-version-latest"] }
 async-compression = { version = "0.4", features = ["tokio", "zstd"] }
 tar = "0.4"
-zstd = "0.13"
+zstd = { version = "0.13", features = ["zstdmt"] }
 # Prometheus metrics. Used to expose a /metrics endpoint for ops/observability
 # on the opt-in GITLAWB_METRICS_ADDR listener. The crate is also the de-facto
 # exposition format encoder in the Rust ecosystem.

crates/gitlawb-node/src/api/issues.rs

@@ -70,10 +70,32 @@ pub async fn create_issue(
 
     let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str);
 
-    // Always release the advisory lock — even on error; upload to Tigris only on success.
-    guard.release(create_result.is_ok()).await;
+    // Always release the advisory lock — even on error; upload to storage only on success.
+    let release_result = guard.release(create_result.is_ok()).await;
 
     create_result.map_err(|e| AppError::Git(e.to_string()))?;
+    // A durable-upload failure is recoverable ONLY while the pending-upload
+    // marker protects the committed mutation (the next successful upload
+    // re-syncs storage). Verify the marker actually exists before choosing to
+    // succeed: if the marker write itself also failed, nothing protects the
+    // mutation from a stale-archive rollback, and the request must fail.
+    // (Succeeding here avoids non-idempotent retries: a retried create mints
+    // a second issue UUID and both eventually publish.)
+    if let Err(e) = release_result {
+        if state
+            .repo_store
+            .pending_marker_exists(&record.owner_did, &record.name)
+        {
+            tracing::error!(repo = %record.name, issue = %issue_id, err = %e,
+                "issue committed locally but durable upload failed — storage re-syncs on next upload");
+        } else {
+            tracing::error!(repo = %record.name, issue = %issue_id, err = %e,
+                "issue committed locally with NO durable protection — failing the request");
+            return Err(AppError::Git(format!(
+                "issue stored locally but durability could not be guaranteed: {e}"
+            )));
+        }
+    }
 
     // Bump trust score for the issue author — increment current score by 0.05
     // (avoids the push_count=0 stuck-at-0.05 bug for agents who only file issues)
@@ -244,11 +266,11 @@ pub async fn close_issue(
             .ok()
             .and_then(|i| i.author),
         Ok(None) => {
-            guard.release(false).await;
+            let _ = guard.release(false).await;
             return Err(AppError::NotFound(format!("issue {issue_id} not found")));
         }
         Err(e) => {
-            guard.release(false).await;
+            let _ = guard.release(false).await;
             return Err(AppError::Git(e.to_string()));
         }
     };
@@ -257,20 +279,37 @@ pub async fn close_issue(
         .as_deref()
         .is_some_and(|a| crate::api::did_matches(&auth.0, a));
     if !is_owner && !is_author {
-        guard.release(false).await;
+        let _ = guard.release(false).await;
         return Err(AppError::Forbidden(
             "only the repo owner or the issue author can close this issue".into(),
         ));
     }
 
     let close_result = git_issues::close_issue(&disk_path, &issue_id);
 
-    // Always release the advisory lock — even on error; upload to Tigris only on success.
-    guard.release(close_result.is_ok()).await;
+    // Always release the advisory lock — even on error; upload to storage only on success.
+    let release_result = guard.release(close_result.is_ok()).await;
 
     let updated = close_result
         .map_err(|e| AppError::Git(e.to_string()))?
         .ok_or_else(|| AppError::RepoNotFound(format!("issue {issue_id} not found")))?;
+    // Recoverable only while the marker protects the committed mutation
+    // (see create_issue).
+    if let Err(e) = release_result {
+        if state
+            .repo_store
+            .pending_marker_exists(&record.owner_did, &record.name)
+        {
+            tracing::error!(repo = %repo, issue = %issue_id, err = %e,
+                "issue close committed locally but durable upload failed — storage re-syncs on next upload");
+        } else {
+            tracing::error!(repo = %repo, issue = %issue_id, err = %e,
+                "issue close committed locally with NO durable protection — failing the request");
+            return Err(AppError::Git(format!(
+                "issue close stored locally but durability could not be guaranteed: {e}"
+            )));
+        }
+    }
 
     let issue: serde_json::Value = serde_json::from_str(&updated)
         .map_err(|e| AppError::BadRequest(format!("invalid issue data: {e}")))?;

crates/gitlawb-node/src/api/pulls.rs

@@ -224,10 +224,31 @@ pub async fn merge_pr(
         &pr.title,
     );
 
-    // Always release the advisory lock — even on error; upload to Tigris only on success.
-    guard.release(merge_result.is_ok()).await;
+    // Always release the advisory lock — even on error; upload to storage only on success.
+    let release_result = guard.release(merge_result.is_ok()).await;
 
     let merge_sha = merge_result.map_err(|e| AppError::Git(e.to_string()))?;
+    // A durable-upload failure is recoverable ONLY while the pending-upload
+    // marker protects the merge commit; verify it exists before choosing to
+    // proceed (proceeding keeps the DB status consistent with the already
+    // merged ref, which a retry cannot un-merge). Without the marker nothing
+    // protects the merge from a stale-archive rollback — fail the request so
+    // the inconsistency is surfaced instead of silently losable.
+    if let Err(e) = release_result {
+        if state
+            .repo_store
+            .pending_marker_exists(&record.owner_did, &record.name)
+        {
+            tracing::error!(repo = %record.name, pr = %pr.id, err = %e,
+                "merge committed locally but durable upload failed — storage re-syncs on next upload");
+        } else {
+            tracing::error!(repo = %record.name, pr = %pr.id, err = %e,
+                "merge committed locally with NO durable protection — failing the request");
+            return Err(AppError::Git(format!(
+                "merge applied locally but durability could not be guaranteed: {e}"
+            )));
+        }
+    }
 
     state.db.merge_pr(&pr.id, &merger_did).await?;
     let _ = state.db.touch_repo(&record.id).await;