From eea0eaf01f51886b66aca9354051cbbfb52cc705 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 02:14:19 -0700 Subject: [PATCH 01/17] docs: design persistent Git object sessions --- .../persistent-git-object-sessions.md | 387 ++++++++++++++++++ docs/design/README.md | 1 + 2 files changed, 388 insertions(+) create mode 100644 docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md diff --git a/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md b/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md new file mode 100644 index 0000000..9b8a096 --- /dev/null +++ b/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md @@ -0,0 +1,387 @@ +--- +title: 'PERF-0052 - Persistent Git Object Sessions' +cycle: '0052' +task_id: 'persistent-git-object-sessions' +legend: 'PERF' +release_home: 'v6.5.2' +issue: 'https://github.com/git-stunts/git-cas/issues/90' +goalpost_issue: 'https://github.com/git-stunts/git-cas/issues/90' +tracker_source: 'github' +status: 'active' +base_commit: '12fd67200641f385d2d756c302bb2e5701beefbc' +owners: + - '@git-stunts' +sponsors: + human: 'James' + agent: 'Codex' +blocking_issues: [] +supersedes: [] +superseded_by: null +created: '2026-07-19' +updated: '2026-07-19' +--- + +# PERF-0052 - Persistent Git Object Sessions + +## Linked Issue + +- [#90 - Reuse bounded Git object sessions](https://github.com/git-stunts/git-cas/issues/90) + +## Linked Tracker + +- Milestone: [`v6.5.2`](https://github.com/git-stunts/git-cas/milestone/12) +- Goalpost issue: [#90](https://github.com/git-stunts/git-cas/issues/90) +- Follow-on derivation design: [#86](https://github.com/git-stunts/git-cas/issues/86) + +## Design Type + +This design is primarily: + +- [x] Runtime/API +- [x] Storage/substrate +- [x] Migration/release +- [ ] CLI/operator +- [x] Docs/public guidance +- [ ] TUI/visual surface +- [x] Test/tooling + +## Decision Summary + +`GitPersistenceAdapter` will lazily own typed `cat-file`, `mktree`, and +`fast-import` sessions supplied by `@git-stunts/plumbing` 3.2.0. Bounded +immutable object and tree reads, blob writes, and tree writes will reuse those +processes. Payload streams will continue to use `readBlobStream()` and will not +be converted into whole-object reads. `ContentAddressableStore.close()` will +deterministically release adapter-owned sessions. + +## Sponsored Human + +An application operator wants selected causal materialization to stay +responsive when the retained graph is much larger than memory, without paying +one process launch per immutable support object or managing Git children. + +## Sponsored Agent + +An agent needs process reuse, bounded residency, and lifecycle state to be +observable through stable tests and receipts, without receiving raw Git OIDs, +session handles, mutable refs, or hidden cleanup obligations. + +## Hill + +By the end of this cycle, a caller can perform a cold selected bundle read with +constant session-process startup for supported object protocols, then close the +CAS deterministically. Real-Git tests compare the same fixture through legacy +fallback and session-capable adapters and prove the optimized path does less +process work without changing the result. + +## Current Truth + +- Every `writeBlob()`, `writeTree()`, `readBlobStream()`, `readTree()`, exact + `readTreeEntry()`, and uncached object-info read invokes + `plumbing.execute()` or `plumbing.executeStream()` independently. + [cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#55-251@12fd67200641f385d2d756c302bb2e5701beefbc`] +- Exact tree-entry and object-info results are immutable and count-bounded, but + full tree objects are not reused. + [cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#35-49@12fd67200641f385d2d756c302bb2e5701beefbc`] +- Bundle validation asks for direct tree edges repeatedly along a selected + fanout path. + [cite: `src/domain/services/BundleService.js#535-610@12fd67200641f385d2d756c302bb2e5701beefbc`] +- Chunk restore deliberately uses `readBlobStream()` when available to enforce + hard memory limits. + [cite: `src/domain/services/ChunkRepository.js#122-165@12fd67200641f385d2d756c302bb2e5701beefbc`] +- The facade constructs one persistence adapter lazily but has no close or + async-disposal contract. + [cite: `index.js#120-340@12fd67200641f385d2d756c302bb2e5701beefbc`] +- PERF-0050 explicitly excluded long-lived Git processes because plumbing did + not yet provide the protocol boundary. Plumbing 3.2.0 now provides that + boundary. + [cite: `docs/design/0050-lazy-bundle-reference-reads/lazy-bundle-reference-reads.md#89-101@12fd67200641f385d2d756c302bb2e5701beefbc`] + +## Problem + +The current cache removes repeated immutable work only after a key repeats. +Every distinct support object on a cold bounded path still starts a Git +process. This makes latency proportional to support-object count even when the +causal aperture is correct and memory remains bounded. It also encourages +downstream applications to invent caches or bypass git-cas, violating the +storage ownership boundary. + +## Scope + +This cycle includes: + +- upgrade `@git-stunts/plumbing` to 3.2.0 +- lazy adapter ownership of typed object sessions +- bounded raw tree decoding and exact path traversal +- count-and-byte-bounded immutable tree reuse +- session-backed bounded blob and object-info reads +- session-backed blob and tree writes with visibility guarantees +- deterministic adapter and facade close semantics +- capability fallback for injected plumbing implementations without sessions +- real-Git process-count, wall-clock, and memory evidence +- Node, Bun, and Deno validation + +## Non-Goals + +This cycle does not include: + +- implementing a Git object database, packfile reader, or ref store +- caching refs, mutable collection state, or arbitrary application payloads +- converting `readBlobStream()` into buffered whole-object reads +- exposing sessions, process handles, Git OIDs, or ref mutation authority +- path-local persistent bundle derivation from #86 +- replacing stock Git or `@git-stunts/plumbing` +- claiming graph-wide memory bounds from a small-fixture latency test + +## Runtime / API Contract + +```ts +class ContentAddressableStore { + close(): Promise; + [Symbol.asyncDispose](): Promise; +} + +class GitPersistenceAdapter { + close(): Promise; + [Symbol.asyncDispose](): Promise; +} +``` + +`close()` releases local resources only. It does not delete objects, update +refs, expire retained content, run garbage collection, or mutate witnesses. +It is idempotent. A close begun while operations are active waits for the +typed protocol queues to settle. New adapter operations after close fail with +a stable closed-resource error. + +The adapter feature-detects each typed plumbing capability independently. +Missing capabilities use the existing command-per-operation implementation. +No fallback behavior changes persisted object identity. + +## User Experience / Product Shape + +There is no new CLI surface. Normal callers use: + +```js +await using cas = await ContentAddressableStore.open({ cwd: '.' }); +const page = await cas.pages.get({ handle }); +``` + +Callers that cannot use explicit resource management call `await cas.close()`. + +## Data / State Model + +| State | Source of truth | Derived state | Invalid states | Reset behavior | Serialization | Determinism assumptions | +| --------------- | --------------------- | ------------------------- | --------------------------------- | ------------------------------------ | ------------- | ----------------------------------- | +| Git objects | Git object database | None | Malformed or missing object | External repair | Git native | OID identifies immutable bytes | +| Session handles | Adapter process state | Lazy promise per protocol | Closed or poisoned session reused | Invalidate; next lawful call reopens | None | Operations serialize per protocol | +| Parsed trees | Immutable tree object | Frozen entry array | Malformed raw tree | Reject; do not cache | None | Same tree OID yields same entries | +| Tree cache | Adapter memory | LRU entries | Count or byte bound exceeded | Evict oldest | None | Cache never changes semantic result | + +## Architecture / Anti-SLUDGE Posture + +| Concern | Decision | +| ------------------- | ------------------------------------------------------------------------------------------------- | +| Domain changes | Add only lifecycle capability to the persistence port; domain services do not learn Git sessions. | +| Port changes | `GitPersistencePort.close()` is a default no-op for compatibility. | +| Adapter changes | `GitPersistenceAdapter` owns protocol sessions and Git tree decoding. | +| Facade changes | `ContentAddressableStore.close()` delegates to its initialized persistence adapter. | +| Plumbing boundary | Plumbing owns process spawning and typed wire protocols; git-cas owns lawful reuse policy. | +| Downstream boundary | git-warp continues to see only git-cas asset, page, bundle, retention, and lifecycle APIs. | + +## Cost / Residency Posture + +- At most one live session exists per supported protocol per adapter. +- Session queues serialize operations and do not duplicate response buffers. +- Parsed trees are bounded by entry count and estimated retained bytes. +- A tree larger than the bounded cat-file read budget falls back to streaming + `ls-tree` behavior and is not cached. +- `readBlobStream()` remains one bounded stream per payload and does not share a + buffered cat-file session. +- Fast-import writes checkpoint before `writeBlob()` resolves so a different + Git process can consume the returned OID. + +## Determinism / Replay / Causality + +Session reuse changes process topology, not object bytes, tree ordering, OIDs, +retention, or witness meaning. Same-input object identity tests remain the +determinism proof. + +## Git Substrate Impact + +The adapter uses three stock Git protocols through plumbing: `cat-file +--batch-command`, `mktree --batch -z`, and `fast-import`. Ref access remains on +the existing ref adapter and is never cached. No new refs or object formats are +introduced. + +## Compatibility / Migration Posture + +This is additive. Injected plumbing doubles without typed-session methods keep +the legacy path. Existing callers are not required to call `close()` unless a +session-capable operation was used, but documentation treats explicit close as +the normal lifecycle contract. Persisted repositories need no migration. + +## Error Contract + +- Missing Git objects map to the existing `GIT_OBJECT_NOT_FOUND` CAS error. +- Object read-budget failures map to the existing size-limit contract. +- Malformed tree bytes map to `TREE_PARSE_ERROR`. +- Protocol failure invalidates the affected session before surfacing. +- A later independent call may open a fresh session; writes are not silently + retried after ambiguous failure. +- Close attempts every opened session and reports aggregate cleanup failure. + +## Security / Trust / Redaction Posture + +Session reuse grants no new repository authority. Commands remain sanitized by +plumbing. Diagnostics count protocol/process activity but do not emit content, +keys, paths outside the repository, or object payloads. + +## Accessibility Posture + +No visual interaction changes. Human-readable lifecycle errors remain concise, +and all benchmark evidence is also committed as structured JSON. + +## Agent Inspectability / Explainability Posture + +Tests expose process openings by protocol, fallback command counts, close +events, and semantic result equality. Agents can distinguish cache hits from +process amortization without timing inference. + +## User-Facing Text / Directionality + +Public documentation adds the sentence: "`close()` releases local resources +only." Text is English and left-to-right. Machine-readable tests and witness +JSON carry the equivalent lifecycle facts. + +## Linked Invariants + +- immutable objects are addressed by content identity +- mutable refs are never cached as immutable metadata +- payload streaming remains bounded by existing limits +- only git-cas owns CAS cache and retention policy for git-warp +- resource ownership is paired with deterministic release + +## Design Alternatives Considered + +### Native Git library + +Rejected for this cycle. The spike found no complete supported replacement for +the required stock Git behavior, and it would move protocol risk into the +application stack. + +### Session ownership in git-warp + +Rejected. It would duplicate CAS policy and expose Git beneath git-cas. + +### Buffer every payload through cat-file + +Rejected. It would regress the hard streaming invariant for graphs and assets +larger than memory. + +### Cache only exact tree entries + +Rejected as insufficient. Distinct entries in one bounded fanout tree would +still reread the same immutable tree object repeatedly. + +### Never close sessions + +Rejected. Hidden live child processes are a correctness and test-isolation bug. + +## Decision + +Adopt typed persistent sessions inside `GitPersistenceAdapter`, bounded tree +reuse, explicit close, and capability fallback. Keep streaming payloads on the +existing stream path. + +## Proof Surface + +- unit tests for tree decoding, session coalescing, error invalidation, close, + fallback, and residency eviction +- facade tests for lazy close and async disposal +- real-Git same-fixture fallback/session command-process comparison +- existing large-stream restore and page-cache tests +- Node, Bun, and Deno suites +- committed JSON witness with counts, timing, and peak memory + +## Implementation Slices + +1. Dependency, lifecycle, and failing tests. +2. Session owner and bounded tree codec/cache. +3. Adapter routing and error normalization. +4. Facade lifecycle and declarations. +5. Real-Git performance witness, docs, and release evidence. + +## Tests To Write First + +1. One adapter opens at most one session per protocol under concurrent calls. +2. Close is idempotent and closes every opened protocol. +3. A failed protocol is invalidated and a later call opens a new process. +4. Same-tree exact lookups require one bounded object read. +5. Tree cache evicts by count and estimated bytes. +6. `readBlobStream()` still delegates to `executeStream()`. +7. A session-capable real-Git read returns the same reference with fewer child + process openings than the fallback adapter. + +## Acceptance Criteria + +- [ ] All acceptance criteria in #90 are proven. +- [ ] Public declarations include close and async disposal. +- [ ] No public session, process, or mutable ref handle is exported. +- [ ] Existing storage identity fixtures remain unchanged. +- [ ] `npm test`, integration suites, platform suites, and lint pass. +- [ ] Witness evidence is committed and linked from the PR. + +## Validation Plan + +1. Run targeted unit tests while implementing. +2. Run real-Git fallback and session paths against one generated fixture. +3. Run full unit and integration suites. +4. Run Node, Bun, and Deno platform validation. +5. Run ESLint, formatting check, declaration accuracy, and release verification. +6. Self-review the complete diff against `origin/main`. + +## Playback / Witness + +The witness packet must answer: + +- Did the same selected reference result survive both paths? +- How many child processes and fallback commands did each path use? +- Did repeated immutable tree access hit bounded reuse? +- Did close release every session? +- Did large streaming behavior remain bounded? + +## Risks + +- Fast-import checkpoint cost may erase write-path gains; benchmark before + claiming improvement. +- Policy timeouts can race an in-flight protocol operation; timeout handling + must invalidate the session. +- Raw tree parsing must support SHA-1 and SHA-256 repositories and malformed + input without out-of-bounds reads. +- Callers that omit close may keep a process alive; docs and async disposal + reduce but cannot eliminate misuse. + +## Follow-On Debt + +- Path-local persistent bundle derivation remains #86. +- Batched streaming chunk reads and writes may require a separate design if + same-fixture evidence shows per-chunk stream processes dominate. +- Doctor may later expose live session counters if operational demand exists. + +## Tracker Disposition + +Issue #90 closes only after implementation, witness, PR review, merge, and npm +publication evidence are complete. New debt becomes a linked GitHub issue. + +## Done Does Not Mean + +- git-warp materialization is automatically fast; it must adopt v6.5.2 and + prove its own benchmark. +- arbitrary graph reads fit in memory. +- Git subprocess cost is zero. +- #86 is complete. + +## Retrospective + +To be written after merge under +`docs/method/retro/0052-persistent-git-object-sessions/`. diff --git a/docs/design/README.md b/docs/design/README.md index 7abd132..36bc929 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -11,6 +11,7 @@ process in [docs/method/process.md](../method/process.md). ## Active METHOD Cycles +- [0052-persistent-git-object-sessions - persistent-git-object-sessions](./0052-persistent-git-object-sessions/persistent-git-object-sessions.md) - [0050-lazy-bundle-reference-reads - lazy-bundle-reference-reads](./0050-lazy-bundle-reference-reads/lazy-bundle-reference-reads.md) - [0049-scoped-staging-workspaces — scoped-staging-workspaces](./0049-scoped-staging-workspaces/scoped-staging-workspaces.md) - [0048-scoped-cache-acquisitions — scoped-cache-acquisitions](./0048-scoped-cache-acquisitions/scoped-cache-acquisitions.md) From 27831926327afc7522b39ab435d29b46b7ac428e Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 03:22:55 -0700 Subject: [PATCH 02/17] perf: reuse bounded Git object sessions --- .../persistent-git-object-sessions.md | 109 ++-- index.d.ts | 14 + index.js | 35 ++ package.json | 2 +- pnpm-lock.yaml | 10 +- .../measure-git-object-sessions.js | 335 ++++++++++++ src/domain/errors/Codes.js | 2 + src/domain/services/PageService.js | 79 +++ .../adapters/GitObjectSessionPool.js | 234 ++++++++ .../adapters/GitPersistenceAdapter.js | 510 +++++++++++++++--- .../codecs/GitTreeObjectCodec.js | 124 +++++ src/ports/GitPersistencePort.js | 26 + src/types/ambient.d.ts | 47 +- .../bundle-reference-performance.test.js | 107 +++- test/unit/domain/services/PageService.test.js | 51 ++ ...dressableStore.application-storage.test.js | 2 +- .../ContentAddressableStore.lifecycle.test.js | 33 ++ .../GitPersistenceAdapter.sessions.test.js | 438 +++++++++++++++ .../codecs/GitTreeObjectCodec.test.js | 70 +++ test/unit/ports/GitPersistencePort.test.js | 4 + test/unit/types/declaration-accuracy.test.js | 13 + 21 files changed, 2133 insertions(+), 112 deletions(-) create mode 100644 scripts/diagnostics/measure-git-object-sessions.js create mode 100644 src/infrastructure/adapters/GitObjectSessionPool.js create mode 100644 src/infrastructure/codecs/GitTreeObjectCodec.js create mode 100644 test/unit/facade/ContentAddressableStore.lifecycle.test.js create mode 100644 test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js create mode 100644 test/unit/infrastructure/codecs/GitTreeObjectCodec.test.js diff --git a/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md b/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md index 9b8a096..b38f8cb 100644 --- a/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md +++ b/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md @@ -47,12 +47,16 @@ This design is primarily: ## Decision Summary -`GitPersistenceAdapter` will lazily own typed `cat-file`, `mktree`, and -`fast-import` sessions supplied by `@git-stunts/plumbing` 3.2.0. Bounded -immutable object and tree reads, blob writes, and tree writes will reuse those -processes. Payload streams will continue to use `readBlobStream()` and will not -be converted into whole-object reads. `ContentAddressableStore.close()` will -deterministically release adapter-owned sessions. +`GitPersistenceAdapter` will lazily own typed `cat-file` and `mktree` sessions +supplied by `@git-stunts/plumbing` 3.2.0. Bounded immutable object and tree +reads plus tree writes will reuse those processes. Explicitly bounded page +batches will use one scoped `fast-import` session that closes before returning. +Individual blob writes remain one-shot because a long-lived `fast-import` +process can remember an object that external pruning has removed and decline to +recreate it. Payload streams continue to use `readBlobStream()` and are not +converted into whole-object reads. `ContentAddressableStore.close()` +deterministically drains started operations, cancels abandoned output streams, +and releases adapter-owned sessions. ## Sponsored Human @@ -115,7 +119,8 @@ This cycle includes: - bounded raw tree decoding and exact path traversal - count-and-byte-bounded immutable tree reuse - session-backed bounded blob and object-info reads -- session-backed blob and tree writes with visibility guarantees +- scoped bulk blob writes and session-backed tree writes with visibility guarantees +- one-shot individual blob writes that remain correct across external pruning - deterministic adapter and facade close semantics - capability fallback for injected plumbing implementations without sessions - real-Git process-count, wall-clock, and memory evidence @@ -137,6 +142,13 @@ This cycle does not include: ```ts class ContentAddressableStore { + pages: { + putBatch(options: { + pages: Array<{ source: Uint8Array | Iterable }>; + maxBatchBytes?: number; + maxBatchPages?: number; + }): Promise>; + }; close(): Promise; [Symbol.asyncDispose](): Promise; } @@ -149,9 +161,11 @@ class GitPersistenceAdapter { `close()` releases local resources only. It does not delete objects, update refs, expire retained content, run garbage collection, or mutate witnesses. -It is idempotent. A close begun while operations are active waits for the -typed protocol queues to settle. New adapter operations after close fail with -a stable closed-resource error. +It is idempotent. A close begun while operations are active blocks new work, +waits for started commands, destroys returned output streams that callers did +not consume, waits for their Git processes, and then closes typed protocol +sessions. New adapter operations after close fail with a stable +closed-resource error. The adapter feature-detects each typed plumbing capability independently. Missing capabilities use the existing command-per-operation implementation. @@ -170,12 +184,13 @@ Callers that cannot use explicit resource management call `await cas.close()`. ## Data / State Model -| State | Source of truth | Derived state | Invalid states | Reset behavior | Serialization | Determinism assumptions | -| --------------- | --------------------- | ------------------------- | --------------------------------- | ------------------------------------ | ------------- | ----------------------------------- | -| Git objects | Git object database | None | Malformed or missing object | External repair | Git native | OID identifies immutable bytes | -| Session handles | Adapter process state | Lazy promise per protocol | Closed or poisoned session reused | Invalidate; next lawful call reopens | None | Operations serialize per protocol | -| Parsed trees | Immutable tree object | Frozen entry array | Malformed raw tree | Reject; do not cache | None | Same tree OID yields same entries | -| Tree cache | Adapter memory | LRU entries | Count or byte bound exceeded | Evict oldest | None | Cache never changes semantic result | +| State | Source of truth | Derived state | Invalid states | Reset behavior | Serialization | Determinism assumptions | +| ---------------- | --------------------- | ------------------------- | --------------------------------- | ------------------------------------ | ------------- | ----------------------------------- | +| Git objects | Git object database | None | Malformed or missing object | External repair | Git native | OID identifies immutable bytes | +| Session handles | Adapter process state | Lazy promise per protocol | Closed or poisoned session reused | Invalidate; next lawful call reopens | None | Operations serialize per protocol | +| Page write batch | Caller input | Bounded byte arrays | Page or byte limit exceeded | Reject before persistence begins | None | Input order determines OID order | +| Parsed trees | Immutable tree object | Frozen entry array | Malformed raw tree | Reject; do not cache | None | Same tree OID yields same entries | +| Tree cache | Adapter memory | LRU entries | Count or byte bound exceeded | Evict oldest | None | Cache never changes semantic result | ## Architecture / Anti-SLUDGE Posture @@ -197,8 +212,14 @@ Callers that cannot use explicit resource management call `await cas.close()`. `ls-tree` behavior and is not cached. - `readBlobStream()` remains one bounded stream per payload and does not share a buffered cat-file session. -- Fast-import writes checkpoint before `writeBlob()` resolves so a different - Git process can consume the returned OID. +- Returned Git output streams remain adapter-owned until consumption completes + or `close()` destroys them and waits for the underlying process. +- `pages.putBatch()` admits at most 256 pages and 32 MiB by default, collecting + and validating the complete bounded batch before persistence begins. +- A bulk fast-import session checkpoints and closes before `putBatch()` + resolves so a different Git process can consume every returned OID. +- Individual `pages.put()` and `writeBlob()` calls remain one-shot. This is a + correctness boundary, not an unimplemented optimization. ## Determinism / Replay / Causality @@ -208,10 +229,11 @@ determinism proof. ## Git Substrate Impact -The adapter uses three stock Git protocols through plumbing: `cat-file ---batch-command`, `mktree --batch -z`, and `fast-import`. Ref access remains on -the existing ref adapter and is never cached. No new refs or object formats are -introduced. +The adapter uses three stock Git protocols through plumbing: persistent +`cat-file --batch-command`, persistent `mktree --batch -z`, and scoped +`fast-import` for an explicitly bounded blob batch. Individual blob writes use +`hash-object` as before. Ref access remains on the existing ref adapter and is +never cached. No new refs or object formats are introduced. ## Compatibility / Migration Posture @@ -226,8 +248,9 @@ the normal lifecycle contract. Persisted repositories need no migration. - Object read-budget failures map to the existing size-limit contract. - Malformed tree bytes map to `TREE_PARSE_ERROR`. - Protocol failure invalidates the affected session before surfacing. -- A later independent call may open a fresh session; writes are not silently - retried after ambiguous failure. +- A typed protocol-process failure receives one retry through a fresh session. + These operations are content-addressed and idempotent; bulk inputs are + materialized once before the first attempt so a retry sees the same sequence. - Close attempts every opened session and reports aggregate cleanup failure. ## Security / Trust / Redaction Posture @@ -287,18 +310,31 @@ still reread the same immutable tree object repeatedly. Rejected. Hidden live child processes are a correctness and test-isolation bug. +### Reuse one fast-import process across individual blob writes + +Rejected after a real-Git counterexample. When an unreachable blob was pruned +between two identical writes, the still-running `fast-import` process retained +its duplicate-object knowledge and returned the old OID without recreating the +missing object. A scoped batch is safe because it checkpoints and closes before +the OIDs cross the method boundary; the next batch starts with fresh Git object +state. + ## Decision -Adopt typed persistent sessions inside `GitPersistenceAdapter`, bounded tree -reuse, explicit close, and capability fallback. Keep streaming payloads on the -existing stream path. +Adopt typed persistent read/tree sessions inside `GitPersistenceAdapter`, +bounded tree reuse, explicit close, capability fallback, and an explicit +bounded page-batch API backed by one scoped fast-import process. Keep individual +blob writes one-shot and streaming payloads on the existing stream path. ## Proof Surface - unit tests for tree decoding, session coalescing, error invalidation, close, - fallback, and residency eviction + active-command draining, abandoned-stream cleanup, fallback, and residency + eviction - facade tests for lazy close and async disposal - real-Git same-fixture fallback/session command-process comparison +- real-Git prune/rewrite regression for individually written blobs +- real-Git individual-versus-batch write process comparison - existing large-stream restore and page-cache tests - Node, Bun, and Deno suites - committed JSON witness with counts, timing, and peak memory @@ -321,6 +357,12 @@ existing stream path. 6. `readBlobStream()` still delegates to `executeStream()`. 7. A session-capable real-Git read returns the same reference with fewer child process openings than the fallback adapter. +8. An individually written unreachable blob can be pruned and recreated with + the same OID. +9. A bounded page batch returns the same handles as individual writes while + opening one scoped fast-import process. +10. Close waits for an active one-shot command and destroys an abandoned Git + output stream before resolving. ## Acceptance Criteria @@ -352,14 +394,17 @@ The witness packet must answer: ## Risks -- Fast-import checkpoint cost may erase write-path gains; benchmark before - claiming improvement. +- Fast-import checkpoint and close cost may erase gains for tiny batches; + benchmark before claiming a wall-clock improvement. +- Low-level `writeBlobs()` callers must still provide a bounded iterable; the + adapter retains its yielded byte arrays until the scoped operation settles. - Policy timeouts can race an in-flight protocol operation; timeout handling must invalidate the session. - Raw tree parsing must support SHA-1 and SHA-256 repositories and malformed input without out-of-bounds reads. -- Callers that omit close may keep a process alive; docs and async disposal - reduce but cannot eliminate misuse. +- Idle retirement bounds reusable-session leaks when callers omit close, but an + abandoned output stream still requires close or eventual process completion; + docs and async disposal reduce but cannot eliminate misuse. ## Follow-On Debt diff --git a/index.d.ts b/index.d.ts index 083c376..e329040 100644 --- a/index.d.ts +++ b/index.d.ts @@ -513,6 +513,7 @@ export declare class CryptoPortBase { /** Abstract port for persisting data to Git's object database. */ export declare class GitPersistencePortBase { writeBlob(content: Uint8Array): Promise; + writeBlobs(contents: Iterable): Promise; writeTree(entries: string[]): Promise; readBlob(oid: string, maxBytes?: number): Promise; readBlobStream(oid: string): Promise>; @@ -529,6 +530,8 @@ export declare class GitPersistencePortBase { readObjectType(oid: string): Promise; readObjectSize(oid: string): Promise; setMaxBlobSize?(maxBlobSize: number): void; + close(): Promise; + [Symbol.asyncDispose](): Promise; } /** Abstract port for Git ref and commit operations. */ @@ -568,6 +571,9 @@ export declare class GitPersistenceAdapter extends GitPersistencePortBase { plumbing: unknown; policy?: unknown; metadataCacheEntries?: number; + sessionIdleTimeoutMs?: number; + treeCacheEntries?: number; + treeCacheBytes?: number; }); setMaxBlobSize(maxBlobSize: number): void; } @@ -1455,6 +1461,11 @@ export type PageSource = export interface PageCapability { put(options: { source: PageSource; maxBytes?: number }): Promise; + putBatch(options: { + pages: Array<{ source: PageSource; maxBytes?: number }>; + maxBatchBytes?: number; + maxBatchPages?: number; + }): Promise>; get(options: { handle: PageHandleInput; maxBytes?: number }): Promise; open(options: { handle: PageHandleInput }): AsyncIterable; } @@ -1723,6 +1734,9 @@ export default class ContentAddressableStore { getService(): Promise; getVaultService(): Promise; getRootSetRegistry(): Promise; + /** Releases local resources only. Stored objects and refs are unchanged. */ + close(): Promise; + [Symbol.asyncDispose](): Promise; static open(options?: ContentAddressableStoreOpenOptions): Promise; diff --git a/index.js b/index.js index 1f734ea..8e337a4 100644 --- a/index.js +++ b/index.js @@ -215,6 +215,7 @@ export default class ContentAddressableStore { }); this.pages = Object.freeze({ put: async (options) => (await this.#getPageService()).put(options), + putBatch: async (options) => (await this.#getPageService()).putBatch(options), get: async (options) => (await this.#getPageService()).get(options), open: (options) => this.#openPage(options), }); @@ -261,6 +262,8 @@ export default class ContentAddressableStore { #stagingWorkspaceRegistry = null; /** @type {RepositoryDoctor|null} */ #repositoryDoctor = null; + #closePromise = null; + #closed = false; #servicePromise = null; /** @@ -269,6 +272,7 @@ export default class ContentAddressableStore { * @returns {Promise} */ async #getService() { + this.#assertOpen(); if (!this.#servicePromise) { this.#servicePromise = this.#initService(); } @@ -608,6 +612,37 @@ export default class ContentAddressableStore { return await this.#getRootSetRegistry(); } + /** + * Releases local adapter resources. This does not mutate stored objects, + * refs, retention, or publication state. + * @returns {Promise} + */ + async close() { + if (this.#closePromise !== null) { + return await this.#closePromise; + } + this.#closed = true; + this.#closePromise = (async () => { + if (this.#servicePromise === null) { + return; + } + const service = await this.#servicePromise; + await service.persistence.close(); + })(); + return await this.#closePromise; + } + + /** @returns {Promise} */ + async [Symbol.asyncDispose]() { + await this.close(); + } + + #assertOpen() { + if (this.#closed) { + throw createCasError('Content-addressable store is closed', ErrorCodes.RESOURCE_CLOSED); + } + } + /** * Factory to create a default JSON CAS from a Git working directory. * diff --git a/package.json b/package.json index da78a6c..6b789c3 100644 --- a/package.json +++ b/package.json @@ -103,7 +103,7 @@ "@flyingrobots/bijou-tui": "^5.0.0", "@flyingrobots/bijou-tui-app": "^5.0.0", "@git-stunts/alfred": "^0.10.0", - "@git-stunts/plumbing": "^3.1.0", + "@git-stunts/plumbing": "^3.2.0", "@git-stunts/vault": "^1.0.1", "cbor-x": "^1.6.0", "commander": "14.0.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 838a709..44153eb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,8 +24,8 @@ importers: specifier: ^0.10.0 version: 0.10.0 '@git-stunts/plumbing': - specifier: ^3.1.0 - version: 3.1.0 + specifier: ^3.2.0 + version: 3.2.0 '@git-stunts/vault': specifier: ^1.0.1 version: 1.0.1 @@ -297,8 +297,8 @@ packages: resolution: {integrity: sha512-0DPhJdKhYTcsPuoOnYIIyvlwaIM7yIx4fQM4Q48abe/VDLTfZef1ubeT1pYio+ZTp1lKXtSG663973ewBbi/yw==} engines: {node: '>=20.0.0'} - '@git-stunts/plumbing@3.1.0': - resolution: {integrity: sha512-SLRySIS8FyTOy3mV43GY0xp/1QPu+Q/5CAbDE1HmL+NfnMhuMe6stiycjCQW10WLzbI6ihP+WKqrCgoMfAdRTw==} + '@git-stunts/plumbing@3.2.0': + resolution: {integrity: sha512-wW3Rzq6KNX0iygU9LpF8odQpcAyylH0KrUxNTDeRJHnwxfIamm7DWBL8lDkP1TqltE+4Bi0H0rjyPtvSkAJJeg==} engines: {bun: '>=1.3.5', deno: '>=2.0.0', node: '>=20.0.0'} '@git-stunts/vault@1.0.1': @@ -1161,7 +1161,7 @@ snapshots: '@git-stunts/alfred@0.10.0': {} - '@git-stunts/plumbing@3.1.0': + '@git-stunts/plumbing@3.2.0': dependencies: zod: 3.25.76 diff --git a/scripts/diagnostics/measure-git-object-sessions.js b/scripts/diagnostics/measure-git-object-sessions.js new file mode 100644 index 0000000..3aa965c --- /dev/null +++ b/scripts/diagnostics/measure-git-object-sessions.js @@ -0,0 +1,335 @@ +import assert from 'node:assert/strict'; +import { execFileSync, fork } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdtempSync, rmSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { fileURLToPath } from 'node:url'; +import ContentAddressableStore from '../../index.js'; +import { createGitPlumbing } from '../../src/infrastructure/createGitPlumbing.js'; + +const WORKER = '--worker'; +const DEFAULT_ITEMS = 32; +const DEFAULT_PAGE_BYTES = 4 * 1024; +const DEFAULT_SAMPLES = 3; +const MAX_BATCH_BYTES = 32 * 1024 * 1024; +const MAX_BATCH_ITEMS = 256; +const MIN_PAGE_BYTES = 4; +const scriptPath = fileURLToPath(import.meta.url); + +if (process.argv[2] === WORKER) { + await emitWorkerResult(JSON.parse(process.argv[3])); +} else { + await runController(); +} + +async function runController() { + const items = positiveSafeInteger(process.argv[2] ?? DEFAULT_ITEMS, 'items'); + const pageBytes = positiveSafeInteger(process.argv[3] ?? DEFAULT_PAGE_BYTES, 'pageBytes'); + const samples = positiveSafeInteger(process.argv[4] ?? DEFAULT_SAMPLES, 'samples'); + assertMeasurementBounds(items, pageBytes); + const root = mkdtempSync(path.join(os.tmpdir(), 'cas-object-session-measure-')); + try { + const readRepo = path.join(root, 'read.git'); + initBare(readRepo); + const handles = await buildReadFixture(readRepo, items, pageBytes); + const reads = await compareModes({ + left: { kind: 'read', mode: 'fallback', repo: readRepo, handles }, + right: { kind: 'read', mode: 'session', repo: readRepo, handles }, + samples, + }); + const writes = await compareModes({ + left: { kind: 'write', mode: 'individual', items, pageBytes }, + right: { kind: 'write', mode: 'batch', items, pageBytes }, + samples, + }); + assert.equal(reads.left.semanticDigest, reads.right.semanticDigest); + assert.equal(writes.left.semanticDigest, writes.right.semanticDigest); + process.stdout.write( + `${JSON.stringify(buildReport({ items, pageBytes, samples, reads, writes }), null, 2)}\n` + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +async function compareModes({ left, right, samples }) { + const leftSamples = []; + const rightSamples = []; + for (let index = 0; index < samples; index += 1) { + const order = index % 2 === 0 ? [left, right] : [right, left]; + for (const options of order) { + const result = await runWorkerProcess(options); + (options === left ? leftSamples : rightSamples).push(result); + } + } + return { left: summarize(leftSamples), right: summarize(rightSamples) }; +} + +function buildReport({ items, pageBytes, samples, reads, writes }) { + return { + schema: 'git-cas.git-object-session-measurement/v1', + generatedAt: new Date().toISOString(), + environment: { + node: process.version, + git: execFileSync('git', ['--version'], { encoding: 'utf8' }).trim(), + platform: process.platform, + architecture: process.arch, + }, + parameters: { items, pageBytes, samples }, + selectedBundleRead: comparison({ + left: reads.left, + right: reads.right, + leftName: 'fallback', + rightName: 'session', + }), + pageWrite: comparison({ + left: writes.left, + right: writes.right, + leftName: 'individual', + rightName: 'batch', + }), + }; +} + +function comparison({ left, right, leftName, rightName }) { + return { + [leftName]: left, + [rightName]: right, + semanticDigestEqual: left.semanticDigest === right.semanticDigest, + processReductionPercent: percentageReduction(left.processCount, right.processCount), + wallReductionPercent: percentageReduction(left.wallMs, right.wallMs), + cpuReductionPercent: percentageReduction(left.cpuMs, right.cpuMs), + }; +} + +async function buildReadFixture(repo, items, pageBytes) { + const cas = new ContentAddressableStore({ plumbing: await createGitPlumbing({ cwd: repo }) }); + try { + const handles = []; + for (let index = 0; index < items; index += 1) { + const page = await cas.pages.put({ source: pageSource(index, pageBytes) }); + const nested = await cas.bundles.put({ members: { page: page.handle } }); + const outer = await cas.bundles.put({ members: { nested: nested.handle } }); + handles.push(outer.handle.toString()); + } + return handles; + } finally { + await cas.close(); + } +} + +async function emitWorkerResult(options) { + try { + const result = + options.kind === 'read' ? await measureReads(options) : await measureWrites(options); + process.send?.({ ok: true, result }); + } catch (error) { + process.send?.({ ok: false, error: error?.stack ?? String(error) }); + process.exitCode = 1; + } finally { + process.disconnect?.(); + } +} + +async function measureReads({ mode, repo, handles }) { + const counted = await countedPlumbing(repo, { sessions: mode === 'session' }); + const cas = new ContentAddressableStore({ plumbing: counted.plumbing }); + const values = []; + const metrics = await timed(async () => { + for (const handle of handles) { + const reference = await cas.bundles.getMemberReference({ handle, path: 'nested' }); + values.push(`${reference.path}\0${reference.type}\0${reference.handle.toString()}`); + } + await cas.close(); + }); + return resultEnvelope(values, counted.snapshot(), metrics); +} + +async function measureWrites({ mode, items, pageBytes }) { + const repo = mkdtempSync(path.join(os.tmpdir(), `cas-${mode}-write-`)); + try { + initBare(repo); + const counted = await countedPlumbing(repo, { sessions: true }); + const cas = new ContentAddressableStore({ plumbing: counted.plumbing }); + const sources = Array.from({ length: items }, (_, index) => pageSource(index, pageBytes)); + let pages; + const metrics = await timed(async () => { + pages = + mode === 'batch' + ? await cas.pages.putBatch({ pages: sources.map((source) => ({ source })) }) + : await putIndividually(cas, sources); + await cas.close(); + }); + return resultEnvelope( + pages.map((page) => page.handle.toString()), + counted.snapshot(), + metrics + ); + } finally { + rmSync(repo, { recursive: true, force: true }); + } +} + +async function putIndividually(cas, sources) { + const pages = []; + for (const source of sources) { + pages.push(await cas.pages.put({ source })); + } + return pages; +} + +async function timed(operation) { + const startedAt = performance.now(); + const startedCpu = process.cpuUsage(); + await operation(); + const cpu = process.cpuUsage(startedCpu); + return { + wallMs: performance.now() - startedAt, + userCpuMs: cpu.user / 1000, + systemCpuMs: cpu.system / 1000, + peakRssBytes: process.resourceUsage().maxRSS * 1024, + }; +} + +async function countedPlumbing(repo, { sessions }) { + const plumbing = await createGitPlumbing({ cwd: repo }); + const counts = new Map(); + const record = (name) => counts.set(name, (counts.get(name) ?? 0) + 1); + const counted = { + execute(options) { + record(operationOf(options.args)); + return plumbing.execute(options); + }, + executeStream(options) { + record(operationOf(options.args)); + return plumbing.executeStream(options); + }, + }; + if (sessions) { + counted.openCatFileSession = sessionOpener(plumbing, counts, 'cat-file'); + counted.openMktreeSession = sessionOpener(plumbing, counts, 'mktree'); + counted.openFastImportSession = sessionOpener(plumbing, counts, 'fast-import'); + } + return { plumbing: counted, snapshot: () => Object.fromEntries(counts) }; +} + +function sessionOpener(plumbing, counts, protocol) { + const method = `open${protocol === 'cat-file' ? 'CatFile' : protocol === 'mktree' ? 'Mktree' : 'FastImport'}Session`; + return (...args) => { + const key = `session:${protocol}`; + counts.set(key, (counts.get(key) ?? 0) + 1); + return plumbing[method](...args); + }; +} + +function operationOf(args) { + if (args[0] === 'cat-file' && args.some((arg) => arg.startsWith('--batch-check='))) { + return 'cat-file:batch-check'; + } + return args[0]; +} + +function resultEnvelope(values, counts, metrics) { + return { + semanticDigest: digest(values), + resultCount: values.length, + processCount: Object.values(counts).reduce((sum, value) => sum + value, 0), + counts, + ...metrics, + }; +} + +function summarize(samples) { + assert(samples.length > 0); + for (const sample of samples.slice(1)) { + assert.equal(sample.semanticDigest, samples[0].semanticDigest); + assert.deepEqual(sample.counts, samples[0].counts); + } + return { + sampleCount: samples.length, + semanticDigest: samples[0].semanticDigest, + resultCount: samples[0].resultCount, + processCount: samples[0].processCount, + counts: samples[0].counts, + wallMs: roundedMedian(samples.map((sample) => sample.wallMs)), + cpuMs: roundedMedian(samples.map((sample) => sample.userCpuMs + sample.systemCpuMs)), + userCpuMs: roundedMedian(samples.map((sample) => sample.userCpuMs)), + systemCpuMs: roundedMedian(samples.map((sample) => sample.systemCpuMs)), + peakRssBytes: Math.round(median(samples.map((sample) => sample.peakRssBytes))), + }; +} + +function runWorkerProcess(options) { + return new Promise((resolve, reject) => { + const child = fork(scriptPath, [WORKER, JSON.stringify(options)], { + stdio: ['ignore', 'ignore', 'inherit', 'ipc'], + }); + let response; + child.once('message', (message) => { + response = message; + }); + child.once('error', reject); + child.once('exit', (code) => { + if (code === 0 && response?.ok) { + resolve(response.result); + } else { + reject(new Error(response?.error ?? `measurement worker exited with code ${code}`)); + } + }); + }); +} + +function pageSource(index, bytes) { + const source = Buffer.alloc(bytes, index % 251); + source.writeUInt32BE(index, 0); + return source; +} + +function initBare(repo) { + execFileSync('git', ['init', '--bare', repo], { stdio: 'ignore' }); +} + +function digest(values) { + const hash = createHash('sha256'); + for (const value of values) { + hash.update(value); + hash.update('\0'); + } + return hash.digest('hex'); +} + +function percentageReduction(before, after) { + return Math.round(((before - after) / before) * 1000) / 10; +} + +function roundedMedian(values) { + return Math.round(median(values) * 1000) / 1000; +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; +} + +function positiveSafeInteger(raw, label) { + const value = Number(raw); + if (!Number.isSafeInteger(value) || value < 1) { + throw new TypeError(`${label} must be a positive safe integer`); + } + return value; +} + +function assertMeasurementBounds(items, pageBytes) { + if (items > MAX_BATCH_ITEMS) { + throw new RangeError(`items must not exceed ${MAX_BATCH_ITEMS}`); + } + if (pageBytes < MIN_PAGE_BYTES) { + throw new RangeError(`pageBytes must be at least ${MIN_PAGE_BYTES}`); + } + if (pageBytes > Math.floor(MAX_BATCH_BYTES / items)) { + throw new RangeError(`items * pageBytes must not exceed ${MAX_BATCH_BYTES}`); + } +} diff --git a/src/domain/errors/Codes.js b/src/domain/errors/Codes.js index 9a58fe7..4a150b8 100644 --- a/src/domain/errors/Codes.js +++ b/src/domain/errors/Codes.js @@ -55,6 +55,7 @@ const ErrorCodes = Object.freeze({ MANIFEST_NOT_FOUND: 'MANIFEST_NOT_FOUND', MISSING_KEY: 'MISSING_KEY', NO_MATCHING_RECIPIENT: 'NO_MATCHING_RECIPIENT', + PAGE_BATCH_LIMIT: 'PAGE_BATCH_LIMIT', PAGE_TOO_LARGE: 'PAGE_TOO_LARGE', PERSISTENCE_CAPABILITY_REQUIRED: 'PERSISTENCE_CAPABILITY_REQUIRED', PORT_NOT_IMPLEMENTED: 'PORT_NOT_IMPLEMENTED', @@ -65,6 +66,7 @@ const ErrorCodes = Object.freeze({ RECIPIENT_ALREADY_EXISTS: 'RECIPIENT_ALREADY_EXISTS', RECIPIENT_NOT_FOUND: 'RECIPIENT_NOT_FOUND', REPOSITORY_INSPECTION_INVALID: 'REPOSITORY_INSPECTION_INVALID', + RESOURCE_CLOSED: 'RESOURCE_CLOSED', RESTORE_TOO_LARGE: 'RESTORE_TOO_LARGE', RETENTION_WITNESS_INVALID: 'RETENTION_WITNESS_INVALID', ROOT_SET_CONFLICT: 'ROOT_SET_CONFLICT', diff --git a/src/domain/services/PageService.js b/src/domain/services/PageService.js index e26871e..33764a8 100644 --- a/src/domain/services/PageService.js +++ b/src/domain/services/PageService.js @@ -9,6 +9,8 @@ import BoundedPromiseCache from '../../helpers/boundedPromiseCache.js'; export const DEFAULT_MAX_PAGE_SIZE = 16 * 1024 * 1024; const DEFAULT_PAGE_CACHE_ENTRIES = 128; const DEFAULT_PAGE_CACHE_BYTES = 8 * 1024 * 1024; +const DEFAULT_PAGE_WRITE_BATCH_BYTES = 32 * 1024 * 1024; +const DEFAULT_PAGE_WRITE_BATCH_PAGES = 256; const DEFAULT_CLOCK = Object.freeze({ now: () => new Date() }); /** @@ -64,6 +66,57 @@ export default class PageService { }); } + /** + * Stores one explicitly bounded page batch. All inputs are validated and + * collected before the persistence batch begins. + * @param {object} options + * @param {Array<{source: Uint8Array|Iterable|AsyncIterable, maxBytes?: number}>} options.pages + * @param {number} [options.maxBatchBytes] + * @param {number} [options.maxBatchPages] + * @returns {Promise>} + */ + async putBatch({ + pages, + maxBatchBytes = DEFAULT_PAGE_WRITE_BATCH_BYTES, + maxBatchPages = DEFAULT_PAGE_WRITE_BATCH_PAGES, + }) { + PageService.#assertBatch(pages, maxBatchBytes, maxBatchPages); + const prepared = []; + let totalBytes = 0; + for (const page of pages) { + if (page === null || typeof page !== 'object' || !Object.hasOwn(page, 'source')) { + throw createCasError('Page batch entry must provide source', ErrorCodes.INVALID_OPTIONS); + } + const bytes = await PageService.#collect(page.source, this.#effectiveLimit(page.maxBytes)); + totalBytes += bytes.length; + if (totalBytes > maxBatchBytes) { + throw createCasError( + 'Page batch exceeds its configured byte limit', + ErrorCodes.PAGE_BATCH_LIMIT, + { observedBytes: totalBytes, maxBatchBytes }, + ); + } + prepared.push({ bytes, observedAt: this.#observedAt() }); + } + + if (prepared.length === 0) { + return Object.freeze([]); + } + const oids = await this.#writeBlobs(prepared.map((page) => page.bytes)); + if (oids.length !== prepared.length) { + throw createCasError( + 'Persistence returned the wrong number of page object identifiers', + ErrorCodes.GIT_ERROR, + { expected: prepared.length, actual: oids.length }, + ); + } + return Object.freeze(oids.map((oid, index) => new StagedPage({ + handle: new PageHandle({ oid }), + size: prepared[index].bytes.length, + observedAt: prepared[index].observedAt, + }))); + } + /** * @param {{ handle: PageHandle|string|object }} options * @returns {AsyncIterable} @@ -160,6 +213,17 @@ export default class PageService { return now.toISOString(); } + async #writeBlobs(contents) { + if (typeof this.#persistence.writeBlobs === 'function') { + return await this.#persistence.writeBlobs(contents); + } + const oids = []; + for (const content of contents) { + oids.push(await this.#persistence.writeBlob(content)); + } + return oids; + } + static async #collect(source, maxBytes) { if (isBytes(source)) { if (source.length > maxBytes) { @@ -217,6 +281,21 @@ export default class PageService { } } + static #assertBatch(pages, maxBatchBytes, maxBatchPages) { + PageService.#assertPositiveLimit(maxBatchBytes, 'Page batch bytes'); + PageService.#assertPositiveLimit(maxBatchPages, 'Page batch pages'); + if (!Array.isArray(pages)) { + throw createCasError('Page batch must be an array', ErrorCodes.INVALID_OPTIONS); + } + if (pages.length > maxBatchPages) { + throw createCasError( + 'Page batch exceeds its configured page limit', + ErrorCodes.PAGE_BATCH_LIMIT, + { observedPages: pages.length, maxBatchPages }, + ); + } + } + static #assertDependencies(persistence, clock) { const methods = ['writeBlob', 'readBlobStream', 'readObjectType', 'readObjectSize']; if (!persistence || methods.some((method) => typeof persistence[method] !== 'function')) { diff --git a/src/infrastructure/adapters/GitObjectSessionPool.js b/src/infrastructure/adapters/GitObjectSessionPool.js new file mode 100644 index 0000000..0e4130e --- /dev/null +++ b/src/infrastructure/adapters/GitObjectSessionPool.js @@ -0,0 +1,234 @@ +import { + GitObjectMissingError, + GitProtocolError, + InvalidArgumentError, +} from '@git-stunts/plumbing'; + +const PROTOCOLS = Object.freeze({ + catFile: Object.freeze({ opener: 'openCatFileSession', abort: 'terminate' }), + fastImport: Object.freeze({ opener: 'openFastImportSession', abort: 'abort' }), + mktree: Object.freeze({ opener: 'openMktreeSession', abort: 'terminate' }), +}); + +/** + * Lazily owns one typed plumbing session per Git object protocol. + */ +export default class GitObjectSessionPool { + #active = new Map(); + #closePromise = null; + #closed = false; + #idleTimeoutMs; + #idleTimers = new Map(); + #plumbing; + #sessions = new Map(); + + constructor({ plumbing, idleTimeoutMs }) { + this.#plumbing = plumbing; + this.#idleTimeoutMs = idleTimeoutMs; + } + + supports(protocol) { + const descriptor = PROTOCOLS[protocol]; + return descriptor !== undefined && typeof this.#plumbing[descriptor.opener] === 'function'; + } + + async info(objectName) { + return await this.#run('catFile', (session) => session.info(objectName), isRecoverableCatError); + } + + async read(objectName, options) { + return await this.#run( + 'catFile', + (session) => session.read(objectName, options), + isRecoverableCatError + ); + } + + async writeBlobs(contents) { + return await this.#run('fastImport', async (session) => { + const oids = []; + for (const content of contents) { + oids.push(await session.writeBlob(content)); + } + await session.checkpoint(); + return Object.freeze(oids); + }); + } + + async writeTree(entries) { + return await this.#run('mktree', (session) => session.write(entries)); + } + + async invalidate(protocol, expectedSession) { + this.#cancelIdle(protocol); + const present = this.#sessions.get(protocol); + this.#sessions.delete(protocol); + let session = expectedSession; + if (session === undefined && present !== undefined) { + session = await present.catch(() => undefined); + } + if (session !== undefined) { + await this.#abort(protocol, session); + } + } + + async retire(protocol) { + this.#cancelIdle(protocol); + const opening = this.#sessions.get(protocol); + this.#sessions.delete(protocol); + if (opening === undefined) { + return; + } + const session = await opening.catch(() => undefined); + if (session === undefined) { + return; + } + try { + await session.close(); + } catch { + await this.#abort(protocol, session).catch(() => {}); + } + } + + async close() { + if (this.#closePromise !== null) { + return await this.#closePromise; + } + this.#closed = true; + for (const protocol of this.#idleTimers.keys()) { + this.#cancelIdle(protocol); + } + const sessions = [...this.#sessions.entries()]; + this.#sessions.clear(); + this.#closePromise = (async () => { + const results = await Promise.allSettled( + sessions.map(async ([, opening]) => { + const session = await opening; + await session.close(); + }) + ); + const failures = results + .filter((result) => result.status === 'rejected') + .map((result) => result.reason); + if (failures.length > 0) { + throw new AggregateError(failures, 'One or more Git object sessions failed to close'); + } + })(); + return await this.#closePromise; + } + + async #run(protocol, operation, recoverable = () => false) { + this.#cancelIdle(protocol); + this.#active.set(protocol, (this.#active.get(protocol) ?? 0) + 1); + try { + return await this.#attempt({ protocol, operation, recoverable, mayRetry: true }); + } finally { + this.#release(protocol); + } + } + + async #attempt({ protocol, operation, recoverable, mayRetry }) { + let session; + try { + session = await this.#session(protocol); + return await operation(session); + } catch (error) { + if (recoverable(error) || error instanceof InvalidArgumentError) { + throw error; + } + await this.#invalidateSafely(protocol, session); + if (mayRetry && error instanceof GitProtocolError) { + return await this.#attempt({ protocol, operation, recoverable, mayRetry: false }); + } + throw error; + } + } + + async #invalidateSafely(protocol, session) { + if (session === undefined) { + return; + } + try { + await this.invalidate(protocol, session); + } catch { + // Preserve the operation failure; cleanup is best effort here. + } + } + + #release(protocol) { + const remaining = (this.#active.get(protocol) ?? 1) - 1; + if (remaining === 0) { + this.#active.delete(protocol); + this.#scheduleIdle(protocol); + return; + } + this.#active.set(protocol, remaining); + } + + async #session(protocol) { + if (this.#closed) { + throw new Error('Git object session pool is closed'); + } + const descriptor = PROTOCOLS[protocol]; + if (descriptor === undefined || typeof this.#plumbing[descriptor.opener] !== 'function') { + throw new Error(`Git object protocol is unavailable: ${protocol}`); + } + let opening = this.#sessions.get(protocol); + if (opening === undefined) { + opening = Promise.resolve().then(() => this.#plumbing[descriptor.opener]()); + this.#sessions.set(protocol, opening); + opening.catch(() => { + if (this.#sessions.get(protocol) === opening) { + this.#sessions.delete(protocol); + } + }); + } + return await opening; + } + + async #abort(protocol, session) { + const method = PROTOCOLS[protocol]?.abort; + if (method !== undefined && typeof session[method] === 'function') { + await session[method](); + } + } + + #scheduleIdle(protocol) { + if (this.#closed || this.#active.has(protocol) || !this.#sessions.has(protocol)) { + return; + } + const opening = this.#sessions.get(protocol); + const timer = setTimeout(() => { + this.#idleTimers.delete(protocol); + if (this.#active.has(protocol) || this.#sessions.get(protocol) !== opening) { + return; + } + this.#sessions.delete(protocol); + void opening + .then((session) => session.close()) + .catch(async () => { + const session = await opening.catch(() => undefined); + if (session !== undefined) { + await this.#abort(protocol, session).catch(() => {}); + } + }); + }, this.#idleTimeoutMs); + timer.unref?.(); + this.#idleTimers.set(protocol, timer); + } + + #cancelIdle(protocol) { + const timer = this.#idleTimers.get(protocol); + if (timer !== undefined) { + clearTimeout(timer); + this.#idleTimers.delete(protocol); + } + } +} + +function isRecoverableCatError(error) { + return ( + error instanceof GitObjectMissingError || + error?.details?.code === 'OBJECT_BUFFER_LIMIT_EXCEEDED' + ); +} diff --git a/src/infrastructure/adapters/GitPersistenceAdapter.js b/src/infrastructure/adapters/GitPersistenceAdapter.js index b1d7883..1036722 100644 --- a/src/infrastructure/adapters/GitPersistenceAdapter.js +++ b/src/infrastructure/adapters/GitPersistenceAdapter.js @@ -2,9 +2,12 @@ import { Policy } from '@git-stunts/alfred'; import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; +import { GitObjectMissingError } from '@git-stunts/plumbing'; import GitPersistencePort from '../../ports/GitPersistencePort.js'; import { CasError, createCasError, ErrorCodes } from '../../domain/errors/index.js'; import BoundedPromiseCache from '../../helpers/boundedPromiseCache.js'; +import GitTreeObjectCodec from '../codecs/GitTreeObjectCodec.js'; +import GitObjectSessionPool from './GitObjectSessionPool.js'; /** * Default resilience policy: 30 s timeout (no retry). @@ -17,6 +20,9 @@ import BoundedPromiseCache from '../../helpers/boundedPromiseCache.js'; const DEFAULT_POLICY = Policy.timeout(30_000); export const DEFAULT_MAX_BLOB_SIZE = 10 * 1024 * 1024; const DEFAULT_METADATA_CACHE_ENTRIES = 2_048; +const DEFAULT_SESSION_IDLE_TIMEOUT_MS = 1_000; +const DEFAULT_TREE_CACHE_BYTES = 8 * 1024 * 1024; +const DEFAULT_TREE_CACHE_ENTRIES = 256; const MIN_READ_BLOB_LIMIT = 1; const MIN_MAX_BLOB_SIZE = 1024; const MIN_METADATA_CACHE_ENTRIES = 1; @@ -33,20 +39,46 @@ const GIT_OBJECT_TYPES = new Set(['blob', 'tree', 'commit', 'tag']); * must not race active operations. */ export default class GitPersistenceAdapter extends GitPersistencePort { + #activeOperations = new Set(); + #activeStreams = new Set(); + #closePromise = null; + #closed = false; #maxBlobSize = DEFAULT_MAX_BLOB_SIZE; #metadataCache; + #sessions; + #treeCache; + #treeReadMaxBytes; /** * @param {Object} options * @param {import('@git-stunts/plumbing').default} options.plumbing - GitPlumbing instance. * @param {import('@git-stunts/alfred').Policy} [options.policy] - Resilience policy (defaults to 30 s timeout, no retry). * @param {number} [options.metadataCacheEntries=2048] - Maximum immutable metadata entries retained by this adapter. + * @param {number} [options.sessionIdleTimeoutMs=1000] - Idle delay before a reusable Git process closes automatically. + * @param {number} [options.treeCacheEntries=256] - Maximum immutable parsed tree objects retained by this adapter. + * @param {number} [options.treeCacheBytes=8388608] - Maximum estimated parsed tree bytes retained by this adapter. */ - constructor({ plumbing, policy, metadataCacheEntries = DEFAULT_METADATA_CACHE_ENTRIES }) { + constructor({ + plumbing, + policy, + metadataCacheEntries = DEFAULT_METADATA_CACHE_ENTRIES, + sessionIdleTimeoutMs = DEFAULT_SESSION_IDLE_TIMEOUT_MS, + treeCacheEntries = DEFAULT_TREE_CACHE_ENTRIES, + treeCacheBytes = DEFAULT_TREE_CACHE_BYTES, + }) { super(); GitPersistenceAdapter.#assertMetadataCacheEntries(metadataCacheEntries); + GitPersistenceAdapter.#assertSessionIdleTimeout(sessionIdleTimeoutMs); + GitPersistenceAdapter.#assertTreeCacheEntries(treeCacheEntries); + GitPersistenceAdapter.#assertTreeCacheBytes(treeCacheBytes); this.plumbing = plumbing; this.policy = policy ?? DEFAULT_POLICY; this.#metadataCache = new BoundedPromiseCache(metadataCacheEntries); + this.#sessions = new GitObjectSessionPool({ plumbing, idleTimeoutMs: sessionIdleTimeoutMs }); + this.#treeCache = new BoundedPromiseCache(treeCacheEntries, { + maxWeight: treeCacheBytes, + weightOf: (tree) => tree.weight, + }); + this.#treeReadMaxBytes = Math.max(MIN_READ_BLOB_LIMIT, treeCacheBytes); } /** @@ -55,14 +87,50 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {Promise} The Git OID of the stored blob. */ async writeBlob(content) { - return this.policy.execute(() => ( + return await this.#runOperation(() => this.#writeBlob(content)); + } + + async #writeBlob(content) { + const oid = await this.policy.execute(() => typeof globalThis.Bun !== 'undefined' ? this.#writeBlobFromTempFile(content) : this.plumbing.execute({ - args: ['hash-object', '-w', '--stdin'], - input: content, - }) - )); + args: ['hash-object', '-w', '--stdin'], + input: content, + }) + ); + await Promise.all([this.#sessions.retire('catFile'), this.#sessions.retire('mktree')]); + return oid; + } + + /** + * Writes a bounded group through one scoped fast-import session when the + * injected plumbing supports it. The session closes before OIDs are exposed, + * so later pruning cannot poison duplicate writes in a reused process. + * @override + * @param {Iterable} contents + * @returns {Promise} + */ + async writeBlobs(contents) { + return await this.#runOperation(async () => { + const replayableContents = [...contents]; + if (!this.#sessions.supports('fastImport')) { + const oids = []; + for (const content of replayableContents) { + oids.push(await this.#writeBlob(content)); + } + return oids; + } + try { + const oids = await this.#executeSession('fastImport', () => + this.#sessions.writeBlobs(replayableContents) + ); + await Promise.all([this.#sessions.retire('catFile'), this.#sessions.retire('mktree')]); + return [...oids]; + } finally { + await this.#sessions.retire('fastImport'); + } + }); } /** @@ -71,12 +139,22 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {Promise} The Git OID of the created tree. */ async writeTree(entries) { - return this.policy.execute(() => - this.plumbing.execute({ - args: ['mktree'], - input: `${entries.join('\n')}\n`, - }), - ); + return await this.#runOperation(async () => { + if (this.#sessions.supports('mktree')) { + const structured = GitTreeObjectCodec.parseMktreeLines(entries); + const oid = await this.#executeSession('mktree', () => + this.#sessions.writeTree(structured) + ); + await this.#sessions.retire('catFile'); + return oid; + } + return this.policy.execute(() => + this.plumbing.execute({ + args: ['mktree'], + input: `${entries.join('\n')}\n`, + }) + ); + }); } /** @@ -86,23 +164,45 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {Promise} The blob content. */ async readBlob(oid, maxBytes) { - const limit = maxBytes === undefined - ? this.#maxBlobSize - : GitPersistenceAdapter.#validatedReadBlobLimit(maxBytes); - const chunks = []; - let bytesRead = 0; - for await (const chunk of await this.readBlobStream(oid)) { - bytesRead += chunk.length; - if (bytesRead > limit) { - throw new CasError( - `Blob ${oid} exceeds safety limit of ${limit} bytes`, - ErrorCodes.RESTORE_TOO_LARGE, - { oid, maxBytes: limit }, - ); + return await this.#runOperation(async () => { + const limit = + maxBytes === undefined + ? this.#maxBlobSize + : GitPersistenceAdapter.#validatedReadBlobLimit(maxBytes); + if (this.#sessions.supports('catFile')) { + let object; + try { + object = await this.#executeSession( + 'catFile', + () => this.#sessions.read(oid, { maxBytes: limit }), + GitPersistenceAdapter.#isRecoverableCatError + ); + } catch (error) { + throw this.#normalizeObjectReadError(error, oid, limit); + } + if (object.type !== 'blob') { + throw createCasError(`Git object is not a blob: ${oid}`, ErrorCodes.GIT_ERROR, { + oid, + actualType: object.type, + }); + } + return GitPersistenceAdapter.#toBuffer(object.content); } - chunks.push(chunk); - } - return Buffer.concat(chunks); + const chunks = []; + let bytesRead = 0; + for await (const chunk of await this.#openBufferStream(['cat-file', 'blob', oid])) { + bytesRead += chunk.length; + if (bytesRead > limit) { + throw new CasError( + `Blob ${oid} exceeds safety limit of ${limit} bytes`, + ErrorCodes.RESTORE_TOO_LARGE, + { oid, maxBytes: limit } + ); + } + chunks.push(chunk); + } + return Buffer.concat(chunks); + }); } /** @@ -113,6 +213,7 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {void} */ setMaxBlobSize(maxBlobSize) { + this.#assertOpen(); GitPersistenceAdapter.#assertMaxBlobSize(maxBlobSize); this.#maxBlobSize = maxBlobSize; } @@ -123,13 +224,7 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {Promise>} The blob content stream. */ async readBlobStream(oid) { - const stream = await this.policy.execute(async () => ( - await this.plumbing.executeStream({ - args: ['cat-file', 'blob', oid], - }) - )); - - return this.#bufferStream(stream); + return await this.#runOperation(() => this.#openBufferStream(['cat-file', 'blob', oid])); } /** @@ -138,6 +233,22 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {Promise>} */ async readTree(treeOid) { + return await this.#runOperation(async () => { + if (this.#sessions.supports('catFile')) { + try { + const tree = await this.#readTreeObject(treeOid); + return tree.entries.map((entry) => ({ ...entry })); + } catch (error) { + if (!GitPersistenceAdapter.#isObjectBufferLimit(error)) { + throw this.#normalizeTreeReadError(error, treeOid); + } + } + } + return await this.#readTreeWithCommand(treeOid); + }); + } + + async #readTreeWithCommand(treeOid) { return this.policy.execute(async () => { const output = await this.plumbing.execute({ args: ['ls-tree', '-z', treeOid], @@ -154,15 +265,35 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {Promise<{ mode: string, type: string, oid: string, name: string }|null>} */ async readTreeEntry(treeOid, treePath) { - const key = `tree\0${treeOid}\0${treePath}`; - const entry = await this.#metadataCache.getOrCreate(key, () => this.policy.execute(async () => { + return await this.#runOperation(async () => { + const key = `tree\0${treeOid}\0${treePath}`; + const entry = await this.#metadataCache.getOrCreate(key, async () => { + let found; + if (this.#sessions.supports('catFile')) { + try { + found = await this.#readExactTreePath(treeOid, treePath); + } catch (error) { + if (!GitPersistenceAdapter.#isObjectBufferLimit(error)) { + throw this.#normalizeTreeReadError(error, treeOid); + } + found = await this.#readTreeEntryWithCommand(treeOid, treePath); + } + } else { + found = await this.#readTreeEntryWithCommand(treeOid, treePath); + } + return found === null ? null : Object.freeze(found); + }); + return entry === null ? null : { ...entry }; + }); + } + + async #readTreeEntryWithCommand(treeOid, treePath) { + return this.policy.execute(async () => { const output = await this.plumbing.execute({ args: ['ls-tree', '-z', treeOid, '--', treePath], }); - const found = GitPersistenceAdapter.#parseTreeOutput(output)[0] || null; - return found === null ? null : Object.freeze(found); - })); - return entry === null ? null : { ...entry }; + return GitPersistenceAdapter.#parseTreeOutput(output)[0] || null; + }); } /** @@ -171,11 +302,9 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {AsyncIterable<{ mode: string, type: string, oid: string, name: string }>} */ async *iterateTree(treeOid) { - const stream = await this.policy.execute(async () => ( - await this.plumbing.executeStream({ - args: ['ls-tree', '-z', treeOid], - }) - )); + const stream = await this.#runOperation(() => + this.#openBufferStream(['ls-tree', '-z', treeOid]) + ); let pending = ''; for await (const chunk of stream) { pending += GitPersistenceAdapter.#toBuffer(chunk).toString(); @@ -200,7 +329,7 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {Promise} Git object type. */ async readObjectType(oid) { - return (await this.#readObjectInfo(oid)).type; + return await this.#runOperation(async () => (await this.#readObjectInfo(oid)).type); } /** @@ -209,7 +338,7 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {Promise} Git object size in bytes. */ async readObjectSize(oid) { - return (await this.#readObjectInfo(oid)).size; + return await this.#runOperation(async () => (await this.#readObjectInfo(oid)).size); } /** @@ -222,10 +351,24 @@ export default class GitPersistenceAdapter extends GitPersistencePort { */ async #readObjectInfo(oid) { return this.#metadataCache.getOrCreate(`object\0${oid}`, async () => { - const rawOutput = await this.policy.execute(() => this.plumbing.execute({ - args: ['cat-file', OBJECT_INFO_ARGUMENT], - input: `${oid}\n`, - })); + if (this.#sessions.supports('catFile')) { + try { + const info = await this.#executeSession( + 'catFile', + () => this.#sessions.info(oid), + GitPersistenceAdapter.#isRecoverableCatError + ); + return Object.freeze({ oid: info.oid, type: info.type, size: info.size }); + } catch (error) { + throw this.#normalizeObjectReadError(error, oid, this.#maxBlobSize); + } + } + const rawOutput = await this.policy.execute(() => + this.plumbing.execute({ + args: ['cat-file', OBJECT_INFO_ARGUMENT], + input: `${oid}\n`, + }) + ); const output = typeof rawOutput === 'string' ? rawOutput.trim() : ''; if (output === `${oid} missing`) { throw new CasError(`Git object not found: ${oid}`, ErrorCodes.GIT_OBJECT_NOT_FOUND, { @@ -251,6 +394,173 @@ export default class GitPersistenceAdapter extends GitPersistencePort { }); } + async #readTreeObject(treeOid) { + return this.#treeCache.getOrCreate(treeOid, async () => { + const object = await this.#executeSession( + 'catFile', + () => this.#sessions.read(treeOid, { maxBytes: this.#treeReadMaxBytes }), + GitPersistenceAdapter.#isRecoverableCatError + ); + if (object.type !== 'tree') { + throw createCasError(`Git object is not a tree: ${treeOid}`, ErrorCodes.TREE_PARSE_ERROR, { + treeOid, + actualType: object.type, + }); + } + try { + return GitTreeObjectCodec.decode(object.content, treeOid); + } catch (error) { + throw createCasError( + `Git tree object is malformed: ${treeOid}`, + ErrorCodes.TREE_PARSE_ERROR, + { treeOid, originalError: error } + ); + } + }); + } + + async #readExactTreePath(treeOid, treePath) { + if (typeof treePath !== 'string' || treePath.length === 0) { + throw createCasError('Git tree path must be a non-empty string', ErrorCodes.INVALID_OPTIONS, { + treeOid, + treePath, + }); + } + const components = treePath.split('/'); + if (components.some((component) => component.length === 0)) { + throw createCasError( + 'Git tree path contains an empty component', + ErrorCodes.INVALID_OPTIONS, + { + treeOid, + treePath, + } + ); + } + + let currentOid = treeOid; + for (let index = 0; index < components.length; index += 1) { + const tree = await this.#readTreeObject(currentOid); + const entry = tree.entries.find((candidate) => candidate.name === components[index]) ?? null; + if (entry === null) { + return null; + } + if (index === components.length - 1) { + return { ...entry, name: treePath }; + } + if (entry.type !== 'tree') { + return null; + } + currentOid = entry.oid; + } + return null; + } + + async #executeSession(protocol, operation, recoverable = () => false) { + try { + return await this.policy.execute(operation); + } catch (error) { + if (!recoverable(error)) { + try { + await this.#sessions.invalidate(protocol); + } catch { + // Preserve the operation or timeout failure. + } + } + throw error; + } + } + + #normalizeObjectReadError(error, oid, maxBytes) { + if (error instanceof GitObjectMissingError) { + return new CasError(`Git object not found: ${oid}`, ErrorCodes.GIT_OBJECT_NOT_FOUND, { oid }); + } + if (GitPersistenceAdapter.#isObjectBufferLimit(error)) { + return new CasError( + `Blob ${oid} exceeds safety limit of ${maxBytes} bytes`, + ErrorCodes.RESTORE_TOO_LARGE, + { oid, maxBytes } + ); + } + return error; + } + + #normalizeTreeReadError(error, treeOid) { + if (error instanceof GitObjectMissingError) { + return new CasError(`Git object not found: ${treeOid}`, ErrorCodes.GIT_OBJECT_NOT_FOUND, { + oid: treeOid, + }); + } + return error; + } + + async close() { + if (this.#closePromise !== null) { + return await this.#closePromise; + } + this.#closed = true; + this.#closePromise = (async () => { + const failures = []; + const active = [...this.#activeOperations]; + await Promise.allSettled(active); + + const streams = [...this.#activeStreams]; + const streamResults = await Promise.allSettled( + streams.map((stream) => GitPersistenceAdapter.#closeStream(stream)) + ); + this.#activeStreams.clear(); + failures.push( + ...streamResults + .filter((result) => result.status === 'rejected') + .map((result) => result.reason) + ); + + try { + await this.#sessions.close(); + } catch (error) { + failures.push(error); + } finally { + this.#metadataCache = null; + this.#treeCache = null; + } + if (failures.length > 0) { + throw new AggregateError(failures, 'Git persistence adapter failed to close cleanly'); + } + })(); + return await this.#closePromise; + } + + async [Symbol.asyncDispose]() { + await this.close(); + } + + #assertOpen() { + if (this.#closed) { + throw createCasError('Git persistence adapter is closed', ErrorCodes.RESOURCE_CLOSED); + } + } + + #runOperation(operation) { + this.#assertOpen(); + const promise = Promise.resolve().then(operation); + this.#activeOperations.add(promise); + void promise.then( + () => this.#activeOperations.delete(promise), + () => this.#activeOperations.delete(promise) + ); + return promise; + } + + static #isObjectBufferLimit(error) { + return error?.details?.code === 'OBJECT_BUFFER_LIMIT_EXCEEDED'; + } + + static #isRecoverableCatError(error) { + return ( + error instanceof GitObjectMissingError || GitPersistenceAdapter.#isObjectBufferLimit(error) + ); + } + /** * @param {number} metadataCacheEntries */ @@ -264,7 +574,40 @@ export default class GitPersistenceAdapter extends GitPersistencePort { throw createCasError( 'Git metadata cache entries must be a positive safe integer', ErrorCodes.INVALID_OPTIONS, - { option: 'metadataCacheEntries', metadataCacheEntries }, + { option: 'metadataCacheEntries', metadataCacheEntries } + ); + } + + static #assertTreeCacheEntries(treeCacheEntries) { + if (Number.isSafeInteger(treeCacheEntries) && treeCacheEntries >= 1) { + return; + } + throw createCasError( + 'Git tree cache entries must be a positive safe integer', + ErrorCodes.INVALID_OPTIONS, + { option: 'treeCacheEntries', treeCacheEntries } + ); + } + + static #assertSessionIdleTimeout(sessionIdleTimeoutMs) { + if (Number.isSafeInteger(sessionIdleTimeoutMs) && sessionIdleTimeoutMs >= 1) { + return; + } + throw createCasError( + 'Git session idle timeout must be a positive safe integer', + ErrorCodes.INVALID_OPTIONS, + { option: 'sessionIdleTimeoutMs', sessionIdleTimeoutMs } + ); + } + + static #assertTreeCacheBytes(treeCacheBytes) { + if (Number.isSafeInteger(treeCacheBytes) && treeCacheBytes >= 0) { + return; + } + throw createCasError( + 'Git tree cache bytes must be a non-negative safe integer', + ErrorCodes.INVALID_OPTIONS, + { option: 'treeCacheBytes', treeCacheBytes } ); } @@ -297,8 +640,33 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {AsyncIterable} */ async *#bufferStream(stream) { - for await (const chunk of stream) { - yield GitPersistenceAdapter.#toBuffer(chunk); + try { + for await (const chunk of stream) { + yield GitPersistenceAdapter.#toBuffer(chunk); + } + } finally { + try { + await GitPersistenceAdapter.#closeStream(stream); + } finally { + this.#activeStreams.delete(stream); + } + } + } + + async #openBufferStream(args) { + const stream = await this.policy.execute( + async () => await this.plumbing.executeStream({ args }) + ); + this.#activeStreams.add(stream); + return this.#bufferStream(stream); + } + + static async #closeStream(stream) { + if (typeof stream.destroy === 'function') { + await stream.destroy(); + } + if (stream.finished !== undefined) { + await stream.finished; } } @@ -321,7 +689,11 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {number} */ static #validatedReadBlobLimit(maxBytes) { - if (!Number.isSafeInteger(maxBytes) || maxBytes < MIN_READ_BLOB_LIMIT || maxBytes > MAX_BLOB_SIZE_LIMIT) { + if ( + !Number.isSafeInteger(maxBytes) || + maxBytes < MIN_READ_BLOB_LIMIT || + maxBytes > MAX_BLOB_SIZE_LIMIT + ) { throw createCasError( `maxBytes must be an integer in [${MIN_READ_BLOB_LIMIT}, ${MAX_BLOB_SIZE_LIMIT}]`, ErrorCodes.INVALID_OPTIONS, @@ -330,7 +702,7 @@ export default class GitPersistenceAdapter extends GitPersistencePort { value: maxBytes, min: MIN_READ_BLOB_LIMIT, max: MAX_BLOB_SIZE_LIMIT, - }, + } ); } return maxBytes; @@ -341,7 +713,11 @@ export default class GitPersistenceAdapter extends GitPersistencePort { * @returns {void} */ static #assertMaxBlobSize(maxBlobSize) { - if (!Number.isInteger(maxBlobSize) || maxBlobSize < MIN_MAX_BLOB_SIZE || maxBlobSize > MAX_BLOB_SIZE_LIMIT) { + if ( + !Number.isInteger(maxBlobSize) || + maxBlobSize < MIN_MAX_BLOB_SIZE || + maxBlobSize > MAX_BLOB_SIZE_LIMIT + ) { throw createCasError( `maxBlobSize must be an integer in [${MIN_MAX_BLOB_SIZE}, ${MAX_BLOB_SIZE_LIMIT}]`, ErrorCodes.INVALID_OPTIONS, @@ -350,7 +726,7 @@ export default class GitPersistenceAdapter extends GitPersistencePort { value: maxBlobSize, min: MIN_MAX_BLOB_SIZE, max: MAX_BLOB_SIZE_LIMIT, - }, + } ); } } @@ -376,19 +752,15 @@ export default class GitPersistenceAdapter extends GitPersistencePort { static #parseTreeEntry(entry) { const tabIndex = entry.indexOf('\t'); if (tabIndex === -1) { - throw new CasError( - `Malformed ls-tree entry: ${entry}`, - ErrorCodes.TREE_PARSE_ERROR, - { rawEntry: entry }, - ); + throw new CasError(`Malformed ls-tree entry: ${entry}`, ErrorCodes.TREE_PARSE_ERROR, { + rawEntry: entry, + }); } const meta = entry.slice(0, tabIndex).split(' '); if (meta.length !== 3) { - throw new CasError( - `Malformed ls-tree entry: ${entry}`, - ErrorCodes.TREE_PARSE_ERROR, - { rawEntry: entry }, - ); + throw new CasError(`Malformed ls-tree entry: ${entry}`, ErrorCodes.TREE_PARSE_ERROR, { + rawEntry: entry, + }); } return { mode: meta[0], diff --git a/src/infrastructure/codecs/GitTreeObjectCodec.js b/src/infrastructure/codecs/GitTreeObjectCodec.js new file mode 100644 index 0000000..5894711 --- /dev/null +++ b/src/infrastructure/codecs/GitTreeObjectCodec.js @@ -0,0 +1,124 @@ +import { utf8ByteLength, utf8Decode } from '../../domain/encoding/utf8.js'; + +const TREE_ENTRY_OVERHEAD_BYTES = 128; +const TYPE_BY_MODE = new Map([ + ['040000', 'tree'], + ['100644', 'blob'], + ['100755', 'blob'], + ['120000', 'blob'], + ['160000', 'commit'], +]); + +/** + * Decodes Git's canonical tree-object bytes at the persistence boundary. + */ +export default class GitTreeObjectCodec { + /** + * @param {Uint8Array} content + * @param {string} treeOid + * @returns {{ entries: ReadonlyArray>, weight: number }} + */ + static decode(content, treeOid) { + const bytes = GitTreeObjectCodec.#bytes(content); + const oidBytes = GitTreeObjectCodec.#oidBytes(treeOid); + const entries = []; + let cursor = 0; + + while (cursor < bytes.length) { + const modeEnd = GitTreeObjectCodec.#find(bytes, 0x20, cursor); + const nameEnd = GitTreeObjectCodec.#find(bytes, 0x00, modeEnd + 1); + const oidEnd = nameEnd + 1 + oidBytes; + if (oidEnd > bytes.length) { + throw new TypeError('Git tree entry has a truncated object identifier'); + } + + const rawMode = utf8Decode(bytes.subarray(cursor, modeEnd)); + const mode = rawMode.padStart(6, '0'); + const type = TYPE_BY_MODE.get(mode); + const name = utf8Decode(bytes.subarray(modeEnd + 1, nameEnd)); + if (type === undefined || name.length === 0 || name.includes('/')) { + throw new TypeError('Git tree entry has an invalid mode or name'); + } + + entries.push( + Object.freeze({ + mode, + type, + oid: GitTreeObjectCodec.#hex(bytes.subarray(nameEnd + 1, oidEnd)), + name, + }) + ); + cursor = oidEnd; + } + + const weight = + bytes.length + + entries.reduce( + (total, entry) => total + TREE_ENTRY_OVERHEAD_BYTES + utf8ByteLength(entry.name), + 0 + ); + return Object.freeze({ entries: Object.freeze(entries), weight }); + } + + /** + * Converts the existing `git mktree` line contract into typed session input. + * @param {string[]} lines + * @returns {Array<{mode: string, type: string, oid: string, name: string}>} + */ + static parseMktreeLines(lines) { + if (!Array.isArray(lines)) { + throw new TypeError('Git tree entries must be an array'); + } + return lines.map((line) => { + if (typeof line !== 'string') { + throw new TypeError('Git tree entry must be a string'); + } + const tab = line.indexOf('\t'); + const fields = tab === -1 ? [] : line.slice(0, tab).split(' '); + const name = tab === -1 ? '' : line.slice(tab + 1); + const [mode, type, oid] = fields; + if ( + fields.length !== 3 || + TYPE_BY_MODE.get(mode) !== type || + !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(oid) || + name.length === 0 || + name.includes('\0') || + name.includes('/') + ) { + throw new TypeError('Git tree entry line is invalid'); + } + return { mode, type, oid, name }; + }); + } + + static #bytes(content) { + if (!(content instanceof Uint8Array)) { + throw new TypeError('Git tree content must be a Uint8Array'); + } + return content; + } + + static #oidBytes(treeOid) { + if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(treeOid)) { + throw new TypeError('Git tree object identifier is invalid'); + } + return treeOid.length / 2; + } + + static #find(bytes, needle, start) { + for (let index = start; index < bytes.length; index += 1) { + if (bytes[index] === needle) { + return index; + } + } + throw new TypeError('Git tree entry is missing a delimiter'); + } + + static #hex(bytes) { + let result = ''; + for (const byte of bytes) { + result += byte.toString(16).padStart(2, '0'); + } + return result; + } +} diff --git a/src/ports/GitPersistencePort.js b/src/ports/GitPersistencePort.js index 01b9afc..0921876 100644 --- a/src/ports/GitPersistencePort.js +++ b/src/ports/GitPersistencePort.js @@ -12,6 +12,20 @@ export default class GitPersistencePort { throw new Error('Not implemented'); } + /** + * Writes a bounded group of blobs. Adapters may override this to amortize + * process startup while preserving visibility before the method resolves. + * @param {Iterable} contents + * @returns {Promise} + */ + async writeBlobs(contents) { + const oids = []; + for (const content of contents) { + oids.push(await this.writeBlob(content)); + } + return oids; + } + /** * Creates a Git tree object from formatted entries. * @param {string[]} _entries - Lines in `git mktree` format. @@ -87,4 +101,16 @@ export default class GitPersistencePort { async readObjectSize(_oid) { throw new Error('Not implemented'); } + + /** + * Releases adapter-owned local resources. The default implementation is a + * no-op so persistence adapters without long-lived resources remain valid. + * @returns {Promise} + */ + async close() {} + + /** @returns {Promise} */ + async [Symbol.asyncDispose]() { + await this.close(); + } } diff --git a/src/types/ambient.d.ts b/src/types/ambient.d.ts index d58201c..3bf7376 100644 --- a/src/types/ambient.d.ts +++ b/src/types/ambient.d.ts @@ -9,12 +9,54 @@ declare module '@git-stunts/plumbing' { env?: Record; } - interface StreamResult { + interface StreamResult extends AsyncIterable { collect(options?: { asString?: boolean; maxBytes?: number }): Promise; + destroy(): Promise; + finished: Promise<{ code: number; stderr: string }>; } type ShellRunner = (options: Record) => Promise; + interface GitObjectInfo { + oid: string; + type: string; + size: number; + } + + interface GitObject extends GitObjectInfo { + content: Uint8Array; + } + + interface GitCatFileSession { + info(objectName: string): Promise; + read(objectName: string, options?: { maxBytes?: number }): Promise; + close(): Promise; + terminate(): Promise; + } + + interface GitMktreeSession { + write( + entries: Array<{ mode: string; type: string; oid: string; name: string }> + ): Promise; + close(): Promise; + terminate(): Promise; + } + + interface GitFastImportSession { + writeBlob(content: string | Uint8Array): Promise; + checkpoint(): Promise; + close(): Promise; + abort(): Promise; + } + + export class GitObjectMissingError extends Error { + details?: Record; + } + + export class GitProtocolError extends Error {} + + export class InvalidArgumentError extends Error {} + export class ShellRunnerFactory { static ENV_BUN: 'bun'; static ENV_DENO: 'deno'; @@ -29,6 +71,9 @@ declare module '@git-stunts/plumbing' { static createRepository(options?: { cwd?: string; env?: string }): Promise; execute(options: ExecuteOptions): Promise; executeStream(options: ExecuteOptions): Promise; + openCatFileSession(): Promise; + openMktreeSession(): Promise; + openFastImportSession(): Promise; } } diff --git a/test/integration/bundle-reference-performance.test.js b/test/integration/bundle-reference-performance.test.js index b87075d..3e1cc39 100644 --- a/test/integration/bundle-reference-performance.test.js +++ b/test/integration/bundle-reference-performance.test.js @@ -44,7 +44,7 @@ function operationOf(args) { return args[0]; } -async function countingReader() { +async function countingReader({ sessions = false } = {}) { const plumbing = await createGitPlumbing({ cwd: repoDir }); const counts = new Map(); const record = (options) => { @@ -61,6 +61,20 @@ async function countingReader() { return plumbing.executeStream(options); }, }; + if (sessions) { + counted.openCatFileSession = (...args) => { + counts.set('session:cat-file', count(counts, 'session:cat-file') + 1); + return plumbing.openCatFileSession(...args); + }; + counted.openMktreeSession = (...args) => { + counts.set('session:mktree', count(counts, 'session:mktree') + 1); + return plumbing.openMktreeSession(...args); + }; + counted.openFastImportSession = (...args) => { + counts.set('session:fast-import', count(counts, 'session:fast-import') + 1); + return plumbing.openFastImportSession(...args); + }; + } return { cas: new ContentAddressableStore({ plumbing: counted }), snapshot() { @@ -104,7 +118,8 @@ beforeAll(async () => { outer = await writer.bundles.put({ members: { nested: nested.handle } }); }); -afterAll(() => { +afterAll(async () => { + await writer.close(); rmSync(repoDir, { recursive: true, force: true }); }); @@ -114,7 +129,7 @@ describe('real-Git immutable page payload reads', () => { const page = await writer.pages.put({ source: Buffer.from('warm page payload') }); await expect(reader.cas.pages.get({ handle: page.handle })).resolves.toEqual( - new Uint8Array(Buffer.from('warm page payload')), + new Uint8Array(Buffer.from('warm page payload')) ); const cold = reader.snapshot(); await reader.cas.pages.get({ handle: page.handle }); @@ -122,6 +137,7 @@ describe('real-Git immutable page payload reads', () => { expect(count(cold, 'cat-file')).toBeGreaterThan(0); expect(total(warm)).toBe(0); + await reader.cas.close(); }); }); @@ -144,6 +160,7 @@ describe('real-Git direct bundle reference reads', () => { expect(count(warm, 'ls-tree')).toBe(0); expect(count(warm, 'cat-file:batch-check')).toBe(0); expect(total(warm)).toBe(0); + await reader.cas.close(); }); it('does less Git work than complete recursive member validation', async () => { @@ -165,5 +182,89 @@ describe('real-Git direct bundle reference reads', () => { expect(count(directCounts, 'cat-file:batch-check')).toBeLessThan( count(completeCounts, 'cat-file:batch-check') ); + await direct.cas.close(); + await complete.cas.close(); + }); +}); + +describe('real-Git persistent object session process count', () => { + it('uses fewer Git child processes through persistent object sessions', async () => { + const fallback = await countingReader(); + const persistent = await countingReader({ sessions: true }); + + const fallbackResult = await fallback.cas.bundles.getMemberReference({ + handle: outer.handle, + path: 'nested', + }); + const persistentResult = await persistent.cas.bundles.getMemberReference({ + handle: outer.handle, + path: 'nested', + }); + const fallbackCounts = fallback.snapshot(); + const persistentCounts = persistent.snapshot(); + + expect(persistentResult).toEqual(fallbackResult); + expect(count(persistentCounts, 'session:cat-file')).toBe(1); + expect(count(persistentCounts, 'ls-tree')).toBe(0); + expect(total(persistentCounts)).toBeLessThan(total(fallbackCounts)); + + await fallback.cas.close(); + await persistent.cas.close(); + }); +}); + +describe('real-Git scoped bulk write process count', () => { + it('stores a bounded page batch through one scoped fast-import process', async () => { + const individual = await countingReader({ sessions: true }); + const batched = await countingReader({ sessions: true }); + const inputs = Array.from({ length: 8 }, (_, index) => Buffer.from(`page-${index}`)); + const individualPages = []; + for (const source of inputs) { + individualPages.push(await individual.cas.pages.put({ source })); + } + const batchPages = await batched.cas.pages.putBatch({ + pages: inputs.map((source) => ({ source })), + }); + const individualCounts = individual.snapshot(); + const batchCounts = batched.snapshot(); + + expect(batchPages.map((page) => page.handle.toString())).toEqual( + individualPages.map((page) => page.handle.toString()) + ); + expect(count(batchCounts, 'session:fast-import')).toBe(1); + expect(total(batchCounts)).toBeLessThan(total(individualCounts)); + + await individual.cas.close(); + await batched.cas.close(); + }); +}); + +describe('real-Git individual blob rewrite after pruning', () => { + it('recreates an individually written blob after external pruning', async () => { + const isolatedRepo = mkdtempSync(path.join(os.tmpdir(), 'cas-pruned-blob-rewrite-')); + const isolatedGit = (args) => spawnSync('git', args, { cwd: isolatedRepo, encoding: 'utf8' }); + isolatedGit(['init', '--bare']); + const cas = new ContentAddressableStore({ + plumbing: await createGitPlumbing({ cwd: isolatedRepo }), + }); + + try { + const source = Buffer.from('rewrite me after prune'); + const first = await cas.pages.put({ source }); + expect(isolatedGit(['cat-file', '-e', first.handle.oid]).status).toBe(0); + + expect(isolatedGit(['prune', '--expire=now']).status).toBe(0); + expect(isolatedGit(['cat-file', '-e', first.handle.oid]).status).not.toBe(0); + + const second = await cas.pages.put({ source }); + expect(second.handle.oid).toBe(first.handle.oid); + expect(isolatedGit(['cat-file', '-e', second.handle.oid]).status).toBe(0); + await expect(cas.pages.get({ handle: second.handle })).resolves.toEqual( + new Uint8Array(source) + ); + } finally { + await cas.close(); + rmSync(isolatedRepo, { recursive: true, force: true }); + } }); }); diff --git a/test/unit/domain/services/PageService.test.js b/test/unit/domain/services/PageService.test.js index 26e893f..af8ceb9 100644 --- a/test/unit/domain/services/PageService.test.js +++ b/test/unit/domain/services/PageService.test.js @@ -80,6 +80,57 @@ describe('PageService', () => { }); }); +describe('PageService.putBatch()', () => { + it('uses one bounded persistence batch and preserves input order', async () => { + const { pages, persistence } = makePages(); + const writeBlobs = vi.spyOn(persistence, 'writeBlobs'); + + const staged = await pages.putBatch({ + pages: [ + { source: Buffer.from('first') }, + { source: (async function* source() { yield Buffer.from('second'); })() }, + ], + maxBatchBytes: 32, + }); + + expect(writeBlobs).toHaveBeenCalledTimes(1); + expect(staged).toHaveLength(2); + expect(staged.map((page) => page.page.size)).toEqual([5, 6]); + await expect(pages.get({ handle: staged[0].handle })).resolves.toEqual( + new Uint8Array(Buffer.from('first')), + ); + await expect(pages.get({ handle: staged[1].handle })).resolves.toEqual( + new Uint8Array(Buffer.from('second')), + ); + }); + + it('rejects count and byte overflow before starting persistence', async () => { + const { pages, persistence } = makePages(); + const writeBlobs = vi.spyOn(persistence, 'writeBlobs'); + + await expect(pages.putBatch({ + pages: [{ source: Buffer.from('a') }, { source: Buffer.from('b') }], + maxBatchPages: 1, + })).rejects.toMatchObject({ code: 'PAGE_BATCH_LIMIT' }); + await expect(pages.putBatch({ + pages: [{ source: Buffer.from('ab') }, { source: Buffer.from('cd') }], + maxBatchBytes: 3, + })).rejects.toMatchObject({ code: 'PAGE_BATCH_LIMIT' }); + expect(writeBlobs).not.toHaveBeenCalled(); + }); + + it('returns an empty frozen result without opening persistence', async () => { + const { pages, persistence } = makePages(); + const writeBlobs = vi.spyOn(persistence, 'writeBlobs'); + + const staged = await pages.putBatch({ pages: [] }); + + expect(staged).toEqual([]); + expect(Object.isFrozen(staged)).toBe(true); + expect(writeBlobs).not.toHaveBeenCalled(); + }); +}); + describe('PageService immutable payload reuse', () => { it('coalesces immutable payload reads and returns caller-owned byte copies', async () => { const { pages, persistence } = makePages(); diff --git a/test/unit/facade/ContentAddressableStore.application-storage.test.js b/test/unit/facade/ContentAddressableStore.application-storage.test.js index 5d2c5d8..0b1973a 100644 --- a/test/unit/facade/ContentAddressableStore.application-storage.test.js +++ b/test/unit/facade/ContentAddressableStore.application-storage.test.js @@ -30,7 +30,7 @@ describe('ContentAddressableStore application storage capabilities', () => { }); expect(Object.keys(cas.assets)).toEqual(['put', 'adopt', 'open']); - expect(Object.keys(cas.pages)).toEqual(['put', 'get', 'open']); + expect(Object.keys(cas.pages)).toEqual(['put', 'putBatch', 'get', 'open']); expect(Object.keys(cas.bundles)).toEqual([ 'put', 'putOrdered', diff --git a/test/unit/facade/ContentAddressableStore.lifecycle.test.js b/test/unit/facade/ContentAddressableStore.lifecycle.test.js new file mode 100644 index 0000000..5c24a6a --- /dev/null +++ b/test/unit/facade/ContentAddressableStore.lifecycle.test.js @@ -0,0 +1,33 @@ +import { describe, expect, it, vi } from 'vitest'; +import ContentAddressableStore from '../../../index.js'; + +function plumbing() { + return { + execute: vi.fn(), + executeStream: vi.fn(), + }; +} + +describe('ContentAddressableStore lifecycle', () => { + it('closes initialized persistence exactly once', async () => { + const cas = new ContentAddressableStore({ plumbing: plumbing() }); + const service = await cas.getService(); + const close = vi.spyOn(service.persistence, 'close'); + + await Promise.all([cas.close(), cas.close(), cas[Symbol.asyncDispose]()]); + + expect(close).toHaveBeenCalledTimes(1); + await expect(cas.getService()).rejects.toMatchObject({ code: 'RESOURCE_CLOSED' }); + }); + + it('does not initialize persistence merely to close an unused facade', async () => { + const cas = new ContentAddressableStore({ plumbing: plumbing() }); + + await cas.close(); + + expect(cas.service).toBeNull(); + await expect(cas.pages.get({ handle: 'unused' })).rejects.toMatchObject({ + code: 'RESOURCE_CLOSED', + }); + }); +}); diff --git a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js new file mode 100644 index 0000000..5cf40a6 --- /dev/null +++ b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js @@ -0,0 +1,438 @@ +import { describe, expect, it, vi } from 'vitest'; +import { GitProtocolError } from '@git-stunts/plumbing'; +import GitPersistenceAdapter from '../../../../src/infrastructure/adapters/GitPersistenceAdapter.js'; + +const noPolicy = { execute: (operation) => operation() }; + +function deferred() { + let reject; + let resolve; + const promise = new Promise((promiseResolve, promiseReject) => { + resolve = promiseResolve; + reject = promiseReject; + }); + return { promise, reject, resolve }; +} + +function rawTree(entries, oidBytes = 20) { + const records = []; + for (const { mode, name, oid } of entries) { + records.push(Buffer.from(`${mode} ${name}\0`)); + records.push(Buffer.from(oid.padEnd(oidBytes * 2, '0').slice(0, oidBytes * 2), 'hex')); + } + return Buffer.concat(records); +} + +function catObject({ oid, type = 'blob', content = Buffer.alloc(0) }) { + return Object.freeze({ + oid, + type, + size: content.length, + content: Uint8Array.from(content), + }); +} + +function sessionPlumbing({ catSessions = [], mktreeSession, fastImportSession } = {}) { + const plumbing = { + execute: vi.fn(), + executeStream: vi.fn(), + }; + if (catSessions.length > 0) { + plumbing.openCatFileSession = vi.fn(); + for (const session of catSessions) { + plumbing.openCatFileSession.mockResolvedValueOnce(session); + } + } + if (mktreeSession !== undefined) { + plumbing.openMktreeSession = vi.fn().mockResolvedValue(mktreeSession); + } + if (fastImportSession !== undefined) { + plumbing.openFastImportSession = vi.fn().mockResolvedValue(fastImportSession); + } + return plumbing; +} + +function fakeCatSession(overrides = {}) { + return { + info: vi.fn(), + read: vi.fn(), + close: vi.fn().mockResolvedValue(undefined), + terminate: vi.fn().mockResolvedValue(undefined), + ...overrides, + }; +} + +describe('GitPersistenceAdapter persistent cat-file reads', () => { + it('coalesces concurrent session opening and serializes bounded object reads', async () => { + const firstOid = 'a'.repeat(40); + const secondOid = 'b'.repeat(40); + const cat = fakeCatSession({ + read: vi.fn(async (oid) => catObject({ oid, content: Buffer.from(oid[0]) })), + }); + const plumbing = sessionPlumbing({ catSessions: [cat] }); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + + await expect( + Promise.all([adapter.readBlob(firstOid), adapter.readBlob(secondOid)]) + ).resolves.toEqual([Buffer.from('a'), Buffer.from('b')]); + + expect(plumbing.openCatFileSession).toHaveBeenCalledTimes(1); + expect(cat.read).toHaveBeenCalledTimes(2); + expect(plumbing.executeStream).not.toHaveBeenCalled(); + }); +}); + +describe('GitPersistenceAdapter cat-file recovery', () => { + it('invalidates a failed session and opens a fresh session on the next call', async () => { + const oid = 'c'.repeat(40); + const failed = fakeCatSession({ + info: vi.fn().mockRejectedValue(new Error('protocol failed')), + }); + const replacement = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid, type: 'tree', size: 12 }), + }); + const plumbing = sessionPlumbing({ catSessions: [failed, replacement] }); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + + await expect(adapter.readObjectType(oid)).rejects.toThrow('protocol failed'); + await expect(adapter.readObjectType(oid)).resolves.toBe('tree'); + + expect(failed.terminate).toHaveBeenCalledTimes(1); + expect(plumbing.openCatFileSession).toHaveBeenCalledTimes(2); + }); + + it('retries one idempotent read after a typed protocol process failure', async () => { + const oid = 'f'.repeat(40); + const failed = fakeCatSession({ + info: vi.fn().mockRejectedValue(new GitProtocolError('process closed', 'test')), + }); + const replacement = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid, type: 'blob', size: 7 }), + }); + const plumbing = sessionPlumbing({ catSessions: [failed, replacement] }); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + + await expect(adapter.readObjectSize(oid)).resolves.toBe(7); + expect(failed.terminate).toHaveBeenCalledTimes(1); + expect(plumbing.openCatFileSession).toHaveBeenCalledTimes(2); + }); +}); + +describe('GitPersistenceAdapter payload streaming', () => { + it('keeps payload streaming on executeStream instead of buffering through cat-file', async () => { + const oid = 'd'.repeat(40); + const cat = fakeCatSession(); + const plumbing = sessionPlumbing({ catSessions: [cat] }); + plumbing.executeStream.mockResolvedValue( + (async function* stream() { + yield Buffer.from('streamed'); + })() + ); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + const chunks = []; + + for await (const chunk of await adapter.readBlobStream(oid)) { + chunks.push(chunk); + } + + expect(Buffer.concat(chunks).toString()).toBe('streamed'); + expect(plumbing.executeStream).toHaveBeenCalledTimes(1); + expect(plumbing.openCatFileSession).not.toHaveBeenCalled(); + }); +}); + +describe('GitPersistenceAdapter immutable tree reuse', () => { + it('reads one tree object for distinct exact lookups and isolates returned entries', async () => { + const treeOid = '1'.repeat(40); + const firstOid = '2'.repeat(40); + const secondOid = '3'.repeat(40); + const content = rawTree([ + { mode: '100644', name: 'first', oid: firstOid }, + { mode: '100644', name: 'second', oid: secondOid }, + ]); + const cat = fakeCatSession({ + read: vi.fn().mockResolvedValue(catObject({ oid: treeOid, type: 'tree', content })), + }); + const adapter = new GitPersistenceAdapter({ + plumbing: sessionPlumbing({ catSessions: [cat] }), + policy: noPolicy, + treeCacheEntries: 2, + treeCacheBytes: 4096, + }); + + const first = await adapter.readTreeEntry(treeOid, 'first'); + first.name = 'mutated'; + await expect(adapter.readTreeEntry(treeOid, 'second')).resolves.toEqual({ + mode: '100644', + type: 'blob', + oid: secondOid, + name: 'second', + }); + await expect(adapter.readTreeEntry(treeOid, 'first')).resolves.toMatchObject({ name: 'first' }); + + expect(cat.read).toHaveBeenCalledTimes(1); + }); +}); + +describe('GitPersistenceAdapter exact tree paths', () => { + it('traverses an exact nested path through bounded tree objects', async () => { + const rootOid = '4'.repeat(40); + const childOid = '5'.repeat(40); + const blobOid = '6'.repeat(40); + const objects = new Map([ + [rootOid, rawTree([{ mode: '40000', name: 'nested', oid: childOid }])], + [childOid, rawTree([{ mode: '100644', name: 'value', oid: blobOid }])], + ]); + const cat = fakeCatSession({ + read: vi.fn(async (oid) => catObject({ oid, type: 'tree', content: objects.get(oid) })), + }); + const adapter = new GitPersistenceAdapter({ + plumbing: sessionPlumbing({ catSessions: [cat] }), + policy: noPolicy, + }); + + await expect(adapter.readTreeEntry(rootOid, 'nested/value')).resolves.toEqual({ + mode: '100644', + type: 'blob', + oid: blobOid, + name: 'nested/value', + }); + expect(cat.read).toHaveBeenCalledTimes(2); + }); +}); + +describe('GitPersistenceAdapter tree cache residency', () => { + it('evicts tree objects by the configured byte bound', async () => { + const firstTree = '7'.repeat(40); + const secondTree = '8'.repeat(40); + const payloads = new Map([ + [firstTree, rawTree([{ mode: '100644', name: 'first', oid: '9'.repeat(40) }])], + [secondTree, rawTree([{ mode: '100644', name: 'second', oid: 'a'.repeat(40) }])], + ]); + const cat = fakeCatSession({ + read: vi.fn(async (oid) => catObject({ oid, type: 'tree', content: payloads.get(oid) })), + }); + const adapter = new GitPersistenceAdapter({ + plumbing: sessionPlumbing({ catSessions: [cat] }), + policy: noPolicy, + metadataCacheEntries: 1, + treeCacheEntries: 2, + treeCacheBytes: 200, + }); + + await adapter.readTreeEntry(firstTree, 'first'); + await adapter.readTreeEntry(secondTree, 'second'); + await adapter.readTreeEntry(firstTree, 'first'); + + expect(cat.read).toHaveBeenCalledTimes(3); + }); +}); + +describe('GitPersistenceAdapter persistent write sessions', () => { + it('checkpoints and closes one scoped bulk blob write before returning OIDs', async () => { + const fastImport = { + writeBlob: vi + .fn() + .mockResolvedValueOnce('a'.repeat(40)) + .mockResolvedValueOnce('b'.repeat(40)), + checkpoint: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockResolvedValue(undefined), + abort: vi.fn().mockResolvedValue(undefined), + }; + const plumbing = sessionPlumbing({ fastImportSession: fastImport }); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + + await expect( + adapter.writeBlobs([Buffer.from('first'), Buffer.from('second')]) + ).resolves.toEqual(['a'.repeat(40), 'b'.repeat(40)]); + + expect(plumbing.openFastImportSession).toHaveBeenCalledTimes(1); + expect(fastImport.writeBlob).toHaveBeenCalledTimes(2); + expect(fastImport.checkpoint).toHaveBeenCalledTimes(1); + expect(fastImport.close).toHaveBeenCalledTimes(1); + expect(plumbing.execute).not.toHaveBeenCalled(); + }); +}); + +describe('GitPersistenceAdapter bulk write recovery', () => { + it('replays a materialized bulk input through a fresh typed session', async () => { + const failed = { + writeBlob: vi.fn().mockRejectedValue(new GitProtocolError('process closed', 'test')), + checkpoint: vi.fn(), + close: vi.fn(), + abort: vi.fn().mockResolvedValue(undefined), + }; + const replacement = { + writeBlob: vi + .fn() + .mockResolvedValueOnce('a'.repeat(40)) + .mockResolvedValueOnce('b'.repeat(40)), + checkpoint: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockResolvedValue(undefined), + abort: vi.fn().mockResolvedValue(undefined), + }; + const plumbing = sessionPlumbing(); + plumbing.openFastImportSession = vi + .fn() + .mockResolvedValueOnce(failed) + .mockResolvedValueOnce(replacement); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + const sources = [Buffer.from('first'), Buffer.from('second')]; + + await expect(adapter.writeBlobs(sources.values())).resolves.toEqual([ + 'a'.repeat(40), + 'b'.repeat(40), + ]); + + expect(failed.abort).toHaveBeenCalledTimes(1); + expect(replacement.writeBlob).toHaveBeenCalledTimes(2); + expect(replacement.close).toHaveBeenCalledTimes(1); + }); +}); + +describe('GitPersistenceAdapter persistent tree writes', () => { + it('converts existing mktree lines into structured session entries', async () => { + const mktree = { + write: vi.fn().mockResolvedValue('c'.repeat(40)), + close: vi.fn().mockResolvedValue(undefined), + terminate: vi.fn().mockResolvedValue(undefined), + }; + const plumbing = sessionPlumbing({ mktreeSession: mktree }); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + + await expect(adapter.writeTree([`${'100644'} blob ${'d'.repeat(40)}\tpage`])).resolves.toBe( + 'c'.repeat(40) + ); + + expect(mktree.write).toHaveBeenCalledWith([ + { mode: '100644', type: 'blob', oid: 'd'.repeat(40), name: 'page' }, + ]); + expect(plumbing.execute).not.toHaveBeenCalled(); + }); +}); + +describe('GitPersistenceAdapter lifecycle', () => { + it('closes every opened session once and rejects later operations', async () => { + const oid = 'e'.repeat(40); + const cat = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid, type: 'blob', size: 1 }), + }); + const mktree = { + write: vi.fn().mockResolvedValue('f'.repeat(40)), + close: vi.fn().mockResolvedValue(undefined), + terminate: vi.fn().mockResolvedValue(undefined), + }; + const fastImport = { + writeBlob: vi.fn().mockResolvedValue('a'.repeat(40)), + checkpoint: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockResolvedValue(undefined), + abort: vi.fn().mockResolvedValue(undefined), + }; + const adapter = new GitPersistenceAdapter({ + plumbing: sessionPlumbing({ + catSessions: [cat], + mktreeSession: mktree, + fastImportSession: fastImport, + }), + policy: noPolicy, + }); + + await adapter.readObjectType(oid); + await adapter.writeTree([]); + await adapter.writeBlobs([Buffer.from('value')]); + + expect(cat.close).toHaveBeenCalledTimes(1); + expect(mktree.close).toHaveBeenCalledTimes(1); + await Promise.all([adapter.close(), adapter.close(), adapter[Symbol.asyncDispose]()]); + + expect(cat.close).toHaveBeenCalledTimes(1); + expect(mktree.close).toHaveBeenCalledTimes(1); + expect(fastImport.close).toHaveBeenCalledTimes(1); + await expect(adapter.readObjectType(oid)).rejects.toMatchObject({ + code: 'RESOURCE_CLOSED', + }); + }); +}); + +describe('GitPersistenceAdapter command shutdown', () => { + it('waits for an active one-shot command before closing protocol sessions', async () => { + const oid = '1'.repeat(40); + const write = deferred(); + const cat = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid, type: 'blob', size: 1 }), + }); + const plumbing = sessionPlumbing({ catSessions: [cat] }); + plumbing.execute.mockReturnValue(write.promise); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + + await adapter.readObjectType(oid); + const pendingWrite = adapter.writeBlob(Buffer.from('value')); + await vi.waitFor(() => expect(plumbing.execute).toHaveBeenCalledTimes(1)); + + let closeSettled = false; + const close = adapter.close().then(() => { + closeSettled = true; + }); + await Promise.resolve(); + + expect(closeSettled).toBe(false); + expect(cat.close).not.toHaveBeenCalled(); + + write.resolve('2'.repeat(40)); + await expect(pendingWrite).resolves.toBe('2'.repeat(40)); + await close; + + expect(cat.close).toHaveBeenCalledTimes(1); + }); +}); + +describe('GitPersistenceAdapter stream shutdown', () => { + it('destroys an abandoned output stream and waits for its Git process', async () => { + const finished = deferred(); + const stream = { + async *[Symbol.asyncIterator]() {}, + destroy: vi.fn().mockResolvedValue(undefined), + finished: finished.promise, + }; + const plumbing = sessionPlumbing(); + plumbing.executeStream.mockResolvedValue(stream); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + + await adapter.readBlobStream('3'.repeat(40)); + let closeSettled = false; + const close = adapter.close().then(() => { + closeSettled = true; + }); + await vi.waitFor(() => expect(stream.destroy).toHaveBeenCalledTimes(1)); + + expect(closeSettled).toBe(false); + finished.resolve({ code: 0, stderr: '' }); + await close; + + expect(closeSettled).toBe(true); + }); +}); + +describe('GitPersistenceAdapter idle lifecycle fallback', () => { + it('retires an idle session when a caller omits explicit close', async () => { + vi.useFakeTimers(); + try { + const oid = '1'.repeat(40); + const cat = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid, type: 'blob', size: 1 }), + }); + const adapter = new GitPersistenceAdapter({ + plumbing: sessionPlumbing({ catSessions: [cat] }), + policy: noPolicy, + sessionIdleTimeoutMs: 10, + }); + + await adapter.readObjectType(oid); + expect(cat.close).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(10); + expect(cat.close).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/test/unit/infrastructure/codecs/GitTreeObjectCodec.test.js b/test/unit/infrastructure/codecs/GitTreeObjectCodec.test.js new file mode 100644 index 0000000..f4d30c2 --- /dev/null +++ b/test/unit/infrastructure/codecs/GitTreeObjectCodec.test.js @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; +import GitTreeObjectCodec from '../../../../src/infrastructure/codecs/GitTreeObjectCodec.js'; + +function rawEntry({ mode, name, oid }) { + return Buffer.concat([Buffer.from(`${mode} ${name}\0`), Buffer.from(oid, 'hex')]); +} + +describe('GitTreeObjectCodec.decode()', () => { + it.each([ + ['SHA-1', 'a'.repeat(40)], + ['SHA-256', 'b'.repeat(64)], + ])('decodes %s object identifiers without changing entry meaning', (_label, oid) => { + const treeOid = 'c'.repeat(oid.length); + const decoded = GitTreeObjectCodec.decode( + rawEntry({ + mode: '100644', + name: 'page', + oid, + }), + treeOid + ); + + expect(decoded.entries).toEqual([{ mode: '100644', type: 'blob', oid, name: 'page' }]); + expect(decoded.weight).toBeGreaterThan(oid.length / 2); + expect(Object.isFrozen(decoded.entries[0])).toBe(true); + }); + + it('normalizes the raw directory mode and preserves commit entries', () => { + const treeOid = 'd'.repeat(40); + const content = Buffer.concat([ + rawEntry({ mode: '40000', name: 'nested', oid: 'e'.repeat(40) }), + rawEntry({ mode: '160000', name: 'submodule', oid: 'f'.repeat(40) }), + ]); + + expect(GitTreeObjectCodec.decode(content, treeOid).entries).toEqual([ + { mode: '040000', type: 'tree', oid: 'e'.repeat(40), name: 'nested' }, + { mode: '160000', type: 'commit', oid: 'f'.repeat(40), name: 'submodule' }, + ]); + }); + + it.each([ + ['missing delimiter', Buffer.from('100644 page')], + ['truncated OID', Buffer.concat([Buffer.from('100644 page\0'), Buffer.alloc(19)])], + ['invalid mode', rawEntry({ mode: '100600', name: 'page', oid: 'a'.repeat(40) })], + ])('rejects a malformed tree with a %s', (_label, content) => { + expect(() => GitTreeObjectCodec.decode(content, 'a'.repeat(40))).toThrow(TypeError); + }); +}); + +describe('GitTreeObjectCodec.parseMktreeLines()', () => { + it('converts the existing persistence line contract', () => { + expect( + GitTreeObjectCodec.parseMktreeLines([ + `100644 blob ${'a'.repeat(40)}\tpage`, + `040000 tree ${'b'.repeat(40)}\tnested`, + ]) + ).toEqual([ + { mode: '100644', type: 'blob', oid: 'a'.repeat(40), name: 'page' }, + { mode: '040000', type: 'tree', oid: 'b'.repeat(40), name: 'nested' }, + ]); + }); + + it.each([ + ['bad OID', '100644 blob nope\tpage'], + ['wrong type', `100644 tree ${'a'.repeat(40)}\tpage`], + ['nested name', `100644 blob ${'a'.repeat(40)}\tnested/page`], + ])('rejects %s before writing protocol bytes', (_label, line) => { + expect(() => GitTreeObjectCodec.parseMktreeLines([line])).toThrow(TypeError); + }); +}); diff --git a/test/unit/ports/GitPersistencePort.test.js b/test/unit/ports/GitPersistencePort.test.js index 58797b2..762e1f8 100644 --- a/test/unit/ports/GitPersistencePort.test.js +++ b/test/unit/ports/GitPersistencePort.test.js @@ -8,6 +8,10 @@ describe('GitPersistencePort – abstract methods', () => { await expect(port.writeBlob(Buffer.alloc(0))).rejects.toThrow('Not implemented'); }); + it('writeBlobs() preserves the writeBlob fallback contract', async () => { + await expect(port.writeBlobs([Buffer.alloc(0)])).rejects.toThrow('Not implemented'); + }); + it('writeTree() throws Not implemented', async () => { await expect(port.writeTree([])).rejects.toThrow('Not implemented'); }); diff --git a/test/unit/types/declaration-accuracy.test.js b/test/unit/types/declaration-accuracy.test.js index bab8980..162ef82 100644 --- a/test/unit/types/declaration-accuracy.test.js +++ b/test/unit/types/declaration-accuracy.test.js @@ -89,6 +89,7 @@ describe('Application-storage declaration accuracy', () => { expect(declarations).toContain('readonly assets: AssetCapability;'); expect(declarations).toContain('readonly bundles: BundleCapability;'); expect(declarations).toContain('readonly pages: PageCapability;'); + expect(declarations).toContain('putBatch(options: {'); expect(declarations).toContain('readonly retention: RetentionCapability;'); expect(declarations).toContain('readonly publications: PublicationCapability;'); expect(declarations).toContain('export declare class CacheSet'); @@ -124,6 +125,18 @@ describe('Application-storage declaration accuracy', () => { }); }); +describe('Persistence lifecycle declaration accuracy', () => { + it('declares bounded tree reuse and deterministic resource release', () => { + const declarations = read('index.d.ts'); + + expect(declarations).toContain('treeCacheEntries?: number;'); + expect(declarations).toContain('treeCacheBytes?: number;'); + expect(declarations).toContain('sessionIdleTimeoutMs?: number;'); + expect(declarations).toContain('close(): Promise;'); + expect(declarations).toContain('[Symbol.asyncDispose](): Promise;'); + }); +}); + describe('Bundle reference declaration accuracy', () => { it('declares direct bundle references and bounded Git metadata caching', () => { const declarations = read('index.d.ts'); From a2ce917fb629f63db643de7cf72035386f48345e Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 03:32:06 -0700 Subject: [PATCH 03/17] fix: preserve legacy persistence adapter types --- index.d.ts | 9 +++-- test/types/public-api-compatibility.ts | 40 ++++++++++++++++++++ test/unit/types/declaration-accuracy.test.js | 3 ++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/index.d.ts b/index.d.ts index e329040..19daf80 100644 --- a/index.d.ts +++ b/index.d.ts @@ -513,7 +513,7 @@ export declare class CryptoPortBase { /** Abstract port for persisting data to Git's object database. */ export declare class GitPersistencePortBase { writeBlob(content: Uint8Array): Promise; - writeBlobs(contents: Iterable): Promise; + writeBlobs?(contents: Iterable): Promise; writeTree(entries: string[]): Promise; readBlob(oid: string, maxBytes?: number): Promise; readBlobStream(oid: string): Promise>; @@ -530,8 +530,8 @@ export declare class GitPersistencePortBase { readObjectType(oid: string): Promise; readObjectSize(oid: string): Promise; setMaxBlobSize?(maxBlobSize: number): void; - close(): Promise; - [Symbol.asyncDispose](): Promise; + close?(): Promise; + [Symbol.asyncDispose]?(): Promise; } /** Abstract port for Git ref and commit operations. */ @@ -575,7 +575,10 @@ export declare class GitPersistenceAdapter extends GitPersistencePortBase { treeCacheEntries?: number; treeCacheBytes?: number; }); + writeBlobs(contents: Iterable): Promise; setMaxBlobSize(maxBlobSize: number): void; + close(): Promise; + [Symbol.asyncDispose](): Promise; } /** Git-backed implementation of the ref port. */ diff --git a/test/types/public-api-compatibility.ts b/test/types/public-api-compatibility.ts index b791a6c..179b624 100644 --- a/test/types/public-api-compatibility.ts +++ b/test/types/public-api-compatibility.ts @@ -1,4 +1,5 @@ import type { + GitPersistencePortBase, GitRefPortBase, RepositoryDiagnosticLimitation, RepositoryDoctorReport, @@ -6,6 +7,45 @@ import type { RetentionRootKind, } from '../../index.d.ts'; +class LegacyGitPersistenceAdapter { + async writeBlob(_content: Uint8Array): Promise { + return 'a'.repeat(40); + } + + async writeTree(_entries: string[]): Promise { + return 'b'.repeat(40); + } + + async readBlob(_oid: string, _maxBytes?: number): Promise { + return new Uint8Array(); + } + + async readBlobStream(_oid: string): Promise> { + return (async function* emptyStream() {})(); + } + + async readTree(_treeOid: string) { + return []; + } + + async readTreeEntry(_treeOid: string, _treePath: string) { + return null; + } + + async *iterateTree(_treeOid: string) {} + + async readObjectType(_oid: string): Promise { + return 'blob'; + } + + async readObjectSize(_oid: string): Promise { + return 0; + } +} + +const legacyPersistence: GitPersistencePortBase = new LegacyGitPersistenceAdapter(); +void legacyPersistence; + class LegacyGitRefAdapter { async resolveRef(_ref: string): Promise { return 'a'.repeat(40); diff --git a/test/unit/types/declaration-accuracy.test.js b/test/unit/types/declaration-accuracy.test.js index 162ef82..a67f71f 100644 --- a/test/unit/types/declaration-accuracy.test.js +++ b/test/unit/types/declaration-accuracy.test.js @@ -132,6 +132,9 @@ describe('Persistence lifecycle declaration accuracy', () => { expect(declarations).toContain('treeCacheEntries?: number;'); expect(declarations).toContain('treeCacheBytes?: number;'); expect(declarations).toContain('sessionIdleTimeoutMs?: number;'); + expect(declarations).toContain('writeBlobs?(contents: Iterable)'); + expect(declarations).toContain('close?(): Promise;'); + expect(declarations).toContain('[Symbol.asyncDispose]?(): Promise;'); expect(declarations).toContain('close(): Promise;'); expect(declarations).toContain('[Symbol.asyncDispose](): Promise;'); }); From 76e0cf57dfe8a1d0a0af49fb1c8a5ef1915fafb5 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 03:39:16 -0700 Subject: [PATCH 04/17] fix: make session recovery generation-safe --- .../adapters/GitObjectSessionPool.js | 86 +++++++++++++------ .../adapters/GitPersistenceAdapter.js | 49 +++-------- .../GitPersistenceAdapter.sessions.test.js | 50 +++++++++++ 3 files changed, 123 insertions(+), 62 deletions(-) diff --git a/src/infrastructure/adapters/GitObjectSessionPool.js b/src/infrastructure/adapters/GitObjectSessionPool.js index 0e4130e..e254444 100644 --- a/src/infrastructure/adapters/GitObjectSessionPool.js +++ b/src/infrastructure/adapters/GitObjectSessionPool.js @@ -9,6 +9,7 @@ const PROTOCOLS = Object.freeze({ fastImport: Object.freeze({ opener: 'openFastImportSession', abort: 'abort' }), mktree: Object.freeze({ opener: 'openMktreeSession', abort: 'terminate' }), }); +const EXECUTE_DIRECTLY = (operation) => operation(); /** * Lazily owns one typed plumbing session per Git object protocol. @@ -32,41 +33,54 @@ export default class GitObjectSessionPool { return descriptor !== undefined && typeof this.#plumbing[descriptor.opener] === 'function'; } - async info(objectName) { - return await this.#run('catFile', (session) => session.info(objectName), isRecoverableCatError); + async info(objectName, execute = EXECUTE_DIRECTLY) { + return await this.#run( + 'catFile', + (session) => execute(() => session.info(objectName)), + isRecoverableCatError + ); } - async read(objectName, options) { + async read(objectName, options, execute = EXECUTE_DIRECTLY) { return await this.#run( 'catFile', - (session) => session.read(objectName, options), + (session) => execute(() => session.read(objectName, options)), isRecoverableCatError ); } - async writeBlobs(contents) { - return await this.#run('fastImport', async (session) => { - const oids = []; - for (const content of contents) { - oids.push(await session.writeBlob(content)); - } - await session.checkpoint(); - return Object.freeze(oids); - }); + async writeBlobs(contents, execute = EXECUTE_DIRECTLY) { + return await this.#run('fastImport', (session) => + execute(async () => { + const oids = []; + for (const content of contents) { + oids.push(await session.writeBlob(content)); + } + await session.checkpoint(); + return Object.freeze(oids); + }) + ); } - async writeTree(entries) { - return await this.#run('mktree', (session) => session.write(entries)); + async writeTree(entries, execute = EXECUTE_DIRECTLY) { + return await this.#run('mktree', (session) => execute(() => session.write(entries))); } async invalidate(protocol, expectedSession) { this.#cancelIdle(protocol); - const present = this.#sessions.get(protocol); - this.#sessions.delete(protocol); - let session = expectedSession; - if (session === undefined && present !== undefined) { - session = await present.catch(() => undefined); + const opening = this.#sessions.get(protocol); + if (expectedSession !== undefined) { + if (opening !== undefined) { + const current = await opening.catch(() => undefined); + if (current === expectedSession && this.#sessions.get(protocol) === opening) { + this.#sessions.delete(protocol); + } + } + await this.#abort(protocol, expectedSession); + return; } + this.#sessions.delete(protocol); + const session = await opening?.catch(() => undefined); if (session !== undefined) { await this.#abort(protocol, session); } @@ -85,8 +99,15 @@ export default class GitObjectSessionPool { } try { await session.close(); - } catch { - await this.#abort(protocol, session).catch(() => {}); + } catch (closeError) { + try { + await this.#abort(protocol, session); + } catch (abortError) { + throw new AggregateError( + [closeError, abortError], + `Git ${protocol} session failed to retire` + ); + } } } @@ -102,9 +123,24 @@ export default class GitObjectSessionPool { this.#sessions.clear(); this.#closePromise = (async () => { const results = await Promise.allSettled( - sessions.map(async ([, opening]) => { - const session = await opening; - await session.close(); + sessions.map(async ([protocol, opening]) => { + let session; + try { + session = await opening; + await session.close(); + } catch (closeError) { + if (session !== undefined) { + try { + await this.#abort(protocol, session); + } catch (abortError) { + throw new AggregateError( + [closeError, abortError], + `Git ${protocol} session failed to close or terminate` + ); + } + } + throw closeError; + } }) ); const failures = results diff --git a/src/infrastructure/adapters/GitPersistenceAdapter.js b/src/infrastructure/adapters/GitPersistenceAdapter.js index 1036722..b4fc1b1 100644 --- a/src/infrastructure/adapters/GitPersistenceAdapter.js +++ b/src/infrastructure/adapters/GitPersistenceAdapter.js @@ -122,8 +122,8 @@ export default class GitPersistenceAdapter extends GitPersistencePort { return oids; } try { - const oids = await this.#executeSession('fastImport', () => - this.#sessions.writeBlobs(replayableContents) + const oids = await this.#sessions.writeBlobs(replayableContents, (operation) => + this.policy.execute(operation) ); await Promise.all([this.#sessions.retire('catFile'), this.#sessions.retire('mktree')]); return [...oids]; @@ -142,8 +142,8 @@ export default class GitPersistenceAdapter extends GitPersistencePort { return await this.#runOperation(async () => { if (this.#sessions.supports('mktree')) { const structured = GitTreeObjectCodec.parseMktreeLines(entries); - const oid = await this.#executeSession('mktree', () => - this.#sessions.writeTree(structured) + const oid = await this.#sessions.writeTree(structured, (operation) => + this.policy.execute(operation) ); await this.#sessions.retire('catFile'); return oid; @@ -172,10 +172,8 @@ export default class GitPersistenceAdapter extends GitPersistencePort { if (this.#sessions.supports('catFile')) { let object; try { - object = await this.#executeSession( - 'catFile', - () => this.#sessions.read(oid, { maxBytes: limit }), - GitPersistenceAdapter.#isRecoverableCatError + object = await this.#sessions.read(oid, { maxBytes: limit }, (operation) => + this.policy.execute(operation) ); } catch (error) { throw this.#normalizeObjectReadError(error, oid, limit); @@ -353,10 +351,8 @@ export default class GitPersistenceAdapter extends GitPersistencePort { return this.#metadataCache.getOrCreate(`object\0${oid}`, async () => { if (this.#sessions.supports('catFile')) { try { - const info = await this.#executeSession( - 'catFile', - () => this.#sessions.info(oid), - GitPersistenceAdapter.#isRecoverableCatError + const info = await this.#sessions.info(oid, (operation) => + this.policy.execute(operation) ); return Object.freeze({ oid: info.oid, type: info.type, size: info.size }); } catch (error) { @@ -396,10 +392,10 @@ export default class GitPersistenceAdapter extends GitPersistencePort { async #readTreeObject(treeOid) { return this.#treeCache.getOrCreate(treeOid, async () => { - const object = await this.#executeSession( - 'catFile', - () => this.#sessions.read(treeOid, { maxBytes: this.#treeReadMaxBytes }), - GitPersistenceAdapter.#isRecoverableCatError + const object = await this.#sessions.read( + treeOid, + { maxBytes: this.#treeReadMaxBytes }, + (operation) => this.policy.execute(operation) ); if (object.type !== 'tree') { throw createCasError(`Git object is not a tree: ${treeOid}`, ErrorCodes.TREE_PARSE_ERROR, { @@ -456,21 +452,6 @@ export default class GitPersistenceAdapter extends GitPersistencePort { return null; } - async #executeSession(protocol, operation, recoverable = () => false) { - try { - return await this.policy.execute(operation); - } catch (error) { - if (!recoverable(error)) { - try { - await this.#sessions.invalidate(protocol); - } catch { - // Preserve the operation or timeout failure. - } - } - throw error; - } - } - #normalizeObjectReadError(error, oid, maxBytes) { if (error instanceof GitObjectMissingError) { return new CasError(`Git object not found: ${oid}`, ErrorCodes.GIT_OBJECT_NOT_FOUND, { oid }); @@ -555,12 +536,6 @@ export default class GitPersistenceAdapter extends GitPersistencePort { return error?.details?.code === 'OBJECT_BUFFER_LIMIT_EXCEEDED'; } - static #isRecoverableCatError(error) { - return ( - error instanceof GitObjectMissingError || GitPersistenceAdapter.#isObjectBufferLimit(error) - ); - } - /** * @param {number} metadataCacheEntries */ diff --git a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js index 5cf40a6..3934147 100644 --- a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js +++ b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js @@ -118,6 +118,36 @@ describe('GitPersistenceAdapter cat-file recovery', () => { }); }); +describe('GitPersistenceAdapter concurrent cat-file recovery', () => { + it('does not let a late old-session failure invalidate its replacement', async () => { + const firstOid = '1'.repeat(40); + const secondOid = '2'.repeat(40); + const firstFailure = deferred(); + const lateFailure = deferred(); + const failed = fakeCatSession({ + info: vi.fn((oid) => (oid === firstOid ? firstFailure.promise : lateFailure.promise)), + }); + const replacement = fakeCatSession({ + info: vi.fn(async (oid) => ({ oid, type: 'blob', size: 1 })), + }); + const plumbing = sessionPlumbing({ catSessions: [failed, replacement] }); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + + const firstRead = adapter.readObjectType(firstOid); + const secondRead = adapter.readObjectType(secondOid); + await vi.waitFor(() => expect(failed.info).toHaveBeenCalledTimes(2)); + + firstFailure.reject(new GitProtocolError('first process failure', 'test')); + await expect(firstRead).resolves.toBe('blob'); + lateFailure.reject(new GitProtocolError('late process failure', 'test')); + await expect(secondRead).resolves.toBe('blob'); + + expect(plumbing.openCatFileSession).toHaveBeenCalledTimes(2); + expect(replacement.info).toHaveBeenCalledTimes(2); + expect(replacement.terminate).not.toHaveBeenCalled(); + }); +}); + describe('GitPersistenceAdapter payload streaming', () => { it('keeps payload streaming on executeStream instead of buffering through cat-file', async () => { const oid = 'd'.repeat(40); @@ -413,6 +443,26 @@ describe('GitPersistenceAdapter stream shutdown', () => { }); }); +describe('GitPersistenceAdapter failed session shutdown', () => { + it('terminates a session whose graceful close fails before reporting the error', async () => { + const oid = '4'.repeat(40); + const cat = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid, type: 'blob', size: 1 }), + close: vi.fn().mockRejectedValue(new Error('graceful close failed')), + }); + const adapter = new GitPersistenceAdapter({ + plumbing: sessionPlumbing({ catSessions: [cat] }), + policy: noPolicy, + }); + + await adapter.readObjectType(oid); + await expect(adapter.close()).rejects.toBeInstanceOf(AggregateError); + + expect(cat.close).toHaveBeenCalledTimes(1); + expect(cat.terminate).toHaveBeenCalledTimes(1); + }); +}); + describe('GitPersistenceAdapter idle lifecycle fallback', () => { it('retires an idle session when a caller omits explicit close', async () => { vi.useFakeTimers(); From 6f29be5a8637b6010ca9497924695ec49c28d1f8 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 03:43:12 -0700 Subject: [PATCH 05/17] fix: surface session retirement failures --- .../adapters/GitObjectSessionPool.js | 1 + .../GitPersistenceAdapter.sessions.test.js | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/infrastructure/adapters/GitObjectSessionPool.js b/src/infrastructure/adapters/GitObjectSessionPool.js index e254444..9508ab1 100644 --- a/src/infrastructure/adapters/GitObjectSessionPool.js +++ b/src/infrastructure/adapters/GitObjectSessionPool.js @@ -108,6 +108,7 @@ export default class GitObjectSessionPool { `Git ${protocol} session failed to retire` ); } + throw closeError; } } diff --git a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js index 3934147..060262a 100644 --- a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js +++ b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js @@ -282,6 +282,27 @@ describe('GitPersistenceAdapter persistent write sessions', () => { expect(fastImport.close).toHaveBeenCalledTimes(1); expect(plumbing.execute).not.toHaveBeenCalled(); }); + + it('aborts and rejects when a scoped bulk session cannot close cleanly', async () => { + const fastImport = { + writeBlob: vi.fn().mockResolvedValue('a'.repeat(40)), + checkpoint: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockRejectedValue(new Error('graceful close failed')), + abort: vi.fn().mockResolvedValue(undefined), + }; + const adapter = new GitPersistenceAdapter({ + plumbing: sessionPlumbing({ fastImportSession: fastImport }), + policy: noPolicy, + }); + + await expect(adapter.writeBlobs([Buffer.from('value')])).rejects.toThrow( + 'graceful close failed' + ); + + expect(fastImport.checkpoint).toHaveBeenCalledTimes(1); + expect(fastImport.close).toHaveBeenCalledTimes(1); + expect(fastImport.abort).toHaveBeenCalledTimes(1); + }); }); describe('GitPersistenceAdapter bulk write recovery', () => { From 907b5f0dc6a3139477820c9748728ea31e02c63e Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 03:52:53 -0700 Subject: [PATCH 06/17] docs: record persistent session witness --- ARCHITECTURE.md | 9 +- BEARING.md | 28 +-- CHANGELOG.md | 17 ++ GUIDE.md | 16 +- README.md | 13 +- ROADMAP.md | 5 +- STATUS.md | 11 +- docs/API.md | 36 +++- .../persistent-git-object-sessions.md | 21 ++- .../witness/git-object-sessions.json | 85 +++++++++ .../witness/verification.md | 166 ++++++++++++++++++ test/unit/docs/release-state.test.js | 2 + 12 files changed, 380 insertions(+), 29 deletions(-) create mode 100644 docs/design/0052-persistent-git-object-sessions/witness/git-object-sessions.json create mode 100644 docs/design/0052-persistent-git-object-sessions/witness/verification.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3be92d7..6e66e6f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -147,6 +147,8 @@ The public entrypoint is [index.js](./index.js). - exposes `retention`, `publications`, `caches`, and `expiringSets` for explicit reachability and lifecycle policy - exposes read-only repository evidence through `diagnostics.doctor()` +- owns explicit local resource closure without coupling closure to object or ref + mutation The facade is orchestration glue. It is not the storage engine itself. @@ -329,7 +331,10 @@ Crypto: Git: - **`GitPersistenceAdapter`** — `GitPersistencePort` implementation using - `@git-stunts/plumbing` to shell out to the `git` CLI. + `@git-stunts/plumbing` to shell out to the `git` CLI. It reuses typed + cat-file and mktree sessions, keeps parsed immutable trees count-and-byte + bounded, uses scoped fast-import only for explicit page batches, preserves + streaming payload reads, and owns deterministic child-process closure. - **`GitRefAdapter`** — `GitRefPort` implementation using `@git-stunts/plumbing`. - **`GitRepositoryInspectionAdapter`** — `RepositoryInspectionPort` @@ -354,6 +359,8 @@ File I/O: - **`JsonCodec`** — `CodecPort` using JSON serialization (default). - **`CborCodec`** — `CodecPort` using CBOR serialization. +- **`GitTreeObjectCodec`** — boundary decoder for canonical SHA-1 and SHA-256 + Git tree bytes plus typed mktree session entries. #### Chunkers (`src/infrastructure/chunkers/`) diff --git a/BEARING.md b/BEARING.md index a640879..6d56e6c 100644 --- a/BEARING.md +++ b/BEARING.md @@ -14,11 +14,12 @@ timeline ## Current State -`v6.2.0` shipped on `2026-07-13`. Application asset, bundle, page, cache, -expiry, witness, and repository-diagnostics APIs now sit above mutable root -sets and the low-level CAS pipeline. npm plus GitHub Releases are the active -publication surfaces. JSR validation is healthy, but JSR publication remains -outside the release workflow. +`v6.5.1` shipped on `2026-07-18`. Application asset, bundle, page, cache, +expiry, witness, and repository-diagnostics APIs sit above mutable root sets +and the low-level CAS pipeline. Direct bundle-reference reads and bounded +immutable metadata/page reuse are published. npm plus GitHub Releases are the +active publication surfaces. JSR validation is healthy, but JSR publication +remains outside the release workflow. What exists now: @@ -66,6 +67,10 @@ What exists now: current-generation retention, and allowlisted compare-and-swap application refs. Staged results and immutable witnesses keep content identity separate from retention claims. +- **Persistent bounded Git object sessions.** The v6.5.2 candidate reuses typed + cat-file and mktree processes behind `GitPersistenceAdapter`, uses one scoped + fast-import process for an explicit page batch, and keeps individual blob + writes one-shot so external pruning cannot poison duplicate writes. - **Migration script.** `scripts/migrate-encryption.js` upgrades legacy v1/v2 manifests to the current scheme identifiers. @@ -121,19 +126,20 @@ These were the active tensions from the previous bearing. All resolved. ## Next Horizon -With v6.2.0 shipped, active work is tracked in GitHub Issues and Milestones. -Repo docs hold design and evidence records, not the active queue. +With v6.5.1 shipped and the v6.5.2 performance candidate under review, active +work is tracked in GitHub Issues and Milestones. Repo docs hold design and +evidence records, not the active queue. The latest landed design record is -[0047-application-storage-cache-boundary](./docs/design/0047-application-storage-cache-boundary/application-storage-cache-boundary.md). +[0052-persistent-git-object-sessions](./docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md). Its release evidence is attached to -[#50](https://github.com/git-stunts/git-cas/issues/50) and the -[`v6.2.0` milestone](https://github.com/git-stunts/git-cas/milestone/3). No +[#90](https://github.com/git-stunts/git-cas/issues/90) and the +[`v6.5.2` milestone](https://github.com/git-stunts/git-cas/milestone/12). No later design is selected here; GitHub owns that decision. The broader horizon remains: -- **TUI modernization.** Carried forward to `v6.3.0`; track +- **TUI modernization.** Carried forward to `v6.6.0`; track [#39](https://github.com/git-stunts/git-cas/issues/39). Keep dashboard and wizard actions sharing executable option-building truth while improving operator ergonomics around long-lived diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e741ad..837511a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Bounded page batches and explicit resource closure** - + `pages.putBatch()` stores an explicitly count-and-byte-bounded page group in + input order, while `ContentAddressableStore.close()` and async disposal drain + started operations and release local adapter resources without mutating + objects, refs, retention, or publication state. + +### Performance + +- **Persistent Git object sessions** - immutable metadata and bounded tree + reads reuse one typed `cat-file` process, tree writes reuse typed `mktree`, + and page batches use one scoped `fast-import` process. Parsed tree reuse is + bounded by entry count and estimated bytes; payload streams remain streaming. + Individual blob writes deliberately remain one-shot so an object pruned by + an external process is recreated correctly on the next write. + ## [6.5.1] — 2026-07-18 ### Performance diff --git a/GUIDE.md b/GUIDE.md index 4311bc5..acfd038 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -99,10 +99,13 @@ for await (const chunk of cas.assets.open({ handle: staged.handle })) { } console.log(retained.witness.root.generation); +await cas.close(); ``` Node's file stream is an `AsyncIterable`; Bun and Deno callers can -supply their native async byte streams through the same contract. +supply their native async byte streams through the same contract. In production +code, call `close()` from `finally` or use async disposal. Closure releases +local resources only and does not change stored objects or retention. ## Application Storage Lifecycle @@ -139,8 +142,9 @@ that is already sorted by canonical member path, so construction does not need to collect the complete member set. ```js -const nodes = await cas.pages.put({ source: encodedNodePage }); -const edges = await cas.pages.put({ source: encodedEdgePage }); +const [nodes, edges] = await cas.pages.putBatch({ + pages: [{ source: encodedNodePage }, { source: encodedEdgePage }], +}); const materialization = await cas.bundles.putOrdered({ members: (async function* members() { yield ['edges/root', edges.handle]; @@ -163,6 +167,12 @@ lookup reads only the descriptor path and selected payload; retention and publication perform full bounded graph validation. Identical page bytes and identically constructed bundles produce identical handles. +`pages.putBatch()` collects and validates the complete explicit batch before +persistence. It accepts at most 256 pages and 32 MiB by default, preserves input +order, and uses one scoped Git write session when the plumbing adapter supports +it. Use individual `pages.put()` calls when inputs should not share one bounded +memory envelope. + Repeated `pages.get()` calls reuse immutable payload reads within the store's bounded page cache. The defaults retain at most 128 payloads and 8 MiB; use `pageCacheEntries` and `pageCacheBytes` to tune that ceiling. Every result is a diff --git a/README.md b/README.md index 7d27e67..c2224f9 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,10 @@ Unlike traditional LFS which moves files to external servers, `git-cas` treats t and deterministic structured bundles through validated handles, then retain or publish them with generation-scoped evidence instead of managing Git objects, trees, or payload OIDs directly. +- **Bounded Git Process Reuse**: Immutable object metadata and tree reads reuse + typed Git sessions behind the adapter, while explicitly bounded page batches + amortize writes without changing content identity or buffering payload + streams. - **Key Lifecycle**: Envelope encryption separates DEKs from KEKs. Rotate passphrases across an entire vault without re-encrypting data blobs. Privacy mode HMAC-hashes slug names to prevent metadata discovery. - **Runtime-Adaptive**: A single core supports Node.js 22+, Bun, and Deno through a strict hexagonal port architecture with runtime-specific crypto adapters. @@ -84,11 +88,13 @@ const retained = await cas.retention.retain({ root: { ref: 'refs/cas/rootsets/my-app', name: 'app/asset' }, policy: 'pinned', }); +await cas.close(); ``` `staged.handle` is a content locator, not a durability promise. The returned retention witness identifies the exact Git generation and tree edge that made -the asset reachable. +the asset reachable. `close()` releases local resources only; it never deletes +objects, updates refs, or changes retention. ## Capability Map @@ -138,8 +144,9 @@ Core capabilities: read-only, and `sweep()` can release only markers whose expiry has passed. - **Application storage**: `assets`, `pages`, `bundles`, `retention`, and `publications` compose streaming CAS writes, bounded structured - materializations, direct-reference or complete-validation member reads, - reachability roots, compare-and-swap refs, and immutable lifecycle evidence. + materializations, explicitly bounded page batches, direct-reference or + complete-validation member reads, reachability roots, compare-and-swap refs, + and immutable lifecycle evidence. - **Scoped staging workspaces**: `workspaces.open()` mirrors application writes behind one renewable temporary RootSet, returns only after each handle is anchored, promotes destination-first, and exposes bounded age, expiry, diff --git a/ROADMAP.md b/ROADMAP.md index e6c2422..ba0db9e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -39,6 +39,7 @@ GitHub wins and this file should be corrected. | [`v6.4.1`](https://github.com/git-stunts/git-cas/milestone/8) | Historical bounded-residency proof closeout | [#38](https://github.com/git-stunts/git-cas/issues/38), [#46](https://github.com/git-stunts/git-cas/issues/46) | | [`v6.5.0`](https://github.com/git-stunts/git-cas/milestone/5) | Bounded lazy bundle references and immutable metadata reads | [#81](https://github.com/git-stunts/git-cas/issues/81) | | [`v6.5.1`](https://github.com/git-stunts/git-cas/milestone/11) | Bounded immutable page payload reuse | [#85](https://github.com/git-stunts/git-cas/issues/85) | +| [`v6.5.2`](https://github.com/git-stunts/git-cas/milestone/12) | Persistent bounded Git object sessions | [#90](https://github.com/git-stunts/git-cas/issues/90) | | [`v6.6.0`](https://github.com/git-stunts/git-cas/milestone/9) | Operator TUI and agent automation follow-through | [#39](https://github.com/git-stunts/git-cas/issues/39), [#40](https://github.com/git-stunts/git-cas/issues/40) | | [`v6.7.0`](https://github.com/git-stunts/git-cas/milestone/10) | Browser and edge read-path exploration | [#41](https://github.com/git-stunts/git-cas/issues/41) | | [`v7.0.0`](https://github.com/git-stunts/git-cas/milestone/6) | Protocol break only if audit requires it | [#42](https://github.com/git-stunts/git-cas/issues/42), only when justified | @@ -47,10 +48,10 @@ GitHub wins and this file should be corrected. The latest landed design record is: -- [0051-bounded-page-payload-reuse](./docs/design/0051-bounded-page-payload-reuse/bounded-page-payload-reuse.md) +- [0052-persistent-git-object-sessions](./docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md) Its GitHub goalpost issue, -[#85](https://github.com/git-stunts/git-cas/issues/85), owns the release +[#90](https://github.com/git-stunts/git-cas/issues/90), owns the release evidence. The design doc is the durable contract; GitHub records completion. The `v6.2.0` slice record is: diff --git a/STATUS.md b/STATUS.md index 7c2567e..2d8c4a9 100644 --- a/STATUS.md +++ b/STATUS.md @@ -18,6 +18,11 @@ - The machine-facing `git cas agent` surface exists and now supports OS-keychain passphrase sources for vault-derived key flows, but parity and portability are still partial. +- **v6.5.2 candidate posture** - persistent bounded Git object sessions, + scoped page-write batches, and deterministic local resource closure are + implemented under [#90](https://github.com/git-stunts/git-cas/issues/90). + Unit, Docker real-Git integration, and Node/Bun/Deno platform gates pass; PR, + review, merge, and publication evidence remain outstanding. - **v6.5.1 artifact posture** — signed tag `v6.5.1` resolves to reviewed merge `49b7d5cb`; npm reports `@git-stunts/git-cas@6.5.1` as `latest` with SLSA provenance, and the final GitHub Release is published. Bounded immutable page @@ -116,13 +121,17 @@ - GitHub Issues are canonical. If this section and GitHub disagree, GitHub wins and this section should be corrected. - Current release goalpost: + [#90 v6.5.2: Reuse bounded Git object sessions](https://github.com/git-stunts/git-cas/issues/90) + under the + [`v6.5.2` milestone](https://github.com/git-stunts/git-cas/milestone/12). +- Next planned goalposts remain [#39 v6.6.0: Operator TUI](https://github.com/git-stunts/git-cas/issues/39) and [#40 v6.6.0: Agent automation follow-through](https://github.com/git-stunts/git-cas/issues/40) under the [`v6.6.0` milestone](https://github.com/git-stunts/git-cas/milestone/9). - The latest landed design record is - [0051-bounded-page-payload-reuse](./docs/design/0051-bounded-page-payload-reuse/bounded-page-payload-reuse.md). + [0052-persistent-git-object-sessions](./docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md). ## Read Next diff --git a/docs/API.md b/docs/API.md index 82232e6..4558737 100644 --- a/docs/API.md +++ b/docs/API.md @@ -182,6 +182,21 @@ Lazily initializes and returns the underlying VaultService instance. const vaultService = await cas.getVaultService(); ``` +#### close + +```javascript +await cas.close(); +``` + +Blocks new facade operations, drains started persistence operations, and +releases adapter-owned Git sessions and streams. The method is idempotent and +is also available as `Symbol.asyncDispose`. + +Closure affects local resources only. It does not delete Git objects, update +refs, run garbage collection, expire retained content, or mutate publication +state. Production code should call `close()` from `finally` when async disposal +syntax is unavailable. + #### store ```javascript @@ -1004,11 +1019,20 @@ Validates an existing git-cas manifest tree and wraps it in the same staged result returned by `assets.put()`. This is a migration bridge for callers that already persisted raw tree OIDs; new application code should exchange handles. -### `pages.put()`, `pages.open()`, and `pages.get()` +### `pages.put()`, `pages.putBatch()`, `pages.open()`, and `pages.get()` ```javascript const staged = await cas.pages.put({ source, maxBytes }); +const stagedBatch = await cas.pages.putBatch({ + pages: [ + { source: firstPage, maxBytes: 4096 }, + { source: secondPage, maxBytes: 4096 }, + ], + maxBatchPages: 16, + maxBatchBytes: 65536, +}); + for await (const chunk of cas.pages.open({ handle: staged.handle })) { consume(chunk); } @@ -1027,6 +1051,14 @@ metadata before streaming it. `get()` additionally collects the page under its effective byte bound. An imported handle above the configured maximum fails with `PAGE_TOO_LARGE` without materializing the blob. +`putBatch()` returns an immutable ordered array of `StagedPage` results. It +validates and collects every page before persistence begins, defaults to at +most 256 pages and 32 MiB across the batch, and rejects count or aggregate-byte +overflow with `PAGE_BATCH_LIMIT`. A session-capable Git adapter writes the +batch through one scoped fast-import process that checkpoints and closes before +the method returns. Individual `put()` calls remain one-shot so an externally +pruned object is recreated correctly on a later identical write. + Successful `get()` payloads share in-flight and completed immutable reads inside one store instance. The default LRU retains at most 128 payloads and 8 MiB of payload bytes; `pageCacheEntries` and `pageCacheBytes` may lower or raise those @@ -3342,12 +3374,14 @@ new CasError({ message, code, meta, documentationUrl }); | `GIT_ERROR` | Underlying Git plumbing command failed | `readManifest()`, `inspectAsset()`, `collectReferencedChunks()` | | `GIT_REF_NOT_FOUND` | Git ref lookup found no ref; vault reads normalize this to empty state | `GitRefAdapter`, `VaultPersistence` | | `INVALID_OPTIONS` | Mutually exclusive options provided or unsupported option value | `store()`, `restore()` | +| `RESOURCE_CLOSED` | A facade or persistence adapter operation began after local resources were closed | `ContentAddressableStore`, `GitPersistenceAdapter` | | `HANDLE_INVALID` | Handle object, canonical token, or staged result is malformed or unsupported | Application handles and staged results | | `HANDLE_KIND_MISMATCH` | A handle has a valid envelope but the wrong content kind | Handle parsing and application storage | | `HANDLE_CODEC_MISMATCH` | Handle manifest codec differs from the active CAS codec | `assets.open()`, `assets.adopt()`, retention, publication | | `HANDLE_TARGET_MISSING` | The repository does not contain the complete object graph referenced by a handle | `assets.open()`, `assets.adopt()`, retention, publication | | `HANDLE_TARGET_TYPE_MISMATCH` | A handle root or referenced object has the wrong Git object type | Application storage validation | | `PAGE_TOO_LARGE` | Page input or imported page blob exceeds the effective byte bound | `pages.put()`, `pages.get()`, `pages.open()` | +| `PAGE_BATCH_LIMIT` | Page batch count or aggregate bytes exceed the explicit batch policy | `pages.putBatch()` | | `BUNDLE_CORRUPT` | Persisted bundle descriptors, edges, ranges, or target summaries disagree | Bundle reads, retention, publication | | `BUNDLE_DESCRIPTOR_LIMIT` | Bundle descriptor bytes exceed admission or repository read policy | Bundle construction and validation | | `BUNDLE_DUPLICATE_PATH` | Two ordered bundle members use the same canonical path | `bundles.putOrdered()` | diff --git a/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md b/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md index b38f8cb..bac7d89 100644 --- a/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md +++ b/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md @@ -248,10 +248,15 @@ the normal lifecycle contract. Persisted repositories need no migration. - Object read-budget failures map to the existing size-limit contract. - Malformed tree bytes map to `TREE_PARSE_ERROR`. - Protocol failure invalidates the affected session before surfacing. +- Invalidation is generation-safe: a late failure from an old process may + terminate that process, but cannot evict a replacement session opened by a + concurrent retry. - A typed protocol-process failure receives one retry through a fresh session. These operations are content-addressed and idempotent; bulk inputs are materialized once before the first attempt so a retry sees the same sequence. - Close attempts every opened session and reports aggregate cleanup failure. + A graceful close or retirement failure force-terminates the affected process + before the failure is surfaced. ## Security / Trust / Redaction Posture @@ -328,9 +333,9 @@ blob writes one-shot and streaming payloads on the existing stream path. ## Proof Surface -- unit tests for tree decoding, session coalescing, error invalidation, close, - active-command draining, abandoned-stream cleanup, fallback, and residency - eviction +- unit tests for tree decoding, session coalescing, generation-safe error + invalidation, close and retirement failure, active-command draining, + abandoned-stream cleanup, fallback, and residency eviction - facade tests for lazy close and async disposal - real-Git same-fixture fallback/session command-process comparison - real-Git prune/rewrite regression for individually written blobs @@ -363,14 +368,16 @@ blob writes one-shot and streaming payloads on the existing stream path. opening one scoped fast-import process. 10. Close waits for an active one-shot command and destroys an abandoned Git output stream before resolving. +11. A late failure from an old session cannot invalidate a concurrently opened + replacement, and failed graceful retirement aborts before rejecting. ## Acceptance Criteria - [ ] All acceptance criteria in #90 are proven. -- [ ] Public declarations include close and async disposal. -- [ ] No public session, process, or mutable ref handle is exported. -- [ ] Existing storage identity fixtures remain unchanged. -- [ ] `npm test`, integration suites, platform suites, and lint pass. +- [x] Public declarations include close and async disposal. +- [x] No public session, process, or mutable ref handle is exported. +- [x] Existing storage identity fixtures remain unchanged. +- [x] `npm test`, integration suites, platform suites, and lint pass. - [ ] Witness evidence is committed and linked from the PR. ## Validation Plan diff --git a/docs/design/0052-persistent-git-object-sessions/witness/git-object-sessions.json b/docs/design/0052-persistent-git-object-sessions/witness/git-object-sessions.json new file mode 100644 index 0000000..02a5d90 --- /dev/null +++ b/docs/design/0052-persistent-git-object-sessions/witness/git-object-sessions.json @@ -0,0 +1,85 @@ +{ + "schema": "git-cas.git-object-session-measurement/v1", + "generatedAt": "2026-07-19T10:46:01.178Z", + "environment": { + "node": "v22.23.1", + "git": "git version 2.43.0", + "platform": "linux", + "architecture": "arm64" + }, + "parameters": { + "items": 32, + "pageBytes": 4096, + "samples": 3 + }, + "selectedBundleRead": { + "fallback": { + "sampleCount": 3, + "semanticDigest": "f33d2c7157761572c4cbb8cb367bcf4f1f4bc1c8cbb34e9bb502b1cfd085a63d", + "resultCount": 32, + "processCount": 225, + "counts": { + "cat-file:batch-check": 64, + "ls-tree": 128, + "cat-file": 33 + }, + "wallMs": 343.911, + "cpuMs": 277.441, + "userCpuMs": 175.93, + "systemCpuMs": 107.204, + "peakRssBytes": 79433728 + }, + "session": { + "sampleCount": 3, + "semanticDigest": "f33d2c7157761572c4cbb8cb367bcf4f1f4bc1c8cbb34e9bb502b1cfd085a63d", + "resultCount": 32, + "processCount": 1, + "counts": { + "session:cat-file": 1 + }, + "wallMs": 54.038, + "cpuMs": 61.958, + "userCpuMs": 55.143, + "systemCpuMs": 6.815, + "peakRssBytes": 73871360 + }, + "semanticDigestEqual": true, + "processReductionPercent": 99.6, + "wallReductionPercent": 84.3, + "cpuReductionPercent": 77.7 + }, + "pageWrite": { + "individual": { + "sampleCount": 3, + "semanticDigest": "6b1abc55f720680d74d1bf24b5b9ef7de1494807b7cd2e339af87f118c2b1d26", + "resultCount": 32, + "processCount": 32, + "counts": { + "hash-object": 32 + }, + "wallMs": 76.503, + "cpuMs": 71.906, + "userCpuMs": 50.053, + "systemCpuMs": 21.853, + "peakRssBytes": 73334784 + }, + "batch": { + "sampleCount": 3, + "semanticDigest": "6b1abc55f720680d74d1bf24b5b9ef7de1494807b7cd2e339af87f118c2b1d26", + "resultCount": 32, + "processCount": 1, + "counts": { + "session:fast-import": 1 + }, + "wallMs": 35, + "cpuMs": 19.785, + "userCpuMs": 16.663, + "systemCpuMs": 3.122, + "peakRssBytes": 72331264 + }, + "semanticDigestEqual": true, + "processReductionPercent": 96.9, + "wallReductionPercent": 54.3, + "cpuReductionPercent": 72.5 + } +} diff --git a/docs/design/0052-persistent-git-object-sessions/witness/verification.md b/docs/design/0052-persistent-git-object-sessions/witness/verification.md new file mode 100644 index 0000000..0817e96 --- /dev/null +++ b/docs/design/0052-persistent-git-object-sessions/witness/verification.md @@ -0,0 +1,166 @@ +# Persistent Git Object Sessions Verification Witness + +Generated: 2026-07-19 03:46:01 PDT + +Issue: [#90](https://github.com/git-stunts/git-cas/issues/90) + +Implementation commit: `27831926327afc7522b39ab435d29b46b7ac428e` + +Compatibility repair commit: `a2ce917fb629f63db643de7cf72035386f48345e` + +Session recovery repair commit: `76e0cf57dfe8a1d0a0af49fb1c8a5ef1915fafb5` + +Session retirement repair commit: `6f29be5a8637b6010ca9497924695ec49c28d1f8` + +## Contract + +`GitPersistenceAdapter` feature-detects typed plumbing sessions, keeps metadata +and parsed tree residency bounded, and preserves command-per-operation fallback +for injected plumbing implementations without those capabilities. + +[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#41-81@27831926327afc7522b39ab435d29b46b7ac428e`] + +The session pool lazily coalesces one process per protocol, invalidates only the +exact poisoned session generation, retries one typed protocol failure through +a fresh process, retires idle sessions, and aggregates explicit close failures. +A late failure from an old process cannot evict its replacement. Graceful close +or retirement failure force-terminates the affected process before surfacing. + +[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#16-59@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#62-145@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#196-226@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#69-153@76e0cf57dfe8a1d0a0af49fb1c8a5ef1915fafb5`] +[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#89-113@6f29be5a8637b6010ca9497924695ec49c28d1f8`] + +The low-level persistence-port declarations keep batch write and lifecycle +capabilities optional for existing structural adapters. The concrete Git +adapter and root facade expose them as required capabilities. + +[cite: `index.d.ts#513-581@a2ce917fb629f63db643de7cf72035386f48345e`] + +A Deno type-check fixture assigns a legacy structural persistence adapter that +does not implement any new optional capability, proving patch-release source +compatibility. + +[cite: `test/types/public-api-compatibility.ts#1-47@a2ce917fb629f63db643de7cf72035386f48345e`] + +Raw Git tree decoding supports SHA-1 and SHA-256 object identifiers, validates +modes and names, freezes decoded entries, and computes an estimated residency +weight for the byte-bounded cache. + +[cite: `src/infrastructure/codecs/GitTreeObjectCodec.js#15-61@27831926327afc7522b39ab435d29b46b7ac428e`] + +## Write Correctness Boundary + +Individual `writeBlob()` calls remain one-shot `hash-object` operations. +Bounded `writeBlobs()` calls materialize their iterable once, write through one +scoped fast-import process, checkpoint it, and retire it before exposing OIDs. + +[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#89-134@27831926327afc7522b39ab435d29b46b7ac428e`] + +The one-shot boundary is required. The real-Git regression aggressively prunes +an individually written unreachable blob, proves the object is absent, writes +the same bytes again, and proves the same OID exists and remains readable. + +[cite: `test/integration/bundle-reference-performance.test.js#242-269@27831926327afc7522b39ab435d29b46b7ac428e`] + +`pages.putBatch()` validates page count and aggregate bytes, collects the whole +explicitly bounded batch before persistence starts, and preserves input order +in its immutable staged results. + +[cite: `src/domain/services/PageService.js#69-118@27831926327afc7522b39ab435d29b46b7ac428e`] + +## Lifecycle + +Every adapter operation registers before asynchronous work starts. `close()` +blocks new operations, drains started commands, destroys abandoned Git output +streams, waits for the corresponding processes, closes typed sessions, and +releases bounded cache residency. + +[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#497-552@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#642-671@27831926327afc7522b39ab435d29b46b7ac428e`] + +Unit tests prove idempotent closure, post-close rejection, active one-shot +draining, abandoned-stream destruction, and idle retirement when explicit +close is omitted. Concurrency coverage proves that a late old-session failure +cannot invalidate its replacement. Shutdown coverage proves that graceful +close and scoped retirement failures force termination before rejection. + +[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#315-354@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#357-437@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#121-148@76e0cf57dfe8a1d0a0af49fb1c8a5ef1915fafb5`] +[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#447-463@76e0cf57dfe8a1d0a0af49fb1c8a5ef1915fafb5`] +[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#261-305@6f29be5a8637b6010ca9497924695ec49c28d1f8`] + +## Stable Performance Gates + +The real-Git integration test requires the session path to return the same +selected reference while opening exactly one cat-file session and fewer total +Git processes than fallback. A separate gate requires batch and individual page +writes to return the same ordered handles while batch opens exactly one +fast-import session and fewer total processes. + +[cite: `test/integration/bundle-reference-performance.test.js#190-239@27831926327afc7522b39ab435d29b46b7ac428e`] + +## Verification Results + +| Command | Result | +| ------------------------------------------------- | -------------------------------------- | +| Focused lifecycle, session, codec, and page units | 4 files; 50 tests passed | +| `pnpm test` | 223 files; 2,070 passed, 2 skipped | +| `pnpm test:integration:node` | 12 files; 196 tests passed in Docker | +| `bats test/platform/runtimes.bats` | Node, Bun, and Deno full suites passed | +| Deno public type compatibility check | Passed with legacy structural adapter | +| `pnpm lint` | Passed | +| `git diff --check` | Passed | + +The direct host `pnpm test:integration` invocation was intentionally rejected +by all 12 integration files because this repository requires +`GIT_STUNTS_DOCKER=1`. The required Docker invocation above passed; no host +result is counted as integration evidence. + +## Measurement Method + +The committed diagnostic creates isolated repositories and forked workers, +alternates comparison order across three samples, verifies equal semantic +digests, counts every one-shot command and typed session opening, and reports +median wall time, process CPU, and peak RSS. + +```sh +docker compose run --build --rm test-node \ + node scripts/diagnostics/measure-git-object-sessions.js 32 4096 3 +``` + +The environment was Linux arm64 with Node `v22.23.1` and Git `2.43.0`. The raw +machine-readable result is +[`git-object-sessions.json`](./git-object-sessions.json). + +## Measurement Results + +| Path | Baseline processes | Candidate processes | Process reduction | Baseline wall | Candidate wall | Observed wall reduction | +| ------------------------------- | -----------------: | ------------------: | ----------------: | ------------: | -------------: | ----------------------: | +| Selected bundle reference reads | 225 | 1 | 99.6% | 343.911 ms | 54.038 ms | 84.3% | +| Bounded page writes | 32 | 1 | 96.9% | 76.503 ms | 35.000 ms | 54.3% | + +Selected reads had equal semantic digests and returned 32 results in both +modes. The session path opened one cat-file process; fallback opened 64 +batch-check, 128 ls-tree, and 33 cat-file processes. Median process CPU fell +from 277.441 ms to 61.958 ms. Peak RSS was 79,433,728 bytes for fallback and +73,871,360 bytes for the session path. + +Page writes had equal semantic digests and returned 32 handles in both modes. +Individual writes opened 32 hash-object processes; the batch opened one scoped +fast-import process. Median process CPU fell from 71.906 ms to 19.785 ms. Peak +RSS was 73,334,784 bytes for individual writes and 72,331,264 bytes for batch. + +Timing and RSS are environment observations, not portable guarantees. Semantic +digest equality and process counts are the stable regression contracts. + +## Residual Scope + +This witness does not prove graph-wide memory boundedness, git-warp application +latency, or elimination of all subprocess cost. Payload streams remain +one-shot and streaming by design. Git-warp must adopt the published git-cas +release, hold cache acquisitions at the correct runtime lifetime, and prove its +own large-graph memory and latency benchmarks before the v19 performance +campaign can close. diff --git a/test/unit/docs/release-state.test.js b/test/unit/docs/release-state.test.js index e4eb1dd..8d86696 100644 --- a/test/unit/docs/release-state.test.js +++ b/test/unit/docs/release-state.test.js @@ -128,6 +128,8 @@ describe('release state docs', () => { expectV650PublishedEvidence(status, v650Publication); expectCurrentV640PublicationEvidence(v640Publication); expect(status).toContain('Current release goalpost:'); + expect(status).toContain('#90 v6.5.2: Reuse bounded Git object sessions'); + expect(status).toContain('0052-persistent-git-object-sessions'); expect(v640Publication).toContain('https://slsa.dev/provenance/v1'); expect(v640Publication).toContain('https://github.com/git-stunts/git-cas/releases/tag/v6.4.0'); }); From 1ab53265a0f95e062817497b56982f4da5059402 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 04:00:08 -0700 Subject: [PATCH 07/17] fix: await idle session retirement --- .../adapters/GitObjectSessionPool.js | 37 +++++++++--- .../GitPersistenceAdapter.sessions.test.js | 60 +++++++++++++++++++ 2 files changed, 89 insertions(+), 8 deletions(-) diff --git a/src/infrastructure/adapters/GitObjectSessionPool.js b/src/infrastructure/adapters/GitObjectSessionPool.js index 9508ab1..fb6b87e 100644 --- a/src/infrastructure/adapters/GitObjectSessionPool.js +++ b/src/infrastructure/adapters/GitObjectSessionPool.js @@ -18,6 +18,8 @@ export default class GitObjectSessionPool { #active = new Map(); #closePromise = null; #closed = false; + #idleFailures = new Map(); + #idleRetirements = new Set(); #idleTimeoutMs; #idleTimers = new Map(); #plumbing; @@ -121,6 +123,7 @@ export default class GitObjectSessionPool { this.#cancelIdle(protocol); } const sessions = [...this.#sessions.entries()]; + const idleRetirements = [...this.#idleRetirements]; this.#sessions.clear(); this.#closePromise = (async () => { const results = await Promise.allSettled( @@ -144,9 +147,11 @@ export default class GitObjectSessionPool { } }) ); + await Promise.allSettled(idleRetirements); const failures = results .filter((result) => result.status === 'rejected') .map((result) => result.reason); + failures.push(...this.#idleFailures.values()); if (failures.length > 0) { throw new AggregateError(failures, 'One or more Git object sessions failed to close'); } @@ -241,19 +246,35 @@ export default class GitObjectSessionPool { return; } this.#sessions.delete(protocol); - void opening - .then((session) => session.close()) - .catch(async () => { - const session = await opening.catch(() => undefined); - if (session !== undefined) { - await this.#abort(protocol, session).catch(() => {}); - } - }); + this.#trackIdleRetirement(protocol, opening); }, this.#idleTimeoutMs); timer.unref?.(); this.#idleTimers.set(protocol, timer); } + #trackIdleRetirement(protocol, opening) { + const retirement = opening.then(async (session) => { + try { + await session.close(); + } catch (closeError) { + try { + await this.#abort(protocol, session); + } catch (abortError) { + throw new AggregateError( + [closeError, abortError], + `Idle Git ${protocol} session failed to close or terminate` + ); + } + throw closeError; + } + }); + const tracked = retirement.catch((error) => { + this.#idleFailures.set(protocol, error); + }); + this.#idleRetirements.add(tracked); + void tracked.finally(() => this.#idleRetirements.delete(tracked)); + } + #cancelIdle(protocol) { const timer = this.#idleTimers.get(protocol); if (timer !== undefined) { diff --git a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js index 060262a..8a20821 100644 --- a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js +++ b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js @@ -507,3 +507,63 @@ describe('GitPersistenceAdapter idle lifecycle fallback', () => { } }); }); + +describe('GitPersistenceAdapter idle retirement shutdown', () => { + it('waits for an in-flight idle retirement before explicit close resolves', async () => { + vi.useFakeTimers(); + try { + const oid = '2'.repeat(40); + const retired = deferred(); + const cat = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid, type: 'blob', size: 1 }), + close: vi.fn().mockReturnValue(retired.promise), + }); + const adapter = new GitPersistenceAdapter({ + plumbing: sessionPlumbing({ catSessions: [cat] }), + policy: noPolicy, + sessionIdleTimeoutMs: 10, + }); + + await adapter.readObjectType(oid); + await vi.advanceTimersByTimeAsync(10); + let closeSettled = false; + const close = adapter.close().then(() => { + closeSettled = true; + }); + await Promise.resolve(); + + expect(closeSettled).toBe(false); + retired.resolve(); + await close; + expect(closeSettled).toBe(true); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe('GitPersistenceAdapter failed idle retirement', () => { + it('reports an idle retirement failure after force-terminating the session', async () => { + vi.useFakeTimers(); + try { + const oid = '3'.repeat(40); + const cat = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid, type: 'blob', size: 1 }), + close: vi.fn().mockRejectedValue(new Error('idle close failed')), + }); + const adapter = new GitPersistenceAdapter({ + plumbing: sessionPlumbing({ catSessions: [cat] }), + policy: noPolicy, + sessionIdleTimeoutMs: 10, + }); + + await adapter.readObjectType(oid); + await vi.advanceTimersByTimeAsync(10); + await expect(adapter.close()).rejects.toBeInstanceOf(AggregateError); + + expect(cat.terminate).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }); +}); From d675ea1cd6c21d83dc94f380e9477235f174dcd6 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 04:04:13 -0700 Subject: [PATCH 08/17] fix: enforce page batch budget while streaming --- src/domain/services/PageService.js | 37 ++++++++++++++----- test/unit/domain/services/PageService.test.js | 17 ++++++++- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/src/domain/services/PageService.js b/src/domain/services/PageService.js index 33764a8..96a3798 100644 --- a/src/domain/services/PageService.js +++ b/src/domain/services/PageService.js @@ -87,15 +87,20 @@ export default class PageService { if (page === null || typeof page !== 'object' || !Object.hasOwn(page, 'source')) { throw createCasError('Page batch entry must provide source', ErrorCodes.INVALID_OPTIONS); } - const bytes = await PageService.#collect(page.source, this.#effectiveLimit(page.maxBytes)); + const pageLimit = this.#effectiveLimit(page.maxBytes); + const remainingBatchBytes = maxBatchBytes - totalBytes; + const overflow = remainingBatchBytes <= pageLimit + ? (observedBytes) => PageService.#batchTooLarge( + totalBytes + observedBytes, + maxBatchBytes, + ) + : PageService.#tooLarge; + const bytes = await PageService.#collectWithOverflow( + page.source, + Math.min(pageLimit, remainingBatchBytes), + overflow, + ); totalBytes += bytes.length; - if (totalBytes > maxBatchBytes) { - throw createCasError( - 'Page batch exceeds its configured byte limit', - ErrorCodes.PAGE_BATCH_LIMIT, - { observedBytes: totalBytes, maxBatchBytes }, - ); - } prepared.push({ bytes, observedAt: this.#observedAt() }); } @@ -225,9 +230,13 @@ export default class PageService { } static async #collect(source, maxBytes) { + return await PageService.#collectWithOverflow(source, maxBytes, PageService.#tooLarge); + } + + static async #collectWithOverflow(source, maxBytes, overflow) { if (isBytes(source)) { if (source.length > maxBytes) { - throw PageService.#tooLarge(source.length, maxBytes); + throw overflow(source.length, maxBytes); } return new Uint8Array(source); } @@ -251,7 +260,7 @@ export default class PageService { } total += bytes.length; if (total > maxBytes) { - throw PageService.#tooLarge(total, maxBytes); + throw overflow(total, maxBytes); } chunks.push(bytes); } @@ -265,6 +274,14 @@ export default class PageService { }); } + static #batchTooLarge(observedBytes, maxBatchBytes) { + return createCasError( + 'Page batch exceeds its configured byte limit', + ErrorCodes.PAGE_BATCH_LIMIT, + { observedBytes, maxBatchBytes }, + ); + } + static #assertLimit(value, label) { if (!Number.isSafeInteger(value) || value < 0) { throw createCasError(`${label} must be a non-negative safe integer`, ErrorCodes.INVALID_OPTIONS, { diff --git a/test/unit/domain/services/PageService.test.js b/test/unit/domain/services/PageService.test.js index af8ceb9..fc3c3be 100644 --- a/test/unit/domain/services/PageService.test.js +++ b/test/unit/domain/services/PageService.test.js @@ -103,19 +103,32 @@ describe('PageService.putBatch()', () => { new Uint8Array(Buffer.from('second')), ); }); +}); +describe('PageService.putBatch() limits', () => { it('rejects count and byte overflow before starting persistence', async () => { const { pages, persistence } = makePages(); const writeBlobs = vi.spyOn(persistence, 'writeBlobs'); + let yieldedChunks = 0; + const overflow = (async function* source() { + yieldedChunks += 1; + yield Buffer.from('cd'); + yieldedChunks += 1; + yield Buffer.from('unreachable'); + })(); await expect(pages.putBatch({ pages: [{ source: Buffer.from('a') }, { source: Buffer.from('b') }], maxBatchPages: 1, })).rejects.toMatchObject({ code: 'PAGE_BATCH_LIMIT' }); await expect(pages.putBatch({ - pages: [{ source: Buffer.from('ab') }, { source: Buffer.from('cd') }], + pages: [{ source: Buffer.from('ab') }, { source: overflow }], maxBatchBytes: 3, - })).rejects.toMatchObject({ code: 'PAGE_BATCH_LIMIT' }); + })).rejects.toMatchObject({ + code: 'PAGE_BATCH_LIMIT', + meta: { observedBytes: 4, maxBatchBytes: 3 }, + }); + expect(yieldedChunks).toBe(1); expect(writeBlobs).not.toHaveBeenCalled(); }); From 22984f83cd204817a734d5cd874a6d1d992f3615 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 04:06:05 -0700 Subject: [PATCH 09/17] fix: serialize idle session replacement --- .../adapters/GitObjectSessionPool.js | 19 +++++++++++++++---- .../GitPersistenceAdapter.sessions.test.js | 11 ++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/infrastructure/adapters/GitObjectSessionPool.js b/src/infrastructure/adapters/GitObjectSessionPool.js index fb6b87e..4f96acb 100644 --- a/src/infrastructure/adapters/GitObjectSessionPool.js +++ b/src/infrastructure/adapters/GitObjectSessionPool.js @@ -19,7 +19,7 @@ export default class GitObjectSessionPool { #closePromise = null; #closed = false; #idleFailures = new Map(); - #idleRetirements = new Set(); + #idleRetirements = new Map(); #idleTimeoutMs; #idleTimers = new Map(); #plumbing; @@ -123,7 +123,7 @@ export default class GitObjectSessionPool { this.#cancelIdle(protocol); } const sessions = [...this.#sessions.entries()]; - const idleRetirements = [...this.#idleRetirements]; + const idleRetirements = [...this.#idleRetirements.values()]; this.#sessions.clear(); this.#closePromise = (async () => { const results = await Promise.allSettled( @@ -216,6 +216,13 @@ export default class GitObjectSessionPool { throw new Error(`Git object protocol is unavailable: ${protocol}`); } let opening = this.#sessions.get(protocol); + if (opening === undefined) { + await this.#idleRetirements.get(protocol); + if (this.#closed) { + throw new Error('Git object session pool is closed'); + } + opening = this.#sessions.get(protocol); + } if (opening === undefined) { opening = Promise.resolve().then(() => this.#plumbing[descriptor.opener]()); this.#sessions.set(protocol, opening); @@ -271,8 +278,12 @@ export default class GitObjectSessionPool { const tracked = retirement.catch((error) => { this.#idleFailures.set(protocol, error); }); - this.#idleRetirements.add(tracked); - void tracked.finally(() => this.#idleRetirements.delete(tracked)); + this.#idleRetirements.set(protocol, tracked); + void tracked.finally(() => { + if (this.#idleRetirements.get(protocol) === tracked) { + this.#idleRetirements.delete(protocol); + } + }); } #cancelIdle(protocol) { diff --git a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js index 8a20821..aa30c76 100644 --- a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js +++ b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js @@ -513,19 +513,25 @@ describe('GitPersistenceAdapter idle retirement shutdown', () => { vi.useFakeTimers(); try { const oid = '2'.repeat(40); + const nextOid = '3'.repeat(40); const retired = deferred(); const cat = fakeCatSession({ info: vi.fn().mockResolvedValue({ oid, type: 'blob', size: 1 }), close: vi.fn().mockReturnValue(retired.promise), }); + const replacement = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid: nextOid, type: 'tree', size: 1 }), + }); + const plumbing = sessionPlumbing({ catSessions: [cat, replacement] }); const adapter = new GitPersistenceAdapter({ - plumbing: sessionPlumbing({ catSessions: [cat] }), + plumbing, policy: noPolicy, sessionIdleTimeoutMs: 10, }); await adapter.readObjectType(oid); await vi.advanceTimersByTimeAsync(10); + const nextRead = adapter.readObjectType(nextOid); let closeSettled = false; const close = adapter.close().then(() => { closeSettled = true; @@ -533,9 +539,12 @@ describe('GitPersistenceAdapter idle retirement shutdown', () => { await Promise.resolve(); expect(closeSettled).toBe(false); + expect(plumbing.openCatFileSession).toHaveBeenCalledTimes(1); retired.resolve(); + await expect(nextRead).resolves.toBe('tree'); await close; expect(closeSettled).toBe(true); + expect(plumbing.openCatFileSession).toHaveBeenCalledTimes(2); } finally { vi.useRealTimers(); } From f6d37902feb4f69cc167455e3ccbe60b1bbb0f5b Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 04:12:13 -0700 Subject: [PATCH 10/17] fix: preserve bulk session failures --- .../adapters/GitPersistenceAdapter.js | 23 ++++++++++++-- .../GitPersistenceAdapter.sessions.test.js | 30 +++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/infrastructure/adapters/GitPersistenceAdapter.js b/src/infrastructure/adapters/GitPersistenceAdapter.js index b4fc1b1..a83017c 100644 --- a/src/infrastructure/adapters/GitPersistenceAdapter.js +++ b/src/infrastructure/adapters/GitPersistenceAdapter.js @@ -121,15 +121,34 @@ export default class GitPersistenceAdapter extends GitPersistencePort { } return oids; } + let result; + let operationFailed = false; + let operationError; try { const oids = await this.#sessions.writeBlobs(replayableContents, (operation) => this.policy.execute(operation) ); await Promise.all([this.#sessions.retire('catFile'), this.#sessions.retire('mktree')]); - return [...oids]; - } finally { + result = [...oids]; + } catch (error) { + operationFailed = true; + operationError = error; + } + try { await this.#sessions.retire('fastImport'); + } catch (retirementError) { + if (operationFailed) { + throw new AggregateError( + [operationError, retirementError], + 'Bulk Git object write and session retirement both failed' + ); + } + throw retirementError; + } + if (operationFailed) { + throw operationError; } + return result; }); } diff --git a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js index aa30c76..c455102 100644 --- a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js +++ b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js @@ -305,6 +305,36 @@ describe('GitPersistenceAdapter persistent write sessions', () => { }); }); +describe('GitPersistenceAdapter bulk write retirement failures', () => { + it('preserves cache and bulk session retirement failures', async () => { + const catCloseError = new Error('cat-file close failed'); + const fastImportCloseError = new Error('fast-import close failed'); + const cat = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid: 'a'.repeat(40), type: 'blob', size: 1 }), + close: vi.fn().mockRejectedValue(catCloseError), + terminate: vi.fn().mockResolvedValue(undefined), + }); + const fastImport = { + writeBlob: vi.fn().mockResolvedValue('b'.repeat(40)), + checkpoint: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockRejectedValue(fastImportCloseError), + abort: vi.fn().mockResolvedValue(undefined), + }; + const adapter = new GitPersistenceAdapter({ + plumbing: sessionPlumbing({ catSessions: [cat], fastImportSession: fastImport }), + policy: noPolicy, + }); + await adapter.readObjectType('a'.repeat(40)); + + const failure = await adapter.writeBlobs([Buffer.from('value')]).catch((error) => error); + + expect(failure).toBeInstanceOf(AggregateError); + expect(failure.errors).toEqual([catCloseError, fastImportCloseError]); + expect(cat.terminate).toHaveBeenCalledTimes(1); + expect(fastImport.abort).toHaveBeenCalledTimes(1); + }); +}); + describe('GitPersistenceAdapter bulk write recovery', () => { it('replays a materialized bulk input through a fresh typed session', async () => { const failed = { From ccbc80d67c2e96d8556690fd04683a34c4b4f19f Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 04:14:07 -0700 Subject: [PATCH 11/17] fix: preserve cache retirement failures --- .../adapters/GitPersistenceAdapter.js | 19 ++++++++++-- .../GitPersistenceAdapter.sessions.test.js | 30 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/infrastructure/adapters/GitPersistenceAdapter.js b/src/infrastructure/adapters/GitPersistenceAdapter.js index a83017c..6c6d7c5 100644 --- a/src/infrastructure/adapters/GitPersistenceAdapter.js +++ b/src/infrastructure/adapters/GitPersistenceAdapter.js @@ -99,7 +99,7 @@ export default class GitPersistenceAdapter extends GitPersistencePort { input: content, }) ); - await Promise.all([this.#sessions.retire('catFile'), this.#sessions.retire('mktree')]); + await this.#retireSessions(['catFile', 'mktree']); return oid; } @@ -128,7 +128,7 @@ export default class GitPersistenceAdapter extends GitPersistencePort { const oids = await this.#sessions.writeBlobs(replayableContents, (operation) => this.policy.execute(operation) ); - await Promise.all([this.#sessions.retire('catFile'), this.#sessions.retire('mktree')]); + await this.#retireSessions(['catFile', 'mktree']); result = [...oids]; } catch (error) { operationFailed = true; @@ -551,6 +551,21 @@ export default class GitPersistenceAdapter extends GitPersistencePort { return promise; } + async #retireSessions(protocols) { + const results = await Promise.allSettled( + protocols.map((protocol) => this.#sessions.retire(protocol)) + ); + const failures = results + .filter((result) => result.status === 'rejected') + .map((result) => result.reason); + if (failures.length === 1) { + throw failures[0]; + } + if (failures.length > 1) { + throw new AggregateError(failures, 'Multiple Git object sessions failed to retire'); + } + } + static #isObjectBufferLimit(error) { return error?.details?.code === 'OBJECT_BUFFER_LIMIT_EXCEEDED'; } diff --git a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js index c455102..cf09e05 100644 --- a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js +++ b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js @@ -335,6 +335,36 @@ describe('GitPersistenceAdapter bulk write retirement failures', () => { }); }); +describe('GitPersistenceAdapter cache session retirement failures', () => { + it('preserves every cache session failure after an object write', async () => { + const catCloseError = new Error('cat-file close failed'); + const mktreeCloseError = new Error('mktree close failed'); + const cat = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid: 'a'.repeat(40), type: 'blob', size: 1 }), + close: vi.fn().mockRejectedValue(catCloseError), + terminate: vi.fn().mockResolvedValue(undefined), + }); + const mktree = { + write: vi.fn().mockResolvedValue('b'.repeat(40)), + close: vi.fn().mockRejectedValue(mktreeCloseError), + terminate: vi.fn().mockResolvedValue(undefined), + }; + const adapter = new GitPersistenceAdapter({ + plumbing: sessionPlumbing({ catSessions: [cat], mktreeSession: mktree }), + policy: noPolicy, + }); + await adapter.writeTree([]); + await adapter.readObjectType('a'.repeat(40)); + + const failure = await adapter.writeBlob(Buffer.from('value')).catch((error) => error); + + expect(failure).toBeInstanceOf(AggregateError); + expect(failure.errors).toEqual([catCloseError, mktreeCloseError]); + expect(cat.terminate).toHaveBeenCalledTimes(1); + expect(mktree.terminate).toHaveBeenCalledTimes(1); + }); +}); + describe('GitPersistenceAdapter bulk write recovery', () => { it('replays a materialized bulk input through a fresh typed session', async () => { const failed = { From 3f1fa9000cdd87d03ef8a1077a1c1e20e486f2de Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 04:25:16 -0700 Subject: [PATCH 12/17] fix: serialize protocol teardown --- .../adapters/GitObjectSessionPool.js | 160 ++++++++++-------- .../GitPersistenceAdapter.sessions.test.js | 79 +++++++++ 2 files changed, 166 insertions(+), 73 deletions(-) diff --git a/src/infrastructure/adapters/GitObjectSessionPool.js b/src/infrastructure/adapters/GitObjectSessionPool.js index 4f96acb..a099adc 100644 --- a/src/infrastructure/adapters/GitObjectSessionPool.js +++ b/src/infrastructure/adapters/GitObjectSessionPool.js @@ -19,10 +19,10 @@ export default class GitObjectSessionPool { #closePromise = null; #closed = false; #idleFailures = new Map(); - #idleRetirements = new Map(); #idleTimeoutMs; #idleTimers = new Map(); #plumbing; + #retirements = new Map(); #sessions = new Map(); constructor({ plumbing, idleTimeoutMs }) { @@ -78,14 +78,23 @@ export default class GitObjectSessionPool { this.#sessions.delete(protocol); } } - await this.#abort(protocol, expectedSession); + await this.#trackRetirement(protocol, () => this.#abort(protocol, expectedSession)); return; } this.#sessions.delete(protocol); - const session = await opening?.catch(() => undefined); - if (session !== undefined) { - await this.#abort(protocol, session); + if (opening === undefined) { + const retirement = this.#retirements.get(protocol); + if (retirement !== undefined) { + await retirement.completion; + } + return; } + await this.#trackRetirement(protocol, async () => { + const session = await opening.catch(() => undefined); + if (session !== undefined) { + await this.#abort(protocol, session); + } + }); } async retire(protocol) { @@ -93,25 +102,18 @@ export default class GitObjectSessionPool { const opening = this.#sessions.get(protocol); this.#sessions.delete(protocol); if (opening === undefined) { + const retirement = this.#retirements.get(protocol); + if (retirement !== undefined) { + await retirement.completion; + } return; } - const session = await opening.catch(() => undefined); - if (session === undefined) { - return; - } - try { - await session.close(); - } catch (closeError) { - try { - await this.#abort(protocol, session); - } catch (abortError) { - throw new AggregateError( - [closeError, abortError], - `Git ${protocol} session failed to retire` - ); + await this.#trackRetirement(protocol, async () => { + const session = await opening.catch(() => undefined); + if (session !== undefined) { + await this.#closeSession(protocol, session, `Git ${protocol} session failed to retire`); } - throw closeError; - } + }); } async close() { @@ -123,31 +125,20 @@ export default class GitObjectSessionPool { this.#cancelIdle(protocol); } const sessions = [...this.#sessions.entries()]; - const idleRetirements = [...this.#idleRetirements.values()]; + const retirements = [...this.#retirements.values()].map((retirement) => retirement.barrier); this.#sessions.clear(); this.#closePromise = (async () => { const results = await Promise.allSettled( sessions.map(async ([protocol, opening]) => { - let session; - try { - session = await opening; - await session.close(); - } catch (closeError) { - if (session !== undefined) { - try { - await this.#abort(protocol, session); - } catch (abortError) { - throw new AggregateError( - [closeError, abortError], - `Git ${protocol} session failed to close or terminate` - ); - } - } - throw closeError; - } + const session = await opening; + await this.#closeSession( + protocol, + session, + `Git ${protocol} session failed to close or terminate` + ); }) ); - await Promise.allSettled(idleRetirements); + await Promise.allSettled(retirements); const failures = results .filter((result) => result.status === 'rejected') .map((result) => result.reason); @@ -178,7 +169,14 @@ export default class GitObjectSessionPool { if (recoverable(error) || error instanceof InvalidArgumentError) { throw error; } - await this.#invalidateSafely(protocol, session); + try { + await this.invalidate(protocol, session); + } catch (invalidationError) { + throw new AggregateError( + [error, invalidationError], + `Git ${protocol} operation and session invalidation both failed` + ); + } if (mayRetry && error instanceof GitProtocolError) { return await this.#attempt({ protocol, operation, recoverable, mayRetry: false }); } @@ -186,17 +184,6 @@ export default class GitObjectSessionPool { } } - async #invalidateSafely(protocol, session) { - if (session === undefined) { - return; - } - try { - await this.invalidate(protocol, session); - } catch { - // Preserve the operation failure; cleanup is best effort here. - } - } - #release(protocol) { const remaining = (this.#active.get(protocol) ?? 1) - 1; if (remaining === 0) { @@ -217,7 +204,15 @@ export default class GitObjectSessionPool { } let opening = this.#sessions.get(protocol); if (opening === undefined) { - await this.#idleRetirements.get(protocol); + let retirement = this.#retirements.get(protocol); + while (retirement !== undefined) { + await retirement.barrier; + const latest = this.#retirements.get(protocol); + if (latest === retirement) { + break; + } + retirement = latest; + } if (this.#closed) { throw new Error('Git object session pool is closed'); } @@ -242,6 +237,19 @@ export default class GitObjectSessionPool { } } + async #closeSession(protocol, session, message) { + try { + await session.close(); + } catch (closeError) { + try { + await this.#abort(protocol, session); + } catch (abortError) { + throw new AggregateError([closeError, abortError], message); + } + throw closeError; + } + } + #scheduleIdle(protocol) { if (this.#closed || this.#active.has(protocol) || !this.#sessions.has(protocol)) { return; @@ -260,30 +268,36 @@ export default class GitObjectSessionPool { } #trackIdleRetirement(protocol, opening) { - const retirement = opening.then(async (session) => { - try { - await session.close(); - } catch (closeError) { - try { - await this.#abort(protocol, session); - } catch (abortError) { - throw new AggregateError( - [closeError, abortError], - `Idle Git ${protocol} session failed to close or terminate` - ); - } - throw closeError; + this.#trackRetirement( + protocol, + async () => { + const session = await opening; + await this.#closeSession( + protocol, + session, + `Idle Git ${protocol} session failed to close or terminate` + ); + }, + true + ); + } + + #trackRetirement(protocol, operation, recordFailure = false) { + const previous = this.#retirements.get(protocol)?.barrier ?? Promise.resolve(); + const completion = previous.then(operation); + const barrier = completion.catch((error) => { + if (recordFailure) { + this.#idleFailures.set(protocol, error); } }); - const tracked = retirement.catch((error) => { - this.#idleFailures.set(protocol, error); - }); - this.#idleRetirements.set(protocol, tracked); - void tracked.finally(() => { - if (this.#idleRetirements.get(protocol) === tracked) { - this.#idleRetirements.delete(protocol); + const retirement = { barrier, completion }; + this.#retirements.set(protocol, retirement); + void barrier.finally(() => { + if (this.#retirements.get(protocol) === retirement) { + this.#retirements.delete(protocol); } }); + return completion; } #cancelIdle(protocol) { diff --git a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js index cf09e05..4313fa1 100644 --- a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js +++ b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js @@ -148,6 +148,54 @@ describe('GitPersistenceAdapter concurrent cat-file recovery', () => { }); }); +describe('GitPersistenceAdapter cat-file invalidation barrier', () => { + it('does not open a replacement before the failed process terminates', async () => { + const firstOid = '3'.repeat(40); + const secondOid = '4'.repeat(40); + const terminated = deferred(); + const failed = fakeCatSession({ + info: vi.fn().mockRejectedValue(new GitProtocolError('process closed', 'test')), + terminate: vi.fn().mockReturnValue(terminated.promise), + }); + const replacement = fakeCatSession({ + info: vi.fn(async (oid) => ({ oid, type: 'blob', size: 1 })), + }); + const plumbing = sessionPlumbing({ catSessions: [failed, replacement] }); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + + const firstRead = adapter.readObjectType(firstOid); + await vi.waitFor(() => expect(failed.terminate).toHaveBeenCalledTimes(1)); + const secondRead = adapter.readObjectType(secondOid); + await Promise.resolve(); + + expect(plumbing.openCatFileSession).toHaveBeenCalledTimes(1); + terminated.resolve(); + await expect(Promise.all([firstRead, secondRead])).resolves.toEqual(['blob', 'blob']); + expect(plumbing.openCatFileSession).toHaveBeenCalledTimes(2); + await adapter.close(); + }); +}); + +describe('GitPersistenceAdapter cat-file invalidation failures', () => { + it('preserves the operation and teardown failures without opening a replacement', async () => { + const operationError = new GitProtocolError('process closed', 'test'); + const terminationError = new Error('termination failed'); + const failed = fakeCatSession({ + info: vi.fn().mockRejectedValue(operationError), + terminate: vi.fn().mockRejectedValue(terminationError), + }); + const replacement = fakeCatSession(); + const plumbing = sessionPlumbing({ catSessions: [failed, replacement] }); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + + const failure = await adapter.readObjectType('5'.repeat(40)).catch((error) => error); + + expect(failure).toBeInstanceOf(AggregateError); + expect(failure.errors).toEqual([operationError, terminationError]); + expect(plumbing.openCatFileSession).toHaveBeenCalledTimes(1); + }); +}); + describe('GitPersistenceAdapter payload streaming', () => { it('keeps payload streaming on executeStream instead of buffering through cat-file', async () => { const oid = 'd'.repeat(40); @@ -365,6 +413,37 @@ describe('GitPersistenceAdapter cache session retirement failures', () => { }); }); +describe('GitPersistenceAdapter explicit retirement barrier', () => { + it('does not open a replacement while the previous process drains', async () => { + const firstOid = 'c'.repeat(40); + const secondOid = 'd'.repeat(40); + const retired = deferred(); + const cat = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid: firstOid, type: 'blob', size: 1 }), + close: vi.fn().mockReturnValue(retired.promise), + }); + const replacement = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid: secondOid, type: 'tree', size: 1 }), + }); + const plumbing = sessionPlumbing({ catSessions: [cat, replacement] }); + plumbing.execute.mockResolvedValue('e'.repeat(40)); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + + await adapter.readObjectType(firstOid); + const write = adapter.writeBlob(Buffer.from('value')); + await vi.waitFor(() => expect(cat.close).toHaveBeenCalledTimes(1)); + const nextRead = adapter.readObjectType(secondOid); + await Promise.resolve(); + + expect(plumbing.openCatFileSession).toHaveBeenCalledTimes(1); + retired.resolve(); + await expect(write).resolves.toBe('e'.repeat(40)); + await expect(nextRead).resolves.toBe('tree'); + expect(plumbing.openCatFileSession).toHaveBeenCalledTimes(2); + await adapter.close(); + }); +}); + describe('GitPersistenceAdapter bulk write recovery', () => { it('replays a materialized bulk input through a fresh typed session', async () => { const failed = { From f04a020b5120aa012f07b29d69a4490991c53b54 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 04:27:19 -0700 Subject: [PATCH 13/17] fix: complete Git process shutdown --- .../adapters/GitPersistenceAdapter.js | 19 +++++-- .../GitPersistenceAdapter.sessions.test.js | 51 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/infrastructure/adapters/GitPersistenceAdapter.js b/src/infrastructure/adapters/GitPersistenceAdapter.js index 6c6d7c5..3e31066 100644 --- a/src/infrastructure/adapters/GitPersistenceAdapter.js +++ b/src/infrastructure/adapters/GitPersistenceAdapter.js @@ -167,12 +167,14 @@ export default class GitPersistenceAdapter extends GitPersistencePort { await this.#sessions.retire('catFile'); return oid; } - return this.policy.execute(() => + const oid = await this.policy.execute(() => this.plumbing.execute({ args: ['mktree'], input: `${entries.join('\n')}\n`, }) ); + await this.#sessions.retire('catFile'); + return oid; }); } @@ -671,11 +673,22 @@ export default class GitPersistenceAdapter extends GitPersistencePort { } static async #closeStream(stream) { + const operations = []; if (typeof stream.destroy === 'function') { - await stream.destroy(); + operations.push(Promise.resolve().then(() => stream.destroy())); } if (stream.finished !== undefined) { - await stream.finished; + operations.push(Promise.resolve(stream.finished)); + } + const results = await Promise.allSettled(operations); + const failures = results + .filter((result) => result.status === 'rejected') + .map((result) => result.reason); + if (failures.length === 1) { + throw failures[0]; + } + if (failures.length > 1) { + throw new AggregateError(failures, 'Git output stream failed to terminate cleanly'); } } diff --git a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js index 4313fa1..c5cc445 100644 --- a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js +++ b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js @@ -501,6 +501,24 @@ describe('GitPersistenceAdapter persistent tree writes', () => { }); }); +describe('GitPersistenceAdapter fallback tree writes', () => { + it('retires the persistent reader after a one-shot tree write', async () => { + const blobOid = 'd'.repeat(40); + const treeOid = 'e'.repeat(40); + const cat = fakeCatSession({ + info: vi.fn().mockResolvedValue({ oid: blobOid, type: 'blob', size: 1 }), + }); + const plumbing = sessionPlumbing({ catSessions: [cat] }); + plumbing.execute.mockResolvedValue(treeOid); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + await adapter.readObjectType(blobOid); + + await expect(adapter.writeTree([])).resolves.toBe(treeOid); + + expect(cat.close).toHaveBeenCalledTimes(1); + }); +}); + describe('GitPersistenceAdapter lifecycle', () => { it('closes every opened session once and rejects later operations', async () => { const oid = 'e'.repeat(40); @@ -603,6 +621,39 @@ describe('GitPersistenceAdapter stream shutdown', () => { }); }); +describe('GitPersistenceAdapter failed stream shutdown', () => { + it('waits for process completion and preserves a destroy failure', async () => { + const destroyError = new Error('stream destroy failed'); + const finished = deferred(); + const stream = { + async *[Symbol.asyncIterator]() {}, + destroy: vi.fn().mockRejectedValue(destroyError), + finished: finished.promise, + }; + const plumbing = sessionPlumbing(); + plumbing.executeStream.mockResolvedValue(stream); + const adapter = new GitPersistenceAdapter({ plumbing, policy: noPolicy }); + await adapter.readBlobStream('4'.repeat(40)); + + let closeSettled = false; + const close = adapter.close().then( + () => ({ status: 'fulfilled' }), + (error) => ({ status: 'rejected', error }) + ); + void close.then(() => { + closeSettled = true; + }); + await vi.waitFor(() => expect(stream.destroy).toHaveBeenCalledTimes(1)); + + expect(closeSettled).toBe(false); + finished.resolve({ code: 1, stderr: 'terminated' }); + const outcome = await close; + expect(outcome).toMatchObject({ status: 'rejected' }); + expect(outcome.error).toBeInstanceOf(AggregateError); + expect(outcome.error.errors).toContain(destroyError); + }); +}); + describe('GitPersistenceAdapter failed session shutdown', () => { it('terminates a session whose graceful close fails before reporting the error', async () => { const oid = '4'.repeat(40); From 6ff3e1c78b3188c1772acb046a5bab123f5412ff Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 04:35:59 -0700 Subject: [PATCH 14/17] docs: qualify benchmark resource scope --- .../measure-git-object-sessions.js | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/scripts/diagnostics/measure-git-object-sessions.js b/scripts/diagnostics/measure-git-object-sessions.js index 3aa965c..cc53b87 100644 --- a/scripts/diagnostics/measure-git-object-sessions.js +++ b/scripts/diagnostics/measure-git-object-sessions.js @@ -77,6 +77,11 @@ function buildReport({ items, pageBytes, samples, reads, writes }) { platform: process.platform, architecture: process.arch, }, + metricScope: { + wallMs: 'worker elapsed time including awaited Git subprocesses', + workerCpuMs: 'Node worker CPU only; excludes Git subprocess CPU', + workerPeakRssBytes: 'Node worker peak RSS only; excludes Git subprocess RSS', + }, parameters: { items, pageBytes, samples }, selectedBundleRead: comparison({ left: reads.left, @@ -100,7 +105,7 @@ function comparison({ left, right, leftName, rightName }) { semanticDigestEqual: left.semanticDigest === right.semanticDigest, processReductionPercent: percentageReduction(left.processCount, right.processCount), wallReductionPercent: percentageReduction(left.wallMs, right.wallMs), - cpuReductionPercent: percentageReduction(left.cpuMs, right.cpuMs), + workerCpuReductionPercent: percentageReduction(left.workerCpuMs, right.workerCpuMs), }; } @@ -187,9 +192,9 @@ async function timed(operation) { const cpu = process.cpuUsage(startedCpu); return { wallMs: performance.now() - startedAt, - userCpuMs: cpu.user / 1000, - systemCpuMs: cpu.system / 1000, - peakRssBytes: process.resourceUsage().maxRSS * 1024, + workerUserCpuMs: cpu.user / 1000, + workerSystemCpuMs: cpu.system / 1000, + workerPeakRssBytes: process.resourceUsage().maxRSS * 1024, }; } @@ -254,10 +259,14 @@ function summarize(samples) { processCount: samples[0].processCount, counts: samples[0].counts, wallMs: roundedMedian(samples.map((sample) => sample.wallMs)), - cpuMs: roundedMedian(samples.map((sample) => sample.userCpuMs + sample.systemCpuMs)), - userCpuMs: roundedMedian(samples.map((sample) => sample.userCpuMs)), - systemCpuMs: roundedMedian(samples.map((sample) => sample.systemCpuMs)), - peakRssBytes: Math.round(median(samples.map((sample) => sample.peakRssBytes))), + workerCpuMs: roundedMedian( + samples.map((sample) => sample.workerUserCpuMs + sample.workerSystemCpuMs) + ), + workerUserCpuMs: roundedMedian(samples.map((sample) => sample.workerUserCpuMs)), + workerSystemCpuMs: roundedMedian(samples.map((sample) => sample.workerSystemCpuMs)), + workerPeakRssBytes: Math.round( + median(samples.map((sample) => sample.workerPeakRssBytes)) + ), }; } From 1819d8572707d8846ffa7ace9847f490cdb6438b Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 04:43:55 -0700 Subject: [PATCH 15/17] fix: block sessions after failed termination --- .../adapters/GitObjectSessionPool.js | 18 +++++++++++++++++- .../GitPersistenceAdapter.sessions.test.js | 2 ++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/infrastructure/adapters/GitObjectSessionPool.js b/src/infrastructure/adapters/GitObjectSessionPool.js index a099adc..7fd4633 100644 --- a/src/infrastructure/adapters/GitObjectSessionPool.js +++ b/src/infrastructure/adapters/GitObjectSessionPool.js @@ -22,6 +22,7 @@ export default class GitObjectSessionPool { #idleTimeoutMs; #idleTimers = new Map(); #plumbing; + #poisoned = new Map(); #retirements = new Map(); #sessions = new Map(); @@ -125,6 +126,7 @@ export default class GitObjectSessionPool { this.#cancelIdle(protocol); } const sessions = [...this.#sessions.entries()]; + const poisoned = new Map(this.#poisoned); const retirements = [...this.#retirements.values()].map((retirement) => retirement.barrier); this.#sessions.clear(); this.#closePromise = (async () => { @@ -143,6 +145,11 @@ export default class GitObjectSessionPool { .filter((result) => result.status === 'rejected') .map((result) => result.reason); failures.push(...this.#idleFailures.values()); + for (const [protocol, error] of poisoned) { + if (!this.#idleFailures.has(protocol)) { + failures.push(error); + } + } if (failures.length > 0) { throw new AggregateError(failures, 'One or more Git object sessions failed to close'); } @@ -198,6 +205,10 @@ export default class GitObjectSessionPool { if (this.#closed) { throw new Error('Git object session pool is closed'); } + const poison = this.#poisoned.get(protocol); + if (poison !== undefined) { + throw poison; + } const descriptor = PROTOCOLS[protocol]; if (descriptor === undefined || typeof this.#plumbing[descriptor.opener] !== 'function') { throw new Error(`Git object protocol is unavailable: ${protocol}`); @@ -233,7 +244,12 @@ export default class GitObjectSessionPool { async #abort(protocol, session) { const method = PROTOCOLS[protocol]?.abort; if (method !== undefined && typeof session[method] === 'function') { - await session[method](); + try { + await session[method](); + } catch (error) { + this.#poisoned.set(protocol, error); + throw error; + } } } diff --git a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js index c5cc445..c766492 100644 --- a/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js +++ b/test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js @@ -192,6 +192,8 @@ describe('GitPersistenceAdapter cat-file invalidation failures', () => { expect(failure).toBeInstanceOf(AggregateError); expect(failure.errors).toEqual([operationError, terminationError]); + await expect(adapter.readObjectType('6'.repeat(40))).rejects.toBe(terminationError); + await expect(adapter.close()).rejects.toBeInstanceOf(AggregateError); expect(plumbing.openCatFileSession).toHaveBeenCalledTimes(1); }); }); From fa61b1957e0cf026accad2c88dbc9e8cd6f51d25 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 04:52:13 -0700 Subject: [PATCH 16/17] docs: finalize persistent session witness --- .../persistent-git-object-sessions.md | 24 ++- .../witness/git-object-sessions.json | 55 +++--- .../witness/verification.md | 174 ++++++++++-------- .../measure-git-object-sessions.js | 4 +- 4 files changed, 145 insertions(+), 112 deletions(-) diff --git a/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md b/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md index bac7d89..808c729 100644 --- a/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md +++ b/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md @@ -205,7 +205,8 @@ Callers that cannot use explicit resource management call `await cas.close()`. ## Cost / Residency Posture -- At most one live session exists per supported protocol per adapter. +- At most one live session exists per supported protocol per adapter. A failed + forced termination poisons that protocol and blocks replacement processes. - Session queues serialize operations and do not duplicate response buffers. - Parsed trees are bounded by entry count and estimated retained bytes. - A tree larger than the bounded cat-file read budget falls back to streaming @@ -251,12 +252,16 @@ the normal lifecycle contract. Persisted repositories need no migration. - Invalidation is generation-safe: a late failure from an old process may terminate that process, but cannot evict a replacement session opened by a concurrent retry. -- A typed protocol-process failure receives one retry through a fresh session. - These operations are content-addressed and idempotent; bulk inputs are - materialized once before the first attempt so a retry sees the same sequence. +- A typed protocol-process failure receives one retry through a fresh session + only after invalidation and forced termination succeed. These operations are + content-addressed and idempotent; bulk inputs are materialized once before + the first attempt so a retry sees the same sequence. +- Failed forced termination poisons the protocol for the adapter lifetime. + Later operations remain blocked and explicit close reports the unresolved + teardown instead of opening beside a potentially live process. - Close attempts every opened session and reports aggregate cleanup failure. A graceful close or retirement failure force-terminates the affected process - before the failure is surfaced. + before the failure is surfaced. Independent cleanup failures are preserved. ## Security / Trust / Redaction Posture @@ -342,7 +347,8 @@ blob writes one-shot and streaming payloads on the existing stream path. - real-Git individual-versus-batch write process comparison - existing large-stream restore and page-cache tests - Node, Bun, and Deno suites -- committed JSON witness with counts, timing, and peak memory +- committed JSON witness with counts, wall timing, and explicitly scoped Node + worker CPU and peak-memory observations ## Implementation Slices @@ -370,6 +376,10 @@ blob writes one-shot and streaming payloads on the existing stream path. output stream before resolving. 11. A late failure from an old session cannot invalidate a concurrently opened replacement, and failed graceful retirement aborts before rejecting. +12. Explicit retirement and invalidation block replacement process creation + until teardown completes. +13. Failed forced termination blocks all later sessions for that protocol and + remains visible to explicit close. ## Acceptance Criteria @@ -412,6 +422,8 @@ The witness packet must answer: - Idle retirement bounds reusable-session leaks when callers omit close, but an abandoned output stream still requires close or eventual process completion; docs and async disposal reduce but cannot eliminate misuse. +- The diagnostic's CPU and RSS fields cover the Node worker, not Git children; + git-warp must add process-tree CPU and large-graph memory gates before v19. ## Follow-On Debt diff --git a/docs/design/0052-persistent-git-object-sessions/witness/git-object-sessions.json b/docs/design/0052-persistent-git-object-sessions/witness/git-object-sessions.json index 02a5d90..a3f51c1 100644 --- a/docs/design/0052-persistent-git-object-sessions/witness/git-object-sessions.json +++ b/docs/design/0052-persistent-git-object-sessions/witness/git-object-sessions.json @@ -1,12 +1,17 @@ { "schema": "git-cas.git-object-session-measurement/v1", - "generatedAt": "2026-07-19T10:46:01.178Z", + "generatedAt": "2026-07-19T11:35:28.075Z", "environment": { "node": "v22.23.1", "git": "git version 2.43.0", "platform": "linux", "architecture": "arm64" }, + "metricScope": { + "wallMs": "worker elapsed time including awaited Git subprocesses", + "workerCpuMs": "Node worker CPU only; excludes Git subprocess CPU", + "workerPeakRssBytes": "Node worker peak RSS only; excludes Git subprocess RSS" + }, "parameters": { "items": 32, "pageBytes": 4096, @@ -23,11 +28,11 @@ "ls-tree": 128, "cat-file": 33 }, - "wallMs": 343.911, - "cpuMs": 277.441, - "userCpuMs": 175.93, - "systemCpuMs": 107.204, - "peakRssBytes": 79433728 + "wallMs": 288.647, + "workerCpuMs": 234.386, + "workerUserCpuMs": 136.508, + "workerSystemCpuMs": 105.143, + "workerPeakRssBytes": 79306752 }, "session": { "sampleCount": 3, @@ -37,16 +42,16 @@ "counts": { "session:cat-file": 1 }, - "wallMs": 54.038, - "cpuMs": 61.958, - "userCpuMs": 55.143, - "systemCpuMs": 6.815, - "peakRssBytes": 73871360 + "wallMs": 50.366, + "workerCpuMs": 59.11, + "workerUserCpuMs": 53.206, + "workerSystemCpuMs": 8.256, + "workerPeakRssBytes": 74203136 }, "semanticDigestEqual": true, "processReductionPercent": 99.6, - "wallReductionPercent": 84.3, - "cpuReductionPercent": 77.7 + "wallReductionPercent": 82.6, + "workerCpuReductionPercent": 74.8 }, "pageWrite": { "individual": { @@ -57,11 +62,11 @@ "counts": { "hash-object": 32 }, - "wallMs": 76.503, - "cpuMs": 71.906, - "userCpuMs": 50.053, - "systemCpuMs": 21.853, - "peakRssBytes": 73334784 + "wallMs": 68.658, + "workerCpuMs": 64.342, + "workerUserCpuMs": 49.411, + "workerSystemCpuMs": 14.931, + "workerPeakRssBytes": 73883648 }, "batch": { "sampleCount": 3, @@ -71,15 +76,15 @@ "counts": { "session:fast-import": 1 }, - "wallMs": 35, - "cpuMs": 19.785, - "userCpuMs": 16.663, - "systemCpuMs": 3.122, - "peakRssBytes": 72331264 + "wallMs": 33.49, + "workerCpuMs": 18.473, + "workerUserCpuMs": 15.18, + "workerSystemCpuMs": 2.472, + "workerPeakRssBytes": 72445952 }, "semanticDigestEqual": true, "processReductionPercent": 96.9, - "wallReductionPercent": 54.3, - "cpuReductionPercent": 72.5 + "wallReductionPercent": 51.2, + "workerCpuReductionPercent": 71.3 } } diff --git a/docs/design/0052-persistent-git-object-sessions/witness/verification.md b/docs/design/0052-persistent-git-object-sessions/witness/verification.md index 0817e96..94d4276 100644 --- a/docs/design/0052-persistent-git-object-sessions/witness/verification.md +++ b/docs/design/0052-persistent-git-object-sessions/witness/verification.md @@ -1,130 +1,142 @@ # Persistent Git Object Sessions Verification Witness -Generated: 2026-07-19 03:46:01 PDT +Generated: 2026-07-19 04:50:45 PDT Issue: [#90](https://github.com/git-stunts/git-cas/issues/90) Implementation commit: `27831926327afc7522b39ab435d29b46b7ac428e` -Compatibility repair commit: `a2ce917fb629f63db643de7cf72035386f48345e` - -Session recovery repair commit: `76e0cf57dfe8a1d0a0af49fb1c8a5ef1915fafb5` - -Session retirement repair commit: `6f29be5a8637b6010ca9497924695ec49c28d1f8` +Final reviewed source commit: `1819d8572707d8846ffa7ace9847f490cdb6438b` ## Contract `GitPersistenceAdapter` feature-detects typed plumbing sessions, keeps metadata and parsed tree residency bounded, and preserves command-per-operation fallback -for injected plumbing implementations without those capabilities. +for injected plumbing implementations without session capabilities. + +[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#41-86@1819d8572707d8846ffa7ace9847f490cdb6438b`] + +The session pool owns at most one live process per protocol during normal +operation. Opening is coalesced, and new sessions wait for explicit retirement, +idle retirement, or invalidation to finish. One typed protocol failure receives +one retry only after teardown succeeds. A late failure from an old generation +cannot evict its replacement. -[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#41-81@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#17-117@1819d8572707d8846ffa7ace9847f490cdb6438b`] +[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#153-225@1819d8572707d8846ffa7ace9847f490cdb6438b`] +[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#244-316@1819d8572707d8846ffa7ace9847f490cdb6438b`] -The session pool lazily coalesces one process per protocol, invalidates only the -exact poisoned session generation, retries one typed protocol failure through -a fresh process, retires idle sessions, and aggregates explicit close failures. -A late failure from an old process cannot evict its replacement. Graceful close -or retirement failure force-terminates the affected process before surfacing. +Graceful retirement failures force termination before surfacing. If an +operation and teardown both fail, both causes survive in an `AggregateError`; +the pool does not open a speculative replacement after failed teardown. -[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#16-59@27831926327afc7522b39ab435d29b46b7ac428e`] -[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#62-145@27831926327afc7522b39ab435d29b46b7ac428e`] -[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#196-226@27831926327afc7522b39ab435d29b46b7ac428e`] -[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#69-153@76e0cf57dfe8a1d0a0af49fb1c8a5ef1915fafb5`] -[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#89-113@6f29be5a8637b6010ca9497924695ec49c28d1f8`] +[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#163-180@1819d8572707d8846ffa7ace9847f490cdb6438b`] +[cite: `src/infrastructure/adapters/GitObjectSessionPool.js#233-251@1819d8572707d8846ffa7ace9847f490cdb6438b`] -The low-level persistence-port declarations keep batch write and lifecycle +Low-level persistence-port declarations keep batch write and lifecycle capabilities optional for existing structural adapters. The concrete Git -adapter and root facade expose them as required capabilities. +adapter and root facade expose deterministic close and async disposal. -[cite: `index.d.ts#513-581@a2ce917fb629f63db643de7cf72035386f48345e`] +[cite: `index.d.ts#513-581@1819d8572707d8846ffa7ace9847f490cdb6438b`] +[cite: `index.d.ts#1723-1742@1819d8572707d8846ffa7ace9847f490cdb6438b`] A Deno type-check fixture assigns a legacy structural persistence adapter that -does not implement any new optional capability, proving patch-release source -compatibility. +does not implement any new optional capability, proving source compatibility. -[cite: `test/types/public-api-compatibility.ts#1-47@a2ce917fb629f63db643de7cf72035386f48345e`] +[cite: `test/types/public-api-compatibility.ts#10-47@1819d8572707d8846ffa7ace9847f490cdb6438b`] Raw Git tree decoding supports SHA-1 and SHA-256 object identifiers, validates modes and names, freezes decoded entries, and computes an estimated residency weight for the byte-bounded cache. -[cite: `src/infrastructure/codecs/GitTreeObjectCodec.js#15-61@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `src/infrastructure/codecs/GitTreeObjectCodec.js#21-92@1819d8572707d8846ffa7ace9847f490cdb6438b`] ## Write Correctness Boundary Individual `writeBlob()` calls remain one-shot `hash-object` operations. Bounded `writeBlobs()` calls materialize their iterable once, write through one -scoped fast-import process, checkpoint it, and retire it before exposing OIDs. +scoped `fast-import` process, checkpoint it, retire dependent readers, and +close it before exposing OIDs. Multiple retirement failures are preserved. -[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#89-134@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#89-158@1819d8572707d8846ffa7ace9847f490cdb6438b`] +[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#545-570@1819d8572707d8846ffa7ace9847f490cdb6438b`] The one-shot boundary is required. The real-Git regression aggressively prunes an individually written unreachable blob, proves the object is absent, writes the same bytes again, and proves the same OID exists and remains readable. -[cite: `test/integration/bundle-reference-performance.test.js#242-269@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `test/integration/bundle-reference-performance.test.js#242-269@1819d8572707d8846ffa7ace9847f490cdb6438b`] -`pages.putBatch()` validates page count and aggregate bytes, collects the whole -explicitly bounded batch before persistence starts, and preserves input order -in its immutable staged results. +`pages.putBatch()` enforces page-count, per-page, and aggregate-byte bounds +while consuming each source. It stops an overflowing stream at the first chunk +that proves the bound, performs no persistence before the full bounded batch is +valid, and preserves input order in immutable staged results. -[cite: `src/domain/services/PageService.js#69-118@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `src/domain/services/PageService.js#78-127@1819d8572707d8846ffa7ace9847f490cdb6438b`] +[cite: `src/domain/services/PageService.js#236-264@1819d8572707d8846ffa7ace9847f490cdb6438b`] ## Lifecycle Every adapter operation registers before asynchronous work starts. `close()` -blocks new operations, drains started commands, destroys abandoned Git output -streams, waits for the corresponding processes, closes typed sessions, and -releases bounded cache residency. +blocks new work, drains started commands, destroys abandoned output streams, +waits for the corresponding Git children even when destruction fails, closes +typed sessions, and releases bounded cache residency. The fallback tree-write +path also retires a persistent reader after object-database mutation. -[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#497-552@27831926327afc7522b39ab435d29b46b7ac428e`] -[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#642-671@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#160-180@1819d8572707d8846ffa7ace9847f490cdb6438b`] +[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#499-558@1819d8572707d8846ffa7ace9847f490cdb6438b`] +[cite: `src/infrastructure/adapters/GitPersistenceAdapter.js#649-694@1819d8572707d8846ffa7ace9847f490cdb6438b`] -Unit tests prove idempotent closure, post-close rejection, active one-shot -draining, abandoned-stream destruction, and idle retirement when explicit -close is omitted. Concurrency coverage proves that a late old-session failure -cannot invalidate its replacement. Shutdown coverage proves that graceful -close and scoped retirement failures force termination before rejection. +Concurrency tests prove invalidation and explicit retirement barriers prevent a +replacement process from opening before the old process terminates. Failure +tests preserve operation, cache-retirement, bulk-retirement, and stream-cleanup +causes. Idle retirement is awaited by explicit close and its failures remain +visible after forced termination. -[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#315-354@27831926327afc7522b39ab435d29b46b7ac428e`] -[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#357-437@27831926327afc7522b39ab435d29b46b7ac428e`] -[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#121-148@76e0cf57dfe8a1d0a0af49fb1c8a5ef1915fafb5`] -[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#447-463@76e0cf57dfe8a1d0a0af49fb1c8a5ef1915fafb5`] -[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#261-305@6f29be5a8637b6010ca9497924695ec49c28d1f8`] +[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#151-203@1819d8572707d8846ffa7ace9847f490cdb6438b`] +[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#356-445@1819d8572707d8846ffa7ace9847f490cdb6438b`] +[cite: `test/unit/infrastructure/adapters/GitPersistenceAdapter.sessions.test.js#597-767@1819d8572707d8846ffa7ace9847f490cdb6438b`] + +The root facade closes lazily and idempotently without initializing an unused +store. Closing releases local resources only; stored objects, refs, retention, +and publication state remain unchanged. + +[cite: `index.js#620-645@1819d8572707d8846ffa7ace9847f490cdb6438b`] +[cite: `test/unit/facade/ContentAddressableStore.lifecycle.test.js#1-33@1819d8572707d8846ffa7ace9847f490cdb6438b`] ## Stable Performance Gates -The real-Git integration test requires the session path to return the same -selected reference while opening exactly one cat-file session and fewer total -Git processes than fallback. A separate gate requires batch and individual page -writes to return the same ordered handles while batch opens exactly one -fast-import session and fewer total processes. +The real-Git integration gate requires the persistent path to return the same +selected reference while opening exactly one `cat-file` session and fewer +total Git processes than fallback. A separate gate requires batch and +individual page writes to return the same ordered handles while batch opens +exactly one `fast-import` session and fewer total processes. -[cite: `test/integration/bundle-reference-performance.test.js#190-239@27831926327afc7522b39ab435d29b46b7ac428e`] +[cite: `test/integration/bundle-reference-performance.test.js#190-239@1819d8572707d8846ffa7ace9847f490cdb6438b`] ## Verification Results | Command | Result | | ------------------------------------------------- | -------------------------------------- | -| Focused lifecycle, session, codec, and page units | 4 files; 50 tests passed | -| `pnpm test` | 223 files; 2,070 passed, 2 skipped | +| Focused lifecycle, session, codec, and page units | 4 files; 59 tests passed | +| `pnpm test` | 223 files; 2,079 passed, 2 skipped | | `pnpm test:integration:node` | 12 files; 196 tests passed in Docker | | `bats test/platform/runtimes.bats` | Node, Bun, and Deno full suites passed | | Deno public type compatibility check | Passed with legacy structural adapter | | `pnpm lint` | Passed | | `git diff --check` | Passed | -The direct host `pnpm test:integration` invocation was intentionally rejected -by all 12 integration files because this repository requires -`GIT_STUNTS_DOCKER=1`. The required Docker invocation above passed; no host -result is counted as integration evidence. +All integration evidence came from the required Docker environment. Host-only +integration runs are not counted. ## Measurement Method The committed diagnostic creates isolated repositories and forked workers, alternates comparison order across three samples, verifies equal semantic -digests, counts every one-shot command and typed session opening, and reports -median wall time, process CPU, and peak RSS. +digests, and counts every one-shot command and typed session opening. + +[cite: `scripts/diagnostics/measure-git-object-sessions.js#27-105@1819d8572707d8846ffa7ace9847f490cdb6438b`] +[cite: `scripts/diagnostics/measure-git-object-sessions.js#136-267@1819d8572707d8846ffa7ace9847f490cdb6438b`] ```sh docker compose run --build --rm test-node \ @@ -135,32 +147,38 @@ The environment was Linux arm64 with Node `v22.23.1` and Git `2.43.0`. The raw machine-readable result is [`git-object-sessions.json`](./git-object-sessions.json). +Wall time includes awaited Git subprocesses. CPU and peak-RSS fields are +explicitly named `workerCpuMs` and `workerPeakRssBytes`: they measure the Node +worker only and exclude Git child CPU and RSS. They are diagnostic observations, +not end-to-end resource claims. + ## Measurement Results | Path | Baseline processes | Candidate processes | Process reduction | Baseline wall | Candidate wall | Observed wall reduction | | ------------------------------- | -----------------: | ------------------: | ----------------: | ------------: | -------------: | ----------------------: | -| Selected bundle reference reads | 225 | 1 | 99.6% | 343.911 ms | 54.038 ms | 84.3% | -| Bounded page writes | 32 | 1 | 96.9% | 76.503 ms | 35.000 ms | 54.3% | +| Selected bundle reference reads | 225 | 1 | 99.6% | 288.647 ms | 50.366 ms | 82.6% | +| Bounded page writes | 32 | 1 | 96.9% | 68.658 ms | 33.490 ms | 51.2% | Selected reads had equal semantic digests and returned 32 results in both -modes. The session path opened one cat-file process; fallback opened 64 -batch-check, 128 ls-tree, and 33 cat-file processes. Median process CPU fell -from 277.441 ms to 61.958 ms. Peak RSS was 79,433,728 bytes for fallback and -73,871,360 bytes for the session path. +modes. The session path opened one `cat-file` process; fallback opened 64 +batch-check, 128 `ls-tree`, and 33 `cat-file` processes. Node-worker CPU was +234.386 ms for fallback and 59.110 ms for sessions. Node-worker peak RSS was +79,306,752 bytes and 74,203,136 bytes, respectively. Page writes had equal semantic digests and returned 32 handles in both modes. -Individual writes opened 32 hash-object processes; the batch opened one scoped -fast-import process. Median process CPU fell from 71.906 ms to 19.785 ms. Peak -RSS was 73,334,784 bytes for individual writes and 72,331,264 bytes for batch. +Individual writes opened 32 `hash-object` processes; the batch opened one +scoped `fast-import` process. Node-worker CPU was 64.342 ms for individual +writes and 18.473 ms for batch. Node-worker peak RSS was 73,883,648 bytes and +72,445,952 bytes, respectively. -Timing and RSS are environment observations, not portable guarantees. Semantic -digest equality and process counts are the stable regression contracts. +Semantic digest equality and process counts are stable regression contracts. +Timing and worker-resource observations are environment-specific evidence. ## Residual Scope -This witness does not prove graph-wide memory boundedness, git-warp application -latency, or elimination of all subprocess cost. Payload streams remain -one-shot and streaming by design. Git-warp must adopt the published git-cas -release, hold cache acquisitions at the correct runtime lifetime, and prove its -own large-graph memory and latency benchmarks before the v19 performance -campaign can close. +This witness does not prove graph-wide memory boundedness, end-to-end +process-tree CPU, git-warp application latency, or elimination of every +subprocess. Payload streams remain one-shot and streaming by design. Git-warp +must adopt the published git-cas release, hold cache acquisitions at the +correct runtime lifetime, and pass its own large-graph memory, process-tree CPU, +and latency gates before the v19 performance campaign can close. diff --git a/scripts/diagnostics/measure-git-object-sessions.js b/scripts/diagnostics/measure-git-object-sessions.js index cc53b87..ea48ad2 100644 --- a/scripts/diagnostics/measure-git-object-sessions.js +++ b/scripts/diagnostics/measure-git-object-sessions.js @@ -264,9 +264,7 @@ function summarize(samples) { ), workerUserCpuMs: roundedMedian(samples.map((sample) => sample.workerUserCpuMs)), workerSystemCpuMs: roundedMedian(samples.map((sample) => sample.workerSystemCpuMs)), - workerPeakRssBytes: Math.round( - median(samples.map((sample) => sample.workerPeakRssBytes)) - ), + workerPeakRssBytes: Math.round(median(samples.map((sample) => sample.workerPeakRssBytes))), }; } From efce7879348072a25759996dd159e35b638c4035 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 19 Jul 2026 06:50:05 -0700 Subject: [PATCH 17/17] fix: close review resource gaps --- BEARING.md | 2 +- README.md | 5 +- ROADMAP.md | 4 +- STATUS.md | 2 +- docs/API.md | 2 +- .../persistent-git-object-sessions.md | 13 +- .../diagnostics/createCountingGitPlumbing.js | 48 +++++++ .../measure-git-object-sessions.js | 74 ++++------ .../bundle-reference-performance.test.js | 130 ++++++++---------- 9 files changed, 142 insertions(+), 138 deletions(-) create mode 100644 scripts/diagnostics/createCountingGitPlumbing.js diff --git a/BEARING.md b/BEARING.md index 6d56e6c..28ce182 100644 --- a/BEARING.md +++ b/BEARING.md @@ -130,7 +130,7 @@ With v6.5.1 shipped and the v6.5.2 performance candidate under review, active work is tracked in GitHub Issues and Milestones. Repo docs hold design and evidence records, not the active queue. -The latest landed design record is +The current design under review is [0052-persistent-git-object-sessions](./docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md). Its release evidence is attached to [#90](https://github.com/git-stunts/git-cas/issues/90) and the diff --git a/README.md b/README.md index c2224f9..44cb480 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,9 @@ Unlike traditional LFS which moves files to external servers, `git-cas` treats t objects, trees, or payload OIDs directly. - **Bounded Git Process Reuse**: Immutable object metadata and tree reads reuse typed Git sessions behind the adapter, while explicitly bounded page batches - amortize writes without changing content identity or buffering payload - streams. + amortize writes without changing content identity. Read and payload streams + remain streaming; complete page-batch inputs are collected and validated only + within the explicit count-and-byte envelope. - **Key Lifecycle**: Envelope encryption separates DEKs from KEKs. Rotate passphrases across an entire vault without re-encrypting data blobs. Privacy mode HMAC-hashes slug names to prevent metadata discovery. - **Runtime-Adaptive**: A single core supports Node.js 22+, Bun, and Deno through a strict hexagonal port architecture with runtime-specific crypto adapters. diff --git a/ROADMAP.md b/ROADMAP.md index ba0db9e..8e57392 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -44,9 +44,9 @@ GitHub wins and this file should be corrected. | [`v6.7.0`](https://github.com/git-stunts/git-cas/milestone/10) | Browser and edge read-path exploration | [#41](https://github.com/git-stunts/git-cas/issues/41) | | [`v7.0.0`](https://github.com/git-stunts/git-cas/milestone/6) | Protocol break only if audit requires it | [#42](https://github.com/git-stunts/git-cas/issues/42), only when justified | -## Latest Landed Design +## Current Design -The latest landed design record is: +The current active design record is: - [0052-persistent-git-object-sessions](./docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md) diff --git a/STATUS.md b/STATUS.md index 2d8c4a9..4aa7e86 100644 --- a/STATUS.md +++ b/STATUS.md @@ -130,7 +130,7 @@ [#40 v6.6.0: Agent automation follow-through](https://github.com/git-stunts/git-cas/issues/40) under the [`v6.6.0` milestone](https://github.com/git-stunts/git-cas/milestone/9). -- The latest landed design record is +- The latest active design record is [0052-persistent-git-object-sessions](./docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md). ## Read Next diff --git a/docs/API.md b/docs/API.md index 4558737..1cfc9a5 100644 --- a/docs/API.md +++ b/docs/API.md @@ -3374,7 +3374,7 @@ new CasError({ message, code, meta, documentationUrl }); | `GIT_ERROR` | Underlying Git plumbing command failed | `readManifest()`, `inspectAsset()`, `collectReferencedChunks()` | | `GIT_REF_NOT_FOUND` | Git ref lookup found no ref; vault reads normalize this to empty state | `GitRefAdapter`, `VaultPersistence` | | `INVALID_OPTIONS` | Mutually exclusive options provided or unsupported option value | `store()`, `restore()` | -| `RESOURCE_CLOSED` | A facade or persistence adapter operation began after local resources were closed | `ContentAddressableStore`, `GitPersistenceAdapter` | +| `RESOURCE_CLOSED` | A facade or persistence adapter operation began while or after closure was in progress | `ContentAddressableStore`, `GitPersistenceAdapter` | | `HANDLE_INVALID` | Handle object, canonical token, or staged result is malformed or unsupported | Application handles and staged results | | `HANDLE_KIND_MISMATCH` | A handle has a valid envelope but the wrong content kind | Handle parsing and application storage | | `HANDLE_CODEC_MISMATCH` | Handle manifest codec differs from the active CAS codec | `assets.open()`, `assets.adopt()`, retention, publication | diff --git a/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md b/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md index 808c729..e9fd2b0 100644 --- a/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md +++ b/docs/design/0052-persistent-git-object-sessions/persistent-git-object-sessions.md @@ -144,7 +144,10 @@ This cycle does not include: class ContentAddressableStore { pages: { putBatch(options: { - pages: Array<{ source: Uint8Array | Iterable }>; + pages: Array<{ + source: Uint8Array | Iterable; + maxBytes?: number; + }>; maxBatchBytes?: number; maxBatchPages?: number; }): Promise>; @@ -239,9 +242,11 @@ never cached. No new refs or object formats are introduced. ## Compatibility / Migration Posture This is additive. Injected plumbing doubles without typed-session methods keep -the legacy path. Existing callers are not required to call `close()` unless a -session-capable operation was used, but documentation treats explicit close as -the normal lifecycle contract. Persisted repositories need no migration. +the legacy path. Existing callers remain source-compatible, but explicit +`close()` is the normal lifecycle contract and is required whenever operations +may remain active or streams may remain unconsumed. That includes fallback +`readBlobStream()` operations and their child processes, not only typed +sessions. Persisted repositories need no migration. ## Error Contract diff --git a/scripts/diagnostics/createCountingGitPlumbing.js b/scripts/diagnostics/createCountingGitPlumbing.js new file mode 100644 index 0000000..08132ec --- /dev/null +++ b/scripts/diagnostics/createCountingGitPlumbing.js @@ -0,0 +1,48 @@ +import { createGitPlumbing } from '../../src/infrastructure/createGitPlumbing.js'; + +export async function createCountingGitPlumbing({ cwd, sessions = false }) { + const plumbing = await createGitPlumbing({ cwd }); + const counts = new Map(); + const record = (operation) => counts.set(operation, (counts.get(operation) ?? 0) + 1); + const counted = { + execute(options) { + record(operationOf(options.args)); + return plumbing.execute(options); + }, + executeStream(options) { + record(operationOf(options.args)); + return plumbing.executeStream(options); + }, + }; + + if (sessions) { + counted.openCatFileSession = sessionOpener(plumbing, record, 'cat-file'); + counted.openMktreeSession = sessionOpener(plumbing, record, 'mktree'); + counted.openFastImportSession = sessionOpener(plumbing, record, 'fast-import'); + } + + return { + plumbing: counted, + snapshot: () => new Map(counts), + }; +} + +function sessionOpener(plumbing, record, protocol) { + const method = + protocol === 'cat-file' + ? 'openCatFileSession' + : protocol === 'mktree' + ? 'openMktreeSession' + : 'openFastImportSession'; + return (...args) => { + record(`session:${protocol}`); + return plumbing[method](...args); + }; +} + +function operationOf(args) { + if (args[0] === 'cat-file' && args.some((arg) => arg.startsWith('--batch-check='))) { + return 'cat-file:batch-check'; + } + return args[0]; +} diff --git a/scripts/diagnostics/measure-git-object-sessions.js b/scripts/diagnostics/measure-git-object-sessions.js index ea48ad2..25f221f 100644 --- a/scripts/diagnostics/measure-git-object-sessions.js +++ b/scripts/diagnostics/measure-git-object-sessions.js @@ -8,6 +8,7 @@ import { performance } from 'node:perf_hooks'; import { fileURLToPath } from 'node:url'; import ContentAddressableStore from '../../index.js'; import { createGitPlumbing } from '../../src/infrastructure/createGitPlumbing.js'; +import { createCountingGitPlumbing } from './createCountingGitPlumbing.js'; const WORKER = '--worker'; const DEFAULT_ITEMS = 32; @@ -139,37 +140,46 @@ async function emitWorkerResult(options) { } async function measureReads({ mode, repo, handles }) { - const counted = await countedPlumbing(repo, { sessions: mode === 'session' }); + const counted = await createCountingGitPlumbing({ + cwd: repo, + sessions: mode === 'session', + }); const cas = new ContentAddressableStore({ plumbing: counted.plumbing }); const values = []; const metrics = await timed(async () => { - for (const handle of handles) { - const reference = await cas.bundles.getMemberReference({ handle, path: 'nested' }); - values.push(`${reference.path}\0${reference.type}\0${reference.handle.toString()}`); + try { + for (const handle of handles) { + const reference = await cas.bundles.getMemberReference({ handle, path: 'nested' }); + values.push(`${reference.path}\0${reference.type}\0${reference.handle.toString()}`); + } + } finally { + await cas.close(); } - await cas.close(); }); - return resultEnvelope(values, counted.snapshot(), metrics); + return resultEnvelope(values, Object.fromEntries(counted.snapshot()), metrics); } async function measureWrites({ mode, items, pageBytes }) { const repo = mkdtempSync(path.join(os.tmpdir(), `cas-${mode}-write-`)); try { initBare(repo); - const counted = await countedPlumbing(repo, { sessions: true }); + const counted = await createCountingGitPlumbing({ cwd: repo, sessions: true }); const cas = new ContentAddressableStore({ plumbing: counted.plumbing }); const sources = Array.from({ length: items }, (_, index) => pageSource(index, pageBytes)); let pages; const metrics = await timed(async () => { - pages = - mode === 'batch' - ? await cas.pages.putBatch({ pages: sources.map((source) => ({ source })) }) - : await putIndividually(cas, sources); - await cas.close(); + try { + pages = + mode === 'batch' + ? await cas.pages.putBatch({ pages: sources.map((source) => ({ source })) }) + : await putIndividually(cas, sources); + } finally { + await cas.close(); + } }); return resultEnvelope( pages.map((page) => page.handle.toString()), - counted.snapshot(), + Object.fromEntries(counted.snapshot()), metrics ); } finally { @@ -198,44 +208,6 @@ async function timed(operation) { }; } -async function countedPlumbing(repo, { sessions }) { - const plumbing = await createGitPlumbing({ cwd: repo }); - const counts = new Map(); - const record = (name) => counts.set(name, (counts.get(name) ?? 0) + 1); - const counted = { - execute(options) { - record(operationOf(options.args)); - return plumbing.execute(options); - }, - executeStream(options) { - record(operationOf(options.args)); - return plumbing.executeStream(options); - }, - }; - if (sessions) { - counted.openCatFileSession = sessionOpener(plumbing, counts, 'cat-file'); - counted.openMktreeSession = sessionOpener(plumbing, counts, 'mktree'); - counted.openFastImportSession = sessionOpener(plumbing, counts, 'fast-import'); - } - return { plumbing: counted, snapshot: () => Object.fromEntries(counts) }; -} - -function sessionOpener(plumbing, counts, protocol) { - const method = `open${protocol === 'cat-file' ? 'CatFile' : protocol === 'mktree' ? 'Mktree' : 'FastImport'}Session`; - return (...args) => { - const key = `session:${protocol}`; - counts.set(key, (counts.get(key) ?? 0) + 1); - return plumbing[method](...args); - }; -} - -function operationOf(args) { - if (args[0] === 'cat-file' && args.some((arg) => arg.startsWith('--batch-check='))) { - return 'cat-file:batch-check'; - } - return args[0]; -} - function resultEnvelope(values, counts, metrics) { return { semanticDigest: digest(values), diff --git a/test/integration/bundle-reference-performance.test.js b/test/integration/bundle-reference-performance.test.js index 3e1cc39..d6797f0 100644 --- a/test/integration/bundle-reference-performance.test.js +++ b/test/integration/bundle-reference-performance.test.js @@ -11,6 +11,7 @@ import { spawnSync } from 'node:child_process'; import os from 'node:os'; import path from 'node:path'; import ContentAddressableStore from '../../index.js'; +import { createCountingGitPlumbing } from '../../scripts/diagnostics/createCountingGitPlumbing.js'; import { createGitPlumbing } from '../../src/infrastructure/createGitPlumbing.js'; if (process.env.GIT_STUNTS_DOCKER !== '1') { @@ -37,52 +38,27 @@ function git(args) { return result.stdout.trim(); } -function operationOf(args) { - if (args[0] === 'cat-file' && args.some((arg) => arg.startsWith('--batch-check='))) { - return 'cat-file:batch-check'; - } - return args[0]; -} - async function countingReader({ sessions = false } = {}) { - const plumbing = await createGitPlumbing({ cwd: repoDir }); - const counts = new Map(); - const record = (options) => { - const operation = operationOf(options.args); - counts.set(operation, (counts.get(operation) ?? 0) + 1); - }; - const counted = { - execute(options) { - record(options); - return plumbing.execute(options); - }, - executeStream(options) { - record(options); - return plumbing.executeStream(options); - }, - }; - if (sessions) { - counted.openCatFileSession = (...args) => { - counts.set('session:cat-file', count(counts, 'session:cat-file') + 1); - return plumbing.openCatFileSession(...args); - }; - counted.openMktreeSession = (...args) => { - counts.set('session:mktree', count(counts, 'session:mktree') + 1); - return plumbing.openMktreeSession(...args); - }; - counted.openFastImportSession = (...args) => { - counts.set('session:fast-import', count(counts, 'session:fast-import') + 1); - return plumbing.openFastImportSession(...args); - }; - } + const counted = await createCountingGitPlumbing({ cwd: repoDir, sessions }); return { - cas: new ContentAddressableStore({ plumbing: counted }), - snapshot() { - return new Map(counts); - }, + cas: new ContentAddressableStore({ plumbing: counted.plumbing }), + snapshot: counted.snapshot, }; } +async function closeAll(...stores) { + const results = await Promise.allSettled(stores.map((store) => store.close())); + const failures = results + .filter((result) => result.status === 'rejected') + .map((result) => result.reason); + if (failures.length === 1) { + throw failures[0]; + } + if (failures.length > 1) { + throw new AggregateError(failures, 'Multiple stores failed to close'); + } +} + function count(snapshot, operation) { return snapshot.get(operation) ?? 0; } @@ -192,24 +168,25 @@ describe('real-Git persistent object session process count', () => { const fallback = await countingReader(); const persistent = await countingReader({ sessions: true }); - const fallbackResult = await fallback.cas.bundles.getMemberReference({ - handle: outer.handle, - path: 'nested', - }); - const persistentResult = await persistent.cas.bundles.getMemberReference({ - handle: outer.handle, - path: 'nested', - }); - const fallbackCounts = fallback.snapshot(); - const persistentCounts = persistent.snapshot(); - - expect(persistentResult).toEqual(fallbackResult); - expect(count(persistentCounts, 'session:cat-file')).toBe(1); - expect(count(persistentCounts, 'ls-tree')).toBe(0); - expect(total(persistentCounts)).toBeLessThan(total(fallbackCounts)); - - await fallback.cas.close(); - await persistent.cas.close(); + try { + const fallbackResult = await fallback.cas.bundles.getMemberReference({ + handle: outer.handle, + path: 'nested', + }); + const persistentResult = await persistent.cas.bundles.getMemberReference({ + handle: outer.handle, + path: 'nested', + }); + const fallbackCounts = fallback.snapshot(); + const persistentCounts = persistent.snapshot(); + + expect(persistentResult).toEqual(fallbackResult); + expect(count(persistentCounts, 'session:cat-file')).toBe(1); + expect(count(persistentCounts, 'ls-tree')).toBe(0); + expect(total(persistentCounts)).toBeLessThan(total(fallbackCounts)); + } finally { + await closeAll(fallback.cas, persistent.cas); + } }); }); @@ -218,24 +195,25 @@ describe('real-Git scoped bulk write process count', () => { const individual = await countingReader({ sessions: true }); const batched = await countingReader({ sessions: true }); const inputs = Array.from({ length: 8 }, (_, index) => Buffer.from(`page-${index}`)); - const individualPages = []; - for (const source of inputs) { - individualPages.push(await individual.cas.pages.put({ source })); + try { + const individualPages = []; + for (const source of inputs) { + individualPages.push(await individual.cas.pages.put({ source })); + } + const batchPages = await batched.cas.pages.putBatch({ + pages: inputs.map((source) => ({ source })), + }); + const individualCounts = individual.snapshot(); + const batchCounts = batched.snapshot(); + + expect(batchPages.map((page) => page.handle.toString())).toEqual( + individualPages.map((page) => page.handle.toString()) + ); + expect(count(batchCounts, 'session:fast-import')).toBe(1); + expect(total(batchCounts)).toBeLessThan(total(individualCounts)); + } finally { + await closeAll(individual.cas, batched.cas); } - const batchPages = await batched.cas.pages.putBatch({ - pages: inputs.map((source) => ({ source })), - }); - const individualCounts = individual.snapshot(); - const batchCounts = batched.snapshot(); - - expect(batchPages.map((page) => page.handle.toString())).toEqual( - individualPages.map((page) => page.handle.toString()) - ); - expect(count(batchCounts, 'session:fast-import')).toBe(1); - expect(total(batchCounts)).toBeLessThan(total(individualCounts)); - - await individual.cas.close(); - await batched.cas.close(); }); });