diff --git a/docs/FLOWS.md b/docs/FLOWS.md index 7552be6279..7c5de88cfb 100644 --- a/docs/FLOWS.md +++ b/docs/FLOWS.md @@ -81,8 +81,8 @@ Provider、Tool、Skill、Memory 和 Session data 都通过创建时传入的必 不会伪造新的 outer round。 - Memory prompt contribution 必须等待结果、清理内容、限制大小并允许失败；terminal extraction 在后台 执行，并保持 epoch、cursor 和 fence 约束。 -- `TapeRecorder.appendToolFact` 在 message projection 完成后写 terminal tool call/result；写入失败不影响 - 当前回复完成。 +- `TapeToolFactWriter.appendToolFact` 在 message projection 完成后写 terminal tool call/result； + 写入失败不影响当前回复完成。 ## 4. ACP 执行 diff --git a/docs/architecture/baselines/agent-system-layered-runtime-baseline.json b/docs/architecture/baselines/agent-system-layered-runtime-baseline.json index 1baa3fce5b..be21f707cc 100644 --- a/docs/architecture/baselines/agent-system-layered-runtime-baseline.json +++ b/docs/architecture/baselines/agent-system-layered-runtime-baseline.json @@ -1,7 +1,7 @@ { "schemaVersion": 2, "goal": "agent-system-layered-runtime", - "headCommit": "d87cb4f69dca7af499e153d9a1d71015e1bfad60", + "headCommit": "df1cd3ae5b19d78e6c3cab79cdc9e4c5938fb2d4", "relevantWorkingTree": { "dirty": false, "files": [] @@ -127,11 +127,12 @@ "src/main/agent/shared/storage/sessionPaths.ts", "src/main/provider/providers/acpProvider.ts", "src/main/session/assignment.ts", - "src/main/session/data/tape.ts", "src/main/session/data/transcript.ts", "src/main/session/lifecycle.ts", "src/main/session/query.ts", - "src/main/session/turn.ts" + "src/main/session/turn.ts", + "src/main/tape/application/sessionTape.ts", + "src/main/tape/ports/capabilities.ts" ], "expectedFiles": { "src/main/agent/acp/instance/acpAgentInstance.ts": true, @@ -155,11 +156,12 @@ "src/main/agent/shared/appSessionService.ts": true, "src/main/provider/providers/acpProvider.ts": true, "src/main/session/assignment.ts": true, - "src/main/session/data/tape.ts": true, "src/main/session/data/transcript.ts": true, "src/main/session/lifecycle.ts": true, "src/main/session/query.ts": true, - "src/main/session/turn.ts": true + "src/main/session/turn.ts": true, + "src/main/tape/application/sessionTape.ts": true, + "src/main/tape/ports/capabilities.ts": true }, "ownerEvidence": { "agentManager": { @@ -192,8 +194,8 @@ "exists": true, "declarationCount": 1 }, - "tapeRecorder": { - "file": "src/main/agent/deepchat/loop/ports.ts", + "tapeToolFactWriter": { + "file": "src/main/tape/ports/capabilities.ts", "exists": true, "declarationCount": 1 }, @@ -297,11 +299,12 @@ "src/main/agent/deepchat/runtime/process.ts", "src/main/provider/providers/acpProvider.ts", "src/main/session/assignment.ts", - "src/main/session/data/tape.ts", "src/main/session/data/transcript.ts", "src/main/session/lifecycle.ts", "src/main/session/query.ts", - "src/main/session/turn.ts" + "src/main/session/turn.ts", + "src/main/tape/application/sessionTape.ts", + "src/main/tape/ports/capabilities.ts" ] }, "contracts": { @@ -366,7 +369,7 @@ "src/shared/contracts/routes/window.routes.ts", "src/shared/contracts/routes/workspace.routes.ts" ], - "sha256": "406ef501353454e8677f773c556b8938141d931229eebabae6fcb9b0c37abe3c" + "sha256": "e624008521f11befaa0497a325cb65b4bf38f36e575dd4285622ad3705955b9e" }, "storage": { "sqlite": { @@ -376,7 +379,7 @@ "src/main/data/schemaTypes.ts" ], "tableIdentifiers": [], - "sha256": "411d0c6078cf4264f2241e1198315e52dcf085342975d344e5d8967333b8129b" + "sha256": "3465d92dff615489b7037de640077ce95bda7a7abbb0144ce84bb144dcd85a9e" }, "memoryDuckDbSidecar": { "files": [ @@ -396,7 +399,7 @@ "src/main/app/mainProcess.ts", "src/main/appMain.ts" ], - "sha256": "dc13adbe8b76f0c4bb463605c780fc83e4a78696b8875d6765e767a4c2dcf70f" + "sha256": "394cd861f9fe06a29159d06f87f83b7ba1866287f129fae2a42a382e7e2e6c95" }, "dependencyMetrics": { "loopFiles": [ diff --git a/docs/architecture/memory-system.md b/docs/architecture/memory-system.md index 65e3ef16cc..d3c17a586a 100644 --- a/docs/architecture/memory-system.md +++ b/docs/architecture/memory-system.md @@ -23,6 +23,9 @@ flowchart LR epoch、cursor 和 fence。 - Session 保存 Memory cursor/settings，不拥有 Memory row 或 vector store。 - App 负责 shutdown/database maintenance 时的全局 fence 和停止顺序，不解释 Memory 业务状态。 +- Memory runtime 通过 `TapeRawEntryReader` 和 `TapeAnchorWriter` 读取执行事实、记录 + `memory/view_assembled` 与 `memory/extract` anchor；Memory routes 只通过 `TapeInspectionReader` + 获取 effective source span 和 manifest DTO，不接收 Tape table 或 raw Tape row。 ## 数据与状态 @@ -61,6 +64,8 @@ Vector store v2 使用 `.v2.duckdb`、plain `FLOAT[]` table 和 exact s ```text terminal turn projection + -> read bounded ingestion projection range + -> rebuild from effective Tape or fall back when projection is stale/unavailable -> collect bounded text chunks -> extraction / reflection -> normalize candidates @@ -76,6 +81,23 @@ terminal turn projection - stale result、partial batch 和 provider cancellation 有明确 terminal outcome； - vector store 异常进入 typed error/quarantine，不得把消息发送永久挂起。 +## Tape 与 ingestion projection 边界 + +`DeepChatMemoryIngestionProjectionTable.readCurrentRange` 在一条只读 SQL 中同时观察 Tape head 和 +projection head。这是明确的基础设施例外：拆成两次查询会让并发 append 产生 false-current 窗口。 +除此之外 Memory 不得直接读取物理 Tape 表。 + +projection current 时只 materialize cursor 区间；head 不一致时，runtime 通过 `TapeRawEntryReader` +构建 effective Tape view 并重建 projection。projection 查询或重建失败时保留既有 Tape fallback 和 +cursor commit 保护，不能因为拆层新增全历史 hot-path 查询，也不能在不完整 projection 上推进 cursor。 + +`TapeRawEntryReader` 只提供 `getBySession`。Memory management route 先验证 memory row 属于请求 Agent， +再用 `getEffectiveMessageSourceSpan` 读取 retraction/replacement 生效后的最小 message DTO；manifest +列表通过 `listMemoryViewManifestsByAgent` 在 storage query 中执行 Agent、Session、message 和 limit +过滤，route 不自行解析 `payload_json` 或 `meta_json`。架构守卫同时扫描 static import、dynamic +import、CommonJS require、type import 和 re-export，Memory route 不能绕过 inspection port 重新取得 +raw reader、facade 或 domain helper。 + ## Privacy 与隔离 - 所有查询显式携带 Agent identity；不得依赖进程全局“当前 Agent”。 @@ -101,7 +123,8 @@ metric 名称、retrieval evaluation 和 artifact upload 的未完成工作保 5. `src/main/memory/infra/vectorStoreManager.ts` 6. `src/main/memory/infra/memoryVectorStore.ts` 7. `src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts` -8. `test/main/memory/` +8. `src/main/tape/ports/capabilities.ts` +9. `test/main/memory/` Memory tests 必须防止旧 `src/main/presenter/memoryPresenter`、HNSW hot path、 无 Agent namespace 查询和无 deadline provider call 回流。 diff --git a/docs/architecture/session-management.md b/docs/architecture/session-management.md index fff32610bb..e80d0d837f 100644 --- a/docs/architecture/session-management.md +++ b/docs/architecture/session-management.md @@ -13,7 +13,8 @@ Session 是可长期保存的产品对象；window、renderer、Remote endpoint | list、restore、status、projection query | `src/main/session/query.ts` | | full delete transaction | `src/main/session/deletion.ts` | | transcript | `src/main/session/data/transcript.ts` | -| Tape / ViewManifest | `src/main/session/data/tape*.ts` | +| Tape public port / composition | `src/main/session/data/index.ts` | +| Tape domain / application / SQLite adapters | `src/main/tape/` | | generation settings / Memory cursor | `src/main/session/data/settings.ts` | | pending input | `src/main/session/data/pendingInputs.ts` | | renderer binding | `src/main/desktop/sessionBinding.ts` | @@ -54,6 +55,33 @@ send input；Remote、Scheduler 和 renderer 不得各自维护不同的默认 - history search 优先使用 search document / FTS path，失败时回退受控 SQL search；坐标和 scroll 归 renderer viewport owner，不写回 Session。 +## Tape boundary + +Session data composition 创建一个 `SessionTape`，对外继续暴露现有 `SessionTapePort`，并按每个 IPC +操作原有的条件和时序调用 `ensureSessionTapeReady`。`src/main/session/data/tape*.ts` 和旧 table path +只是显式冻结并标记 deprecated 的 compatibility re-export，不再拥有 Tape policy 或 persistence。 + +- transcript 只接收 `TapeMessageFactWriter`；message replace/retract 与对应 Tape fact 继续共享调用方 + SQLite transaction； +- settings/compaction 只接收 anchor reader/writer 与 lifecycle admin；summary 更新和 anchor append + 继续使用同一个 connection； +- loop runner 分别接收 reconciliation、ViewManifest reader/writer 和 tool fact writer；Turn coordinator + 与 ACP compatibility 只接收 reconciliation；Memory 和 routes 分别接收自己的最小 Tape capability， + 不接收物理 table； +- Tape 的运行中修订通过 append 表达；物理 delete/reset 只由 Session lifecycle 触发。 + +`SessionTranscript` 和 `SessionSettingsStore` 不提供隐式 `new SessionTape(...)` fallback，必须由正常 +composition 注入共享 connection 上的最小 capability。legacy import 作为 migration consumer 复用 +`sessionData.tapeStore` 的 message fact writer，不再构造第二个 facade，避免隐藏的独立 writer 或事务 +上下文。 + +`clearSessionMessages` 会创建新的 Tape incarnation：pending input 删除、transcript 删除和 Tape reset +位于同一个外层 SQLite transaction；Tape 内部的 entry、mutation projection、search/FTS projection +删除和新 bootstrap 作为 savepoint 嵌套。lifecycle、cleanup 或 bootstrap hard failure 会同时恢复上述 +数据并完整保留旧 incarnation。mutation projection 沿用 fail-open：新 bootstrap 的 projection apply +失败时旧 projection row 已删除且 meta 标 stale。最终 Session delete 不创建新 incarnation，继续遵循 +下面的 staged cleanup 顺序。 + ## Binding `DesktopSessionBinding` 维护 `webContentsId -> sessionId`： @@ -70,7 +98,7 @@ send input；Remote、Scheduler 和 renderer 不得各自维护不同的默认 ```text cleanup both backend caches without hydration -> remove ACP durable binding - -> remove transcript / Tape / pending / settings + -> remove transcript / Tape lifecycle data / pending / settings -> clear permission and active Skill state -> delete app-session row ``` diff --git a/docs/architecture/tape-layering/plan.md b/docs/architecture/tape-layering/plan.md new file mode 100644 index 0000000000..f54ae5d399 --- /dev/null +++ b/docs/architecture/tape-layering/plan.md @@ -0,0 +1,269 @@ +# Tape Layering Refactor Implementation Plan + +## Target Structure + +The implementation will create a top-level Tape subsystem: + +```text +src/main/tape/ + domain/ pure entry, fact, view, manifest, lineage, and replay logic + ports/ storage and consumer capability interfaces + application/ fact, reconciliation, recall, lineage, view/replay, and fork services + infrastructure/sqlite/ SQLite entry store, query SQL, search projection, and lifecycle adapter +``` + +Existing `src/main/session/data/tape*.ts` and table modules will become compatibility re-exports +where an old import path is still part of the current internal contract. New production imports +will target `@/tape/*`. + +## Domain and Port Design + +Tape entry rows, append inputs, source identities, entry references, fact provenance, and tool fact +inputs move out of Agent and table modules into Tape-owned types. Effective-view selection, +ViewManifest hashing, lineage validation, stored-manifest validation, and replay hashing remain +pure. Database row parsing and trace-evidence reads remain in the application layer. + +The primary ports are: + +- `TapeEntryStore`: append, anchor/event append helpers, and read/query operations. It has no + destructive method. `TapeBootstrapStore` and `TapeTransactionRunner` are separate application + composition capabilities. +- `TapeSearchProjectionStore`: rebuildable search projection behavior. +- `TapeToolFactWriter`: the single `appendToolFact` capability used by the Agent loop. +- `TapeMessageFactWriter`: message, replacement, and retraction fact operations used by transcript. +- `TapeReconciliationPort`: bootstrap and transcript reconciliation used by the loop runner, Turn + coordinator, and ACP compatibility projection. +- `TapeViewManifestReader` and `TapeViewManifestWriter`: the manifest capabilities used by the + loop runner without exposing the full facade. +- `TapeAnchorReader`: only the latest reconstruction-anchor read required by settings. +- `TapeAnchorWriter`: narrow anchor append capability for settings and Memory. +- `TapeRawEntryReader`: only `getBySession`, retained for effective-view rebuilding in Memory. +- `TapeInspectionReader`: effective source spans and Memory ViewManifest inspection DTOs; no raw + entry rows cross this boundary. +- `TapeLifecycleAdmin`: Session-owned delete and reset operations across entries and projections. +- Explicit transcript and trace evidence read ports used by reconciliation and replay. + +One application object may implement multiple interfaces. Composition injects the narrow interface +at each call site. + +## Application Services + +The current `SessionTape` behavior is divided without changing method semantics: + +1. **TapeFactService** owns message, tool, generic anchor, handoff, and fork-message fact appends. +2. **TapeReconcilerService** owns bootstrap, legacy transcript backfill, and legacy summary-anchor + repair. +3. **TapeRecallService** owns info, search, context windows, anchor listing, and authorized source + resolution needed by recall. +4. **TapeLineageService** owns link validation, frozen child heads, authorization, and lineage + receipts. +5. **TapeViewReplayService** owns ViewManifest append and source assembly, manifest listing, replay + exports, Memory inspection projection, and explicit trace-evidence reads. +6. **TapeForkService** owns only fork creation, delta merge, discard, and external lifecycle + receipts. + +`SessionTape` becomes a compatibility facade that constructs these services and forwards the +existing methods. `SessionTapePort` in Session contracts remains unchanged. + +## SQLite Infrastructure + +The large entry table module is separated into Tape-owned row and append types, reusable effective +query SQL, a normal SQLite entry store, and a lifecycle adapter. Table names, indexes, SQL +predicates, provenance uniqueness, and payload serialization remain byte-compatible. + +Physical entry deletion is removed from the normal entry-store interface. `TapeLifecycleAdmin` +coordinates entry deletion, search projection deletion, and reset bootstrap. A reset executes +entry and mutation-projection deletion, search and FTS deletion, and new bootstrap creation in one +transaction. Startup legacy import may continue to execute whole-database cleanup SQL because it +rebuilds persisted state before normal runtime composition. + +The Memory ingestion projection retains its current single SQL statement that compares +`MAX(deepchat_tape_entries.entry_id)` with the projection metadata head. Moving this comparison to +two independent port calls would introduce a freshness race and an extra query, so it is an +allowlisted read-only infrastructure dependency. + +## Composition and Data Flow + +Session data composition will create the entry store and Tape services before constructing +transcript and settings: + +```text +SQLite connection + -> Tape stores and services + -> Transcript with TapeMessageFactWriter + -> Settings with anchor and lifecycle capabilities + -> SessionTape facade and existing SessionTapePort adapter +``` + +Runtime composition passes reconciliation, ViewManifest read/write, and tool-fact capabilities to +the loop runner; only reconciliation to the Turn coordinator and ACP adapter; raw-row and anchor +capabilities to the Memory coordinator; and `TapeInspectionReader` to Memory routes. No +application consumer gets the concrete entry table. Transcript and settings have no concrete +facade default: normal composition injects their capabilities from the shared `SessionTape`, and +legacy import reuses that composition-owned `TapeMessageFactWriter` instead of constructing a +second facade. + +`ensureSessionTapeReady` remains at the current Session port boundary. Search and context requests +with linked-source scopes keep their existing conditional reconciliation behavior. + +## Transaction Boundaries + +- Transcript deletion and retry truncation append retractions inside the same SQLite transaction + that deletes projection rows. +- Summary compare-and-set appends its reconstruction anchor inside the same transaction that + updates summary state. +- Clear-time pending-input deletion, transcript deletion, and Tape reset run in one outer + shared-connection transaction. The Tape generation transaction nests as a savepoint, so a reset + or bootstrap failure restores every clear-time data family. +- Reset deletes entries, mutation projection, search projection, FTS metadata, and FTS rows and + appends the new bootstrap within one shared-connection transaction. A propagated transition + failure restores the old incarnation. The pre-existing fail-open mutation-projection append + policy may instead commit the new Tape with that derivative marked stale after old projection + rows have been removed. +- Fork discard performs the same atomic cleanup attempt. Cleanup failure rolls that attempt back + but still appends a fail-closed discard receipt, preserving the non-blocking contract. +- Final Session deletion keeps its staged lifecycle ordering and does not create a replacement + incarnation. Its Tape entry and search-projection cleanup still runs as one generation + transaction before the Session row is removed. +- FTS is a rebuildable derivative. If session-row deletion from the virtual table fails, the + adapter drops the FTS table and clears freshness metadata inside the same lifecycle transaction; + the next search recreates and repopulates it. Failure to drop the damaged derivative remains a + hard transaction failure rather than committing a mixed generation. +- Port implementations use the same connection provider and remain synchronous, so extracting a + service does not cross a transaction boundary. + +Contract tests will force failures between paired operations and verify rollback or unchanged +behavior where the current implementation is atomic. + +## Compatibility Strategy + +- Keep all shared DTOs and `SessionTapePort` signatures unchanged. +- Preserve old internal exported symbol names through explicit, frozen compatibility re-exports + marked as deprecated. +- Keep methods that existed on the historically exported concrete SQLite classes, while excluding + them from application-facing protocols. Remove non-historical raw-row forwarding helpers from + the facade. +- Use `TapeViewManifestAssemblySources` for the complete application source set and + `TapeViewManifestLookupMaps` for pure domain lookups. Preserve each historical + `TapeViewManifestSourceMaps` shape only at its original legacy import path. +- Advance the rebuildable search projection to version 3 so same-head version 2 rows from a + possible interrupted legacy reset are never trusted. The first current-Tape search performs a + one-time rebuild; linked read-only search uses the existing effective-Tape fallback until a + current rebuild exists. +- Preserve schema SQL, existing rows, canonical policy identifiers, hashes, source identities, + provenance keys, error messages where tested, and bounded query limits. +- Preserve projection failure fallback and best-effort fork projection cleanup. +- Keep trace evidence distinct from transcript projection in replay dependencies. + +## Follow-up Hardening + +1. Replace the deleted monolithic Tape test in the Memory native scope with every split suite that + contains `itIfSqlite` or `describeIfSqlite`, and make the scope validator discover this required + coverage independently. +2. Route final Session Tape deletion through `deleteTapeGeneration` and recover a failed FTS row + delete by invalidating and dropping the derivative within the generation transaction. +3. Prune pre-version-3 and metadata-orphaned projection rows during schema initialization without + rebuilding every current projection eagerly. +4. Freeze legacy shim export surfaces; remove non-historical facade raw-row helpers, and retain + historical concrete-store methods because the exported classes are compatibility contracts. +5. Rename the canonical domain ViewManifest lookup-map type and make Memory route boundary scans + reject static, dynamic, CommonJS, type-import, and re-export bypasses. +6. Reuse the composition-owned Tape fact writer in legacy import, document the same-connection + anchor requirement, cache SQLite FTS capability per connection, and replace exception-by-missing + mock behavior with explicit failure fixtures. +7. Put the complete `clearMessages` mutation set inside one shared-connection transaction and + document that a failed best-effort fork cleanup leaves permanent, non-retried residue that is + nevertheless fail-closed for merge and identifier reuse. + +## Test Strategy + +1. Record the current seven-file baseline: 120 passed and 26 native-SQLite-gated skipped tests. +2. Mechanically split the monolithic test suite by application-service boundary without changing + assertions or skip gates. +3. Add characterization coverage for reconciliation ordering, transaction atomicity, projection + fallback, and lifecycle reset. +4. Add contract coverage for append-only correction, frozen-head authorization, fork delta merge, + ViewManifest hashes, replay evidence, and projection rebuild equivalence. +5. Add source-boundary tests that reject domain reverse/runtime imports, production legacy Tape + imports, concrete facade imports from capability-scoped consumers, Memory route capability + expansion, the project SQLite driver, and non-allowlisted table access. Exercise each guard with + table-driven negative fixtures. +6. Add native-scope discovery, corrupt-FTS recovery, stale-projection cleanup, bootstrap rollback, + frozen-shim, and non-static Memory route import coverage. +7. Run the full main-process suite, Tape scale suite, type checks, formatting, i18n validation, and + lint before handoff. + +## Commit and Review Strategy + +Each implementation slice remains green and receives a local conventional commit. Before every +commit, review the complete unstaged and staged diff for hidden side effects, compatibility, +boundary cases, performance, security, misleading names, missing tests, and long-term maintenance +cost. Fix all findings and repeat validation before committing. + +The initial implementation commits were: + +1. `docs(tape): specify layering refactor` +2. `test(tape): split behavior contracts` +3. `refactor(tape): establish domain ports` +4. `refactor(tape): split application services` +5. `refactor(tape): close storage bypasses` +6. `test(tape): enforce layer boundaries` +7. `test(tape): align writer mock naming` +8. `docs(tape): refresh architecture map` + +The review remediation commits are: + +1. `fix(tape): make generation resets atomic` +2. `refactor(tape): narrow consumer ports` +3. `refactor(tape): align service ownership` +4. `test(tape): harden layer boundaries` +5. `docs(tape): clarify layer contracts` + +The cumulative review added these focused fixes before the documentation commit: + +1. `test(tape): guard semantics compatibility path` +2. `test(tape): guard project sqlite driver` +3. `fix(tape): invalidate legacy projections` +4. `refactor(tape): require capability injection` +5. `refactor(tape): narrow anchor reader port` +6. `refactor(tape): narrow storage protocols` + +The post-review hardening added these local commits: + +1. `docs(tape): specify follow-up hardening` +2. `test(memory): restore native tape scope` +3. `test(tape): restore layered settings fixture` +4. `fix(tape): harden generation cleanup` +5. `refactor(tape): harden compatibility boundaries` +6. `test(tape): repair memory type contracts` + +No commit is pushed. The final review compares the complete branch with `dev`. + +## Final Validation Record + +The final focused gates passed: + +- Memory scope discovery classified 65 files with 3 explicit exemptions, and the independent + Memory type gate passed all 65 scoped files. +- Memory behavior passed 749 tests across 46 files. +- Native SQLite and Tape coverage passed 242 tests across 13 files; the 2 skipped tests are + Windows-only handle-locking cases on the current macOS host. +- Memory performance passed all 8 tests, including the 10k/100k Tape range-bound comparison. +- Full node and renderer type checks, formatting, i18n validation, and lint passed. +- The architecture baseline generator passed and refreshed the canonical snapshot against the + final verified code commit. + +The full main-process command completed with 390 passing files and 3 failing files: 4,471 tests +passed, 2 were skipped, and 9 failed. Each affected file was then run in an isolated detached +worktree at the exact `dev` baseline (`e84428b66`), reproducing the same 6 failures in +`mainDatabase.test.ts`, 1 failure in `schedulerService.test.ts`, and 2 failures in +`sessionDataMigrations.sqlite.test.ts`. These are pre-existing baseline failures, not branch +regressions; the project-wide main-process gate therefore remains red for reasons outside this Tape +refactor. + +## Rollback + +The work is organized into locally reviewable commits. Reverting must proceed in reverse order +because later composition and boundary changes depend on the earlier domain and port extraction. +The unchanged schema and compatibility re-exports allow a complete branch rollback without a data +migration. diff --git a/docs/architecture/tape-layering/spec.md b/docs/architecture/tape-layering/spec.md new file mode 100644 index 0000000000..7d57b0c63d --- /dev/null +++ b/docs/architecture/tape-layering/spec.md @@ -0,0 +1,213 @@ +# Tape Layering Refactor Specification + +## Background + +Before this refactor, DeepChat's Tape implementation had strong runtime semantics but weak module +boundaries. The main `SessionTape` implementation combined fact writing, migration and +reconciliation, search and context recall, ViewManifest and replay assembly, subagent lineage, +and fork management in one large module. The SQLite entry table also exposed destructive +lifecycle operations beside normal append and read operations. + +Several consumers bypassed `SessionTape` and depended directly on the SQLite table. Transcript +writes, Memory ingestion, Memory management routes, Session settings, and startup migration each +used a different subset of Tape behavior, but the table-shaped dependency gave them more +authority than they needed. Tape types also flowed in the wrong direction because the Tape layer +imported Agent loop port types. + +This refactor adopts Bub's useful dependency pattern—domain primitives, narrow store protocols, +application services, and independent view selection—without copying Bub's simpler schema or +reset semantics. DeepChat retains its stronger revision, retraction, ViewManifest, frozen-head, +and fork contracts. + +## Goals + +1. Establish a top-level `src/main/tape/` subsystem with explicit domain, port, application, and + SQLite infrastructure boundaries. +2. Split `SessionTape` along its existing cohesive behavior groups while retaining a compatibility + facade. +3. Replace raw table dependencies with the smallest capability required by each consumer. +4. Keep destructive reset and delete operations outside the append-only entry store contract. +5. Remove the Tape-to-Agent reverse dependency and delete unused `TapeRecorder` capabilities. +6. Preserve all persisted data, public IPC behavior, runtime ordering, transaction boundaries, + failure fallbacks, and performance characteristics. +7. Add enforceable dependency and behavioral contracts so the layering does not regress. + +## Data Families + +The refactor keeps three distinct data families: + +| Data family | Role | Authority | +| --- | --- | --- | +| Tape facts | Append-only execution facts, anchors, manifests, lineage, and fork receipts | Tape | +| Transcript projection | UI-oriented structured messages and a legacy backfill source | Session data | +| Trace evidence | Provider request and terminal execution evidence used by replay | Session trace storage | + +Replay may combine Tape facts with trace evidence through explicit read ports. This is not a reason +to treat trace evidence as transcript data or to move it into the Tape entry schema. + +## Required Invariants + +- Entries in an active Tape are append-only. Known facts are never updated in place. +- Corrections and deletions of projected messages are represented by appended replacement or + retraction facts. +- Anchors are reconstruction points and never imply deletion of earlier entries. +- Compaction changes the selected view, not the retained history. +- Fork merge appends only the fork delta and a merge receipt to the parent. +- Cross-Tape reads require an explicit direct-child lineage fact and remain bounded by the stored + child head. +- Search projections are rebuildable derivatives. Projection failures retain the existing bounded + effective-view fallback. +- Destructive Session cleanup is a lifecycle operation, not a normal Tape store operation. +- A reset that creates a new Tape incarnation deletes entries, mutation projection state, and + search projection state and appends the new bootstrap anchor in one SQLite transaction. +- A discarded fork is fail-closed for merge and identifier reuse even when best-effort physical + cleanup fails. Failed cleanup leaves permanent inert residue; no automatic retry is scheduled. + +## Capability Boundaries + +| Consumer | Allowed capability | +| --- | --- | +| DeepChat loop runner | `TapeReconciliationPort`, `TapeViewManifestReader`, `TapeViewManifestWriter`, and `TapeToolFactWriter` | +| Turn coordinator and ACP compatibility adapter | `TapeReconciliationPort` | +| Session transcript | `TapeMessageFactWriter` | +| Memory runtime | `TapeRawEntryReader` and `TapeAnchorWriter` | +| Session settings and compaction | `TapeAnchorReader`, `TapeAnchorWriter`, and `TapeLifecycleAdmin` | +| Memory management routes | `TapeInspectionReader` | +| Session IPC | Existing `SessionTapePort` facade | + +A single implementation may satisfy several ports, but each consumer receives only the +structural type it needs. `TapeRawEntryReader` exposes only `getBySession`. The inspection port +returns purpose-built effective-message and Memory ViewManifest DTOs; it never returns a physical +Tape row. `TapeAnchorReader` exposes only the latest reconstruction anchor required by settings. +Transcript and settings require these capabilities to be injected; only normal or migration +composition may provide them, and only normal Session composition constructs the concrete facade. +Legacy import reuses the composition-owned `TapeMessageFactWriter`. `TapeViewManifestWriter` +intentionally exposes a `void` append contract because its consumer does not observe the stored +row; the concrete facade's richer return value remains an internal compatibility detail. + +`TapeViewManifestAssemblySources` names the complete source set assembled by the application +service. `TapeViewManifestLookupMaps` names the smaller domain lookup map used by pure +ViewManifest builders. The two historical `TapeViewManifestSourceMaps` exports remain available +only from their respective legacy compatibility modules, where each is an explicit deprecated +alias of the original shape. + +## Direct Storage Access Inventory + +The implementation must account for every current physical-table access: + +- `session/data/tape.ts`: compatibility re-export of the new facade; production imports use + `@/tape/*` directly. +- `session/data/transcript.ts`: legitimate message fact producer; migrated to + `TapeMessageFactWriter` while preserving same-connection transactions. +- `session/data/settings.ts`: bootstrap, reconstruction-anchor reads, summary/reset anchors, and + destructive cleanup; migrated to anchor and lifecycle capabilities. +- `agent/deepchat/runtime/deepChatRuntimeCoordinator.ts`: composition root that distributes + reconciliation, fact, manifest, raw-read, and anchor capabilities to narrower consumers. +- `memory/routes.ts` and app composition: use `TapeInspectionReader`; effective source spans and + Memory ViewManifest records cross the boundary only as domain DTOs. +- `memory/data/tables/deepchatMemoryIngestionProjection.ts`: one-statement freshness comparison + between Tape head and projection head; retained as an explicit read-only infrastructure + exception to preserve atomicity and query count. +- `app/startupMigrations/legacyChatImportService.ts`: destructive whole-database rebuild; retained + as an explicit startup-migration exception while reusing the composition-owned message fact + writer. +- Schema catalog and database security table-name lists: metadata, not runtime Tape access. + +## Generation and Failure Semantics + +`resetSessionTape` performs entry deletion, mutation-projection deletion, search-projection and FTS +deletion, and new bootstrap creation inside one transaction on the shared Session SQLite +connection. Any propagated lifecycle, cleanup, or bootstrap failure restores the complete prior +incarnation. The existing fail-open mutation-projection append policy remains: if applying the new +bootstrap to that derived projection fails, its metadata is invalidated after old rows have been +removed, so the new Tape can commit without trusting partial projection state. Search-projection +session cleanup is also internally transactional so it cannot leave base, metadata, and FTS rows +at different generations. + +Fork discard makes one atomic cleanup attempt. If cleanup succeeds, cleanup and the discard +receipt commit together. If cleanup fails, the cleanup attempt rolls back, the parent still +appends the discard receipt, and the failure remains non-blocking. Merge checks an existing merge +receipt first for idempotency and then rejects a discard receipt before reading the fork. Creating +a fork with an explicitly discarded identifier also fails closed. + +Context projection reads use `getByEntryIdsIfCurrent`, which verifies projection version and the +projection metadata head against the current Tape head supplied by the synchronous caller in the +same SQL statement that reads the requested rows. A non-current projection is ignored and summary +or reference context is rebuilt from the current effective Tape. Projection version 3 invalidates +version 2 data that may have survived an interrupted pre-atomic reset with the same entry-id head; +current search rebuilds that derivative on demand, while read-only linked search retains its +effective-Tape fallback. + +Corrupt or unavailable FTS storage must not block deletion of authoritative Tape or transcript +state. A failed FTS row deletion invalidates FTS metadata and drops the rebuildable virtual table +before the lifecycle transaction continues; if that recovery operation itself cannot complete, +the enclosing Tape generation transaction fails atomically. Startup removes base and FTS +projection rows owned only by pre-version-3 metadata so inert legacy text does not remain on disk +or force linked read-only searches onto permanent fallback paths. + +`clearMessages` places pending-input deletion, transcript deletion, and Tape reset inside one +outer transaction on that same connection. The Tape generation transaction becomes a nested +savepoint, so any reset, projection cleanup, or bootstrap failure restores all three data families +instead of leaving transcript and Tape at different generations. + +The Memory native-test manifest must include every split Tape suite that contains an active native +SQLite gate. Scope validation discovers these gated suites independently from the manifest so +removing or renaming one cannot silently eliminate the only real-SQLite lifecycle and FTS CI +coverage. + +## Acceptance Criteria + +1. `src/main/tape/domain/` does not import Agent, Session, Memory, App, SQLite, Electron, or logging + runtime modules. +2. Agent execution consumers compile against reconciliation, manifest, and fact ports rather than + concrete `SessionTape` or the deleted broad `TapeRecorder` interface. +3. Runtime, transcript, settings, routes, and normal application composition do not receive a + `DeepChatTapeEntriesTable` instance. +4. `TapeEntryStore` exposes no reset or delete method. +5. Existing `SessionTapePort`, persisted schema, table names, entry payloads, View policy IDs, and + renderer contracts remain unchanged. +6. Transcript mutation plus Tape correction, summary mutation plus anchor append, and clear-time + pending/transcript deletion plus Tape reset share their required transaction context. +7. `ensureSessionTapeReady` remains idempotent and runs at the same Session port boundaries. +8. Projection search fallback remains unchanged. Fork cleanup failure remains non-blocking while + its discard receipt prevents later merge or identifier reuse. +9. Baseline Tape tests remain green; the pre-refactor baseline is 120 passed and 26 + environment-gated skipped tests across seven files. +10. Tape scale coverage confirms bounded tail materialization and no added full-history query on + the Memory projection fast path. +11. Architecture tests reject forbidden imports, production use of legacy compatibility paths, + concrete facade imports from capability-scoped consumers, Memory route capability expansion, + the actual SQLite driver, and new physical-table bypasses. Negative fixtures prove that each + guard recognizes the prohibited dependency. +12. No remote Git operations are performed as part of this work. +13. Native Memory CI discovers and executes every SQLite-gated Tape suite after test splitting. +14. Corrupt FTS cleanup cannot leave transcript, Tape entries, and base search projection in + different generations; pre-version-3 projection data is removed during schema initialization. +15. Legacy Tape modules expose frozen deprecated export lists, while canonical modules use + unambiguous ViewManifest source-map names. +16. Historical concrete SQLite-class methods remain available through the frozen legacy class + exports, but non-historical raw-row helpers are absent from the `SessionTape` facade and every + application-facing port. + +## Constraints + +- Use synchronous ports where the current SQLite operation is synchronous; do not introduce + artificial asynchronous transaction boundaries. +- Preserve ordering, idempotency keys, hashes, error classes, and fallback logging semantics. +- Compatibility re-exports may remain at old module paths to control import churn. +- New SDD artifacts in this directory must use English prose. +- Every local commit requires a complete unstaged and staged diff review plus relevant validation. + +## Non-Goals + +- No database schema or data migration. +- No archive-on-reset behavior. +- No change to compaction, context selection, or ViewManifest policy. +- No redesign of transcript or trace storage. +- No renderer or IPC feature change. +- No GitHub issue, pull request, branch push, or other remote mutation. + +## Open Questions + +None. The implementation decisions required for this refactor are recorded in this specification +and the accompanying plan. diff --git a/docs/architecture/tape-layering/tasks.md b/docs/architecture/tape-layering/tasks.md new file mode 100644 index 0000000000..54314337c2 --- /dev/null +++ b/docs/architecture/tape-layering/tasks.md @@ -0,0 +1,114 @@ +# Tape Layering Refactor Tasks + +## Specification and Baseline + +- [x] Record the pre-refactor Tape baseline: 120 passed and 26 environment-gated skipped tests. +- [x] Write the English `spec.md`, `plan.md`, and `tasks.md` artifacts. +- [x] Confirm the SDD artifacts contain no unresolved clarification markers or non-English prose. +- [x] Review and commit the SDD slice. + +## Behavior Characterization + +- [x] Split the monolithic Tape suite by reconciliation, recall, lineage, view/replay, and fork + behavior. +- [x] Preserve every existing assertion and environment skip gate during the mechanical split. +- [x] Add transaction, reconciliation-order, and projection-fallback characterization coverage. +- [x] Review and commit the characterization slice. + +## Domain and Ports + +- [x] Move Tape-owned entry, source, provenance, and fact types out of Agent and SQLite modules. +- [x] Move effective-view and ViewManifest pure logic into `src/main/tape/domain/`. +- [x] Introduce normal storage and narrow consumer capability ports. +- [x] Replace the broad `TapeRecorder` dependency with `TapeToolFactWriter`. +- [x] Preserve old module paths through compatibility re-exports. +- [x] Review and commit the domain and port slice. + +## Application Services + +- [x] Extract fact, reconciliation, recall, lineage, view/replay, and fork services. +- [x] Convert `SessionTape` into a compatibility facade. +- [x] Preserve `SessionTapePort` and current reconciliation timing. +- [x] Review and commit the application-service slice. + +## Infrastructure and Bypass Closure + +- [x] Separate normal SQLite entry operations from destructive lifecycle operations. +- [x] Inject message fact capabilities into transcript without changing transaction boundaries. +- [x] Inject anchor and lifecycle capabilities into Session settings. +- [x] Replace Memory runtime and route table access with explicit capabilities. +- [x] Document and allowlist startup migration and Memory projection infrastructure exceptions. +- [x] Add lifecycle, transaction, projection, and authorization tests. +- [x] Review and commit the storage-boundary slice. + +## Architecture Enforcement + +- [x] Add domain dependency-direction tests. +- [x] Add a physical Tape table access guard with a narrow explicit allowlist. +- [x] Review and commit the architecture-guard slice. + +## Review Remediation + +- [x] Make reset generation replacement atomic across entries, mutation projection, search + projection, FTS state, and the new bootstrap. +- [x] Make fork cleanup atomic while preserving a non-blocking, fail-closed discard receipt. +- [x] Read context projection rows only when projection version and metadata head match the + synchronous caller's current Tape head in the same SQL statement. +- [x] Replace concrete `SessionTape` dependencies in the loop runner, Turn coordinator, and ACP + compatibility adapter with narrow ports. +- [x] Reduce `TapeRawEntryReader` to the Memory runtime's `getBySession` requirement. +- [x] Replace Memory route raw rows with effective-source and ViewManifest inspection DTOs. +- [x] Remove production imports of legacy Tape compatibility paths. +- [x] Move handoff, generic anchor, and fork-message fact ownership into `TapeFactService`. +- [x] Restrict `TapeForkService` to fork lifecycle behavior. +- [x] Move stored-manifest validation and replay hash helpers into the domain layer. +- [x] Rename the complete source-set DTO to `TapeViewManifestAssemblySources` and retain its old + name only as a deprecated legacy-path alias. +- [x] Harden architecture guards for main-layer, SQLite, Electron, logging, legacy-path, and Memory + route violations, including table-driven negative fixtures. +- [x] Cover every legacy compatibility wrapper and the project's actual SQLite driver in the + boundary guards. +- [x] Invalidate pre-atomic version 2 search projections and verify same-head context fallback. +- [x] Require capability injection for transcript and settings, with concrete facade construction + limited to composition boundaries. +- [x] Keep capability-scoped consumers off the concrete facade through a negative-tested guard. +- [x] Remove unused anchor and storage operations from application-facing protocols while keeping + concrete compatibility methods intact. +- [x] Refresh the canonical agent-system architecture baseline with the new Tape owner evidence. + +## Documentation and Final Validation + +- [x] Synchronize the English SDD with the implemented capabilities, transactions, ownership, and + failure semantics. +- [x] Update Tape, Session, and Memory architecture references in their existing languages. +- [x] Run the Tape contract and scale suites. +- [x] Run the full main-process test suite and Memory performance suite. +- [x] Run full type checks, formatting, i18n validation, and lint. +- [x] Scan all three SDD artifacts for non-English prose and unresolved clarification markers. +- [x] Review the complete `dev...HEAD` diff and fix every finding. +- [x] Prepare the completed task checklist and final documentation slice for a local commit. +- [x] Confirm no unexpected working-tree files exist and no remote push has been performed before + the final commit. + +## Post-Review Hardening + +- [x] Replace the stale monolithic Tape path in Memory native CI with all SQLite-gated split suites. +- [x] Make scope validation discover required native Tape coverage independently from the manifest. +- [x] Make final Session Tape deletion reuse the atomic generation lifecycle helper. +- [x] Recover failed FTS row deletion by dropping and invalidating the rebuildable derivative. +- [x] Remove pre-version-3 and metadata-orphaned search projection rows at schema initialization. +- [x] Add direct reset-bootstrap rollback and corrupt-FTS lifecycle regression tests. +- [x] Put pending-input deletion, transcript deletion, and Tape reset in one clear-time transaction. +- [x] Freeze and deprecate legacy Tape compatibility export lists without removing old symbols. +- [x] Use distinct canonical names for application assembly sources and domain lookup maps. +- [x] Audit unused concrete storage helpers and retain those present on historically exported + SQLite classes while removing non-historical facade raw-row forwarding methods. +- [x] Detect dynamic import, CommonJS require, type import, and re-export Memory route bypasses. +- [x] Reuse the composition-owned Tape message writer in legacy import. +- [x] Document the same-connection transaction requirement for settings anchor writers. +- [x] Cache FTS capability detection per SQLite connection. +- [x] Replace fallback tests that depend on missing mock methods with explicit failures. +- [x] Document permanent fork residue when best-effort discard cleanup fails. +- [x] Run focused native scope, lifecycle, recall, boundary, migration, and compatibility tests. +- [x] Run full validation, reproduce the nine unrelated main-suite failures on the exact `dev` + baseline, and review every new local commit without pushing. diff --git a/docs/architecture/tape-system.md b/docs/architecture/tape-system.md index 745d9d30f6..b5eeb6174b 100644 --- a/docs/architecture/tape-system.md +++ b/docs/architecture/tape-system.md @@ -3,19 +3,85 @@ Tape 是 Session 同寿命的 append-only execution fact log。它保存可回放事实、anchor、ViewManifest 和 Subagent lineage；message transcript 是面向 UI 的 projection，不是 Tape 的替代品。 -## 所有权和数据 +## 所有权和分层 | 能力 | 当前 owner | | --- | --- | -| append/query/replay | `src/main/session/data/tape.ts` | -| effective view | `src/main/session/data/tapeEffectiveView.ts` | -| facts | `src/main/session/data/tapeFacts.ts` | -| ViewManifest storage | `src/main/session/data/tapeViewManifest.ts` | +| entry/fact/ref、effective semantics、ViewManifest/replay 纯逻辑 | `src/main/tape/domain/` | +| 消费方能力和 storage ports | `src/main/tape/ports/` | +| Fact、Reconciler、Recall、Lineage、View/Replay、Fork services | `src/main/tape/application/` | +| `SessionTape` 兼容 facade | `src/main/tape/application/sessionTape.ts` | +| append/read/query store | `src/main/tape/infrastructure/sqlite/tapeEntryStore.ts` | +| search projection | `src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts` | +| 物理 lifecycle delete | `src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts` | | runtime assembly | `src/main/agent/deepchat/runtime/tapeViewAssembler.ts` | | policy selection | `src/main/agent/deepchat/runtime/tapeViewPolicy.ts` | | model-facing tools | `src/main/tool/agentTools/agentTapeTools.ts` | Tape entry 只能 append。更正、压缩和 handoff 通过新 fact/anchor 表达，不原地改写旧 entry。 +anchor 改变后续读取起点或重建状态，但不删除被覆盖的历史。 + +```mermaid +flowchart TD + Consumers["Agent / Transcript / Memory / Settings / IPC"] --> Ports["Tape capability ports"] + Ports --> Facade["SessionTape compatibility facade"] + Facade --> Services["Six application services"] + Services --> Stores["Entry store / Search projection / Lifecycle adapter"] + Stores --> SQLite["Shared Session SQLite connection"] +``` + +`src/main/session/data/tape*.ts` 和旧 table modules 只保留显式、冻结且标记 deprecated 的 +compatibility re-export。新代码必须从 `src/main/tape/` 或能力 port 导入，不能把兼容路径重新当作 +owner，也不能通过 canonical module 的新增导出隐式扩大旧路径合同。 + +## 能力端口和组合 + +| 消费方 | 允许依赖的 Tape 能力 | +| --- | --- | +| DeepChat loop runner | `TapeReconciliationPort`、`TapeViewManifestReader`、`TapeViewManifestWriter`、`TapeToolFactWriter` | +| Turn coordinator / ACP compatibility | `TapeReconciliationPort` | +| Transcript | `TapeMessageFactWriter` | +| Memory runtime | `TapeRawEntryReader`、`TapeAnchorWriter` | +| Settings / compaction | `TapeAnchorReader`、`TapeAnchorWriter`、`TapeLifecycleAdmin` | +| Memory routes | `TapeInspectionReader` | +| IPC / Session data | 现有 `SessionTapePort` | + +`createSessionDataFromDatabase` 组合一个 `SessionTape`，把窄能力传给 transcript 和 settings，并在既有 +IPC boundary 按原时序执行 `ensureSessionTapeReady`。facade 只做 service 组合和兼容转发，不承载新的 +domain policy；外部方法的签名、同步/异步行为、异常和 fallback 语义保持稳定。 + +`TapeRawEntryReader` 只暴露 Memory runtime 实际需要的 `getBySession`。Memory routes 使用的 +`TapeInspectionReader` 只返回 effective message source span 与 Memory ViewManifest DTO，不返回 +`DeepChatTapeEntryRow`。完整的 manifest assembly source set 命名为 +`TapeViewManifestAssemblySources`，domain lookup map 命名为 `TapeViewManifestLookupMaps`；两种历史 +`TapeViewManifestSourceMaps` 形状只在各自原有 compatibility path 作为 type alias 保留。 +`TapeAnchorReader` 只暴露 settings 实际使用的 latest reconstruction anchor；transcript/settings 必须 +由 composition 注入 port，不允许在 consumer 内隐式构造 concrete facade。 + +## 存储与事务边界 + +- `TapeEntryStore` 只负责 append/read/query；物理删除由独立 lifecycle adapter 执行，只服务于 + Session lifecycle（包含 fork Session cleanup），不属于运行中 Tape 语义。 +- transcript message mutation 与 replacement/retraction fact、summary compare-and-set 与 anchor append + 使用同一个 SQLite connection 和调用方 transaction，拆层不能拆开其原子边界。 +- `clearMessages` 在同一外层 transaction 中删除 pending input、transcript projection 并 reset Tape； + Tape generation transaction 作为 savepoint 嵌套，任一 hard failure 会同时恢复三类数据。 +- `resetSessionTape` 在同一 transaction 内删除 entry、mutation projection、search/FTS projection 并 + 创建新 bootstrap；lifecycle/cleanup/bootstrap 的 hard failure 会恢复旧 incarnation。既有 mutation + projection append fail-open 策略仍可提交新 Tape，但旧 projection row 已删除且 meta 会标 stale。 +- context projection 通过单条 `getByEntryIdsIfCurrent` SQL 校验 projection version、projection meta head + 与同步调用方提供的 current Tape head，并读取请求行；不 current 时从 effective Tape 重建 + summary/ref context。 +- search projection 升为 version 3，一次性拒绝可能来自 pre-atomic reset、恰好复用相同 head 的 version + 2 row；current search 按需重建，linked read-only search 在重建前沿用 effective-Tape fallback。 +- search projection 可以重建；projection 不可用或 coverage 不完整时回退 effective Tape search，fork + cleanup 的 projection 删除失败仍不阻断主流程，但 discard receipt 会使后续 merge 和相同 fork ID + 的显式复用 fail closed。 +- legacy chat import 的全表删除是 migration-only 例外，但消息 fact writer 复用 composition 已创建的 + `SessionTape` capability，不再另建 facade；Memory ingestion projection 为避免并发窗口，可以在一条 + 只读 SQL 中同时比较 Tape head 和 projection head。除此之外消费方不得访问物理 Tape 表。 +- reset 物理删除当前 Session Tape 后重新 bootstrap；本阶段没有 archive-on-reset，不能把 reset 解释成 + append-only 运行语义的一部分。 ## View 和 provenance @@ -54,20 +120,33 @@ read boundary 兼容，新写入必须使用 canonical policy/provenance。 `tape_info`、`tape_anchors` 是 diagnostic；`tape_handoff` 是 runtime-only。五个名称全部 reserved， MCP 不能 shadow，持久化 disabled-tool 配置也不能关闭 system capability。 -## Subagent lineage +## Fork 和 Subagent lineage Subagent 使用独立 Session 和独立 Tape。完成后父 Session append 一个 link，固定 child Tape head： - 查询时只读该 frozen head，不自动读取 child 后续 entry； - child entries 不复制进父 Tape； -- discard/merge 只改变父 Session 的 lineage fact，不篡改 child 历史； +- 只有显式授权的直接 child 可以跨 Tape 读取；missing、recreated 或 incarnation 不匹配必须 fail + closed； - 非直接 child、未授权 Session 或递归 Subagent 不能通过 Tape tool 越权读取。 +普通 fork merge 只把 fork head 相对基线的 delta 作为新 entry append 到父 Tape，并追加 merge receipt； +不得改写父 Tape 旧 entry，也不得把整份 fork 历史重复复制。discard 和重复 merge 保持既有审计、幂等及 +best-effort projection cleanup 语义。discard cleanup 成功时与 receipt 一起提交；cleanup 失败时回滚本次 +cleanup、仍 append receipt 并记录 warning。此时残留 fork 是永久惰性数据，本阶段没有自动或后台重试 +路径；discard receipt 仍保证它不能再次 merge，也不能用相同 fork ID 显式复用。 + ## 回放和兼容 Replay 必须保持 entry order、role、tool call/result pairing、anchor cursor 和 policy version。未知旧 fact 可以按兼容规则跳过或映射，但不能静默改变已知 fact 的含义。测试至少覆盖正常 chat、resume、tool interaction、compaction、context pressure、Subagent frozen head 和旧 manifest 读取。 -关键测试位于 `test/main/session/`、`test/main/agent/deepchat/` 和 `test/main/tool/`。历史的 Tape -increment SDD 已合并到本文，详细实施顺序从 Git 历史查询。 +stored manifest validation、legacy `hashVersion` normalization、entry-id collection 和 replay slice hash +属于 `src/main/tape/domain/replay.ts` 的纯逻辑；SQLite row parsing、message trace 和 terminal evidence +读取仍属于 `TapeViewReplayService`，不能反向放进 domain。 + +关键行为测试位于 `test/main/session/data/tape*.test.ts`，分层守护位于 +`test/main/tape/layerBoundaries.test.ts`；runtime 和 tool 契约继续位于 +`test/main/agent/deepchat/` 与 `test/main/tool/`。历史的 Tape increment SDD 已合并到本文，详细实施 +顺序从 Git 历史查询。 diff --git a/scripts/check-memory-test-scope.mjs b/scripts/check-memory-test-scope.mjs index 993f806530..1ce061efc7 100644 --- a/scripts/check-memory-test-scope.mjs +++ b/scripts/check-memory-test-scope.mjs @@ -6,6 +6,9 @@ const CATEGORY_NAMES = ['behavior', 'native', 'eval', 'perf'] const MEMORY_IMPORT = /from ['"][^'"]*(?:\/memory(?:\/|['"])|agent-memory|memory\.routes|agentMemory|recallKeyword)[^'"]*['"]/ const TEST_FILE = /\.(?:test|spec)\.[cm]?[jt]sx?$|\.perf\.[cm]?[jt]sx?$/ +const NATIVE_TAPE_TEST = + /^test\/main\/session\/data\/(?:tape[^/]*|tables\/deepchatTapeEntriesTable)\.(?:test|spec)\.ts$/ +const NATIVE_SQLITE_GATE = /\b(?:itIfSqlite|describeIfSqlite)\b/ function walkTestFiles(rootDir) { const testRoot = join(rootDir, 'test', 'main') @@ -30,11 +33,24 @@ function isMemoryOwned(path, readContent) { return MEMORY_IMPORT.test(readContent(path)) } +export function findRequiredNativeTapeTests(paths, readContent) { + return paths.filter( + (path) => NATIVE_TAPE_TEST.test(path) && NATIVE_SQLITE_GATE.test(readContent(path)) + ) +} + +function discoverRequiredNativeTapeTests(rootDir) { + return findRequiredNativeTapeTests(walkTestFiles(rootDir), (path) => + readFileSync(join(rootDir, path), 'utf8') + ) +} + export function validateMemoryTestScope({ rootDir, manifest, existingPaths, discoveredPaths, + requiredNativePaths = [], fileContents, readFile }) { @@ -101,6 +117,12 @@ export function validateMemoryTestScope({ } } + for (const path of requiredNativePaths) { + if (ownerByPath.get(path) !== 'native') { + errors.push(`Required native Tape test is not classified as native: ${path}`) + } + } + return errors.sort() } @@ -108,7 +130,8 @@ function runCli() { const scriptDirectory = dirname(fileURLToPath(import.meta.url)) const rootDir = resolve(scriptDirectory, '..') const manifest = JSON.parse(readFileSync(join(rootDir, 'test/memory-test-scope.json'), 'utf8')) - const errors = validateMemoryTestScope({ rootDir, manifest }) + const requiredNativePaths = discoverRequiredNativeTapeTests(rootDir) + const errors = validateMemoryTestScope({ rootDir, manifest, requiredNativePaths }) if (errors.length > 0) { console.error(['Memory test scope validation failed:', ...errors.map((error) => `- ${error}`)].join('\n')) process.exitCode = 1 diff --git a/scripts/generate-architecture-baseline.mjs b/scripts/generate-architecture-baseline.mjs index 2172da73c4..ec66e5d901 100644 --- a/scripts/generate-architecture-baseline.mjs +++ b/scripts/generate-architecture-baseline.mjs @@ -24,7 +24,8 @@ const AGENT_SYSTEM_RUNTIME_BOUNDARY_FILES = [ 'src/main/agent/deepchat/runtime/process.ts', 'src/main/agent/deepchat/runtime/dispatch.ts', 'src/main/session/data/transcript.ts', - 'src/main/session/data/tape.ts', + 'src/main/tape/application/sessionTape.ts', + 'src/main/tape/ports/capabilities.ts', 'src/main/provider/providers/acpProvider.ts' ] const AGENT_SYSTEM_EXPECTED_FILES = [ @@ -73,7 +74,11 @@ const AGENT_SYSTEM_OWNER_EVIDENCE = [ 'src/main/agent/deepchat/loop/deepChatLoopEngine.ts', /\bclass DeepChatLoopEngine\b/g ], - ['tapeRecorder', 'src/main/agent/deepchat/loop/ports.ts', /\binterface TapeRecorder\b/g], + [ + 'tapeToolFactWriter', + 'src/main/tape/ports/capabilities.ts', + /\binterface TapeToolFactWriter\b/g + ], [ 'memoryRuntimeCoordinator', 'src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts', diff --git a/src/main/agent/acp/compatibility/adapters.ts b/src/main/agent/acp/compatibility/adapters.ts index 3bfc85a76d..fb1b5c1956 100644 --- a/src/main/agent/acp/compatibility/adapters.ts +++ b/src/main/agent/acp/compatibility/adapters.ts @@ -24,8 +24,8 @@ import { type IoParams, type StreamState } from '@/agent/deepchat/runtime/types' -import { SessionTranscript } from '@/session/data/transcript' -import { SessionTape } from '@/session/data/tape' +import type { SessionTranscript } from '@/session/data/transcript' +import type { TapeReconciliationPort } from '@/tape/ports/capabilities' import { buildPersistableMessageTracePayload } from '@/agent/deepchat/runtime/messageTracePayload' interface ProjectionState { @@ -36,7 +36,7 @@ interface ProjectionState { export interface AcpCompatibilityProjectionAdapterOptions { messageStore: SessionTranscript - tapeService: SessionTape + tapeReconciliation: TapeReconciliationPort writeViewManifest: (input: AcpViewManifestInput) => void | Promise setStatus: (status: 'generating' | 'idle' | 'error') => void publishEvent: DeepChatEventPublisher @@ -56,8 +56,8 @@ export class AcpCompatibilityProjectionAdapter implements AcpCompatibilityProjec sessionId: AcpViewManifestInput['sessionId'] userContent: Parameters[2] }): AcpProjectionHandle { - const { messageStore, tapeService } = this.options - tapeService.ensureSessionTapeReady(input.sessionId, messageStore) + const { messageStore, tapeReconciliation } = this.options + tapeReconciliation.ensureSessionTapeReady(input.sessionId, messageStore) const userMessageId = messageStore.createUserMessage( input.sessionId, messageStore.getNextOrderSeq(input.sessionId), diff --git a/src/main/agent/acp/compatibility/dependencies.ts b/src/main/agent/acp/compatibility/dependencies.ts index 32b6b1908d..42755c8e51 100644 --- a/src/main/agent/acp/compatibility/dependencies.ts +++ b/src/main/agent/acp/compatibility/dependencies.ts @@ -17,7 +17,7 @@ import { createUserChatMessage } from '@/agent/deepchat/runtime/contextBuilder' import { resolveEffectiveActiveSkillNames } from '@/agent/deepchat/resources/systemPromptBuilder' import type { SessionSettingsStore } from '@/session/data/settings' import type { SessionTranscript } from '@/session/data/transcript' -import type { SessionTape } from '@/session/data/tape' +import type { TapeReconciliationPort } from '@/tape/ports/capabilities' import type { DeepChatToolResolver } from '@/agent/deepchat/runtime/toolResolver' import type { DeepChatEventPublisher, @@ -34,7 +34,7 @@ export interface AcpCompatibilityDependencyBuilderDependencies { providerRuntime: Pick sessionStore: SessionSettingsStore messageStore: SessionTranscript - tapeService: SessionTape + tapeReconciliation: TapeReconciliationPort toolResolver: DeepChatToolResolver appendViewManifest(input: AcpViewManifestInput): void setStatus(sessionId: string, status: DeepChatSessionState['status']): void @@ -84,7 +84,7 @@ export function createAcpCompatibilityDependencies( publishEvent: dependencies.publishEvent, publishSessionUpdate: dependencies.publishSessionUpdate, messageStore: dependencies.messageStore, - tapeService: dependencies.tapeService, + tapeReconciliation: dependencies.tapeReconciliation, writeViewManifest: async (manifest) => dependencies.appendViewManifest(manifest), setStatus: (status) => dependencies.setStatus(sessionId, status) }) diff --git a/src/main/agent/deepchat/loop/ports.ts b/src/main/agent/deepchat/loop/ports.ts index 69a939dce1..42bb831725 100644 --- a/src/main/agent/deepchat/loop/ports.ts +++ b/src/main/agent/deepchat/loop/ports.ts @@ -1,11 +1,10 @@ import type { AppSessionId } from '@/agent/shared/agentSessionIds' -import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' +import type { AssistantMessageBlock } from '@shared/types/agent-interface' import type { ChatMessage } from '@shared/types/core/chat-message' import type { LLMCoreStreamEvent } from '@shared/types/core/llm-events' import type { MCPToolCall, MCPToolDefinition, MCPToolResponse } from '@shared/types/core/mcp' import type { ToolCallOptions, ToolPermissionPreCheckResult } from '@shared/types/tool' import type { ModelConfig } from '@shared/types/provider' -import type { DeepChatTapeViewManifest } from '@shared/types/tape-view-manifest' import type { MemorySessionHandle } from '@/agent/deepchat/memory/memoryPromptContributor' export interface ProviderRequest { @@ -162,54 +161,6 @@ export interface ToolResultPort { }): Promise } -export interface TapeEntryRef { - sessionId: AppSessionId - entryId: number -} - -export interface TapeHead { - sessionId: AppSessionId - latestEntryId: number -} - -export interface TapeFactProvenance { - source: 'message' | 'tool_call' | 'tool_result' | 'runtime_event' - sourceId: string - sequence: number -} - -export interface TapeToolFactInput { - sessionId: AppSessionId - messageId: string - orderSeq: number - blockIndex: number - block: AssistantMessageBlock - provenance: TapeFactProvenance -} - -export interface TapeAnchorInput { - sessionId: AppSessionId - name: string - state: Readonly> - meta: Readonly> - provenance: TapeFactProvenance -} - -export interface TapeEffectiveView { - sessionId: AppSessionId - records: ChatMessageRecord[] -} - -export interface TapeRecorder { - ensureSession(input: { sessionId: AppSessionId }): Promise - appendUserMessage(input: { record: ChatMessageRecord }): Promise - appendViewManifest(manifest: DeepChatTapeViewManifest): Promise - appendAssistantFact(input: { record: ChatMessageRecord }): Promise - appendToolFact(input: TapeToolFactInput): Promise - appendAnchor(input: TapeAnchorInput): Promise - readEffectiveView(input: { sessionId: AppSessionId }): Promise -} - export interface OutputSink { update(input: { runId: string diff --git a/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts b/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts index 75fecf0aa7..59719edcc0 100644 --- a/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts +++ b/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts @@ -4,13 +4,14 @@ import { appendMemorySectionWithManifest } from '@/memory/injection' import type { MemoryExecutionToken, MemoryRuntimePort } from '@/memory/injection' import { BUILTIN_DEEPCHAT_AGENT_ID } from '@/agent/repository' import { withSoftDeadline } from '@/memory/core/asyncDeadline' -import { buildEffectiveTapeView } from '@/session/data/tapeEffectiveView' +import { buildEffectiveTapeView } from '@/tape/domain/effectiveView' import type { DeepChatMemoryIngestionCurrentRange, DeepChatMemoryIngestionProjectionInput, DeepChatMemoryIngestionProjectionRow } from '@/memory/data/tables/deepchatMemoryIngestionProjection' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeAnchorWriter, TapeRawEntryReader } from '@/tape/ports/capabilities' import { MEMORY_EXTRACTION_CHUNKS_PER_QUEUE_TASK, buildMemoryExtractionChunks, @@ -68,13 +69,8 @@ export interface MemoryRuntimeCoordinatorDependencies { getMemoryCursorOrderSeq(sessionId: string): number | null updateMemoryCursorOrderSeq(sessionId: string, orderSeq: number): void rewindMemoryCursorOrderSeq(sessionId: string, orderSeq: number): void - getTapeRows(sessionId: string): DeepChatTapeEntryRow[] - appendTapeAnchor(input: { - sessionId: string - name: string - state: Record - meta?: Record - }): void + tapeReader: TapeRawEntryReader + tapeAnchorWriter: TapeAnchorWriter getIngestionProjection(): MemoryIngestionProjection | undefined } @@ -193,7 +189,7 @@ export class MemoryRuntimeCoordinator implements MemoryPromptContributor, Memory } if (this.canContinueExecution(sessionId, executionToken)) { try { - this.deps.appendTapeAnchor({ + this.deps.tapeAnchorWriter.appendAnchor({ sessionId, name: 'memory/view_assembled', state: assembled.manifest as unknown as Record, @@ -406,7 +402,7 @@ export class MemoryRuntimeCoordinator implements MemoryPromptContributor, Memory this.deps.updateMemoryCursorOrderSeq(sessionId, chunk.cursorCommitOrderSeq) } if (result.createdIds.length > 0) { - this.deps.appendTapeAnchor({ + this.deps.tapeAnchorWriter.appendAnchor({ sessionId, name: 'memory/extract', state: { @@ -678,7 +674,7 @@ export class MemoryRuntimeCoordinator implements MemoryPromptContributor, Memory maxEntryId: number, projection: MemoryIngestionProjection ): { rows: DeepChatMemoryIngestionProjectionRow[]; cursorCommitAllowed: boolean } { - const view = buildEffectiveTapeView(this.deps.getTapeRows(sessionId)) + const view = buildEffectiveTapeView(this.deps.tapeReader.getBySession(sessionId)) const projectionRows = this.projectionRowsFromEffectiveView(sessionId, view) try { projection.replaceSession(sessionId, projectionRows, maxEntryId) @@ -732,7 +728,7 @@ export class MemoryRuntimeCoordinator implements MemoryPromptContributor, Memory cursorCommitAllowed: boolean ): { rows: DeepChatMemoryIngestionProjectionRow[]; cursorCommitAllowed: boolean } | null { try { - const view = buildEffectiveTapeView(this.deps.getTapeRows(sessionId)) + const view = buildEffectiveTapeView(this.deps.tapeReader.getBySession(sessionId)) const rows = this.projectionRowsFromEffectiveView(sessionId, view) return { rows: this.filterIngestionRange(rows, fromOrderSeqExclusive, toOrderSeqInclusive), diff --git a/src/main/agent/deepchat/runtime/contextBuilder.ts b/src/main/agent/deepchat/runtime/contextBuilder.ts index 705316c019..cd2deb0e7d 100644 --- a/src/main/agent/deepchat/runtime/contextBuilder.ts +++ b/src/main/agent/deepchat/runtime/contextBuilder.ts @@ -14,7 +14,7 @@ import { estimateMessageTokens, estimateMessagesTokens } from '@shared/utils/messageTokens' -import { isCompactionRecord } from '@/session/data/tapeViewManifest' +import { isCompactionRecord } from '@/tape/domain/viewManifest' export { estimateMessagesTokens } from '@shared/utils/messageTokens' diff --git a/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts b/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts index 99257feccd..7f1773be95 100644 --- a/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts +++ b/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts @@ -66,8 +66,13 @@ import { createTapeViewManifest, resolveTapeViewManifestPolicy, type TapeViewContextSelection -} from '@/session/data/tapeViewManifest' -import type { SessionTape } from '@/session/data/tape' +} from '@/tape/domain/viewManifest' +import type { + TapeReconciliationPort, + TapeToolFactWriter, + TapeViewManifestReader, + TapeViewManifestWriter +} from '@/tape/ports/capabilities' import type { AgentTraceSettingsPort } from '@/agent/traceSettings' import type { DeepChatToolResolver } from '@/agent/deepchat/runtime/toolResolver' import type { @@ -179,7 +184,10 @@ export interface DeepChatLoopRunnerPorts { traceSettings: AgentTraceSettingsPort sessionStore: SessionSettingsStore messageStore: SessionTranscript - tapeService: SessionTape + tapeReconciliation: TapeReconciliationPort + tapeViewManifestReader: TapeViewManifestReader + tapeViewManifestWriter: TapeViewManifestWriter + tapeToolFactWriter: TapeToolFactWriter pendingInputCoordinator: SessionPendingInputs toolResolver: DeepChatToolResolver providerPermissionCoordinator: ProviderPermissionCoordinator @@ -391,7 +399,8 @@ export class DeepChatLoopRunner { const traceEnabled = this.ports.traceSettings.isEnabled() const initialRequestSeq = Math.max( - this.ports.tapeService.listViewManifestsByMessage(sessionId, messageId)[0]?.requestSeq ?? 0, + this.ports.tapeViewManifestReader.listViewManifestsByMessage(sessionId, messageId)[0] + ?.requestSeq ?? 0, this.ports.messageStore.getMaxMessageTraceRequestSeq(messageId) ) const temperature = generationSettings.temperature @@ -754,7 +763,7 @@ export class DeepChatLoopRunner { }, io: { messageStore: this.ports.messageStore, - tapeRecorder: this.ports.tapeService, + tapeToolFactWriter: this.ports.tapeToolFactWriter, publishEvent: this.ports.publishEvent, publishSessionUpdate: this.ports.publishSessionUpdate } @@ -770,7 +779,7 @@ export class DeepChatLoopRunner { } appendTapeViewManifest(params: AppendTapeViewManifestInput): void { - const sourceMaps = this.ports.tapeService.getViewManifestSourceMaps( + const sourceMaps = this.ports.tapeViewManifestReader.getViewManifestSourceMaps( params.sessionId, params.messageId ) @@ -799,7 +808,7 @@ export class DeepChatLoopRunner { supportsAudioInput: params.supportsAudioInput, traceDebugEnabled: params.traceDebugEnabled }) - this.ports.tapeService.appendViewManifest(manifest) + this.ports.tapeViewManifestWriter.appendViewManifest(manifest) } emitRateLimitWaitingMessage( @@ -893,7 +902,10 @@ export class DeepChatLoopRunner { prepareCompaction: async (systemPrompt) => { const prepared = await this.ports.inputPreparationCoordinator.prepareExisting({ ensureHistory: () => - this.ports.tapeService.ensureSessionTapeReady(params.sessionId, this.ports.messageStore) + this.ports.tapeReconciliation.ensureSessionTapeReady( + params.sessionId, + this.ports.messageStore + ) .historyRecords, prepareIntent: async (historyRecords) => await this.ports.compactionService.prepareForContextPressureRecovery({ diff --git a/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts b/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts index d4eee9c9b7..8c1b365a6f 100644 --- a/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts +++ b/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts @@ -329,11 +329,8 @@ export class DeepChatRuntimeCoordinator { this.sqlitePresenter.deepchatSessionsTable.updateMemoryCursorOrderSeq(sessionId, orderSeq), rewindMemoryCursorOrderSeq: (sessionId, orderSeq) => this.sqlitePresenter.deepchatSessionsTable.rewindMemoryCursorOrderSeq(sessionId, orderSeq), - getTapeRows: (sessionId) => - this.sqlitePresenter.deepchatTapeEntriesTable.getBySession(sessionId), - appendTapeAnchor: (input) => { - this.sqlitePresenter.deepchatTapeEntriesTable.appendAnchor(input) - }, + tapeReader: this.tapeService, + tapeAnchorWriter: this.tapeService, getIngestionProjection: runtimePorts.getMemoryIngestionProjection }) this.memoryPromptContributor = this.memoryCoordinator @@ -413,7 +410,10 @@ export class DeepChatRuntimeCoordinator { traceSettings: this.traceSettings, sessionStore: this.sessionStore, messageStore: this.messageStore, - tapeService: this.tapeService, + tapeReconciliation: this.tapeService, + tapeViewManifestReader: this.tapeService, + tapeViewManifestWriter: this.tapeService, + tapeToolFactWriter: this.tapeService, pendingInputCoordinator: this.pendingInputCoordinator, toolResolver: this.toolResolver, providerPermissionCoordinator: this.providerPermissionCoordinator, @@ -460,7 +460,7 @@ export class DeepChatRuntimeCoordinator { toolService: this.toolService, sessionStore: this.sessionStore, messageStore: this.messageStore, - tapeService: this.tapeService, + tapeReconciliation: this.tapeService, pendingInputCoordinator: this.pendingInputCoordinator, toolResolver: this.toolResolver, compactionService: this.compactionService, @@ -578,7 +578,7 @@ export class DeepChatRuntimeCoordinator { providerRuntime: this.providerRuntime, sessionStore: this.sessionStore, messageStore: this.messageStore, - tapeService: this.tapeService, + tapeReconciliation: this.tapeService, toolResolver: this.toolResolver, appendViewManifest: (manifest) => { this.loopRunner.appendTapeViewManifest({ diff --git a/src/main/agent/deepchat/runtime/process.ts b/src/main/agent/deepchat/runtime/process.ts index 8f0c6679f5..b75acefbc0 100644 --- a/src/main/agent/deepchat/runtime/process.ts +++ b/src/main/agent/deepchat/runtime/process.ts @@ -31,7 +31,7 @@ import { } from '@/agent/deepchat/loop/deepChatLoopEngine' import { emitDeepChatLoopNotification } from '@/agent/deepchat/loop/notificationObserver' import type { OutputSink } from '@/agent/deepchat/loop/ports' -import { buildTapeToolFactInputs } from '@/session/data/tapeFacts' +import { buildTapeToolFactInputs } from '@/tape/application/factPersistence' const UNKNOWN_CONTEXT_LIMIT = Number.MAX_SAFE_INTEGER const USER_CANCELED_GENERATION_ERROR = 'common.error.userCanceledGeneration' @@ -739,7 +739,7 @@ export async function processStream(params: ProcessParams): Promise sessionStore: SessionSettingsStore messageStore: SessionTranscript - tapeService: SessionTape + tapeReconciliation: TapeReconciliationPort pendingInputCoordinator: SessionPendingInputs toolResolver: DeepChatToolResolver compactionService: CompactionService @@ -352,7 +352,10 @@ export class TurnCoordinator { ensureHistory: () => this.ports.runSynchronousPreStreamStep(sessionId, 'tape-ready', () => getTapeContextHistoryRecords( - this.ports.tapeService.ensureSessionTapeReady(sessionId, this.ports.messageStore) + this.ports.tapeReconciliation.ensureSessionTapeReady( + sessionId, + this.ports.messageStore + ) .historyRecords ) ), @@ -840,7 +843,10 @@ export class TurnCoordinator { sessionId, 'tape-ready', () => - this.ports.tapeService.ensureSessionTapeReady(sessionId, this.ports.messageStore) + this.ports.tapeReconciliation.ensureSessionTapeReady( + sessionId, + this.ports.messageStore + ) .historyRecords ), refreshHistory: () => @@ -848,7 +854,10 @@ export class TurnCoordinator { sessionId, 'tape-ready', () => - this.ports.tapeService.ensureSessionTapeReady(sessionId, this.ports.messageStore) + this.ports.tapeReconciliation.ensureSessionTapeReady( + sessionId, + this.ports.messageStore + ) .historyRecords ), prepareIntent: async (historyRecords) => { diff --git a/src/main/agent/deepchat/runtime/types.ts b/src/main/agent/deepchat/runtime/types.ts index 176f20c674..7986eef96c 100644 --- a/src/main/agent/deepchat/runtime/types.ts +++ b/src/main/agent/deepchat/runtime/types.ts @@ -17,11 +17,11 @@ import type { DeepChatLoopNotificationObserver, PendingToolInteractionOrigin, PersistedToolBatchState, - TapeRecorder, ToolCatalogPort, ToolExecutionPort, ToolResultPort } from '@/agent/deepchat/loop/ports' +import type { TapeToolFactWriter } from '@/tape/ports/capabilities' export interface InterleavedReasoningConfig { preserveReasoningContent: boolean @@ -91,7 +91,7 @@ export type ProcessIoParams = Pick< IoParams, 'messageStore' | 'publishEvent' | 'publishSessionUpdate' > & { - tapeRecorder: Pick + tapeToolFactWriter: TapeToolFactWriter } export interface ProcessControlCollaborators { diff --git a/src/main/app/composition.ts b/src/main/app/composition.ts index fc74d0499d..d4d93e143f 100644 --- a/src/main/app/composition.ts +++ b/src/main/app/composition.ts @@ -386,7 +386,8 @@ export async function createMainProcessControl(dependencies: { appDatabase, sessionData.database, projectDatabase, - memoryDatabase + memoryDatabase, + sessionData.tapeStore ) usageStatsService = new UsageStatsService( sessionData.database, @@ -986,7 +987,8 @@ export async function createMainProcessControl(dependencies: { transcript: sessionData.transcript, settings: sessionData.settings, pendingInputs: sessionData.pendingInputs, - runtime: deepChatRuntimeCoordinator + runtime: deepChatRuntimeCoordinator, + runInTransaction: (operation) => sessionData.database.getDatabase().transaction(operation)() }) memoryIngestionObserver = deepChatRuntimeCoordinator.memoryIngestionObserver acpAgentRuntime = new AcpAgentRuntime( @@ -1608,7 +1610,7 @@ export async function createMainProcessControl(dependencies: { const memoryRoutes = createMemoryRoutes({ memoryService, getAgentType: (agentId) => agentSettings.getAgentType(agentId), - getTapeEntries: () => sessionData.database.deepchatTapeEntriesTable, + getTapeInspection: () => sessionData.tapeStore, getAuditEntries: () => memoryDatabase.agentMemoryAuditTable }) const desktopRoutes = createDesktopRoutes({ diff --git a/src/main/app/startupMigrations/legacyChatImportService.ts b/src/main/app/startupMigrations/legacyChatImportService.ts index 89904f61f7..acd72273f5 100644 --- a/src/main/app/startupMigrations/legacyChatImportService.ts +++ b/src/main/app/startupMigrations/legacyChatImportService.ts @@ -12,6 +12,7 @@ import { isReasoningEffort } from '@shared/types/model-db' import { resolveAcpAgentAlias } from '@shared/utils/acpAgentAlias' import { SessionTranscript } from '@/session/data/transcript' import { SessionDatabase } from '@/session/data/database' +import type { TapeMessageFactWriter } from '@/tape/ports/capabilities' import type { ProjectDatabase } from '@/project/data/database' import type { AppDatabase } from '@/app/data/database' import type { MemoryDatabase } from '@/memory/data/database' @@ -44,13 +45,14 @@ export class LegacyChatImportService { sessionDatabase: SessionDatabase, projectDatabase: ProjectDatabase, memoryDatabase: MemoryDatabase, + tapeFacts: TapeMessageFactWriter, sourceDbPath?: string ) { this.appDatabase = appDatabase this.sessionDatabase = sessionDatabase this.projectDatabase = projectDatabase this.memoryDatabase = memoryDatabase - this.messageStore = new SessionTranscript(this.sessionDatabase) + this.messageStore = new SessionTranscript(this.sessionDatabase, tapeFacts) this.sourceDbPath = sourceDbPath ?? path.join(app.getPath('userData'), 'app_db', 'chat.db') } @@ -157,6 +159,8 @@ export class LegacyChatImportService { private async clearImportedSessionData(): Promise { const db = this.sessionDatabase.getDatabase() db.transaction(() => { + // Migration-only exception: overwrite import intentionally clears all legacy-owned tables + // in one transaction, including Tape and its projections. db.exec(` DELETE FROM deepchat_message_search_results; DELETE FROM deepchat_search_documents; diff --git a/src/main/data/schemaCatalog.ts b/src/main/data/schemaCatalog.ts index 5922b76c10..55f4a11899 100644 --- a/src/main/data/schemaCatalog.ts +++ b/src/main/data/schemaCatalog.ts @@ -19,9 +19,9 @@ import { DeepChatMessageSearchResultsTable } from '@/session/data/tables/deepcha import { DeepChatSearchDocumentsTable } from '@/session/data/tables/deepchatSearchDocuments' import { DeepChatPendingInputsTable } from '@/session/data/tables/deepchatPendingInputs' import { DeepChatUsageStatsTable } from '@/session/data/tables/deepchatUsageStats' -import { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' +import { DeepChatTapeEntriesTable } from '@/tape/infrastructure/sqlite/tapeEntryStore' import { DeepChatMemoryIngestionProjectionTable } from '@/memory/data/tables/deepchatMemoryIngestionProjection' -import { DeepChatTapeSearchProjectionTable } from '@/session/data/tables/deepchatTapeSearchProjection' +import { DeepChatTapeSearchProjectionTable } from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' import { DeepChatSessionMetadataTable } from '@/session/data/tables/deepchatSessionMetadata' import { LegacyImportStatusTable } from '@/app/data/tables/legacyImportStatus' import { AgentsTable } from '@/agent/data/tables/agents' diff --git a/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts b/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts index 557247e3ba..c59dfc857d 100644 --- a/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts +++ b/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts @@ -1,6 +1,7 @@ import type Database from 'better-sqlite3-multiple-ciphers' import { BaseTable } from '@/data/baseTable' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeMutationProjection } from '@/tape/ports/storage' import type { MemoryPerfObserver } from '../../../memory/ports' import { readTapeMessageRetractionId, @@ -9,7 +10,7 @@ import { tapeEntryToMessageRecord, tapeMessageRank, tapeToolRank -} from '@/session/data/tables/deepchatTapeEffectiveSemantics' +} from '@/tape/domain/effectiveSemantics' export const DEEPCHAT_MEMORY_INGESTION_PROJECTION_VERSION = 1 @@ -92,7 +93,10 @@ const MEMORY_INGESTION_PROJECTION_SQL = ` ); ` -export class DeepChatMemoryIngestionProjectionTable extends BaseTable { +export class DeepChatMemoryIngestionProjectionTable + extends BaseTable + implements TapeMutationProjection +{ constructor( db: Database.Database, private readonly perfObserver?: MemoryPerfObserver @@ -267,6 +271,8 @@ export class DeepChatMemoryIngestionProjectionTable extends BaseTable { toOrderSeqInclusive: number ): DeepChatMemoryIngestionCurrentRange { this.perfObserver?.increment('repositoryCalls') + // Read-only infrastructure exception: the Tape head and projection head must be observed by + // one SQL statement so concurrent appends cannot create a false-current projection window. const records = this.db .prepare( `WITH state AS ( diff --git a/src/main/memory/routes.ts b/src/main/memory/routes.ts index 055b5a20c5..c9f8f5b4d9 100644 --- a/src/main/memory/routes.ts +++ b/src/main/memory/routes.ts @@ -40,7 +40,7 @@ import { type MemoryUpdateResult } from '@shared/contracts/routes/memory.routes' import { createRouteMap, type DeepchatRouteMap } from '@/routes/routeRegistry' -import { buildEffectiveTapeView } from '@/session/data/tapeEffectiveView' +import type { TapeInspectionReader } from '@/tape/ports/capabilities' import type { MemoryConflictPair, MemoryConflictResolution, @@ -57,7 +57,9 @@ const MEMORY_PERSONA_STATES = ['draft', 'active', 'superseded', 'rejected'] as c type MemoryPersonaState = (typeof MEMORY_PERSONA_STATES)[number] const MEMORY_PERSONA_STATE_SET: ReadonlySet = new Set(MEMORY_PERSONA_STATES) -export function formatMemorySourceRecordContent(record: ChatMessageRecord): string { +export function formatMemorySourceRecordContent( + record: Pick +): string { try { const parsed = JSON.parse(record.content) as unknown if (record.role === 'user') { @@ -180,81 +182,6 @@ function toMemoryAuditEventDto(row: AgentMemoryAuditRow) { } } -function deriveSelectedMemoryIds(value: unknown): string[] | null { - if (!Array.isArray(value)) return null - const ids = new Set() - for (const item of value) { - const id = - typeof item === 'string' - ? item - : item && typeof item === 'object' && !Array.isArray(item) - ? (item as Record).id - : null - if (typeof id === 'string' && id.length > 0) ids.add(id) - } - return [...ids] -} - -interface MemoryTapeEntryRow { - session_id: string - entry_id: number - kind: 'event' | 'anchor' | 'message' | 'tool_call' | 'tool_result' - name: string | null - source_type: - | 'session' - | 'message' - | 'assistant_block' - | 'tool_call' - | 'tool_result' - | 'runtime_event' - | 'migration' - | 'summary' - | 'fork' - | 'subagent' - | null - source_id: string | null - source_seq: number | null - provenance_key: string | null - payload_json: string - meta_json: string - created_at: number -} - -function toMemoryViewManifestDto(row: MemoryTapeEntryRow) { - const payload = parseJsonRecord(row.payload_json) - const meta = parseJsonRecord(row.meta_json) - const manifest = - payload.state && typeof payload.state === 'object' && !Array.isArray(payload.state) - ? (payload.state as Record) - : null - if (!manifest) return null - const readNumber = (value: unknown): number => - typeof value === 'number' && Number.isFinite(value) ? value : 0 - return { - sessionId: row.session_id, - messageId: typeof meta.messageId === 'string' ? meta.messageId : null, - entryId: row.entry_id, - policyVersion: - typeof manifest.policyVersion === 'number' && Number.isFinite(manifest.policyVersion) - ? manifest.policyVersion - : null, - tokenBudget: readNumber(manifest.tokenBudget), - estimatedTokens: readNumber(manifest.estimatedTokens), - selectedCount: Array.isArray(manifest.selected) ? manifest.selected.length : 0, - selectedIds: deriveSelectedMemoryIds(manifest.selected), - droppedCount: Array.isArray(manifest.dropped) ? manifest.dropped.length : 0, - queryHash: typeof manifest.queryHash === 'string' ? manifest.queryHash : null, - createdAt: row.created_at - } -} - -type MemoryTapeEntries = { - getBySession(sessionId: string): MemoryTapeEntryRow[] - listMemoryViewManifestAnchorsByAgent( - agentId: string, - options: { sessionId?: string; limit?: number; messageId?: string } - ): MemoryTapeEntryRow[] -} type MemoryAuditEntries = { listByAgent(agentId: string, options: MemoryAuditListOptions): AgentMemoryAuditRow[] } @@ -316,7 +243,7 @@ interface MemoryRouteService { export function createMemoryRoutes(deps: { memoryService: MemoryRouteService getAgentType(agentId: string): Promise - getTapeEntries(): MemoryTapeEntries + getTapeInspection(): TapeInspectionReader getAuditEntries(): MemoryAuditEntries }): DeepchatRouteMap { const { memoryService } = deps @@ -325,9 +252,9 @@ export function createMemoryRoutes(deps: { if (!row || row.agent_id !== agentId || !row.source_session) return null const sourceEntryIds = parseAgentMemorySourceEntryIds(row.source_entry_ids) if (!sourceEntryIds?.length) return null - const sourceSet = new Set(sourceEntryIds) - const entries = buildEffectiveTapeView(deps.getTapeEntries().getBySession(row.source_session)) - .messageEntries.filter((entry) => sourceSet.has(entry.entryId)) + const entries = deps + .getTapeInspection() + .getEffectiveMessageSourceSpan(row.source_session, sourceEntryIds) .map((entry) => ({ entryId: entry.entryId, role: entry.record.role, @@ -519,15 +446,12 @@ export function createMemoryRoutes(deps: { } const limit = input.limit ?? 100 const manifests = deps - .getTapeEntries() - .listMemoryViewManifestAnchorsByAgent(input.agentId, { + .getTapeInspection() + .listMemoryViewManifestsByAgent(input.agentId, { sessionId: input.sessionId, limit, messageId: input.messageId }) - .map(toMemoryViewManifestDto) - .filter((manifest): manifest is NonNullable => Boolean(manifest)) - .filter((manifest) => !input.messageId || manifest.messageId === input.messageId) .slice(0, limit) return memoryListViewManifestsRoute.output.parse({ manifests }) } diff --git a/src/main/session/data/database.ts b/src/main/session/data/database.ts index 9e6d48024b..a28c7c4c59 100644 --- a/src/main/session/data/database.ts +++ b/src/main/session/data/database.ts @@ -15,11 +15,10 @@ import { DeepChatMessageSearchResultsTable } from './tables/deepchatMessageSearc import { DeepChatSearchDocumentsTable } from './tables/deepchatSearchDocuments' import { DeepChatPendingInputsTable } from './tables/deepchatPendingInputs' import { DeepChatUsageStatsTable } from './tables/deepchatUsageStats' -import { - DeepChatTapeEntriesTable, - type DeepChatTapeMutationProjection -} from './tables/deepchatTapeEntries' -import { DeepChatTapeSearchProjectionTable } from './tables/deepchatTapeSearchProjection' +import { DeepChatTapeEntriesTable } from '@/tape/infrastructure/sqlite/tapeEntryStore' +import type { TapeMutationProjection } from '@/tape/ports/storage' +import { SqliteTapeLifecycleAdapter } from '@/tape/infrastructure/sqlite/tapeLifecycleAdapter' +import { DeepChatTapeSearchProjectionTable } from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' import { DeepChatSessionMetadataTable } from './tables/deepchatSessionMetadata' import { NewSessionActiveSkillsTable } from './tables/newSessionActiveSkills' import { NewSessionDisabledAgentToolsTable } from './tables/newSessionDisabledAgentTools' @@ -27,7 +26,7 @@ import { NewSessionDisabledAgentToolsTable } from './tables/newSessionDisabledAg export class SessionDatabase { constructor( private readonly connection: DatabaseConnectionProvider, - private readonly getTapeMutationProjection?: () => DeepChatTapeMutationProjection + private readonly getTapeMutationProjection?: () => TapeMutationProjection ) {} getDatabase() { @@ -98,6 +97,10 @@ export class SessionDatabase { return new DeepChatTapeEntriesTable(this.getDatabase(), this.getTapeMutationProjection?.()) } + get tapeLifecycle() { + return new SqliteTapeLifecycleAdapter(this.getDatabase(), this.getTapeMutationProjection?.()) + } + get deepchatTapeSearchProjectionTable() { return new DeepChatTapeSearchProjectionTable(this.getDatabase()) } diff --git a/src/main/session/data/index.ts b/src/main/session/data/index.ts index bf528a77a3..a64f168ae5 100644 --- a/src/main/session/data/index.ts +++ b/src/main/session/data/index.ts @@ -1,17 +1,17 @@ import type { DatabaseConnectionProvider } from '@/data/databaseConnection' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeMutationProjection } from '@/tape/ports/storage' import type { SessionTapePort } from './contracts' import { SessionPendingInputStore } from './pendingInputStore' import { SessionPendingInputs } from './pendingInputs' import { SessionSettingsStore } from './settings' -import { normalizeTapeHandoffState, SessionTape } from './tape' +import { normalizeTapeHandoffState, SessionTape } from '@/tape/application/sessionTape' import { SessionTranscript } from './transcript' import { SessionDatabase } from './database' -import type { DeepChatTapeMutationProjection } from './tables/deepchatTapeEntries' export function createSessionData( connection: DatabaseConnectionProvider, - getTapeMutationProjection: (() => DeepChatTapeMutationProjection) | undefined, + getTapeMutationProjection: (() => TapeMutationProjection) | undefined, events: SessionDataEvents ) { const database = new SessionDatabase(connection, getTapeMutationProjection) @@ -26,8 +26,8 @@ export function createSessionDataFromDatabase( database: SessionDatabase, events: SessionDataEvents ) { - const transcript = new SessionTranscript(database) const tapeStore = new SessionTape(database) + const transcript = new SessionTranscript(database, tapeStore) const pendingInputStore = new SessionPendingInputStore(database) const ensureTape = (sessionId: string) => tapeStore.ensureSessionTapeReady(sessionId, transcript) const toTapeAnchor = (row: DeepChatTapeEntryRow) => ({ @@ -80,7 +80,7 @@ export function createSessionDataFromDatabase( return { database, - settings: new SessionSettingsStore(database), + settings: new SessionSettingsStore(database, tapeStore), transcript, tape, tapeStore, diff --git a/src/main/session/data/settings.ts b/src/main/session/data/settings.ts index 344edcd8b3..c6831d1821 100644 --- a/src/main/session/data/settings.ts +++ b/src/main/session/data/settings.ts @@ -1,7 +1,12 @@ import { SessionDatabase } from './database' import type { PermissionMode, SessionGenerationSettings } from '@shared/types/agent-interface' import type { DeepChatSessionSummaryRow } from '@/session/data/tables/deepchatSessions' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { + TapeAnchorReader, + TapeAnchorWriter, + TapeLifecycleAdmin +} from '@/tape/ports/capabilities' export type SessionSummaryState = { summaryText: string | null @@ -131,9 +136,14 @@ function summaryStatesEqual(left: SessionSummaryState, right: SessionSummaryStat export class SessionSettingsStore { private database: SessionDatabase + private readonly tape: TapeAnchorReader & TapeAnchorWriter & TapeLifecycleAdmin - constructor(database: SessionDatabase) { + constructor( + database: SessionDatabase, + tape: TapeAnchorReader & TapeAnchorWriter & TapeLifecycleAdmin + ) { this.database = database + this.tape = tape } create( @@ -150,7 +160,7 @@ export class SessionSettingsStore { permissionMode, generationSettings ) - this.database.deepchatTapeEntriesTable.ensureBootstrapAnchor(id) + this.tape.initializeSessionTape(id) } get(id: string) { @@ -158,8 +168,7 @@ export class SessionSettingsStore { } delete(id: string): void { - this.database.deepchatTapeEntriesTable.deleteBySession(id) - this.database.deepchatTapeSearchProjectionTable.deleteBySession(id) + this.tape.deleteSessionTape(id) this.database.deepchatSessionsTable.delete(id) } @@ -198,8 +207,7 @@ export class SessionSettingsStore { } getSummaryState(id: string): SessionSummaryState { - const tapeTable = this.database.deepchatTapeEntriesTable - const tapeState = summaryStateFromTapeAnchor(tapeTable.getLatestReconstructionAnchor(id)) + const tapeState = summaryStateFromTapeAnchor(this.tape.getLatestReconstructionAnchor(id)) if (tapeState) { return tapeState } @@ -208,9 +216,7 @@ export class SessionSettingsStore { } getReconstructionAnchorPromptState(id: string): ReconstructionAnchorPromptState | null { - return reconstructionAnchorPromptStateFromRow( - this.database.deepchatTapeEntriesTable.getLatestReconstructionAnchor(id) - ) + return reconstructionAnchorPromptStateFromRow(this.tape.getLatestReconstructionAnchor(id)) } updateSummaryState(id: string, state: SessionSummaryState): void { @@ -224,8 +230,7 @@ export class SessionSettingsStore { tapeAnchor?: SummaryTapeAnchorInput ): SummaryStateCompareAndSetResult { const applyUpdate = (): boolean => { - const tapeTable = this.database.deepchatTapeEntriesTable - const latestTapeAnchor = tapeTable.getLatestReconstructionAnchor(id) + const latestTapeAnchor = this.tape.getLatestReconstructionAnchor(id) const currentState = this.getSummaryState(id) if (!summaryStatesEqual(currentState, expectedState)) { return false @@ -236,7 +241,7 @@ export class SessionSettingsStore { this.database.deepchatSessionsTable.updateSummaryState(id, nextState) if (tapeAnchor) { - tapeTable.appendAnchor({ + this.tape.appendAnchor({ sessionId: id, name: tapeAnchor.name, state: tapeAnchor.state, @@ -265,7 +270,7 @@ export class SessionSettingsStore { resetSummaryState(id: string): void { const reset = (): void => { this.database.deepchatSessionsTable.resetSummaryState(id) - this.database.deepchatTapeEntriesTable.appendAnchor({ + this.tape.appendAnchor({ sessionId: id, name: 'summary/reset', state: { @@ -278,8 +283,6 @@ export class SessionSettingsStore { } resetTape(id: string): void { - this.database.deepchatTapeEntriesTable.deleteBySession(id) - this.database.deepchatTapeSearchProjectionTable.deleteBySession(id) - this.database.deepchatTapeEntriesTable.ensureBootstrapAnchor(id) + this.tape.resetSessionTape(id) } } diff --git a/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts b/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts index 932234b62f..3223a637b4 100644 --- a/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts +++ b/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts @@ -1,161 +1,16 @@ -import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' -import type { DeepChatTapeEntryRow } from './deepchatTapeEntries' - -const TERMINAL_TAPE_TOOL_STATUSES = new Set(['success', 'error']) - -export interface DeepChatTapeToolIdentity { - key: string - messageId: string -} - -export function parseTapeJsonObject(raw: string): Record { - try { - const parsed = JSON.parse(raw) as unknown - if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { - return parsed as Record - } - } catch {} - return {} -} - -export function parseNestedTapeJsonObject(value: unknown): Record { - if (typeof value === 'string') { - return parseTapeJsonObject(value) - } - if (value && typeof value === 'object' && !Array.isArray(value)) { - return value as Record - } - return {} -} - -export function parseAssistantBlocks(rawContent: string): AssistantMessageBlock[] { - try { - const parsed = JSON.parse(rawContent) as unknown - return Array.isArray(parsed) ? (parsed as AssistantMessageBlock[]) : [] - } catch { - return [] - } -} - -export function messageRecordHasFinalToolUse(record: ChatMessageRecord): boolean { - if (record.role !== 'assistant' || (record.status !== 'sent' && record.status !== 'error')) { - return false - } - const blocks = parseAssistantBlocks(record.content) - const pendingInteractionToolIds = new Set( - blocks.flatMap((block) => - block.type === 'action' && - (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && - block.status === 'pending' && - typeof block.tool_call?.id === 'string' - ? [block.tool_call.id] - : [] - ) - ) - return blocks.some( - (block) => - block.type === 'tool_call' && - (block.status === 'success' || block.status === 'error') && - typeof block.tool_call?.id === 'string' && - !pendingInteractionToolIds.has(block.tool_call.id) - ) -} - -function isMessageStatus(value: unknown): value is ChatMessageRecord['status'] { - return value === 'pending' || value === 'sent' || value === 'error' -} - -export function tapeEntryToMessageRecord(row: DeepChatTapeEntryRow): ChatMessageRecord | null { - if (row.kind !== 'message') { - return null - } - - const payload = parseTapeJsonObject(row.payload_json) - const record = payload.record - if (!record || typeof record !== 'object' || Array.isArray(record)) { - return null - } - - const candidate = record as Partial - if ( - typeof candidate.id !== 'string' || - typeof candidate.sessionId !== 'string' || - typeof candidate.orderSeq !== 'number' || - (candidate.role !== 'user' && candidate.role !== 'assistant') || - typeof candidate.content !== 'string' - ) { - return null - } - - return { - id: candidate.id, - sessionId: candidate.sessionId, - orderSeq: candidate.orderSeq, - role: candidate.role, - content: candidate.content, - status: isMessageStatus(candidate.status) ? candidate.status : 'sent', - isContextEdge: typeof candidate.isContextEdge === 'number' ? candidate.isContextEdge : 0, - metadata: typeof candidate.metadata === 'string' ? candidate.metadata : '{}', - traceCount: typeof candidate.traceCount === 'number' ? candidate.traceCount : 0, - createdAt: typeof candidate.createdAt === 'number' ? candidate.createdAt : row.created_at, - updatedAt: typeof candidate.updatedAt === 'number' ? candidate.updatedAt : row.created_at - } -} - -export function tapeMessageRank(record: ChatMessageRecord, includePending: boolean): number { - if (record.status === 'sent' || record.status === 'error') { - return 2 - } - return includePending && record.status === 'pending' ? 1 : 0 -} - -export function readTapeMessageRetractionId(row: DeepChatTapeEntryRow): string | null { - if (row.kind !== 'event' || row.name !== 'message/retracted') { - return null - } - - const payload = parseTapeJsonObject(row.payload_json) - const data = parseNestedTapeJsonObject(payload.data) - return typeof data.messageId === 'string' ? data.messageId : null -} - -export function readTapeToolStatus(row: DeepChatTapeEntryRow): string | null { - const meta = parseTapeJsonObject(row.meta_json) - return typeof meta.status === 'string' ? meta.status : null -} - -export function tapeToolRank(row: DeepChatTapeEntryRow, includePending: boolean): number { - const status = readTapeToolStatus(row) - if (status === 'pending') { - return includePending ? 1 : 0 - } - return status !== null && TERMINAL_TAPE_TOOL_STATUSES.has(status) ? 2 : 0 -} - -export function readTapeToolIdentity(row: DeepChatTapeEntryRow): DeepChatTapeToolIdentity | null { - if (row.kind !== 'tool_call' && row.kind !== 'tool_result') { - return null - } - - const payload = parseTapeJsonObject(row.payload_json) - const messageId = payload.messageId - if (typeof messageId !== 'string' || messageId.length === 0) { - return null - } - - let toolCallId: unknown - if (row.kind === 'tool_call') { - toolCallId = parseNestedTapeJsonObject(payload.toolCall).id - } else { - toolCallId = payload.toolCallId - } - - if (typeof toolCallId !== 'string' || toolCallId.length === 0) { - return null - } - - return { - key: `${row.kind}:${messageId}:${toolCallId}`, - messageId - } -} +/** @deprecated Import Tape semantics from `@/tape/domain/effectiveSemantics`. */ +export { + messageRecordHasFinalToolUse, + parseAssistantBlocks, + parseNestedTapeJsonObject, + parseTapeJsonObject, + readTapeMessageRetractionId, + readTapeToolIdentity, + readTapeToolStatus, + tapeEntryToMessageRecord, + tapeMessageRank, + tapeToolRank +} from '@/tape/domain/effectiveSemantics' + +/** @deprecated Import Tape semantic types from `@/tape/domain/effectiveSemantics`. */ +export type { DeepChatTapeToolIdentity } from '@/tape/domain/effectiveSemantics' diff --git a/src/main/session/data/tables/deepchatTapeEntries.ts b/src/main/session/data/tables/deepchatTapeEntries.ts index 2987280341..c34220272c 100644 --- a/src/main/session/data/tables/deepchatTapeEntries.ts +++ b/src/main/session/data/tables/deepchatTapeEntries.ts @@ -1,1158 +1,22 @@ -import Database from 'better-sqlite3-multiple-ciphers' -import logger from '@shared/logger' -import { BaseTable } from '@/data/baseTable' -import { randomUUID } from 'crypto' - -export type DeepChatTapeEntryKind = 'event' | 'anchor' | 'message' | 'tool_call' | 'tool_result' - -export const TAPE_INCARNATION_META_KEY = 'tapeIncarnationId' - -export type DeepChatTapeSourceType = - | 'session' - | 'message' - | 'assistant_block' - | 'tool_call' - | 'tool_result' - | 'runtime_event' - | 'migration' - | 'summary' - | 'fork' - | 'subagent' - -export interface DeepChatTapeEntryRow { - session_id: string - entry_id: number - kind: DeepChatTapeEntryKind - name: string | null - source_type: DeepChatTapeSourceType | null - source_id: string | null - source_seq: number | null - provenance_key: string | null - payload_json: string - meta_json: string - created_at: number -} - -export interface DeepChatTapeSourceInput { - type: DeepChatTapeSourceType - id: string - seq?: number | null -} - -export interface DeepChatTapeAppendInput { - sessionId: string - kind: DeepChatTapeEntryKind - name?: string | null - source?: DeepChatTapeSourceInput | null - provenanceKey?: string | null - payload: Record - meta?: Record - createdAt?: number - idempotent?: boolean -} - -export interface DeepChatTapeSearchInput { - limit?: number - kinds?: DeepChatTapeEntryKind[] - startCreatedAt?: number - endCreatedAt?: number -} - -export interface DeepChatTapeReadSource { - sessionId: string - maxEntryId: number -} - -export interface DeepChatTapeMutationProjection { - applyAppendedEntry(row: DeepChatTapeEntryRow, previousSessionMaxEntryId: number): boolean - invalidateSession(sessionId: string): void - deleteBySession(sessionId: string): void -} - -export const SUMMARY_ANCHOR_NAMES = [ - 'compaction/auto', - 'compaction/manual', - 'compaction/context_pressure', - 'compaction/resume', - 'compaction/migrated_summary', - 'auto_handoff/context_overflow', - 'summary/reset' -] as const - -const RECONSTRUCTION_ANCHOR_NAMES = SUMMARY_ANCHOR_NAMES - -const TAPE_ENTRY_INDEX_SQL = ` - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_kind - ON deepchat_tape_entries(session_id, kind, entry_id); - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_name - ON deepchat_tape_entries(session_id, name, entry_id); - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_source - ON deepchat_tape_entries(session_id, source_type, source_id, source_seq); - CREATE UNIQUE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_provenance - ON deepchat_tape_entries(session_id, provenance_key) - WHERE provenance_key IS NOT NULL; -` - -function safeJsonStringify(value: Record | undefined): string { - return JSON.stringify(value ?? {}) -} - -function buildProvenanceKey(input: DeepChatTapeAppendInput): string | null { - if (input.provenanceKey !== undefined) { - return input.provenanceKey - } - if (!input.source?.type || !input.source.id) { - return null - } - return [ - input.source.type, - input.source.id, - input.source.seq ?? 0, - input.kind, - input.name ?? '' - ].join(':') -} - -function escapeLikePattern(value: string): string { - return value.replace(/[\\%_]/g, (character) => `\\${character}`) -} - -// Three LIKE parameters per field group stay below SQLite's portable 999-variable floor after -// source, filter, and limit bindings. -const MAX_TAPE_SEARCH_TOKEN_CLAUSES = 256 - -function tokenizeDeepChatTapeSearchQuery(value: string): string[] { - return value - .split(/\s+/) - .map((token) => token.trim()) - .filter(Boolean) -} - -export function buildDeepChatTapeFtsMatch(value: string): string { - const tokens = tokenizeDeepChatTapeSearchQuery(value) - const values = - tokens.length > 1 && tokens.length <= MAX_TAPE_SEARCH_TOKEN_CLAUSES ? tokens : [value] - return values.map((token) => `"${token.replace(/"/g, '""')}"`).join(' AND ') -} - -export function buildDeepChatTapeLikeSearchPredicate( - fieldExpressions: readonly [string, ...string[]], - normalizedQuery: string -): { sql: string; params: string[] } { - const fieldClause = `(${fieldExpressions - .map((field) => `${field} LIKE ? ESCAPE '\\'`) - .join(' OR ')})` - const queryClauses = [fieldClause] - const queryPattern = `%${escapeLikePattern(normalizedQuery)}%` - const params = fieldExpressions.map(() => queryPattern) - const tokens = tokenizeDeepChatTapeSearchQuery(normalizedQuery) - - if (tokens.length > 1 && tokens.length <= MAX_TAPE_SEARCH_TOKEN_CLAUSES) { - queryClauses.push(`(${tokens.map(() => fieldClause).join(' AND ')})`) - for (const token of tokens) { - const tokenPattern = `%${escapeLikePattern(token)}%` - params.push(...fieldExpressions.map(() => tokenPattern)) - } - } - - return { - sql: `(${queryClauses.join(' OR ')})`, - params - } -} - -export function normalizeDeepChatTapeReadSources( - sources: readonly DeepChatTapeReadSource[] -): DeepChatTapeReadSource[] { - const maxEntryIdBySession = new Map() - for (const source of sources) { - const sessionId = source.sessionId.trim() - if (!sessionId || !Number.isSafeInteger(source.maxEntryId) || source.maxEntryId < 0) { - continue - } - maxEntryIdBySession.set( - sessionId, - Math.max(maxEntryIdBySession.get(sessionId) ?? 0, source.maxEntryId) - ) - } - return [...maxEntryIdBySession.entries()] - .map(([sessionId, maxEntryId]) => ({ sessionId, maxEntryId })) - .sort((left, right) => - left.sessionId < right.sessionId ? -1 : left.sessionId > right.sessionId ? 1 : 0 - ) -} - -export function serializeDeepChatTapeReadSources( - sources: readonly DeepChatTapeReadSource[] -): string { - return JSON.stringify(normalizeDeepChatTapeReadSources(sources)) -} - -const AUTHORIZED_TAPE_SOURCES_CTE_SQL = ` - authorized_sources(session_id, max_entry_id) AS ( - SELECT - json_extract(value, '$.sessionId'), - CAST(json_extract(value, '$.maxEntryId') AS INTEGER) - FROM json_each(?) - ) -` - -function effectiveTapeMessagePredicateSql(alias: string): string { - return ` - json_type(${alias}.payload_json, '$.record') = 'object' - AND typeof(json_extract(${alias}.payload_json, '$.record.id')) = 'text' - AND typeof(json_extract(${alias}.payload_json, '$.record.sessionId')) = 'text' - AND typeof(json_extract(${alias}.payload_json, '$.record.orderSeq')) IN ('integer', 'real') - AND json_extract(${alias}.payload_json, '$.record.role') IN ('user', 'assistant') - AND typeof(json_extract(${alias}.payload_json, '$.record.content')) = 'text' - AND ( - json_extract(${alias}.payload_json, '$.record.status') IS NULL - OR json_extract(${alias}.payload_json, '$.record.status') != 'pending' - ) - ` -} - -function tapeRetractionMessageIdSql(alias: string): string { - return ` - CASE - WHEN json_type(${alias}.payload_json, '$.data') = 'object' - THEN json_extract(${alias}.payload_json, '$.data.messageId') - WHEN json_type(${alias}.payload_json, '$.data') = 'text' - AND json_valid(json_extract(${alias}.payload_json, '$.data')) - THEN json_extract(json_extract(${alias}.payload_json, '$.data'), '$.messageId') - ELSE NULL - END - ` -} - -function tapeToolCallIdSql(alias: string): string { - return ` - CASE - WHEN ${alias}.kind = 'tool_result' - THEN json_extract(${alias}.payload_json, '$.toolCallId') - WHEN json_type(${alias}.payload_json, '$.toolCall') = 'object' - THEN json_extract(${alias}.payload_json, '$.toolCall.id') - WHEN json_type(${alias}.payload_json, '$.toolCall') = 'text' - AND json_valid(json_extract(${alias}.payload_json, '$.toolCall')) - THEN json_extract(json_extract(${alias}.payload_json, '$.toolCall'), '$.id') - ELSE NULL - END - ` -} - -// These read-only SQL forms mirror deepchatTapeEffectiveSemantics. Search uses correlated -// candidate validation to avoid materializing a whole linked Tape; context uses the complete -// effective CTE because it needs stable neighboring-row positions. Native tests cover parity for -// replacement, retraction, head cutoff, and window ordering. -const EFFECTIVE_TAPE_ROWS_CTE_SQL = ` - bounded_rows AS ( - SELECT tape.* - FROM deepchat_tape_entries AS tape - INNER JOIN authorized_sources AS source - ON source.session_id = tape.session_id - AND tape.entry_id <= source.max_entry_id - ), - raw_message_candidates AS ( - SELECT - bounded_rows.*, - json_extract(payload_json, '$.record.id') AS message_id - FROM bounded_rows - WHERE kind = 'message' - ), - message_candidates AS ( - SELECT * - FROM raw_message_candidates - WHERE ${effectiveTapeMessagePredicateSql('raw_message_candidates')} - ), - ranked_messages AS ( - SELECT - message_candidates.*, - ROW_NUMBER() OVER ( - PARTITION BY session_id, message_id - ORDER BY entry_id DESC - ) AS candidate_rank - FROM message_candidates - ), - effective_message_rows AS ( - SELECT ranked_messages.* - FROM ranked_messages - WHERE candidate_rank = 1 - AND NOT EXISTS ( - SELECT 1 - FROM bounded_rows AS retraction - WHERE retraction.session_id = ranked_messages.session_id - AND retraction.kind = 'event' - AND retraction.name = 'message/retracted' - AND retraction.entry_id > ranked_messages.entry_id - AND (${tapeRetractionMessageIdSql('retraction')}) = ranked_messages.message_id - ) - ), - raw_tool_candidates AS ( - SELECT - bounded_rows.*, - json_extract(payload_json, '$.messageId') AS message_id, - (${tapeToolCallIdSql('bounded_rows')}) AS tool_call_id, - json_extract(meta_json, '$.status') AS tool_status - FROM bounded_rows - WHERE kind IN ('tool_call', 'tool_result') - ), - tool_candidates AS ( - SELECT * - FROM raw_tool_candidates - WHERE typeof(message_id) = 'text' - AND length(message_id) > 0 - AND typeof(tool_call_id) = 'text' - AND length(tool_call_id) > 0 - AND tool_status IN ('success', 'error') - ), - ranked_tools AS ( - SELECT - tool_candidates.*, - ROW_NUMBER() OVER ( - PARTITION BY session_id, kind, message_id, tool_call_id - ORDER BY entry_id DESC - ) AS candidate_rank - FROM tool_candidates - ), - effective_rows AS ( - SELECT - session_id, entry_id, kind, name, source_type, source_id, source_seq, - provenance_key, payload_json, meta_json, created_at - FROM bounded_rows - WHERE kind = 'anchor' - UNION ALL - SELECT - session_id, entry_id, kind, name, source_type, source_id, source_seq, - provenance_key, payload_json, meta_json, created_at - FROM bounded_rows - WHERE kind = 'event' - AND (name IS NULL OR name NOT IN ( - 'message/retracted', - 'message/compaction_indicator', - 'migration/backfill' - )) - UNION ALL - SELECT - session_id, entry_id, kind, name, source_type, source_id, source_seq, - provenance_key, payload_json, meta_json, created_at - FROM effective_message_rows - UNION ALL - SELECT - ranked_tools.session_id, - ranked_tools.entry_id, - ranked_tools.kind, - ranked_tools.name, - ranked_tools.source_type, - ranked_tools.source_id, - ranked_tools.source_seq, - ranked_tools.provenance_key, - ranked_tools.payload_json, - ranked_tools.meta_json, - ranked_tools.created_at - FROM ranked_tools - INNER JOIN effective_message_rows AS message - ON message.session_id = ranked_tools.session_id - AND message.message_id = ranked_tools.message_id - WHERE ranked_tools.candidate_rank = 1 - ) -` - -const EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL = ` - candidate.kind = 'anchor' - OR ( - candidate.kind = 'event' - AND (candidate.name IS NULL OR candidate.name NOT IN ( - 'message/retracted', - 'message/compaction_indicator', - 'migration/backfill' - )) - ) - OR ( - candidate.kind = 'message' - AND ${effectiveTapeMessagePredicateSql('candidate')} - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS later_message - WHERE later_message.session_id = candidate.session_id - AND later_message.entry_id > candidate.entry_id - AND later_message.entry_id <= source.max_entry_id - AND later_message.kind = 'message' - AND ${effectiveTapeMessagePredicateSql('later_message')} - AND json_extract(later_message.payload_json, '$.record.id') = - json_extract(candidate.payload_json, '$.record.id') - ) - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS retraction - WHERE retraction.session_id = candidate.session_id - AND retraction.entry_id > candidate.entry_id - AND retraction.entry_id <= source.max_entry_id - AND retraction.kind = 'event' - AND retraction.name = 'message/retracted' - AND (${tapeRetractionMessageIdSql('retraction')}) = - json_extract(candidate.payload_json, '$.record.id') - ) - ) - OR ( - candidate.kind IN ('tool_call', 'tool_result') - AND json_extract(candidate.meta_json, '$.status') IN ('success', 'error') - AND typeof(json_extract(candidate.payload_json, '$.messageId')) = 'text' - AND length(json_extract(candidate.payload_json, '$.messageId')) > 0 - AND typeof((${tapeToolCallIdSql('candidate')})) = 'text' - AND length((${tapeToolCallIdSql('candidate')})) > 0 - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS later_tool - WHERE later_tool.session_id = candidate.session_id - AND later_tool.entry_id > candidate.entry_id - AND later_tool.entry_id <= source.max_entry_id - AND later_tool.kind = candidate.kind - AND json_extract(later_tool.meta_json, '$.status') IN ('success', 'error') - AND json_extract(later_tool.payload_json, '$.messageId') = - json_extract(candidate.payload_json, '$.messageId') - AND (${tapeToolCallIdSql('later_tool')}) = (${tapeToolCallIdSql('candidate')}) - ) - AND EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS message - WHERE message.session_id = candidate.session_id - AND message.entry_id <= source.max_entry_id - AND message.kind = 'message' - AND ${effectiveTapeMessagePredicateSql('message')} - AND json_extract(message.payload_json, '$.record.id') = - json_extract(candidate.payload_json, '$.messageId') - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS later_message - WHERE later_message.session_id = message.session_id - AND later_message.entry_id > message.entry_id - AND later_message.entry_id <= source.max_entry_id - AND later_message.kind = 'message' - AND ${effectiveTapeMessagePredicateSql('later_message')} - AND json_extract(later_message.payload_json, '$.record.id') = - json_extract(message.payload_json, '$.record.id') - ) - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS retraction - WHERE retraction.session_id = message.session_id - AND retraction.entry_id > message.entry_id - AND retraction.entry_id <= source.max_entry_id - AND retraction.kind = 'event' - AND retraction.name = 'message/retracted' - AND (${tapeRetractionMessageIdSql('retraction')}) = - json_extract(message.payload_json, '$.record.id') - ) - ) - ) -` - -export class DeepChatTapeEntriesTable extends BaseTable { - constructor( - db: Database.Database, - private readonly mutationProjection?: DeepChatTapeMutationProjection - ) { - super(db, 'deepchat_tape_entries') - } - - getCreateTableSQL(): string { - return ` - CREATE TABLE IF NOT EXISTS deepchat_tape_entries ( - session_id TEXT NOT NULL, - entry_id INTEGER NOT NULL, - kind TEXT NOT NULL, - name TEXT, - source_type TEXT, - source_id TEXT, - source_seq INTEGER, - provenance_key TEXT, - payload_json TEXT NOT NULL DEFAULT '{}', - meta_json TEXT NOT NULL DEFAULT '{}', - created_at INTEGER NOT NULL, - PRIMARY KEY (session_id, entry_id) - ); - ${TAPE_ENTRY_INDEX_SQL} - ` - } - - public createTable(): void { - if (!this.tableExists()) { - this.db.exec(this.getCreateTableSQL()) - return - } - this.ensureProvenanceColumns() - this.db.exec(TAPE_ENTRY_INDEX_SQL) - } - - getMigrationSQL(_version: number): string | null { - return null - } - - getLatestVersion(): number { - return 0 - } - - runInTransaction(operation: () => T): T { - return this.db.transaction(operation)() - } - - append(input: DeepChatTapeAppendInput): DeepChatTapeEntryRow { - const append = this.db.transaction(() => { - const provenanceKey = buildProvenanceKey(input) - if (input.idempotent && provenanceKey) { - const existing = this.getByProvenanceKey(input.sessionId, provenanceKey) - if (existing) { - return existing - } - } - - const createdAt = input.createdAt ?? Date.now() - const previousSessionMaxEntryId = this.getMaxEntryId(input.sessionId) - const row = { - session_id: input.sessionId, - entry_id: previousSessionMaxEntryId + 1, - kind: input.kind, - name: input.name ?? null, - source_type: input.source?.type ?? null, - source_id: input.source?.id ?? null, - source_seq: input.source?.seq ?? null, - provenance_key: provenanceKey, - payload_json: safeJsonStringify(input.payload), - meta_json: safeJsonStringify(input.meta), - created_at: createdAt - } satisfies DeepChatTapeEntryRow - - try { - this.db - .prepare( - `INSERT INTO deepchat_tape_entries ( - session_id, - entry_id, - kind, - name, - source_type, - source_id, - source_seq, - provenance_key, - payload_json, - meta_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - .run( - row.session_id, - row.entry_id, - row.kind, - row.name, - row.source_type, - row.source_id, - row.source_seq, - row.provenance_key, - row.payload_json, - row.meta_json, - row.created_at - ) - } catch (error) { - if (input.idempotent && provenanceKey) { - const existing = this.getByProvenanceKey(input.sessionId, provenanceKey) - if (existing) { - return existing - } - } - throw error - } - - if (this.mutationProjection) { - try { - const applyProjection = this.db.transaction(() => - this.mutationProjection?.applyAppendedEntry(row, previousSessionMaxEntryId) - ) - applyProjection() - } catch (error) { - this.mutationProjection.invalidateSession(row.session_id) - logger.warn( - `[Tape] memory ingestion projection append failed; session marked stale: ${String(error)}` - ) - } - } - - return row - }) - - return append() - } - - appendAnchor(input: { - sessionId: string - name: string - state: Record - meta?: Record - source?: DeepChatTapeSourceInput | null - provenanceKey?: string | null - createdAt?: number - idempotent?: boolean - }): DeepChatTapeEntryRow { - return this.append({ - sessionId: input.sessionId, - kind: 'anchor', - name: input.name, - source: input.source, - provenanceKey: input.provenanceKey, - payload: { - name: input.name, - state: input.state - }, - meta: input.meta, - createdAt: input.createdAt, - idempotent: input.idempotent - }) - } - - appendEvent(input: { - sessionId: string - name: string - data: Record - meta?: Record - source?: DeepChatTapeSourceInput | null - provenanceKey?: string | null - createdAt?: number - idempotent?: boolean - }): DeepChatTapeEntryRow { - return this.append({ - sessionId: input.sessionId, - kind: 'event', - name: input.name, - source: input.source, - provenanceKey: input.provenanceKey, - payload: { - name: input.name, - data: input.data - }, - meta: input.meta, - createdAt: input.createdAt, - idempotent: input.idempotent - }) - } - - ensureBootstrapAnchor(sessionId: string): void { - const existing = this.db - .prepare( - `SELECT entry_id - FROM deepchat_tape_entries - WHERE session_id = ? AND kind = 'anchor' - ORDER BY entry_id ASC - LIMIT 1` - ) - .get(sessionId) as { entry_id: number } | undefined - - if (existing) { - return - } - - this.appendAnchor({ - sessionId, - name: 'session/start', - source: { - type: 'session', - id: sessionId, - seq: 0 - }, - state: { - owner: 'human' - }, - meta: { - [TAPE_INCARNATION_META_KEY]: randomUUID() - }, - idempotent: true - }) - } - - getBySession(sessionId: string): DeepChatTapeEntryRow[] { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - ORDER BY entry_id ASC` - ) - .all(sessionId) as DeepChatTapeEntryRow[] - } - - getSubagentLineageEvents(sessionId: string): DeepChatTapeEntryRow[] { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - AND kind = 'event' - AND name IN ('subagent/tape_linked', 'fork/merge') - ORDER BY entry_id ASC` - ) - .all(sessionId) as DeepChatTapeEntryRow[] - } - - getFirstEntriesBySessions(sessionIds: string[]): DeepChatTapeEntryRow[] { - const ids = [...new Set(sessionIds.map((id) => id.trim()).filter(Boolean))] - if (ids.length === 0) { - return [] - } - return this.db - .prepare( - `WITH requested_sessions(session_id) AS ( - SELECT value FROM json_each(?) - ), - first_entries(session_id, entry_id) AS ( - SELECT tape.session_id, MIN(tape.entry_id) - FROM deepchat_tape_entries AS tape - INNER JOIN requested_sessions AS requested - ON requested.session_id = tape.session_id - GROUP BY tape.session_id - ) - SELECT tape.* - FROM deepchat_tape_entries AS tape - INNER JOIN first_entries AS first - ON first.session_id = tape.session_id - AND first.entry_id = tape.entry_id - ORDER BY tape.session_id ASC` - ) - .all(JSON.stringify(ids)) as DeepChatTapeEntryRow[] - } - - getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND entry_id <= ? - ORDER BY entry_id ASC` - ) - .all(sessionId, maxEntryId) as DeepChatTapeEntryRow[] - } - - listMemoryViewManifestAnchorsBySessions( - sessionIds: string[], - optionsOrLimit: number | { limit?: number; messageId?: string } = 100 - ): DeepChatTapeEntryRow[] { - const uniqueSessionIds = [...new Set(sessionIds.filter((id) => id.trim().length > 0))] - if (uniqueSessionIds.length === 0) { - return [] - } - const options = typeof optionsOrLimit === 'number' ? { limit: optionsOrLimit } : optionsOrLimit - const cappedLimit = Math.min(Math.max(Math.floor(options.limit ?? 100), 1), 500) - const placeholders = uniqueSessionIds.map(() => '?').join(', ') - const whereClauses = [ - `session_id IN (${placeholders})`, - "kind = 'anchor'", - "name = 'memory/view_assembled'" - ] - const params: Array = [...uniqueSessionIds] - if (options.messageId) { - whereClauses.push("json_extract(meta_json, '$.messageId') = ?") - params.push(options.messageId) - } - params.push(cappedLimit) - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE ${whereClauses.join(' AND ')} - ORDER BY created_at DESC, entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeEntryRow[] - } - - listMemoryViewManifestAnchorsByAgent( - agentId: string, - options: { sessionId?: string; limit?: number; messageId?: string } = {} - ): DeepChatTapeEntryRow[] { - const cappedLimit = Math.min(Math.max(Math.floor(options.limit ?? 100), 1), 500) - const whereClauses = [ - 'sessions.agent_id = ?', - "tape.kind = 'anchor'", - "tape.name = 'memory/view_assembled'" - ] - const params: Array = [agentId] - if (options.sessionId) { - whereClauses.push('tape.session_id = ?') - params.push(options.sessionId) - } - if (options.messageId) { - whereClauses.push("json_extract(tape.meta_json, '$.messageId') = ?") - params.push(options.messageId) - } - params.push(cappedLimit) - return this.db - .prepare( - `SELECT tape.* - FROM deepchat_tape_entries AS tape - INNER JOIN new_sessions AS sessions - ON sessions.id = tape.session_id - WHERE ${whereClauses.join(' AND ')} - ORDER BY tape.created_at DESC, tape.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeEntryRow[] - } - - getEntriesAfter(sessionId: string, entryId: number): DeepChatTapeEntryRow[] { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND entry_id > ? - ORDER BY entry_id ASC` - ) - .all(sessionId, entryId) as DeepChatTapeEntryRow[] - } - - getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND kind = 'anchor' - ORDER BY entry_id DESC - LIMIT 1` - ) - .get(sessionId) as DeepChatTapeEntryRow | undefined - } - - getAnchors(sessionId: string, limit: number = 20): DeepChatTapeEntryRow[] { - const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) - const rows = this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND kind = 'anchor' - ORDER BY entry_id DESC - LIMIT ?` - ) - .all(sessionId, cappedLimit) as DeepChatTapeEntryRow[] - - return rows.reverse() - } - - getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { - const placeholders = SUMMARY_ANCHOR_NAMES.map(() => '?').join(', ') - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - AND kind = 'anchor' - AND name IN (${placeholders}) - ORDER BY entry_id DESC - LIMIT 1` - ) - .get(sessionId, ...SUMMARY_ANCHOR_NAMES) as DeepChatTapeEntryRow | undefined - } - - getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { - const placeholders = RECONSTRUCTION_ANCHOR_NAMES.map(() => '?').join(', ') - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - AND kind = 'anchor' - AND ( - name IN (${placeholders}) - OR name LIKE 'handoff/%' - OR name LIKE 'auto_handoff/%' - ) - ORDER BY entry_id DESC - LIMIT 1` - ) - .get(sessionId, ...RECONSTRUCTION_ANCHOR_NAMES) as DeepChatTapeEntryRow | undefined - } - - getByProvenanceKey(sessionId: string, provenanceKey: string): DeepChatTapeEntryRow | undefined { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND provenance_key = ? - LIMIT 1` - ) - .get(sessionId, provenanceKey) as DeepChatTapeEntryRow | undefined - } - - getMaxEntryId(sessionId: string): number { - const row = this.db - .prepare( - `SELECT MAX(entry_id) AS max_entry_id - FROM deepchat_tape_entries - WHERE session_id = ?` - ) - .get(sessionId) as { max_entry_id: number | null } | undefined - return row?.max_entry_id ?? 0 - } - - getMaxEntryIdsBySessions(sessionIds: string[]): Map { - const ids = [...new Set(sessionIds.map((id) => id.trim()).filter(Boolean))] - const maxEntryIdBySession = new Map(ids.map((id) => [id, 0])) - if (ids.length === 0) { - return maxEntryIdBySession - } - const rows = this.db - .prepare( - `WITH requested_sessions(session_id) AS ( - SELECT value FROM json_each(?) - ) - SELECT tape.session_id, MAX(tape.entry_id) AS max_entry_id - FROM deepchat_tape_entries AS tape - INNER JOIN requested_sessions AS requested - ON requested.session_id = tape.session_id - GROUP BY tape.session_id` - ) - .all(JSON.stringify(ids)) as Array<{ session_id: string; max_entry_id: number }> - for (const row of rows) { - maxEntryIdBySession.set(row.session_id, row.max_entry_id) - } - return maxEntryIdBySession - } - - countAnchorsBySession(sessionId: string): number { - const row = this.db - .prepare( - `SELECT COUNT(*) AS count - FROM deepchat_tape_entries - WHERE session_id = ? AND kind = 'anchor'` - ) - .get(sessionId) as { count: number } | undefined - return row?.count ?? 0 - } - - countEntriesAfter(sessionId: string, entryId: number): number { - const row = this.db - .prepare( - `SELECT COUNT(*) AS count - FROM deepchat_tape_entries - WHERE session_id = ? AND entry_id > ?` - ) - .get(sessionId, entryId) as { count: number } | undefined - return row?.count ?? 0 - } - - countBySession(sessionId: string): number { - const row = this.db - .prepare( - `SELECT COUNT(*) AS count - FROM deepchat_tape_entries - WHERE session_id = ?` - ) - .get(sessionId) as { count: number } | undefined - return row?.count ?? 0 - } - - search( - sessionId: string, - query: string, - options: DeepChatTapeSearchInput = {} - ): DeepChatTapeEntryRow[] { - const normalizedQuery = query.trim() - if (!normalizedQuery) { - return [] - } - const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 - const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) - const queryPredicate = buildDeepChatTapeLikeSearchPredicate( - ['payload_json', 'meta_json', 'name'], - normalizedQuery - ) - const whereClauses = ['session_id = ?', queryPredicate.sql] - const params: Array = [sessionId, ...queryPredicate.params] - - if (options.kinds?.length) { - whereClauses.push(`kind IN (${options.kinds.map(() => '?').join(', ')})`) - params.push(...options.kinds) - } - - if (Number.isFinite(options.startCreatedAt)) { - whereClauses.push('created_at >= ?') - params.push(options.startCreatedAt as number) - } - - if (Number.isFinite(options.endCreatedAt)) { - whereClauses.push('created_at <= ?') - params.push(options.endCreatedAt as number) - } - - params.push(cappedLimit) - - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE ${whereClauses.join(' AND ')} - ORDER BY entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeEntryRow[] - } - - searchEffectiveSourcesAtHeads( - sources: readonly DeepChatTapeReadSource[], - query: string, - options: DeepChatTapeSearchInput = {} - ): DeepChatTapeEntryRow[] { - const normalizedSources = normalizeDeepChatTapeReadSources(sources) - const normalizedQuery = query.trim() - if (normalizedSources.length === 0 || !normalizedQuery) { - return [] - } - - const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 - const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) - const queryPredicate = buildDeepChatTapeLikeSearchPredicate( - ['candidate.payload_json', 'candidate.meta_json', 'candidate.name'], - normalizedQuery - ) - const whereClauses = [queryPredicate.sql] - const params: Array = [ - serializeDeepChatTapeReadSources(normalizedSources), - ...queryPredicate.params - ] - - if (options.kinds?.length) { - whereClauses.push(`candidate.kind IN (${options.kinds.map(() => '?').join(', ')})`) - params.push(...options.kinds) - } - if (Number.isFinite(options.startCreatedAt)) { - whereClauses.push('candidate.created_at >= ?') - params.push(options.startCreatedAt as number) - } - if (Number.isFinite(options.endCreatedAt)) { - whereClauses.push('candidate.created_at <= ?') - params.push(options.endCreatedAt as number) - } - params.push(cappedLimit) - - return this.db - .prepare( - `WITH - ${AUTHORIZED_TAPE_SOURCES_CTE_SQL} - SELECT candidate.* - FROM deepchat_tape_entries AS candidate - INNER JOIN authorized_sources AS source - ON source.session_id = candidate.session_id - AND candidate.entry_id <= source.max_entry_id - WHERE ${whereClauses.join(' AND ')} - AND (${EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL}) - ORDER BY candidate.created_at DESC, candidate.session_id ASC, candidate.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeEntryRow[] - } - - getEffectiveContextRowsAtHead( - source: DeepChatTapeReadSource, - entryIds: number[], - options: { before: number; after: number; limit: number } - ): DeepChatTapeEntryRow[] { - const normalizedSource = normalizeDeepChatTapeReadSources([source])[0] - const requestedEntryIds = [ - ...new Set(entryIds.filter((entryId) => Number.isSafeInteger(entryId) && entryId > 0)) - ].sort((left, right) => left - right) - if (!normalizedSource || requestedEntryIds.length === 0) { - return [] - } - const before = Math.min(Math.max(Math.floor(options.before), 0), 20) - const after = Math.min(Math.max(Math.floor(options.after), 0), 20) - const limit = Math.min(Math.max(Math.floor(options.limit), 1), 100) - - return this.db - .prepare( - `WITH - ${AUTHORIZED_TAPE_SOURCES_CTE_SQL}, - ${EFFECTIVE_TAPE_ROWS_CTE_SQL}, - ordered_rows AS ( - SELECT - effective_rows.*, - ROW_NUMBER() OVER (ORDER BY entry_id ASC) AS row_position - FROM effective_rows - ), - requested_ids(entry_id, request_ordinal) AS ( - SELECT CAST(value AS INTEGER), CAST(key AS INTEGER) - FROM json_each(?) - ), - requested_positions AS ( - SELECT - requested_ids.request_ordinal, - ordered_rows.entry_id, - ordered_rows.row_position - FROM requested_ids - INNER JOIN ordered_rows - ON ordered_rows.entry_id = requested_ids.entry_id - ), - context_candidates AS ( - SELECT - ordered_rows.*, - 0 AS priority_group, - requested_positions.request_ordinal, - 0 AS neighbor_position - FROM requested_positions - INNER JOIN ordered_rows - ON ordered_rows.entry_id = requested_positions.entry_id - UNION ALL - SELECT - ordered_rows.*, - 1 AS priority_group, - requested_positions.request_ordinal, - ordered_rows.row_position AS neighbor_position - FROM requested_positions - INNER JOIN ordered_rows - ON ordered_rows.row_position BETWEEN requested_positions.row_position - ? - AND requested_positions.row_position + ? - AND ordered_rows.entry_id != requested_positions.entry_id - ), - ranked_context_candidates AS ( - SELECT - context_candidates.*, - ROW_NUMBER() OVER ( - PARTITION BY session_id, entry_id - ORDER BY priority_group, request_ordinal, neighbor_position - ) AS duplicate_rank - FROM context_candidates - ) - SELECT - session_id, entry_id, kind, name, source_type, source_id, source_seq, - provenance_key, payload_json, meta_json, created_at - FROM ranked_context_candidates - WHERE duplicate_rank = 1 - ORDER BY priority_group, request_ordinal, neighbor_position - LIMIT ?` - ) - .all( - serializeDeepChatTapeReadSources([normalizedSource]), - JSON.stringify(requestedEntryIds), - before, - after, - limit - ) as DeepChatTapeEntryRow[] - } - - deleteBySession(sessionId: string): void { - const remove = this.db.transaction(() => { - this.db.prepare('DELETE FROM deepchat_tape_entries WHERE session_id = ?').run(sessionId) - this.mutationProjection?.deleteBySession(sessionId) - }) - remove() - } - - private ensureProvenanceColumns(): void { - const columns: Array<[string, string]> = [ - ['source_type', 'TEXT'], - ['source_id', 'TEXT'], - ['source_seq', 'INTEGER'], - ['provenance_key', 'TEXT'] - ] - for (const [columnName, columnType] of columns) { - if (!this.hasColumn(columnName)) { - this.db.exec(`ALTER TABLE deepchat_tape_entries ADD COLUMN ${columnName} ${columnType}`) - } - } - } -} +/** @deprecated Import the SQLite Tape store from `@/tape/infrastructure/sqlite/tapeEntryStore`. */ +export { + buildDeepChatTapeFtsMatch, + buildDeepChatTapeLikeSearchPredicate, + DeepChatTapeEntriesTable, + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources, + SUMMARY_ANCHOR_NAMES, + TAPE_INCARNATION_META_KEY +} from '@/tape/infrastructure/sqlite/tapeEntryStore' + +/** @deprecated Import SQLite Tape types from `@/tape/infrastructure/sqlite/tapeEntryStore`. */ +export type { + DeepChatTapeAppendInput, + DeepChatTapeEntryKind, + DeepChatTapeEntryRow, + DeepChatTapeMutationProjection, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + DeepChatTapeSourceInput, + DeepChatTapeSourceType +} from '@/tape/infrastructure/sqlite/tapeEntryStore' diff --git a/src/main/session/data/tables/deepchatTapeSearchProjection.ts b/src/main/session/data/tables/deepchatTapeSearchProjection.ts index 46660b03f3..6869f22cea 100644 --- a/src/main/session/data/tables/deepchatTapeSearchProjection.ts +++ b/src/main/session/data/tables/deepchatTapeSearchProjection.ts @@ -1,926 +1,14 @@ -import Database from 'better-sqlite3-multiple-ciphers' -import { BaseTable } from '@/data/baseTable' -import { - buildDeepChatTapeFtsMatch, - buildDeepChatTapeLikeSearchPredicate, - normalizeDeepChatTapeReadSources, - serializeDeepChatTapeReadSources -} from './deepchatTapeEntries' -import type { - DeepChatTapeReadSource, - DeepChatTapeEntryKind, - DeepChatTapeSearchInput, - DeepChatTapeSourceType -} from './deepchatTapeEntries' - -export const DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION = 2 - -export interface DeepChatTapeSearchProjectionInput { - sessionId: string - entryId: number - kind: DeepChatTapeEntryKind - name: string | null - sourceType: DeepChatTapeSourceType | null - sourceId: string | null - sourceSeq: number | null - searchText: string - summaryText: string - refs: Record - createdAt: number -} - -export interface DeepChatTapeSearchProjectionRow { - session_id: string - entry_id: number - kind: DeepChatTapeEntryKind - name: string | null - source_type: DeepChatTapeSourceType | null - source_id: string | null - source_seq: number | null - search_text: string - summary_text: string - refs_json: string - created_at: number -} - -export interface DeepChatTapeSearchProjectionResultRow extends DeepChatTapeSearchProjectionRow { - score: number | null -} - -export interface DeepChatTapeSearchProjectionMeta { - projectionVersion: number - maxEntryId: number -} - -export interface DeepChatTapeSearchProjectionReadResult { - rows: DeepChatTapeSearchProjectionResultRow[] - coveredSources: DeepChatTapeReadSource[] -} - -type FtsCapability = { available: boolean; tokenizer: 'trigram' | 'unicode61' } - -const TAPE_SEARCH_PROJECTION_INDEX_SQL = ` - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_kind - ON deepchat_tape_search_projection(session_id, kind, entry_id); - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_created - ON deepchat_tape_search_projection(session_id, created_at, entry_id); -` - -function normalizeLimit(limit: number | undefined): number { - return Math.min(Math.max(Math.floor(Number.isFinite(limit) ? (limit as number) : 20), 1), 100) -} - -function safeJsonStringify(value: Record): string { - return JSON.stringify(value ?? {}) -} - -function parseRefs(value: string): Record { - try { - const parsed = JSON.parse(value) - return parsed && typeof parsed === 'object' && !Array.isArray(parsed) - ? (parsed as Record) - : {} - } catch { - return {} - } -} - -export class DeepChatTapeSearchProjectionTable extends BaseTable { - private ftsCapability: FtsCapability | undefined - private ftsReady = false - - constructor(db: Database.Database) { - super(db, 'deepchat_tape_search_projection') - } - - getCreateTableSQL(): string { - return ` - CREATE TABLE IF NOT EXISTS deepchat_tape_search_projection ( - session_id TEXT NOT NULL, - entry_id INTEGER NOT NULL, - kind TEXT NOT NULL, - name TEXT, - source_type TEXT, - source_id TEXT, - source_seq INTEGER, - search_text TEXT NOT NULL, - summary_text TEXT NOT NULL, - refs_json TEXT NOT NULL DEFAULT '{}', - created_at INTEGER NOT NULL, - PRIMARY KEY (session_id, entry_id) - ); - CREATE TABLE IF NOT EXISTS deepchat_tape_search_projection_meta ( - session_id TEXT PRIMARY KEY, - projection_version INTEGER NOT NULL, - max_entry_id INTEGER NOT NULL, - updated_at INTEGER NOT NULL - ); - CREATE TABLE IF NOT EXISTS deepchat_tape_search_fts_meta ( - session_id TEXT PRIMARY KEY, - projection_version INTEGER NOT NULL, - max_entry_id INTEGER NOT NULL, - updated_at INTEGER NOT NULL - ); - ${TAPE_SEARCH_PROJECTION_INDEX_SQL} - ` - } - - override createTable(): void { - this.db.exec(this.getCreateTableSQL()) - if (!this.ftsReady) { - this.ensureFtsIndex() - } - } - - getMigrationSQL(_version: number): string | null { - return null - } - - getLatestVersion(): number { - return 0 - } - - getSessionMeta(sessionId: string): DeepChatTapeSearchProjectionMeta | null { - const row = this.db - .prepare( - `SELECT projection_version, max_entry_id - FROM deepchat_tape_search_projection_meta - WHERE session_id = ?` - ) - .get(sessionId) as - | { - projection_version: number - max_entry_id: number - } - | undefined - if (!row) return null - return { - projectionVersion: row.projection_version, - maxEntryId: row.max_entry_id - } - } - - isCurrent( - sessionId: string, - maxEntryId: number, - projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - ): boolean { - const row = this.getSessionMeta(sessionId) - return row?.projectionVersion === projectionVersion && row.maxEntryId === maxEntryId - } - - getProjectedEntryIds(sessionId: string): number[] { - return ( - this.db - .prepare( - `SELECT entry_id - FROM deepchat_tape_search_projection - WHERE session_id = ? - ORDER BY entry_id ASC` - ) - .all(sessionId) as Array<{ entry_id: number }> - ).map((row) => row.entry_id) - } - - appendSession( - sessionId: string, - rows: DeepChatTapeSearchProjectionInput[], - maxEntryId: number, - projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - ): void { - const previousMeta = this.getSessionMeta(sessionId) - try { - this.db.transaction(() => { - if (!this.ftsReady) { - this.clearSessionFtsForBaseWrite(sessionId) - } - this.insertProjectionRows(rows) - if (this.ftsReady) { - if (previousMeta && this.isFtsCurrent(sessionId, previousMeta)) { - this.insertFtsRows(rows) - } else { - this.replaceSessionFtsRows(sessionId, this.getSessionProjectionInputs(sessionId)) - } - this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) - } - this.upsertMeta(sessionId, projectionVersion, maxEntryId) - })() - } catch (error) { - if (this.ftsReady) { - this.ftsReady = false - } - throw error - } - } - - replaceSession( - sessionId: string, - rows: DeepChatTapeSearchProjectionInput[], - maxEntryId: number, - projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - ): void { - try { - this.db.transaction(() => { - if (this.ftsReady) { - this.db - .prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?') - .run(sessionId) - this.db - .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') - .run(sessionId) - } else { - this.clearSessionFtsForBaseWrite(sessionId) - } - this.db - .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') - .run(sessionId) - this.db - .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') - .run(sessionId) - this.insertProjectionRows(rows) - if (this.ftsReady) { - this.insertFtsRows(rows) - this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) - } - this.upsertMeta(sessionId, projectionVersion, maxEntryId) - })() - } catch (error) { - if (this.ftsReady) { - this.ftsReady = false - } - throw error - } - } - - private insertProjectionRows(rows: DeepChatTapeSearchProjectionInput[]): void { - if (!rows.length) return - const insertProjection = this.db.prepare( - `INSERT OR REPLACE INTO deepchat_tape_search_projection ( - session_id, - entry_id, - kind, - name, - source_type, - source_id, - source_seq, - search_text, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - for (const row of rows) { - insertProjection.run( - row.sessionId, - row.entryId, - row.kind, - row.name, - row.sourceType, - row.sourceId, - row.sourceSeq, - row.searchText, - row.summaryText, - safeJsonStringify(row.refs), - row.createdAt - ) - } - } - - private upsertMeta(sessionId: string, projectionVersion: number, maxEntryId: number): void { - this.db - .prepare( - `INSERT INTO deepchat_tape_search_projection_meta ( - session_id, - projection_version, - max_entry_id, - updated_at - ) - VALUES (?, ?, ?, ?) - ON CONFLICT(session_id) DO UPDATE SET - projection_version = excluded.projection_version, - max_entry_id = excluded.max_entry_id, - updated_at = excluded.updated_at` - ) - .run(sessionId, projectionVersion, maxEntryId, Date.now()) - } - - private getFtsMeta(sessionId: string): DeepChatTapeSearchProjectionMeta | null { - const row = this.db - .prepare( - `SELECT projection_version, max_entry_id - FROM deepchat_tape_search_fts_meta - WHERE session_id = ?` - ) - .get(sessionId) as - | { - projection_version: number - max_entry_id: number - } - | undefined - if (!row) return null - return { - projectionVersion: row.projection_version, - maxEntryId: row.max_entry_id - } - } - - private isFtsCurrent(sessionId: string, meta: DeepChatTapeSearchProjectionMeta): boolean { - const row = this.getFtsMeta(sessionId) - return row?.projectionVersion === meta.projectionVersion && row.maxEntryId === meta.maxEntryId - } - - private upsertFtsMeta(sessionId: string, projectionVersion: number, maxEntryId: number): void { - this.db - .prepare( - `INSERT INTO deepchat_tape_search_fts_meta ( - session_id, - projection_version, - max_entry_id, - updated_at - ) - VALUES (?, ?, ?, ?) - ON CONFLICT(session_id) DO UPDATE SET - projection_version = excluded.projection_version, - max_entry_id = excluded.max_entry_id, - updated_at = excluded.updated_at` - ) - .run(sessionId, projectionVersion, maxEntryId, Date.now()) - } - - getByEntryIds(sessionId: string, entryIds: number[]): DeepChatTapeSearchProjectionRow[] { - const ids = [...new Set(entryIds.filter((id) => Number.isInteger(id) && id > 0))] - if (!ids.length) return [] - const placeholders = ids.map(() => '?').join(', ') - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_search_projection - WHERE session_id = ? AND entry_id IN (${placeholders}) - ORDER BY entry_id ASC` - ) - .all(sessionId, ...ids) as DeepChatTapeSearchProjectionRow[] - } - - searchSourcesReadOnly( - sources: readonly DeepChatTapeReadSource[], - query: string, - options: DeepChatTapeSearchInput = {} - ): DeepChatTapeSearchProjectionReadResult { - const normalizedSources = normalizeDeepChatTapeReadSources(sources) - const coveredSources = this.getCurrentReadSources( - normalizedSources, - 'deepchat_tape_search_projection_meta' - ) - const normalizedQuery = query.trim() - if (coveredSources.length === 0 || !normalizedQuery) { - return { rows: [], coveredSources } - } - if (coveredSources.length !== normalizedSources.length) { - return { rows: [], coveredSources } - } - - const limit = normalizeLimit(options.limit) - const ordered: DeepChatTapeSearchProjectionResultRow[] = [] - const seen = new Set() - const collect = (rows: DeepChatTapeSearchProjectionResultRow[]): void => { - for (const row of rows) { - const key = `${row.session_id}:${row.entry_id}` - if (seen.has(key)) continue - seen.add(key) - ordered.push(row) - } - } - - if (this.ftsReady) { - const ftsSources = this.getCurrentReadSources(coveredSources, 'deepchat_tape_search_fts_meta') - if (ftsSources.length === coveredSources.length) { - try { - collect(this.searchFtsSourcesReadOnly(ftsSources, normalizedQuery, options, limit)) - } catch { - // The base projection remains a complete read-only fallback. - } - } - } - if (ordered.length < limit) { - collect(this.searchLikeSourcesReadOnly(coveredSources, normalizedQuery, options, limit)) - } - - return { rows: ordered.slice(0, limit), coveredSources } - } - - search( - sessionId: string, - query: string, - options: DeepChatTapeSearchInput = {} - ): DeepChatTapeSearchProjectionResultRow[] { - const normalized = query.trim() - if (!normalized) return [] - const limit = normalizeLimit(options.limit) - const ordered: DeepChatTapeSearchProjectionResultRow[] = [] - const seen = new Set() - const collect = (rows: DeepChatTapeSearchProjectionResultRow[]): void => { - for (const row of rows) { - if (seen.has(row.entry_id)) continue - seen.add(row.entry_id) - ordered.push(row) - } - } - this.recoverSessionFts(sessionId) - if (this.ftsReady) { - collect(this.searchFts(sessionId, normalized, options, limit)) - } - if (!this.ftsReady || ordered.length < limit) { - collect(this.searchLike(sessionId, normalized, options, limit)) - } - return ordered.slice(0, limit) - } - - deleteBySession(sessionId: string): void { - this.db - .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') - .run(sessionId) - this.db - .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') - .run(sessionId) - this.deleteSessionFts(sessionId) - } - - clearAll(): void { - this.db.prepare('DELETE FROM deepchat_tape_search_projection').run() - this.db.prepare('DELETE FROM deepchat_tape_search_projection_meta').run() - if (this.ftsMetaTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() - } - this.clearFts() - } - - private detectFtsCapability(): FtsCapability { - if (this.ftsCapability) return this.ftsCapability - const probe = (tokenizer: string): boolean => { - const name = `temp.tape_search_fts_probe_${tokenizer}` - try { - this.db.exec( - `CREATE VIRTUAL TABLE IF NOT EXISTS ${name} USING fts5(c, tokenize='${tokenizer}');` - ) - this.db.exec(`DROP TABLE IF EXISTS ${name};`) - return true - } catch { - return false - } - } - if (probe('trigram')) this.ftsCapability = { available: true, tokenizer: 'trigram' } - else if (probe('unicode61')) this.ftsCapability = { available: true, tokenizer: 'unicode61' } - else this.ftsCapability = { available: false, tokenizer: 'unicode61' } - return this.ftsCapability - } - - private ensureFtsIndex(): void { - const capability = this.detectFtsCapability() - if (!capability.available) { - this.ftsReady = false - return - } - try { - this.db.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS deepchat_tape_search_fts USING fts5( - search_text, - name, - session_id UNINDEXED, - entry_id UNINDEXED, - kind UNINDEXED, - source_type UNINDEXED, - source_id UNINDEXED, - source_seq UNINDEXED, - summary_text UNINDEXED, - refs_json UNINDEXED, - created_at UNINDEXED, - tokenize='${capability.tokenizer}' - ); - `) - this.ftsReady = true - } catch { - this.ftsReady = false - } - } - - private toProjectionInput( - row: DeepChatTapeSearchProjectionRow - ): DeepChatTapeSearchProjectionInput { - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - sourceType: row.source_type, - sourceId: row.source_id, - sourceSeq: row.source_seq, - searchText: row.search_text, - summaryText: row.summary_text, - refs: parseRefs(row.refs_json), - createdAt: row.created_at - } - } - - private getSessionProjectionInputs(sessionId: string): DeepChatTapeSearchProjectionInput[] { - return ( - this.db - .prepare( - `SELECT * - FROM deepchat_tape_search_projection - WHERE session_id = ? - ORDER BY entry_id ASC` - ) - .all(sessionId) as DeepChatTapeSearchProjectionRow[] - ).map((row) => this.toProjectionInput(row)) - } - - private ftsTableExists(): boolean { - const row = this.db - .prepare( - `SELECT name - FROM sqlite_master - WHERE type = 'table' AND name = 'deepchat_tape_search_fts' - LIMIT 1` - ) - .get() as { name: string } | undefined - return Boolean(row) - } - - private ftsMetaTableExists(): boolean { - const row = this.db - .prepare( - `SELECT name - FROM sqlite_master - WHERE type = 'table' AND name = 'deepchat_tape_search_fts_meta' - LIMIT 1` - ) - .get() as { name: string } | undefined - return Boolean(row) - } - - private clearSessionFtsForBaseWrite(sessionId: string): void { - if (this.ftsMetaTableExists()) { - this.db - .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') - .run(sessionId) - } - if (this.ftsTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) - } - } - - private getProjectionRowId(sessionId: string, entryId: number): number { - const row = this.db - .prepare( - `SELECT rowid - FROM deepchat_tape_search_projection - WHERE session_id = ? AND entry_id = ?` - ) - .get(sessionId, entryId) as { rowid: number } | undefined - if (!row) { - throw new Error(`Missing tape search projection row for ${sessionId}:${entryId}`) - } - return row.rowid - } - - private insertFtsRows(rows: DeepChatTapeSearchProjectionInput[]): void { - if (!rows.length) return - const insertFts = this.db.prepare( - `INSERT INTO deepchat_tape_search_fts ( - rowid, - search_text, - name, - session_id, - entry_id, - kind, - source_type, - source_id, - source_seq, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - for (const row of rows) { - this.db - .prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ? AND entry_id = ?') - .run(row.sessionId, row.entryId) - insertFts.run( - this.getProjectionRowId(row.sessionId, row.entryId), - row.searchText, - row.name ?? '', - row.sessionId, - row.entryId, - row.kind, - row.sourceType, - row.sourceId, - row.sourceSeq, - row.summaryText, - safeJsonStringify(row.refs), - row.createdAt - ) - } - } - - private replaceSessionFtsRows( - sessionId: string, - rows: DeepChatTapeSearchProjectionInput[] - ): void { - this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) - this.insertFtsRows(rows) - } - - private replaceSessionFts( - sessionId: string, - rows: DeepChatTapeSearchProjectionInput[], - projectionVersion: number, - maxEntryId: number - ): boolean { - if (!this.ftsReady) return false - try { - this.db.transaction(() => { - this.replaceSessionFtsRows(sessionId, rows) - this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) - })() - return true - } catch { - this.ftsReady = false - return false - } - } - - private recoverSessionFts(sessionId: string): void { - const meta = this.getSessionMeta(sessionId) - if (!meta) { - this.deleteSessionFts(sessionId) - return - } - if (this.ftsReady && this.isFtsCurrent(sessionId, meta)) return - if (!this.ftsReady) { - this.ensureFtsIndex() - } - if (!this.ftsReady) return - if (this.isFtsCurrent(sessionId, meta)) return - this.replaceSessionFts( - sessionId, - this.getSessionProjectionInputs(sessionId), - meta.projectionVersion, - meta.maxEntryId - ) - } - - hasFtsReadyForTesting(): boolean { - return this.ftsReady - } - - disableFtsForTesting(): void { - this.ftsReady = false - } - - dropFtsForTesting(): void { - this.db.exec('DROP TABLE IF EXISTS deepchat_tape_search_fts') - if (this.ftsMetaTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() - } - this.ftsReady = false - } - - private deleteSessionFts(sessionId: string): void { - if (this.ftsMetaTableExists()) { - this.db - .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') - .run(sessionId) - } - if (!this.ftsTableExists()) return - try { - this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) - } catch { - this.ftsReady = false - } - } - - private clearFts(): void { - if (this.ftsMetaTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() - } - if (!this.ftsTableExists()) return - try { - this.db.prepare('DELETE FROM deepchat_tape_search_fts').run() - } catch { - this.ftsReady = false - } - } - - private searchFts( - sessionId: string, - normalized: string, - options: DeepChatTapeSearchInput, - limit: number - ): DeepChatTapeSearchProjectionResultRow[] { - const match = buildDeepChatTapeFtsMatch(normalized) - const whereClauses = [ - 'deepchat_tape_search_fts MATCH ?', - 'deepchat_tape_search_fts.session_id = ?', - 'projection.session_id = ?' - ] - const params: Array = [match, sessionId, sessionId] - this.addFilters(whereClauses, params, options, true, 'projection') - params.push(limit) - try { - return this.db - .prepare( - `SELECT - projection.session_id, - projection.entry_id, - projection.kind, - projection.name, - projection.source_type, - projection.source_id, - projection.source_seq, - projection.search_text, - projection.summary_text, - projection.refs_json, - projection.created_at, - bm25(deepchat_tape_search_fts) AS score - FROM deepchat_tape_search_fts - INNER JOIN deepchat_tape_search_projection AS projection - ON projection.session_id = deepchat_tape_search_fts.session_id - AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) - AND projection.search_text = deepchat_tape_search_fts.search_text - WHERE ${whereClauses.join(' AND ')} - ORDER BY score ASC, projection.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeSearchProjectionResultRow[] - } catch { - return [] - } - } - - private getCurrentReadSources( - sources: readonly DeepChatTapeReadSource[], - metaTable: 'deepchat_tape_search_projection_meta' | 'deepchat_tape_search_fts_meta' - ): DeepChatTapeReadSource[] { - const normalizedSources = normalizeDeepChatTapeReadSources(sources) - if (normalizedSources.length === 0) { - return [] - } - const rows = this.db - .prepare( - `WITH requested_sources(session_id, max_entry_id) AS ( - SELECT - json_extract(value, '$.sessionId'), - CAST(json_extract(value, '$.maxEntryId') AS INTEGER) - FROM json_each(?) - ) - SELECT requested.session_id, requested.max_entry_id - FROM requested_sources AS requested - INNER JOIN ${metaTable} AS meta - ON meta.session_id = requested.session_id - AND meta.max_entry_id = requested.max_entry_id - AND meta.projection_version = ? - ORDER BY requested.session_id ASC` - ) - .all( - serializeDeepChatTapeReadSources(normalizedSources), - DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - ) as Array<{ session_id: string; max_entry_id: number }> - return rows.map((row) => ({ sessionId: row.session_id, maxEntryId: row.max_entry_id })) - } - - private searchFtsSourcesReadOnly( - sources: readonly DeepChatTapeReadSource[], - normalized: string, - options: DeepChatTapeSearchInput, - limit: number - ): DeepChatTapeSearchProjectionResultRow[] { - const match = buildDeepChatTapeFtsMatch(normalized) - const whereClauses = ['deepchat_tape_search_fts MATCH ?'] - const params: Array = [serializeDeepChatTapeReadSources(sources), match] - this.addFilters(whereClauses, params, options, true, 'projection') - params.push(limit) - return this.db - .prepare( - `WITH authorized_sources(session_id, max_entry_id) AS ( - SELECT - json_extract(value, '$.sessionId'), - CAST(json_extract(value, '$.maxEntryId') AS INTEGER) - FROM json_each(?) - ) - SELECT - projection.session_id, - projection.entry_id, - projection.kind, - projection.name, - projection.source_type, - projection.source_id, - projection.source_seq, - projection.search_text, - projection.summary_text, - projection.refs_json, - projection.created_at, - bm25(deepchat_tape_search_fts) AS score - FROM deepchat_tape_search_fts - INNER JOIN deepchat_tape_search_projection AS projection - ON projection.session_id = deepchat_tape_search_fts.session_id - AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) - AND projection.search_text = deepchat_tape_search_fts.search_text - INNER JOIN authorized_sources AS source - ON source.session_id = projection.session_id - AND projection.entry_id <= source.max_entry_id - WHERE ${whereClauses.join(' AND ')} - ORDER BY score ASC, projection.created_at DESC, projection.session_id ASC, - projection.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeSearchProjectionResultRow[] - } - - private searchLikeSourcesReadOnly( - sources: readonly DeepChatTapeReadSource[], - normalized: string, - options: DeepChatTapeSearchInput, - limit: number - ): DeepChatTapeSearchProjectionResultRow[] { - const queryPredicate = buildDeepChatTapeLikeSearchPredicate( - ['projection.search_text', 'projection.summary_text', 'projection.name'], - normalized - ) - const whereClauses = [queryPredicate.sql] - const params: Array = [serializeDeepChatTapeReadSources(sources)] - params.push(...queryPredicate.params) - this.addFilters(whereClauses, params, options, false, 'projection') - params.push(limit) - return this.db - .prepare( - `WITH authorized_sources(session_id, max_entry_id) AS ( - SELECT - json_extract(value, '$.sessionId'), - CAST(json_extract(value, '$.maxEntryId') AS INTEGER) - FROM json_each(?) - ) - SELECT projection.*, NULL AS score - FROM deepchat_tape_search_projection AS projection - INNER JOIN authorized_sources AS source - ON source.session_id = projection.session_id - AND projection.entry_id <= source.max_entry_id - WHERE ${whereClauses.join(' AND ')} - ORDER BY projection.created_at DESC, projection.session_id ASC, projection.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeSearchProjectionResultRow[] - } - - private searchLike( - sessionId: string, - normalized: string, - options: DeepChatTapeSearchInput, - limit: number - ): DeepChatTapeSearchProjectionResultRow[] { - const queryPredicate = buildDeepChatTapeLikeSearchPredicate( - ['search_text', 'summary_text', 'name'], - normalized - ) - const whereClauses = ['session_id = ?', queryPredicate.sql] - const params: Array = [sessionId] - params.push(...queryPredicate.params) - this.addFilters(whereClauses, params, options) - params.push(limit) - return this.db - .prepare( - `SELECT *, NULL AS score - FROM deepchat_tape_search_projection - WHERE ${whereClauses.join(' AND ')} - ORDER BY entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeSearchProjectionResultRow[] - } - - private addFilters( - whereClauses: string[], - params: Array, - options: DeepChatTapeSearchInput, - castCreatedAt = false, - tableAlias?: string - ): void { - const column = (name: string) => (tableAlias ? `${tableAlias}.${name}` : name) - if (options.kinds?.length) { - whereClauses.push(`${column('kind')} IN (${options.kinds.map(() => '?').join(', ')})`) - params.push(...options.kinds) - } - const createdAtColumn = castCreatedAt - ? `CAST(${column('created_at')} AS INTEGER)` - : column('created_at') - if (Number.isFinite(options.startCreatedAt)) { - whereClauses.push(`${createdAtColumn} >= ?`) - params.push(options.startCreatedAt as number) - } - if (Number.isFinite(options.endCreatedAt)) { - whereClauses.push(`${createdAtColumn} <= ?`) - params.push(options.endCreatedAt as number) - } - } -} +/** @deprecated Import the SQLite projection from `@/tape/infrastructure/sqlite/tapeSearchProjectionStore`. */ +export { + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, + DeepChatTapeSearchProjectionTable +} from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' + +/** @deprecated Import projection types from `@/tape/infrastructure/sqlite/tapeSearchProjectionStore`. */ +export type { + DeepChatTapeSearchProjectionInput, + DeepChatTapeSearchProjectionMeta, + DeepChatTapeSearchProjectionReadResult, + DeepChatTapeSearchProjectionResultRow, + DeepChatTapeSearchProjectionRow +} from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' diff --git a/src/main/session/data/tape.ts b/src/main/session/data/tape.ts index a0908c9d42..6915099d38 100644 --- a/src/main/session/data/tape.ts +++ b/src/main/session/data/tape.ts @@ -1,2631 +1,19 @@ -import { SessionDatabase } from './database' -import { nanoid } from 'nanoid' -import { createHash } from 'crypto' -import logger from 'electron-log' -import type { - AgentTapeAnchorResult, - AgentTapeAnchorsOptions, - AgentTapeContextEntry, - AgentTapeContextOptions, - AgentTapeContextResult, - AgentTapeHandoffState, - AgentTapeSearchOptions, - AgentTapeViewScope, - ChatMessageRecord, - SubagentTapeLinkInput, - SubagentTapeLinkOutcome, - SubagentTapeLinkReceipt -} from '@shared/types/agent-interface' -import type { - DeepChatTapeViewExcludedRange, - DeepChatTapeViewManifest, - DeepChatTapeViewManifestRecord -} from '@shared/types/tape-view-manifest' -import type { - DeepChatCausalObservationReadOptions, - DeepChatCausalObservationRequest, - DeepChatCausalObservationSlice, - DeepChatTapeReplayEntrySnapshot, - DeepChatTapeReplayExportOptions, - DeepChatTapeReplaySlice, - DeepChatTapeReplayTraceSnapshot -} from '@shared/types/tape-replay' -import type { SessionTranscript } from '@/session/data/transcript' -import { - SUMMARY_ANCHOR_NAMES, - TAPE_INCARNATION_META_KEY, - type DeepChatTapeReadSource, - type DeepChatTapeEntryRow, - type DeepChatTapeSearchInput -} from '@/session/data/tables/deepchatTapeEntries' -import type { - DeepChatTapeSearchProjectionInput, - DeepChatTapeSearchProjectionResultRow, - DeepChatTapeSearchProjectionRow -} from '@/session/data/tables/deepchatTapeSearchProjection' -import type { DeepChatMessageTraceRow } from '@/session/data/tables/deepchatMessageTraces' -import { appendMessageRecordToTape, appendTapeToolFact } from '@/session/data/tapeFacts' -import type { TapeEntryRef, TapeRecorder, TapeToolFactInput } from '@/agent/deepchat/loop/ports' -import { - buildEffectiveTapeView, - getLastEffectiveTokenUsage, - searchEffectiveTapeRows -} from '@/session/data/tapeEffectiveView' -import { - hashJson, - TAPE_VIEW_MANIFEST_EVENT_NAME, - verifyTapeViewManifestHash -} from '@/session/data/tapeViewManifest' - -export type TapeMigrationState = 'none' | 'ready' - -export type TapeBackfillResult = { - sessionId: string - migrationState: TapeMigrationState - messageCount: number - maxOrderSeq: number - appendedFactCount: number - historyRecords: ChatMessageRecord[] -} - -export type TapeInfo = { - sessionId: string - entries: number - anchors: number - lastAnchor: string | null - lastAnchorEntryId: number | null - entriesSinceLastAnchor: number - lastTokenUsage: number | null - migrationState: TapeMigrationState -} - -export type TapeSearchResult = { - sessionId: string - entryId: number - kind: string - name: string | null - createdAt: number - summary?: string - refs?: Record - score?: number -} - -export type TapeAnchorResult = AgentTapeAnchorResult - -export type TapeForkHandle = { - parentSessionId: string - forkId: string - forkSessionId: string - parentHeadEntryId: number -} - -export type TapeViewManifestSourceMaps = { - latestEntryId: number - anchorEntryIds: number[] - reconstructionAnchorEntryIds: number[] - reconstructionAnchorEntryId: number | null - entryIdByMessageId: Map - toolCallEntryIdByToolId: Map - toolResultEntryIdByToolId: Map -} - -const BOOTSTRAP_ANCHOR_NAME = 'session/start' -const DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY = 2048 -const DEFAULT_CONTEXT_MAX_TOTAL_BYTES = 16384 -const MAX_CONTEXT_MAX_BYTES_PER_ENTRY = 8192 -const MAX_CONTEXT_MAX_TOTAL_BYTES = 65536 - -// Mirrors getLatestReconstructionAnchor: the anchor set that owns the summary -// cursor and prompt-visible reconstruction state (summaries, handoffs). -function isReconstructionAnchorName(name: string | null): boolean { - if (name === null) { - return false - } - return ( - (SUMMARY_ANCHOR_NAMES as readonly string[]).includes(name) || - name.startsWith('handoff/') || - name.startsWith('auto_handoff/') - ) -} - -function parseJsonObject(raw: string): Record { - try { - const parsed = JSON.parse(raw) as unknown - if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { - return parsed as Record - } - } catch {} - return {} -} - -function parseJsonValue(raw: string): unknown { - try { - return JSON.parse(raw) as unknown - } catch { - return null - } -} - -const SUBAGENT_TAPE_LINK_EVENT_NAME = 'subagent/tape_linked' -const SUBAGENT_TAPE_LINK_VERSION = 2 -const TAPE_IDENTITY_PATTERN = /^[a-f0-9]{64}$/ -const SUBAGENT_TAPE_LINK_OUTCOMES = new Set([ - 'completed', - 'error', - 'cancelled' -]) - -type SubagentTapeLinkSnapshot = { - linkEntryId: number - childSessionId: string - childHeadEntryId: number - childEntryCount: number - outcome: SubagentTapeLinkOutcome - childTapeIdentity: string | null -} - -type ParsedSubagentTapeLink = { - snapshot: SubagentTapeLinkSnapshot - frozenInput: SubagentTapeLinkInput -} - -type LinkedTapeSourceResolution = { - sources: DeepChatTapeReadSource[] - unavailableSourceIds: Set -} - -export type AgentTapeViewErrorCode = - | 'current_tape_unavailable' - | 'linked_tape_unavailable' - | 'linked_tape_unauthorized' - -export class AgentTapeViewError extends Error { - readonly name = 'AgentTapeViewError' - - constructor( - readonly code: AgentTapeViewErrorCode, - readonly parentSessionId: string, - readonly sourceSessionId: string, - message: string - ) { - super(message) - } -} - -export function normalizeSubagentTapeLinkInput( - input: SubagentTapeLinkInput -): SubagentTapeLinkInput { - const normalized = { - parentSessionId: input.parentSessionId.trim(), - childSessionId: input.childSessionId.trim(), - runId: input.runId.trim(), - taskId: input.taskId.trim(), - slotId: input.slotId.trim(), - taskTitle: compactText(input.taskTitle, 500), - outcome: input.outcome, - resultSummary: input.resultSummary?.trim() ? compactText(input.resultSummary, 2000) : null - } - for (const [name, value] of Object.entries(normalized)) { - if (name === 'resultSummary' || name === 'outcome') continue - if (typeof value !== 'string' || !value) { - throw new Error(`Subagent Tape link ${name} is required.`) - } - } - if (normalized.parentSessionId === normalized.childSessionId) { - throw new Error('Subagent Tape link child must differ from its parent.') - } - if (!SUBAGENT_TAPE_LINK_OUTCOMES.has(normalized.outcome)) { - throw new Error(`Invalid subagent Tape link outcome: ${String(normalized.outcome)}`) - } - return normalized -} - -function isUnmarkedLegacyTape(row: DeepChatTapeEntryRow): boolean { - const meta = parseJsonValue(row.meta_json) - return ( - meta !== null && - typeof meta === 'object' && - !Array.isArray(meta) && - !Object.prototype.hasOwnProperty.call(meta, TAPE_INCARNATION_META_KEY) - ) -} - -function computeTapeIdentity(row: DeepChatTapeEntryRow): string { - return createHash('sha256') - .update( - JSON.stringify([ - row.session_id, - row.entry_id, - row.kind, - row.name, - row.source_type, - row.source_id, - row.source_seq, - row.provenance_key, - row.payload_json, - row.meta_json, - row.created_at - ]) - ) - .digest('hex') -} - -function subagentTapeLinkProvenanceKey(input: SubagentTapeLinkInput): string { - // This version belongs to the stable task-identity key, independently of the evolving event - // payload's linkVersion. - const identityHash = createHash('sha256') - .update( - JSON.stringify([input.parentSessionId, input.childSessionId, input.runId, input.taskId]) - ) - .digest('hex') - return `subagent:tape-link:v1:${identityHash}` -} - -function parseSubagentTapeLink(row: DeepChatTapeEntryRow): ParsedSubagentTapeLink | null { - const payload = parseJsonObject(row.payload_json) - const data = - payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) - ? (payload.data as Record) - : {} - const childSessionId = data.childSessionId - const childHeadEntryId = data.childHeadEntryId - const childEntryCount = data.childEntryCount - const outcome = data.outcome - const runId = data.runId - const taskId = data.taskId - const slotId = data.slotId - const taskTitle = data.taskTitle - const resultSummary = data.resultSummary - const linkVersion = data.linkVersion - const childTapeIdentity = data.childTapeIdentity - const hasValidLinkVersion = - (linkVersion === 1 && childTapeIdentity === undefined) || - (linkVersion === SUBAGENT_TAPE_LINK_VERSION && - typeof childTapeIdentity === 'string' && - TAPE_IDENTITY_PATTERN.test(childTapeIdentity)) - if ( - row.kind !== 'event' || - row.name !== SUBAGENT_TAPE_LINK_EVENT_NAME || - typeof childSessionId !== 'string' || - !childSessionId || - typeof childHeadEntryId !== 'number' || - !Number.isSafeInteger(childHeadEntryId) || - childHeadEntryId < 0 || - typeof childEntryCount !== 'number' || - !Number.isSafeInteger(childEntryCount) || - childEntryCount < 0 || - typeof outcome !== 'string' || - !SUBAGENT_TAPE_LINK_OUTCOMES.has(outcome as SubagentTapeLinkOutcome) || - typeof runId !== 'string' || - typeof taskId !== 'string' || - typeof slotId !== 'string' || - typeof taskTitle !== 'string' || - (resultSummary !== null && typeof resultSummary !== 'string') || - !hasValidLinkVersion || - row.source_type !== 'subagent' || - row.source_id !== childSessionId || - row.source_seq !== childHeadEntryId || - childEntryCount > childHeadEntryId - ) { - return null - } - - let normalizedInput: SubagentTapeLinkInput - try { - normalizedInput = normalizeSubagentTapeLinkInput({ - parentSessionId: row.session_id, - childSessionId, - runId, - taskId, - slotId, - taskTitle, - outcome: outcome as SubagentTapeLinkOutcome, - resultSummary - }) - } catch { - return null - } - if ( - normalizedInput.parentSessionId !== row.session_id || - normalizedInput.childSessionId !== childSessionId || - normalizedInput.runId !== runId || - normalizedInput.taskId !== taskId || - normalizedInput.slotId !== slotId || - normalizedInput.taskTitle !== taskTitle || - normalizedInput.resultSummary !== resultSummary || - row.provenance_key !== subagentTapeLinkProvenanceKey(normalizedInput) - ) { - return null - } - - return { - snapshot: { - linkEntryId: row.entry_id, - childSessionId, - childHeadEntryId, - childEntryCount, - outcome: outcome as SubagentTapeLinkOutcome, - childTapeIdentity: - linkVersion === SUBAGENT_TAPE_LINK_VERSION ? (childTapeIdentity as string) : null - }, - frozenInput: normalizedInput - } -} - -function parseSubagentTapeLinkSnapshot(row: DeepChatTapeEntryRow): SubagentTapeLinkSnapshot | null { - return parseSubagentTapeLink(row)?.snapshot ?? null -} - -function parseLegacyExternalTapeLinkSnapshot( - row: DeepChatTapeEntryRow -): SubagentTapeLinkSnapshot | null { - if (row.kind !== 'event' || row.name !== 'fork/merge' || row.source_type !== 'fork') { - return null - } - const payload = parseJsonObject(row.payload_json) - const data = - payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) - ? (payload.data as Record) - : {} - const childSessionId = data.forkSessionId - const forkId = data.forkId - const referencedEntryCount = data.referencedEntryCount - if ( - typeof childSessionId !== 'string' || - !childSessionId || - forkId !== childSessionId || - row.source_id !== childSessionId || - row.source_seq !== 0 || - row.provenance_key !== `fork:${row.session_id}:${childSessionId}:external-merge:event` || - typeof referencedEntryCount !== 'number' || - !Number.isSafeInteger(referencedEntryCount) || - referencedEntryCount <= 0 - ) { - return null - } - return { - linkEntryId: row.entry_id, - childSessionId, - childHeadEntryId: referencedEntryCount, - childEntryCount: referencedEntryCount, - outcome: 'completed', - childTapeIdentity: null - } -} - -function readForkMergeReceiptCount( - row: DeepChatTapeEntryRow, - parentSessionId: string, - forkId: string, - forkSessionIdValue: string -): number { - const payload = parseJsonObject(row.payload_json) - const data = - payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) - ? (payload.data as Record) - : {} - const mergedCount = data.mergedCount - const forkHeadEntryId = data.forkHeadEntryId - const hasValidLegacyOrCurrentHead = - forkHeadEntryId === undefined || - (typeof forkHeadEntryId === 'number' && - Number.isSafeInteger(forkHeadEntryId) && - forkHeadEntryId >= 0) - if ( - row.session_id !== parentSessionId || - row.kind !== 'event' || - row.name !== 'fork/merge' || - row.source_type !== 'fork' || - row.source_id !== forkId || - row.source_seq !== 0 || - row.provenance_key !== `fork:${parentSessionId}:${forkId}:merge:event` || - data.forkId !== forkId || - data.forkSessionId !== forkSessionIdValue || - typeof mergedCount !== 'number' || - !Number.isSafeInteger(mergedCount) || - mergedCount < 0 || - !hasValidLegacyOrCurrentHead || - (typeof forkHeadEntryId === 'number' && mergedCount > forkHeadEntryId) - ) { - throw new Error(`Stored fork merge receipt is malformed: ${row.entry_id}`) - } - return mergedCount -} - -function assertValidForkStart( - row: DeepChatTapeEntryRow | undefined, - parentSessionId: string, - forkId: string, - forkSessionIdValue: string -): void { - if (!row) { - throw new Error(`Fork ${forkId} does not exist or has been discarded.`) - } - const payload = parseJsonObject(row.payload_json) - const state = - payload.state && typeof payload.state === 'object' && !Array.isArray(payload.state) - ? (payload.state as Record) - : {} - const parentHeadEntryId = state.parentHeadEntryId - const hasValidLegacyOrCurrentHead = - parentHeadEntryId === undefined || - (typeof parentHeadEntryId === 'number' && - Number.isSafeInteger(parentHeadEntryId) && - parentHeadEntryId >= 0) - if ( - row.session_id !== forkSessionIdValue || - row.kind !== 'anchor' || - row.name !== 'fork/start' || - row.source_type !== 'fork' || - row.source_id !== forkId || - row.source_seq !== 0 || - row.provenance_key !== `fork:${parentSessionId}:${forkId}:start` || - state.parentSessionId !== parentSessionId || - !hasValidLegacyOrCurrentHead - ) { - throw new Error(`Stored fork start is malformed: ${row.entry_id}`) - } -} - -function toSubagentTapeLinkReceipt(row: DeepChatTapeEntryRow): SubagentTapeLinkReceipt { - const snapshot = parseSubagentTapeLinkSnapshot(row) - if (!snapshot) { - throw new Error(`Stored subagent Tape link receipt is malformed: ${row.entry_id}`) - } - return { - linkEntry: { - sessionId: row.session_id, - entryId: snapshot.linkEntryId - }, - childSessionId: snapshot.childSessionId, - childHeadEntryId: snapshot.childHeadEntryId, - childEntryCount: snapshot.childEntryCount, - outcome: snapshot.outcome - } -} - -function assertSubagentTapeLinkMatchesInput( - row: DeepChatTapeEntryRow, - input: SubagentTapeLinkInput -): void { - const parsed = parseSubagentTapeLink(row) - if (!parsed) { - throw new Error(`Stored subagent Tape link receipt is malformed: ${row.entry_id}`) - } - const storedInput = parsed.frozenInput - const storedKeys = Object.keys(storedInput) as Array - if ( - storedKeys.length !== Object.keys(input).length || - storedKeys.some((key) => storedInput[key] !== input[key]) - ) { - throw new Error( - `Subagent Tape link conflicts with finalized task ${input.runId}/${input.taskId}.` - ) - } -} - -function compactText(value: string, maxLength = 1000): string { - const normalized = value.replace(/\s+/g, ' ').trim() - if (normalized.length <= maxLength) return normalized - return `${normalized.slice(0, Math.max(0, maxLength - 3)).trimEnd()}...` -} - -function stringifyForSummary(value: unknown, maxLength = 1000): string { - if (typeof value === 'string') return compactText(value, maxLength) - if (value === null || value === undefined) return '' - try { - return compactText(JSON.stringify(value), maxLength) - } catch { - return compactText(String(value), maxLength) - } -} - -function truncateToUtf8Bytes(text: string, maxBytes: number): { text: string; truncated: boolean } { - const normalized = text.trim() - if (maxBytes <= 0) { - return { text: '', truncated: normalized.length > 0 } - } - if (maxBytes < 3) { - return { text: '', truncated: normalized.length > 0 } - } - if (Buffer.byteLength(normalized, 'utf8') <= maxBytes) { - return { text: normalized, truncated: false } - } - let bytes = 0 - let output = '' - for (const character of normalized) { - const nextBytes = Buffer.byteLength(character, 'utf8') - if (bytes + nextBytes > Math.max(0, maxBytes - 3)) break - output += character - bytes += nextBytes - } - return { text: `${output.trimEnd()}...`, truncated: true } -} - -function normalizeContextByteLimit( - value: number | undefined, - fallback: number, - max: number -): number { - if (!Number.isFinite(value)) return fallback - return Math.min(Math.max(Math.floor(value as number), 0), max) -} - -function uniqueStrings(values: string[], limit = 10): string[] { - const seen = new Set() - const result: string[] = [] - for (const value of values) { - const normalized = value.trim() - if (!normalized || seen.has(normalized)) continue - seen.add(normalized) - result.push(normalized) - if (result.length >= limit) break - } - return result -} - -function extractFilePaths(text: string): string[] { - const matches = [ - ...text.matchAll( - /(?:^|[\s"'`([{<])((?:[A-Za-z]:\\|\/|\.{1,2}\/|[\w.-]+\/)[^\s"'`<>{}(),;!?]+(?:[/\\][^\s"'`<>{}(),;!?]+)*)/g - ) - ].map((match) => match[1].replace(/[.:]+$/g, '')) - return uniqueStrings(matches ?? []) -} - -function extractErrorCodes(text: string): string[] { - const matches = text.match(/\b(?:E[A-Z0-9_]{3,}|[A-Z][A-Z0-9_]*Error)\b/g) - return uniqueStrings(matches ?? []) -} - -function collectKeyedStrings( - value: unknown, - keys: Set, - output: string[] = [], - depth = 0 -): string[] { - if (depth > 4 || output.length >= 10 || !value || typeof value !== 'object') return output - if (Array.isArray(value)) { - for (const item of value) collectKeyedStrings(item, keys, output, depth + 1) - return output - } - for (const [key, nested] of Object.entries(value as Record)) { - if (keys.has(key) && typeof nested === 'string' && nested.trim()) { - output.push(compactText(nested, 500)) - if (output.length >= 10) return output - } - collectKeyedStrings(nested, keys, output, depth + 1) - if (output.length >= 10) return output - } - return output -} - -function collectUserMessageAttachmentRefs(files: unknown): { - searchText: string[] - filePaths: string[] - fileNames: string[] -} { - const searchText: string[] = [] - const filePaths: string[] = [] - const fileNames: string[] = [] - if (!Array.isArray(files)) { - return { searchText, filePaths, fileNames } - } - for (const file of files) { - if (!isRecordObject(file)) continue - const path = typeof file.path === 'string' ? file.path : '' - const name = typeof file.name === 'string' ? file.name : '' - const metadata = isRecordObject(file.metadata) ? file.metadata : null - const metadataFileName = - metadata && typeof metadata.fileName === 'string' ? metadata.fileName : '' - if (path) { - filePaths.push(compactText(path, 500)) - searchText.push(compactText(path, 500)) - } - for (const value of [name, metadataFileName]) { - if (!value) continue - fileNames.push(compactText(value, 500)) - searchText.push(compactText(value, 500)) - } - } - return { - searchText: uniqueStrings(searchText, 20), - filePaths: uniqueStrings(filePaths, 20), - fileNames: uniqueStrings(fileNames, 20) - } -} - -type UserMessageProjectionText = { - text: string - attachmentRefs: { - searchText: string[] - filePaths: string[] - fileNames: string[] - } -} - -function emptyUserMessageAttachmentRefs(): UserMessageProjectionText['attachmentRefs'] { - return { searchText: [], filePaths: [], fileNames: [] } -} - -function parseUserMessageProjectionText(content: string): UserMessageProjectionText { - const parsed = parseJsonValue(content) - if (isRecordObject(parsed) && typeof parsed.text === 'string') { - const attachmentRefs = collectUserMessageAttachmentRefs(parsed.files) - const parts = [parsed.text] - if (Array.isArray(parsed.files) && parsed.files.length > 0) { - parts.push(`files:${parsed.files.length}`) - parts.push(...attachmentRefs.searchText) - } - if (Array.isArray(parsed.links) && parsed.links.length > 0) { - parts.push(`links:${parsed.links.length}`) - } - return { text: parts.join(' '), attachmentRefs } - } - return { text: content, attachmentRefs: emptyUserMessageAttachmentRefs() } -} - -function getUserMessageProjectionText( - row: DeepChatTapeEntryRow, - payload: Record -): UserMessageProjectionText | null { - if (row.kind !== 'message' || !isRecordObject(payload.record)) return null - const record = payload.record - const role = typeof record.role === 'string' ? record.role : 'message' - if (role === 'assistant') return null - const content = typeof record.content === 'string' ? record.content : '' - return parseUserMessageProjectionText(content) -} - -function collectUserMessageAttachmentRefsFromPayload(payload: Record): { - searchText: string[] - filePaths: string[] - fileNames: string[] -} { - if (!isRecordObject(payload.record)) { - return { searchText: [], filePaths: [], fileNames: [] } - } - const content = typeof payload.record.content === 'string' ? payload.record.content : '' - const parsed = parseJsonValue(content) - return isRecordObject(parsed) - ? collectUserMessageAttachmentRefs(parsed.files) - : { searchText: [], filePaths: [], fileNames: [] } -} - -function readUserMessageText(content: string, parsed?: UserMessageProjectionText): string { - return parsed?.text ?? parseUserMessageProjectionText(content).text -} - -function readAssistantMessageText(content: string): string { - const parsed = parseJsonValue(content) - if (!Array.isArray(parsed)) return content - const parts: string[] = [] - for (const block of parsed) { - if (!isRecordObject(block)) continue - if (typeof block.content === 'string' && block.content.trim()) { - parts.push(block.content) - continue - } - const toolCall = block.tool_call - if (isRecordObject(toolCall)) { - const name = typeof toolCall.name === 'string' ? toolCall.name : 'unknown' - const params = typeof toolCall.params === 'string' ? toolCall.params : '' - const response = typeof toolCall.response === 'string' ? toolCall.response : '' - parts.push(`tool ${name} ${params} ${response}`.trim()) - } - } - return parts.join(' ') -} - -function summarizeTapeRow( - row: DeepChatTapeEntryRow, - payload: Record, - userMessage?: UserMessageProjectionText | null -): string { - if (row.kind === 'message') { - const record = payload.record - if (isRecordObject(record)) { - const role = typeof record.role === 'string' ? record.role : 'message' - const content = typeof record.content === 'string' ? record.content : '' - const text = - role === 'assistant' - ? readAssistantMessageText(content) - : readUserMessageText(content, userMessage ?? undefined) - return compactText(`${role}: ${text}`, 1200) - } - } - - if (row.kind === 'tool_call') { - const toolCall = payload.toolCall - if (isRecordObject(toolCall)) { - const name = typeof toolCall.name === 'string' ? toolCall.name : (row.name ?? 'unknown') - const params = typeof toolCall.params === 'string' ? toolCall.params : '' - return compactText(`tool_call ${name}: ${params}`, 1200) - } - } - - if (row.kind === 'tool_result') { - const response = typeof payload.response === 'string' ? payload.response : payload - return compactText( - `tool_result ${row.name ?? 'unknown'}: ${stringifyForSummary(response)}`, - 1200 - ) - } - - if (row.kind === 'anchor') { - const state = isRecordObject(payload.state) ? payload.state : payload - const summary = typeof state.summary === 'string' ? state.summary : stringifyForSummary(state) - return compactText(`anchor ${row.name ?? 'unknown'}: ${summary}`, 1200) - } - - if (row.kind === 'event') { - const data = isRecordObject(payload.data) ? payload.data : payload - return compactText(`event ${row.name ?? 'unknown'}: ${stringifyForSummary(data)}`, 1200) - } - - return compactText(`${row.kind} ${row.name ?? ''}`.trim(), 1200) -} - -function buildTapeRowEvidenceText( - row: DeepChatTapeEntryRow, - payload: Record, - meta: Record, - userMessage?: UserMessageProjectionText | null -): string { - const parts: string[] = [] - if (row.kind === 'message' && isRecordObject(payload.record)) { - const record = payload.record - const content = typeof record.content === 'string' ? record.content : '' - const role = typeof record.role === 'string' ? record.role : 'message' - parts.push( - role === 'assistant' - ? readAssistantMessageText(content) - : readUserMessageText(content, userMessage ?? undefined) - ) - } else if (row.kind === 'tool_call' && isRecordObject(payload.toolCall)) { - const toolCall = payload.toolCall - parts.push(stringifyForSummary(toolCall.name, 200)) - parts.push(stringifyForSummary(toolCall.params, 3000)) - } else if (row.kind === 'tool_result') { - parts.push(stringifyForSummary(payload.response ?? payload, 4000)) - } else if (row.kind === 'anchor') { - parts.push(stringifyForSummary(isRecordObject(payload.state) ? payload.state : payload, 4000)) - } else if (row.kind === 'event') { - parts.push(stringifyForSummary(isRecordObject(payload.data) ? payload.data : payload, 4000)) - } else { - parts.push(stringifyForSummary(payload, 4000)) - } - if (typeof meta.status === 'string') parts.push(`status:${meta.status}`) - return compactText(parts.filter(Boolean).join('\n'), 5000) -} - -function setRef(target: Record, key: string, value: unknown): void { - if (value !== null && value !== undefined && value !== '') { - target[key] = value - } -} - -function enrichTapeRowRefs( - refs: Record, - payload: Record, - meta: Record, - evidenceText: string, - userMessage?: UserMessageProjectionText | null -): void { - const attachmentRefs = - userMessage?.attachmentRefs ?? collectUserMessageAttachmentRefsFromPayload(payload) - const filePaths = uniqueStrings( - [...extractFilePaths(evidenceText), ...attachmentRefs.filePaths], - 20 - ) - const errorCodes = extractErrorCodes(evidenceText) - const commands = uniqueStrings( - [ - ...collectKeyedStrings(payload, new Set(['command', 'cmd', 'script', 'shellCommand'])), - ...collectKeyedStrings(meta, new Set(['command', 'cmd', 'script', 'shellCommand'])) - ], - 10 - ) - setRef(refs, 'filePaths', filePaths.length ? filePaths : null) - setRef(refs, 'fileNames', attachmentRefs.fileNames.length ? attachmentRefs.fileNames : null) - setRef(refs, 'commands', commands.length ? commands : null) - setRef(refs, 'errorCodes', errorCodes.length ? errorCodes : null) - for (const key of ['exitCode', 'exitStatus', 'code']) { - const value = payload[key] ?? meta[key] - if (typeof value === 'number' || typeof value === 'string') { - setRef(refs, key, value) - } - } -} - -function buildTapeRowRefs( - row: DeepChatTapeEntryRow, - payload: Record, - meta: Record, - userMessage?: UserMessageProjectionText | null, - evidenceText?: string -): Record { - const refs: Record = {} - setRef(refs, 'sourceType', row.source_type) - setRef(refs, 'sourceId', row.source_id) - setRef(refs, 'sourceSeq', row.source_seq) - setRef(refs, 'status', typeof meta.status === 'string' ? meta.status : null) - - if (row.kind === 'message' && isRecordObject(payload.record)) { - const record = payload.record - setRef(refs, 'messageId', record.id) - setRef(refs, 'orderSeq', record.orderSeq) - setRef(refs, 'role', record.role) - setRef(refs, 'messageStatus', record.status) - } else if (row.kind === 'tool_call' && isRecordObject(payload.toolCall)) { - const toolCall = payload.toolCall - setRef(refs, 'messageId', payload.messageId) - setRef(refs, 'orderSeq', payload.orderSeq) - setRef(refs, 'toolCallId', toolCall.id) - setRef(refs, 'toolName', toolCall.name ?? row.name) - setRef(refs, 'serverName', toolCall.serverName) - } else if (row.kind === 'tool_result') { - setRef(refs, 'messageId', payload.messageId) - setRef(refs, 'orderSeq', payload.orderSeq) - setRef(refs, 'toolCallId', payload.toolCallId) - setRef(refs, 'toolName', row.name) - } else if (row.kind === 'anchor') { - setRef(refs, 'anchorName', row.name) - } else if (row.kind === 'event') { - setRef(refs, 'eventName', row.name) - } - - enrichTapeRowRefs( - refs, - payload, - meta, - evidenceText ?? buildTapeRowEvidenceText(row, payload, meta, userMessage), - userMessage - ) - return refs -} - -function parseProjectionRefs(raw: string): Record { - const parsed = parseJsonObject(raw) - return parsed -} - -function normalizeContextWindowValue(value: number | undefined, fallback: number): number { - if (!Number.isFinite(value)) return fallback - return Math.min(Math.max(Math.floor(value as number), 0), 20) -} - -function normalizeContextLimit(value: number | undefined): number { - if (!Number.isFinite(value)) return 50 - return Math.min(Math.max(Math.floor(value as number), 1), 100) -} - -function readToolFactStatus(row: DeepChatTapeEntryRow): string | null { - const status = parseJsonObject(row.meta_json).status - return typeof status === 'string' ? status : null -} - -function readToolFactToolCallId(row: DeepChatTapeEntryRow): string | null { - const payload = parseJsonObject(row.payload_json) - if (row.kind === 'tool_call') { - const toolCall = payload.toolCall - if (toolCall && typeof toolCall === 'object' && !Array.isArray(toolCall)) { - const id = (toolCall as Record).id - return typeof id === 'string' && id.length > 0 ? id : null - } - return null - } - const toolCallId = payload.toolCallId - return typeof toolCallId === 'string' && toolCallId.length > 0 ? toolCallId : null -} - -function readToolFactMessageId(row: DeepChatTapeEntryRow): string | null { - const messageId = parseJsonObject(row.payload_json).messageId - return typeof messageId === 'string' && messageId.length > 0 ? messageId : null -} - -function parseSearchBoundary(value: string | undefined, name: string): number | undefined { - const trimmed = value?.trim() - if (!trimmed) { - return undefined - } - - const numericValue = Number(trimmed) - if (Number.isFinite(numericValue)) { - return numericValue - } - - const parsedDate = Date.parse(trimmed) - if (Number.isFinite(parsedDate)) { - return parsedDate - } - - throw new Error(`${name} must be an ISO date/time or millisecond timestamp.`) -} - -function toTapeSearchInput(options: AgentTapeSearchOptions | undefined): DeepChatTapeSearchInput { - return { - limit: options?.limit, - kinds: options?.kinds, - startCreatedAt: parseSearchBoundary(options?.start, 'start'), - endCreatedAt: parseSearchBoundary(options?.end, 'end') - } -} - -function normalizeTapeViewScope(scope: AgentTapeViewScope | undefined): AgentTapeViewScope { - if (scope === undefined || scope === 'current') return 'current' - if (scope === 'linked_subagents' || scope === 'current_and_linked') return scope - throw new Error(`Invalid Tape view scope: ${String(scope)}`) -} - -function normalizeTapeSearchLimit(value: number | undefined): number { - if (!Number.isFinite(value)) return 20 - return Math.min(Math.max(Math.floor(value as number), 1), 100) -} - -function compareTapeSearchResults(left: TapeSearchResult, right: TapeSearchResult): number { - const leftHasScore = typeof left.score === 'number' && Number.isFinite(left.score) - const rightHasScore = typeof right.score === 'number' && Number.isFinite(right.score) - if (leftHasScore && rightHasScore && left.score !== right.score) { - return (left.score as number) - (right.score as number) - } - if (leftHasScore !== rightHasScore) { - return leftHasScore ? -1 : 1 - } - if (left.createdAt !== right.createdAt) { - return right.createdAt - left.createdAt - } - if (left.sessionId !== right.sessionId) { - return left.sessionId < right.sessionId ? -1 : 1 - } - return right.entryId - left.entryId -} - -function migrationProvenanceKey(sessionId: string): string { - return `migration:${sessionId}:message-backfill:v1` -} - -function legacySummaryProvenanceKey(sessionId: string): string { - return `summary:${sessionId}:legacy-summary:v1` -} - -function normalizeHandoffName(name: string): string { - const trimmed = name.trim() - if (!trimmed) { - return 'handoff/manual' - } - if (trimmed.startsWith('handoff/') || trimmed.startsWith('auto_handoff/')) { - return trimmed - } - return `handoff/${trimmed}` -} - -function normalizePositiveInteger(value: unknown): number | null { - if (typeof value === 'number' && Number.isFinite(value)) { - return Math.max(1, Math.floor(value)) - } - return null -} - -function hasOwnKey(value: Record, key: string): boolean { - return Object.prototype.hasOwnProperty.call(value, key) -} - -function buildOrderSeqRange(records: ChatMessageRecord[]): Record | null { - if (records.length === 0) { - return null - } - - return { - fromOrderSeq: records[0].orderSeq, - toOrderSeq: records[records.length - 1].orderSeq - } -} - -function enrichHandoffState( - state: Record, - historyRecords: ChatMessageRecord[] -): Record { - const maxOrderSeq = historyRecords.reduce( - (currentMax, record) => Math.max(currentMax, record.orderSeq), - 0 - ) - const cursorOrderSeq = - normalizePositiveInteger(state.cursorOrderSeq ?? state.summaryCursorOrderSeq) ?? maxOrderSeq + 1 - const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) - const enrichedState: Record = { - ...state, - cursorOrderSeq - } - - if (!hasOwnKey(enrichedState, 'range')) { - enrichedState.range = buildOrderSeqRange(sourceRecords) - } - - const sourceMessageIds = enrichedState.sourceMessageIds - if (!Array.isArray(sourceMessageIds) || sourceMessageIds.some((id) => typeof id !== 'string')) { - enrichedState.sourceMessageIds = sourceRecords.map((record) => record.id) - } - - return enrichedState -} - -export function normalizeTapeHandoffState(state: unknown): AgentTapeHandoffState { - if (!state || typeof state !== 'object' || Array.isArray(state)) { - throw new Error('Tape handoff requires a non-empty summary.') - } - - const summary = (state as Record).summary - if (typeof summary !== 'string' || !summary.trim()) { - throw new Error('Tape handoff requires a non-empty summary.') - } - - return { - ...(state as Record), - summary: summary.trim() - } -} - -function forkSessionId(parentSessionId: string, forkId: string): string { - return `${parentSessionId}::fork::${forkId}` -} - -function isEntryIdPrefix(prefix: number[], values: number[]): boolean { - if (prefix.length > values.length) return false - for (let index = 0; index < prefix.length; index += 1) { - if (prefix[index] !== values[index]) return false - } - return true -} - -function hashString(value: string): string { - return createHash('sha256').update(value).digest('hex') -} - -function isPositiveInteger(value: number): boolean { - return Number.isInteger(value) && value > 0 -} - -function collectEntryIds(values: Array): number[] { - return [...new Set(values.filter((value): value is number => typeof value === 'number'))].sort( - (left, right) => left - right - ) -} - -const VIEW_POLICIES = new Set([ - 'legacy_context_v1', - 'legacy_context_shadow', - 'resume_shadow', - 'tool_loop_shadow', - 'context_pressure_recovery_shadow' -]) - -const VIEW_ENTRY_REASONS = new Set([ - 'system_prompt', - 'selected_history', - 'new_user_input', - 'resume_target', - 'tool_loop_message' -]) - -const VIEW_EXCLUDED_REASONS = new Set([ - 'before_summary_cursor', - 'compaction_indicator', - 'pending_not_context_history', - 'out_of_budget', - 'empty_after_formatting', - 'superseded', - 'retracted' -]) - -function isRecordObject(value: unknown): value is Record { - return Boolean(value && typeof value === 'object' && !Array.isArray(value)) -} - -function isNullableString(value: unknown): value is string | null { - return value === null || typeof value === 'string' -} - -function isNullableNumber(value: unknown): value is number | null { - return value === null || typeof value === 'number' -} - -function isViewEntryRef(value: unknown): value is DeepChatTapeViewManifest['included'][number] { - if (!isRecordObject(value)) { - return false - } - - return ( - isNullableNumber(value.entryId) && - isNullableString(value.messageId) && - isNullableNumber(value.orderSeq) && - (value.role === 'system' || - value.role === 'user' || - value.role === 'assistant' || - value.role === 'tool' || - value.role === null) && - (value.source === 'tape' || value.source === 'synthetic') && - typeof value.reason === 'string' && - VIEW_ENTRY_REASONS.has(value.reason) - ) -} - -function isViewExcludedRef(value: unknown): value is DeepChatTapeViewManifest['excluded'][number] { - if (!isRecordObject(value)) { - return false - } - - return ( - isNullableNumber(value.entryId) && - isNullableString(value.messageId) && - isNullableNumber(value.orderSeq) && - typeof value.reason === 'string' && - VIEW_EXCLUDED_REASONS.has(value.reason) - ) -} - -function isViewExcludedRange(value: unknown): value is DeepChatTapeViewExcludedRange { - if (!isRecordObject(value)) { - return false - } - - return ( - typeof value.fromOrderSeq === 'number' && - typeof value.toOrderSeq === 'number' && - typeof value.count === 'number' && - typeof value.reason === 'string' && - VIEW_EXCLUDED_REASONS.has(value.reason) - ) -} - -function hasNumberFields(value: unknown, fields: string[]): value is Record { - if (!isRecordObject(value)) { - return false - } - - return fields.every((field) => typeof value[field] === 'number') -} - -function hasStringFields(value: unknown, fields: string[]): value is Record { - if (!isRecordObject(value)) { - return false - } - - return fields.every((field) => typeof value[field] === 'string') -} - -function isViewManifestMeta(value: unknown): value is DeepChatTapeViewManifest['meta'] { - if (!isRecordObject(value)) { - return false - } - - return ( - typeof value.providerId === 'string' && - typeof value.modelId === 'string' && - typeof value.summaryCursorOrderSeq === 'number' && - typeof value.supportsVision === 'boolean' && - typeof value.supportsAudioInput === 'boolean' && - typeof value.traceDebugEnabled === 'boolean' - ) -} - -function isViewManifest(value: unknown, sessionId: string): value is DeepChatTapeViewManifest { - if (!isRecordObject(value)) { - return false - } - - return ( - (value.schemaVersion === 1 || value.schemaVersion === 2) && - typeof value.hashVersion === 'number' && - value.sessionId === sessionId && - typeof value.viewId === 'string' && - typeof value.messageId === 'string' && - typeof value.requestSeq === 'number' && - (value.taskType === 'chat' || value.taskType === 'resume' || value.taskType === 'tool_loop') && - typeof value.policy === 'string' && - VIEW_POLICIES.has(value.policy) && - (typeof value.policyVersion === 'number' || value.policyVersion === null) && - value.contextBuilderVersion === 'legacy-v1' && - typeof value.latestEntryId === 'number' && - Array.isArray(value.anchorEntryIds) && - value.anchorEntryIds.every((entryId) => typeof entryId === 'number') && - (value.reconstructionAnchorEntryId === undefined || - isNullableNumber(value.reconstructionAnchorEntryId)) && - (value.excludedRanges === undefined || - (Array.isArray(value.excludedRanges) && value.excludedRanges.every(isViewExcludedRange))) && - Array.isArray(value.included) && - value.included.every(isViewEntryRef) && - Array.isArray(value.excluded) && - value.excluded.every(isViewExcludedRef) && - hasNumberFields(value.tokenBudget, [ - 'contextLength', - 'requestedMaxTokens', - 'effectiveMaxTokens', - 'reserveTokens', - 'toolReserveTokens', - 'estimatedPromptTokens' - ]) && - hasStringFields(value.hashes, ['promptHash', 'toolDefinitionsHash', 'manifestHash']) && - isViewManifestMeta(value.meta) && - typeof value.assembledAt === 'number' - ) -} - -function withReplaySliceHash( - slice: Omit & { - hashes: Omit & { sliceHash: '' } - } -): DeepChatTapeReplaySlice { - const sliceForHash = { ...slice } as Partial - delete sliceForHash.createdAt - delete sliceForHash.integrity - return { - ...slice, - hashes: { - ...slice.hashes, - sliceHash: hashJson(sliceForHash) - } - } -} - -export class SessionTape implements Pick { - constructor(private readonly database: SessionDatabase) {} - - private get table(): SessionDatabase['deepchatTapeEntriesTable'] { - return this.database.deepchatTapeEntriesTable - } - - private get searchProjectionTable(): SessionDatabase['deepchatTapeSearchProjectionTable'] { - return this.database.deepchatTapeSearchProjectionTable - } - - ensureSessionTapeReady(sessionId: string, messageStore: SessionTranscript): TapeBackfillResult { - const table = this.table - const historyRecords = messageStore - .getMessages(sessionId) - .sort((left, right) => left.orderSeq - right.orderSeq) - const maxOrderSeq = historyRecords.reduce( - (currentMax, record) => Math.max(currentMax, record.orderSeq), - 0 - ) - - table.ensureBootstrapAnchor(sessionId) - - let appendedFactCount = 0 - for (const record of historyRecords) { - appendedFactCount += appendMessageRecordToTape(table, record, 'backfill') - } - - this.backfillLegacySummaryAnchor(sessionId, historyRecords) - - table.appendEvent({ - sessionId, - name: 'migration/backfill', - source: { - type: 'migration', - id: 'message-backfill', - seq: 1 - }, - provenanceKey: migrationProvenanceKey(sessionId), - data: { - source: 'deepchat_messages', - messageCount: historyRecords.length, - maxOrderSeq - }, - idempotent: true - }) - - return { - sessionId, - migrationState: 'ready', - messageCount: historyRecords.length, - maxOrderSeq, - appendedFactCount, - historyRecords: this.getMessageRecords(sessionId) - } - } - - appendMessageRecord(record: ChatMessageRecord): number { - return appendMessageRecordToTape(this.table, record, 'live') - } - - async appendToolFact(input: TapeToolFactInput): Promise { - const row = appendTapeToolFact(this.table, input, 'live', 'tool_loop') - if (!row) throw new Error('Tape tool fact was not appendable.') - return { sessionId: input.sessionId, entryId: row.entry_id } - } - - getMessageRecords(sessionId: string): ChatMessageRecord[] { - return buildEffectiveTapeView(this.table.getBySession(sessionId), { includePending: true }) - .messageRecords - } - - info(sessionId: string): TapeInfo { - const table = this.table - const lastAnchor = table.getLatestAnchor(sessionId) - const rows = table.getBySession(sessionId) - return { - sessionId, - entries: table.countBySession(sessionId), - anchors: table.countAnchorsBySession(sessionId), - lastAnchor: lastAnchor?.name ?? null, - lastAnchorEntryId: lastAnchor?.entry_id ?? null, - entriesSinceLastAnchor: lastAnchor - ? table.countEntriesAfter(sessionId, lastAnchor.entry_id) - : 0, - lastTokenUsage: getLastEffectiveTokenUsage(rows), - migrationState: table.getByProvenanceKey(sessionId, migrationProvenanceKey(sessionId)) - ? 'ready' - : 'none' - } - } - - search(sessionId: string, query: string, options?: AgentTapeSearchOptions): TapeSearchResult[] { - const scope = normalizeTapeViewScope(options?.scope) - if (!query.trim()) { - return [] - } - if (scope === 'current') { - return this.searchCurrentTape(sessionId, query, options) - } - - const resolution = this.resolveLinkedTapeSources(sessionId) - if (resolution.unavailableSourceIds.size > 0) { - const sourceSessionId = [...resolution.unavailableSourceIds].sort()[0] - throw new AgentTapeViewError( - 'linked_tape_unavailable', - sessionId, - sourceSessionId, - `Linked Tape ${sourceSessionId} is unavailable.` - ) - } - const sources = [...resolution.sources] - if (scope === 'current_and_linked') { - sources.push({ sessionId, maxEntryId: this.table?.getMaxEntryId(sessionId) ?? 0 }) - } - return this.searchTapeSourcesReadOnly(sources, query, options) - } - - private searchCurrentTape( - sessionId: string, - query: string, - options?: AgentTapeSearchOptions - ): TapeSearchResult[] { - const table = this.table - const searchInput = toTapeSearchInput(options) - const projectionTable = this.searchProjectionTable - let skipProjectionSearch = false - - if (projectionTable) { - try { - const maxEntryId = table.getMaxEntryId(sessionId) - if (projectionTable.isCurrent(sessionId, maxEntryId)) { - return projectionTable - .search(sessionId, query, searchInput) - .map((row) => this.toProjectedSearchResult(row, undefined)) - } - } catch (error) { - skipProjectionSearch = true - logger.warn( - `[Tape] projection fast-path search failed; falling back to effective search: ${String(error)}` - ) - } - } - - const rows = table.getBySession(sessionId) - const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows - const preparedProjectionTable = skipProjectionSearch - ? null - : this.ensureSearchProjection(sessionId, rows, effectiveRows) - if (!preparedProjectionTable) { - return searchEffectiveTapeRows(rows, query, searchInput).map((row) => - this.toSearchResult(row) - ) - } - - const rowByEntryId = new Map(effectiveRows.map((row) => [row.entry_id, row])) - try { - return preparedProjectionTable - .search(sessionId, query, searchInput) - .map((row) => this.toProjectedSearchResult(row, rowByEntryId.get(row.entry_id))) - } catch (error) { - logger.warn( - `[Tape] projection search failed; falling back to effective search: ${String(error)}` - ) - return searchEffectiveTapeRows(rows, query, searchInput).map((row) => - this.toSearchResult(row) - ) - } - } - - getContext( - sessionId: string, - entryIds: number[], - options: AgentTapeContextOptions = {} - ): AgentTapeContextResult { - const sourceSessionId = options.sourceSessionId?.trim() || sessionId - if (sourceSessionId !== sessionId) { - return this.getLinkedTapeContext(sessionId, sourceSessionId, entryIds, options) - } - - const requestedEntryIds = [ - ...new Set(entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0)) - ].sort((left, right) => left - right) - const table = this.table - if (!table || requestedEntryIds.length === 0) { - return { - sessionId, - sourceSessionId, - requestedEntryIds, - matchedEntryIds: [], - entries: [] - } - } - - const rows = table.getBySession(sessionId) - const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows - const indexByEntryId = new Map(effectiveRows.map((row, index) => [row.entry_id, index])) - const before = normalizeContextWindowValue(options.before, 2) - const after = normalizeContextWindowValue(options.after, 2) - const limit = normalizeContextLimit(options.limit) - const maxBytesPerEntry = normalizeContextByteLimit( - options.maxBytesPerEntry, - DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY, - MAX_CONTEXT_MAX_BYTES_PER_ENTRY - ) - const maxTotalBytes = normalizeContextByteLimit( - options.maxTotalBytes, - DEFAULT_CONTEXT_MAX_TOTAL_BYTES, - MAX_CONTEXT_MAX_TOTAL_BYTES - ) - const selectedIndexes = new Set() - const requestedIndexes: number[] = [] - const windowIndexes: number[] = [] - - for (const entryId of requestedEntryIds) { - const index = indexByEntryId.get(entryId) - if (index === undefined) continue - requestedIndexes.push(index) - for ( - let cursor = Math.max(0, index - before); - cursor <= Math.min(effectiveRows.length - 1, index + after); - cursor += 1 - ) { - if (cursor === index) continue - windowIndexes.push(cursor) - } - } - - for (const index of requestedIndexes) { - if (selectedIndexes.size >= limit) break - selectedIndexes.add(index) - } - for (const index of windowIndexes) { - if (selectedIndexes.size >= limit) break - selectedIndexes.add(index) - } - - const selectedRows = [...selectedIndexes] - .sort((left, right) => left - right) - .map((index) => effectiveRows[index]) - let projectionRows = new Map() - try { - projectionRows = new Map( - this.searchProjectionTable - .getByEntryIds( - sessionId, - selectedRows.map((row) => row.entry_id) - ) - .map((row) => [row.entry_id, row]) - ) - } catch { - projectionRows = new Map() - } - let usedBytes = 0 - const entries: AgentTapeContextEntry[] = [] - const priorityIndexes = [...requestedIndexes, ...windowIndexes].filter( - (index, offset, indexes) => { - return selectedIndexes.has(index) && indexes.indexOf(index) === offset - } - ) - for (const index of priorityIndexes) { - const row = effectiveRows[index] - const remaining = Math.max(0, maxTotalBytes - usedBytes) - if (remaining <= 0) break - const maxEntryBytes = Math.min(maxBytesPerEntry, remaining) - if (maxEntryBytes <= 0) break - const entry = this.toContextEntry(row, projectionRows.get(row.entry_id), maxEntryBytes) - if (entry.evidence.bytes <= 0) continue - usedBytes += entry.evidence.bytes - entries.push(entry) - } - entries.sort((left, right) => left.entryId - right.entryId) - const returnedEntryIds = new Set(entries.map((entry) => entry.entryId)) - - return { - sessionId, - sourceSessionId, - requestedEntryIds, - matchedEntryIds: requestedEntryIds.filter((entryId) => returnedEntryIds.has(entryId)), - entries - } - } - - private resolveLinkedTapeSources(parentSessionId: string): LinkedTapeSourceResolution { - const table = this.table - const sessionTable = this.database.newSessionsTable - if (!table || !sessionTable?.get(parentSessionId)) { - throw new AgentTapeViewError( - 'current_tape_unavailable', - parentSessionId, - parentSessionId, - `Current Tape ${parentSessionId} is unavailable.` - ) - } - - const parsedSnapshots = table - .getSubagentLineageEvents(parentSessionId) - .map((row) => parseSubagentTapeLinkSnapshot(row) ?? parseLegacyExternalTapeLinkSnapshot(row)) - .filter((snapshot): snapshot is SubagentTapeLinkSnapshot => snapshot !== null) - const latestSnapshotByChild = new Map() - for (const snapshot of parsedSnapshots) { - const current = latestSnapshotByChild.get(snapshot.childSessionId) - if (!current || snapshot.linkEntryId > current.linkEntryId) { - latestSnapshotByChild.set(snapshot.childSessionId, snapshot) - } - } - const snapshots = [...latestSnapshotByChild.values()] - const childSessionIds = [...new Set(snapshots.map((snapshot) => snapshot.childSessionId))] - const childById = new Map(sessionTable.getMany(childSessionIds).map((row) => [row.id, row])) - const unavailableSourceIds = new Set() - const snapshotBySource = new Map() - - for (const snapshot of snapshots) { - const child = childById.get(snapshot.childSessionId) - if (!child) { - unavailableSourceIds.add(snapshot.childSessionId) - continue - } - if (child.session_kind !== 'subagent' || child.parent_session_id !== parentSessionId) { - continue - } - snapshotBySource.set(snapshot.childSessionId, snapshot) - } - - const authorizedSourceIds = [...snapshotBySource.keys()] - const firstEntryBySource = new Map( - table.getFirstEntriesBySessions(authorizedSourceIds).map((row) => [row.session_id, row]) - ) - const liveHeads = table.getMaxEntryIdsBySessions(authorizedSourceIds) - const availableSources: DeepChatTapeReadSource[] = [] - for (const [sourceSessionId, snapshot] of snapshotBySource) { - const firstEntry = firstEntryBySource.get(sourceSessionId) - const identityMatches = snapshot.childTapeIdentity - ? firstEntry !== undefined && computeTapeIdentity(firstEntry) === snapshot.childTapeIdentity - : firstEntry !== undefined && isUnmarkedLegacyTape(firstEntry) - if (!identityMatches || (liveHeads.get(sourceSessionId) ?? 0) < snapshot.childHeadEntryId) { - unavailableSourceIds.add(sourceSessionId) - continue - } - availableSources.push({ - sessionId: sourceSessionId, - maxEntryId: snapshot.childHeadEntryId - }) - } - - return { - sources: availableSources.sort((left, right) => - left.sessionId < right.sessionId ? -1 : left.sessionId > right.sessionId ? 1 : 0 - ), - unavailableSourceIds - } - } - - private searchTapeSourcesReadOnly( - sources: DeepChatTapeReadSource[], - query: string, - options: AgentTapeSearchOptions | undefined - ): TapeSearchResult[] { - const table = this.table - if (!table || !query.trim() || sources.length === 0) { - return [] - } - const searchInput = toTapeSearchInput(options) - const limit = normalizeTapeSearchLimit(options?.limit) - const results: TapeSearchResult[] = [] - let uncoveredSources = sources - - try { - const projected = this.searchProjectionTable?.searchSourcesReadOnly( - sources, - query, - searchInput - ) - if (projected) { - const coveredHeadBySource = new Map( - projected.coveredSources.map((source) => [source.sessionId, source.maxEntryId]) - ) - const hasCompleteProjection = - coveredHeadBySource.size === sources.length && - sources.every((source) => coveredHeadBySource.get(source.sessionId) === source.maxEntryId) - if (hasCompleteProjection) { - results.push(...projected.rows.map((row) => this.toProjectedSearchResult(row, undefined))) - uncoveredSources = [] - } - } - } catch (error) { - logger.warn( - `[Tape] linked projection search failed; using read-only Tape fallback: ${String(error)}` - ) - uncoveredSources = sources - } - - if (uncoveredSources.length > 0) { - results.push( - ...table - .searchEffectiveSourcesAtHeads(uncoveredSources, query, searchInput) - .map((row) => this.toSearchResult(row)) - ) - } - - const seen = new Set() - return results - .sort(compareTapeSearchResults) - .filter((result) => { - const key = `${result.sessionId}:${result.entryId}` - if (seen.has(key)) return false - seen.add(key) - return true - }) - .slice(0, limit) - } - - private getLinkedTapeContext( - parentSessionId: string, - sourceSessionId: string, - entryIds: number[], - options: AgentTapeContextOptions - ): AgentTapeContextResult { - const resolution = this.resolveLinkedTapeSources(parentSessionId) - if (resolution.unavailableSourceIds.has(sourceSessionId)) { - throw new AgentTapeViewError( - 'linked_tape_unavailable', - parentSessionId, - sourceSessionId, - `Linked Tape ${sourceSessionId} is unavailable.` - ) - } - const source = resolution.sources.find((candidate) => candidate.sessionId === sourceSessionId) - if (!source) { - throw new AgentTapeViewError( - 'linked_tape_unauthorized', - parentSessionId, - sourceSessionId, - `Tape ${sourceSessionId} is not an authorized direct child of ${parentSessionId}.` - ) - } - - const requestedEntryIds = [ - ...new Set(entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0)) - ].sort((left, right) => left - right) - const table = this.table - if (!table || requestedEntryIds.length === 0) { - return { - sessionId: parentSessionId, - sourceSessionId, - requestedEntryIds, - matchedEntryIds: [], - entries: [] - } - } - - const before = normalizeContextWindowValue(options.before, 2) - const after = normalizeContextWindowValue(options.after, 2) - const limit = normalizeContextLimit(options.limit) - const maxBytesPerEntry = normalizeContextByteLimit( - options.maxBytesPerEntry, - DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY, - MAX_CONTEXT_MAX_BYTES_PER_ENTRY - ) - const maxTotalBytes = normalizeContextByteLimit( - options.maxTotalBytes, - DEFAULT_CONTEXT_MAX_TOTAL_BYTES, - MAX_CONTEXT_MAX_TOTAL_BYTES - ) - const rows = table.getEffectiveContextRowsAtHead(source, requestedEntryIds, { - before, - after, - limit - }) - - let usedBytes = 0 - const entries: AgentTapeContextEntry[] = [] - for (const row of rows) { - const remaining = Math.max(0, maxTotalBytes - usedBytes) - if (remaining <= 0) break - const maxEntryBytes = Math.min(maxBytesPerEntry, remaining) - const entry = this.toContextEntry(row, undefined, maxEntryBytes) - if (entry.evidence.bytes <= 0) continue - usedBytes += entry.evidence.bytes - entries.push(entry) - } - entries.sort((left, right) => left.entryId - right.entryId) - const returnedEntryIds = new Set(entries.map((entry) => entry.entryId)) - - return { - sessionId: parentSessionId, - sourceSessionId, - requestedEntryIds, - matchedEntryIds: requestedEntryIds.filter((entryId) => returnedEntryIds.has(entryId)), - entries - } - } - - anchors(sessionId: string, options: AgentTapeAnchorsOptions = {}): TapeAnchorResult[] { - return this.table.getAnchors(sessionId, options.limit).map((row) => this.toAnchorResult(row)) - } - - getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestSourceMaps { - const table = this.table - const rows = table.getBySession(sessionId) - const entryIdByMessageId = new Map() - const toolCallEntryIdByToolId = new Map() - const toolResultEntryIdByToolId = new Map() - let latestEntryId = 0 - const anchorEntryIds: number[] = [] - let reconstructionAnchorEntryId: number | null = null - let bootstrapAnchorEntryId: number | null = null - - for (const row of rows) { - latestEntryId = Math.max(latestEntryId, row.entry_id) - if (row.kind === 'anchor') { - anchorEntryIds.push(row.entry_id) - if (isReconstructionAnchorName(row.name)) { - if (reconstructionAnchorEntryId === null || row.entry_id > reconstructionAnchorEntryId) { - reconstructionAnchorEntryId = row.entry_id - } - } else if (row.name === BOOTSTRAP_ANCHOR_NAME) { - bootstrapAnchorEntryId = row.entry_id - } - continue - } - if (row.kind === 'message' && row.source_type === 'message' && row.source_id) { - entryIdByMessageId.set(row.source_id, row.entry_id) - continue - } - if (row.kind === 'tool_call' || row.kind === 'tool_result') { - if (messageId && readToolFactMessageId(row) !== messageId) { - continue - } - const toolCallId = readToolFactToolCallId(row) - if (!toolCallId || readToolFactStatus(row) === 'pending') { - continue - } - const target = - row.kind === 'tool_call' ? toolCallEntryIdByToolId : toolResultEntryIdByToolId - target.set(toolCallId, row.entry_id) - } - } - - const reconstructionAnchorEntryIds = - reconstructionAnchorEntryId !== null - ? [reconstructionAnchorEntryId] - : bootstrapAnchorEntryId !== null - ? [bootstrapAnchorEntryId] - : [] - - return { - latestEntryId, - anchorEntryIds, - reconstructionAnchorEntryIds, - reconstructionAnchorEntryId, - entryIdByMessageId, - toolCallEntryIdByToolId, - toolResultEntryIdByToolId - } - } - - appendViewManifest(manifest: DeepChatTapeViewManifest): DeepChatTapeEntryRow { - const table = this.table - table.ensureBootstrapAnchor(manifest.sessionId) - return table.appendEvent({ - sessionId: manifest.sessionId, - name: TAPE_VIEW_MANIFEST_EVENT_NAME, - source: { - type: 'runtime_event', - id: manifest.messageId, - seq: manifest.requestSeq - }, - provenanceKey: `view:${manifest.sessionId}:${manifest.messageId}:${manifest.requestSeq}:${manifest.hashes.manifestHash}`, - data: { - manifest - }, - meta: { - viewId: manifest.viewId, - requestSeq: manifest.requestSeq, - taskType: manifest.taskType, - policy: manifest.policy, - policyVersion: manifest.policyVersion - }, - createdAt: manifest.assembledAt, - idempotent: true - }) - } - - listViewManifestsByMessage( - sessionId: string, - messageId: string - ): DeepChatTapeViewManifestRecord[] { - const table = this.table - return table - .getBySession(sessionId) - .filter( - (row) => - row.kind === 'event' && - row.name === TAPE_VIEW_MANIFEST_EVENT_NAME && - row.source_type === 'runtime_event' && - row.source_id === messageId - ) - .map((row) => this.toViewManifestRecord(row)) - .filter((record): record is DeepChatTapeViewManifestRecord => Boolean(record)) - .sort((left, right) => right.requestSeq - left.requestSeq || right.entryId - left.entryId) - } - - exportReplaySlice( - sessionId: string, - messageId: string, - options: DeepChatTapeReplayExportOptions = {} - ): DeepChatTapeReplaySlice | null { - if (options.requestSeq !== undefined && !isPositiveInteger(options.requestSeq)) { - throw new Error('requestSeq must be a positive integer.') - } - - const manifests = this.listViewManifestsByMessage(sessionId, messageId) - const manifestRecord = - options.requestSeq === undefined - ? manifests[0] - : manifests.find((record) => record.requestSeq === options.requestSeq) - if (!manifestRecord) { - return null - } - - return this.buildReplaySlice(sessionId, messageId, manifestRecord, options) - } - - readCausalObservationSlice( - sessionId: string, - messageId: string, - options: DeepChatCausalObservationReadOptions = {} - ): DeepChatCausalObservationSlice { - if (options.requestSeq !== undefined && !isPositiveInteger(options.requestSeq)) { - throw new Error('requestSeq must be a positive integer.') - } - - const rows = this.table.getBySession(sessionId) - const manifestRows = rows.filter( - (row) => - row.kind === 'event' && - row.name === TAPE_VIEW_MANIFEST_EVENT_NAME && - row.source_type === 'runtime_event' && - row.source_id === messageId - ) - const traces = this.database.deepchatMessageTracesTable - .listByMessageId(messageId) - .filter( - (row) => - row.session_id === sessionId && - row.message_id === messageId && - isPositiveInteger(row.request_seq) - ) - - const requestSeq = - options.requestSeq ?? - [...manifestRows.map((row) => row.source_seq), ...traces.map((row) => row.request_seq)] - .filter((value): value is number => typeof value === 'number' && isPositiveInteger(value)) - .reduce((latest, value) => Math.max(latest ?? value, value), null) - - let request: DeepChatCausalObservationRequest - if (requestSeq === null) { - request = { state: 'request_unavailable', requestSeq: null, trace: null } - } else { - const selectedManifestRows = manifestRows.filter((row) => row.source_seq === requestSeq) - const manifestRecord = selectedManifestRows - .map((row) => this.toViewManifestRecord(row)) - .find((record) => record?.messageId === messageId && record.requestSeq === requestSeq) - const trace = traces.find((row) => row.request_seq === requestSeq) ?? null - - if (manifestRecord) { - request = { - state: 'manifest_bound', - requestSeq, - replay: this.buildReplaySlice(sessionId, messageId, manifestRecord, options) - } - } else { - request = { - state: selectedManifestRows.length > 0 ? 'manifest_malformed' : 'manifest_missing', - requestSeq, - trace: trace - ? this.toReplayTraceSnapshot(trace, options.includeTracePayload === true) - : null - } - } - } - - const outputEntries = buildEffectiveTapeView(rows, { includePending: false }) - .rows.filter( - (row) => - (row.kind === 'message' && - row.source_type === 'message' && - row.source_id === messageId) || - ((row.kind === 'tool_call' || row.kind === 'tool_result') && - readToolFactMessageId(row) === messageId) - ) - .map((row) => this.toReplayEntrySnapshot(row, options.includeTapePayloads === true)) - const message = this.database.deepchatMessagesTable.get(messageId) - const terminalMessage = - message?.session_id === sessionId && - message.role === 'assistant' && - (message.status === 'sent' || message.status === 'error') - ? { - status: message.status, - orderSeq: message.order_seq, - createdAt: message.created_at, - updatedAt: message.updated_at, - contentHash: hashString(message.content), - metadataHash: hashString(message.metadata) - } - : null - - return { - schemaVersion: 1, - sessionId, - messageId, - request, - output: { - correlation: 'message_only', - entries: outputEntries, - terminalMessage - }, - runtime: - options.currentRuntimeStatus === undefined - ? { scope: 'unavailable', status: null, eventHistory: 'not_persisted' } - : { - scope: 'current_only', - status: options.currentRuntimeStatus, - eventHistory: 'not_persisted' - } - } - } - - private buildReplaySlice( - sessionId: string, - messageId: string, - manifestRecord: DeepChatTapeViewManifestRecord, - options: DeepChatTapeReplayExportOptions - ): DeepChatTapeReplaySlice { - const table = this.table - const manifest = manifestRecord.manifest - const includedEntryIds = collectEntryIds(manifest.included.map((ref) => ref.entryId)) - const excludedEntryIds = collectEntryIds(manifest.excluded.map((ref) => ref.entryId)) - const anchorEntryIds = collectEntryIds(manifest.anchorEntryIds) - const selectedEntryIds = new Set([ - manifestRecord.entryId, - ...includedEntryIds, - ...excludedEntryIds, - ...anchorEntryIds - ]) - const entries = table - .getBySession(sessionId) - .filter((row) => selectedEntryIds.has(row.entry_id)) - .map((row) => this.toReplayEntrySnapshot(row, options.includeTapePayloads === true)) - - const trace = this.findReplayTrace(sessionId, messageId, manifestRecord.requestSeq) - const createdAt = Date.now() - const sliceBase: Omit & { - hashes: Omit & { sliceHash: '' } - } = { - schemaVersion: 1 as const, - sliceId: `replay_${hashJson({ - sessionId, - messageId, - requestSeq: manifestRecord.requestSeq, - manifestHash: manifest.hashes.manifestHash - }).slice(0, 16)}`, - sessionId, - messageId, - requestSeq: manifestRecord.requestSeq, - mode: trace ? 'trace_bound' : 'manifest_only', - manifestRecord, - trace: trace ? this.toReplayTraceSnapshot(trace, options.includeTracePayload === true) : null, - entries, - refs: { - manifestEntryId: manifestRecord.entryId, - includedEntryIds, - excludedEntryIds, - anchorEntryIds - }, - hashes: { - manifestHash: manifest.hashes.manifestHash, - sliceHash: '' - }, - integrity: manifestRecord.integrity, - createdAt - } - - return withReplaySliceHash(sliceBase) - } - - handoff( - sessionId: string, - name: string, - state: AgentTapeHandoffState, - meta: Record = {} - ): DeepChatTapeEntryRow { - const normalizedState = normalizeTapeHandoffState(state) - const table = this.table - table.ensureBootstrapAnchor(sessionId) - const handoffState = enrichHandoffState(normalizedState, this.getMessageRecords(sessionId)) - return table.appendAnchor({ - sessionId, - name: normalizeHandoffName(name), - source: { - type: 'runtime_event', - id: `handoff:${Date.now()}`, - seq: 0 - }, - state: handoffState, - meta: { - ...meta, - handoff: true - } - }) - } - - handoffResult( - sessionId: string, - name: string, - state: AgentTapeHandoffState, - meta: Record = {} - ): TapeAnchorResult { - return this.toAnchorResult(this.handoff(sessionId, name, state, meta)) - } - - createFork(parentSessionId: string, forkId: string = nanoid()): TapeForkHandle { - const table = this.table - const forkIdValue = forkId.trim() || nanoid() - const forkSessionIdValue = forkSessionId(parentSessionId, forkIdValue) - const parentHeadEntryId = table.getMaxEntryId(parentSessionId) - table.ensureBootstrapAnchor(forkSessionIdValue) - const parentAnchor = table.getLatestAnchor(parentSessionId) - const forkStart = table.appendAnchor({ - sessionId: forkSessionIdValue, - name: 'fork/start', - source: { - type: 'fork', - id: forkIdValue, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkIdValue}:start`, - state: { - parentSessionId, - parentHeadEntryId, - parentLastAnchorEntryId: parentAnchor?.entry_id ?? null, - parentLastAnchorName: parentAnchor?.name ?? null - }, - idempotent: true - }) - const forkStartPayload = parseJsonObject(forkStart.payload_json) - const persistedState = - forkStartPayload.state && - typeof forkStartPayload.state === 'object' && - !Array.isArray(forkStartPayload.state) - ? (forkStartPayload.state as Record) - : {} - const persistedParentHeadEntryId = persistedState.parentHeadEntryId - return { - parentSessionId, - forkId: forkIdValue, - forkSessionId: forkSessionIdValue, - parentHeadEntryId: - typeof persistedParentHeadEntryId === 'number' && - Number.isSafeInteger(persistedParentHeadEntryId) && - persistedParentHeadEntryId >= 0 - ? persistedParentHeadEntryId - : parentHeadEntryId - } - } - - appendForkMessageRecord(handle: TapeForkHandle, record: ChatMessageRecord): number { - return appendMessageRecordToTape( - this.table, - { - ...record, - sessionId: handle.forkSessionId - }, - 'live' - ) - } - - mergeFork(parentSessionId: string, forkId: string): number { - const table = this.table - const forkSessionIdValue = forkSessionId(parentSessionId, forkId) - const mergeProvenanceKey = `fork:${parentSessionId}:${forkId}:merge:event` - - return table.runInTransaction(() => { - const existingReceipt = table.getByProvenanceKey(parentSessionId, mergeProvenanceKey) - if (existingReceipt) { - return readForkMergeReceiptCount( - existingReceipt, - parentSessionId, - forkId, - forkSessionIdValue - ) - } - - assertValidForkStart( - table.getByProvenanceKey(forkSessionIdValue, `fork:${parentSessionId}:${forkId}:start`), - parentSessionId, - forkId, - forkSessionIdValue - ) - - const forkHeadEntryId = table.getMaxEntryId(forkSessionIdValue) - const forkEntries = table - .getBySessionUpToEntryId(forkSessionIdValue, forkHeadEntryId) - .filter( - (entry) => - !( - entry.kind === 'anchor' && - (entry.name === 'session/start' || entry.name === 'fork/start') - ) - ) - - for (const entry of forkEntries) { - table.append({ - sessionId: parentSessionId, - kind: entry.kind, - name: entry.name, - source: { - type: 'fork', - id: forkId, - seq: entry.entry_id - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:merge:${entry.entry_id}`, - payload: parseJsonObject(entry.payload_json), - meta: { - ...parseJsonObject(entry.meta_json), - forkId, - forkSessionId: forkSessionIdValue, - mergedFromEntryId: entry.entry_id - }, - createdAt: entry.created_at, - idempotent: true - }) - } - - table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/merge', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: mergeProvenanceKey, - data: { - forkId, - forkSessionId: forkSessionIdValue, - forkHeadEntryId, - mergedCount: forkEntries.length - }, - idempotent: true - }) - - return forkEntries.length - }) - } - - discardFork(parentSessionId: string, forkId: string): void { - const table = this.table - const forkSessionIdValue = forkSessionId(parentSessionId, forkId) - table.deleteBySession(forkSessionIdValue) - try { - this.searchProjectionTable.deleteBySession(forkSessionIdValue) - } catch (error) { - logger.warn(`[Tape] failed to delete fork search projection: ${String(error)}`) - } - table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/discard', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:discard:event`, - data: { - forkId, - forkSessionId: forkSessionIdValue - }, - idempotent: true - }) - } - - recordExternalForkMerge( - parentSessionId: string, - forkSessionIdValue: string, - forkId: string, - meta: Record = {} - ): DeepChatTapeEntryRow { - const table = this.table - const referencedEntryCount = table.countBySession(forkSessionIdValue) - return table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/merge', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:external-merge:event`, - data: { - forkId, - forkSessionId: forkSessionIdValue, - referencedEntryCount, - ...meta - }, - idempotent: true - }) - } - - recordExternalForkDiscard( - parentSessionId: string, - forkSessionIdValue: string, - forkId: string, - meta: Record = {} - ): DeepChatTapeEntryRow { - const table = this.table - return table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/discard', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:external-discard:event`, - data: { - forkId, - forkSessionId: forkSessionIdValue, - ...meta - }, - idempotent: true - }) - } - - linkSubagentTape(input: SubagentTapeLinkInput): SubagentTapeLinkReceipt { - const table = this.table - const normalized = normalizeSubagentTapeLinkInput(input) - const provenanceKey = subagentTapeLinkProvenanceKey(normalized) - return table.runInTransaction(() => { - const existing = table.getByProvenanceKey(normalized.parentSessionId, provenanceKey) - if (existing) { - assertSubagentTapeLinkMatchesInput(existing, normalized) - const receipt = toSubagentTapeLinkReceipt(existing) - return receipt - } - - const childFirstEntry = table.getFirstEntriesBySessions([normalized.childSessionId])[0] - if (!childFirstEntry || childFirstEntry.session_id !== normalized.childSessionId) { - throw new Error(`Subagent Tape ${normalized.childSessionId} is unavailable.`) - } - const childTapeIdentity = computeTapeIdentity(childFirstEntry) - const childHeadEntryId = table.getMaxEntryId(normalized.childSessionId) - const childEntryCount = table.countBySession(normalized.childSessionId) - const row = table.appendEvent({ - sessionId: normalized.parentSessionId, - name: SUBAGENT_TAPE_LINK_EVENT_NAME, - source: { - type: 'subagent', - id: normalized.childSessionId, - seq: childHeadEntryId - }, - provenanceKey, - data: { - linkVersion: SUBAGENT_TAPE_LINK_VERSION, - childSessionId: normalized.childSessionId, - childHeadEntryId, - childEntryCount, - childTapeIdentity, - runId: normalized.runId, - taskId: normalized.taskId, - slotId: normalized.slotId, - taskTitle: normalized.taskTitle, - outcome: normalized.outcome, - resultSummary: normalized.resultSummary - }, - idempotent: true - }) - assertSubagentTapeLinkMatchesInput(row, normalized) - const receipt = toSubagentTapeLinkReceipt(row) - return receipt - }) - } - - private backfillLegacySummaryAnchor( - sessionId: string, - historyRecords: ChatMessageRecord[] - ): void { - const table = this.table - if (table.getLatestSummaryAnchor(sessionId)) { - return - } - - const legacyState = this.database.deepchatSessionsTable.getSummaryState(sessionId) - if (!legacyState) { - return - } - - const summary = legacyState.summary_text?.trim() - if (!summary) { - return - } - - const cursorOrderSeq = Math.max(1, legacyState.summary_cursor_order_seq ?? 1) - const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) - table.appendAnchor({ - sessionId, - name: 'compaction/migrated_summary', - source: { - type: 'summary', - id: 'legacy-summary', - seq: 1 - }, - provenanceKey: legacySummaryProvenanceKey(sessionId), - state: { - summary, - cursorOrderSeq, - range: - sourceRecords.length > 0 - ? { - fromOrderSeq: sourceRecords[0].orderSeq, - toOrderSeq: sourceRecords[sourceRecords.length - 1].orderSeq - } - : null, - sourceMessageIds: sourceRecords.map((record) => record.id), - migratedFrom: 'deepchat_sessions.summary_text' - }, - idempotent: true, - createdAt: legacyState.summary_updated_at ?? undefined - }) - } - - private ensureSearchProjection( - sessionId: string, - rows: DeepChatTapeEntryRow[], - effectiveRows: DeepChatTapeEntryRow[] - ): SessionDatabase['deepchatTapeSearchProjectionTable'] | null { - const projectionTable = this.searchProjectionTable - const maxEntryId = rows.reduce((max, row) => Math.max(max, row.entry_id), 0) - try { - if (!projectionTable.isCurrent(sessionId, maxEntryId)) { - const meta = projectionTable.getSessionMeta(sessionId) - const projectedEntryIds = projectionTable.getProjectedEntryIds(sessionId) - const effectiveEntryIds = effectiveRows.map((row) => row.entry_id) - const canAppend = - !!meta && - projectionTable.isCurrent(sessionId, meta.maxEntryId) && - meta.maxEntryId <= maxEntryId && - isEntryIdPrefix(projectedEntryIds, effectiveEntryIds) - if (canAppend) { - projectionTable.appendSession( - sessionId, - effectiveRows.slice(projectedEntryIds.length).map((row) => this.toProjectionInput(row)), - maxEntryId - ) - } else { - projectionTable.replaceSession( - sessionId, - effectiveRows.map((row) => this.toProjectionInput(row)), - maxEntryId - ) - } - } - return projectionTable - } catch { - return null - } - } - - private toProjectionInput(row: DeepChatTapeEntryRow): DeepChatTapeSearchProjectionInput { - const payload = parseJsonObject(row.payload_json) - const meta = parseJsonObject(row.meta_json) - const userMessage = getUserMessageProjectionText(row, payload) - const summaryText = summarizeTapeRow(row, payload, userMessage) - const evidenceText = buildTapeRowEvidenceText(row, payload, meta, userMessage) - const refs = buildTapeRowRefs(row, payload, meta, userMessage, evidenceText) - const searchText = [ - row.kind, - row.name ?? '', - summaryText, - evidenceText, - Object.values(refs) - .map((value) => stringifyForSummary(value)) - .join(' ') - ] - .filter(Boolean) - .join('\n') - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - sourceType: row.source_type, - sourceId: row.source_id, - sourceSeq: row.source_seq, - searchText, - summaryText, - refs, - createdAt: row.created_at - } - } - - private toProjectedSearchResult( - row: DeepChatTapeSearchProjectionResultRow, - _sourceRow: DeepChatTapeEntryRow | undefined - ): TapeSearchResult { - const score = - typeof row.score === 'number' && Number.isFinite(row.score) ? row.score : undefined - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - createdAt: row.created_at, - summary: row.summary_text, - refs: parseProjectionRefs(row.refs_json), - ...(score === undefined ? {} : { score }) - } - } - - private toContextEntry( - row: DeepChatTapeEntryRow, - projectionRow: DeepChatTapeSearchProjectionRow | undefined, - maxBytes: number - ): AgentTapeContextEntry { - const fallbackProjection = projectionRow ? null : this.toProjectionInput(row) - const payload = parseJsonObject(row.payload_json) - const meta = parseJsonObject(row.meta_json) - const evidenceSource = buildTapeRowEvidenceText(row, payload, meta) - const clipped = truncateToUtf8Bytes(evidenceSource, maxBytes) - const bytes = Buffer.byteLength(clipped.text, 'utf8') - return { - entryId: row.entry_id, - kind: row.kind, - name: row.name, - summary: projectionRow?.summary_text ?? fallbackProjection?.summaryText ?? '', - refs: projectionRow - ? parseProjectionRefs(projectionRow.refs_json) - : (fallbackProjection?.refs ?? {}), - evidence: { - text: clipped.text, - truncated: clipped.truncated, - bytes - }, - createdAt: row.created_at - } - } - - private toSearchResult(row: DeepChatTapeEntryRow): TapeSearchResult { - const projection = this.toProjectionInput(row) - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - createdAt: row.created_at, - summary: projection.summaryText, - refs: projection.refs - } - } - - private toAnchorResult(row: DeepChatTapeEntryRow): TapeAnchorResult { - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - payload: parseJsonObject(row.payload_json), - meta: parseJsonObject(row.meta_json), - createdAt: row.created_at - } - } - - private toViewManifestRecord(row: DeepChatTapeEntryRow): DeepChatTapeViewManifestRecord | null { - const payload = parseJsonObject(row.payload_json) - const data = payload.data - const rawManifest = - data && typeof data === 'object' && !Array.isArray(data) - ? (data as Record).manifest - : undefined - const manifest = - isRecordObject(rawManifest) && rawManifest.hashVersion === undefined - ? { ...rawManifest, hashVersion: 1 } - : rawManifest - if (!isViewManifest(manifest, row.session_id)) { - return null - } - - return { - sessionId: row.session_id, - messageId: manifest.messageId, - requestSeq: manifest.requestSeq, - entryId: row.entry_id, - createdAt: row.created_at, - integrity: verifyTapeViewManifestHash(manifest), - manifest - } - } - - private findReplayTrace( - sessionId: string, - messageId: string, - requestSeq: number - ): DeepChatMessageTraceRow | null { - const traceTable = this.database.deepchatMessageTracesTable - return ( - traceTable - .listByMessageId(messageId) - .find((row) => row.session_id === sessionId && row.request_seq === requestSeq) ?? null - ) - } - - private toReplayEntrySnapshot( - row: DeepChatTapeEntryRow, - includePayloads: boolean - ): DeepChatTapeReplayEntrySnapshot { - const snapshot: DeepChatTapeReplayEntrySnapshot = { - entryId: row.entry_id, - kind: row.kind, - name: row.name, - sourceType: row.source_type, - sourceId: row.source_id, - sourceSeq: row.source_seq, - provenanceKey: row.provenance_key, - payloadHash: hashString(row.payload_json), - metaHash: hashString(row.meta_json), - createdAt: row.created_at - } - - if (includePayloads) { - snapshot.payload = parseJsonObject(row.payload_json) - snapshot.meta = parseJsonObject(row.meta_json) - } - - return snapshot - } - - private toReplayTraceSnapshot( - row: DeepChatMessageTraceRow, - includePayload: boolean - ): DeepChatTapeReplayTraceSnapshot { - const snapshot: DeepChatTapeReplayTraceSnapshot = { - id: row.id, - requestSeq: row.request_seq, - providerId: row.provider_id, - modelId: row.model_id, - endpoint: row.endpoint, - headersHash: hashString(row.headers_json), - bodyHash: hashString(row.body_json), - truncated: row.truncated === 1, - createdAt: row.created_at - } - - if (includePayload) { - snapshot.headersJson = row.headers_json - snapshot.bodyJson = row.body_json - } - - return snapshot - } -} +/** @deprecated Import Tape application APIs from `@/tape/application/sessionTape`. */ +export { + AgentTapeViewError, + normalizeSubagentTapeLinkInput, + normalizeTapeHandoffState, + SessionTape +} from '@/tape/application/sessionTape' + +/** @deprecated Import Tape application types from `@/tape/application/sessionTape`. */ +export type { + AgentTapeViewErrorCode, + TapeAnchorResult, + TapeBackfillResult, + TapeForkHandle, + TapeInfo, + TapeMigrationState, + TapeSearchResult, + TapeViewManifestAssemblySources as TapeViewManifestSourceMaps +} from '@/tape/application/sessionTape' diff --git a/src/main/session/data/tapeEffectiveView.ts b/src/main/session/data/tapeEffectiveView.ts index b49135e2bc..a8ee4faa81 100644 --- a/src/main/session/data/tapeEffectiveView.ts +++ b/src/main/session/data/tapeEffectiveView.ts @@ -1,255 +1,9 @@ -import type { ChatMessageRecord } from '@shared/types/agent-interface' -import type { - DeepChatTapeEntryKind, - DeepChatTapeEntryRow, - DeepChatTapeSearchInput -} from '@/session/data/tables/deepchatTapeEntries' -import { - parseNestedTapeJsonObject, - readTapeMessageRetractionId, - readTapeToolIdentity, - tapeEntryToMessageRecord, - tapeMessageRank, - tapeToolRank -} from '@/session/data/tables/deepchatTapeEffectiveSemantics' - -export interface EffectiveMessageEntry { - entryId: number - record: ChatMessageRecord -} - -export interface EffectiveTapeView { - rows: DeepChatTapeEntryRow[] - messageRecords: ChatMessageRecord[] - /** Effective messages paired with their tape entry_id, ordered by orderSeq (for lineage). */ - messageEntries: EffectiveMessageEntry[] -} - -interface EffectiveTapeViewOptions { - includePending?: boolean - includeAuditEvents?: boolean -} - -type EffectiveMessageCandidate = { - row: DeepChatTapeEntryRow - record: ChatMessageRecord -} - -function compareSqliteBinaryText(left: string, right: string): number { - return Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')) -} - -function toNonNegativeInteger(value: unknown): number | null { - if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { - return null - } - return Math.floor(value) -} - -function readTokenUsage(metadata: Record): number | null { - const totalTokens = toNonNegativeInteger(metadata.totalTokens ?? metadata.total_tokens) - if (totalTokens !== null) { - return totalTokens - } - - const inputTokens = toNonNegativeInteger(metadata.inputTokens ?? metadata.input_tokens) - const outputTokens = toNonNegativeInteger(metadata.outputTokens ?? metadata.output_tokens) - if (inputTokens !== null || outputTokens !== null) { - return (inputTokens ?? 0) + (outputTokens ?? 0) - } - - return null -} - -function shouldReplaceMessage( - current: EffectiveMessageCandidate | undefined, - next: EffectiveMessageCandidate, - includePending: boolean -): boolean { - if (!current) { - return true - } - - const currentRank = tapeMessageRank(current.record, includePending) - const nextRank = tapeMessageRank(next.record, includePending) - if (nextRank > currentRank) { - return true - } - if (nextRank < currentRank) { - return false - } - return next.row.entry_id > current.row.entry_id -} - -function isAuditEvent(row: DeepChatTapeEntryRow): boolean { - return ( - row.name === 'message/retracted' || - row.name === 'message/compaction_indicator' || - row.name === 'migration/backfill' - ) -} - -function shouldReplaceToolRow( - current: DeepChatTapeEntryRow | undefined, - next: DeepChatTapeEntryRow, - includePending: boolean -): boolean { - if (!current) { - return true - } - - const currentRank = tapeToolRank(current, includePending) - const nextRank = tapeToolRank(next, includePending) - if (nextRank > currentRank) { - return true - } - if (nextRank < currentRank) { - return false - } - return next.entry_id > current.entry_id -} - -function matchesKinds( - row: DeepChatTapeEntryRow, - kinds: DeepChatTapeEntryKind[] | undefined -): boolean { - return !kinds?.length || kinds.includes(row.kind) -} - -function matchesCreatedAt(row: DeepChatTapeEntryRow, options: DeepChatTapeSearchInput): boolean { - if ( - Number.isFinite(options.startCreatedAt) && - row.created_at < (options.startCreatedAt as number) - ) { - return false - } - if (Number.isFinite(options.endCreatedAt) && row.created_at > (options.endCreatedAt as number)) { - return false - } - return true -} - -function matchesQuery(row: DeepChatTapeEntryRow, normalizedQuery: string): boolean { - const haystack = `${row.payload_json}\n${row.meta_json}\n${row.name ?? ''}`.toLowerCase() - return haystack.includes(normalizedQuery) -} - -export function buildEffectiveTapeView( - rows: DeepChatTapeEntryRow[], - options: EffectiveTapeViewOptions = {} -): EffectiveTapeView { - const includePending = options.includePending === true - const includeAuditEvents = options.includeAuditEvents === true - const messageCandidates = new Map() - const retractedMessageIds = new Set() - const toolRows = new Map() - const anchorRows: DeepChatTapeEntryRow[] = [] - const eventRows: DeepChatTapeEntryRow[] = [] - - for (const row of [...rows].sort((left, right) => left.entry_id - right.entry_id)) { - if (row.kind === 'anchor') { - anchorRows.push(row) - continue - } - - if (row.kind === 'event') { - const retractedMessageId = readTapeMessageRetractionId(row) - if (retractedMessageId) { - messageCandidates.delete(retractedMessageId) - retractedMessageIds.add(retractedMessageId) - } - if (includeAuditEvents || !isAuditEvent(row)) { - eventRows.push(row) - } - continue - } - - if (row.kind === 'message') { - const record = tapeEntryToMessageRecord(row) - if (!record) { - continue - } - const rank = tapeMessageRank(record, includePending) - if (rank === 0) { - continue - } - const candidate = { row, record } - if (shouldReplaceMessage(messageCandidates.get(record.id), candidate, includePending)) { - messageCandidates.set(record.id, candidate) - retractedMessageIds.delete(record.id) - } - continue - } - - const identity = readTapeToolIdentity(row) - if (!identity || tapeToolRank(row, includePending) === 0) { - continue - } - const current = toolRows.get(identity.key)?.row - if (shouldReplaceToolRow(current, row, includePending)) { - toolRows.set(identity.key, { row, messageId: identity.messageId }) - } - } - - const messageRows = [...messageCandidates.values()] - .filter((candidate) => !retractedMessageIds.has(candidate.record.id)) - .sort( - (left, right) => - left.record.orderSeq - right.record.orderSeq || - compareSqliteBinaryText(left.record.id, right.record.id) - ) - const effectiveMessageIds = new Set(messageRows.map((candidate) => candidate.record.id)) - const effectiveToolRows = [...toolRows.values()] - .filter((candidate) => effectiveMessageIds.has(candidate.messageId)) - .map((candidate) => candidate.row) - const effectiveRows = [ - ...anchorRows, - ...eventRows, - ...messageRows.map((candidate) => candidate.row), - ...effectiveToolRows - ].sort((left, right) => left.entry_id - right.entry_id) - - return { - rows: effectiveRows, - messageRecords: messageRows.map((candidate) => candidate.record), - messageEntries: messageRows.map((candidate) => ({ - entryId: candidate.row.entry_id, - record: candidate.record - })) - } -} - -export function searchEffectiveTapeRows( - rows: DeepChatTapeEntryRow[], - query: string, - options: DeepChatTapeSearchInput = {} -): DeepChatTapeEntryRow[] { - const normalizedQuery = query.trim().toLowerCase() - if (!normalizedQuery) { - return [] - } - - const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 - const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) - return buildEffectiveTapeView(rows, { includePending: false }) - .rows.filter((row) => matchesKinds(row, options.kinds)) - .filter((row) => matchesCreatedAt(row, options)) - .filter((row) => matchesQuery(row, normalizedQuery)) - .sort((left, right) => right.entry_id - left.entry_id) - .slice(0, cappedLimit) -} - -export function getLastEffectiveTokenUsage(rows: DeepChatTapeEntryRow[]): number | null { - const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows - for (let index = effectiveRows.length - 1; index >= 0; index -= 1) { - const record = tapeEntryToMessageRecord(effectiveRows[index]) - if (!record || record.role !== 'assistant') { - continue - } - const usage = readTokenUsage(parseNestedTapeJsonObject(record.metadata)) - if (usage !== null) { - return usage - } - } - return null -} +/** @deprecated Import effective Tape helpers from `@/tape/domain/effectiveView`. */ +export { + buildEffectiveTapeView, + getLastEffectiveTokenUsage, + searchEffectiveTapeRows +} from '@/tape/domain/effectiveView' + +/** @deprecated Import effective Tape types from `@/tape/domain/effectiveView`. */ +export type { EffectiveMessageEntry, EffectiveTapeView } from '@/tape/domain/effectiveView' diff --git a/src/main/session/data/tapeFacts.ts b/src/main/session/data/tapeFacts.ts index e8e4028124..9d06afbce1 100644 --- a/src/main/session/data/tapeFacts.ts +++ b/src/main/session/data/tapeFacts.ts @@ -1,352 +1,14 @@ -import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' -import type { TapeToolFactInput } from '@/agent/deepchat/loop/ports' -import { toAppSessionId } from '@/agent/shared/agentSessionIds' -import type { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' -import { buildEffectiveTapeView } from './tapeEffectiveView' -import { hashJson } from './tapeViewManifest' -import { parseAssistantBlocks } from '@/session/data/tables/deepchatTapeEffectiveSemantics' - -export { tapeEntryToMessageRecord } from '@/session/data/tables/deepchatTapeEffectiveSemantics' - -export type TapeFactSource = 'live' | 'backfill' | 'repair' - -function readCompactionStatus(record: ChatMessageRecord): string | null { - try { - const parsed = JSON.parse(record.metadata) as { - messageType?: string - compactionStatus?: unknown - } - if (parsed.messageType !== 'compaction') { - return null - } - return typeof parsed.compactionStatus === 'string' ? parsed.compactionStatus : record.status - } catch { - return null - } -} - -function shouldUseRevisionProvenance(record: ChatMessageRecord, source: TapeFactSource): boolean { - return source === 'repair' || record.status !== 'sent' -} - -function buildMessageProvenanceKey( - record: ChatMessageRecord, - source: TapeFactSource -): string | undefined { - if (!shouldUseRevisionProvenance(record, source)) { - return undefined - } - return `message:${record.id}:revision:${record.status}:${record.updatedAt}` -} - -function buildToolFactProvenanceKey( - kind: 'tool_call' | 'tool_result', - messageId: string, - toolCallId: string, - payload: Record -): string { - return `${kind}:${messageId}:${toolCallId}:${hashJson(payload)}` -} - -function collectPendingInteractionToolIds(blocks: AssistantMessageBlock[]): Set { - const ids = new Set() - for (const block of blocks) { - if ( - block.type === 'action' && - (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && - block.status === 'pending' && - typeof block.tool_call?.id === 'string' && - block.tool_call.id.length > 0 - ) { - ids.add(block.tool_call.id) - } - } - return ids -} - -export function buildTapeToolFactInputs(record: ChatMessageRecord): TapeToolFactInput[] { - if (record.role !== 'assistant') return [] - - const inputs: TapeToolFactInput[] = [] - const blocks = parseAssistantBlocks(record.content) - const pendingInteractionToolIds = collectPendingInteractionToolIds(blocks) - blocks.forEach((block, blockIndex) => { - if (block.type !== 'tool_call' || !block.tool_call) return - if (block.status !== 'success' && block.status !== 'error') return - - const toolCallId = block.tool_call.id - if (!toolCallId || pendingInteractionToolIds.has(toolCallId)) return - const sourceId = `${record.id}:${toolCallId}` - const factBlock = - block.timestamp === undefined ? { ...block, timestamp: record.updatedAt } : block - inputs.push({ - sessionId: toAppSessionId(record.sessionId), - messageId: record.id, - orderSeq: record.orderSeq, - blockIndex, - block: factBlock, - provenance: { source: 'tool_call', sourceId, sequence: blockIndex } - }) - if (typeof block.tool_call.response === 'string' && block.tool_call.response.length > 0) { - inputs.push({ - sessionId: toAppSessionId(record.sessionId), - messageId: record.id, - orderSeq: record.orderSeq, - blockIndex, - block: factBlock, - provenance: { source: 'tool_result', sourceId, sequence: blockIndex } - }) - } - }) - return inputs -} - -export function appendTapeToolFact( - table: DeepChatTapeEntriesTable, - input: TapeToolFactInput, - source: TapeFactSource, - reason?: string -): DeepChatTapeEntryRow | null { - const block = input.block - const toolCall = block.type === 'tool_call' ? block.tool_call : undefined - if ( - !toolCall?.id || - (block.status !== 'success' && block.status !== 'error') || - (input.provenance.source !== 'tool_call' && input.provenance.source !== 'tool_result') - ) { - return null - } - - table.ensureBootstrapAnchor(input.sessionId) - const meta = reason - ? { source, role: 'assistant', status: block.status, reason } - : { source, role: 'assistant', status: block.status } - if (input.provenance.source === 'tool_call') { - const payload = { - messageId: input.messageId, - orderSeq: input.orderSeq, - toolCall: { - id: toolCall.id, - name: toolCall.name, - params: toolCall.params, - serverName: toolCall.server_name, - serverIcons: toolCall.server_icons, - serverDescription: toolCall.server_description - } - } - return table.append({ - sessionId: input.sessionId, - kind: 'tool_call', - name: toolCall.name || 'unknown', - source: { - type: input.provenance.source, - id: input.provenance.sourceId, - seq: input.provenance.sequence - }, - provenanceKey: buildToolFactProvenanceKey('tool_call', input.messageId, toolCall.id, payload), - payload, - meta, - createdAt: block.timestamp, - idempotent: true - }) - } - - if (typeof toolCall.response !== 'string' || toolCall.response.length === 0) return null - const payload = { - messageId: input.messageId, - orderSeq: input.orderSeq, - toolCallId: toolCall.id, - response: toolCall.response, - rtkApplied: toolCall.rtkApplied, - rtkMode: toolCall.rtkMode, - rtkFallbackReason: toolCall.rtkFallbackReason, - imagePreviews: toolCall.imagePreviews - } - return table.append({ - sessionId: input.sessionId, - kind: 'tool_result', - name: toolCall.name || 'unknown', - source: { - type: input.provenance.source, - id: input.provenance.sourceId, - seq: input.provenance.sequence - }, - provenanceKey: buildToolFactProvenanceKey('tool_result', input.messageId, toolCall.id, payload), - payload, - meta, - createdAt: block.timestamp, - idempotent: true - }) -} - -export function appendToolFactsToTape( - table: DeepChatTapeEntriesTable, - record: ChatMessageRecord, - source: TapeFactSource, - reason?: string -): number { - if (record.role !== 'assistant') { - return 0 - } - - return buildTapeToolFactInputs(record).reduce( - (appended, input) => appended + (appendTapeToolFact(table, input, source, reason) ? 1 : 0), - 0 - ) -} - -export function appendMessageRecordToTape( - table: DeepChatTapeEntriesTable, - record: ChatMessageRecord, - source: TapeFactSource -): number { - table.ensureBootstrapAnchor(record.sessionId) - - const compactionStatus = readCompactionStatus(record) - if (compactionStatus) { - table.appendEvent({ - sessionId: record.sessionId, - name: 'message/compaction_indicator', - source: { - type: 'message', - id: record.id, - seq: record.updatedAt - }, - provenanceKey: `message:${record.id}:compaction_indicator:${compactionStatus}:${record.updatedAt}`, - data: { - messageId: record.id, - orderSeq: record.orderSeq, - status: compactionStatus, - metadata: record.metadata - }, - meta: { - source, - status: compactionStatus - }, - createdAt: record.updatedAt, - idempotent: true - }) - return 1 - } - - table.append({ - sessionId: record.sessionId, - kind: 'message', - name: `message/${record.role}`, - source: { - type: 'message', - id: record.id, - seq: 0 - }, - provenanceKey: buildMessageProvenanceKey(record, source), - payload: { - record: { - id: record.id, - sessionId: record.sessionId, - orderSeq: record.orderSeq, - role: record.role, - content: record.content, - status: record.status, - isContextEdge: record.isContextEdge, - metadata: record.metadata, - traceCount: record.traceCount, - createdAt: record.createdAt, - updatedAt: record.updatedAt - } - }, - meta: { - source, - orderSeq: record.orderSeq, - role: record.role, - status: record.status - }, - createdAt: record.createdAt, - idempotent: true - }) - - return 1 + appendToolFactsToTape(table, record, source) -} - -export function appendMessageReplacementToTape( - table: DeepChatTapeEntriesTable, - record: ChatMessageRecord, - reason: string -): number { - table.ensureBootstrapAnchor(record.sessionId) - table.append({ - sessionId: record.sessionId, - kind: 'message', - name: `message/${record.role}`, - source: { - type: 'message', - id: record.id, - seq: record.updatedAt - }, - provenanceKey: `message:${record.id}:revision:${record.updatedAt}`, - payload: { - record: { - id: record.id, - sessionId: record.sessionId, - orderSeq: record.orderSeq, - role: record.role, - content: record.content, - status: record.status, - isContextEdge: record.isContextEdge, - metadata: record.metadata, - traceCount: record.traceCount, - createdAt: record.createdAt, - updatedAt: record.updatedAt - } - }, - meta: { - source: 'live', - correction: true, - reason, - orderSeq: record.orderSeq, - role: record.role, - status: record.status - }, - createdAt: record.updatedAt, - idempotent: true - }) - - return 1 + appendToolFactsToTape(table, record, 'repair') -} - -export function appendMessageRetractionToTape( - table: DeepChatTapeEntriesTable, - record: ChatMessageRecord, - reason: string -): number { - table.ensureBootstrapAnchor(record.sessionId) - table.appendEvent({ - sessionId: record.sessionId, - name: 'message/retracted', - source: { - type: 'message', - id: record.id, - seq: Date.now() - }, - provenanceKey: null, - data: { - messageId: record.id, - orderSeq: record.orderSeq, - role: record.role, - reason - }, - meta: { - source: 'live', - correction: true - }, - idempotent: false - }) - - return 1 -} - -export function tapeEntriesToEffectiveMessageRecords( - rows: DeepChatTapeEntryRow[] -): ChatMessageRecord[] { - return buildEffectiveTapeView(rows, { includePending: true }).messageRecords -} +/** @deprecated Import Tape fact helpers from `@/tape/application/factPersistence`. */ +export { + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + appendTapeToolFact, + appendToolFactsToTape, + buildTapeToolFactInputs, + tapeEntriesToEffectiveMessageRecords, + tapeEntryToMessageRecord +} from '@/tape/application/factPersistence' + +/** @deprecated Import Tape fact types from `@/tape/application/factPersistence`. */ +export type { TapeFactSource } from '@/tape/application/factPersistence' diff --git a/src/main/session/data/tapeViewManifest.ts b/src/main/session/data/tapeViewManifest.ts index 95794a955d..529d6f7707 100644 --- a/src/main/session/data/tapeViewManifest.ts +++ b/src/main/session/data/tapeViewManifest.ts @@ -1,362 +1,25 @@ -import { createHash } from 'crypto' -import type { ChatMessage } from '@shared/types/core/chat-message' -import type { MCPToolDefinition } from '@shared/types/core/mcp' -import type { ChatMessageRecord } from '@shared/types/agent-interface' -import type { - DeepChatTapeViewEntryRef, - DeepChatTapeViewExcludedRange, - DeepChatTapeViewExcludedRef, - DeepChatTapeViewManifest, - DeepChatTapeViewManifestIntegrity, - DeepChatTapeViewPolicy, - DeepChatTapeViewTaskType, - DeepChatTapeViewTokenBudget -} from '@shared/types/tape-view-manifest' -import { estimateMessagesTokens } from '@shared/utils/messageTokens' - -export type ContextSummaryCursorMetadata = { - summaryCursorOrderSeq: number - preCursorOrderSeqMin: number | null - preCursorOrderSeqMax: number | null - preCursorCount: number -} - -export function isCompactionRecord(record: ChatMessageRecord): boolean { - try { - const metadata = JSON.parse(record.metadata) as { messageType?: string } - return metadata.messageType === 'compaction' - } catch { - return false - } -} - -export const TAPE_VIEW_MANIFEST_EVENT_NAME = 'view/assembled' -export const TAPE_VIEW_CONTEXT_BUILDER_VERSION = 'legacy-v1' as const - -export type TapeViewManifestSourceMaps = { - entryIdByMessageId?: Map - toolCallEntryIdByToolId?: Map - toolResultEntryIdByToolId?: Map -} - -export type TapeViewManifestBuildInput = { - sessionId: string - messageId: string - requestSeq: number - taskType: DeepChatTapeViewTaskType - policy: DeepChatTapeViewPolicy - policyVersion?: number | null - messages: ChatMessage[] - tools: MCPToolDefinition[] - latestEntryId: number - anchorEntryIds: number[] - reconstructionAnchorEntryId?: number | null - included: DeepChatTapeViewEntryRef[] - excluded: DeepChatTapeViewExcludedRef[] - summaryCursor?: ContextSummaryCursorMetadata - tokenBudget: Omit - providerId: string - modelId: string - summaryCursorOrderSeq: number - supportsVision: boolean - supportsAudioInput: boolean - traceDebugEnabled: boolean - assembledAt?: number -} - -export type TapeViewManifestPolicyInput = { - recoveredFromContextPressure: boolean - isInitialViewRequest: boolean - viewPolicy?: DeepChatTapeViewPolicy - viewPolicyVersion?: number | null -} - -export type TapeViewManifestPolicyResult = { - policy: DeepChatTapeViewPolicy - policyVersion: number | null -} - -export type TapeViewContextSelection = { - includedRecords: Array<{ - record: ChatMessageRecord - reason: DeepChatTapeViewEntryRef['reason'] - }> - excludedRecords: Array<{ - record: ChatMessageRecord - reason: DeepChatTapeViewExcludedRef['reason'] - }> - summaryCursor?: ContextSummaryCursorMetadata - includesSystemPrompt: boolean - newUserMessageId?: string | null -} - -export function resolveTapeViewManifestPolicy( - input: TapeViewManifestPolicyInput -): TapeViewManifestPolicyResult { - if (input.recoveredFromContextPressure) { - return { - policy: 'context_pressure_recovery_shadow', - policyVersion: null - } - } - - if (input.isInitialViewRequest && input.viewPolicy) { - return { - policy: input.viewPolicy, - policyVersion: input.viewPolicyVersion ?? null - } - } - - return { - policy: 'tool_loop_shadow', - policyVersion: null - } -} - -function normalizeForStableJson(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map(normalizeForStableJson) - } - - if (!value || typeof value !== 'object') { - return value - } - - const record = value as Record - return Object.keys(record) - .sort() - .reduce>((result, key) => { - const nested = record[key] - if (nested !== undefined) { - result[key] = normalizeForStableJson(nested) - } - return result - }, {}) -} - -export function stableJsonStringify(value: unknown): string { - return JSON.stringify(normalizeForStableJson(value)) -} - -export function hashJson(value: unknown): string { - return createHash('sha256').update(stableJsonStringify(value)).digest('hex') -} - -export const TAPE_VIEW_MANIFEST_HASH_VERSION = 2 - -function buildManifestHash(manifest: DeepChatTapeViewManifest): string { - const hashable: Record = { ...manifest } - delete hashable.assembledAt - delete hashable.viewId - hashable.hashes = { - promptHash: manifest.hashes.promptHash, - toolDefinitionsHash: manifest.hashes.toolDefinitionsHash - } - return hashJson(hashable) -} - -function buildExcludedRanges( - summaryCursor?: ContextSummaryCursorMetadata -): DeepChatTapeViewExcludedRange[] { - if ( - !summaryCursor || - summaryCursor.preCursorCount === 0 || - summaryCursor.preCursorOrderSeqMin === null || - summaryCursor.preCursorOrderSeqMax === null - ) { - return [] - } - return [ - { - fromOrderSeq: summaryCursor.preCursorOrderSeqMin, - toOrderSeq: summaryCursor.preCursorOrderSeqMax, - count: summaryCursor.preCursorCount, - reason: 'before_summary_cursor' - } - ] -} - -export function verifyTapeViewManifestHash( - manifest: DeepChatTapeViewManifest -): DeepChatTapeViewManifestIntegrity { - if (manifest.hashVersion !== TAPE_VIEW_MANIFEST_HASH_VERSION) { - return 'unverified' - } - return buildManifestHash(manifest) === manifest.hashes.manifestHash ? 'valid' : 'invalid' -} - -export function createTapeViewManifest( - input: TapeViewManifestBuildInput -): DeepChatTapeViewManifest { - const assembledAt = input.assembledAt ?? Date.now() - const excludedRanges = buildExcludedRanges(input.summaryCursor) - const draft: DeepChatTapeViewManifest = { - schemaVersion: 2, - hashVersion: TAPE_VIEW_MANIFEST_HASH_VERSION, - viewId: '', - sessionId: input.sessionId, - messageId: input.messageId, - requestSeq: input.requestSeq, - taskType: input.taskType, - policy: input.policy, - policyVersion: input.policyVersion ?? null, - contextBuilderVersion: TAPE_VIEW_CONTEXT_BUILDER_VERSION, - latestEntryId: input.latestEntryId, - anchorEntryIds: [...input.anchorEntryIds], - ...(input.reconstructionAnchorEntryId !== undefined - ? { reconstructionAnchorEntryId: input.reconstructionAnchorEntryId } - : {}), - included: input.included.map((entry) => ({ ...entry })), - excluded: input.excluded.map((entry) => ({ ...entry })), - ...(excludedRanges.length > 0 ? { excludedRanges } : {}), - tokenBudget: { - ...input.tokenBudget, - estimatedPromptTokens: estimateMessagesTokens(input.messages) - }, - hashes: { - promptHash: hashJson(input.messages), - toolDefinitionsHash: hashJson(input.tools), - manifestHash: '' - }, - meta: { - providerId: input.providerId, - modelId: input.modelId, - summaryCursorOrderSeq: input.summaryCursorOrderSeq, - supportsVision: input.supportsVision, - supportsAudioInput: input.supportsAudioInput, - traceDebugEnabled: input.traceDebugEnabled - }, - assembledAt - } - - const manifestHash = buildManifestHash(draft) - return { - ...draft, - viewId: `view_${manifestHash.slice(0, 16)}`, - hashes: { ...draft.hashes, manifestHash } - } -} - -export function buildIncludedRefs( - selection: TapeViewContextSelection, - sourceMaps: TapeViewManifestSourceMaps = {} -): DeepChatTapeViewEntryRef[] { - const refs: DeepChatTapeViewEntryRef[] = [] - - if (selection.includesSystemPrompt) { - refs.push({ - entryId: null, - messageId: null, - orderSeq: null, - role: 'system', - source: 'synthetic', - reason: 'system_prompt' - }) - } - - for (const item of selection.includedRecords) { - refs.push({ - entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, - messageId: item.record.id, - orderSeq: item.record.orderSeq, - role: item.record.role, - source: sourceMaps.entryIdByMessageId?.has(item.record.id) ? 'tape' : 'synthetic', - reason: item.reason - }) - } - - if (selection.newUserMessageId) { - refs.push({ - entryId: sourceMaps.entryIdByMessageId?.get(selection.newUserMessageId) ?? null, - messageId: selection.newUserMessageId, - orderSeq: null, - role: 'user', - source: sourceMaps.entryIdByMessageId?.has(selection.newUserMessageId) ? 'tape' : 'synthetic', - reason: 'new_user_input' - }) - } - - return refs -} - -export function buildExcludedRefs( - selection: TapeViewContextSelection, - sourceMaps: TapeViewManifestSourceMaps = {} -): DeepChatTapeViewExcludedRef[] { - return selection.excludedRecords.map((item) => ({ - entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, - messageId: item.record.id, - orderSeq: item.record.orderSeq, - reason: item.reason - })) -} - -export function buildRequestRefs( - messages: ChatMessage[], - sourceMaps: TapeViewManifestSourceMaps = {} -): DeepChatTapeViewEntryRef[] { - const lastToolCallIndex = new Map() - const lastToolResultIndex = new Map() - messages.forEach((message, index) => { - if (message.role === 'assistant' && message.tool_calls?.length) { - for (const toolCall of message.tool_calls) { - lastToolCallIndex.set(toolCall.id, index) - } - } else if (message.role === 'tool' && message.tool_call_id) { - lastToolResultIndex.set(message.tool_call_id, index) - } - }) - - const refs: DeepChatTapeViewEntryRef[] = [] - messages.forEach((message, index) => { - if (message.role === 'assistant' && message.tool_calls?.length) { - for (const toolCall of message.tool_calls) { - const entryId = - lastToolCallIndex.get(toolCall.id) === index - ? (sourceMaps.toolCallEntryIdByToolId?.get(toolCall.id) ?? null) - : null - refs.push({ - entryId, - messageId: null, - orderSeq: null, - role: 'assistant', - source: entryId === null ? 'synthetic' : 'tape', - reason: 'tool_loop_message' - }) - } - return - } - - if (message.role === 'tool' && message.tool_call_id) { - const entryId = - lastToolResultIndex.get(message.tool_call_id) === index - ? (sourceMaps.toolResultEntryIdByToolId?.get(message.tool_call_id) ?? null) - : null - refs.push({ - entryId, - messageId: null, - orderSeq: null, - role: 'tool', - source: entryId === null ? 'synthetic' : 'tape', - reason: 'tool_loop_message' - }) - return - } - - refs.push({ - entryId: null, - messageId: null, - orderSeq: null, - role: message.role, - source: 'synthetic', - reason: - message.role === 'system' - ? 'system_prompt' - : message.role === 'tool' - ? 'tool_loop_message' - : 'selected_history' - }) - }) - - return refs -} +/** @deprecated Import ViewManifest helpers from `@/tape/domain/viewManifest`. */ +export { + buildExcludedRefs, + buildIncludedRefs, + buildRequestRefs, + createTapeViewManifest, + hashJson, + isCompactionRecord, + resolveTapeViewManifestPolicy, + stableJsonStringify, + TAPE_VIEW_CONTEXT_BUILDER_VERSION, + TAPE_VIEW_MANIFEST_EVENT_NAME, + TAPE_VIEW_MANIFEST_HASH_VERSION, + verifyTapeViewManifestHash +} from '@/tape/domain/viewManifest' + +/** @deprecated Import ViewManifest types from `@/tape/domain/viewManifest`. */ +export type { + ContextSummaryCursorMetadata, + TapeViewContextSelection, + TapeViewManifestBuildInput, + TapeViewManifestLookupMaps as TapeViewManifestSourceMaps, + TapeViewManifestPolicyInput, + TapeViewManifestPolicyResult +} from '@/tape/domain/viewManifest' diff --git a/src/main/session/data/transcript.ts b/src/main/session/data/transcript.ts index 4e986d9e3d..f1d9764af3 100644 --- a/src/main/session/data/transcript.ts +++ b/src/main/session/data/transcript.ts @@ -23,11 +23,7 @@ import { resolveUsageModelId, resolveUsageProviderId } from '@/session/usageStats' -import { - appendMessageRecordToTape, - appendMessageReplacementToTape, - appendMessageRetractionToTape -} from '@/session/data/tapeFacts' +import type { TapeMessageFactWriter } from '@/tape/ports/capabilities' function shouldConvertPendingBlockToError( status: AssistantMessageBlock['status'] @@ -128,9 +124,11 @@ function extractSearchableMessageContent(rawContent: string): string { export class SessionTranscript { private database: SessionDatabase + private readonly tapeFacts: TapeMessageFactWriter - constructor(database: SessionDatabase) { + constructor(database: SessionDatabase, tapeFacts: TapeMessageFactWriter) { this.database = database + this.tapeFacts = tapeFacts } private runInDatabaseTransaction(operation: () => T): T { @@ -372,11 +370,7 @@ export class SessionTranscript { } const updated = this.getMessage(messageId) if (updated) { - appendMessageReplacementToTape( - this.database.deepchatTapeEntriesTable, - updated, - 'message_content_updated' - ) + this.tapeFacts.appendMessageReplacement(updated, 'message_content_updated') } return } @@ -394,11 +388,7 @@ export class SessionTranscript { } const updated = this.getMessage(messageId) if (updated) { - appendMessageReplacementToTape( - this.database.deepchatTapeEntriesTable, - updated, - 'message_content_updated' - ) + this.tapeFacts.appendMessageReplacement(updated, 'message_content_updated') } } @@ -421,11 +411,7 @@ export class SessionTranscript { this.runInDatabaseTransaction(() => { const record = this.getMessage(messageId) if (record) { - appendMessageRetractionToTape( - this.database.deepchatTapeEntriesTable, - record, - 'message_deleted' - ) + this.tapeFacts.appendMessageRetraction(record, 'message_deleted') } this.database.deepchatSearchDocumentsTable.delete(`message:${messageId}`) this.database.deepchatAssistantBlocksTable.delete(messageId) @@ -444,11 +430,7 @@ export class SessionTranscript { (record) => record.orderSeq >= fromOrderSeq ) for (const record of records) { - appendMessageRetractionToTape( - this.database.deepchatTapeEntriesTable, - record, - 'messages_deleted_from_order_seq' - ) + this.tapeFacts.appendMessageRetraction(record, 'messages_deleted_from_order_seq') } const messageIds = records.map((record) => record.id) if (messageIds.length > 0) { @@ -685,7 +667,7 @@ export class SessionTranscript { if (!record) { return } - appendMessageRecordToTape(this.database.deepchatTapeEntriesTable, record, 'live') + this.tapeFacts.appendMessageRecord(record) } private toRecord(row: DeepChatMessageRow): ChatMessageRecord { diff --git a/src/main/session/transcriptMutations.ts b/src/main/session/transcriptMutations.ts index eb15255dee..c3bbf21d27 100644 --- a/src/main/session/transcriptMutations.ts +++ b/src/main/session/transcriptMutations.ts @@ -20,6 +20,7 @@ export interface SessionTranscriptMutationDependencies { settings: SessionSettingsStore pendingInputs: SessionPendingInputs runtime: SessionTranscriptRuntimePort + runInTransaction(operation: () => T): T } export class SessionTranscriptMutations { @@ -27,9 +28,11 @@ export class SessionTranscriptMutations { async clearMessages(sessionId: string): Promise { await this.dependencies.runtime.prepareClearMessages(sessionId) - this.dependencies.pendingInputs.deleteBySession(sessionId) - this.dependencies.transcript.deleteBySession(sessionId) - this.dependencies.settings.resetTape(sessionId) + this.dependencies.runInTransaction(() => { + this.dependencies.pendingInputs.deleteBySession(sessionId) + this.dependencies.transcript.deleteBySession(sessionId) + this.dependencies.settings.resetTape(sessionId) + }) this.dependencies.runtime.finishClearMessages(sessionId) } diff --git a/src/main/tape/application/common.ts b/src/main/tape/application/common.ts new file mode 100644 index 0000000000..5eb330afa3 --- /dev/null +++ b/src/main/tape/application/common.ts @@ -0,0 +1,29 @@ +export function parseJsonObject(raw: string): Record { + try { + const parsed = JSON.parse(raw) as unknown + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record + } + } catch {} + return {} +} + +export function parseJsonValue(raw: string): unknown { + try { + return JSON.parse(raw) as unknown + } catch { + return null + } +} + +export function isEntryIdPrefix(prefix: number[], values: number[]): boolean { + if (prefix.length > values.length) return false + for (let index = 0; index < prefix.length; index += 1) { + if (prefix[index] !== values[index]) return false + } + return true +} + +export function migrationProvenanceKey(sessionId: string): string { + return `migration:${sessionId}:message-backfill:v1` +} diff --git a/src/main/tape/application/contracts.ts b/src/main/tape/application/contracts.ts new file mode 100644 index 0000000000..4c537623cb --- /dev/null +++ b/src/main/tape/application/contracts.ts @@ -0,0 +1,39 @@ +import type { AgentTapeAnchorResult } from '@shared/types/agent-interface' +import type { TapeMigrationState } from '../ports/capabilities' + +export type { + TapeBackfillResult, + TapeMigrationState, + TapeViewManifestAssemblySources +} from '../ports/capabilities' + +export type TapeInfo = { + sessionId: string + entries: number + anchors: number + lastAnchor: string | null + lastAnchorEntryId: number | null + entriesSinceLastAnchor: number + lastTokenUsage: number | null + migrationState: TapeMigrationState +} + +export type TapeSearchResult = { + sessionId: string + entryId: number + kind: string + name: string | null + createdAt: number + summary?: string + refs?: Record + score?: number +} + +export type TapeAnchorResult = AgentTapeAnchorResult + +export type TapeForkHandle = { + parentSessionId: string + forkId: string + forkSessionId: string + parentHeadEntryId: number +} diff --git a/src/main/tape/application/factPersistence.ts b/src/main/tape/application/factPersistence.ts new file mode 100644 index 0000000000..a320777677 --- /dev/null +++ b/src/main/tape/application/factPersistence.ts @@ -0,0 +1,352 @@ +import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' +import { toTapeSessionId, type TapeFactSource, type TapeToolFactInput } from '@/tape/domain/facts' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeBootstrapStore, TapeEntryStore } from '@/tape/ports/storage' +import { buildEffectiveTapeView } from '@/tape/domain/effectiveView' +import { parseAssistantBlocks } from '@/tape/domain/effectiveSemantics' +import { hashJson } from '@/tape/domain/viewManifest' + +export { tapeEntryToMessageRecord } from '@/tape/domain/effectiveSemantics' +export type { TapeFactSource } from '@/tape/domain/facts' + +type TapeFactStore = Pick & TapeBootstrapStore + +function readCompactionStatus(record: ChatMessageRecord): string | null { + try { + const parsed = JSON.parse(record.metadata) as { + messageType?: string + compactionStatus?: unknown + } + if (parsed.messageType !== 'compaction') { + return null + } + return typeof parsed.compactionStatus === 'string' ? parsed.compactionStatus : record.status + } catch { + return null + } +} + +function shouldUseRevisionProvenance(record: ChatMessageRecord, source: TapeFactSource): boolean { + return source === 'repair' || record.status !== 'sent' +} + +function buildMessageProvenanceKey( + record: ChatMessageRecord, + source: TapeFactSource +): string | undefined { + if (!shouldUseRevisionProvenance(record, source)) { + return undefined + } + return `message:${record.id}:revision:${record.status}:${record.updatedAt}` +} + +function buildToolFactProvenanceKey( + kind: 'tool_call' | 'tool_result', + messageId: string, + toolCallId: string, + payload: Record +): string { + return `${kind}:${messageId}:${toolCallId}:${hashJson(payload)}` +} + +function collectPendingInteractionToolIds(blocks: AssistantMessageBlock[]): Set { + const ids = new Set() + for (const block of blocks) { + if ( + block?.type === 'action' && + (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && + block.status === 'pending' && + typeof block.tool_call?.id === 'string' && + block.tool_call.id.length > 0 + ) { + ids.add(block.tool_call.id) + } + } + return ids +} + +export function buildTapeToolFactInputs(record: ChatMessageRecord): TapeToolFactInput[] { + if (record.role !== 'assistant') return [] + + const inputs: TapeToolFactInput[] = [] + const blocks = parseAssistantBlocks(record.content) + const pendingInteractionToolIds = collectPendingInteractionToolIds(blocks) + blocks.forEach((block, blockIndex) => { + if (block?.type !== 'tool_call' || !block.tool_call) return + if (block.status !== 'success' && block.status !== 'error') return + + const toolCallId = block.tool_call.id + if (!toolCallId || pendingInteractionToolIds.has(toolCallId)) return + const sourceId = `${record.id}:${toolCallId}` + const factBlock = + block.timestamp === undefined ? { ...block, timestamp: record.updatedAt } : block + inputs.push({ + sessionId: toTapeSessionId(record.sessionId), + messageId: record.id, + orderSeq: record.orderSeq, + blockIndex, + block: factBlock, + provenance: { source: 'tool_call', sourceId, sequence: blockIndex } + }) + if (typeof block.tool_call.response === 'string' && block.tool_call.response.length > 0) { + inputs.push({ + sessionId: toTapeSessionId(record.sessionId), + messageId: record.id, + orderSeq: record.orderSeq, + blockIndex, + block: factBlock, + provenance: { source: 'tool_result', sourceId, sequence: blockIndex } + }) + } + }) + return inputs +} + +export function appendTapeToolFact( + table: TapeFactStore, + input: TapeToolFactInput, + source: TapeFactSource, + reason?: string +): DeepChatTapeEntryRow | null { + const block = input.block + const toolCall = block.type === 'tool_call' ? block.tool_call : undefined + if ( + !toolCall?.id || + (block.status !== 'success' && block.status !== 'error') || + (input.provenance.source !== 'tool_call' && input.provenance.source !== 'tool_result') + ) { + return null + } + + table.ensureBootstrapAnchor(input.sessionId) + const meta = reason + ? { source, role: 'assistant', status: block.status, reason } + : { source, role: 'assistant', status: block.status } + if (input.provenance.source === 'tool_call') { + const payload = { + messageId: input.messageId, + orderSeq: input.orderSeq, + toolCall: { + id: toolCall.id, + name: toolCall.name, + params: toolCall.params, + serverName: toolCall.server_name, + serverIcons: toolCall.server_icons, + serverDescription: toolCall.server_description + } + } + return table.append({ + sessionId: input.sessionId, + kind: 'tool_call', + name: toolCall.name || 'unknown', + source: { + type: input.provenance.source, + id: input.provenance.sourceId, + seq: input.provenance.sequence + }, + provenanceKey: buildToolFactProvenanceKey('tool_call', input.messageId, toolCall.id, payload), + payload, + meta, + createdAt: block.timestamp, + idempotent: true + }) + } + + if (typeof toolCall.response !== 'string' || toolCall.response.length === 0) return null + const payload = { + messageId: input.messageId, + orderSeq: input.orderSeq, + toolCallId: toolCall.id, + response: toolCall.response, + rtkApplied: toolCall.rtkApplied, + rtkMode: toolCall.rtkMode, + rtkFallbackReason: toolCall.rtkFallbackReason, + imagePreviews: toolCall.imagePreviews + } + return table.append({ + sessionId: input.sessionId, + kind: 'tool_result', + name: toolCall.name || 'unknown', + source: { + type: input.provenance.source, + id: input.provenance.sourceId, + seq: input.provenance.sequence + }, + provenanceKey: buildToolFactProvenanceKey('tool_result', input.messageId, toolCall.id, payload), + payload, + meta, + createdAt: block.timestamp, + idempotent: true + }) +} + +export function appendToolFactsToTape( + table: TapeFactStore, + record: ChatMessageRecord, + source: TapeFactSource, + reason?: string +): number { + if (record.role !== 'assistant') { + return 0 + } + + return buildTapeToolFactInputs(record).reduce( + (appended, input) => appended + (appendTapeToolFact(table, input, source, reason) ? 1 : 0), + 0 + ) +} + +export function appendMessageRecordToTape( + table: TapeFactStore, + record: ChatMessageRecord, + source: TapeFactSource +): number { + table.ensureBootstrapAnchor(record.sessionId) + + const compactionStatus = readCompactionStatus(record) + if (compactionStatus) { + table.appendEvent({ + sessionId: record.sessionId, + name: 'message/compaction_indicator', + source: { + type: 'message', + id: record.id, + seq: record.updatedAt + }, + provenanceKey: `message:${record.id}:compaction_indicator:${compactionStatus}:${record.updatedAt}`, + data: { + messageId: record.id, + orderSeq: record.orderSeq, + status: compactionStatus, + metadata: record.metadata + }, + meta: { + source, + status: compactionStatus + }, + createdAt: record.updatedAt, + idempotent: true + }) + return 1 + } + + table.append({ + sessionId: record.sessionId, + kind: 'message', + name: `message/${record.role}`, + source: { + type: 'message', + id: record.id, + seq: 0 + }, + provenanceKey: buildMessageProvenanceKey(record, source), + payload: { + record: { + id: record.id, + sessionId: record.sessionId, + orderSeq: record.orderSeq, + role: record.role, + content: record.content, + status: record.status, + isContextEdge: record.isContextEdge, + metadata: record.metadata, + traceCount: record.traceCount, + createdAt: record.createdAt, + updatedAt: record.updatedAt + } + }, + meta: { + source, + orderSeq: record.orderSeq, + role: record.role, + status: record.status + }, + createdAt: record.createdAt, + idempotent: true + }) + + return 1 + appendToolFactsToTape(table, record, source) +} + +export function appendMessageReplacementToTape( + table: TapeFactStore, + record: ChatMessageRecord, + reason: string +): number { + table.ensureBootstrapAnchor(record.sessionId) + table.append({ + sessionId: record.sessionId, + kind: 'message', + name: `message/${record.role}`, + source: { + type: 'message', + id: record.id, + seq: record.updatedAt + }, + provenanceKey: `message:${record.id}:revision:${record.updatedAt}`, + payload: { + record: { + id: record.id, + sessionId: record.sessionId, + orderSeq: record.orderSeq, + role: record.role, + content: record.content, + status: record.status, + isContextEdge: record.isContextEdge, + metadata: record.metadata, + traceCount: record.traceCount, + createdAt: record.createdAt, + updatedAt: record.updatedAt + } + }, + meta: { + source: 'live', + correction: true, + reason, + orderSeq: record.orderSeq, + role: record.role, + status: record.status + }, + createdAt: record.updatedAt, + idempotent: true + }) + + return 1 + appendToolFactsToTape(table, record, 'repair') +} + +export function appendMessageRetractionToTape( + table: TapeFactStore, + record: ChatMessageRecord, + reason: string +): number { + table.ensureBootstrapAnchor(record.sessionId) + table.appendEvent({ + sessionId: record.sessionId, + name: 'message/retracted', + source: { + type: 'message', + id: record.id, + seq: Date.now() + }, + provenanceKey: null, + data: { + messageId: record.id, + orderSeq: record.orderSeq, + role: record.role, + reason + }, + meta: { + source: 'live', + correction: true + }, + idempotent: false + }) + + return 1 +} + +export function tapeEntriesToEffectiveMessageRecords( + rows: DeepChatTapeEntryRow[] +): ChatMessageRecord[] { + return buildEffectiveTapeView(rows, { includePending: true }).messageRecords +} diff --git a/src/main/tape/application/factService.ts b/src/main/tape/application/factService.ts new file mode 100644 index 0000000000..36684e5a76 --- /dev/null +++ b/src/main/tape/application/factService.ts @@ -0,0 +1,166 @@ +import type { AgentTapeHandoffState, ChatMessageRecord } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryRow, TapeAnchorAppendInput } from '../domain/entry' +import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' +import { buildEffectiveTapeView } from '../domain/effectiveView' +import type { + TapeAnchorWriter, + TapeMessageFactWriter, + TapeToolFactWriter +} from '../ports/capabilities' +import type { TapeApplicationProviders } from '../ports/application' +import { + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + appendTapeToolFact +} from './factPersistence' +import { parseJsonObject } from './common' +import type { TapeAnchorResult } from './contracts' + +type TapeFactProviders = Pick + +function normalizeHandoffName(name: string): string { + const trimmed = name.trim() + if (!trimmed) return 'handoff/manual' + if (trimmed.startsWith('handoff/') || trimmed.startsWith('auto_handoff/')) return trimmed + return `handoff/${trimmed}` +} + +function normalizePositiveInteger(value: unknown): number | null { + if (typeof value !== 'number' || !Number.isFinite(value)) return null + return Math.max(1, Math.floor(value)) +} + +function buildOrderSeqRange(records: ChatMessageRecord[]): Record | null { + if (records.length === 0) return null + return { + fromOrderSeq: records[0].orderSeq, + toOrderSeq: records[records.length - 1].orderSeq + } +} + +function enrichHandoffState( + state: Record, + historyRecords: ChatMessageRecord[] +): Record { + const maxOrderSeq = historyRecords.reduce( + (currentMax, record) => Math.max(currentMax, record.orderSeq), + 0 + ) + const cursorOrderSeq = + normalizePositiveInteger(state.cursorOrderSeq ?? state.summaryCursorOrderSeq) ?? maxOrderSeq + 1 + const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) + const enrichedState: Record = { ...state, cursorOrderSeq } + + if (!Object.prototype.hasOwnProperty.call(enrichedState, 'range')) { + enrichedState.range = buildOrderSeqRange(sourceRecords) + } + + const sourceMessageIds = enrichedState.sourceMessageIds + if (!Array.isArray(sourceMessageIds) || sourceMessageIds.some((id) => typeof id !== 'string')) { + enrichedState.sourceMessageIds = sourceRecords.map((record) => record.id) + } + + return enrichedState +} + +export function normalizeTapeHandoffState(state: unknown): AgentTapeHandoffState { + if (!state || typeof state !== 'object' || Array.isArray(state)) { + throw new Error('Tape handoff requires a non-empty summary.') + } + + const summary = (state as Record).summary + if (typeof summary !== 'string' || !summary.trim()) { + throw new Error('Tape handoff requires a non-empty summary.') + } + + return { + ...(state as Record), + summary: summary.trim() + } +} + +export class TapeFactService + implements TapeToolFactWriter, TapeMessageFactWriter, TapeAnchorWriter +{ + constructor(private readonly providers: TapeFactProviders) {} + + private get table() { + return this.providers.getEntryStore() + } + + appendMessageRecord(record: ChatMessageRecord): number { + return appendMessageRecordToTape(this.table, record, 'live') + } + + appendMessageRecordForSession(sessionId: string, record: ChatMessageRecord): number { + return appendMessageRecordToTape(this.table, { ...record, sessionId }, 'live') + } + + appendMessageReplacement(record: ChatMessageRecord, reason: string): number { + return appendMessageReplacementToTape(this.table, record, reason) + } + + appendMessageRetraction(record: ChatMessageRecord, reason: string): number { + return appendMessageRetractionToTape(this.table, record, reason) + } + + async appendToolFact(input: TapeToolFactInput): Promise { + const row = appendTapeToolFact(this.table, input, 'live', 'tool_loop') + if (!row) throw new Error('Tape tool fact was not appendable.') + return { sessionId: input.sessionId, entryId: row.entry_id } + } + + getMessageRecords(sessionId: string): ChatMessageRecord[] { + return buildEffectiveTapeView(this.table.getBySession(sessionId), { includePending: true }) + .messageRecords + } + + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow { + return this.table.appendAnchor(input) + } + + handoff( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): DeepChatTapeEntryRow { + const normalizedState = normalizeTapeHandoffState(state) + const table = this.table + table.ensureBootstrapAnchor(sessionId) + const handoffState = enrichHandoffState(normalizedState, this.getMessageRecords(sessionId)) + return table.appendAnchor({ + sessionId, + name: normalizeHandoffName(name), + source: { + type: 'runtime_event', + id: `handoff:${Date.now()}`, + seq: 0 + }, + state: handoffState, + meta: { + ...meta, + handoff: true + } + }) + } + + handoffResult( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): TapeAnchorResult { + const row = this.handoff(sessionId, name, state, meta) + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + payload: parseJsonObject(row.payload_json), + meta: parseJsonObject(row.meta_json), + createdAt: row.created_at + } + } +} diff --git a/src/main/tape/application/forkService.ts b/src/main/tape/application/forkService.ts new file mode 100644 index 0000000000..3dd69371a5 --- /dev/null +++ b/src/main/tape/application/forkService.ts @@ -0,0 +1,335 @@ +import { nanoid } from 'nanoid' +import logger from 'electron-log' +import type { DeepChatTapeEntryRow } from '../domain/entry' +import type { TapeApplicationProviders } from '../ports/application' +import { deleteTapeGeneration } from './generationLifecycle' +import { parseJsonObject } from './common' +import type { TapeForkHandle } from './contracts' + +type TapeForkProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getEntryLifecycleStore' | 'getSearchProjectionStore' +> + +function readForkMergeReceiptCount( + row: DeepChatTapeEntryRow, + parentSessionId: string, + forkId: string, + forkSessionIdValue: string +): number { + const payload = parseJsonObject(row.payload_json) + const data = + payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) + ? (payload.data as Record) + : {} + const mergedCount = data.mergedCount + const forkHeadEntryId = data.forkHeadEntryId + const hasValidLegacyOrCurrentHead = + forkHeadEntryId === undefined || + (typeof forkHeadEntryId === 'number' && + Number.isSafeInteger(forkHeadEntryId) && + forkHeadEntryId >= 0) + if ( + row.session_id !== parentSessionId || + row.kind !== 'event' || + row.name !== 'fork/merge' || + row.source_type !== 'fork' || + row.source_id !== forkId || + row.source_seq !== 0 || + row.provenance_key !== `fork:${parentSessionId}:${forkId}:merge:event` || + data.forkId !== forkId || + data.forkSessionId !== forkSessionIdValue || + typeof mergedCount !== 'number' || + !Number.isSafeInteger(mergedCount) || + mergedCount < 0 || + !hasValidLegacyOrCurrentHead || + (typeof forkHeadEntryId === 'number' && mergedCount > forkHeadEntryId) + ) { + throw new Error(`Stored fork merge receipt is malformed: ${row.entry_id}`) + } + return mergedCount +} + +function assertValidForkStart( + row: DeepChatTapeEntryRow | undefined, + parentSessionId: string, + forkId: string, + forkSessionIdValue: string +): void { + if (!row) { + throw new Error(`Fork ${forkId} does not exist or has been discarded.`) + } + const payload = parseJsonObject(row.payload_json) + const state = + payload.state && typeof payload.state === 'object' && !Array.isArray(payload.state) + ? (payload.state as Record) + : {} + const parentHeadEntryId = state.parentHeadEntryId + const hasValidLegacyOrCurrentHead = + parentHeadEntryId === undefined || + (typeof parentHeadEntryId === 'number' && + Number.isSafeInteger(parentHeadEntryId) && + parentHeadEntryId >= 0) + if ( + row.session_id !== forkSessionIdValue || + row.kind !== 'anchor' || + row.name !== 'fork/start' || + row.source_type !== 'fork' || + row.source_id !== forkId || + row.source_seq !== 0 || + row.provenance_key !== `fork:${parentSessionId}:${forkId}:start` || + state.parentSessionId !== parentSessionId || + !hasValidLegacyOrCurrentHead + ) { + throw new Error(`Stored fork start is malformed: ${row.entry_id}`) + } +} + +function forkSessionId(parentSessionId: string, forkId: string): string { + return `${parentSessionId}::fork::${forkId}` +} + +function forkDiscardProvenanceKey(parentSessionId: string, forkId: string): string { + return `fork:${parentSessionId}:${forkId}:discard:event` +} + +function forkMergeProvenanceKey(parentSessionId: string, forkId: string): string { + return `fork:${parentSessionId}:${forkId}:merge:event` +} + +export class TapeForkService { + constructor(private readonly providers: TapeForkProviders) {} + + private get table() { + return this.providers.getEntryStore() + } + + createFork(parentSessionId: string, forkId: string = nanoid()): TapeForkHandle { + const table = this.table + const forkIdValue = forkId.trim() || nanoid() + if ( + table.getByProvenanceKey( + parentSessionId, + forkMergeProvenanceKey(parentSessionId, forkIdValue) + ) + ) { + throw new Error(`Fork ${forkIdValue} has already been merged and cannot be reused.`) + } + if ( + table.getByProvenanceKey( + parentSessionId, + forkDiscardProvenanceKey(parentSessionId, forkIdValue) + ) + ) { + throw new Error(`Fork ${forkIdValue} has been discarded and cannot be reused.`) + } + const forkSessionIdValue = forkSessionId(parentSessionId, forkIdValue) + const parentHeadEntryId = table.getMaxEntryId(parentSessionId) + table.ensureBootstrapAnchor(forkSessionIdValue) + const parentAnchor = table.getLatestAnchor(parentSessionId) + const forkStart = table.appendAnchor({ + sessionId: forkSessionIdValue, + name: 'fork/start', + source: { + type: 'fork', + id: forkIdValue, + seq: 0 + }, + provenanceKey: `fork:${parentSessionId}:${forkIdValue}:start`, + state: { + parentSessionId, + parentHeadEntryId, + parentLastAnchorEntryId: parentAnchor?.entry_id ?? null, + parentLastAnchorName: parentAnchor?.name ?? null + }, + idempotent: true + }) + const forkStartPayload = parseJsonObject(forkStart.payload_json) + const persistedState = + forkStartPayload.state && + typeof forkStartPayload.state === 'object' && + !Array.isArray(forkStartPayload.state) + ? (forkStartPayload.state as Record) + : {} + const persistedParentHeadEntryId = persistedState.parentHeadEntryId + return { + parentSessionId, + forkId: forkIdValue, + forkSessionId: forkSessionIdValue, + parentHeadEntryId: + typeof persistedParentHeadEntryId === 'number' && + Number.isSafeInteger(persistedParentHeadEntryId) && + persistedParentHeadEntryId >= 0 + ? persistedParentHeadEntryId + : parentHeadEntryId + } + } + + mergeFork(parentSessionId: string, forkId: string): number { + const table = this.table + const forkSessionIdValue = forkSessionId(parentSessionId, forkId) + const mergeProvenanceKey = forkMergeProvenanceKey(parentSessionId, forkId) + + return table.runInTransaction(() => { + const existingReceipt = table.getByProvenanceKey(parentSessionId, mergeProvenanceKey) + if (existingReceipt) { + return readForkMergeReceiptCount( + existingReceipt, + parentSessionId, + forkId, + forkSessionIdValue + ) + } + + if ( + table.getByProvenanceKey(parentSessionId, forkDiscardProvenanceKey(parentSessionId, forkId)) + ) { + throw new Error(`Fork ${forkId} does not exist or has been discarded.`) + } + + assertValidForkStart( + table.getByProvenanceKey(forkSessionIdValue, `fork:${parentSessionId}:${forkId}:start`), + parentSessionId, + forkId, + forkSessionIdValue + ) + + const forkHeadEntryId = table.getMaxEntryId(forkSessionIdValue) + const forkEntries = table + .getBySessionUpToEntryId(forkSessionIdValue, forkHeadEntryId) + .filter( + (entry) => + !( + entry.kind === 'anchor' && + (entry.name === 'session/start' || entry.name === 'fork/start') + ) + ) + + for (const entry of forkEntries) { + table.append({ + sessionId: parentSessionId, + kind: entry.kind, + name: entry.name, + source: { + type: 'fork', + id: forkId, + seq: entry.entry_id + }, + provenanceKey: `fork:${parentSessionId}:${forkId}:merge:${entry.entry_id}`, + payload: parseJsonObject(entry.payload_json), + meta: { + ...parseJsonObject(entry.meta_json), + forkId, + forkSessionId: forkSessionIdValue, + mergedFromEntryId: entry.entry_id + }, + createdAt: entry.created_at, + idempotent: true + }) + } + + table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/merge', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: mergeProvenanceKey, + data: { + forkId, + forkSessionId: forkSessionIdValue, + forkHeadEntryId, + mergedCount: forkEntries.length + }, + idempotent: true + }) + + return forkEntries.length + }) + } + + discardFork(parentSessionId: string, forkId: string): void { + const table = this.table + const forkSessionIdValue = forkSessionId(parentSessionId, forkId) + let cleanupError: unknown + table.runInTransaction(() => { + try { + deleteTapeGeneration(this.providers, forkSessionIdValue) + } catch (error) { + cleanupError = error + } + table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/discard', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: forkDiscardProvenanceKey(parentSessionId, forkId), + data: { + forkId, + forkSessionId: forkSessionIdValue + }, + idempotent: true + }) + }) + if (cleanupError) { + logger.warn(`[Tape] failed to delete fork Tape generation: ${String(cleanupError)}`) + } + } + + recordExternalForkMerge( + parentSessionId: string, + forkSessionIdValue: string, + forkId: string, + meta: Record = {} + ): DeepChatTapeEntryRow { + const table = this.table + const referencedEntryCount = table.countBySession(forkSessionIdValue) + return table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/merge', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: `fork:${parentSessionId}:${forkId}:external-merge:event`, + data: { + ...meta, + forkId, + forkSessionId: forkSessionIdValue, + referencedEntryCount + }, + idempotent: true + }) + } + + recordExternalForkDiscard( + parentSessionId: string, + forkSessionIdValue: string, + forkId: string, + meta: Record = {} + ): DeepChatTapeEntryRow { + const table = this.table + return table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/discard', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: `fork:${parentSessionId}:${forkId}:external-discard:event`, + data: { + ...meta, + forkId, + forkSessionId: forkSessionIdValue + }, + idempotent: true + }) + } +} diff --git a/src/main/tape/application/generationLifecycle.ts b/src/main/tape/application/generationLifecycle.ts new file mode 100644 index 0000000000..54f4dac3f0 --- /dev/null +++ b/src/main/tape/application/generationLifecycle.ts @@ -0,0 +1,29 @@ +import type { TapeApplicationProviders } from '../ports/application' + +type TapeGenerationLifecycleProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getEntryLifecycleStore' | 'getSearchProjectionStore' +> + +export function deleteTapeGeneration( + providers: TapeGenerationLifecycleProviders, + sessionId: string +): void { + const table = providers.getEntryStore() + table.runInTransaction(() => { + providers.getEntryLifecycleStore().deleteBySession(sessionId) + providers.getSearchProjectionStore().deleteBySession(sessionId) + }) +} + +export function resetTapeGeneration( + providers: TapeGenerationLifecycleProviders, + sessionId: string +): void { + const table = providers.getEntryStore() + table.runInTransaction(() => { + providers.getEntryLifecycleStore().deleteBySession(sessionId) + providers.getSearchProjectionStore().deleteBySession(sessionId) + table.ensureBootstrapAnchor(sessionId) + }) +} diff --git a/src/main/tape/application/lineageService.ts b/src/main/tape/application/lineageService.ts new file mode 100644 index 0000000000..42aba422c7 --- /dev/null +++ b/src/main/tape/application/lineageService.ts @@ -0,0 +1,458 @@ +import { createHash } from 'crypto' +import type { + SubagentTapeLinkInput, + SubagentTapeLinkOutcome, + SubagentTapeLinkReceipt +} from '@shared/types/agent-interface' +import { + TAPE_INCARNATION_META_KEY, + type DeepChatTapeEntryRow, + type DeepChatTapeReadSource +} from '../domain/entry' +import type { TapeApplicationProviders } from '../ports/application' +import { parseJsonObject, parseJsonValue } from './common' + +type TapeLineageProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getLineageSessionReader' +> + +function compactText(value: string, maxLength = 1000): string { + const normalized = value.replace(/\s+/g, ' ').trim() + if (normalized.length <= maxLength) return normalized + return `${normalized.slice(0, Math.max(0, maxLength - 3)).trimEnd()}...` +} + +const SUBAGENT_TAPE_LINK_EVENT_NAME = 'subagent/tape_linked' + +const SUBAGENT_TAPE_LINK_VERSION = 2 + +const TAPE_IDENTITY_PATTERN = /^[a-f0-9]{64}$/ + +const SUBAGENT_TAPE_LINK_OUTCOMES = new Set([ + 'completed', + 'error', + 'cancelled' +]) + +type SubagentTapeLinkSnapshot = { + linkEntryId: number + childSessionId: string + childHeadEntryId: number + childEntryCount: number + outcome: SubagentTapeLinkOutcome + childTapeIdentity: string | null +} + +type ParsedSubagentTapeLink = { + snapshot: SubagentTapeLinkSnapshot + frozenInput: SubagentTapeLinkInput +} + +export type LinkedTapeSourceResolution = { + sources: DeepChatTapeReadSource[] + unavailableSourceIds: Set +} + +export type AgentTapeViewErrorCode = + | 'current_tape_unavailable' + | 'linked_tape_unavailable' + | 'linked_tape_unauthorized' + +export class AgentTapeViewError extends Error { + readonly name = 'AgentTapeViewError' + + constructor( + readonly code: AgentTapeViewErrorCode, + readonly parentSessionId: string, + readonly sourceSessionId: string, + message: string + ) { + super(message) + } +} + +export function normalizeSubagentTapeLinkInput( + input: SubagentTapeLinkInput +): SubagentTapeLinkInput { + const normalized = { + parentSessionId: input.parentSessionId.trim(), + childSessionId: input.childSessionId.trim(), + runId: input.runId.trim(), + taskId: input.taskId.trim(), + slotId: input.slotId.trim(), + taskTitle: compactText(input.taskTitle, 500), + outcome: input.outcome, + resultSummary: input.resultSummary?.trim() ? compactText(input.resultSummary, 2000) : null + } + for (const [name, value] of Object.entries(normalized)) { + if (name === 'resultSummary' || name === 'outcome') continue + if (typeof value !== 'string' || !value) { + throw new Error(`Subagent Tape link ${name} is required.`) + } + } + if (normalized.parentSessionId === normalized.childSessionId) { + throw new Error('Subagent Tape link child must differ from its parent.') + } + if (!SUBAGENT_TAPE_LINK_OUTCOMES.has(normalized.outcome)) { + throw new Error(`Invalid subagent Tape link outcome: ${String(normalized.outcome)}`) + } + return normalized +} + +function isUnmarkedLegacyTape(row: DeepChatTapeEntryRow): boolean { + const meta = parseJsonValue(row.meta_json) + return ( + meta !== null && + typeof meta === 'object' && + !Array.isArray(meta) && + !Object.prototype.hasOwnProperty.call(meta, TAPE_INCARNATION_META_KEY) + ) +} + +function computeTapeIdentity(row: DeepChatTapeEntryRow): string { + return createHash('sha256') + .update( + JSON.stringify([ + row.session_id, + row.entry_id, + row.kind, + row.name, + row.source_type, + row.source_id, + row.source_seq, + row.provenance_key, + row.payload_json, + row.meta_json, + row.created_at + ]) + ) + .digest('hex') +} + +function subagentTapeLinkProvenanceKey(input: SubagentTapeLinkInput): string { + // This version belongs to the stable task-identity key, independently of the evolving event + // payload's linkVersion. + const identityHash = createHash('sha256') + .update( + JSON.stringify([input.parentSessionId, input.childSessionId, input.runId, input.taskId]) + ) + .digest('hex') + return `subagent:tape-link:v1:${identityHash}` +} + +function parseSubagentTapeLink(row: DeepChatTapeEntryRow): ParsedSubagentTapeLink | null { + const payload = parseJsonObject(row.payload_json) + const data = + payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) + ? (payload.data as Record) + : {} + const childSessionId = data.childSessionId + const childHeadEntryId = data.childHeadEntryId + const childEntryCount = data.childEntryCount + const outcome = data.outcome + const runId = data.runId + const taskId = data.taskId + const slotId = data.slotId + const taskTitle = data.taskTitle + const resultSummary = data.resultSummary + const linkVersion = data.linkVersion + const childTapeIdentity = data.childTapeIdentity + const hasValidLinkVersion = + (linkVersion === 1 && childTapeIdentity === undefined) || + (linkVersion === SUBAGENT_TAPE_LINK_VERSION && + typeof childTapeIdentity === 'string' && + TAPE_IDENTITY_PATTERN.test(childTapeIdentity)) + if ( + row.kind !== 'event' || + row.name !== SUBAGENT_TAPE_LINK_EVENT_NAME || + typeof childSessionId !== 'string' || + !childSessionId || + typeof childHeadEntryId !== 'number' || + !Number.isSafeInteger(childHeadEntryId) || + childHeadEntryId < 0 || + typeof childEntryCount !== 'number' || + !Number.isSafeInteger(childEntryCount) || + childEntryCount < 0 || + typeof outcome !== 'string' || + !SUBAGENT_TAPE_LINK_OUTCOMES.has(outcome as SubagentTapeLinkOutcome) || + typeof runId !== 'string' || + typeof taskId !== 'string' || + typeof slotId !== 'string' || + typeof taskTitle !== 'string' || + (resultSummary !== null && typeof resultSummary !== 'string') || + !hasValidLinkVersion || + row.source_type !== 'subagent' || + row.source_id !== childSessionId || + row.source_seq !== childHeadEntryId || + childEntryCount > childHeadEntryId + ) { + return null + } + + let normalizedInput: SubagentTapeLinkInput + try { + normalizedInput = normalizeSubagentTapeLinkInput({ + parentSessionId: row.session_id, + childSessionId, + runId, + taskId, + slotId, + taskTitle, + outcome: outcome as SubagentTapeLinkOutcome, + resultSummary + }) + } catch { + return null + } + if ( + normalizedInput.parentSessionId !== row.session_id || + normalizedInput.childSessionId !== childSessionId || + normalizedInput.runId !== runId || + normalizedInput.taskId !== taskId || + normalizedInput.slotId !== slotId || + normalizedInput.taskTitle !== taskTitle || + normalizedInput.resultSummary !== resultSummary || + row.provenance_key !== subagentTapeLinkProvenanceKey(normalizedInput) + ) { + return null + } + + return { + snapshot: { + linkEntryId: row.entry_id, + childSessionId, + childHeadEntryId, + childEntryCount, + outcome: outcome as SubagentTapeLinkOutcome, + childTapeIdentity: + linkVersion === SUBAGENT_TAPE_LINK_VERSION ? (childTapeIdentity as string) : null + }, + frozenInput: normalizedInput + } +} + +function parseSubagentTapeLinkSnapshot(row: DeepChatTapeEntryRow): SubagentTapeLinkSnapshot | null { + return parseSubagentTapeLink(row)?.snapshot ?? null +} + +function parseLegacyExternalTapeLinkSnapshot( + row: DeepChatTapeEntryRow +): SubagentTapeLinkSnapshot | null { + if (row.kind !== 'event' || row.name !== 'fork/merge' || row.source_type !== 'fork') { + return null + } + const payload = parseJsonObject(row.payload_json) + const data = + payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) + ? (payload.data as Record) + : {} + const childSessionId = data.forkSessionId + const forkId = data.forkId + const referencedEntryCount = data.referencedEntryCount + if ( + typeof childSessionId !== 'string' || + !childSessionId || + forkId !== childSessionId || + row.source_id !== childSessionId || + row.source_seq !== 0 || + row.provenance_key !== `fork:${row.session_id}:${childSessionId}:external-merge:event` || + typeof referencedEntryCount !== 'number' || + !Number.isSafeInteger(referencedEntryCount) || + referencedEntryCount <= 0 + ) { + return null + } + return { + linkEntryId: row.entry_id, + childSessionId, + childHeadEntryId: referencedEntryCount, + childEntryCount: referencedEntryCount, + outcome: 'completed', + childTapeIdentity: null + } +} + +function toSubagentTapeLinkReceipt(row: DeepChatTapeEntryRow): SubagentTapeLinkReceipt { + const snapshot = parseSubagentTapeLinkSnapshot(row) + if (!snapshot) { + throw new Error(`Stored subagent Tape link receipt is malformed: ${row.entry_id}`) + } + return { + linkEntry: { + sessionId: row.session_id, + entryId: snapshot.linkEntryId + }, + childSessionId: snapshot.childSessionId, + childHeadEntryId: snapshot.childHeadEntryId, + childEntryCount: snapshot.childEntryCount, + outcome: snapshot.outcome + } +} + +function assertSubagentTapeLinkMatchesInput( + row: DeepChatTapeEntryRow, + input: SubagentTapeLinkInput +): void { + const parsed = parseSubagentTapeLink(row) + if (!parsed) { + throw new Error(`Stored subagent Tape link receipt is malformed: ${row.entry_id}`) + } + const storedInput = parsed.frozenInput + const storedKeys = Object.keys(storedInput) as Array + if ( + storedKeys.length !== Object.keys(input).length || + storedKeys.some((key) => storedInput[key] !== input[key]) + ) { + throw new Error( + `Subagent Tape link conflicts with finalized task ${input.runId}/${input.taskId}.` + ) + } +} + +export class TapeLineageService { + constructor(private readonly providers: TapeLineageProviders) {} + + private get table() { + return this.providers.getEntryStore() + } + + resolveLinkedTapeSources(parentSessionId: string): LinkedTapeSourceResolution { + const table = this.table + const sessionTable = this.providers.getLineageSessionReader() + if (!table || !sessionTable?.get(parentSessionId)) { + throw new AgentTapeViewError( + 'current_tape_unavailable', + parentSessionId, + parentSessionId, + `Current Tape ${parentSessionId} is unavailable.` + ) + } + + const parsedSnapshots = table + .getSubagentLineageEvents(parentSessionId) + .map((row) => parseSubagentTapeLinkSnapshot(row) ?? parseLegacyExternalTapeLinkSnapshot(row)) + .filter((snapshot): snapshot is SubagentTapeLinkSnapshot => snapshot !== null) + const latestSnapshotByChild = new Map() + for (const snapshot of parsedSnapshots) { + const current = latestSnapshotByChild.get(snapshot.childSessionId) + if (!current || snapshot.linkEntryId > current.linkEntryId) { + latestSnapshotByChild.set(snapshot.childSessionId, snapshot) + } + } + const snapshots = [...latestSnapshotByChild.values()] + const childSessionIds = [...new Set(snapshots.map((snapshot) => snapshot.childSessionId))] + const childById = new Map(sessionTable.getMany(childSessionIds).map((row) => [row.id, row])) + const unavailableSourceIds = new Set() + const snapshotBySource = new Map() + + for (const snapshot of snapshots) { + const child = childById.get(snapshot.childSessionId) + if (!child) { + unavailableSourceIds.add(snapshot.childSessionId) + continue + } + if (child.session_kind !== 'subagent' || child.parent_session_id !== parentSessionId) { + continue + } + snapshotBySource.set(snapshot.childSessionId, snapshot) + } + + const authorizedSourceIds = [...snapshotBySource.keys()] + const firstEntryBySource = new Map( + table.getFirstEntriesBySessions(authorizedSourceIds).map((row) => [row.session_id, row]) + ) + const liveHeads = table.getMaxEntryIdsBySessions(authorizedSourceIds) + const availableSources: DeepChatTapeReadSource[] = [] + for (const [sourceSessionId, snapshot] of snapshotBySource) { + const firstEntry = firstEntryBySource.get(sourceSessionId) + const identityMatches = snapshot.childTapeIdentity + ? firstEntry !== undefined && computeTapeIdentity(firstEntry) === snapshot.childTapeIdentity + : firstEntry !== undefined && isUnmarkedLegacyTape(firstEntry) + if (!identityMatches || (liveHeads.get(sourceSessionId) ?? 0) < snapshot.childHeadEntryId) { + unavailableSourceIds.add(sourceSessionId) + continue + } + availableSources.push({ + sessionId: sourceSessionId, + maxEntryId: snapshot.childHeadEntryId + }) + } + + return { + sources: availableSources.sort((left, right) => + left.sessionId < right.sessionId ? -1 : left.sessionId > right.sessionId ? 1 : 0 + ), + unavailableSourceIds + } + } + + linkSubagentTape(input: SubagentTapeLinkInput): SubagentTapeLinkReceipt { + const table = this.table + const sessionTable = this.providers.getLineageSessionReader() + const normalized = normalizeSubagentTapeLinkInput(input) + const provenanceKey = subagentTapeLinkProvenanceKey(normalized) + return table.runInTransaction(() => { + const existing = table.getByProvenanceKey(normalized.parentSessionId, provenanceKey) + if (existing) { + assertSubagentTapeLinkMatchesInput(existing, normalized) + const receipt = toSubagentTapeLinkReceipt(existing) + return receipt + } + + const sessionById = new Map( + sessionTable + .getMany([normalized.parentSessionId, normalized.childSessionId]) + .map((session) => [session.id, session]) + ) + const parent = sessionById.get(normalized.parentSessionId) + const child = sessionById.get(normalized.childSessionId) + if ( + !parent || + !child || + child.session_kind !== 'subagent' || + child.parent_session_id !== normalized.parentSessionId + ) { + throw new Error( + `Session ${normalized.childSessionId} is not a direct subagent child of ` + + `${normalized.parentSessionId}.` + ) + } + + const childFirstEntry = table.getFirstEntriesBySessions([normalized.childSessionId])[0] + if (!childFirstEntry || childFirstEntry.session_id !== normalized.childSessionId) { + throw new Error(`Subagent Tape ${normalized.childSessionId} is unavailable.`) + } + const childTapeIdentity = computeTapeIdentity(childFirstEntry) + const childHeadEntryId = table.getMaxEntryId(normalized.childSessionId) + const childEntryCount = table.countBySession(normalized.childSessionId) + const row = table.appendEvent({ + sessionId: normalized.parentSessionId, + name: SUBAGENT_TAPE_LINK_EVENT_NAME, + source: { + type: 'subagent', + id: normalized.childSessionId, + seq: childHeadEntryId + }, + provenanceKey, + data: { + linkVersion: SUBAGENT_TAPE_LINK_VERSION, + childSessionId: normalized.childSessionId, + childHeadEntryId, + childEntryCount, + childTapeIdentity, + runId: normalized.runId, + taskId: normalized.taskId, + slotId: normalized.slotId, + taskTitle: normalized.taskTitle, + outcome: normalized.outcome, + resultSummary: normalized.resultSummary + }, + idempotent: true + }) + assertSubagentTapeLinkMatchesInput(row, normalized) + const receipt = toSubagentTapeLinkReceipt(row) + return receipt + }) + } +} diff --git a/src/main/tape/application/recallProjection.ts b/src/main/tape/application/recallProjection.ts new file mode 100644 index 0000000000..e5336b5519 --- /dev/null +++ b/src/main/tape/application/recallProjection.ts @@ -0,0 +1,466 @@ +import type { AgentTapeSearchOptions, AgentTapeViewScope } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryRow, DeepChatTapeSearchInput } from '../domain/entry' +import { parseJsonObject, parseJsonValue } from './common' +import type { TapeSearchResult } from './contracts' + +function isRecordObject(value: unknown): value is Record { + return Boolean(value && typeof value === 'object' && !Array.isArray(value)) +} + +function compactText(value: string, maxLength = 1000): string { + const normalized = value.replace(/\s+/g, ' ').trim() + if (normalized.length <= maxLength) return normalized + return `${normalized.slice(0, Math.max(0, maxLength - 3)).trimEnd()}...` +} + +export function stringifyForSummary(value: unknown, maxLength = 1000): string { + if (typeof value === 'string') return compactText(value, maxLength) + if (value === null || value === undefined) return '' + try { + return compactText(JSON.stringify(value), maxLength) + } catch { + return compactText(String(value), maxLength) + } +} + +export function truncateToUtf8Bytes( + text: string, + maxBytes: number +): { text: string; truncated: boolean } { + const normalized = text.trim() + if (maxBytes <= 0) { + return { text: '', truncated: normalized.length > 0 } + } + if (maxBytes < 3) { + return { text: '', truncated: normalized.length > 0 } + } + if (Buffer.byteLength(normalized, 'utf8') <= maxBytes) { + return { text: normalized, truncated: false } + } + let bytes = 0 + let output = '' + for (const character of normalized) { + const nextBytes = Buffer.byteLength(character, 'utf8') + if (bytes + nextBytes > Math.max(0, maxBytes - 3)) break + output += character + bytes += nextBytes + } + return { text: `${output.trimEnd()}...`, truncated: true } +} + +export function normalizeContextByteLimit( + value: number | undefined, + fallback: number, + max: number +): number { + if (!Number.isFinite(value)) return fallback + return Math.min(Math.max(Math.floor(value as number), 0), max) +} + +function uniqueStrings(values: string[], limit = 10): string[] { + const seen = new Set() + const result: string[] = [] + for (const value of values) { + const normalized = value.trim() + if (!normalized || seen.has(normalized)) continue + seen.add(normalized) + result.push(normalized) + if (result.length >= limit) break + } + return result +} + +function extractFilePaths(text: string): string[] { + const matches = [ + ...text.matchAll( + /(?:^|[\s"'`([{<])((?:[A-Za-z]:\\|\/|\.{1,2}\/|[\w.-]+\/)[^\s"'`<>{}(),;!?]+(?:[/\\][^\s"'`<>{}(),;!?]+)*)/g + ) + ].map((match) => match[1].replace(/[.:]+$/g, '')) + return uniqueStrings(matches ?? []) +} + +function extractErrorCodes(text: string): string[] { + const matches = text.match(/\b(?:E[A-Z0-9_]{3,}|[A-Z][A-Z0-9_]*Error)\b/g) + return uniqueStrings(matches ?? []) +} + +function collectKeyedStrings( + value: unknown, + keys: Set, + output: string[] = [], + depth = 0 +): string[] { + if (depth > 4 || output.length >= 10 || !value || typeof value !== 'object') return output + if (Array.isArray(value)) { + for (const item of value) collectKeyedStrings(item, keys, output, depth + 1) + return output + } + for (const [key, nested] of Object.entries(value as Record)) { + if (keys.has(key) && typeof nested === 'string' && nested.trim()) { + output.push(compactText(nested, 500)) + if (output.length >= 10) return output + } + collectKeyedStrings(nested, keys, output, depth + 1) + if (output.length >= 10) return output + } + return output +} + +function collectUserMessageAttachmentRefs(files: unknown): { + searchText: string[] + filePaths: string[] + fileNames: string[] +} { + const searchText: string[] = [] + const filePaths: string[] = [] + const fileNames: string[] = [] + if (!Array.isArray(files)) { + return { searchText, filePaths, fileNames } + } + for (const file of files) { + if (!isRecordObject(file)) continue + const path = typeof file.path === 'string' ? file.path : '' + const name = typeof file.name === 'string' ? file.name : '' + const metadata = isRecordObject(file.metadata) ? file.metadata : null + const metadataFileName = + metadata && typeof metadata.fileName === 'string' ? metadata.fileName : '' + if (path) { + filePaths.push(compactText(path, 500)) + searchText.push(compactText(path, 500)) + } + for (const value of [name, metadataFileName]) { + if (!value) continue + fileNames.push(compactText(value, 500)) + searchText.push(compactText(value, 500)) + } + } + return { + searchText: uniqueStrings(searchText, 20), + filePaths: uniqueStrings(filePaths, 20), + fileNames: uniqueStrings(fileNames, 20) + } +} + +type UserMessageProjectionText = { + text: string + attachmentRefs: { + searchText: string[] + filePaths: string[] + fileNames: string[] + } +} + +function emptyUserMessageAttachmentRefs(): UserMessageProjectionText['attachmentRefs'] { + return { searchText: [], filePaths: [], fileNames: [] } +} + +function parseUserMessageProjectionText(content: string): UserMessageProjectionText { + const parsed = parseJsonValue(content) + if (isRecordObject(parsed) && typeof parsed.text === 'string') { + const attachmentRefs = collectUserMessageAttachmentRefs(parsed.files) + const parts = [parsed.text] + if (Array.isArray(parsed.files) && parsed.files.length > 0) { + parts.push(`files:${parsed.files.length}`) + parts.push(...attachmentRefs.searchText) + } + if (Array.isArray(parsed.links) && parsed.links.length > 0) { + parts.push(`links:${parsed.links.length}`) + } + return { text: parts.join(' '), attachmentRefs } + } + return { text: content, attachmentRefs: emptyUserMessageAttachmentRefs() } +} + +export function getUserMessageProjectionText( + row: DeepChatTapeEntryRow, + payload: Record +): UserMessageProjectionText | null { + if (row.kind !== 'message' || !isRecordObject(payload.record)) return null + const record = payload.record + const role = typeof record.role === 'string' ? record.role : 'message' + if (role === 'assistant') return null + const content = typeof record.content === 'string' ? record.content : '' + return parseUserMessageProjectionText(content) +} + +function collectUserMessageAttachmentRefsFromPayload(payload: Record): { + searchText: string[] + filePaths: string[] + fileNames: string[] +} { + if (!isRecordObject(payload.record)) { + return { searchText: [], filePaths: [], fileNames: [] } + } + const content = typeof payload.record.content === 'string' ? payload.record.content : '' + const parsed = parseJsonValue(content) + return isRecordObject(parsed) + ? collectUserMessageAttachmentRefs(parsed.files) + : { searchText: [], filePaths: [], fileNames: [] } +} + +function readUserMessageText(content: string, parsed?: UserMessageProjectionText): string { + return parsed?.text ?? parseUserMessageProjectionText(content).text +} + +function readAssistantMessageText(content: string): string { + const parsed = parseJsonValue(content) + if (!Array.isArray(parsed)) return content + const parts: string[] = [] + for (const block of parsed) { + if (!isRecordObject(block)) continue + if (typeof block.content === 'string' && block.content.trim()) { + parts.push(block.content) + continue + } + const toolCall = block.tool_call + if (isRecordObject(toolCall)) { + const name = typeof toolCall.name === 'string' ? toolCall.name : 'unknown' + const params = typeof toolCall.params === 'string' ? toolCall.params : '' + const response = typeof toolCall.response === 'string' ? toolCall.response : '' + parts.push(`tool ${name} ${params} ${response}`.trim()) + } + } + return parts.join(' ') +} + +export function summarizeTapeRow( + row: DeepChatTapeEntryRow, + payload: Record, + userMessage?: UserMessageProjectionText | null +): string { + if (row.kind === 'message') { + const record = payload.record + if (isRecordObject(record)) { + const role = typeof record.role === 'string' ? record.role : 'message' + const content = typeof record.content === 'string' ? record.content : '' + const text = + role === 'assistant' + ? readAssistantMessageText(content) + : readUserMessageText(content, userMessage ?? undefined) + return compactText(`${role}: ${text}`, 1200) + } + } + + if (row.kind === 'tool_call') { + const toolCall = payload.toolCall + if (isRecordObject(toolCall)) { + const name = typeof toolCall.name === 'string' ? toolCall.name : (row.name ?? 'unknown') + const params = typeof toolCall.params === 'string' ? toolCall.params : '' + return compactText(`tool_call ${name}: ${params}`, 1200) + } + } + + if (row.kind === 'tool_result') { + const response = typeof payload.response === 'string' ? payload.response : payload + return compactText( + `tool_result ${row.name ?? 'unknown'}: ${stringifyForSummary(response)}`, + 1200 + ) + } + + if (row.kind === 'anchor') { + const state = isRecordObject(payload.state) ? payload.state : payload + const summary = typeof state.summary === 'string' ? state.summary : stringifyForSummary(state) + return compactText(`anchor ${row.name ?? 'unknown'}: ${summary}`, 1200) + } + + if (row.kind === 'event') { + const data = isRecordObject(payload.data) ? payload.data : payload + return compactText(`event ${row.name ?? 'unknown'}: ${stringifyForSummary(data)}`, 1200) + } + + return compactText(`${row.kind} ${row.name ?? ''}`.trim(), 1200) +} + +export function buildTapeRowEvidenceText( + row: DeepChatTapeEntryRow, + payload: Record, + meta: Record, + userMessage?: UserMessageProjectionText | null +): string { + const parts: string[] = [] + if (row.kind === 'message' && isRecordObject(payload.record)) { + const record = payload.record + const content = typeof record.content === 'string' ? record.content : '' + const role = typeof record.role === 'string' ? record.role : 'message' + parts.push( + role === 'assistant' + ? readAssistantMessageText(content) + : readUserMessageText(content, userMessage ?? undefined) + ) + } else if (row.kind === 'tool_call' && isRecordObject(payload.toolCall)) { + const toolCall = payload.toolCall + parts.push(stringifyForSummary(toolCall.name, 200)) + parts.push(stringifyForSummary(toolCall.params, 3000)) + } else if (row.kind === 'tool_result') { + parts.push(stringifyForSummary(payload.response ?? payload, 4000)) + } else if (row.kind === 'anchor') { + parts.push(stringifyForSummary(isRecordObject(payload.state) ? payload.state : payload, 4000)) + } else if (row.kind === 'event') { + parts.push(stringifyForSummary(isRecordObject(payload.data) ? payload.data : payload, 4000)) + } else { + parts.push(stringifyForSummary(payload, 4000)) + } + if (typeof meta.status === 'string') parts.push(`status:${meta.status}`) + return compactText(parts.filter(Boolean).join('\n'), 5000) +} + +function setRef(target: Record, key: string, value: unknown): void { + if (value !== null && value !== undefined && value !== '') { + target[key] = value + } +} + +function enrichTapeRowRefs( + refs: Record, + payload: Record, + meta: Record, + evidenceText: string, + userMessage?: UserMessageProjectionText | null +): void { + const attachmentRefs = + userMessage?.attachmentRefs ?? collectUserMessageAttachmentRefsFromPayload(payload) + const filePaths = uniqueStrings( + [...extractFilePaths(evidenceText), ...attachmentRefs.filePaths], + 20 + ) + const errorCodes = extractErrorCodes(evidenceText) + const commands = uniqueStrings( + [ + ...collectKeyedStrings(payload, new Set(['command', 'cmd', 'script', 'shellCommand'])), + ...collectKeyedStrings(meta, new Set(['command', 'cmd', 'script', 'shellCommand'])) + ], + 10 + ) + setRef(refs, 'filePaths', filePaths.length ? filePaths : null) + setRef(refs, 'fileNames', attachmentRefs.fileNames.length ? attachmentRefs.fileNames : null) + setRef(refs, 'commands', commands.length ? commands : null) + setRef(refs, 'errorCodes', errorCodes.length ? errorCodes : null) + for (const key of ['exitCode', 'exitStatus', 'code']) { + const value = payload[key] ?? meta[key] + if (typeof value === 'number' || typeof value === 'string') { + setRef(refs, key, value) + } + } +} + +export function buildTapeRowRefs( + row: DeepChatTapeEntryRow, + payload: Record, + meta: Record, + userMessage?: UserMessageProjectionText | null, + evidenceText?: string +): Record { + const refs: Record = {} + setRef(refs, 'sourceType', row.source_type) + setRef(refs, 'sourceId', row.source_id) + setRef(refs, 'sourceSeq', row.source_seq) + setRef(refs, 'status', typeof meta.status === 'string' ? meta.status : null) + + if (row.kind === 'message' && isRecordObject(payload.record)) { + const record = payload.record + setRef(refs, 'messageId', record.id) + setRef(refs, 'orderSeq', record.orderSeq) + setRef(refs, 'role', record.role) + setRef(refs, 'messageStatus', record.status) + } else if (row.kind === 'tool_call' && isRecordObject(payload.toolCall)) { + const toolCall = payload.toolCall + setRef(refs, 'messageId', payload.messageId) + setRef(refs, 'orderSeq', payload.orderSeq) + setRef(refs, 'toolCallId', toolCall.id) + setRef(refs, 'toolName', toolCall.name ?? row.name) + setRef(refs, 'serverName', toolCall.serverName) + } else if (row.kind === 'tool_result') { + setRef(refs, 'messageId', payload.messageId) + setRef(refs, 'orderSeq', payload.orderSeq) + setRef(refs, 'toolCallId', payload.toolCallId) + setRef(refs, 'toolName', row.name) + } else if (row.kind === 'anchor') { + setRef(refs, 'anchorName', row.name) + } else if (row.kind === 'event') { + setRef(refs, 'eventName', row.name) + } + + enrichTapeRowRefs( + refs, + payload, + meta, + evidenceText ?? buildTapeRowEvidenceText(row, payload, meta, userMessage), + userMessage + ) + return refs +} + +export function parseProjectionRefs(raw: string): Record { + const parsed = parseJsonObject(raw) + return parsed +} + +export function normalizeContextWindowValue(value: number | undefined, fallback: number): number { + if (!Number.isFinite(value)) return fallback + return Math.min(Math.max(Math.floor(value as number), 0), 20) +} + +export function normalizeContextLimit(value: number | undefined): number { + if (!Number.isFinite(value)) return 50 + return Math.min(Math.max(Math.floor(value as number), 1), 100) +} + +function parseSearchBoundary(value: string | undefined, name: string): number | undefined { + const trimmed = value?.trim() + if (!trimmed) { + return undefined + } + + const numericValue = Number(trimmed) + if (Number.isFinite(numericValue)) { + return numericValue + } + + const parsedDate = Date.parse(trimmed) + if (Number.isFinite(parsedDate)) { + return parsedDate + } + + throw new Error(`${name} must be an ISO date/time or millisecond timestamp.`) +} + +export function toTapeSearchInput( + options: AgentTapeSearchOptions | undefined +): DeepChatTapeSearchInput { + return { + limit: options?.limit, + kinds: options?.kinds, + startCreatedAt: parseSearchBoundary(options?.start, 'start'), + endCreatedAt: parseSearchBoundary(options?.end, 'end') + } +} + +export function normalizeTapeViewScope(scope: AgentTapeViewScope | undefined): AgentTapeViewScope { + if (scope === undefined || scope === 'current') return 'current' + if (scope === 'linked_subagents' || scope === 'current_and_linked') return scope + throw new Error(`Invalid Tape view scope: ${String(scope)}`) +} + +export function normalizeTapeSearchLimit(value: number | undefined): number { + if (!Number.isFinite(value)) return 20 + return Math.min(Math.max(Math.floor(value as number), 1), 100) +} + +export function compareTapeSearchResults(left: TapeSearchResult, right: TapeSearchResult): number { + const leftHasScore = typeof left.score === 'number' && Number.isFinite(left.score) + const rightHasScore = typeof right.score === 'number' && Number.isFinite(right.score) + if (leftHasScore && rightHasScore && left.score !== right.score) { + return (left.score as number) - (right.score as number) + } + if (leftHasScore !== rightHasScore) { + return leftHasScore ? -1 : 1 + } + if (left.createdAt !== right.createdAt) { + return right.createdAt - left.createdAt + } + if (left.sessionId !== right.sessionId) { + return left.sessionId < right.sessionId ? -1 : 1 + } + return right.entryId - left.entryId +} diff --git a/src/main/tape/application/recallService.ts b/src/main/tape/application/recallService.ts new file mode 100644 index 0000000000..93a4e74f0f --- /dev/null +++ b/src/main/tape/application/recallService.ts @@ -0,0 +1,584 @@ +import logger from 'electron-log' +import type { + AgentTapeAnchorsOptions, + AgentTapeContextEntry, + AgentTapeContextOptions, + AgentTapeContextResult, + AgentTapeSearchOptions +} from '@shared/types/agent-interface' +import { + buildEffectiveTapeView, + getLastEffectiveTokenUsage, + searchEffectiveTapeRows +} from '../domain/effectiveView' +import type { DeepChatTapeEntryRow, DeepChatTapeReadSource } from '../domain/entry' +import type { + TapeApplicationProviders, + TapeSearchProjectionInput as DeepChatTapeSearchProjectionInput, + TapeSearchProjectionResultRow as DeepChatTapeSearchProjectionResultRow, + TapeSearchProjectionRow as DeepChatTapeSearchProjectionRow, + TapeSearchProjectionStore +} from '../ports/application' +import type { TapeEffectiveMessageSourceEntry } from '../ports/capabilities' +import { isEntryIdPrefix, migrationProvenanceKey, parseJsonObject } from './common' +import type { TapeAnchorResult, TapeInfo, TapeSearchResult } from './contracts' +import { AgentTapeViewError, type TapeLineageService } from './lineageService' +import { + buildTapeRowEvidenceText, + buildTapeRowRefs, + compareTapeSearchResults, + getUserMessageProjectionText, + normalizeContextByteLimit, + normalizeContextLimit, + normalizeContextWindowValue, + normalizeTapeSearchLimit, + normalizeTapeViewScope, + parseProjectionRefs, + stringifyForSummary, + summarizeTapeRow, + toTapeSearchInput, + truncateToUtf8Bytes +} from './recallProjection' + +type TapeRecallProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getSearchProjectionStore' +> + +const DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY = 2048 +const DEFAULT_CONTEXT_MAX_TOTAL_BYTES = 16384 +const MAX_CONTEXT_MAX_BYTES_PER_ENTRY = 8192 +const MAX_CONTEXT_MAX_TOTAL_BYTES = 65536 + +export class TapeRecallService { + constructor( + private readonly providers: TapeRecallProviders, + private readonly lineage: TapeLineageService + ) {} + + private get table() { + return this.providers.getEntryStore() + } + + private get searchProjectionTable() { + return this.providers.getSearchProjectionStore() + } + + info(sessionId: string): TapeInfo { + const table = this.table + const lastAnchor = table.getLatestAnchor(sessionId) + const rows = table.getBySession(sessionId) + return { + sessionId, + entries: table.countBySession(sessionId), + anchors: table.countAnchorsBySession(sessionId), + lastAnchor: lastAnchor?.name ?? null, + lastAnchorEntryId: lastAnchor?.entry_id ?? null, + entriesSinceLastAnchor: lastAnchor + ? table.countEntriesAfter(sessionId, lastAnchor.entry_id) + : 0, + lastTokenUsage: getLastEffectiveTokenUsage(rows), + migrationState: table.getByProvenanceKey(sessionId, migrationProvenanceKey(sessionId)) + ? 'ready' + : 'none' + } + } + + getEffectiveMessageSourceSpan( + sessionId: string, + entryIds: number[] + ): TapeEffectiveMessageSourceEntry[] { + const requestedEntryIds = new Set( + entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0) + ) + if (requestedEntryIds.size === 0) return [] + return buildEffectiveTapeView(this.table.getBySession(sessionId), { includePending: false }) + .messageEntries.filter((entry) => requestedEntryIds.has(entry.entryId)) + .map((entry) => ({ + entryId: entry.entryId, + record: { + role: entry.record.role, + content: entry.record.content, + orderSeq: entry.record.orderSeq + } + })) + } + + search(sessionId: string, query: string, options?: AgentTapeSearchOptions): TapeSearchResult[] { + const scope = normalizeTapeViewScope(options?.scope) + if (!query.trim()) { + return [] + } + if (scope === 'current') { + return this.searchCurrentTape(sessionId, query, options) + } + + const resolution = this.lineage.resolveLinkedTapeSources(sessionId) + if (resolution.unavailableSourceIds.size > 0) { + const sourceSessionId = [...resolution.unavailableSourceIds].sort()[0] + throw new AgentTapeViewError( + 'linked_tape_unavailable', + sessionId, + sourceSessionId, + `Linked Tape ${sourceSessionId} is unavailable.` + ) + } + const sources = [...resolution.sources] + if (scope === 'current_and_linked') { + sources.push({ sessionId, maxEntryId: this.table?.getMaxEntryId(sessionId) ?? 0 }) + } + return this.searchTapeSourcesReadOnly(sources, query, options) + } + + private searchCurrentTape( + sessionId: string, + query: string, + options?: AgentTapeSearchOptions + ): TapeSearchResult[] { + const table = this.table + const searchInput = toTapeSearchInput(options) + const projectionTable = this.searchProjectionTable + let skipProjectionSearch = false + + if (projectionTable) { + try { + const maxEntryId = table.getMaxEntryId(sessionId) + if (projectionTable.isCurrent(sessionId, maxEntryId)) { + return projectionTable + .search(sessionId, query, searchInput) + .map((row) => this.toProjectedSearchResult(row, undefined)) + } + } catch (error) { + skipProjectionSearch = true + logger.warn( + `[Tape] projection fast-path search failed; falling back to effective search: ${String(error)}` + ) + } + } + + const rows = table.getBySession(sessionId) + const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows + const preparedProjectionTable = skipProjectionSearch + ? null + : this.ensureSearchProjection(sessionId, rows, effectiveRows) + if (!preparedProjectionTable) { + return searchEffectiveTapeRows(rows, query, searchInput).map((row) => + this.toSearchResult(row) + ) + } + + const rowByEntryId = new Map(effectiveRows.map((row) => [row.entry_id, row])) + try { + return preparedProjectionTable + .search(sessionId, query, searchInput) + .map((row) => this.toProjectedSearchResult(row, rowByEntryId.get(row.entry_id))) + } catch (error) { + logger.warn( + `[Tape] projection search failed; falling back to effective search: ${String(error)}` + ) + return searchEffectiveTapeRows(rows, query, searchInput).map((row) => + this.toSearchResult(row) + ) + } + } + + getContext( + sessionId: string, + entryIds: number[], + options: AgentTapeContextOptions = {} + ): AgentTapeContextResult { + const sourceSessionId = options.sourceSessionId?.trim() || sessionId + if (sourceSessionId !== sessionId) { + return this.getLinkedTapeContext(sessionId, sourceSessionId, entryIds, options) + } + + const requestedEntryIds = [ + ...new Set(entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0)) + ].sort((left, right) => left - right) + const table = this.table + if (!table || requestedEntryIds.length === 0) { + return { + sessionId, + sourceSessionId, + requestedEntryIds, + matchedEntryIds: [], + entries: [] + } + } + + const rows = table.getBySession(sessionId) + const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows + const indexByEntryId = new Map(effectiveRows.map((row, index) => [row.entry_id, index])) + const before = normalizeContextWindowValue(options.before, 2) + const after = normalizeContextWindowValue(options.after, 2) + const limit = normalizeContextLimit(options.limit) + const maxBytesPerEntry = normalizeContextByteLimit( + options.maxBytesPerEntry, + DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY, + MAX_CONTEXT_MAX_BYTES_PER_ENTRY + ) + const maxTotalBytes = normalizeContextByteLimit( + options.maxTotalBytes, + DEFAULT_CONTEXT_MAX_TOTAL_BYTES, + MAX_CONTEXT_MAX_TOTAL_BYTES + ) + const selectedIndexes = new Set() + const requestedIndexes: number[] = [] + const windowIndexes: number[] = [] + + for (const entryId of requestedEntryIds) { + const index = indexByEntryId.get(entryId) + if (index === undefined) continue + requestedIndexes.push(index) + for ( + let cursor = Math.max(0, index - before); + cursor <= Math.min(effectiveRows.length - 1, index + after); + cursor += 1 + ) { + if (cursor === index) continue + windowIndexes.push(cursor) + } + } + + for (const index of requestedIndexes) { + if (selectedIndexes.size >= limit) break + selectedIndexes.add(index) + } + for (const index of windowIndexes) { + if (selectedIndexes.size >= limit) break + selectedIndexes.add(index) + } + + const selectedRows = [...selectedIndexes] + .sort((left, right) => left - right) + .map((index) => effectiveRows[index]) + let projectionRows = new Map() + try { + const maxEntryId = rows.reduce((max, row) => Math.max(max, row.entry_id), 0) + projectionRows = new Map( + this.searchProjectionTable + .getByEntryIdsIfCurrent( + sessionId, + maxEntryId, + selectedRows.map((row) => row.entry_id) + ) + .map((row) => [row.entry_id, row]) + ) + } catch { + projectionRows = new Map() + } + let usedBytes = 0 + const entries: AgentTapeContextEntry[] = [] + const priorityIndexes = [...requestedIndexes, ...windowIndexes].filter( + (index, offset, indexes) => { + return selectedIndexes.has(index) && indexes.indexOf(index) === offset + } + ) + for (const index of priorityIndexes) { + const row = effectiveRows[index] + const remaining = Math.max(0, maxTotalBytes - usedBytes) + if (remaining <= 0) break + const maxEntryBytes = Math.min(maxBytesPerEntry, remaining) + if (maxEntryBytes <= 0) break + const entry = this.toContextEntry(row, projectionRows.get(row.entry_id), maxEntryBytes) + if (entry.evidence.bytes <= 0) continue + usedBytes += entry.evidence.bytes + entries.push(entry) + } + entries.sort((left, right) => left.entryId - right.entryId) + const returnedEntryIds = new Set(entries.map((entry) => entry.entryId)) + + return { + sessionId, + sourceSessionId, + requestedEntryIds, + matchedEntryIds: requestedEntryIds.filter((entryId) => returnedEntryIds.has(entryId)), + entries + } + } + + private searchTapeSourcesReadOnly( + sources: DeepChatTapeReadSource[], + query: string, + options: AgentTapeSearchOptions | undefined + ): TapeSearchResult[] { + const table = this.table + if (!table || !query.trim() || sources.length === 0) { + return [] + } + const searchInput = toTapeSearchInput(options) + const limit = normalizeTapeSearchLimit(options?.limit) + const results: TapeSearchResult[] = [] + let uncoveredSources = sources + + try { + const projected = this.searchProjectionTable?.searchSourcesReadOnly( + sources, + query, + searchInput + ) + if (projected) { + const coveredHeadBySource = new Map( + projected.coveredSources.map((source) => [source.sessionId, source.maxEntryId]) + ) + const hasCompleteProjection = + coveredHeadBySource.size === sources.length && + sources.every((source) => coveredHeadBySource.get(source.sessionId) === source.maxEntryId) + if (hasCompleteProjection) { + results.push(...projected.rows.map((row) => this.toProjectedSearchResult(row, undefined))) + uncoveredSources = [] + } + } + } catch (error) { + logger.warn( + `[Tape] linked projection search failed; using read-only Tape fallback: ${String(error)}` + ) + uncoveredSources = sources + } + + if (uncoveredSources.length > 0) { + results.push( + ...table + .searchEffectiveSourcesAtHeads(uncoveredSources, query, searchInput) + .map((row) => this.toSearchResult(row)) + ) + } + + const seen = new Set() + return results + .sort(compareTapeSearchResults) + .filter((result) => { + const key = `${result.sessionId}:${result.entryId}` + if (seen.has(key)) return false + seen.add(key) + return true + }) + .slice(0, limit) + } + + private getLinkedTapeContext( + parentSessionId: string, + sourceSessionId: string, + entryIds: number[], + options: AgentTapeContextOptions + ): AgentTapeContextResult { + const resolution = this.lineage.resolveLinkedTapeSources(parentSessionId) + if (resolution.unavailableSourceIds.has(sourceSessionId)) { + throw new AgentTapeViewError( + 'linked_tape_unavailable', + parentSessionId, + sourceSessionId, + `Linked Tape ${sourceSessionId} is unavailable.` + ) + } + const source = resolution.sources.find((candidate) => candidate.sessionId === sourceSessionId) + if (!source) { + throw new AgentTapeViewError( + 'linked_tape_unauthorized', + parentSessionId, + sourceSessionId, + `Tape ${sourceSessionId} is not an authorized direct child of ${parentSessionId}.` + ) + } + + const requestedEntryIds = [ + ...new Set(entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0)) + ].sort((left, right) => left - right) + const table = this.table + if (!table || requestedEntryIds.length === 0) { + return { + sessionId: parentSessionId, + sourceSessionId, + requestedEntryIds, + matchedEntryIds: [], + entries: [] + } + } + + const before = normalizeContextWindowValue(options.before, 2) + const after = normalizeContextWindowValue(options.after, 2) + const limit = normalizeContextLimit(options.limit) + const maxBytesPerEntry = normalizeContextByteLimit( + options.maxBytesPerEntry, + DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY, + MAX_CONTEXT_MAX_BYTES_PER_ENTRY + ) + const maxTotalBytes = normalizeContextByteLimit( + options.maxTotalBytes, + DEFAULT_CONTEXT_MAX_TOTAL_BYTES, + MAX_CONTEXT_MAX_TOTAL_BYTES + ) + const rows = table.getEffectiveContextRowsAtHead(source, requestedEntryIds, { + before, + after, + limit + }) + + let usedBytes = 0 + const entries: AgentTapeContextEntry[] = [] + for (const row of rows) { + const remaining = Math.max(0, maxTotalBytes - usedBytes) + if (remaining <= 0) break + const maxEntryBytes = Math.min(maxBytesPerEntry, remaining) + const entry = this.toContextEntry(row, undefined, maxEntryBytes) + if (entry.evidence.bytes <= 0) continue + usedBytes += entry.evidence.bytes + entries.push(entry) + } + entries.sort((left, right) => left.entryId - right.entryId) + const returnedEntryIds = new Set(entries.map((entry) => entry.entryId)) + + return { + sessionId: parentSessionId, + sourceSessionId, + requestedEntryIds, + matchedEntryIds: requestedEntryIds.filter((entryId) => returnedEntryIds.has(entryId)), + entries + } + } + + anchors(sessionId: string, options: AgentTapeAnchorsOptions = {}): TapeAnchorResult[] { + return this.table.getAnchors(sessionId, options.limit).map((row) => this.toAnchorResult(row)) + } + + private ensureSearchProjection( + sessionId: string, + rows: DeepChatTapeEntryRow[], + effectiveRows: DeepChatTapeEntryRow[] + ): TapeSearchProjectionStore | null { + const projectionTable = this.searchProjectionTable + const maxEntryId = rows.reduce((max, row) => Math.max(max, row.entry_id), 0) + try { + if (!projectionTable.isCurrent(sessionId, maxEntryId)) { + const meta = projectionTable.getSessionMeta(sessionId) + const projectedEntryIds = projectionTable.getProjectedEntryIds(sessionId) + const effectiveEntryIds = effectiveRows.map((row) => row.entry_id) + const canAppend = + !!meta && + projectionTable.isCurrent(sessionId, meta.maxEntryId) && + meta.maxEntryId <= maxEntryId && + isEntryIdPrefix(projectedEntryIds, effectiveEntryIds) + if (canAppend) { + projectionTable.appendSession( + sessionId, + effectiveRows.slice(projectedEntryIds.length).map((row) => this.toProjectionInput(row)), + maxEntryId + ) + } else { + projectionTable.replaceSession( + sessionId, + effectiveRows.map((row) => this.toProjectionInput(row)), + maxEntryId + ) + } + } + return projectionTable + } catch { + return null + } + } + + private toProjectionInput(row: DeepChatTapeEntryRow): DeepChatTapeSearchProjectionInput { + const payload = parseJsonObject(row.payload_json) + const meta = parseJsonObject(row.meta_json) + const userMessage = getUserMessageProjectionText(row, payload) + const summaryText = summarizeTapeRow(row, payload, userMessage) + const evidenceText = buildTapeRowEvidenceText(row, payload, meta, userMessage) + const refs = buildTapeRowRefs(row, payload, meta, userMessage, evidenceText) + const searchText = [ + row.kind, + row.name ?? '', + summaryText, + evidenceText, + Object.values(refs) + .map((value) => stringifyForSummary(value)) + .join(' ') + ] + .filter(Boolean) + .join('\n') + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + sourceType: row.source_type, + sourceId: row.source_id, + sourceSeq: row.source_seq, + searchText, + summaryText, + refs, + createdAt: row.created_at + } + } + + private toProjectedSearchResult( + row: DeepChatTapeSearchProjectionResultRow, + _sourceRow: DeepChatTapeEntryRow | undefined + ): TapeSearchResult { + const score = + typeof row.score === 'number' && Number.isFinite(row.score) ? row.score : undefined + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + createdAt: row.created_at, + summary: row.summary_text, + refs: parseProjectionRefs(row.refs_json), + ...(score === undefined ? {} : { score }) + } + } + + private toContextEntry( + row: DeepChatTapeEntryRow, + projectionRow: DeepChatTapeSearchProjectionRow | undefined, + maxBytes: number + ): AgentTapeContextEntry { + const fallbackProjection = projectionRow ? null : this.toProjectionInput(row) + const payload = parseJsonObject(row.payload_json) + const meta = parseJsonObject(row.meta_json) + const evidenceSource = buildTapeRowEvidenceText(row, payload, meta) + const clipped = truncateToUtf8Bytes(evidenceSource, maxBytes) + const bytes = Buffer.byteLength(clipped.text, 'utf8') + return { + entryId: row.entry_id, + kind: row.kind, + name: row.name, + summary: projectionRow?.summary_text ?? fallbackProjection?.summaryText ?? '', + refs: projectionRow + ? parseProjectionRefs(projectionRow.refs_json) + : (fallbackProjection?.refs ?? {}), + evidence: { + text: clipped.text, + truncated: clipped.truncated, + bytes + }, + createdAt: row.created_at + } + } + + private toSearchResult(row: DeepChatTapeEntryRow): TapeSearchResult { + const projection = this.toProjectionInput(row) + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + createdAt: row.created_at, + summary: projection.summaryText, + refs: projection.refs + } + } + + private toAnchorResult(row: DeepChatTapeEntryRow): TapeAnchorResult { + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + payload: parseJsonObject(row.payload_json), + meta: parseJsonObject(row.meta_json), + createdAt: row.created_at + } + } +} diff --git a/src/main/tape/application/reconcilerService.ts b/src/main/tape/application/reconcilerService.ts new file mode 100644 index 0000000000..b8379cbda2 --- /dev/null +++ b/src/main/tape/application/reconcilerService.ts @@ -0,0 +1,123 @@ +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { TapeApplicationProviders } from '../ports/application' +import type { TapeBackfillResult, TapeTranscriptReader } from '../ports/capabilities' +import { appendMessageRecordToTape } from './factPersistence' +import type { TapeFactService } from './factService' +import { migrationProvenanceKey } from './common' + +type TapeReconcilerProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getLegacySummaryReader' +> + +function legacySummaryProvenanceKey(sessionId: string): string { + return `summary:${sessionId}:legacy-summary:v1` +} + +export class TapeReconcilerService { + constructor( + private readonly providers: TapeReconcilerProviders, + private readonly facts: TapeFactService + ) {} + + private get table() { + return this.providers.getEntryStore() + } + + ensureSessionTapeReady( + sessionId: string, + messageStore: TapeTranscriptReader + ): TapeBackfillResult { + const table = this.table + const historyRecords = [...messageStore.getMessages(sessionId)].sort( + (left, right) => left.orderSeq - right.orderSeq + ) + const maxOrderSeq = historyRecords.reduce( + (currentMax, record) => Math.max(currentMax, record.orderSeq), + 0 + ) + + table.ensureBootstrapAnchor(sessionId) + + let appendedFactCount = 0 + for (const record of historyRecords) { + appendedFactCount += appendMessageRecordToTape(table, record, 'backfill') + } + + this.backfillLegacySummaryAnchor(sessionId, historyRecords) + + table.appendEvent({ + sessionId, + name: 'migration/backfill', + source: { + type: 'migration', + id: 'message-backfill', + seq: 1 + }, + provenanceKey: migrationProvenanceKey(sessionId), + data: { + source: 'deepchat_messages', + messageCount: historyRecords.length, + maxOrderSeq + }, + idempotent: true + }) + + return { + sessionId, + migrationState: 'ready', + messageCount: historyRecords.length, + maxOrderSeq, + appendedFactCount, + historyRecords: this.facts.getMessageRecords(sessionId) + } + } + + private backfillLegacySummaryAnchor( + sessionId: string, + historyRecords: ChatMessageRecord[] + ): void { + const table = this.table + if (table.getLatestSummaryAnchor(sessionId)) { + return + } + + const legacyState = this.providers.getLegacySummaryReader().getSummaryState(sessionId) + if (!legacyState) { + return + } + + const summary = legacyState.summary_text?.trim() + if (!summary) { + return + } + + const cursorOrderSeq = Math.max(1, legacyState.summary_cursor_order_seq ?? 1) + const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) + table.appendAnchor({ + sessionId, + name: 'compaction/migrated_summary', + source: { + type: 'summary', + id: 'legacy-summary', + seq: 1 + }, + provenanceKey: legacySummaryProvenanceKey(sessionId), + state: { + summary, + cursorOrderSeq, + range: + sourceRecords.length > 0 + ? { + fromOrderSeq: sourceRecords[0].orderSeq, + toOrderSeq: sourceRecords[sourceRecords.length - 1].orderSeq + } + : null, + sourceMessageIds: sourceRecords.map((record) => record.id), + migratedFrom: 'deepchat_sessions.summary_text' + }, + idempotent: true, + createdAt: legacyState.summary_updated_at ?? undefined + }) + } +} diff --git a/src/main/tape/application/sessionTape.ts b/src/main/tape/application/sessionTape.ts new file mode 100644 index 0000000000..8a1fccd651 --- /dev/null +++ b/src/main/tape/application/sessionTape.ts @@ -0,0 +1,282 @@ +import type { + AgentTapeAnchorsOptions, + AgentTapeContextOptions, + AgentTapeContextResult, + AgentTapeHandoffState, + AgentTapeSearchOptions, + ChatMessageRecord, + SubagentTapeLinkInput, + SubagentTapeLinkReceipt +} from '@shared/types/agent-interface' +import type { + DeepChatTapeViewManifest, + DeepChatTapeViewManifestRecord +} from '@shared/types/tape-view-manifest' +import type { + DeepChatCausalObservationReadOptions, + DeepChatCausalObservationSlice, + DeepChatTapeReplayExportOptions, + DeepChatTapeReplaySlice +} from '@shared/types/tape-replay' +import type { DeepChatTapeEntryRow, TapeAnchorAppendInput } from '../domain/entry' +import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' +import type { + TapeAnchorReader, + TapeAnchorWriter, + TapeEffectiveMessageSourceEntry, + TapeInspectionReader, + TapeLifecycleAdmin, + TapeMessageFactWriter, + TapeRawEntryReader, + TapeReconciliationPort, + TapeToolFactWriter, + TapeTranscriptReader, + TapeMemoryViewManifestInspection, + TapeViewManifestReader, + TapeViewManifestWriter +} from '../ports/capabilities' +import { + createTapeApplicationProviders, + type TapeApplicationDatabase, + type TapeApplicationProviders +} from '../ports/application' +import type { + TapeAnchorResult, + TapeBackfillResult, + TapeForkHandle, + TapeInfo, + TapeMigrationState, + TapeSearchResult, + TapeViewManifestAssemblySources +} from './contracts' +import { normalizeTapeHandoffState, TapeFactService } from './factService' +import { TapeForkService } from './forkService' +import { deleteTapeGeneration, resetTapeGeneration } from './generationLifecycle' +import { + AgentTapeViewError, + normalizeSubagentTapeLinkInput, + TapeLineageService, + type AgentTapeViewErrorCode +} from './lineageService' +import { TapeRecallService } from './recallService' +import { TapeReconcilerService } from './reconcilerService' +import { TapeViewReplayService } from './viewReplayService' + +export type { + AgentTapeViewErrorCode, + TapeAnchorResult, + TapeBackfillResult, + TapeForkHandle, + TapeInfo, + TapeMigrationState, + TapeSearchResult, + TapeViewManifestAssemblySources +} +export { AgentTapeViewError, normalizeSubagentTapeLinkInput, normalizeTapeHandoffState } + +export class SessionTape + implements + TapeToolFactWriter, + TapeMessageFactWriter, + TapeRawEntryReader, + TapeReconciliationPort, + TapeViewManifestReader, + TapeViewManifestWriter, + TapeAnchorReader, + TapeAnchorWriter, + TapeInspectionReader, + TapeLifecycleAdmin +{ + private readonly providers: TapeApplicationProviders + private readonly facts: TapeFactService + private readonly reconciler: TapeReconcilerService + private readonly recall: TapeRecallService + private readonly lineage: TapeLineageService + private readonly viewReplay: TapeViewReplayService + private readonly forks: TapeForkService + + constructor(database: TapeApplicationDatabase) { + this.providers = createTapeApplicationProviders(database) + this.facts = new TapeFactService(this.providers) + this.lineage = new TapeLineageService(this.providers) + this.reconciler = new TapeReconcilerService(this.providers, this.facts) + this.recall = new TapeRecallService(this.providers, this.lineage) + this.viewReplay = new TapeViewReplayService(this.providers) + this.forks = new TapeForkService(this.providers) + } + + ensureSessionTapeReady( + sessionId: string, + messageStore: TapeTranscriptReader + ): TapeBackfillResult { + return this.reconciler.ensureSessionTapeReady(sessionId, messageStore) + } + + appendMessageRecord(record: ChatMessageRecord): number { + return this.facts.appendMessageRecord(record) + } + + appendMessageReplacement(record: ChatMessageRecord, reason: string): number { + return this.facts.appendMessageReplacement(record, reason) + } + + appendMessageRetraction(record: ChatMessageRecord, reason: string): number { + return this.facts.appendMessageRetraction(record, reason) + } + + appendToolFact(input: TapeToolFactInput): Promise { + return this.facts.appendToolFact(input) + } + + getMessageRecords(sessionId: string): ChatMessageRecord[] { + return this.facts.getMessageRecords(sessionId) + } + + info(sessionId: string): TapeInfo { + return this.recall.info(sessionId) + } + + search(sessionId: string, query: string, options?: AgentTapeSearchOptions): TapeSearchResult[] { + return this.recall.search(sessionId, query, options) + } + + getContext( + sessionId: string, + entryIds: number[], + options: AgentTapeContextOptions = {} + ): AgentTapeContextResult { + return this.recall.getContext(sessionId, entryIds, options) + } + + anchors(sessionId: string, options: AgentTapeAnchorsOptions = {}): TapeAnchorResult[] { + return this.recall.anchors(sessionId, options) + } + + getViewManifestSourceMaps( + sessionId: string, + messageId?: string + ): TapeViewManifestAssemblySources { + return this.viewReplay.getViewManifestSourceMaps(sessionId, messageId) + } + + appendViewManifest(manifest: DeepChatTapeViewManifest): DeepChatTapeEntryRow { + return this.viewReplay.appendViewManifest(manifest) + } + + listViewManifestsByMessage( + sessionId: string, + messageId: string + ): DeepChatTapeViewManifestRecord[] { + return this.viewReplay.listViewManifestsByMessage(sessionId, messageId) + } + + exportReplaySlice( + sessionId: string, + messageId: string, + options: DeepChatTapeReplayExportOptions = {} + ): DeepChatTapeReplaySlice | null { + return this.viewReplay.exportReplaySlice(sessionId, messageId, options) + } + + readCausalObservationSlice( + sessionId: string, + messageId: string, + options: DeepChatCausalObservationReadOptions = {} + ): DeepChatCausalObservationSlice { + return this.viewReplay.readCausalObservationSlice(sessionId, messageId, options) + } + + handoff( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): DeepChatTapeEntryRow { + return this.facts.handoff(sessionId, name, state, meta) + } + + handoffResult( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): TapeAnchorResult { + return this.facts.handoffResult(sessionId, name, state, meta) + } + + createFork(parentSessionId: string, forkId?: string): TapeForkHandle { + return this.forks.createFork(parentSessionId, forkId) + } + + appendForkMessageRecord(handle: TapeForkHandle, record: ChatMessageRecord): number { + return this.facts.appendMessageRecordForSession(handle.forkSessionId, record) + } + + mergeFork(parentSessionId: string, forkId: string): number { + return this.forks.mergeFork(parentSessionId, forkId) + } + + discardFork(parentSessionId: string, forkId: string): void { + this.forks.discardFork(parentSessionId, forkId) + } + + recordExternalForkMerge( + parentSessionId: string, + forkSessionId: string, + forkId: string, + meta: Record = {} + ): DeepChatTapeEntryRow { + return this.forks.recordExternalForkMerge(parentSessionId, forkSessionId, forkId, meta) + } + + recordExternalForkDiscard( + parentSessionId: string, + forkSessionId: string, + forkId: string, + meta: Record = {} + ): DeepChatTapeEntryRow { + return this.forks.recordExternalForkDiscard(parentSessionId, forkSessionId, forkId, meta) + } + + linkSubagentTape(input: SubagentTapeLinkInput): SubagentTapeLinkReceipt { + return this.lineage.linkSubagentTape(input) + } + + getBySession(sessionId: string): DeepChatTapeEntryRow[] { + return this.providers.getEntryStore().getBySession(sessionId) + } + + getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + return this.providers.getEntryStore().getLatestReconstructionAnchor(sessionId) + } + + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow { + return this.facts.appendAnchor(input) + } + + getEffectiveMessageSourceSpan( + sessionId: string, + entryIds: number[] + ): TapeEffectiveMessageSourceEntry[] { + return this.recall.getEffectiveMessageSourceSpan(sessionId, entryIds) + } + + listMemoryViewManifestsByAgent( + agentId: string, + options?: { sessionId?: string; limit?: number; messageId?: string } + ): TapeMemoryViewManifestInspection[] { + return this.viewReplay.listMemoryViewManifestsByAgent(agentId, options) + } + + initializeSessionTape(sessionId: string): void { + this.providers.getEntryStore().ensureBootstrapAnchor(sessionId) + } + + deleteSessionTape(sessionId: string): void { + deleteTapeGeneration(this.providers, sessionId) + } + + resetSessionTape(sessionId: string): void { + resetTapeGeneration(this.providers, sessionId) + } +} diff --git a/src/main/tape/application/viewReplayService.ts b/src/main/tape/application/viewReplayService.ts new file mode 100644 index 0000000000..1e1f8dbff0 --- /dev/null +++ b/src/main/tape/application/viewReplayService.ts @@ -0,0 +1,523 @@ +import type { + DeepChatTapeViewManifest, + DeepChatTapeViewManifestRecord +} from '@shared/types/tape-view-manifest' +import type { + DeepChatCausalObservationReadOptions, + DeepChatCausalObservationRequest, + DeepChatCausalObservationSlice, + DeepChatTapeReplayEntrySnapshot, + DeepChatTapeReplayExportOptions, + DeepChatTapeReplaySlice, + DeepChatTapeReplayTraceSnapshot +} from '@shared/types/tape-replay' +import { SUMMARY_ANCHOR_NAMES, type DeepChatTapeEntryRow } from '../domain/entry' +import { buildEffectiveTapeView } from '../domain/effectiveView' +import { + collectEntryIds, + hashString, + isPositiveInteger, + normalizeStoredTapeViewManifest, + withReplaySliceHash +} from '../domain/replay' +import { + hashJson, + TAPE_VIEW_MANIFEST_EVENT_NAME, + verifyTapeViewManifestHash +} from '../domain/viewManifest' +import type { + TapeApplicationProviders, + TapeMessageTraceRow as DeepChatMessageTraceRow +} from '../ports/application' +import type { TapeMemoryViewManifestInspection } from '../ports/capabilities' +import { parseJsonObject } from './common' +import type { TapeViewManifestAssemblySources } from './contracts' + +type TapeViewReplayProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getMessageTraceReader' | 'getTerminalMessageReader' +> + +const BOOTSTRAP_ANCHOR_NAME = 'session/start' + +function isReconstructionAnchorName(name: string | null): boolean { + if (name === null) { + return false + } + return ( + (SUMMARY_ANCHOR_NAMES as readonly string[]).includes(name) || + name.startsWith('handoff/') || + name.startsWith('auto_handoff/') + ) +} + +function readToolFactStatus(row: DeepChatTapeEntryRow): string | null { + const status = parseJsonObject(row.meta_json).status + return typeof status === 'string' ? status : null +} + +function readToolFactToolCallId(row: DeepChatTapeEntryRow): string | null { + const payload = parseJsonObject(row.payload_json) + if (row.kind === 'tool_call') { + const toolCall = payload.toolCall + if (toolCall && typeof toolCall === 'object' && !Array.isArray(toolCall)) { + const id = (toolCall as Record).id + return typeof id === 'string' && id.length > 0 ? id : null + } + return null + } + const toolCallId = payload.toolCallId + return typeof toolCallId === 'string' && toolCallId.length > 0 ? toolCallId : null +} + +function readToolFactMessageId(row: DeepChatTapeEntryRow): string | null { + const messageId = parseJsonObject(row.payload_json).messageId + return typeof messageId === 'string' && messageId.length > 0 ? messageId : null +} + +function deriveSelectedMemoryIds(value: unknown): string[] | null { + if (!Array.isArray(value)) return null + const ids = new Set() + for (const item of value) { + const id = + typeof item === 'string' + ? item + : item && typeof item === 'object' && !Array.isArray(item) + ? (item as Record).id + : null + if (typeof id === 'string' && id.length > 0) ids.add(id) + } + return [...ids] +} + +function toMemoryViewManifestInspection( + row: DeepChatTapeEntryRow +): TapeMemoryViewManifestInspection | null { + const payload = parseJsonObject(row.payload_json) + const meta = parseJsonObject(row.meta_json) + const manifest = + payload.state && typeof payload.state === 'object' && !Array.isArray(payload.state) + ? (payload.state as Record) + : null + if (!manifest) return null + const readNumber = (value: unknown): number => + typeof value === 'number' && Number.isFinite(value) ? value : 0 + return { + sessionId: row.session_id, + messageId: typeof meta.messageId === 'string' ? meta.messageId : null, + entryId: row.entry_id, + policyVersion: + typeof manifest.policyVersion === 'number' && Number.isFinite(manifest.policyVersion) + ? manifest.policyVersion + : null, + tokenBudget: readNumber(manifest.tokenBudget), + estimatedTokens: readNumber(manifest.estimatedTokens), + selectedCount: Array.isArray(manifest.selected) ? manifest.selected.length : 0, + selectedIds: deriveSelectedMemoryIds(manifest.selected), + droppedCount: Array.isArray(manifest.dropped) ? manifest.dropped.length : 0, + queryHash: typeof manifest.queryHash === 'string' ? manifest.queryHash : null, + createdAt: row.created_at + } +} + +export class TapeViewReplayService { + constructor(private readonly providers: TapeViewReplayProviders) {} + + private get table() { + return this.providers.getEntryStore() + } + + getViewManifestSourceMaps( + sessionId: string, + messageId?: string + ): TapeViewManifestAssemblySources { + const table = this.table + const rows = table.getBySession(sessionId) + const entryIdByMessageId = new Map() + const toolCallEntryIdByToolId = new Map() + const toolResultEntryIdByToolId = new Map() + let latestEntryId = 0 + const anchorEntryIds: number[] = [] + let reconstructionAnchorEntryId: number | null = null + let bootstrapAnchorEntryId: number | null = null + + for (const row of rows) { + latestEntryId = Math.max(latestEntryId, row.entry_id) + if (row.kind === 'anchor') { + anchorEntryIds.push(row.entry_id) + if (isReconstructionAnchorName(row.name)) { + if (reconstructionAnchorEntryId === null || row.entry_id > reconstructionAnchorEntryId) { + reconstructionAnchorEntryId = row.entry_id + } + } else if (row.name === BOOTSTRAP_ANCHOR_NAME) { + bootstrapAnchorEntryId = row.entry_id + } + continue + } + if (row.kind === 'message' && row.source_type === 'message' && row.source_id) { + entryIdByMessageId.set(row.source_id, row.entry_id) + continue + } + if (row.kind === 'tool_call' || row.kind === 'tool_result') { + if (messageId && readToolFactMessageId(row) !== messageId) { + continue + } + const toolCallId = readToolFactToolCallId(row) + if (!toolCallId || readToolFactStatus(row) === 'pending') { + continue + } + const target = + row.kind === 'tool_call' ? toolCallEntryIdByToolId : toolResultEntryIdByToolId + target.set(toolCallId, row.entry_id) + } + } + + const reconstructionAnchorEntryIds = + reconstructionAnchorEntryId !== null + ? [reconstructionAnchorEntryId] + : bootstrapAnchorEntryId !== null + ? [bootstrapAnchorEntryId] + : [] + + return { + latestEntryId, + anchorEntryIds, + reconstructionAnchorEntryIds, + reconstructionAnchorEntryId, + entryIdByMessageId, + toolCallEntryIdByToolId, + toolResultEntryIdByToolId + } + } + + listMemoryViewManifestsByAgent( + agentId: string, + options?: { sessionId?: string; limit?: number; messageId?: string } + ): TapeMemoryViewManifestInspection[] { + return this.table + .listMemoryViewManifestAnchorsByAgent(agentId, options) + .map(toMemoryViewManifestInspection) + .filter((manifest): manifest is TapeMemoryViewManifestInspection => manifest !== null) + .filter((manifest) => !options?.messageId || manifest.messageId === options.messageId) + } + + appendViewManifest(manifest: DeepChatTapeViewManifest): DeepChatTapeEntryRow { + const table = this.table + table.ensureBootstrapAnchor(manifest.sessionId) + return table.appendEvent({ + sessionId: manifest.sessionId, + name: TAPE_VIEW_MANIFEST_EVENT_NAME, + source: { + type: 'runtime_event', + id: manifest.messageId, + seq: manifest.requestSeq + }, + provenanceKey: `view:${manifest.sessionId}:${manifest.messageId}:${manifest.requestSeq}:${manifest.hashes.manifestHash}`, + data: { + manifest + }, + meta: { + viewId: manifest.viewId, + requestSeq: manifest.requestSeq, + taskType: manifest.taskType, + policy: manifest.policy, + policyVersion: manifest.policyVersion + }, + createdAt: manifest.assembledAt, + idempotent: true + }) + } + + listViewManifestsByMessage( + sessionId: string, + messageId: string + ): DeepChatTapeViewManifestRecord[] { + const table = this.table + return table + .getBySession(sessionId) + .filter( + (row) => + row.kind === 'event' && + row.name === TAPE_VIEW_MANIFEST_EVENT_NAME && + row.source_type === 'runtime_event' && + row.source_id === messageId + ) + .map((row) => this.toViewManifestRecord(row)) + .filter((record): record is DeepChatTapeViewManifestRecord => Boolean(record)) + .sort((left, right) => right.requestSeq - left.requestSeq || right.entryId - left.entryId) + } + + exportReplaySlice( + sessionId: string, + messageId: string, + options: DeepChatTapeReplayExportOptions = {} + ): DeepChatTapeReplaySlice | null { + if (options.requestSeq !== undefined && !isPositiveInteger(options.requestSeq)) { + throw new Error('requestSeq must be a positive integer.') + } + + const manifests = this.listViewManifestsByMessage(sessionId, messageId) + const manifestRecord = + options.requestSeq === undefined + ? manifests[0] + : manifests.find((record) => record.requestSeq === options.requestSeq) + if (!manifestRecord) { + return null + } + + return this.buildReplaySlice(sessionId, messageId, manifestRecord, options) + } + + readCausalObservationSlice( + sessionId: string, + messageId: string, + options: DeepChatCausalObservationReadOptions = {} + ): DeepChatCausalObservationSlice { + if (options.requestSeq !== undefined && !isPositiveInteger(options.requestSeq)) { + throw new Error('requestSeq must be a positive integer.') + } + + const rows = this.table.getBySession(sessionId) + const manifestRows = rows.filter( + (row) => + row.kind === 'event' && + row.name === TAPE_VIEW_MANIFEST_EVENT_NAME && + row.source_type === 'runtime_event' && + row.source_id === messageId + ) + const traces = this.providers + .getMessageTraceReader() + .listByMessageId(messageId) + .filter( + (row) => + row.session_id === sessionId && + row.message_id === messageId && + isPositiveInteger(row.request_seq) + ) + + const requestSeq = + options.requestSeq ?? + [...manifestRows.map((row) => row.source_seq), ...traces.map((row) => row.request_seq)] + .filter((value): value is number => typeof value === 'number' && isPositiveInteger(value)) + .reduce((latest, value) => Math.max(latest ?? value, value), null) + + let request: DeepChatCausalObservationRequest + if (requestSeq === null) { + request = { state: 'request_unavailable', requestSeq: null, trace: null } + } else { + const selectedManifestRows = manifestRows.filter((row) => row.source_seq === requestSeq) + const manifestRecord = selectedManifestRows + .map((row) => this.toViewManifestRecord(row)) + .find((record) => record?.messageId === messageId && record.requestSeq === requestSeq) + const trace = traces.find((row) => row.request_seq === requestSeq) ?? null + + if (manifestRecord) { + request = { + state: 'manifest_bound', + requestSeq, + replay: this.buildReplaySlice(sessionId, messageId, manifestRecord, options) + } + } else { + request = { + state: selectedManifestRows.length > 0 ? 'manifest_malformed' : 'manifest_missing', + requestSeq, + trace: trace + ? this.toReplayTraceSnapshot(trace, options.includeTracePayload === true) + : null + } + } + } + + const outputEntries = buildEffectiveTapeView(rows, { includePending: false }) + .rows.filter( + (row) => + (row.kind === 'message' && + row.source_type === 'message' && + row.source_id === messageId) || + ((row.kind === 'tool_call' || row.kind === 'tool_result') && + readToolFactMessageId(row) === messageId) + ) + .map((row) => this.toReplayEntrySnapshot(row, options.includeTapePayloads === true)) + const message = this.providers.getTerminalMessageReader().get(messageId) + const terminalMessage = + message?.session_id === sessionId && + message.role === 'assistant' && + (message.status === 'sent' || message.status === 'error') + ? { + status: message.status, + orderSeq: message.order_seq, + createdAt: message.created_at, + updatedAt: message.updated_at, + contentHash: hashString(message.content), + metadataHash: hashString(message.metadata) + } + : null + + return { + schemaVersion: 1, + sessionId, + messageId, + request, + output: { + correlation: 'message_only', + entries: outputEntries, + terminalMessage + }, + runtime: + options.currentRuntimeStatus === undefined + ? { scope: 'unavailable', status: null, eventHistory: 'not_persisted' } + : { + scope: 'current_only', + status: options.currentRuntimeStatus, + eventHistory: 'not_persisted' + } + } + } + + private buildReplaySlice( + sessionId: string, + messageId: string, + manifestRecord: DeepChatTapeViewManifestRecord, + options: DeepChatTapeReplayExportOptions + ): DeepChatTapeReplaySlice { + const table = this.table + const manifest = manifestRecord.manifest + const includedEntryIds = collectEntryIds(manifest.included.map((ref) => ref.entryId)) + const excludedEntryIds = collectEntryIds(manifest.excluded.map((ref) => ref.entryId)) + const anchorEntryIds = collectEntryIds(manifest.anchorEntryIds) + const selectedEntryIds = new Set([ + manifestRecord.entryId, + ...includedEntryIds, + ...excludedEntryIds, + ...anchorEntryIds + ]) + const entries = table + .getBySession(sessionId) + .filter((row) => selectedEntryIds.has(row.entry_id)) + .map((row) => this.toReplayEntrySnapshot(row, options.includeTapePayloads === true)) + + const trace = this.findReplayTrace(sessionId, messageId, manifestRecord.requestSeq) + const createdAt = Date.now() + const sliceBase: Omit & { + hashes: Omit & { sliceHash: '' } + } = { + schemaVersion: 1 as const, + sliceId: `replay_${hashJson({ + sessionId, + messageId, + requestSeq: manifestRecord.requestSeq, + manifestHash: manifest.hashes.manifestHash + }).slice(0, 16)}`, + sessionId, + messageId, + requestSeq: manifestRecord.requestSeq, + mode: trace ? 'trace_bound' : 'manifest_only', + manifestRecord, + trace: trace ? this.toReplayTraceSnapshot(trace, options.includeTracePayload === true) : null, + entries, + refs: { + manifestEntryId: manifestRecord.entryId, + includedEntryIds, + excludedEntryIds, + anchorEntryIds + }, + hashes: { + manifestHash: manifest.hashes.manifestHash, + sliceHash: '' + }, + integrity: manifestRecord.integrity, + createdAt + } + + return withReplaySliceHash(sliceBase) + } + + private toViewManifestRecord(row: DeepChatTapeEntryRow): DeepChatTapeViewManifestRecord | null { + const payload = parseJsonObject(row.payload_json) + const data = payload.data + const rawManifest = + data && typeof data === 'object' && !Array.isArray(data) + ? (data as Record).manifest + : undefined + const manifest = normalizeStoredTapeViewManifest(rawManifest, row.session_id) + if ( + !manifest || + manifest.messageId !== row.source_id || + manifest.requestSeq !== row.source_seq + ) { + return null + } + + return { + sessionId: row.session_id, + messageId: manifest.messageId, + requestSeq: manifest.requestSeq, + entryId: row.entry_id, + createdAt: row.created_at, + integrity: verifyTapeViewManifestHash(manifest), + manifest + } + } + + private findReplayTrace( + sessionId: string, + messageId: string, + requestSeq: number + ): DeepChatMessageTraceRow | null { + const traceTable = this.providers.getMessageTraceReader() + return ( + traceTable + .listByMessageId(messageId) + .find((row) => row.session_id === sessionId && row.request_seq === requestSeq) ?? null + ) + } + + private toReplayEntrySnapshot( + row: DeepChatTapeEntryRow, + includePayloads: boolean + ): DeepChatTapeReplayEntrySnapshot { + const snapshot: DeepChatTapeReplayEntrySnapshot = { + entryId: row.entry_id, + kind: row.kind, + name: row.name, + sourceType: row.source_type, + sourceId: row.source_id, + sourceSeq: row.source_seq, + provenanceKey: row.provenance_key, + payloadHash: hashString(row.payload_json), + metaHash: hashString(row.meta_json), + createdAt: row.created_at + } + + if (includePayloads) { + snapshot.payload = parseJsonObject(row.payload_json) + snapshot.meta = parseJsonObject(row.meta_json) + } + + return snapshot + } + + private toReplayTraceSnapshot( + row: DeepChatMessageTraceRow, + includePayload: boolean + ): DeepChatTapeReplayTraceSnapshot { + const snapshot: DeepChatTapeReplayTraceSnapshot = { + id: row.id, + requestSeq: row.request_seq, + providerId: row.provider_id, + modelId: row.model_id, + endpoint: row.endpoint, + headersHash: hashString(row.headers_json), + bodyHash: hashString(row.body_json), + truncated: row.truncated === 1, + createdAt: row.created_at + } + + if (includePayload) { + snapshot.headersJson = row.headers_json + snapshot.bodyJson = row.body_json + } + + return snapshot + } +} diff --git a/src/main/tape/domain/effectiveSemantics.ts b/src/main/tape/domain/effectiveSemantics.ts new file mode 100644 index 0000000000..a50091b1a9 --- /dev/null +++ b/src/main/tape/domain/effectiveSemantics.ts @@ -0,0 +1,161 @@ +import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryRow } from './entry' + +const TERMINAL_TAPE_TOOL_STATUSES = new Set(['success', 'error']) + +export interface DeepChatTapeToolIdentity { + key: string + messageId: string +} + +export function parseTapeJsonObject(raw: string): Record { + try { + const parsed = JSON.parse(raw) as unknown + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record + } + } catch {} + return {} +} + +export function parseNestedTapeJsonObject(value: unknown): Record { + if (typeof value === 'string') { + return parseTapeJsonObject(value) + } + if (value && typeof value === 'object' && !Array.isArray(value)) { + return value as Record + } + return {} +} + +export function parseAssistantBlocks(rawContent: string): AssistantMessageBlock[] { + try { + const parsed = JSON.parse(rawContent) as unknown + return Array.isArray(parsed) ? (parsed as AssistantMessageBlock[]) : [] + } catch { + return [] + } +} + +export function messageRecordHasFinalToolUse(record: ChatMessageRecord): boolean { + if (record.role !== 'assistant' || (record.status !== 'sent' && record.status !== 'error')) { + return false + } + const blocks = parseAssistantBlocks(record.content) + const pendingInteractionToolIds = new Set( + blocks.flatMap((block) => + block?.type === 'action' && + (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && + block.status === 'pending' && + typeof block.tool_call?.id === 'string' + ? [block.tool_call.id] + : [] + ) + ) + return blocks.some( + (block) => + block?.type === 'tool_call' && + (block.status === 'success' || block.status === 'error') && + typeof block.tool_call?.id === 'string' && + !pendingInteractionToolIds.has(block.tool_call.id) + ) +} + +function isMessageStatus(value: unknown): value is ChatMessageRecord['status'] { + return value === 'pending' || value === 'sent' || value === 'error' +} + +export function tapeEntryToMessageRecord(row: DeepChatTapeEntryRow): ChatMessageRecord | null { + if (row.kind !== 'message') { + return null + } + + const payload = parseTapeJsonObject(row.payload_json) + const record = payload.record + if (!record || typeof record !== 'object' || Array.isArray(record)) { + return null + } + + const candidate = record as Partial + if ( + typeof candidate.id !== 'string' || + typeof candidate.sessionId !== 'string' || + typeof candidate.orderSeq !== 'number' || + (candidate.role !== 'user' && candidate.role !== 'assistant') || + typeof candidate.content !== 'string' + ) { + return null + } + + return { + id: candidate.id, + sessionId: candidate.sessionId, + orderSeq: candidate.orderSeq, + role: candidate.role, + content: candidate.content, + status: isMessageStatus(candidate.status) ? candidate.status : 'sent', + isContextEdge: typeof candidate.isContextEdge === 'number' ? candidate.isContextEdge : 0, + metadata: typeof candidate.metadata === 'string' ? candidate.metadata : '{}', + traceCount: typeof candidate.traceCount === 'number' ? candidate.traceCount : 0, + createdAt: typeof candidate.createdAt === 'number' ? candidate.createdAt : row.created_at, + updatedAt: typeof candidate.updatedAt === 'number' ? candidate.updatedAt : row.created_at + } +} + +export function tapeMessageRank(record: ChatMessageRecord, includePending: boolean): number { + if (record.status === 'sent' || record.status === 'error') { + return 2 + } + return includePending && record.status === 'pending' ? 1 : 0 +} + +export function readTapeMessageRetractionId(row: DeepChatTapeEntryRow): string | null { + if (row.kind !== 'event' || row.name !== 'message/retracted') { + return null + } + + const payload = parseTapeJsonObject(row.payload_json) + const data = parseNestedTapeJsonObject(payload.data) + return typeof data.messageId === 'string' ? data.messageId : null +} + +export function readTapeToolStatus(row: DeepChatTapeEntryRow): string | null { + const meta = parseTapeJsonObject(row.meta_json) + return typeof meta.status === 'string' ? meta.status : null +} + +export function tapeToolRank(row: DeepChatTapeEntryRow, includePending: boolean): number { + const status = readTapeToolStatus(row) + if (status === 'pending') { + return includePending ? 1 : 0 + } + return status !== null && TERMINAL_TAPE_TOOL_STATUSES.has(status) ? 2 : 0 +} + +export function readTapeToolIdentity(row: DeepChatTapeEntryRow): DeepChatTapeToolIdentity | null { + if (row.kind !== 'tool_call' && row.kind !== 'tool_result') { + return null + } + + const payload = parseTapeJsonObject(row.payload_json) + const messageId = payload.messageId + if (typeof messageId !== 'string' || messageId.length === 0) { + return null + } + + let toolCallId: unknown + if (row.kind === 'tool_call') { + toolCallId = parseNestedTapeJsonObject(payload.toolCall).id + } else { + toolCallId = payload.toolCallId + } + + if (typeof toolCallId !== 'string' || toolCallId.length === 0) { + return null + } + + return { + key: `${row.kind}:${messageId}:${toolCallId}`, + messageId + } +} diff --git a/src/main/tape/domain/effectiveView.ts b/src/main/tape/domain/effectiveView.ts new file mode 100644 index 0000000000..377341ace1 --- /dev/null +++ b/src/main/tape/domain/effectiveView.ts @@ -0,0 +1,251 @@ +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryKind, DeepChatTapeEntryRow, DeepChatTapeSearchInput } from './entry' +import { + parseNestedTapeJsonObject, + readTapeMessageRetractionId, + readTapeToolIdentity, + tapeEntryToMessageRecord, + tapeMessageRank, + tapeToolRank +} from './effectiveSemantics' + +export interface EffectiveMessageEntry { + entryId: number + record: ChatMessageRecord +} + +export interface EffectiveTapeView { + rows: DeepChatTapeEntryRow[] + messageRecords: ChatMessageRecord[] + /** Effective messages paired with their tape entry_id, ordered by orderSeq (for lineage). */ + messageEntries: EffectiveMessageEntry[] +} + +interface EffectiveTapeViewOptions { + includePending?: boolean + includeAuditEvents?: boolean +} + +type EffectiveMessageCandidate = { + row: DeepChatTapeEntryRow + record: ChatMessageRecord +} + +function compareSqliteBinaryText(left: string, right: string): number { + return Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')) +} + +function toNonNegativeInteger(value: unknown): number | null { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + return null + } + return Math.floor(value) +} + +function readTokenUsage(metadata: Record): number | null { + const totalTokens = toNonNegativeInteger(metadata.totalTokens ?? metadata.total_tokens) + if (totalTokens !== null) { + return totalTokens + } + + const inputTokens = toNonNegativeInteger(metadata.inputTokens ?? metadata.input_tokens) + const outputTokens = toNonNegativeInteger(metadata.outputTokens ?? metadata.output_tokens) + if (inputTokens !== null || outputTokens !== null) { + return (inputTokens ?? 0) + (outputTokens ?? 0) + } + + return null +} + +function shouldReplaceMessage( + current: EffectiveMessageCandidate | undefined, + next: EffectiveMessageCandidate, + includePending: boolean +): boolean { + if (!current) { + return true + } + + const currentRank = tapeMessageRank(current.record, includePending) + const nextRank = tapeMessageRank(next.record, includePending) + if (nextRank > currentRank) { + return true + } + if (nextRank < currentRank) { + return false + } + return next.row.entry_id > current.row.entry_id +} + +function isAuditEvent(row: DeepChatTapeEntryRow): boolean { + return ( + row.name === 'message/retracted' || + row.name === 'message/compaction_indicator' || + row.name === 'migration/backfill' + ) +} + +function shouldReplaceToolRow( + current: DeepChatTapeEntryRow | undefined, + next: DeepChatTapeEntryRow, + includePending: boolean +): boolean { + if (!current) { + return true + } + + const currentRank = tapeToolRank(current, includePending) + const nextRank = tapeToolRank(next, includePending) + if (nextRank > currentRank) { + return true + } + if (nextRank < currentRank) { + return false + } + return next.entry_id > current.entry_id +} + +function matchesKinds( + row: DeepChatTapeEntryRow, + kinds: DeepChatTapeEntryKind[] | undefined +): boolean { + return !kinds?.length || kinds.includes(row.kind) +} + +function matchesCreatedAt(row: DeepChatTapeEntryRow, options: DeepChatTapeSearchInput): boolean { + if ( + Number.isFinite(options.startCreatedAt) && + row.created_at < (options.startCreatedAt as number) + ) { + return false + } + if (Number.isFinite(options.endCreatedAt) && row.created_at > (options.endCreatedAt as number)) { + return false + } + return true +} + +function matchesQuery(row: DeepChatTapeEntryRow, normalizedQuery: string): boolean { + const haystack = `${row.payload_json}\n${row.meta_json}\n${row.name ?? ''}`.toLowerCase() + return haystack.includes(normalizedQuery) +} + +export function buildEffectiveTapeView( + rows: DeepChatTapeEntryRow[], + options: EffectiveTapeViewOptions = {} +): EffectiveTapeView { + const includePending = options.includePending === true + const includeAuditEvents = options.includeAuditEvents === true + const messageCandidates = new Map() + const retractedMessageIds = new Set() + const toolRows = new Map() + const anchorRows: DeepChatTapeEntryRow[] = [] + const eventRows: DeepChatTapeEntryRow[] = [] + + for (const row of [...rows].sort((left, right) => left.entry_id - right.entry_id)) { + if (row.kind === 'anchor') { + anchorRows.push(row) + continue + } + + if (row.kind === 'event') { + const retractedMessageId = readTapeMessageRetractionId(row) + if (retractedMessageId) { + messageCandidates.delete(retractedMessageId) + retractedMessageIds.add(retractedMessageId) + } + if (includeAuditEvents || !isAuditEvent(row)) { + eventRows.push(row) + } + continue + } + + if (row.kind === 'message') { + const record = tapeEntryToMessageRecord(row) + if (!record) { + continue + } + const rank = tapeMessageRank(record, includePending) + if (rank === 0) { + continue + } + const candidate = { row, record } + if (shouldReplaceMessage(messageCandidates.get(record.id), candidate, includePending)) { + messageCandidates.set(record.id, candidate) + retractedMessageIds.delete(record.id) + } + continue + } + + const identity = readTapeToolIdentity(row) + if (!identity || tapeToolRank(row, includePending) === 0) { + continue + } + const current = toolRows.get(identity.key)?.row + if (shouldReplaceToolRow(current, row, includePending)) { + toolRows.set(identity.key, { row, messageId: identity.messageId }) + } + } + + const messageRows = [...messageCandidates.values()] + .filter((candidate) => !retractedMessageIds.has(candidate.record.id)) + .sort( + (left, right) => + left.record.orderSeq - right.record.orderSeq || + compareSqliteBinaryText(left.record.id, right.record.id) + ) + const effectiveMessageIds = new Set(messageRows.map((candidate) => candidate.record.id)) + const effectiveToolRows = [...toolRows.values()] + .filter((candidate) => effectiveMessageIds.has(candidate.messageId)) + .map((candidate) => candidate.row) + const effectiveRows = [ + ...anchorRows, + ...eventRows, + ...messageRows.map((candidate) => candidate.row), + ...effectiveToolRows + ].sort((left, right) => left.entry_id - right.entry_id) + + return { + rows: effectiveRows, + messageRecords: messageRows.map((candidate) => candidate.record), + messageEntries: messageRows.map((candidate) => ({ + entryId: candidate.row.entry_id, + record: candidate.record + })) + } +} + +export function searchEffectiveTapeRows( + rows: DeepChatTapeEntryRow[], + query: string, + options: DeepChatTapeSearchInput = {} +): DeepChatTapeEntryRow[] { + const normalizedQuery = query.trim().toLowerCase() + if (!normalizedQuery) { + return [] + } + + const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 + const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) + return buildEffectiveTapeView(rows, { includePending: false }) + .rows.filter((row) => matchesKinds(row, options.kinds)) + .filter((row) => matchesCreatedAt(row, options)) + .filter((row) => matchesQuery(row, normalizedQuery)) + .sort((left, right) => right.entry_id - left.entry_id) + .slice(0, cappedLimit) +} + +export function getLastEffectiveTokenUsage(rows: DeepChatTapeEntryRow[]): number | null { + const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows + for (let index = effectiveRows.length - 1; index >= 0; index -= 1) { + const record = tapeEntryToMessageRecord(effectiveRows[index]) + if (!record || record.role !== 'assistant') { + continue + } + const usage = readTokenUsage(parseNestedTapeJsonObject(record.metadata)) + if (usage !== null) { + return usage + } + } + return null +} diff --git a/src/main/tape/domain/entry.ts b/src/main/tape/domain/entry.ts new file mode 100644 index 0000000000..2bfbaf30e3 --- /dev/null +++ b/src/main/tape/domain/entry.ts @@ -0,0 +1,118 @@ +export type DeepChatTapeEntryKind = 'event' | 'anchor' | 'message' | 'tool_call' | 'tool_result' + +export const TAPE_INCARNATION_META_KEY = 'tapeIncarnationId' + +export type DeepChatTapeSourceType = + | 'session' + | 'message' + | 'assistant_block' + | 'tool_call' + | 'tool_result' + | 'runtime_event' + | 'migration' + | 'summary' + | 'fork' + | 'subagent' + +export interface DeepChatTapeEntryRow { + session_id: string + entry_id: number + kind: DeepChatTapeEntryKind + name: string | null + source_type: DeepChatTapeSourceType | null + source_id: string | null + source_seq: number | null + provenance_key: string | null + payload_json: string + meta_json: string + created_at: number +} + +export interface DeepChatTapeSourceInput { + type: DeepChatTapeSourceType + id: string + seq?: number | null +} + +export interface DeepChatTapeAppendInput { + sessionId: string + kind: DeepChatTapeEntryKind + name?: string | null + source?: DeepChatTapeSourceInput | null + provenanceKey?: string | null + payload: Record + meta?: Record + createdAt?: number + idempotent?: boolean +} + +export interface TapeAnchorAppendInput { + sessionId: string + name: string + state: Record + meta?: Record + source?: DeepChatTapeSourceInput | null + provenanceKey?: string | null + createdAt?: number + idempotent?: boolean +} + +export interface TapeEventAppendInput { + sessionId: string + name: string + data: Record + meta?: Record + source?: DeepChatTapeSourceInput | null + provenanceKey?: string | null + createdAt?: number + idempotent?: boolean +} + +export interface DeepChatTapeSearchInput { + limit?: number + kinds?: DeepChatTapeEntryKind[] + startCreatedAt?: number + endCreatedAt?: number +} + +export interface DeepChatTapeReadSource { + sessionId: string + maxEntryId: number +} + +export const SUMMARY_ANCHOR_NAMES = [ + 'compaction/auto', + 'compaction/manual', + 'compaction/context_pressure', + 'compaction/resume', + 'compaction/migrated_summary', + 'auto_handoff/context_overflow', + 'summary/reset' +] as const + +export function normalizeDeepChatTapeReadSources( + sources: readonly DeepChatTapeReadSource[] +): DeepChatTapeReadSource[] { + const maxEntryIdBySession = new Map() + for (const source of sources) { + const sessionId = source.sessionId.trim() + if (!sessionId || !Number.isSafeInteger(source.maxEntryId) || source.maxEntryId < 0) { + continue + } + maxEntryIdBySession.set( + sessionId, + Math.max(maxEntryIdBySession.get(sessionId) ?? 0, source.maxEntryId) + ) + } + return [...maxEntryIdBySession.entries()] + .map(([sessionId, maxEntryId]) => ({ sessionId, maxEntryId })) + .sort((left, right) => + left.sessionId < right.sessionId ? -1 : left.sessionId > right.sessionId ? 1 : 0 + ) +} + +export function serializeDeepChatTapeReadSources( + sources: readonly DeepChatTapeReadSource[] +): string { + return JSON.stringify(normalizeDeepChatTapeReadSources(sources)) +} diff --git a/src/main/tape/domain/facts.ts b/src/main/tape/domain/facts.ts new file mode 100644 index 0000000000..2c30b6abbe --- /dev/null +++ b/src/main/tape/domain/facts.ts @@ -0,0 +1,29 @@ +import type { AssistantMessageBlock } from '@shared/types/agent-interface' + +declare const tapeSessionIdBrand: unique symbol + +export type TapeSessionId = string & { readonly [tapeSessionIdBrand]: 'TapeSessionId' } + +export const toTapeSessionId = (value: string): TapeSessionId => value as TapeSessionId + +export interface TapeEntryRef { + sessionId: TapeSessionId + entryId: number +} + +export interface TapeFactProvenance { + source: 'message' | 'tool_call' | 'tool_result' | 'runtime_event' + sourceId: string + sequence: number +} + +export interface TapeToolFactInput { + sessionId: TapeSessionId + messageId: string + orderSeq: number + blockIndex: number + block: AssistantMessageBlock + provenance: TapeFactProvenance +} + +export type TapeFactSource = 'live' | 'backfill' | 'repair' diff --git a/src/main/tape/domain/replay.ts b/src/main/tape/domain/replay.ts new file mode 100644 index 0000000000..c9b5e75b37 --- /dev/null +++ b/src/main/tape/domain/replay.ts @@ -0,0 +1,193 @@ +import { createHash } from 'crypto' +import type { + DeepChatTapeViewExcludedRange, + DeepChatTapeViewManifest +} from '@shared/types/tape-view-manifest' +import type { DeepChatTapeReplaySlice } from '@shared/types/tape-replay' +import { hashJson } from './viewManifest' + +const VIEW_POLICIES = new Set([ + 'legacy_context_v1', + 'legacy_context_shadow', + 'resume_shadow', + 'tool_loop_shadow', + 'context_pressure_recovery_shadow' +]) + +const VIEW_ENTRY_REASONS = new Set([ + 'system_prompt', + 'selected_history', + 'new_user_input', + 'resume_target', + 'tool_loop_message' +]) + +const VIEW_EXCLUDED_REASONS = new Set([ + 'before_summary_cursor', + 'compaction_indicator', + 'pending_not_context_history', + 'out_of_budget', + 'empty_after_formatting', + 'superseded', + 'retracted' +]) + +function isRecordObject(value: unknown): value is Record { + return Boolean(value && typeof value === 'object' && !Array.isArray(value)) +} + +function isNullableString(value: unknown): value is string | null { + return value === null || typeof value === 'string' +} + +function isNullableNumber(value: unknown): value is number | null { + return value === null || typeof value === 'number' +} + +function isViewEntryRef(value: unknown): value is DeepChatTapeViewManifest['included'][number] { + if (!isRecordObject(value)) return false + + return ( + isNullableNumber(value.entryId) && + isNullableString(value.messageId) && + isNullableNumber(value.orderSeq) && + (value.role === 'system' || + value.role === 'user' || + value.role === 'assistant' || + value.role === 'tool' || + value.role === null) && + (value.source === 'tape' || value.source === 'synthetic') && + typeof value.reason === 'string' && + VIEW_ENTRY_REASONS.has(value.reason) + ) +} + +function isViewExcludedRef(value: unknown): value is DeepChatTapeViewManifest['excluded'][number] { + if (!isRecordObject(value)) return false + + return ( + isNullableNumber(value.entryId) && + isNullableString(value.messageId) && + isNullableNumber(value.orderSeq) && + typeof value.reason === 'string' && + VIEW_EXCLUDED_REASONS.has(value.reason) + ) +} + +function isViewExcludedRange(value: unknown): value is DeepChatTapeViewExcludedRange { + if (!isRecordObject(value)) return false + + return ( + typeof value.fromOrderSeq === 'number' && + typeof value.toOrderSeq === 'number' && + typeof value.count === 'number' && + typeof value.reason === 'string' && + VIEW_EXCLUDED_REASONS.has(value.reason) + ) +} + +function hasNumberFields(value: unknown, fields: string[]): value is Record { + if (!isRecordObject(value)) return false + return fields.every((field) => typeof value[field] === 'number') +} + +function hasStringFields(value: unknown, fields: string[]): value is Record { + if (!isRecordObject(value)) return false + return fields.every((field) => typeof value[field] === 'string') +} + +function isViewManifestMeta(value: unknown): value is DeepChatTapeViewManifest['meta'] { + if (!isRecordObject(value)) return false + + return ( + typeof value.providerId === 'string' && + typeof value.modelId === 'string' && + typeof value.summaryCursorOrderSeq === 'number' && + typeof value.supportsVision === 'boolean' && + typeof value.supportsAudioInput === 'boolean' && + typeof value.traceDebugEnabled === 'boolean' + ) +} + +export function isTapeViewManifest( + value: unknown, + sessionId: string +): value is DeepChatTapeViewManifest { + if (!isRecordObject(value)) return false + + return ( + (value.schemaVersion === 1 || value.schemaVersion === 2) && + typeof value.hashVersion === 'number' && + value.sessionId === sessionId && + typeof value.viewId === 'string' && + typeof value.messageId === 'string' && + typeof value.requestSeq === 'number' && + (value.taskType === 'chat' || value.taskType === 'resume' || value.taskType === 'tool_loop') && + typeof value.policy === 'string' && + VIEW_POLICIES.has(value.policy) && + (typeof value.policyVersion === 'number' || value.policyVersion === null) && + value.contextBuilderVersion === 'legacy-v1' && + typeof value.latestEntryId === 'number' && + Array.isArray(value.anchorEntryIds) && + value.anchorEntryIds.every((entryId) => typeof entryId === 'number') && + (value.reconstructionAnchorEntryId === undefined || + isNullableNumber(value.reconstructionAnchorEntryId)) && + (value.excludedRanges === undefined || + (Array.isArray(value.excludedRanges) && value.excludedRanges.every(isViewExcludedRange))) && + Array.isArray(value.included) && + value.included.every(isViewEntryRef) && + Array.isArray(value.excluded) && + value.excluded.every(isViewExcludedRef) && + hasNumberFields(value.tokenBudget, [ + 'contextLength', + 'requestedMaxTokens', + 'effectiveMaxTokens', + 'reserveTokens', + 'toolReserveTokens', + 'estimatedPromptTokens' + ]) && + hasStringFields(value.hashes, ['promptHash', 'toolDefinitionsHash', 'manifestHash']) && + isViewManifestMeta(value.meta) && + typeof value.assembledAt === 'number' + ) +} + +export function normalizeStoredTapeViewManifest( + value: unknown, + sessionId: string +): DeepChatTapeViewManifest | null { + const candidate = + isRecordObject(value) && value.hashVersion === undefined ? { ...value, hashVersion: 1 } : value + return isTapeViewManifest(candidate, sessionId) ? candidate : null +} + +export function hashString(value: string): string { + return createHash('sha256').update(value).digest('hex') +} + +export function isPositiveInteger(value: number): boolean { + return Number.isInteger(value) && value > 0 +} + +export function collectEntryIds(values: Array): number[] { + return [...new Set(values.filter((value): value is number => typeof value === 'number'))].sort( + (left, right) => left - right + ) +} + +export function withReplaySliceHash( + slice: Omit & { + hashes: Omit & { sliceHash: '' } + } +): DeepChatTapeReplaySlice { + const sliceForHash = { ...slice } as Partial + delete sliceForHash.createdAt + delete sliceForHash.integrity + return { + ...slice, + hashes: { + ...slice.hashes, + sliceHash: hashJson(sliceForHash) + } + } +} diff --git a/src/main/tape/domain/viewManifest.ts b/src/main/tape/domain/viewManifest.ts new file mode 100644 index 0000000000..9f4ac268f0 --- /dev/null +++ b/src/main/tape/domain/viewManifest.ts @@ -0,0 +1,363 @@ +import { createHash } from 'crypto' +import type { ChatMessage } from '@shared/types/core/chat-message' +import type { MCPToolDefinition } from '@shared/types/core/mcp' +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { + DeepChatTapeViewEntryRef, + DeepChatTapeViewExcludedRange, + DeepChatTapeViewExcludedRef, + DeepChatTapeViewManifest, + DeepChatTapeViewManifestIntegrity, + DeepChatTapeViewPolicy, + DeepChatTapeViewTaskType, + DeepChatTapeViewTokenBudget +} from '@shared/types/tape-view-manifest' +import { estimateMessagesTokens } from '@shared/utils/messageTokens' + +export type ContextSummaryCursorMetadata = { + summaryCursorOrderSeq: number + preCursorOrderSeqMin: number | null + preCursorOrderSeqMax: number | null + preCursorCount: number +} + +export function isCompactionRecord(record: ChatMessageRecord): boolean { + try { + const metadata = JSON.parse(record.metadata) as { messageType?: string } + return metadata.messageType === 'compaction' + } catch { + return false + } +} + +/** Stable event name persisted for deterministic view reconstruction. */ +export const TAPE_VIEW_MANIFEST_EVENT_NAME = 'view/assembled' +export const TAPE_VIEW_CONTEXT_BUILDER_VERSION = 'legacy-v1' as const + +export type TapeViewManifestLookupMaps = { + entryIdByMessageId?: Map + toolCallEntryIdByToolId?: Map + toolResultEntryIdByToolId?: Map +} + +export type TapeViewManifestBuildInput = { + sessionId: string + messageId: string + requestSeq: number + taskType: DeepChatTapeViewTaskType + policy: DeepChatTapeViewPolicy + policyVersion?: number | null + messages: ChatMessage[] + tools: MCPToolDefinition[] + latestEntryId: number + anchorEntryIds: number[] + reconstructionAnchorEntryId?: number | null + included: DeepChatTapeViewEntryRef[] + excluded: DeepChatTapeViewExcludedRef[] + summaryCursor?: ContextSummaryCursorMetadata + tokenBudget: Omit + providerId: string + modelId: string + summaryCursorOrderSeq: number + supportsVision: boolean + supportsAudioInput: boolean + traceDebugEnabled: boolean + assembledAt?: number +} + +export type TapeViewManifestPolicyInput = { + recoveredFromContextPressure: boolean + isInitialViewRequest: boolean + viewPolicy?: DeepChatTapeViewPolicy + viewPolicyVersion?: number | null +} + +export type TapeViewManifestPolicyResult = { + policy: DeepChatTapeViewPolicy + policyVersion: number | null +} + +export type TapeViewContextSelection = { + includedRecords: Array<{ + record: ChatMessageRecord + reason: DeepChatTapeViewEntryRef['reason'] + }> + excludedRecords: Array<{ + record: ChatMessageRecord + reason: DeepChatTapeViewExcludedRef['reason'] + }> + summaryCursor?: ContextSummaryCursorMetadata + includesSystemPrompt: boolean + newUserMessageId?: string | null +} + +export function resolveTapeViewManifestPolicy( + input: TapeViewManifestPolicyInput +): TapeViewManifestPolicyResult { + if (input.recoveredFromContextPressure) { + return { + policy: 'context_pressure_recovery_shadow', + policyVersion: null + } + } + + if (input.isInitialViewRequest && input.viewPolicy) { + return { + policy: input.viewPolicy, + policyVersion: input.viewPolicyVersion ?? null + } + } + + return { + policy: 'tool_loop_shadow', + policyVersion: null + } +} + +function normalizeForStableJson(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(normalizeForStableJson) + } + + if (!value || typeof value !== 'object') { + return value + } + + const record = value as Record + return Object.keys(record) + .sort() + .reduce>((result, key) => { + const nested = record[key] + if (nested !== undefined) { + result[key] = normalizeForStableJson(nested) + } + return result + }, {}) +} + +export function stableJsonStringify(value: unknown): string { + return JSON.stringify(normalizeForStableJson(value)) +} + +export function hashJson(value: unknown): string { + return createHash('sha256').update(stableJsonStringify(value)).digest('hex') +} + +export const TAPE_VIEW_MANIFEST_HASH_VERSION = 2 + +function buildManifestHash(manifest: DeepChatTapeViewManifest): string { + const hashable: Record = { ...manifest } + delete hashable.assembledAt + delete hashable.viewId + hashable.hashes = { + promptHash: manifest.hashes.promptHash, + toolDefinitionsHash: manifest.hashes.toolDefinitionsHash + } + return hashJson(hashable) +} + +function buildExcludedRanges( + summaryCursor?: ContextSummaryCursorMetadata +): DeepChatTapeViewExcludedRange[] { + if ( + !summaryCursor || + summaryCursor.preCursorCount === 0 || + summaryCursor.preCursorOrderSeqMin === null || + summaryCursor.preCursorOrderSeqMax === null + ) { + return [] + } + return [ + { + fromOrderSeq: summaryCursor.preCursorOrderSeqMin, + toOrderSeq: summaryCursor.preCursorOrderSeqMax, + count: summaryCursor.preCursorCount, + reason: 'before_summary_cursor' + } + ] +} + +export function verifyTapeViewManifestHash( + manifest: DeepChatTapeViewManifest +): DeepChatTapeViewManifestIntegrity { + if (manifest.hashVersion !== TAPE_VIEW_MANIFEST_HASH_VERSION) { + return 'unverified' + } + return buildManifestHash(manifest) === manifest.hashes.manifestHash ? 'valid' : 'invalid' +} + +export function createTapeViewManifest( + input: TapeViewManifestBuildInput +): DeepChatTapeViewManifest { + const assembledAt = input.assembledAt ?? Date.now() + const excludedRanges = buildExcludedRanges(input.summaryCursor) + const draft: DeepChatTapeViewManifest = { + schemaVersion: 2, + hashVersion: TAPE_VIEW_MANIFEST_HASH_VERSION, + viewId: '', + sessionId: input.sessionId, + messageId: input.messageId, + requestSeq: input.requestSeq, + taskType: input.taskType, + policy: input.policy, + policyVersion: input.policyVersion ?? null, + contextBuilderVersion: TAPE_VIEW_CONTEXT_BUILDER_VERSION, + latestEntryId: input.latestEntryId, + anchorEntryIds: [...input.anchorEntryIds], + ...(input.reconstructionAnchorEntryId !== undefined + ? { reconstructionAnchorEntryId: input.reconstructionAnchorEntryId } + : {}), + included: input.included.map((entry) => ({ ...entry })), + excluded: input.excluded.map((entry) => ({ ...entry })), + ...(excludedRanges.length > 0 ? { excludedRanges } : {}), + tokenBudget: { + ...input.tokenBudget, + estimatedPromptTokens: estimateMessagesTokens(input.messages) + }, + hashes: { + promptHash: hashJson(input.messages), + toolDefinitionsHash: hashJson(input.tools), + manifestHash: '' + }, + meta: { + providerId: input.providerId, + modelId: input.modelId, + summaryCursorOrderSeq: input.summaryCursorOrderSeq, + supportsVision: input.supportsVision, + supportsAudioInput: input.supportsAudioInput, + traceDebugEnabled: input.traceDebugEnabled + }, + assembledAt + } + + const manifestHash = buildManifestHash(draft) + return { + ...draft, + viewId: `view_${manifestHash.slice(0, 16)}`, + hashes: { ...draft.hashes, manifestHash } + } +} + +export function buildIncludedRefs( + selection: TapeViewContextSelection, + sourceMaps: TapeViewManifestLookupMaps = {} +): DeepChatTapeViewEntryRef[] { + const refs: DeepChatTapeViewEntryRef[] = [] + + if (selection.includesSystemPrompt) { + refs.push({ + entryId: null, + messageId: null, + orderSeq: null, + role: 'system', + source: 'synthetic', + reason: 'system_prompt' + }) + } + + for (const item of selection.includedRecords) { + refs.push({ + entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, + messageId: item.record.id, + orderSeq: item.record.orderSeq, + role: item.record.role, + source: sourceMaps.entryIdByMessageId?.has(item.record.id) ? 'tape' : 'synthetic', + reason: item.reason + }) + } + + if (selection.newUserMessageId) { + refs.push({ + entryId: sourceMaps.entryIdByMessageId?.get(selection.newUserMessageId) ?? null, + messageId: selection.newUserMessageId, + orderSeq: null, + role: 'user', + source: sourceMaps.entryIdByMessageId?.has(selection.newUserMessageId) ? 'tape' : 'synthetic', + reason: 'new_user_input' + }) + } + + return refs +} + +export function buildExcludedRefs( + selection: TapeViewContextSelection, + sourceMaps: TapeViewManifestLookupMaps = {} +): DeepChatTapeViewExcludedRef[] { + return selection.excludedRecords.map((item) => ({ + entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, + messageId: item.record.id, + orderSeq: item.record.orderSeq, + reason: item.reason + })) +} + +export function buildRequestRefs( + messages: ChatMessage[], + sourceMaps: TapeViewManifestLookupMaps = {} +): DeepChatTapeViewEntryRef[] { + const lastToolCallIndex = new Map() + const lastToolResultIndex = new Map() + messages.forEach((message, index) => { + if (message.role === 'assistant' && message.tool_calls?.length) { + for (const toolCall of message.tool_calls) { + lastToolCallIndex.set(toolCall.id, index) + } + } else if (message.role === 'tool' && message.tool_call_id) { + lastToolResultIndex.set(message.tool_call_id, index) + } + }) + + const refs: DeepChatTapeViewEntryRef[] = [] + messages.forEach((message, index) => { + if (message.role === 'assistant' && message.tool_calls?.length) { + for (const toolCall of message.tool_calls) { + const entryId = + lastToolCallIndex.get(toolCall.id) === index + ? (sourceMaps.toolCallEntryIdByToolId?.get(toolCall.id) ?? null) + : null + refs.push({ + entryId, + messageId: null, + orderSeq: null, + role: 'assistant', + source: entryId === null ? 'synthetic' : 'tape', + reason: 'tool_loop_message' + }) + } + return + } + + if (message.role === 'tool' && message.tool_call_id) { + const entryId = + lastToolResultIndex.get(message.tool_call_id) === index + ? (sourceMaps.toolResultEntryIdByToolId?.get(message.tool_call_id) ?? null) + : null + refs.push({ + entryId, + messageId: null, + orderSeq: null, + role: 'tool', + source: entryId === null ? 'synthetic' : 'tape', + reason: 'tool_loop_message' + }) + return + } + + refs.push({ + entryId: null, + messageId: null, + orderSeq: null, + role: message.role, + source: 'synthetic', + reason: + message.role === 'system' + ? 'system_prompt' + : message.role === 'tool' + ? 'tool_loop_message' + : 'selected_history' + }) + }) + + return refs +} diff --git a/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts b/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts new file mode 100644 index 0000000000..1965e58aaf --- /dev/null +++ b/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts @@ -0,0 +1,1070 @@ +import Database from 'better-sqlite3-multiple-ciphers' +import logger from '@shared/logger' +import { BaseTable } from '@/data/baseTable' +import { randomUUID } from 'crypto' +import { + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources, + SUMMARY_ANCHOR_NAMES, + TAPE_INCARNATION_META_KEY, + type DeepChatTapeAppendInput, + type DeepChatTapeEntryRow, + type DeepChatTapeReadSource, + type DeepChatTapeSearchInput, + type TapeAnchorAppendInput, + type TapeEventAppendInput +} from '@/tape/domain/entry' +import type { + TapeBootstrapStore, + TapeEntryStore, + TapeMutationProjection, + TapeTransactionRunner +} from '@/tape/ports/storage' + +export { + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources, + SUMMARY_ANCHOR_NAMES, + TAPE_INCARNATION_META_KEY +} from '@/tape/domain/entry' +export type { + DeepChatTapeAppendInput, + DeepChatTapeEntryKind, + DeepChatTapeEntryRow, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + DeepChatTapeSourceInput, + DeepChatTapeSourceType, + TapeAnchorAppendInput, + TapeEventAppendInput +} from '@/tape/domain/entry' + +export type DeepChatTapeMutationProjection = TapeMutationProjection + +const RECONSTRUCTION_ANCHOR_NAMES = SUMMARY_ANCHOR_NAMES + +const TAPE_ENTRY_INDEX_SQL = ` + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_kind + ON deepchat_tape_entries(session_id, kind, entry_id); + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_name + ON deepchat_tape_entries(session_id, name, entry_id); + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_source + ON deepchat_tape_entries(session_id, source_type, source_id, source_seq); + CREATE UNIQUE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_provenance + ON deepchat_tape_entries(session_id, provenance_key) + WHERE provenance_key IS NOT NULL; +` + +function safeJsonStringify(value: Record | undefined): string { + return JSON.stringify(value ?? {}) +} + +function buildProvenanceKey(input: DeepChatTapeAppendInput): string | null { + if (input.provenanceKey !== undefined) { + return input.provenanceKey + } + if (!input.source?.type || !input.source.id) { + return null + } + return [ + input.source.type, + input.source.id, + input.source.seq ?? 0, + input.kind, + input.name ?? '' + ].join(':') +} + +function escapeLikePattern(value: string): string { + return value.replace(/[\\%_]/g, (character) => `\\${character}`) +} + +// Three LIKE parameters per field group stay below SQLite's portable 999-variable floor after +// source, filter, and limit bindings. +const MAX_TAPE_SEARCH_TOKEN_CLAUSES = 256 + +function tokenizeDeepChatTapeSearchQuery(value: string): string[] { + return value + .split(/\s+/) + .map((token) => token.trim()) + .filter(Boolean) +} + +export function buildDeepChatTapeFtsMatch(value: string): string { + const tokens = tokenizeDeepChatTapeSearchQuery(value) + const values = + tokens.length > 1 && tokens.length <= MAX_TAPE_SEARCH_TOKEN_CLAUSES ? tokens : [value] + return values.map((token) => `"${token.replace(/"/g, '""')}"`).join(' AND ') +} + +export function buildDeepChatTapeLikeSearchPredicate( + fieldExpressions: readonly [string, ...string[]], + normalizedQuery: string +): { sql: string; params: string[] } { + const fieldClause = `(${fieldExpressions + .map((field) => `${field} LIKE ? ESCAPE '\\'`) + .join(' OR ')})` + const queryClauses = [fieldClause] + const queryPattern = `%${escapeLikePattern(normalizedQuery)}%` + const params = fieldExpressions.map(() => queryPattern) + const tokens = tokenizeDeepChatTapeSearchQuery(normalizedQuery) + + if (tokens.length > 1 && tokens.length <= MAX_TAPE_SEARCH_TOKEN_CLAUSES) { + queryClauses.push(`(${tokens.map(() => fieldClause).join(' AND ')})`) + for (const token of tokens) { + const tokenPattern = `%${escapeLikePattern(token)}%` + params.push(...fieldExpressions.map(() => tokenPattern)) + } + } + + return { + sql: `(${queryClauses.join(' OR ')})`, + params + } +} + +const AUTHORIZED_TAPE_SOURCES_CTE_SQL = ` + authorized_sources(session_id, max_entry_id) AS ( + SELECT + json_extract(value, '$.sessionId'), + CAST(json_extract(value, '$.maxEntryId') AS INTEGER) + FROM json_each(?) + ) +` + +function effectiveTapeMessagePredicateSql(alias: string): string { + return ` + json_type(${alias}.payload_json, '$.record') = 'object' + AND typeof(json_extract(${alias}.payload_json, '$.record.id')) = 'text' + AND typeof(json_extract(${alias}.payload_json, '$.record.sessionId')) = 'text' + AND typeof(json_extract(${alias}.payload_json, '$.record.orderSeq')) IN ('integer', 'real') + AND json_extract(${alias}.payload_json, '$.record.role') IN ('user', 'assistant') + AND typeof(json_extract(${alias}.payload_json, '$.record.content')) = 'text' + AND ( + json_extract(${alias}.payload_json, '$.record.status') IS NULL + OR json_extract(${alias}.payload_json, '$.record.status') != 'pending' + ) + ` +} + +function tapeRetractionMessageIdSql(alias: string): string { + return ` + CASE + WHEN json_type(${alias}.payload_json, '$.data') = 'object' + THEN json_extract(${alias}.payload_json, '$.data.messageId') + WHEN json_type(${alias}.payload_json, '$.data') = 'text' + AND json_valid(json_extract(${alias}.payload_json, '$.data')) + THEN json_extract(json_extract(${alias}.payload_json, '$.data'), '$.messageId') + ELSE NULL + END + ` +} + +function tapeToolCallIdSql(alias: string): string { + return ` + CASE + WHEN ${alias}.kind = 'tool_result' + THEN json_extract(${alias}.payload_json, '$.toolCallId') + WHEN json_type(${alias}.payload_json, '$.toolCall') = 'object' + THEN json_extract(${alias}.payload_json, '$.toolCall.id') + WHEN json_type(${alias}.payload_json, '$.toolCall') = 'text' + AND json_valid(json_extract(${alias}.payload_json, '$.toolCall')) + THEN json_extract(json_extract(${alias}.payload_json, '$.toolCall'), '$.id') + ELSE NULL + END + ` +} + +// These read-only SQL forms mirror deepchatTapeEffectiveSemantics. Search uses correlated +// candidate validation to avoid materializing a whole linked Tape; context uses the complete +// effective CTE because it needs stable neighboring-row positions. Native tests cover parity for +// replacement, retraction, head cutoff, and window ordering. +const EFFECTIVE_TAPE_ROWS_CTE_SQL = ` + bounded_rows AS ( + SELECT tape.* + FROM deepchat_tape_entries AS tape + INNER JOIN authorized_sources AS source + ON source.session_id = tape.session_id + AND tape.entry_id <= source.max_entry_id + ), + raw_message_candidates AS ( + SELECT + bounded_rows.*, + json_extract(payload_json, '$.record.id') AS message_id + FROM bounded_rows + WHERE kind = 'message' + ), + message_candidates AS ( + SELECT * + FROM raw_message_candidates + WHERE ${effectiveTapeMessagePredicateSql('raw_message_candidates')} + ), + ranked_messages AS ( + SELECT + message_candidates.*, + ROW_NUMBER() OVER ( + PARTITION BY session_id, message_id + ORDER BY entry_id DESC + ) AS candidate_rank + FROM message_candidates + ), + effective_message_rows AS ( + SELECT ranked_messages.* + FROM ranked_messages + WHERE candidate_rank = 1 + AND NOT EXISTS ( + SELECT 1 + FROM bounded_rows AS retraction + WHERE retraction.session_id = ranked_messages.session_id + AND retraction.kind = 'event' + AND retraction.name = 'message/retracted' + AND retraction.entry_id > ranked_messages.entry_id + AND (${tapeRetractionMessageIdSql('retraction')}) = ranked_messages.message_id + ) + ), + raw_tool_candidates AS ( + SELECT + bounded_rows.*, + json_extract(payload_json, '$.messageId') AS message_id, + (${tapeToolCallIdSql('bounded_rows')}) AS tool_call_id, + json_extract(meta_json, '$.status') AS tool_status + FROM bounded_rows + WHERE kind IN ('tool_call', 'tool_result') + ), + tool_candidates AS ( + SELECT * + FROM raw_tool_candidates + WHERE typeof(message_id) = 'text' + AND length(message_id) > 0 + AND typeof(tool_call_id) = 'text' + AND length(tool_call_id) > 0 + AND tool_status IN ('success', 'error') + ), + ranked_tools AS ( + SELECT + tool_candidates.*, + ROW_NUMBER() OVER ( + PARTITION BY session_id, kind, message_id, tool_call_id + ORDER BY entry_id DESC + ) AS candidate_rank + FROM tool_candidates + ), + effective_rows AS ( + SELECT + session_id, entry_id, kind, name, source_type, source_id, source_seq, + provenance_key, payload_json, meta_json, created_at + FROM bounded_rows + WHERE kind = 'anchor' + UNION ALL + SELECT + session_id, entry_id, kind, name, source_type, source_id, source_seq, + provenance_key, payload_json, meta_json, created_at + FROM bounded_rows + WHERE kind = 'event' + AND (name IS NULL OR name NOT IN ( + 'message/retracted', + 'message/compaction_indicator', + 'migration/backfill' + )) + UNION ALL + SELECT + session_id, entry_id, kind, name, source_type, source_id, source_seq, + provenance_key, payload_json, meta_json, created_at + FROM effective_message_rows + UNION ALL + SELECT + ranked_tools.session_id, + ranked_tools.entry_id, + ranked_tools.kind, + ranked_tools.name, + ranked_tools.source_type, + ranked_tools.source_id, + ranked_tools.source_seq, + ranked_tools.provenance_key, + ranked_tools.payload_json, + ranked_tools.meta_json, + ranked_tools.created_at + FROM ranked_tools + INNER JOIN effective_message_rows AS message + ON message.session_id = ranked_tools.session_id + AND message.message_id = ranked_tools.message_id + WHERE ranked_tools.candidate_rank = 1 + ) +` + +const EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL = ` + candidate.kind = 'anchor' + OR ( + candidate.kind = 'event' + AND (candidate.name IS NULL OR candidate.name NOT IN ( + 'message/retracted', + 'message/compaction_indicator', + 'migration/backfill' + )) + ) + OR ( + candidate.kind = 'message' + AND ${effectiveTapeMessagePredicateSql('candidate')} + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS later_message + WHERE later_message.session_id = candidate.session_id + AND later_message.entry_id > candidate.entry_id + AND later_message.entry_id <= source.max_entry_id + AND later_message.kind = 'message' + AND ${effectiveTapeMessagePredicateSql('later_message')} + AND json_extract(later_message.payload_json, '$.record.id') = + json_extract(candidate.payload_json, '$.record.id') + ) + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS retraction + WHERE retraction.session_id = candidate.session_id + AND retraction.entry_id > candidate.entry_id + AND retraction.entry_id <= source.max_entry_id + AND retraction.kind = 'event' + AND retraction.name = 'message/retracted' + AND (${tapeRetractionMessageIdSql('retraction')}) = + json_extract(candidate.payload_json, '$.record.id') + ) + ) + OR ( + candidate.kind IN ('tool_call', 'tool_result') + AND json_extract(candidate.meta_json, '$.status') IN ('success', 'error') + AND typeof(json_extract(candidate.payload_json, '$.messageId')) = 'text' + AND length(json_extract(candidate.payload_json, '$.messageId')) > 0 + AND typeof((${tapeToolCallIdSql('candidate')})) = 'text' + AND length((${tapeToolCallIdSql('candidate')})) > 0 + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS later_tool + WHERE later_tool.session_id = candidate.session_id + AND later_tool.entry_id > candidate.entry_id + AND later_tool.entry_id <= source.max_entry_id + AND later_tool.kind = candidate.kind + AND json_extract(later_tool.meta_json, '$.status') IN ('success', 'error') + AND json_extract(later_tool.payload_json, '$.messageId') = + json_extract(candidate.payload_json, '$.messageId') + AND (${tapeToolCallIdSql('later_tool')}) = (${tapeToolCallIdSql('candidate')}) + ) + AND EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS message + WHERE message.session_id = candidate.session_id + AND message.entry_id <= source.max_entry_id + AND message.kind = 'message' + AND ${effectiveTapeMessagePredicateSql('message')} + AND json_extract(message.payload_json, '$.record.id') = + json_extract(candidate.payload_json, '$.messageId') + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS later_message + WHERE later_message.session_id = message.session_id + AND later_message.entry_id > message.entry_id + AND later_message.entry_id <= source.max_entry_id + AND later_message.kind = 'message' + AND ${effectiveTapeMessagePredicateSql('later_message')} + AND json_extract(later_message.payload_json, '$.record.id') = + json_extract(message.payload_json, '$.record.id') + ) + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS retraction + WHERE retraction.session_id = message.session_id + AND retraction.entry_id > message.entry_id + AND retraction.entry_id <= source.max_entry_id + AND retraction.kind = 'event' + AND retraction.name = 'message/retracted' + AND (${tapeRetractionMessageIdSql('retraction')}) = + json_extract(message.payload_json, '$.record.id') + ) + ) + ) +` + +export class DeepChatTapeEntriesTable + extends BaseTable + implements TapeEntryStore, TapeTransactionRunner, TapeBootstrapStore +{ + constructor( + db: Database.Database, + private readonly mutationProjection?: DeepChatTapeMutationProjection + ) { + super(db, 'deepchat_tape_entries') + } + + getCreateTableSQL(): string { + return ` + CREATE TABLE IF NOT EXISTS deepchat_tape_entries ( + session_id TEXT NOT NULL, + entry_id INTEGER NOT NULL, + kind TEXT NOT NULL, + name TEXT, + source_type TEXT, + source_id TEXT, + source_seq INTEGER, + provenance_key TEXT, + payload_json TEXT NOT NULL DEFAULT '{}', + meta_json TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, entry_id) + ); + ${TAPE_ENTRY_INDEX_SQL} + ` + } + + public createTable(): void { + if (!this.tableExists()) { + this.db.exec(this.getCreateTableSQL()) + return + } + this.ensureProvenanceColumns() + this.db.exec(TAPE_ENTRY_INDEX_SQL) + } + + getMigrationSQL(_version: number): string | null { + return null + } + + getLatestVersion(): number { + return 0 + } + + runInTransaction(operation: () => T): T { + return this.db.transaction(operation)() + } + + append(input: DeepChatTapeAppendInput): DeepChatTapeEntryRow { + const append = this.db.transaction(() => { + const provenanceKey = buildProvenanceKey(input) + if (input.idempotent && provenanceKey) { + const existing = this.getByProvenanceKey(input.sessionId, provenanceKey) + if (existing) { + return existing + } + } + + const createdAt = input.createdAt ?? Date.now() + const previousSessionMaxEntryId = this.getMaxEntryId(input.sessionId) + const row = { + session_id: input.sessionId, + entry_id: previousSessionMaxEntryId + 1, + kind: input.kind, + name: input.name ?? null, + source_type: input.source?.type ?? null, + source_id: input.source?.id ?? null, + source_seq: input.source?.seq ?? null, + provenance_key: provenanceKey, + payload_json: safeJsonStringify(input.payload), + meta_json: safeJsonStringify(input.meta), + created_at: createdAt + } satisfies DeepChatTapeEntryRow + + try { + this.db + .prepare( + `INSERT INTO deepchat_tape_entries ( + session_id, + entry_id, + kind, + name, + source_type, + source_id, + source_seq, + provenance_key, + payload_json, + meta_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run( + row.session_id, + row.entry_id, + row.kind, + row.name, + row.source_type, + row.source_id, + row.source_seq, + row.provenance_key, + row.payload_json, + row.meta_json, + row.created_at + ) + } catch (error) { + if (input.idempotent && provenanceKey) { + const existing = this.getByProvenanceKey(input.sessionId, provenanceKey) + if (existing) { + return existing + } + } + throw error + } + + if (this.mutationProjection) { + try { + const applyProjection = this.db.transaction(() => + this.mutationProjection?.applyAppendedEntry(row, previousSessionMaxEntryId) + ) + applyProjection() + } catch (error) { + this.mutationProjection.invalidateSession(row.session_id) + logger.warn( + `[Tape] memory ingestion projection append failed; session marked stale: ${String(error)}` + ) + } + } + + return row + }) + + return append() + } + + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow { + return this.append({ + sessionId: input.sessionId, + kind: 'anchor', + name: input.name, + source: input.source, + provenanceKey: input.provenanceKey, + payload: { + name: input.name, + state: input.state + }, + meta: input.meta, + createdAt: input.createdAt, + idempotent: input.idempotent + }) + } + + appendEvent(input: TapeEventAppendInput): DeepChatTapeEntryRow { + return this.append({ + sessionId: input.sessionId, + kind: 'event', + name: input.name, + source: input.source, + provenanceKey: input.provenanceKey, + payload: { + name: input.name, + data: input.data + }, + meta: input.meta, + createdAt: input.createdAt, + idempotent: input.idempotent + }) + } + + ensureBootstrapAnchor(sessionId: string): void { + const existing = this.db + .prepare( + `SELECT entry_id + FROM deepchat_tape_entries + WHERE session_id = ? AND kind = 'anchor' + ORDER BY entry_id ASC + LIMIT 1` + ) + .get(sessionId) as { entry_id: number } | undefined + + if (existing) { + return + } + + this.appendAnchor({ + sessionId, + name: 'session/start', + source: { + type: 'session', + id: sessionId, + seq: 0 + }, + state: { + owner: 'human' + }, + meta: { + [TAPE_INCARNATION_META_KEY]: randomUUID() + }, + idempotent: true + }) + } + + getBySession(sessionId: string): DeepChatTapeEntryRow[] { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? + ORDER BY entry_id ASC` + ) + .all(sessionId) as DeepChatTapeEntryRow[] + } + + getSubagentLineageEvents(sessionId: string): DeepChatTapeEntryRow[] { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? + AND kind = 'event' + AND name IN ('subagent/tape_linked', 'fork/merge') + ORDER BY entry_id ASC` + ) + .all(sessionId) as DeepChatTapeEntryRow[] + } + + getFirstEntriesBySessions(sessionIds: string[]): DeepChatTapeEntryRow[] { + const ids = [...new Set(sessionIds.map((id) => id.trim()).filter(Boolean))] + if (ids.length === 0) { + return [] + } + return this.db + .prepare( + `WITH requested_sessions(session_id) AS ( + SELECT value FROM json_each(?) + ), + first_entries(session_id, entry_id) AS ( + SELECT tape.session_id, MIN(tape.entry_id) + FROM deepchat_tape_entries AS tape + INNER JOIN requested_sessions AS requested + ON requested.session_id = tape.session_id + GROUP BY tape.session_id + ) + SELECT tape.* + FROM deepchat_tape_entries AS tape + INNER JOIN first_entries AS first + ON first.session_id = tape.session_id + AND first.entry_id = tape.entry_id + ORDER BY tape.session_id ASC` + ) + .all(JSON.stringify(ids)) as DeepChatTapeEntryRow[] + } + + getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND entry_id <= ? + ORDER BY entry_id ASC` + ) + .all(sessionId, maxEntryId) as DeepChatTapeEntryRow[] + } + + listMemoryViewManifestAnchorsBySessions( + sessionIds: string[], + optionsOrLimit: number | { limit?: number; messageId?: string } = 100 + ): DeepChatTapeEntryRow[] { + const uniqueSessionIds = [...new Set(sessionIds.filter((id) => id.trim().length > 0))] + if (uniqueSessionIds.length === 0) { + return [] + } + const options = typeof optionsOrLimit === 'number' ? { limit: optionsOrLimit } : optionsOrLimit + const cappedLimit = Math.min(Math.max(Math.floor(options.limit ?? 100), 1), 500) + const placeholders = uniqueSessionIds.map(() => '?').join(', ') + const whereClauses = [ + `session_id IN (${placeholders})`, + "kind = 'anchor'", + "name = 'memory/view_assembled'" + ] + const params: Array = [...uniqueSessionIds] + if (options.messageId) { + whereClauses.push("json_extract(meta_json, '$.messageId') = ?") + params.push(options.messageId) + } + params.push(cappedLimit) + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE ${whereClauses.join(' AND ')} + ORDER BY created_at DESC, entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeEntryRow[] + } + + listMemoryViewManifestAnchorsByAgent( + agentId: string, + options: { sessionId?: string; limit?: number; messageId?: string } = {} + ): DeepChatTapeEntryRow[] { + const cappedLimit = Math.min(Math.max(Math.floor(options.limit ?? 100), 1), 500) + const whereClauses = [ + 'sessions.agent_id = ?', + "tape.kind = 'anchor'", + "tape.name = 'memory/view_assembled'" + ] + const params: Array = [agentId] + if (options.sessionId) { + whereClauses.push('tape.session_id = ?') + params.push(options.sessionId) + } + if (options.messageId) { + whereClauses.push("json_extract(tape.meta_json, '$.messageId') = ?") + params.push(options.messageId) + } + params.push(cappedLimit) + return this.db + .prepare( + `SELECT tape.* + FROM deepchat_tape_entries AS tape + INNER JOIN new_sessions AS sessions + ON sessions.id = tape.session_id + WHERE ${whereClauses.join(' AND ')} + ORDER BY tape.created_at DESC, tape.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeEntryRow[] + } + + getEntriesAfter(sessionId: string, entryId: number): DeepChatTapeEntryRow[] { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND entry_id > ? + ORDER BY entry_id ASC` + ) + .all(sessionId, entryId) as DeepChatTapeEntryRow[] + } + + getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND kind = 'anchor' + ORDER BY entry_id DESC + LIMIT 1` + ) + .get(sessionId) as DeepChatTapeEntryRow | undefined + } + + getAnchors(sessionId: string, limit: number = 20): DeepChatTapeEntryRow[] { + const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) + const rows = this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND kind = 'anchor' + ORDER BY entry_id DESC + LIMIT ?` + ) + .all(sessionId, cappedLimit) as DeepChatTapeEntryRow[] + + return rows.reverse() + } + + getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + const placeholders = SUMMARY_ANCHOR_NAMES.map(() => '?').join(', ') + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? + AND kind = 'anchor' + AND name IN (${placeholders}) + ORDER BY entry_id DESC + LIMIT 1` + ) + .get(sessionId, ...SUMMARY_ANCHOR_NAMES) as DeepChatTapeEntryRow | undefined + } + + getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + const placeholders = RECONSTRUCTION_ANCHOR_NAMES.map(() => '?').join(', ') + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? + AND kind = 'anchor' + AND ( + name IN (${placeholders}) + OR name LIKE 'handoff/%' + OR name LIKE 'auto_handoff/%' + ) + ORDER BY entry_id DESC + LIMIT 1` + ) + .get(sessionId, ...RECONSTRUCTION_ANCHOR_NAMES) as DeepChatTapeEntryRow | undefined + } + + getByProvenanceKey(sessionId: string, provenanceKey: string): DeepChatTapeEntryRow | undefined { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND provenance_key = ? + LIMIT 1` + ) + .get(sessionId, provenanceKey) as DeepChatTapeEntryRow | undefined + } + + getMaxEntryId(sessionId: string): number { + const row = this.db + .prepare( + `SELECT MAX(entry_id) AS max_entry_id + FROM deepchat_tape_entries + WHERE session_id = ?` + ) + .get(sessionId) as { max_entry_id: number | null } | undefined + return row?.max_entry_id ?? 0 + } + + getMaxEntryIdsBySessions(sessionIds: string[]): Map { + const ids = [...new Set(sessionIds.map((id) => id.trim()).filter(Boolean))] + const maxEntryIdBySession = new Map(ids.map((id) => [id, 0])) + if (ids.length === 0) { + return maxEntryIdBySession + } + const rows = this.db + .prepare( + `WITH requested_sessions(session_id) AS ( + SELECT value FROM json_each(?) + ) + SELECT tape.session_id, MAX(tape.entry_id) AS max_entry_id + FROM deepchat_tape_entries AS tape + INNER JOIN requested_sessions AS requested + ON requested.session_id = tape.session_id + GROUP BY tape.session_id` + ) + .all(JSON.stringify(ids)) as Array<{ session_id: string; max_entry_id: number }> + for (const row of rows) { + maxEntryIdBySession.set(row.session_id, row.max_entry_id) + } + return maxEntryIdBySession + } + + countAnchorsBySession(sessionId: string): number { + const row = this.db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_entries + WHERE session_id = ? AND kind = 'anchor'` + ) + .get(sessionId) as { count: number } | undefined + return row?.count ?? 0 + } + + countEntriesAfter(sessionId: string, entryId: number): number { + const row = this.db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_entries + WHERE session_id = ? AND entry_id > ?` + ) + .get(sessionId, entryId) as { count: number } | undefined + return row?.count ?? 0 + } + + countBySession(sessionId: string): number { + const row = this.db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_entries + WHERE session_id = ?` + ) + .get(sessionId) as { count: number } | undefined + return row?.count ?? 0 + } + + search( + sessionId: string, + query: string, + options: DeepChatTapeSearchInput = {} + ): DeepChatTapeEntryRow[] { + const normalizedQuery = query.trim() + if (!normalizedQuery) { + return [] + } + const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 + const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) + const queryPredicate = buildDeepChatTapeLikeSearchPredicate( + ['payload_json', 'meta_json', 'name'], + normalizedQuery + ) + const whereClauses = ['session_id = ?', queryPredicate.sql] + const params: Array = [sessionId, ...queryPredicate.params] + + if (options.kinds?.length) { + whereClauses.push(`kind IN (${options.kinds.map(() => '?').join(', ')})`) + params.push(...options.kinds) + } + + if (Number.isFinite(options.startCreatedAt)) { + whereClauses.push('created_at >= ?') + params.push(options.startCreatedAt as number) + } + + if (Number.isFinite(options.endCreatedAt)) { + whereClauses.push('created_at <= ?') + params.push(options.endCreatedAt as number) + } + + params.push(cappedLimit) + + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE ${whereClauses.join(' AND ')} + ORDER BY entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeEntryRow[] + } + + searchEffectiveSourcesAtHeads( + sources: readonly DeepChatTapeReadSource[], + query: string, + options: DeepChatTapeSearchInput = {} + ): DeepChatTapeEntryRow[] { + const normalizedSources = normalizeDeepChatTapeReadSources(sources) + const normalizedQuery = query.trim() + if (normalizedSources.length === 0 || !normalizedQuery) { + return [] + } + + const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 + const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) + const queryPredicate = buildDeepChatTapeLikeSearchPredicate( + ['candidate.payload_json', 'candidate.meta_json', 'candidate.name'], + normalizedQuery + ) + const whereClauses = [queryPredicate.sql] + const params: Array = [ + serializeDeepChatTapeReadSources(normalizedSources), + ...queryPredicate.params + ] + + if (options.kinds?.length) { + whereClauses.push(`candidate.kind IN (${options.kinds.map(() => '?').join(', ')})`) + params.push(...options.kinds) + } + if (Number.isFinite(options.startCreatedAt)) { + whereClauses.push('candidate.created_at >= ?') + params.push(options.startCreatedAt as number) + } + if (Number.isFinite(options.endCreatedAt)) { + whereClauses.push('candidate.created_at <= ?') + params.push(options.endCreatedAt as number) + } + params.push(cappedLimit) + + return this.db + .prepare( + `WITH + ${AUTHORIZED_TAPE_SOURCES_CTE_SQL} + SELECT candidate.* + FROM deepchat_tape_entries AS candidate + INNER JOIN authorized_sources AS source + ON source.session_id = candidate.session_id + AND candidate.entry_id <= source.max_entry_id + WHERE ${whereClauses.join(' AND ')} + AND (${EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL}) + ORDER BY candidate.created_at DESC, candidate.session_id ASC, candidate.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeEntryRow[] + } + + getEffectiveContextRowsAtHead( + source: DeepChatTapeReadSource, + entryIds: number[], + options: { before: number; after: number; limit: number } + ): DeepChatTapeEntryRow[] { + const normalizedSource = normalizeDeepChatTapeReadSources([source])[0] + const requestedEntryIds = [ + ...new Set(entryIds.filter((entryId) => Number.isSafeInteger(entryId) && entryId > 0)) + ].sort((left, right) => left - right) + if (!normalizedSource || requestedEntryIds.length === 0) { + return [] + } + const before = Math.min(Math.max(Math.floor(options.before), 0), 20) + const after = Math.min(Math.max(Math.floor(options.after), 0), 20) + const limit = Math.min(Math.max(Math.floor(options.limit), 1), 100) + + return this.db + .prepare( + `WITH + ${AUTHORIZED_TAPE_SOURCES_CTE_SQL}, + ${EFFECTIVE_TAPE_ROWS_CTE_SQL}, + ordered_rows AS ( + SELECT + effective_rows.*, + ROW_NUMBER() OVER (ORDER BY entry_id ASC) AS row_position + FROM effective_rows + ), + requested_ids(entry_id, request_ordinal) AS ( + SELECT CAST(value AS INTEGER), CAST(key AS INTEGER) + FROM json_each(?) + ), + requested_positions AS ( + SELECT + requested_ids.request_ordinal, + ordered_rows.entry_id, + ordered_rows.row_position + FROM requested_ids + INNER JOIN ordered_rows + ON ordered_rows.entry_id = requested_ids.entry_id + ), + context_candidates AS ( + SELECT + ordered_rows.*, + 0 AS priority_group, + requested_positions.request_ordinal, + 0 AS neighbor_position + FROM requested_positions + INNER JOIN ordered_rows + ON ordered_rows.entry_id = requested_positions.entry_id + UNION ALL + SELECT + ordered_rows.*, + 1 AS priority_group, + requested_positions.request_ordinal, + ordered_rows.row_position AS neighbor_position + FROM requested_positions + INNER JOIN ordered_rows + ON ordered_rows.row_position BETWEEN requested_positions.row_position - ? + AND requested_positions.row_position + ? + AND ordered_rows.entry_id != requested_positions.entry_id + ), + ranked_context_candidates AS ( + SELECT + context_candidates.*, + ROW_NUMBER() OVER ( + PARTITION BY session_id, entry_id + ORDER BY priority_group, request_ordinal, neighbor_position + ) AS duplicate_rank + FROM context_candidates + ) + SELECT + session_id, entry_id, kind, name, source_type, source_id, source_seq, + provenance_key, payload_json, meta_json, created_at + FROM ranked_context_candidates + WHERE duplicate_rank = 1 + ORDER BY priority_group, request_ordinal, neighbor_position + LIMIT ?` + ) + .all( + serializeDeepChatTapeReadSources([normalizedSource]), + JSON.stringify(requestedEntryIds), + before, + after, + limit + ) as DeepChatTapeEntryRow[] + } + + private ensureProvenanceColumns(): void { + const columns: Array<[string, string]> = [ + ['source_type', 'TEXT'], + ['source_id', 'TEXT'], + ['source_seq', 'INTEGER'], + ['provenance_key', 'TEXT'] + ] + for (const [columnName, columnType] of columns) { + if (!this.hasColumn(columnName)) { + this.db.exec(`ALTER TABLE deepchat_tape_entries ADD COLUMN ${columnName} ${columnType}`) + } + } + } +} diff --git a/src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts b/src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts new file mode 100644 index 0000000000..f245c8d9aa --- /dev/null +++ b/src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts @@ -0,0 +1,17 @@ +import type Database from 'better-sqlite3-multiple-ciphers' +import type { TapeEntryLifecycleStore, TapeMutationProjection } from '../../ports/storage' + +export class SqliteTapeLifecycleAdapter implements TapeEntryLifecycleStore { + constructor( + private readonly db: Database.Database, + private readonly mutationProjection?: TapeMutationProjection + ) {} + + deleteBySession(sessionId: string): void { + const remove = this.db.transaction(() => { + this.db.prepare('DELETE FROM deepchat_tape_entries WHERE session_id = ?').run(sessionId) + this.mutationProjection?.deleteBySession(sessionId) + }) + remove() + } +} diff --git a/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts new file mode 100644 index 0000000000..3c4d8db625 --- /dev/null +++ b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts @@ -0,0 +1,1009 @@ +import Database from 'better-sqlite3-multiple-ciphers' +import { BaseTable } from '@/data/baseTable' +import { buildDeepChatTapeFtsMatch, buildDeepChatTapeLikeSearchPredicate } from './tapeEntryStore' +import { + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources +} from '@/tape/domain/entry' +import type { DeepChatTapeReadSource, DeepChatTapeSearchInput } from '@/tape/domain/entry' +import type { + TapeSearchProjectionInput, + TapeSearchProjectionMeta, + TapeSearchProjectionReadResult, + TapeSearchProjectionResultRow, + TapeSearchProjectionRow, + TapeSearchProjectionStore +} from '@/tape/ports/application' + +// Version 3 invalidates projections that may predate atomic Tape generation transitions. A +// matching entry-id head alone cannot prove that a version 2 row belongs to the current +// incarnation after a previously interrupted reset. +export const DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION = 3 + +export type DeepChatTapeSearchProjectionInput = TapeSearchProjectionInput +export type DeepChatTapeSearchProjectionRow = TapeSearchProjectionRow +export type DeepChatTapeSearchProjectionResultRow = TapeSearchProjectionResultRow +export type DeepChatTapeSearchProjectionMeta = TapeSearchProjectionMeta +export type DeepChatTapeSearchProjectionReadResult = TapeSearchProjectionReadResult + +type FtsCapability = { available: boolean; tokenizer: 'trigram' | 'unicode61' } + +const FTS_CAPABILITY_BY_DATABASE = new WeakMap() + +const TAPE_SEARCH_PROJECTION_INDEX_SQL = ` + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_kind + ON deepchat_tape_search_projection(session_id, kind, entry_id); + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_created + ON deepchat_tape_search_projection(session_id, created_at, entry_id); +` + +function normalizeLimit(limit: number | undefined): number { + return Math.min(Math.max(Math.floor(Number.isFinite(limit) ? (limit as number) : 20), 1), 100) +} + +function safeJsonStringify(value: Record): string { + return JSON.stringify(value ?? {}) +} + +function parseRefs(value: string): Record { + try { + const parsed = JSON.parse(value) + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : {} + } catch { + return {} + } +} + +export class DeepChatTapeSearchProjectionTable + extends BaseTable + implements TapeSearchProjectionStore +{ + private ftsCapability: FtsCapability | undefined + private ftsReady = false + + constructor(db: Database.Database) { + super(db, 'deepchat_tape_search_projection') + } + + getCreateTableSQL(): string { + return ` + CREATE TABLE IF NOT EXISTS deepchat_tape_search_projection ( + session_id TEXT NOT NULL, + entry_id INTEGER NOT NULL, + kind TEXT NOT NULL, + name TEXT, + source_type TEXT, + source_id TEXT, + source_seq INTEGER, + search_text TEXT NOT NULL, + summary_text TEXT NOT NULL, + refs_json TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, entry_id) + ); + CREATE TABLE IF NOT EXISTS deepchat_tape_search_projection_meta ( + session_id TEXT PRIMARY KEY, + projection_version INTEGER NOT NULL, + max_entry_id INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS deepchat_tape_search_fts_meta ( + session_id TEXT PRIMARY KEY, + projection_version INTEGER NOT NULL, + max_entry_id INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + ${TAPE_SEARCH_PROJECTION_INDEX_SQL} + ` + } + + override createTable(): void { + this.db.exec(this.getCreateTableSQL()) + this.pruneInvalidProjectionRows() + if (!this.ftsReady) { + this.ensureFtsIndex() + } + } + + getMigrationSQL(_version: number): string | null { + return null + } + + getLatestVersion(): number { + return 0 + } + + getSessionMeta(sessionId: string): DeepChatTapeSearchProjectionMeta | null { + const row = this.db + .prepare( + `SELECT projection_version, max_entry_id + FROM deepchat_tape_search_projection_meta + WHERE session_id = ?` + ) + .get(sessionId) as + | { + projection_version: number + max_entry_id: number + } + | undefined + if (!row) return null + return { + projectionVersion: row.projection_version, + maxEntryId: row.max_entry_id + } + } + + isCurrent( + sessionId: string, + maxEntryId: number, + projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ): boolean { + const row = this.getSessionMeta(sessionId) + return row?.projectionVersion === projectionVersion && row.maxEntryId === maxEntryId + } + + getProjectedEntryIds(sessionId: string): number[] { + return ( + this.db + .prepare( + `SELECT entry_id + FROM deepchat_tape_search_projection + WHERE session_id = ? + ORDER BY entry_id ASC` + ) + .all(sessionId) as Array<{ entry_id: number }> + ).map((row) => row.entry_id) + } + + appendSession( + sessionId: string, + rows: DeepChatTapeSearchProjectionInput[], + maxEntryId: number, + projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ): void { + const previousMeta = this.getSessionMeta(sessionId) + try { + this.db.transaction(() => { + if (!this.ftsReady) { + this.clearSessionFtsForBaseWrite(sessionId) + } + this.insertProjectionRows(rows) + if (this.ftsReady) { + if (previousMeta && this.isFtsCurrent(sessionId, previousMeta)) { + this.insertFtsRows(rows) + } else { + this.replaceSessionFtsRows(sessionId, this.getSessionProjectionInputs(sessionId)) + } + this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) + } + this.upsertMeta(sessionId, projectionVersion, maxEntryId) + })() + } catch (error) { + if (this.ftsReady) { + this.ftsReady = false + } + throw error + } + } + + replaceSession( + sessionId: string, + rows: DeepChatTapeSearchProjectionInput[], + maxEntryId: number, + projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ): void { + try { + this.db.transaction(() => { + if (this.ftsReady) { + this.db + .prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?') + .run(sessionId) + this.db + .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') + .run(sessionId) + } else { + this.clearSessionFtsForBaseWrite(sessionId) + } + this.db + .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') + .run(sessionId) + this.db + .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') + .run(sessionId) + this.insertProjectionRows(rows) + if (this.ftsReady) { + this.insertFtsRows(rows) + this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) + } + this.upsertMeta(sessionId, projectionVersion, maxEntryId) + })() + } catch (error) { + if (this.ftsReady) { + this.ftsReady = false + } + throw error + } + } + + private insertProjectionRows(rows: DeepChatTapeSearchProjectionInput[]): void { + if (!rows.length) return + const insertProjection = this.db.prepare( + `INSERT OR REPLACE INTO deepchat_tape_search_projection ( + session_id, + entry_id, + kind, + name, + source_type, + source_id, + source_seq, + search_text, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + for (const row of rows) { + insertProjection.run( + row.sessionId, + row.entryId, + row.kind, + row.name, + row.sourceType, + row.sourceId, + row.sourceSeq, + row.searchText, + row.summaryText, + safeJsonStringify(row.refs), + row.createdAt + ) + } + } + + private upsertMeta(sessionId: string, projectionVersion: number, maxEntryId: number): void { + this.db + .prepare( + `INSERT INTO deepchat_tape_search_projection_meta ( + session_id, + projection_version, + max_entry_id, + updated_at + ) + VALUES (?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + projection_version = excluded.projection_version, + max_entry_id = excluded.max_entry_id, + updated_at = excluded.updated_at` + ) + .run(sessionId, projectionVersion, maxEntryId, Date.now()) + } + + private getFtsMeta(sessionId: string): DeepChatTapeSearchProjectionMeta | null { + const row = this.db + .prepare( + `SELECT projection_version, max_entry_id + FROM deepchat_tape_search_fts_meta + WHERE session_id = ?` + ) + .get(sessionId) as + | { + projection_version: number + max_entry_id: number + } + | undefined + if (!row) return null + return { + projectionVersion: row.projection_version, + maxEntryId: row.max_entry_id + } + } + + private isFtsCurrent(sessionId: string, meta: DeepChatTapeSearchProjectionMeta): boolean { + const row = this.getFtsMeta(sessionId) + return row?.projectionVersion === meta.projectionVersion && row.maxEntryId === meta.maxEntryId + } + + private upsertFtsMeta(sessionId: string, projectionVersion: number, maxEntryId: number): void { + this.db + .prepare( + `INSERT INTO deepchat_tape_search_fts_meta ( + session_id, + projection_version, + max_entry_id, + updated_at + ) + VALUES (?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + projection_version = excluded.projection_version, + max_entry_id = excluded.max_entry_id, + updated_at = excluded.updated_at` + ) + .run(sessionId, projectionVersion, maxEntryId, Date.now()) + } + + getByEntryIds(sessionId: string, entryIds: number[]): DeepChatTapeSearchProjectionRow[] { + const ids = [...new Set(entryIds.filter((id) => Number.isInteger(id) && id > 0))] + if (!ids.length) return [] + const placeholders = ids.map(() => '?').join(', ') + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_search_projection + WHERE session_id = ? AND entry_id IN (${placeholders}) + ORDER BY entry_id ASC` + ) + .all(sessionId, ...ids) as DeepChatTapeSearchProjectionRow[] + } + + getByEntryIdsIfCurrent( + sessionId: string, + maxEntryId: number, + entryIds: number[], + projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ): DeepChatTapeSearchProjectionRow[] { + const ids = [...new Set(entryIds.filter((id) => Number.isInteger(id) && id > 0))] + if (!ids.length) return [] + const placeholders = ids.map(() => '?').join(', ') + return this.db + .prepare( + `SELECT projection.* + FROM deepchat_tape_search_projection AS projection + INNER JOIN deepchat_tape_search_projection_meta AS meta + ON meta.session_id = projection.session_id + AND meta.projection_version = ? + AND meta.max_entry_id = ? + WHERE projection.session_id = ? AND projection.entry_id IN (${placeholders}) + ORDER BY projection.entry_id ASC` + ) + .all(projectionVersion, maxEntryId, sessionId, ...ids) as DeepChatTapeSearchProjectionRow[] + } + + searchSourcesReadOnly( + sources: readonly DeepChatTapeReadSource[], + query: string, + options: DeepChatTapeSearchInput = {} + ): DeepChatTapeSearchProjectionReadResult { + const normalizedSources = normalizeDeepChatTapeReadSources(sources) + const coveredSources = this.getCurrentReadSources( + normalizedSources, + 'deepchat_tape_search_projection_meta' + ) + const normalizedQuery = query.trim() + if (coveredSources.length === 0 || !normalizedQuery) { + return { rows: [], coveredSources } + } + if (coveredSources.length !== normalizedSources.length) { + return { rows: [], coveredSources } + } + + const limit = normalizeLimit(options.limit) + const ordered: DeepChatTapeSearchProjectionResultRow[] = [] + const seen = new Set() + const collect = (rows: DeepChatTapeSearchProjectionResultRow[]): void => { + for (const row of rows) { + const key = `${row.session_id}:${row.entry_id}` + if (seen.has(key)) continue + seen.add(key) + ordered.push(row) + } + } + + if (this.ftsReady) { + const ftsSources = this.getCurrentReadSources(coveredSources, 'deepchat_tape_search_fts_meta') + if (ftsSources.length === coveredSources.length) { + try { + collect(this.searchFtsSourcesReadOnly(ftsSources, normalizedQuery, options, limit)) + } catch { + // The base projection remains a complete read-only fallback. + } + } + } + if (ordered.length < limit) { + collect(this.searchLikeSourcesReadOnly(coveredSources, normalizedQuery, options, limit)) + } + + return { rows: ordered.slice(0, limit), coveredSources } + } + + search( + sessionId: string, + query: string, + options: DeepChatTapeSearchInput = {} + ): DeepChatTapeSearchProjectionResultRow[] { + const normalized = query.trim() + if (!normalized) return [] + const limit = normalizeLimit(options.limit) + const ordered: DeepChatTapeSearchProjectionResultRow[] = [] + const seen = new Set() + const collect = (rows: DeepChatTapeSearchProjectionResultRow[]): void => { + for (const row of rows) { + if (seen.has(row.entry_id)) continue + seen.add(row.entry_id) + ordered.push(row) + } + } + this.recoverSessionFts(sessionId) + if (this.ftsReady) { + collect(this.searchFts(sessionId, normalized, options, limit)) + } + if (!this.ftsReady || ordered.length < limit) { + collect(this.searchLike(sessionId, normalized, options, limit)) + } + return ordered.slice(0, limit) + } + + deleteBySession(sessionId: string): void { + this.db.transaction(() => { + this.db + .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') + .run(sessionId) + this.db + .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') + .run(sessionId) + this.deleteSessionFts(sessionId, true) + })() + } + + clearAll(): void { + this.db.transaction(() => { + this.db.prepare('DELETE FROM deepchat_tape_search_projection').run() + this.db.prepare('DELETE FROM deepchat_tape_search_projection_meta').run() + if (this.ftsMetaTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() + } + this.clearFts(true) + })() + } + + private detectFtsCapability(): FtsCapability { + if (this.ftsCapability) return this.ftsCapability + const cached = FTS_CAPABILITY_BY_DATABASE.get(this.db) + if (cached) { + this.ftsCapability = cached + return cached + } + const probe = (tokenizer: string): boolean => { + const name = `temp.tape_search_fts_probe_${tokenizer}` + try { + this.db.exec( + `CREATE VIRTUAL TABLE IF NOT EXISTS ${name} USING fts5(c, tokenize='${tokenizer}');` + ) + this.db.exec(`DROP TABLE IF EXISTS ${name};`) + return true + } catch { + return false + } + } + if (probe('trigram')) this.ftsCapability = { available: true, tokenizer: 'trigram' } + else if (probe('unicode61')) this.ftsCapability = { available: true, tokenizer: 'unicode61' } + else this.ftsCapability = { available: false, tokenizer: 'unicode61' } + FTS_CAPABILITY_BY_DATABASE.set(this.db, this.ftsCapability) + return this.ftsCapability + } + + private ensureFtsIndex(): void { + const capability = this.detectFtsCapability() + if (!capability.available) { + this.ftsReady = false + return + } + try { + this.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS deepchat_tape_search_fts USING fts5( + search_text, + name, + session_id UNINDEXED, + entry_id UNINDEXED, + kind UNINDEXED, + source_type UNINDEXED, + source_id UNINDEXED, + source_seq UNINDEXED, + summary_text UNINDEXED, + refs_json UNINDEXED, + created_at UNINDEXED, + tokenize='${capability.tokenizer}' + ); + `) + this.ftsReady = true + } catch { + this.ftsReady = false + } + } + + private toProjectionInput( + row: DeepChatTapeSearchProjectionRow + ): DeepChatTapeSearchProjectionInput { + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + sourceType: row.source_type, + sourceId: row.source_id, + sourceSeq: row.source_seq, + searchText: row.search_text, + summaryText: row.summary_text, + refs: parseRefs(row.refs_json), + createdAt: row.created_at + } + } + + private getSessionProjectionInputs(sessionId: string): DeepChatTapeSearchProjectionInput[] { + return ( + this.db + .prepare( + `SELECT * + FROM deepchat_tape_search_projection + WHERE session_id = ? + ORDER BY entry_id ASC` + ) + .all(sessionId) as DeepChatTapeSearchProjectionRow[] + ).map((row) => this.toProjectionInput(row)) + } + + private ftsTableExists(): boolean { + const row = this.db + .prepare( + `SELECT name + FROM sqlite_master + WHERE type = 'table' AND name = 'deepchat_tape_search_fts' + LIMIT 1` + ) + .get() as { name: string } | undefined + return Boolean(row) + } + + private ftsMetaTableExists(): boolean { + const row = this.db + .prepare( + `SELECT name + FROM sqlite_master + WHERE type = 'table' AND name = 'deepchat_tape_search_fts_meta' + LIMIT 1` + ) + .get() as { name: string } | undefined + return Boolean(row) + } + + private clearSessionFtsForBaseWrite(sessionId: string): void { + if (this.ftsMetaTableExists()) { + this.db + .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') + .run(sessionId) + } + if (this.ftsTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) + } + } + + private getProjectionRowId(sessionId: string, entryId: number): number { + const row = this.db + .prepare( + `SELECT rowid + FROM deepchat_tape_search_projection + WHERE session_id = ? AND entry_id = ?` + ) + .get(sessionId, entryId) as { rowid: number } | undefined + if (!row) { + throw new Error(`Missing tape search projection row for ${sessionId}:${entryId}`) + } + return row.rowid + } + + private insertFtsRows(rows: DeepChatTapeSearchProjectionInput[]): void { + if (!rows.length) return + const insertFts = this.db.prepare( + `INSERT INTO deepchat_tape_search_fts ( + rowid, + search_text, + name, + session_id, + entry_id, + kind, + source_type, + source_id, + source_seq, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + for (const row of rows) { + this.db + .prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ? AND entry_id = ?') + .run(row.sessionId, row.entryId) + insertFts.run( + this.getProjectionRowId(row.sessionId, row.entryId), + row.searchText, + row.name ?? '', + row.sessionId, + row.entryId, + row.kind, + row.sourceType, + row.sourceId, + row.sourceSeq, + row.summaryText, + safeJsonStringify(row.refs), + row.createdAt + ) + } + } + + private replaceSessionFtsRows( + sessionId: string, + rows: DeepChatTapeSearchProjectionInput[] + ): void { + this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) + this.insertFtsRows(rows) + } + + private replaceSessionFts( + sessionId: string, + rows: DeepChatTapeSearchProjectionInput[], + projectionVersion: number, + maxEntryId: number + ): boolean { + if (!this.ftsReady) return false + try { + this.db.transaction(() => { + this.replaceSessionFtsRows(sessionId, rows) + this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) + })() + return true + } catch { + this.ftsReady = false + return false + } + } + + private recoverSessionFts(sessionId: string): void { + const meta = this.getSessionMeta(sessionId) + if (!meta) { + this.deleteSessionFts(sessionId) + return + } + if (this.ftsReady && this.isFtsCurrent(sessionId, meta)) return + if (!this.ftsReady) { + this.ensureFtsIndex() + } + if (!this.ftsReady) return + if (this.isFtsCurrent(sessionId, meta)) return + this.replaceSessionFts( + sessionId, + this.getSessionProjectionInputs(sessionId), + meta.projectionVersion, + meta.maxEntryId + ) + } + + hasFtsReadyForTesting(): boolean { + return this.ftsReady + } + + disableFtsForTesting(): void { + this.ftsReady = false + } + + dropFtsForTesting(): void { + this.dropFtsIndex() + } + + private deleteSessionFts(sessionId: string, dropOnFailure = false): void { + if (this.ftsMetaTableExists()) { + this.db + .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') + .run(sessionId) + } + if (!this.ftsTableExists()) return + try { + this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) + } catch { + this.ftsReady = false + if (dropOnFailure) { + this.dropFtsIndex() + } + } + } + + private clearFts(dropOnFailure = false): void { + if (this.ftsMetaTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() + } + if (!this.ftsTableExists()) return + try { + this.db.prepare('DELETE FROM deepchat_tape_search_fts').run() + } catch { + this.ftsReady = false + if (dropOnFailure) { + this.dropFtsIndex() + } + } + } + + private dropFtsIndex(): void { + this.ftsReady = false + this.db.exec('DROP TABLE IF EXISTS deepchat_tape_search_fts') + if (this.ftsMetaTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() + } + } + + private pruneInvalidProjectionRows(): void { + this.db.transaction(() => { + this.db + .prepare( + `DELETE FROM deepchat_tape_search_projection + WHERE NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_search_projection_meta AS meta + WHERE meta.session_id = deepchat_tape_search_projection.session_id + AND meta.projection_version >= ? + )` + ) + .run(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION) + this.db + .prepare( + `DELETE FROM deepchat_tape_search_projection_meta + WHERE projection_version < ?` + ) + .run(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION) + + if (this.ftsTableExists()) { + try { + this.db + .prepare( + `DELETE FROM deepchat_tape_search_fts + WHERE NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_search_fts_meta AS meta + WHERE meta.session_id = deepchat_tape_search_fts.session_id + AND meta.projection_version >= ? + ) + OR NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_search_projection_meta AS meta + WHERE meta.session_id = deepchat_tape_search_fts.session_id + AND meta.projection_version >= ? + )` + ) + .run(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION) + } catch { + this.dropFtsIndex() + } + } + if (this.ftsMetaTableExists()) { + this.db + .prepare( + `DELETE FROM deepchat_tape_search_fts_meta + WHERE projection_version < ? + OR NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_search_projection_meta AS projection_meta + WHERE projection_meta.session_id = deepchat_tape_search_fts_meta.session_id + AND projection_meta.projection_version >= ? + )` + ) + .run(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION) + } + })() + } + + private searchFts( + sessionId: string, + normalized: string, + options: DeepChatTapeSearchInput, + limit: number + ): DeepChatTapeSearchProjectionResultRow[] { + const match = buildDeepChatTapeFtsMatch(normalized) + const whereClauses = [ + 'deepchat_tape_search_fts MATCH ?', + 'deepchat_tape_search_fts.session_id = ?', + 'projection.session_id = ?' + ] + const params: Array = [match, sessionId, sessionId] + this.addFilters(whereClauses, params, options, true, 'projection') + params.push(limit) + try { + return this.db + .prepare( + `SELECT + projection.session_id, + projection.entry_id, + projection.kind, + projection.name, + projection.source_type, + projection.source_id, + projection.source_seq, + projection.search_text, + projection.summary_text, + projection.refs_json, + projection.created_at, + bm25(deepchat_tape_search_fts) AS score + FROM deepchat_tape_search_fts + INNER JOIN deepchat_tape_search_projection AS projection + ON projection.session_id = deepchat_tape_search_fts.session_id + AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) + AND projection.search_text = deepchat_tape_search_fts.search_text + WHERE ${whereClauses.join(' AND ')} + ORDER BY score ASC, projection.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeSearchProjectionResultRow[] + } catch { + try { + this.dropFtsIndex() + } catch { + this.ftsReady = false + } + return [] + } + } + + private getCurrentReadSources( + sources: readonly DeepChatTapeReadSource[], + metaTable: 'deepchat_tape_search_projection_meta' | 'deepchat_tape_search_fts_meta' + ): DeepChatTapeReadSource[] { + const normalizedSources = normalizeDeepChatTapeReadSources(sources) + if (normalizedSources.length === 0) { + return [] + } + const rows = this.db + .prepare( + `WITH requested_sources(session_id, max_entry_id) AS ( + SELECT + json_extract(value, '$.sessionId'), + CAST(json_extract(value, '$.maxEntryId') AS INTEGER) + FROM json_each(?) + ) + SELECT requested.session_id, requested.max_entry_id + FROM requested_sources AS requested + INNER JOIN ${metaTable} AS meta + ON meta.session_id = requested.session_id + AND meta.max_entry_id = requested.max_entry_id + AND meta.projection_version = ? + ORDER BY requested.session_id ASC` + ) + .all( + serializeDeepChatTapeReadSources(normalizedSources), + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ) as Array<{ session_id: string; max_entry_id: number }> + return rows.map((row) => ({ sessionId: row.session_id, maxEntryId: row.max_entry_id })) + } + + private searchFtsSourcesReadOnly( + sources: readonly DeepChatTapeReadSource[], + normalized: string, + options: DeepChatTapeSearchInput, + limit: number + ): DeepChatTapeSearchProjectionResultRow[] { + const match = buildDeepChatTapeFtsMatch(normalized) + const whereClauses = ['deepchat_tape_search_fts MATCH ?'] + const params: Array = [serializeDeepChatTapeReadSources(sources), match] + this.addFilters(whereClauses, params, options, true, 'projection') + params.push(limit) + return this.db + .prepare( + `WITH authorized_sources(session_id, max_entry_id) AS ( + SELECT + json_extract(value, '$.sessionId'), + CAST(json_extract(value, '$.maxEntryId') AS INTEGER) + FROM json_each(?) + ) + SELECT + projection.session_id, + projection.entry_id, + projection.kind, + projection.name, + projection.source_type, + projection.source_id, + projection.source_seq, + projection.search_text, + projection.summary_text, + projection.refs_json, + projection.created_at, + bm25(deepchat_tape_search_fts) AS score + FROM deepchat_tape_search_fts + INNER JOIN deepchat_tape_search_projection AS projection + ON projection.session_id = deepchat_tape_search_fts.session_id + AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) + AND projection.search_text = deepchat_tape_search_fts.search_text + INNER JOIN authorized_sources AS source + ON source.session_id = projection.session_id + AND projection.entry_id <= source.max_entry_id + WHERE ${whereClauses.join(' AND ')} + ORDER BY score ASC, projection.created_at DESC, projection.session_id ASC, + projection.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeSearchProjectionResultRow[] + } + + private searchLikeSourcesReadOnly( + sources: readonly DeepChatTapeReadSource[], + normalized: string, + options: DeepChatTapeSearchInput, + limit: number + ): DeepChatTapeSearchProjectionResultRow[] { + const queryPredicate = buildDeepChatTapeLikeSearchPredicate( + ['projection.search_text', 'projection.summary_text', 'projection.name'], + normalized + ) + const whereClauses = [queryPredicate.sql] + const params: Array = [serializeDeepChatTapeReadSources(sources)] + params.push(...queryPredicate.params) + this.addFilters(whereClauses, params, options, false, 'projection') + params.push(limit) + return this.db + .prepare( + `WITH authorized_sources(session_id, max_entry_id) AS ( + SELECT + json_extract(value, '$.sessionId'), + CAST(json_extract(value, '$.maxEntryId') AS INTEGER) + FROM json_each(?) + ) + SELECT projection.*, NULL AS score + FROM deepchat_tape_search_projection AS projection + INNER JOIN authorized_sources AS source + ON source.session_id = projection.session_id + AND projection.entry_id <= source.max_entry_id + WHERE ${whereClauses.join(' AND ')} + ORDER BY projection.created_at DESC, projection.session_id ASC, projection.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeSearchProjectionResultRow[] + } + + private searchLike( + sessionId: string, + normalized: string, + options: DeepChatTapeSearchInput, + limit: number + ): DeepChatTapeSearchProjectionResultRow[] { + const queryPredicate = buildDeepChatTapeLikeSearchPredicate( + ['search_text', 'summary_text', 'name'], + normalized + ) + const whereClauses = ['session_id = ?', queryPredicate.sql] + const params: Array = [sessionId] + params.push(...queryPredicate.params) + this.addFilters(whereClauses, params, options) + params.push(limit) + return this.db + .prepare( + `SELECT *, NULL AS score + FROM deepchat_tape_search_projection + WHERE ${whereClauses.join(' AND ')} + ORDER BY entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeSearchProjectionResultRow[] + } + + private addFilters( + whereClauses: string[], + params: Array, + options: DeepChatTapeSearchInput, + castCreatedAt = false, + tableAlias?: string + ): void { + const column = (name: string) => (tableAlias ? `${tableAlias}.${name}` : name) + if (options.kinds?.length) { + whereClauses.push(`${column('kind')} IN (${options.kinds.map(() => '?').join(', ')})`) + params.push(...options.kinds) + } + const createdAtColumn = castCreatedAt + ? `CAST(${column('created_at')} AS INTEGER)` + : column('created_at') + if (Number.isFinite(options.startCreatedAt)) { + whereClauses.push(`${createdAtColumn} >= ?`) + params.push(options.startCreatedAt as number) + } + if (Number.isFinite(options.endCreatedAt)) { + whereClauses.push(`${createdAtColumn} <= ?`) + params.push(options.endCreatedAt as number) + } + } +} diff --git a/src/main/tape/ports/application.ts b/src/main/tape/ports/application.ts new file mode 100644 index 0000000000..c1b220f029 --- /dev/null +++ b/src/main/tape/ports/application.ts @@ -0,0 +1,179 @@ +import type { + DeepChatTapeEntryKind, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + DeepChatTapeSourceType +} from '../domain/entry' +import type { + TapeBootstrapStore, + TapeEntryLifecycleStore, + TapeEntryStore, + TapeTransactionRunner +} from './storage' + +export interface TapeSearchProjectionInput { + sessionId: string + entryId: number + kind: DeepChatTapeEntryKind + name: string | null + sourceType: DeepChatTapeSourceType | null + sourceId: string | null + sourceSeq: number | null + searchText: string + summaryText: string + refs: Record + createdAt: number +} + +export interface TapeSearchProjectionRow { + session_id: string + entry_id: number + kind: DeepChatTapeEntryKind + name: string | null + source_type: DeepChatTapeSourceType | null + source_id: string | null + source_seq: number | null + search_text: string + summary_text: string + refs_json: string + created_at: number +} + +export interface TapeSearchProjectionResultRow extends TapeSearchProjectionRow { + score: number | null +} + +export interface TapeSearchProjectionMeta { + projectionVersion: number + maxEntryId: number +} + +export interface TapeSearchProjectionReadResult { + rows: TapeSearchProjectionResultRow[] + coveredSources: DeepChatTapeReadSource[] +} + +export interface TapeSearchProjectionStore { + getSessionMeta(sessionId: string): TapeSearchProjectionMeta | null + isCurrent(sessionId: string, maxEntryId: number, projectionVersion?: number): boolean + getProjectedEntryIds(sessionId: string): number[] + appendSession( + sessionId: string, + rows: TapeSearchProjectionInput[], + maxEntryId: number, + projectionVersion?: number + ): void + replaceSession( + sessionId: string, + rows: TapeSearchProjectionInput[], + maxEntryId: number, + projectionVersion?: number + ): void + getByEntryIdsIfCurrent( + sessionId: string, + maxEntryId: number, + entryIds: number[], + projectionVersion?: number + ): TapeSearchProjectionRow[] + searchSourcesReadOnly( + sources: readonly DeepChatTapeReadSource[], + query: string, + options?: DeepChatTapeSearchInput + ): TapeSearchProjectionReadResult + search( + sessionId: string, + query: string, + options?: DeepChatTapeSearchInput + ): TapeSearchProjectionResultRow[] + deleteBySession(sessionId: string): void +} + +export interface TapeLineageSessionRow { + id: string + session_kind: 'regular' | 'subagent' + parent_session_id: string | null +} + +export interface TapeLineageSessionReader { + get(sessionId: string): TapeLineageSessionRow | undefined + getMany(sessionIds: string[]): TapeLineageSessionRow[] +} + +export interface TapeLegacySummaryState { + summary_text: string | null + summary_cursor_order_seq: number | null + summary_updated_at: number | null +} + +export interface TapeLegacySummaryReader { + getSummaryState(sessionId: string): TapeLegacySummaryState | null +} + +export interface TapeMessageTraceRow { + id: string + message_id: string + session_id: string + provider_id: string + model_id: string + request_seq: number + endpoint: string + headers_json: string + body_json: string + truncated: number + created_at: number +} + +export interface TapeMessageTraceReader { + listByMessageId(messageId: string): TapeMessageTraceRow[] +} + +export interface TapeTerminalMessageRow { + session_id: string + order_seq: number + role: 'user' | 'assistant' + content: string + status: 'pending' | 'sent' | 'error' + metadata: string + created_at: number + updated_at: number +} + +export interface TapeTerminalMessageReader { + get(messageId: string): TapeTerminalMessageRow | undefined +} + +export type TapeApplicationEntryStore = TapeEntryStore & TapeTransactionRunner & TapeBootstrapStore + +export interface TapeApplicationDatabase { + readonly deepchatTapeEntriesTable: TapeApplicationEntryStore + readonly tapeLifecycle: TapeEntryLifecycleStore + readonly deepchatTapeSearchProjectionTable: TapeSearchProjectionStore + readonly newSessionsTable: TapeLineageSessionReader + readonly deepchatSessionsTable: TapeLegacySummaryReader + readonly deepchatMessageTracesTable: TapeMessageTraceReader + readonly deepchatMessagesTable: TapeTerminalMessageReader +} + +export interface TapeApplicationProviders { + getEntryStore(): TapeApplicationEntryStore + getEntryLifecycleStore(): TapeEntryLifecycleStore + getSearchProjectionStore(): TapeSearchProjectionStore + getLineageSessionReader(): TapeLineageSessionReader + getLegacySummaryReader(): TapeLegacySummaryReader + getMessageTraceReader(): TapeMessageTraceReader + getTerminalMessageReader(): TapeTerminalMessageReader +} + +export function createTapeApplicationProviders( + database: TapeApplicationDatabase +): TapeApplicationProviders { + return { + getEntryStore: () => database.deepchatTapeEntriesTable, + getEntryLifecycleStore: () => database.tapeLifecycle, + getSearchProjectionStore: () => database.deepchatTapeSearchProjectionTable, + getLineageSessionReader: () => database.newSessionsTable, + getLegacySummaryReader: () => database.deepchatSessionsTable, + getMessageTraceReader: () => database.deepchatMessageTracesTable, + getTerminalMessageReader: () => database.deepchatMessagesTable + } +} diff --git a/src/main/tape/ports/capabilities.ts b/src/main/tape/ports/capabilities.ts new file mode 100644 index 0000000000..68e5898aa7 --- /dev/null +++ b/src/main/tape/ports/capabilities.ts @@ -0,0 +1,108 @@ +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { + DeepChatTapeViewManifest, + DeepChatTapeViewManifestRecord +} from '@shared/types/tape-view-manifest' +import type { DeepChatTapeEntryRow, TapeAnchorAppendInput } from '../domain/entry' +import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' + +export type TapeMigrationState = 'none' | 'ready' + +export type TapeBackfillResult = { + sessionId: string + migrationState: TapeMigrationState + messageCount: number + maxOrderSeq: number + appendedFactCount: number + historyRecords: ChatMessageRecord[] +} + +export interface TapeTranscriptReader { + getMessages(sessionId: string): ChatMessageRecord[] +} + +export interface TapeReconciliationPort { + ensureSessionTapeReady(sessionId: string, messageStore: TapeTranscriptReader): TapeBackfillResult +} + +export type TapeViewManifestAssemblySources = { + latestEntryId: number + anchorEntryIds: number[] + reconstructionAnchorEntryIds: number[] + reconstructionAnchorEntryId: number | null + entryIdByMessageId: Map + toolCallEntryIdByToolId: Map + toolResultEntryIdByToolId: Map +} + +export interface TapeViewManifestReader { + getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestAssemblySources + listViewManifestsByMessage(sessionId: string, messageId: string): DeepChatTapeViewManifestRecord[] +} + +export interface TapeViewManifestWriter { + appendViewManifest(manifest: DeepChatTapeViewManifest): void +} + +export interface TapeToolFactWriter { + appendToolFact(input: TapeToolFactInput): Promise +} + +export interface TapeMessageFactWriter { + appendMessageRecord(record: ChatMessageRecord): number + appendMessageReplacement(record: ChatMessageRecord, reason: string): number + appendMessageRetraction(record: ChatMessageRecord, reason: string): number +} + +export interface TapeRawEntryReader { + getBySession(sessionId: string): DeepChatTapeEntryRow[] +} + +export interface TapeAnchorReader { + getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined +} + +export interface TapeAnchorWriter { + /** + * Compound Session settings updates require this writer to share the caller's synchronous + * SQLite connection and transaction context. Cross-connection or asynchronous adapters cannot + * preserve summary-and-anchor atomicity. + */ + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow +} + +export interface TapeInspectionReader { + getEffectiveMessageSourceSpan( + sessionId: string, + entryIds: number[] + ): TapeEffectiveMessageSourceEntry[] + listMemoryViewManifestsByAgent( + agentId: string, + options?: { sessionId?: string; limit?: number; messageId?: string } + ): TapeMemoryViewManifestInspection[] +} + +export interface TapeEffectiveMessageSourceEntry { + entryId: number + record: Pick +} + +export interface TapeMemoryViewManifestInspection { + sessionId: string + messageId: string | null + entryId: number + policyVersion: number | null + tokenBudget: number + estimatedTokens: number + selectedCount: number + selectedIds: string[] | null + droppedCount: number + queryHash: string | null + createdAt: number +} + +export interface TapeLifecycleAdmin { + initializeSessionTape(sessionId: string): void + deleteSessionTape(sessionId: string): void + resetSessionTape(sessionId: string): void +} diff --git a/src/main/tape/ports/storage.ts b/src/main/tape/ports/storage.ts new file mode 100644 index 0000000000..1a33d390a0 --- /dev/null +++ b/src/main/tape/ports/storage.ts @@ -0,0 +1,67 @@ +import type { + DeepChatTapeAppendInput, + DeepChatTapeEntryRow, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + TapeAnchorAppendInput, + TapeEventAppendInput +} from '../domain/entry' + +export interface TapeMutationProjection { + applyAppendedEntry(row: DeepChatTapeEntryRow, previousSessionMaxEntryId: number): boolean + invalidateSession(sessionId: string): void + deleteBySession(sessionId: string): void +} + +/** Append/read/query persistence only. Physical deletion belongs to TapeEntryLifecycleStore. */ +export interface TapeEntryStore { + append(input: DeepChatTapeAppendInput): DeepChatTapeEntryRow + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow + appendEvent(input: TapeEventAppendInput): DeepChatTapeEntryRow + getBySession(sessionId: string): DeepChatTapeEntryRow[] + getSubagentLineageEvents(sessionId: string): DeepChatTapeEntryRow[] + getFirstEntriesBySessions(sessionIds: string[]): DeepChatTapeEntryRow[] + getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] + listMemoryViewManifestAnchorsByAgent( + agentId: string, + options?: { sessionId?: string; limit?: number; messageId?: string } + ): DeepChatTapeEntryRow[] + getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined + getAnchors(sessionId: string, limit?: number): DeepChatTapeEntryRow[] + getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined + getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined + getByProvenanceKey(sessionId: string, provenanceKey: string): DeepChatTapeEntryRow | undefined + getMaxEntryId(sessionId: string): number + getMaxEntryIdsBySessions(sessionIds: string[]): Map + countAnchorsBySession(sessionId: string): number + countEntriesAfter(sessionId: string, entryId: number): number + countBySession(sessionId: string): number + search( + sessionId: string, + query: string, + options?: DeepChatTapeSearchInput + ): DeepChatTapeEntryRow[] + searchEffectiveSourcesAtHeads( + sources: readonly DeepChatTapeReadSource[], + query: string, + options?: DeepChatTapeSearchInput + ): DeepChatTapeEntryRow[] + getEffectiveContextRowsAtHead( + source: DeepChatTapeReadSource, + entryIds: number[], + options: { before: number; after: number; limit: number } + ): DeepChatTapeEntryRow[] +} + +export interface TapeTransactionRunner { + runInTransaction(operation: () => T): T +} + +/** Transitional bootstrap capability until bootstrap orchestration lives in application services. */ +export interface TapeBootstrapStore { + ensureBootstrapAnchor(sessionId: string): void +} + +export interface TapeEntryLifecycleStore { + deleteBySession(sessionId: string): void +} diff --git a/test/main/agent/acp/compatibility/adapters.test.ts b/test/main/agent/acp/compatibility/adapters.test.ts index 9388d1a4f5..62d7a732b9 100644 --- a/test/main/agent/acp/compatibility/adapters.test.ts +++ b/test/main/agent/acp/compatibility/adapters.test.ts @@ -182,13 +182,13 @@ function createProjectionHarness() { ) } } as unknown as MainDatabase - const messageStore = new SessionTranscript(sqlitePresenter) const tapeService = new SessionTape(sqlitePresenter) + const messageStore = new SessionTranscript(sqlitePresenter, tapeService) const adapter = new AcpCompatibilityProjectionAdapter({ publishEvent: publishDeepchatEvent, publishSessionUpdate: vi.fn(), messageStore, - tapeService, + tapeReconciliation: tapeService, writeViewManifest: vi.fn(), setStatus: vi.fn() }) diff --git a/test/main/agent/deepchat/memory/memoryRuntimeCoordinator.test.ts b/test/main/agent/deepchat/memory/memoryRuntimeCoordinator.test.ts index ee02cefd3a..2b724c9616 100644 --- a/test/main/agent/deepchat/memory/memoryRuntimeCoordinator.test.ts +++ b/test/main/agent/deepchat/memory/memoryRuntimeCoordinator.test.ts @@ -95,6 +95,8 @@ function createHarness() { replaceSession: vi.fn(), invalidateSession: vi.fn() } + const getTapeRows = vi.fn(() => tapeRows) + const appendTapeAnchor = vi.fn() const deps = { memoryPort: port as any, getSessionAgentId: vi.fn(() => 'agent-a'), @@ -116,8 +118,16 @@ function createHarness() { rewindMemoryCursorOrderSeq: vi.fn((_sessionId: string, orderSeq: number) => { cursor = orderSeq }), - getTapeRows: vi.fn(() => tapeRows), - appendTapeAnchor: vi.fn(), + tapeReader: { + getBySession: getTapeRows, + getBySessionUpToEntryId: vi.fn((_sessionId: string, maxEntryId: number) => + tapeRows.filter((row) => row.entry_id <= maxEntryId) + ), + getMaxEntryId: vi.fn(() => tapeRows.at(-1)?.entry_id ?? 0) + }, + tapeAnchorWriter: { appendAnchor: appendTapeAnchor }, + getTapeRows, + appendTapeAnchor, getIngestionProjection: vi.fn(() => projection) } const coordinator = new MemoryRuntimeCoordinator(deps) diff --git a/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts b/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts index 16498a5563..c85b15a1ca 100644 --- a/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts +++ b/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts @@ -322,6 +322,7 @@ function createMockSqlitePresenter() { delete: vi.fn() }, deepchatTapeEntriesTable: (deepchatTapeEntriesTable = { + runInTransaction: vi.fn((operation: () => unknown) => operation()), ensureBootstrapAnchor: vi.fn(), append: vi.fn((input: any) => { const provenanceKey = @@ -443,10 +444,12 @@ function createMockSqlitePresenter() { memoryIngestionProjectionCurrent = false }) }), + tapeLifecycle: deepchatTapeEntriesTable, deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn(), isCurrent: vi.fn().mockReturnValue(false), - getByEntryIds: vi.fn().mockReturnValue([]) + getByEntryIds: vi.fn().mockReturnValue([]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) }, deepchatMemoryIngestionProjectionTable: { readCurrentRange: vi.fn( @@ -874,7 +877,9 @@ describe('DeepChatRuntimeCoordinator', () => { transcript: sessionData.transcript, settings: sessionData.settings, pendingInputs: sessionData.pendingInputs, - runtime: agent + runtime: agent, + runInTransaction: (operation) => + sessionData.database.getDatabase().transaction(operation)() }) }) diff --git a/test/main/agent/deepchat/runtime/process.test.ts b/test/main/agent/deepchat/runtime/process.test.ts index 6c1b7a5c90..75de6fe02b 100644 --- a/test/main/agent/deepchat/runtime/process.test.ts +++ b/test/main/agent/deepchat/runtime/process.test.ts @@ -160,7 +160,7 @@ function makeStreamEvents(...events: LLMCoreStreamEvent[]): LLMCoreStreamEvent[] describe('processStream', () => { let messageStore: ReturnType - let tapeRecorder: { appendToolFact: ReturnType } + let tapeToolFactWriter: { appendToolFact: ReturnType } let tempHome: string | null = null let homedirSpy: ReturnType | null = null @@ -168,7 +168,7 @@ describe('processStream', () => { vi.useFakeTimers() vi.clearAllMocks() messageStore = createMockMessageStore() - tapeRecorder = { + tapeToolFactWriter = { appendToolFact: vi.fn(async (input) => ({ sessionId: input.sessionId, entryId: 1 @@ -241,7 +241,7 @@ describe('processStream', () => { permissionMode: 'full_access', io: { messageStore, - tapeRecorder, + tapeToolFactWriter, publishEvent: publishDeepchatEventMock, publishSessionUpdate: vi.fn() }, @@ -273,7 +273,7 @@ describe('processStream', () => { messageStore.setMessageError.mockImplementation(() => { order.push('message:error') }) - tapeRecorder.appendToolFact.mockImplementation(async (input) => { + tapeToolFactWriter.appendToolFact.mockImplementation(async (input) => { order.push(`tape:${input.provenance.source}`) return { sessionId: input.sessionId, entryId: 1 } }) @@ -374,7 +374,7 @@ describe('processStream', () => { 'renderer:update', 'renderer:complete' ]) - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() expect(JSON.parse(messageStore.finalizeAssistantMessage.mock.calls[0][2])).toMatchObject({ provider: 'openai', model: 'gpt-4' @@ -410,7 +410,7 @@ describe('processStream', () => { ]) expect(messageStore.finalizeAssistantMessage).toHaveBeenCalledTimes(1) expect(messageStore.setMessageError).toHaveBeenCalledTimes(1) - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() }) it('persists each tool round before entering the next provider round', async () => { @@ -470,9 +470,9 @@ describe('processStream', () => { 'renderer:update', 'renderer:complete' ]) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(6) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(6) expect( - tapeRecorder.appendToolFact.mock.calls.map(([input]) => [ + tapeToolFactWriter.appendToolFact.mock.calls.map(([input]) => [ input.provenance.source, input.provenance.sourceId, input.provenance.sequence @@ -487,8 +487,8 @@ describe('processStream', () => { ]) }) - it('keeps the tool loop fail-open when TapeRecorder rejects a fact', async () => { - tapeRecorder.appendToolFact.mockRejectedValue(new Error('tape unavailable')) + it('keeps the tool loop fail-open when TapeToolFactWriter rejects a fact', async () => { + tapeToolFactWriter.appendToolFact.mockRejectedValue(new Error('tape unavailable')) const coreStream = createToolThenCompleteStream('action') const result = await processStream( @@ -501,7 +501,7 @@ describe('processStream', () => { expect(result.status).toBe('completed') expect(coreStream).toHaveBeenCalledTimes(2) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(1) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(1) }) it('persists a paused tool round before its terminal projection', async () => { @@ -553,7 +553,7 @@ describe('processStream', () => { 'renderer:update', 'renderer:complete' ]) - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() }) it('accounts for executed tools before pausing a mixed tool batch', async () => { @@ -705,7 +705,7 @@ describe('processStream', () => { ]) expect(messageStore.setMessageError).toHaveBeenCalled() expect(messageStore.finalizeAssistantMessage).not.toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() expectDeepchatEvent('chat.stream.failed', { sessionId: 's1', messageId: 'm1', @@ -738,7 +738,7 @@ describe('processStream', () => { const abortMetadata = JSON.parse(messageStore.setMessageError.mock.calls[0][2]) expect(abortMetadata).toMatchObject({ provider: 'openai', model: 'gpt-4' }) expect(messageStore.finalizeAssistantMessage).not.toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() expectDeepchatEvent('chat.stream.failed', { sessionId: 's1', messageId: 'm1', @@ -767,7 +767,7 @@ describe('processStream', () => { }) expect(order).toEqual([...TOOL_ROUND_COMMIT_ORDER, ...ERROR_TERMINAL_COMMIT_ORDER]) expect(coreStream).toHaveBeenCalledTimes(1) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) it('does not snapshot an oversized tool batch that never executes', async () => { @@ -807,7 +807,7 @@ describe('processStream', () => { ...COMPLETED_TERMINAL_COMMIT_ORDER ]) expect(toolService.callTool).not.toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() }) it('persists a terminal tool-output error before the failed projection', async () => { @@ -831,7 +831,7 @@ describe('processStream', () => { expect(order).toEqual([...TOOL_ROUND_COMMIT_ORDER, ...ERROR_TERMINAL_COMMIT_ORDER]) expect(coreStream).toHaveBeenCalledTimes(1) expect(messageStore.setMessageError).toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) it('settles a post-stream abort without a tool Tape snapshot', async () => { @@ -848,7 +848,7 @@ describe('processStream', () => { expect(result.status).toBe('aborted') expect(order).toEqual(['renderer:update', 'message:update', ...ERROR_TERMINAL_COMMIT_ORDER]) - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() }) it('persists the completed tool batch before a post-tool abort', async () => { @@ -878,7 +878,7 @@ describe('processStream', () => { expect(toolService.callTool).toHaveBeenCalledTimes(1) expect(messageStore.setMessageError).toHaveBeenCalled() expect(messageStore.finalizeAssistantMessage).not.toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) it('keeps a tool-local AbortError as a tool failure while the run remains active', async () => { @@ -929,7 +929,7 @@ describe('processStream', () => { }) ]) ) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) it('persists the completed batch before settling for pending input', async () => { @@ -953,7 +953,7 @@ describe('processStream', () => { }) expect(order).toEqual([...TOOL_ROUND_COMMIT_ORDER, ...COMPLETED_TERMINAL_COMMIT_ORDER]) expect(coreStream).toHaveBeenCalledTimes(1) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) }) diff --git a/test/main/app/compositionBoundaries.test.ts b/test/main/app/compositionBoundaries.test.ts index 95af980b7e..6222d0e172 100644 --- a/test/main/app/compositionBoundaries.test.ts +++ b/test/main/app/compositionBoundaries.test.ts @@ -10,6 +10,9 @@ describe('session boundary composition', () => { ) expect(compositionSource.match(/new LegacyChatImportService\(/g)).toHaveLength(1) + expect(compositionSource).toMatch( + /new LegacyChatImportService\([^)]*memoryDatabase,\s*sessionData\.tapeStore/ + ) expect(compositionSource).toContain( 'legacyChatImportService.repairImportedLegacySessionSkills(conversationId)' ) diff --git a/test/main/app/startupMigrations/legacyChatImportService.test.ts b/test/main/app/startupMigrations/legacyChatImportService.test.ts index de24962edc..f2d083b8a1 100644 --- a/test/main/app/startupMigrations/legacyChatImportService.test.ts +++ b/test/main/app/startupMigrations/legacyChatImportService.test.ts @@ -123,6 +123,11 @@ describe('LegacyChatImportService', () => { sqlitePresenter as any, sqlitePresenter as any, sqlitePresenter as any, + { + appendMessageRecord: vi.fn(() => 0), + appendMessageReplacement: vi.fn(() => 0), + appendMessageRetraction: vi.fn(() => 0) + }, '/mock/legacy.db' ) }) diff --git a/test/main/evals/nativeAgent/harness.ts b/test/main/evals/nativeAgent/harness.ts index 7b8ce7fc05..3656bd979c 100644 --- a/test/main/evals/nativeAgent/harness.ts +++ b/test/main/evals/nativeAgent/harness.ts @@ -499,7 +499,7 @@ export async function runNativeAgentEvalScenario( messageStore, publishEvent: vi.fn(), publishSessionUpdate: vi.fn(), - tapeRecorder: { + tapeToolFactWriter: { appendToolFact: async () => ({ sessionId, entryId: 1 }) } } diff --git a/test/main/memory/deepchatMemoryIngestionProjection.test.ts b/test/main/memory/deepchatMemoryIngestionProjection.test.ts index 5549ae0550..564875c303 100644 --- a/test/main/memory/deepchatMemoryIngestionProjection.test.ts +++ b/test/main/memory/deepchatMemoryIngestionProjection.test.ts @@ -1,5 +1,6 @@ import { describe, expect, vi } from 'vitest' import { buildEffectiveTapeView } from '@/session/data/tapeEffectiveView' +import { SqliteTapeLifecycleAdapter } from '@/tape/infrastructure/sqlite/tapeLifecycleAdapter' import { Database, nativeSqliteItIf } from '../nativeSqliteHarness' const entriesModule = Database ? await import('@/session/data/tables/deepchatTapeEntries') : null @@ -25,7 +26,8 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { projection.createTable() const tape = new TapeTableCtor(db, projection) tape.createTable() - return { db, projection, tape } + const lifecycle = new SqliteTapeLifecycleAdapter(db, projection) + return { db, lifecycle, projection, tape } } function appendMessage( @@ -243,7 +245,7 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { itIfSqlite( 'marks retractions stale and keeps a final tool-before-message sequence current', () => { - const { db, projection, tape } = createTables() + const { db, lifecycle, projection, tape } = createTables() try { appendMessage(tape, { id: 'm1', @@ -259,8 +261,7 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { }) expect(projection.isCurrent('s1', tape.getMaxEntryId('s1'))).toBe(false) - projection.deleteBySession('s1') - tape.deleteBySession('s1') + lifecycle.deleteBySession('s1') appendToolCall(tape, { messageId: 'future-message', toolCallId: 'tool-before-message', @@ -434,7 +435,7 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { ) itIfSqlite('cleans projection rows and meta with the authoritative session delete', () => { - const { db, projection, tape } = createTables() + const { db, lifecycle, projection, tape } = createTables() try { appendMessage(tape, { id: 'm1', @@ -442,7 +443,7 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { status: 'sent', content: 'first' }) - tape.deleteBySession('s1') + lifecycle.deleteBySession('s1') expect(tape.getBySession('s1')).toEqual([]) expect(projection.listRange('s1', 0, 10)).toEqual([]) diff --git a/test/main/routes/dispatcher.test.ts b/test/main/routes/dispatcher.test.ts index a788bcc968..5c79a6e5e2 100644 --- a/test/main/routes/dispatcher.test.ts +++ b/test/main/routes/dispatcher.test.ts @@ -14,6 +14,7 @@ import type { IYoBrowserPresenter } from '@shared/types/desktop' import type { MainDatabase } from '@/data/mainDatabase' +import type { TapeInspectionReader } from '@/tape/ports/capabilities' import type { OAuthServicePort } from '@shared/types/oauth' import type { DialogServicePort } from '@shared/types/dialog' import type { DeviceServicePort } from '@shared/types/device' @@ -1271,12 +1272,12 @@ function createRuntime() { repairSchema: vi.fn().mockResolvedValue(databaseRepairReport), agentMemoryAuditTable: { listByAgent: vi.fn(() => []) - }, - deepchatTapeEntriesTable: { - getBySession: vi.fn(() => []), - listMemoryViewManifestAnchorsByAgent: vi.fn(() => []) } } as unknown as MainDatabase + const tapeInspection: TapeInspectionReader = { + getEffectiveMessageSourceSpan: vi.fn(() => []), + listMemoryViewManifestsByAgent: vi.fn(() => []) + } const cronJob = { id: 'cron-1', name: 'Cron smoke', @@ -1515,7 +1516,7 @@ function createRuntime() { const memoryRoutes = createMemoryRoutes({ memoryService, getAgentType: (agentId) => providerSettings.getAgentType(agentId), - getTapeEntries: () => (sqlitePresenter as any).deepchatTapeEntriesTable, + getTapeInspection: () => tapeInspection, getAuditEntries: () => (sqlitePresenter as any).agentMemoryAuditTable }) const desktopRoutes = createDesktopRoutes({ @@ -1754,6 +1755,7 @@ function createRuntime() { remoteService, shortcutPresenter, sqlitePresenter, + tapeInspection, windowPresenter, deviceService, appDataReset, @@ -2380,61 +2382,29 @@ describe('dispatchDeepchatRoute', () => { }) it('filters memory view manifests by message before applying the requested limit', async () => { - const { runtime, providerSettings } = createRuntime() + const { runtime, providerSettings, tapeInspection } = createRuntime() vi.mocked(providerSettings.getAgentType).mockResolvedValueOnce('deepchat') const listSessions = vi.fn() - const listMemoryViewManifestAnchorsByAgent = vi.fn().mockReturnValue([ - { - session_id: 's1', - entry_id: 20, - kind: 'anchor', - name: 'memory/view_assembled', - source_type: 'memory', - source_id: 'msg-new', - source_seq: 0, - provenance_key: null, - payload_json: JSON.stringify({ - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 10, - selected: ['new'], - dropped: [], - queryHash: 'newhash' - } - }), - meta_json: JSON.stringify({ messageId: 'msg-new' }), - created_at: 200 - }, - { - session_id: 's1', - entry_id: 10, - kind: 'anchor', - name: 'memory/view_assembled', - source_type: 'memory', - source_id: 'msg-old', - source_seq: 0, - provenance_key: null, - payload_json: JSON.stringify({ - state: { - policyVersion: 1, - tokenBudget: 900, - estimatedTokens: 9, - selected: ['old'], - dropped: ['drop'], - queryHash: 'oldhash' - } - }), - meta_json: JSON.stringify({ messageId: 'msg-old' }), - created_at: 100 - } - ]) + const listMemoryViewManifestsByAgent = vi + .mocked(tapeInspection.listMemoryViewManifestsByAgent) + .mockReturnValue([ + { + sessionId: 's1', + messageId: 'msg-old', + entryId: 10, + policyVersion: 1, + tokenBudget: 900, + estimatedTokens: 9, + selectedCount: 1, + selectedIds: ['old'], + droppedCount: 1, + queryHash: 'oldhash', + createdAt: 100 + } + ]) ;(runtime as any).sqlitePresenter = { newSessionsTable: { list: listSessions - }, - deepchatTapeEntriesTable: { - listMemoryViewManifestAnchorsByAgent } } @@ -2446,7 +2416,7 @@ describe('dispatchDeepchatRoute', () => { ) expect(listSessions).not.toHaveBeenCalled() - expect(listMemoryViewManifestAnchorsByAgent).toHaveBeenCalledWith('a', { + expect(listMemoryViewManifestsByAgent).toHaveBeenCalledWith('a', { sessionId: 's1', limit: 1, messageId: 'msg-old' @@ -2465,46 +2435,24 @@ describe('dispatchDeepchatRoute', () => { }) }) - it('derives selected memory ids from string and object manifest selections', async () => { - const { runtime, providerSettings } = createRuntime() + it('returns Tape inspection manifest DTOs without exposing raw rows', async () => { + const { runtime, providerSettings, tapeInspection } = createRuntime() vi.mocked(providerSettings.getAgentType).mockResolvedValueOnce('deepchat') - const listMemoryViewManifestAnchorsByAgent = vi.fn().mockReturnValue([ - { - session_id: 's1', - entry_id: 30, - kind: 'anchor', - name: 'memory/view_assembled', - source_type: 'memory', - source_id: 'msg-1', - source_seq: 0, - provenance_key: null, - payload_json: JSON.stringify({ - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 10, - selected: [ - 'm-string', - { id: 'm-object' }, - 'm-string', - { id: 'm-object' }, - { nope: 'ignored' }, - 3 - ], - dropped: [], - queryHash: 'hash' - } - }), - meta_json: JSON.stringify({ messageId: 'msg-1' }), - created_at: 300 + vi.mocked(tapeInspection.listMemoryViewManifestsByAgent).mockReturnValue([ + { + sessionId: 's1', + messageId: 'msg-1', + entryId: 30, + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: 10, + selectedCount: 6, + selectedIds: ['m-string', 'm-object'], + droppedCount: 0, + queryHash: 'hash', + createdAt: 300 } ]) - ;(runtime as any).sqlitePresenter = { - deepchatTapeEntriesTable: { - listMemoryViewManifestAnchorsByAgent - } - } - const result = await dispatchDeepchatRoute( runtime, 'memory.listViewManifests', @@ -2520,6 +2468,48 @@ describe('dispatchDeepchatRoute', () => { }) ] }) + expect(result.manifests[0]).not.toHaveProperty('payload_json') + expect(result.manifests[0]).not.toHaveProperty('meta_json') + }) + + it('reads memory source spans through the DTO-only Tape inspection port', async () => { + const { runtime, tapeInspection } = createRuntime() + const getManagementVisibleByIds = vi.fn().mockReturnValue([ + { + id: 'memory-1', + agent_id: 'deepchat', + source_session: 's1', + source_entry_ids: '[2,3]' + } + ]) + const getEffectiveMessageSourceSpan = vi + .mocked(tapeInspection.getEffectiveMessageSourceSpan) + .mockReturnValue([ + { + entryId: 2, + record: { + role: 'user', + orderSeq: 1, + content: JSON.stringify({ text: 'source context' }) + } + } + ]) + ;(runtime as any).memoryService = { getManagementVisibleByIds } + + const result = await dispatchDeepchatRoute( + runtime, + 'memory.getSourceSpan', + { agentId: 'deepchat', memoryId: 'memory-1' }, + { webContentsId: 42, windowId: 7 } + ) + + expect(getEffectiveMessageSourceSpan).toHaveBeenCalledWith('s1', [2, 3]) + expect(result).toEqual({ + span: { + sessionId: 's1', + entries: [{ entryId: 2, role: 'user', content: 'source context', orderSeq: 1 }] + } + }) }) it('dispatches memory.getByIds with deepchat guard and input order projection', async () => { @@ -2702,13 +2692,8 @@ describe('dispatchDeepchatRoute', () => { }) it('returns no memory view manifests for missing or non-DeepChat agents', async () => { - const { runtime, providerSettings } = createRuntime() - const listMemoryViewManifestAnchorsByAgent = vi.fn() - ;(runtime as any).sqlitePresenter = { - deepchatTapeEntriesTable: { - listMemoryViewManifestAnchorsByAgent - } - } + const { runtime, providerSettings, tapeInspection } = createRuntime() + const listMemoryViewManifestsByAgent = vi.mocked(tapeInspection.listMemoryViewManifestsByAgent) vi.mocked(providerSettings.getAgentType) .mockResolvedValueOnce(null) .mockResolvedValueOnce('acp') @@ -2729,7 +2714,7 @@ describe('dispatchDeepchatRoute', () => { { webContentsId: 42, windowId: 7 } ) ).resolves.toEqual({ manifests: [] }) - expect(listMemoryViewManifestAnchorsByAgent).not.toHaveBeenCalled() + expect(listMemoryViewManifestsByAgent).not.toHaveBeenCalled() }) it('dispatches bounded memory pages and returns an opaque keyset cursor', async () => { @@ -2803,41 +2788,31 @@ describe('dispatchDeepchatRoute', () => { }) it('does not expand all sessions when listing memory view manifests', async () => { - const { runtime, providerSettings } = createRuntime() + const { runtime, providerSettings, tapeInspection } = createRuntime() vi.mocked(providerSettings.getAgentType).mockResolvedValueOnce('deepchat') const listSessions = vi.fn(() => Array.from({ length: 1200 }, (_, index) => ({ id: `s-${index}` })) ) - const listMemoryViewManifestAnchorsByAgent = vi.fn().mockReturnValue([ - { - session_id: 's-1199', - entry_id: 1, - kind: 'anchor', - name: 'memory/view_assembled', - source_type: 'memory', - source_id: 'msg-1', - source_seq: 0, - provenance_key: null, - payload_json: JSON.stringify({ - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 10, - selected: ['m1'], - dropped: [], - queryHash: 'hash' - } - }), - meta_json: JSON.stringify({ messageId: 'msg-1' }), - created_at: 100 - } - ]) + const listMemoryViewManifestsByAgent = vi + .mocked(tapeInspection.listMemoryViewManifestsByAgent) + .mockReturnValue([ + { + sessionId: 's-1199', + messageId: 'msg-1', + entryId: 1, + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: 10, + selectedCount: 1, + selectedIds: ['m1'], + droppedCount: 0, + queryHash: 'hash', + createdAt: 100 + } + ]) ;(runtime as any).sqlitePresenter = { newSessionsTable: { list: listSessions - }, - deepchatTapeEntriesTable: { - listMemoryViewManifestAnchorsByAgent } } @@ -2849,7 +2824,7 @@ describe('dispatchDeepchatRoute', () => { ) expect(listSessions).not.toHaveBeenCalled() - expect(listMemoryViewManifestAnchorsByAgent).toHaveBeenCalledWith('a', { + expect(listMemoryViewManifestsByAgent).toHaveBeenCalledWith('a', { sessionId: undefined, limit: 100, messageId: undefined diff --git a/test/main/scripts/memoryTestScope.test.ts b/test/main/scripts/memoryTestScope.test.ts index d7c6d1604b..4c5117c4e0 100644 --- a/test/main/scripts/memoryTestScope.test.ts +++ b/test/main/scripts/memoryTestScope.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it, vi } from 'vitest' -import { validateMemoryTestScope } from '../../../scripts/check-memory-test-scope.mjs' +import { + findRequiredNativeTapeTests, + validateMemoryTestScope +} from '../../../scripts/check-memory-test-scope.mjs' const rootDir = process.cwd() const baseManifest = { @@ -120,4 +123,66 @@ describe('memory test scope guard', () => { ).toEqual([]) expect(readFile).not.toHaveBeenCalled() }) + + it('requires discovered native Tape tests to stay in the native scope', () => { + const requiredPath = 'tape-native.test.ts' + const paths = new Set([...existingPaths, requiredPath]) + const contents = new Map([...fileContents, [requiredPath, 'export {}']]) + + expect( + validateMemoryTestScope({ + rootDir, + manifest: baseManifest, + existingPaths: paths, + discoveredPaths: [], + requiredNativePaths: [requiredPath], + fileContents: contents + }) + ).toContainEqual(expect.stringContaining('not classified as native')) + + const manifest = structuredClone(baseManifest) + manifest.native.push(requiredPath) + expect( + validateMemoryTestScope({ + rootDir, + manifest, + existingPaths: paths, + discoveredPaths: [], + requiredNativePaths: [requiredPath], + fileContents: contents + }) + ).toEqual([]) + }) + + it('discovers only split Tape suites with native SQLite gates', () => { + const paths = [ + 'test/main/session/data/tapeRecall.test.ts', + 'test/main/session/data/tapeReplay.spec.ts', + 'test/main/session/data/tapeReconciler.test.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.spec.ts', + 'test/main/memory/agentMemoryTable.test.ts' + ] + const contents = new Map([ + ['test/main/session/data/tapeRecall.test.ts', 'itIfSqlite(\'uses SQLite\', () => {})'], + ['test/main/session/data/tapeReplay.spec.ts', 'describeIfSqlite(\'uses SQLite\', () => {})'], + ['test/main/session/data/tapeReconciler.test.ts', 'it(\'stays portable\', () => {})'], + [ + 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts', + 'describeIfSqlite(\'uses SQLite\', () => {})' + ], + [ + 'test/main/session/data/tables/deepchatTapeEntriesTable.spec.ts', + 'itIfSqlite(\'uses SQLite\', () => {})' + ], + ['test/main/memory/agentMemoryTable.test.ts', 'itIfSqlite(\'outside Tape\', () => {})'] + ]) + + expect(findRequiredNativeTapeTests(paths, (path) => contents.get(path) ?? '')).toEqual([ + 'test/main/session/data/tapeRecall.test.ts', + 'test/main/session/data/tapeReplay.spec.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.spec.ts' + ]) + }) }) diff --git a/test/main/session/data/settings.test.ts b/test/main/session/data/settings.test.ts index 357e85514e..76f9394c57 100644 --- a/test/main/session/data/settings.test.ts +++ b/test/main/session/data/settings.test.ts @@ -7,12 +7,22 @@ const sqlitePresenterModule = sqliteModule const sessionStoreModule = sqliteModule ? await import('../../../../src/main/session/data/settings') : null +const sessionDatabaseModule = sqliteModule + ? await import('../../../../src/main/session/data/database') + : null +const sessionTapeModule = sqliteModule + ? await import('../../../../src/main/tape/application/sessionTape') + : null const Database = sqliteModule?.default const MainDatabase = sqlitePresenterModule?.MainDatabase const SessionSettingsStore = sessionStoreModule?.SessionSettingsStore +const SessionDatabase = sessionDatabaseModule?.SessionDatabase +const SessionTape = sessionTapeModule?.SessionTape const MainDatabaseCtor = MainDatabase! const SessionSettingsStoreCtor = SessionSettingsStore! +const SessionDatabaseCtor = SessionDatabase! +const SessionTapeCtor = SessionTape! let sqliteAvailable = false if (Database) { @@ -29,18 +39,20 @@ const describeIfSqlite = sqliteAvailable ? describe : describe.skip describeIfSqlite('SessionSettingsStore tape summary state', () => { function createStore() { - const sqlitePresenter = new MainDatabaseCtor(':memory:') - const store = new SessionSettingsStoreCtor(sqlitePresenter) - return { sqlitePresenter, store } + const connection = new MainDatabaseCtor(':memory:') + const database = new SessionDatabaseCtor(connection) + const tape = new SessionTapeCtor(database) + const store = new SessionSettingsStoreCtor(database, tape) + return { connection, database, store } } it('creates a bootstrap anchor for each session', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') - store.create('s2', 'openai', 'gpt-4o-mini') + store.create('s1', 'openai', 'gpt-4o', 'full_access') + store.create('s2', 'openai', 'gpt-4o-mini', 'full_access') - expect(sqlitePresenter.deepchatTapeEntriesTable.getBySession('s1')).toMatchObject([ + expect(database.deepchatTapeEntriesTable.getBySession('s1')).toMatchObject([ { session_id: 's1', entry_id: 1, @@ -48,7 +60,7 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { name: 'session/start' } ]) - expect(sqlitePresenter.deepchatTapeEntriesTable.getBySession('s2')).toMatchObject([ + expect(database.deepchatTapeEntriesTable.getBySession('s2')).toMatchObject([ { session_id: 's2', entry_id: 1, @@ -57,13 +69,13 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { } ]) - sqlitePresenter.close() + connection.close() }) it('prefers compaction summary anchors over legacy summary columns', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', { summaryText: 'legacy summary', summaryCursorOrderSeq: 2, @@ -101,24 +113,24 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { } }) expect(store.getSummaryState('s1')).toEqual(result.currentState) - expect(sqlitePresenter.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toMatchObject({ + expect(database.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toMatchObject({ name: 'compaction/manual', created_at: 100 }) - sqlitePresenter.close() + connection.close() }) it('uses handoff anchors as context reconstruction state', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', { summaryText: 'legacy summary', summaryCursorOrderSeq: 2, summaryUpdatedAt: 50 }) - sqlitePresenter.deepchatTapeEntriesTable.appendAnchor({ + database.deepchatTapeEntriesTable.appendAnchor({ sessionId: 's1', name: 'handoff/manual', state: { @@ -134,14 +146,14 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryUpdatedAt: 120 }) - sqlitePresenter.close() + connection.close() }) it('uses handoff cursor even when handoff state has no summary', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') - sqlitePresenter.deepchatTapeEntriesTable.appendAnchor({ + store.create('s1', 'openai', 'gpt-4o', 'full_access') + database.deepchatTapeEntriesTable.appendAnchor({ sessionId: 's1', name: 'handoff/manual', state: { @@ -157,19 +169,19 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryUpdatedAt: null }) - sqlitePresenter.close() + connection.close() }) it('compares summary state against tape reconstruction anchors before writing compaction anchors', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', { summaryText: 'legacy summary', summaryCursorOrderSeq: 2, summaryUpdatedAt: 50 }) - sqlitePresenter.deepchatTapeEntriesTable.appendAnchor({ + database.deepchatTapeEntriesTable.appendAnchor({ sessionId: 's1', name: 'handoff/manual', state: { @@ -208,21 +220,19 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryUpdatedAt: 200 } }) - expect( - sqlitePresenter.deepchatTapeEntriesTable.getLatestReconstructionAnchor('s1') - ).toMatchObject({ + expect(database.deepchatTapeEntriesTable.getLatestReconstructionAnchor('s1')).toMatchObject({ name: 'compaction/auto', created_at: 200 }) - sqlitePresenter.close() + connection.close() }) it('does not apply no-anchor summary updates over tape-backed state', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') - sqlitePresenter.deepchatTapeEntriesTable.appendAnchor({ + store.create('s1', 'openai', 'gpt-4o', 'full_access') + database.deepchatTapeEntriesTable.appendAnchor({ sessionId: 's1', name: 'handoff/manual', state: { @@ -256,13 +266,13 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { }) expect(store.getSummaryState('s1')).toEqual(result.currentState) - sqlitePresenter.close() + connection.close() }) it('does not write a stale anchor when summary compare-and-set fails', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', { summaryText: 'newer summary', summaryCursorOrderSeq: 5, @@ -298,15 +308,59 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryUpdatedAt: 200 } }) - expect(sqlitePresenter.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toBeUndefined() + expect(database.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toBeUndefined() + + connection.close() + }) + + it('rolls back summary state when the matching tape anchor cannot be appended', () => { + const { connection, database, store } = createStore() + const originalState = { + summaryText: 'stable summary', + summaryCursorOrderSeq: 3, + summaryUpdatedAt: 50 + } + + store.create('s1', 'openai', 'gpt-4o', 'full_access') + store.updateSummaryState('s1', originalState) + database.getDatabase().exec(` + CREATE TRIGGER fail_compaction_anchor + BEFORE INSERT ON deepchat_tape_entries + WHEN NEW.name = 'compaction/failing' + BEGIN + SELECT RAISE(ABORT, 'forced tape anchor failure'); + END; + `) + + expect(() => + store.compareAndSetSummaryState( + 's1', + originalState, + { + summaryText: 'must roll back', + summaryCursorOrderSeq: 6, + summaryUpdatedAt: 100 + }, + { + name: 'compaction/failing', + state: { + summary: 'must roll back', + cursorOrderSeq: 6 + } + } + ) + ).toThrow('forced tape anchor failure') + + expect(store.getSummaryState('s1')).toEqual(originalState) + expect(database.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toBeUndefined() - sqlitePresenter.close() + connection.close() }) it('uses reset anchors to invalidate older compaction anchors', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.compareAndSetSummaryState( 's1', { @@ -335,10 +389,10 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryCursorOrderSeq: 1, summaryUpdatedAt: null }) - expect(sqlitePresenter.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toMatchObject({ + expect(database.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toMatchObject({ name: 'summary/reset' }) - sqlitePresenter.close() + connection.close() }) }) diff --git a/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts b/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts index 13e7c9e9d3..aa8efc4a2c 100644 --- a/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts +++ b/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts @@ -1,12 +1,17 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' const sqliteModule = await import('better-sqlite3-multiple-ciphers').catch(() => null) const tableModule = sqliteModule ? await import('@/session/data/tables/deepchatTapeEntries') : null +const lifecycleModule = sqliteModule + ? await import('@/tape/infrastructure/sqlite/tapeLifecycleAdapter') + : null const Database = sqliteModule?.default const DeepChatTapeEntriesTable = tableModule?.DeepChatTapeEntriesTable +const SqliteTapeLifecycleAdapter = lifecycleModule?.SqliteTapeLifecycleAdapter const DatabaseCtor = Database! const DeepChatTapeEntriesTableCtor = DeepChatTapeEntriesTable! +const SqliteTapeLifecycleAdapterCtor = SqliteTapeLifecycleAdapter! let sqliteAvailable = false if (Database) { @@ -26,7 +31,8 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { const db = new DatabaseCtor(':memory:') const table = new DeepChatTapeEntriesTableCtor(db) table.createTable() - return { db, table } + const lifecycle = new SqliteTapeLifecycleAdapterCtor(db) + return { db, table, lifecycle } } it('keeps memory/persona anchors out of context reconstruction (C7, AC-7.3)', () => { @@ -86,7 +92,7 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { }) it('assigns a new Tape incarnation when a session Tape is rebuilt', () => { - const { db, table } = createTable() + const { db, table, lifecycle } = createTable() table.ensureBootstrapAnchor('s1') table.ensureBootstrapAnchor('s2') @@ -101,7 +107,7 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { expect(secondIncarnation).toEqual(expect.any(String)) expect(secondIncarnation).not.toBe(firstIncarnation) - table.deleteBySession('s1') + lifecycle.deleteBySession('s1') table.ensureBootstrapAnchor('s1') const rebuiltIncarnation = JSON.parse( table.getFirstEntriesBySessions(['s1'])[0].meta_json @@ -112,6 +118,43 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { db.close() }) + it('deletes Tape and its mutation projection through the lifecycle adapter', () => { + const db = new DatabaseCtor(':memory:') + const projection = { + applyAppendedEntry: vi.fn(() => true), + invalidateSession: vi.fn(), + deleteBySession: vi.fn() + } + const table = new DeepChatTapeEntriesTableCtor(db, projection) + table.createTable() + const lifecycle = new SqliteTapeLifecycleAdapterCtor(db, projection) + table.ensureBootstrapAnchor('s1') + + lifecycle.deleteBySession('s1') + + expect(table.getBySession('s1')).toEqual([]) + expect(projection.deleteBySession).toHaveBeenCalledWith('s1') + db.close() + }) + + it('rolls back Tape deletion when mutation projection cleanup fails', () => { + const db = new DatabaseCtor(':memory:') + const table = new DeepChatTapeEntriesTableCtor(db) + table.createTable() + table.ensureBootstrapAnchor('s1') + const lifecycle = new SqliteTapeLifecycleAdapterCtor(db, { + applyAppendedEntry: vi.fn(() => true), + invalidateSession: vi.fn(), + deleteBySession: vi.fn(() => { + throw new Error('projection cleanup failed') + }) + }) + + expect(() => lifecycle.deleteBySession('s1')).toThrow('projection cleanup failed') + expect(table.getBySession('s1')).toHaveLength(1) + db.close() + }) + it('tracks the latest summary-related anchor only within the requested session', () => { const { db, table } = createTable() diff --git a/test/main/session/data/tape.test.ts b/test/main/session/data/tape.test.ts deleted file mode 100644 index cb5521f7cb..0000000000 --- a/test/main/session/data/tape.test.ts +++ /dev/null @@ -1,5531 +0,0 @@ -import { performance } from 'node:perf_hooks' -import { describe, expect, it, vi } from 'vitest' -import { buildContext } from '@/agent/deepchat/runtime/contextBuilder' -import { toAppSessionId } from '@/agent/shared/agentSessionIds' -import { SessionTape } from '@/session/data/tape' -import { buildEffectiveTapeView, searchEffectiveTapeRows } from '@/session/data/tapeEffectiveView' -import { - createTapeViewManifest, - type TapeViewManifestBuildInput -} from '@/session/data/tapeViewManifest' -import { - appendMessageRecordToTape, - appendMessageReplacementToTape, - appendMessageRetractionToTape, - appendToolFactsToTape -} from '@/session/data/tapeFacts' -import { buildRequestRefs } from '@/session/data/tapeViewManifest' -import { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' -import { - DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, - DeepChatTapeSearchProjectionTable -} from '@/session/data/tables/deepchatTapeSearchProjection' -import { DeepChatMemoryIngestionProjectionTable } from '@/memory/data/tables/deepchatMemoryIngestionProjection' -import { DeepChatMessagesTable } from '@/session/data/tables/deepchatMessages' -import { DeepChatMessageTracesTable } from '@/session/data/tables/deepchatMessageTraces' -import { DeepChatSessionsTable } from '@/session/data/tables/deepchatSessions' -import { NewSessionsTable } from '@/session/data/tables/newSessions' -import type { ChatMessageRecord } from '@shared/types/agent-interface' - -const sqliteModule = await import('better-sqlite3-multiple-ciphers').catch(() => null) -const Database = sqliteModule?.default -const DatabaseCtor = Database! -const sqliteSkipReason = 'skipped: better-sqlite3-multiple-ciphers is unavailable' -const requireNativeSqlite = process.env.DEEPCHAT_REQUIRE_NATIVE_SQLITE === '1' - -let sqliteAvailable = false -if (Database) { - try { - const smokeDb = new Database(':memory:') - smokeDb.close() - sqliteAvailable = true - } catch { - sqliteAvailable = false - } -} - -const itIfSqlite = sqliteAvailable - ? it - : requireNativeSqlite - ? (name: string, _test: () => unknown, timeout?: number) => - it( - name, - () => { - throw new Error(sqliteSkipReason) - }, - timeout - ) - : it.skip - -function createTapeTableMock() { - const entries: any[] = [] - let tapeIncarnationSequence = 0 - const table = { - ensureBootstrapAnchor: vi.fn((sessionId: string) => { - if ( - entries.some((entry) => entry.session_id === sessionId && entry.name === 'session/start') - ) { - return - } - table.appendAnchor({ - sessionId, - name: 'session/start', - source: { type: 'session', id: sessionId, seq: 0 }, - state: { owner: 'human' }, - meta: { tapeIncarnationId: `test-tape-${++tapeIncarnationSequence}` }, - idempotent: true - }) - }), - append: vi.fn((input: any) => { - const provenanceKey = - input.provenanceKey !== undefined - ? input.provenanceKey - : input.source - ? [ - input.source.type, - input.source.id, - input.source.seq ?? 0, - input.kind, - input.name ?? '' - ].join(':') - : null - const existing = input.idempotent - ? entries.find( - (entry) => - entry.session_id === input.sessionId && entry.provenance_key === provenanceKey - ) - : null - if (existing) { - return existing - } - const row = { - session_id: input.sessionId, - entry_id: - Math.max( - 0, - ...entries - .filter((entry) => entry.session_id === input.sessionId) - .map((entry) => entry.entry_id) - ) + 1, - kind: input.kind, - name: input.name ?? null, - source_type: input.source?.type ?? null, - source_id: input.source?.id ?? null, - source_seq: input.source?.seq ?? null, - provenance_key: provenanceKey, - payload_json: JSON.stringify(input.payload ?? {}), - meta_json: JSON.stringify(input.meta ?? {}), - created_at: input.createdAt ?? Date.now() - } - entries.push(row) - return row - }), - appendAnchor: vi.fn((input: any) => - table.append({ - ...input, - kind: 'anchor', - payload: { name: input.name, state: input.state } - }) - ), - appendEvent: vi.fn((input: any) => - table.append({ - ...input, - kind: 'event', - payload: { name: input.name, data: input.data } - }) - ), - runInTransaction: vi.fn((operation: () => unknown) => { - const snapshot = entries.map((entry) => ({ ...entry })) - try { - return operation() - } catch (error) { - entries.splice(0, entries.length, ...snapshot) - throw error - } - }), - getBySession: vi.fn((sessionId: string) => - entries.filter((entry) => entry.session_id === sessionId) - ), - getSubagentLineageEvents: vi.fn((sessionId: string) => - entries.filter( - (entry) => - entry.session_id === sessionId && - entry.kind === 'event' && - (entry.name === 'subagent/tape_linked' || entry.name === 'fork/merge') - ) - ), - getFirstEntriesBySessions: vi.fn((sessionIds: string[]) => - [...new Set(sessionIds)] - .flatMap((sessionId) => { - const first = entries - .filter((entry) => entry.session_id === sessionId) - .sort((left, right) => left.entry_id - right.entry_id)[0] - return first ? [first] : [] - }) - .sort((left, right) => left.session_id.localeCompare(right.session_id)) - ), - getBySessionUpToEntryId: vi.fn((sessionId: string, maxEntryId: number) => - entries.filter((entry) => entry.session_id === sessionId && entry.entry_id <= maxEntryId) - ), - getMaxEntryId: vi.fn((sessionId: string) => - Math.max( - 0, - ...entries.filter((entry) => entry.session_id === sessionId).map((entry) => entry.entry_id) - ) - ), - getMaxEntryIdsBySessions: vi.fn( - (sessionIds: string[]) => - new Map( - sessionIds.map((sessionId) => [ - sessionId, - Math.max( - 0, - ...entries - .filter((entry) => entry.session_id === sessionId) - .map((entry) => entry.entry_id) - ) - ]) - ) - ), - getLatestAnchor: vi.fn( - (sessionId: string) => - entries - .filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor') - .sort((left, right) => right.entry_id - left.entry_id)[0] - ), - getAnchors: vi.fn((sessionId: string, limit: number = 20) => - entries - .filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor') - .sort((left, right) => right.entry_id - left.entry_id) - .slice(0, Math.min(Math.max(Math.floor(limit), 1), 100)) - .reverse() - ), - getLatestSummaryAnchor: vi.fn( - (sessionId: string) => - entries - .filter( - (entry) => - entry.session_id === sessionId && - entry.kind === 'anchor' && - ['compaction/migrated_summary', 'compaction/manual', 'summary/reset'].includes( - entry.name - ) - ) - .sort((left, right) => right.entry_id - left.entry_id)[0] - ), - getByProvenanceKey: vi.fn((sessionId: string, provenanceKey: string) => - entries.find( - (entry) => entry.session_id === sessionId && entry.provenance_key === provenanceKey - ) - ), - countBySession: vi.fn( - (sessionId: string) => entries.filter((entry) => entry.session_id === sessionId).length - ), - countAnchorsBySession: vi.fn( - (sessionId: string) => - entries.filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor').length - ), - countEntriesAfter: vi.fn( - (sessionId: string, entryId: number) => - entries.filter((entry) => entry.session_id === sessionId && entry.entry_id > entryId).length - ), - search: vi.fn((sessionId: string, query: string, options: any = {}) => { - const normalizedQuery = query.trim() - if (!normalizedQuery) { - return [] - } - const limit = Number.isFinite(options.limit) ? Math.floor(options.limit) : 20 - return entries - .filter((entry) => entry.session_id === sessionId) - .filter( - (entry) => - entry.payload_json.includes(normalizedQuery) || - entry.meta_json.includes(normalizedQuery) || - entry.name?.includes(normalizedQuery) - ) - .filter((entry) => !options.kinds?.length || options.kinds.includes(entry.kind)) - .filter( - (entry) => - !Number.isFinite(options.startCreatedAt) || entry.created_at >= options.startCreatedAt - ) - .filter( - (entry) => - !Number.isFinite(options.endCreatedAt) || entry.created_at <= options.endCreatedAt - ) - .sort((left, right) => right.entry_id - left.entry_id) - .slice(0, Math.min(Math.max(limit, 1), 100)) - }), - searchEffectiveSourcesAtHeads: vi.fn((sources: any[], query: string, options: any = {}) => - sources - .flatMap((source) => - searchEffectiveTapeRows( - entries.filter( - (entry) => - entry.session_id === source.sessionId && entry.entry_id <= source.maxEntryId - ), - query, - { ...options, limit: 100 } - ) - ) - .sort( - (left, right) => - right.created_at - left.created_at || - left.session_id.localeCompare(right.session_id) || - right.entry_id - left.entry_id - ) - .slice(0, Math.min(Math.max(Math.floor(options.limit ?? 20), 1), 100)) - ), - getEffectiveContextRowsAtHead: vi.fn( - ( - source: any, - entryIds: number[], - options: { before: number; after: number; limit: number } - ) => { - const effectiveRows = buildEffectiveTapeView( - entries.filter( - (entry) => entry.session_id === source.sessionId && entry.entry_id <= source.maxEntryId - ), - { includePending: false } - ).rows - const indexesByEntryId = new Map( - effectiveRows.map((entry, index) => [entry.entry_id, index]) - ) - const indexes: number[] = [] - for (const entryId of entryIds) { - const index = indexesByEntryId.get(entryId) - if (index !== undefined) indexes.push(index) - } - for (const entryId of entryIds) { - const index = indexesByEntryId.get(entryId) - if (index === undefined) continue - for ( - let cursor = Math.max(0, index - options.before); - cursor <= Math.min(effectiveRows.length - 1, index + options.after); - cursor += 1 - ) { - if (cursor !== index) indexes.push(cursor) - } - } - return [...new Set(indexes)].slice(0, options.limit).map((index) => effectiveRows[index]) - } - ), - deleteBySession: vi.fn((sessionId: string) => { - for (let index = entries.length - 1; index >= 0; index -= 1) { - if (entries[index].session_id === sessionId) { - entries.splice(index, 1) - } - } - }) - } - return { table, entries } -} - -function createRecord(overrides: Partial): ChatMessageRecord { - return { - id: 'm1', - sessionId: 's1', - orderSeq: 1, - role: 'user', - content: JSON.stringify({ text: 'hello', files: [], links: [], search: false, think: false }), - status: 'sent', - isContextEdge: 0, - metadata: '{}', - traceCount: 0, - createdAt: 100, - updatedAt: 100, - ...overrides - } -} - -function createTraceRow(overrides: Record = {}) { - return { - id: 'trace-1', - message_id: 'a1', - session_id: 's1', - provider_id: 'openai', - model_id: 'gpt-4o', - request_seq: 1, - endpoint: 'https://api.openai.test/v1/chat/completions', - headers_json: '{"authorization":"[redacted]"}', - body_json: '{"messages":[{"role":"user","content":"hello"}]}', - truncated: 0, - created_at: 300, - ...overrides - } -} - -function createMessageRow(overrides: Record = {}) { - return { - id: 'a1', - session_id: 's1', - order_seq: 2, - role: 'assistant', - content: '[{"type":"content","content":"done","status":"success"}]', - status: 'sent', - is_context_edge: 0, - metadata: '{"totalTokens":10}', - created_at: 200, - updated_at: 300, - ...overrides - } -} - -function createObservationManifest( - overrides: Partial[0]> = {} -) { - return createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user', content: 'hello' }], - tools: [], - latestEntryId: 0, - anchorEntryIds: [], - included: [], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: true, - assembledAt: 200, - ...overrides - }) -} - -function createTapeService( - table: unknown, - traceRows: Array> = [], - messageRows: Array> = [] -) { - return new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: { - deleteBySession: vi.fn(), - getByEntryIds: vi.fn().mockReturnValue([]) - }, - deepchatMessageTracesTable: { - listByMessageId: vi.fn((messageId: string) => - traceRows.filter((row) => row.message_id === messageId) - ) - }, - deepchatMessagesTable: { - get: vi.fn((messageId: string) => messageRows.find((row) => row.id === messageId)) - }, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) -} - -function createLinkedTapeService( - table: unknown, - sessions: Array<{ - id: string - session_kind: 'regular' | 'subagent' - parent_session_id: string | null - }>, - projectionTable?: unknown -) { - const sessionById = new Map(sessions.map((session) => [session.id, session])) - return { - service: new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - newSessionsTable: { - get: vi.fn((sessionId: string) => sessionById.get(sessionId)), - getMany: vi.fn((sessionIds: string[]) => - sessionIds.flatMap((sessionId) => { - const session = sessionById.get(sessionId) - return session ? [session] : [] - }) - ) - }, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any), - sessionById - } -} - -function createSubagentLinkInput(parentSessionId: string, childSessionId: string) { - return { - parentSessionId, - childSessionId, - runId: `run-${childSessionId}`, - taskId: `task-${childSessionId}`, - slotId: 'reviewer', - taskTitle: `Review ${childSessionId}`, - outcome: 'completed' as const, - resultSummary: 'Done' - } -} - -function appendObservationIsolationFacts(table: unknown) { - const original = createRecord({ id: 'u1', orderSeq: 1, createdAt: 100, updatedAt: 100 }) - const edited = createRecord({ - id: 'u1', - orderSeq: 1, - content: JSON.stringify({ - text: 'edited', - files: [], - links: [], - search: false, - think: false - }), - createdAt: 100, - updatedAt: 150 - }) - const retracted = createRecord({ id: 'u2', orderSeq: 2, createdAt: 160, updatedAt: 160 }) - const pending = createRecord({ - id: 'a1', - orderSeq: 3, - role: 'assistant', - status: 'pending', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'pending', - timestamp: 200, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}' } - } - ]), - createdAt: 200, - updatedAt: 200 - }) - const final = createRecord({ - id: 'a1', - orderSeq: 3, - role: 'assistant', - status: 'sent', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 300, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'tape-result-secret' - } - } - ]), - metadata: '{"totalTokens":12}', - createdAt: 200, - updatedAt: 300 - }) - - appendMessageRecordToTape(table as any, original, 'live') - appendMessageReplacementToTape(table as any, edited, 'test_edit') - appendMessageRecordToTape(table as any, retracted, 'live') - appendMessageRetractionToTape(table as any, retracted, 'test_delete') - appendMessageRecordToTape(table as any, pending, 'live') - appendMessageRecordToTape(table as any, final, 'live') - - return { edited, final } -} - -function stripObservationPayloadOptIns(value: T): T { - const copy = structuredClone(value) as any - const stripEntryPayloads = (entries: any[] | undefined) => { - for (const entry of entries ?? []) { - delete entry.payload - delete entry.meta - } - } - - if (copy.request?.state === 'manifest_bound') { - stripEntryPayloads(copy.request.replay.entries) - delete copy.request.replay.hashes.sliceHash - if (copy.request.replay.trace) { - delete copy.request.replay.trace.headersJson - delete copy.request.replay.trace.bodyJson - } - } else if (copy.request?.trace) { - delete copy.request.trace.headersJson - delete copy.request.trace.bodyJson - } - stripEntryPayloads(copy.output?.entries) - return copy -} - -function createSpies(names: string[]) { - return Object.fromEntries(names.map((name) => [name, vi.fn()])) as Record< - string, - ReturnType - > -} - -function trackMemoryPropertyAccess(target: T) { - const memoryPropertyAccess = vi.fn() - return { - memoryPropertyAccess, - presenter: new Proxy(target, { - get(value, property, receiver) { - if (typeof property === 'string' && /memory/i.test(property)) { - memoryPropertyAccess(property) - } - return Reflect.get(value, property, receiver) - } - }) - } -} - -function readObservationMatrix(service: SessionTape) { - return { - defaultObservation: service.readCausalObservationSlice('s1', 'a1'), - repeatedObservation: service.readCausalObservationSlice('s1', 'a1'), - explicitObservation: service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }), - optInObservation: service.readCausalObservationSlice('s1', 'a1', { - includeTapePayloads: true, - includeTracePayload: true - }), - traceOnlyObservation: service.readCausalObservationSlice('s1', 'a-trace') - } -} - -describe('SessionTape', () => { - it('backfills message and tool facts idempotently before returning tape records', () => { - const { table, entries } = createTapeTableMock() - const assistantBlocks = [ - { - type: 'tool_call', - status: 'success', - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } - } - ] - const records = [ - createRecord({ id: 'u1', orderSeq: 1 }), - createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify(assistantBlocks), - createdAt: 120, - updatedAt: 120 - }) - ] - const messageStore = { - getMessages: vi.fn().mockReturnValue(records) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const first = service.ensureSessionTapeReady('s1', messageStore as any) - const second = service.ensureSessionTapeReady('s1', messageStore as any) - - expect(first.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) - expect(second.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) - expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) - expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) - expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) - expect(entries.filter((entry) => entry.name === 'migration/backfill')).toHaveLength(1) - }) - - it('appends live tool facts through the stable recorder port idempotently', async () => { - const { table, entries } = createTapeTableMock() - const service = createTapeService(table) - const input = { - sessionId: toAppSessionId('s1'), - messageId: 'a1', - orderSeq: 2, - blockIndex: 0, - block: { - type: 'tool_call' as const, - status: 'success' as const, - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } - }, - provenance: { source: 'tool_call' as const, sourceId: 'a1:tc1', sequence: 0 } - } - - const first = await service.appendToolFact(input) - const second = await service.appendToolFact(input) - - expect(second).toEqual(first) - expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) - expect(JSON.parse(entries.find((entry) => entry.kind === 'tool_call').meta_json)).toEqual({ - source: 'live', - role: 'assistant', - status: 'success', - reason: 'tool_loop' - }) - }) - - it('reports info, search, and handoff within one session scope', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([ - createRecord({ id: 'u1' }), - createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { type: 'content', content: 'answer', status: 'success', timestamp: 101 } - ]), - metadata: JSON.stringify({ totalTokens: 9 }), - createdAt: 101, - updatedAt: 101 - }) - ]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - service.handoff('s1', 'phase_done', { summary: ' done ' }) - const handoffAnchor = entries.find((entry) => entry.name === 'handoff/phase_done') - - expect(service.info('s1')).toMatchObject({ - sessionId: 's1', - anchors: 2, - lastAnchor: 'handoff/phase_done', - lastTokenUsage: 9, - migrationState: 'ready' - }) - expect(JSON.parse(handoffAnchor.payload_json).state).toMatchObject({ - summary: 'done', - cursorOrderSeq: 3, - range: { - fromOrderSeq: 1, - toOrderSeq: 2 - }, - sourceMessageIds: ['u1', 'a1'] - }) - expect(service.search('s1', 'hello')).toHaveLength(1) - expect( - service.search('s1', 'hello', { kinds: ['message'], start: '1970-01-01T00:00:00.000Z' }) - ).toHaveLength(1) - expect(service.search('s1', 'hello', { kinds: ['anchor'] })).toHaveLength(0) - expect(service.search('s1', 'hello', { end: '99' })).toHaveLength(0) - expect(() => service.search('s1', 'hello', { start: 'not-a-date' })).toThrow( - 'start must be an ISO date/time or millisecond timestamp.' - ) - expect(service.anchors('s1')).toMatchObject([ - { sessionId: 's1', name: 'session/start' }, - { sessionId: 's1', name: 'handoff/phase_done' } - ]) - expect(service.anchors('s1', { limit: 1 })).toMatchObject([ - { sessionId: 's1', name: 'handoff/phase_done' } - ]) - expect(service.search('s2', 'hello')).toHaveLength(0) - }) - - it('projects fallback tape search into compact results and bounded context', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'Run the dev server', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.append({ - sessionId: 's1', - kind: 'tool_result', - name: 'shell', - source: { type: 'tool_result', id: 'm1:tc1', seq: 0 }, - payload: { - messageId: 'm1', - orderSeq: 2, - toolCallId: 'tc1', - command: 'pnpm run dev', - exitStatus: 1, - response: 'Command failed with EADDRINUSE in /tmp/deepchat.log' - }, - meta: { source: 'live', status: 'error' }, - createdAt: 110 - }) - - const hits = service.search('s1', 'pnpm run dev', { limit: 5 }) - expect(hits).toHaveLength(1) - expect(hits[0]).toMatchObject({ - kind: 'tool_result', - summary: expect.stringContaining('EADDRINUSE'), - refs: { - toolCallId: 'tc1', - commands: expect.arrayContaining(['pnpm run dev']), - filePaths: expect.arrayContaining(['/tmp/deepchat.log']), - errorCodes: expect.arrayContaining(['EADDRINUSE']), - exitStatus: 1 - } - }) - expect(hits[0]).not.toHaveProperty('payload') - expect(hits[0]).not.toHaveProperty('meta') - - const context = service.getContext('s1', [hits[0].entryId], { - before: 0, - after: 0, - maxBytesPerEntry: 12, - maxTotalBytes: 12 - }) - expect(context.matchedEntryIds).toEqual([hits[0].entryId]) - expect(context.entries[0]).toMatchObject({ - entryId: hits[0].entryId, - evidence: { truncated: true } - }) - expect(context.entries[0].evidence.bytes).toBeLessThanOrEqual(12) - expect(context.entries[0]).not.toHaveProperty('payload') - expect(context.entries[0]).not.toHaveProperty('meta') - - const exhaustedContext = service.getContext('s1', [hits[0].entryId], { - before: 0, - after: 0, - maxTotalBytes: 0 - }) - expect(exhaustedContext.entries).toEqual([]) - expect(exhaustedContext.matchedEntryIds).toEqual([]) - }) - - it('projects user message attachment metadata into search text and refs', () => { - const { table } = createTapeTableMock() - const projectionTable = { - isCurrent: vi.fn().mockReturnValue(false), - getSessionMeta: vi.fn().mockReturnValue(null), - getProjectedEntryIds: vi.fn().mockReturnValue([]), - appendSession: vi.fn(), - replaceSession: vi.fn(), - search: vi.fn().mockReturnValue([]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm-file', seq: 0 }, - payload: { - record: createRecord({ - id: 'm-file', - content: JSON.stringify({ - text: 'Please review the attachment', - files: [ - { - name: 'a.md', - path: '/tmp/a.md', - content: 'raw attachment body should not be projected', - metadata: { fileName: 'workspace-a.md' } - } - ], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - - service.search('s1', '/tmp/a.md', { limit: 5 }) - - expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) - const projectedRows = projectionTable.replaceSession.mock.calls[0][1] - expect(projectedRows[0]).toMatchObject({ - entryId: 1, - refs: { - filePaths: expect.arrayContaining(['/tmp/a.md']), - fileNames: expect.arrayContaining(['a.md', 'workspace-a.md']) - } - }) - expect(projectedRows[0].searchText).toContain('/tmp/a.md') - expect(projectedRows[0].searchText).toContain('a.md') - expect(projectedRows[0].searchText).toContain('workspace-a.md') - expect(projectedRows[0].searchText).not.toContain('raw attachment body should not be projected') - }) - - it('preserves relative file paths in projected refs', () => { - const { table } = createTapeTableMock() - let projectedRows: any[] = [] - const projectionTable = { - isCurrent: vi.fn().mockReturnValue(false), - getSessionMeta: vi.fn().mockReturnValue(null), - getProjectedEntryIds: vi.fn().mockReturnValue([]), - appendSession: vi.fn(), - replaceSession: vi.fn((_sessionId: string, rows: any[]) => { - projectedRows = rows - }), - search: vi.fn().mockReturnValue([]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm-relative', seq: 0 }, - payload: { - record: createRecord({ - id: 'm-relative', - content: JSON.stringify({ - text: 'Touched src/main/index.ts, ./lib/util.ts, ../shared/types.ts, test/main/foo/bar.ts, /usr/local/bin/deploy, and https://example.com/not-a-file', - files: [], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - - service.search('s1', 'src/main/index.ts', { limit: 5 }) - - expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) - const filePaths = projectedRows[0].refs.filePaths - expect(filePaths).toEqual( - expect.arrayContaining([ - 'src/main/index.ts', - './lib/util.ts', - '../shared/types.ts', - 'test/main/foo/bar.ts', - '/usr/local/bin/deploy' - ]) - ) - expect(filePaths).not.toContain('/main/index.ts') - expect(filePaths).not.toContain('/lib/util.ts') - expect(filePaths).not.toContain('/main/foo/bar.ts') - expect(filePaths).not.toContain('example.com/not-a-file') - }) - - it('rebuilds old tape projection versions before attachment path search', () => { - const { table } = createTapeTableMock() - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm-file', seq: 0 }, - payload: { - record: createRecord({ - id: 'm-file', - content: JSON.stringify({ - text: 'Please review the migrated attachment', - files: [ - { - name: 'legacy.md', - path: '/tmp/legacy.md', - content: 'raw migrated body must stay out of projection', - metadata: { fileName: 'legacy-workspace.md' } - } - ], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - let storedVersion = 1 - let storedMaxEntryId = 1 - let projectedRows: any[] = [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm-file', - sourceSeq: 0, - searchText: 'message/user Please review the migrated attachment', - summaryText: 'user: Please review the migrated attachment', - refs: { messageId: 'm-file' }, - createdAt: 100 - } - ] - const projectionTable = { - isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => { - return ( - storedVersion === DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION && - storedMaxEntryId === maxEntryId - ) - }), - getSessionMeta: vi.fn(() => ({ - projectionVersion: storedVersion, - maxEntryId: storedMaxEntryId - })), - getProjectedEntryIds: vi.fn().mockReturnValue([1]), - appendSession: vi.fn(), - replaceSession: vi.fn((_sessionId: string, rows: any[], maxEntryId: number) => { - projectedRows = rows - storedVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - storedMaxEntryId = maxEntryId - }), - search: vi.fn((_sessionId: string, query: string) => { - return projectedRows - .filter((row) => row.searchText.includes(query)) - .map((row) => ({ - session_id: row.sessionId, - entry_id: row.entryId, - kind: row.kind, - name: row.name, - source_type: row.sourceType, - source_id: row.sourceId, - source_seq: row.sourceSeq, - search_text: row.searchText, - summary_text: row.summaryText, - refs_json: JSON.stringify(row.refs), - created_at: row.createdAt, - score: 1 - })) - }) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const hits = service.search('s1', '/tmp/legacy.md', { limit: 5 }) - - expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) - expect(projectionTable.appendSession).not.toHaveBeenCalled() - expect(hits).toHaveLength(1) - expect(hits[0].refs).toMatchObject({ - filePaths: expect.arrayContaining(['/tmp/legacy.md']), - fileNames: expect.arrayContaining(['legacy.md', 'legacy-workspace.md']) - }) - expect(projectedRows[0].searchText).not.toContain( - 'raw migrated body must stay out of projection' - ) - }) - - it('prioritizes requested tape context entries before window entries under byte caps', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'before-1', seq: 0 }, - payload: { - record: createRecord({ - id: 'before-1', - content: JSON.stringify({ - text: 'before entry consumes the tiny byte budget first', - files: [], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'before-2', seq: 0 }, - payload: { - record: createRecord({ - id: 'before-2', - content: JSON.stringify({ - text: 'second before entry also appears earlier', - files: [], - links: [] - }), - createdAt: 110, - updatedAt: 110 - }) - }, - meta: { source: 'live', orderSeq: 2, role: 'user' }, - createdAt: 110 - }) - const target = table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'target', seq: 0 }, - payload: { - record: createRecord({ - id: 'target', - content: JSON.stringify({ - text: 'target-ok consumes the available budget', - files: [], - links: [] - }), - createdAt: 120, - updatedAt: 120 - }) - }, - meta: { source: 'live', orderSeq: 3, role: 'user' }, - createdAt: 120 - }) - - const context = service.getContext('s1', [target.entry_id], { - before: 2, - after: 0, - limit: 3, - maxBytesPerEntry: 18, - maxTotalBytes: 18 - }) - - expect(context.matchedEntryIds).toEqual([target.entry_id]) - expect(context.entries.map((entry) => entry.entryId)).toEqual([target.entry_id]) - expect(context.entries[0].evidence.text).toContain('target-ok') - }) - - it('binds tape projection LIKE fallback params for single and multi-term queries', () => { - const all = vi.fn().mockReturnValue([]) - const db = { - prepare: vi.fn((sql: string) => ({ - all: (...params: unknown[]) => all(sql, params), - get: vi.fn().mockReturnValue(undefined), - run: vi.fn() - })) - } - const projectionTable = new DeepChatTapeSearchProjectionTable(db as any) - ;(projectionTable as any).recoverSessionFts = vi.fn() - - projectionTable.search('s1', 'Redis', { limit: 5 }) - expect(all).toHaveBeenLastCalledWith( - expect.stringContaining('FROM deepchat_tape_search_projection'), - ['s1', '%Redis%', '%Redis%', '%Redis%', 5] - ) - - projectionTable.search('s1', 'Redis TTL', { limit: 5 }) - expect(all).toHaveBeenLastCalledWith( - expect.stringContaining('FROM deepchat_tape_search_projection'), - [ - 's1', - '%Redis TTL%', - '%Redis TTL%', - '%Redis TTL%', - '%Redis%', - '%Redis%', - '%Redis%', - '%TTL%', - '%TTL%', - '%TTL%', - 5 - ] - ) - - const oversizedQuery = Array.from({ length: 257 }, (_, index) => `term-${index}`).join(' ') - projectionTable.search('s1', oversizedQuery, { limit: 5 }) - expect(all).toHaveBeenLastCalledWith( - expect.stringContaining('FROM deepchat_tape_search_projection'), - ['s1', `%${oversizedQuery}%`, `%${oversizedQuery}%`, `%${oversizedQuery}%`, 5] - ) - }) - - it('uses current tape projection without loading full session rows', () => { - const { table } = createTapeTableMock() - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'Redis compact marker', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.getBySession.mockClear() - const projectionTable = { - isCurrent: vi.fn().mockReturnValue(true), - getSessionMeta: vi.fn(), - getProjectedEntryIds: vi.fn(), - appendSession: vi.fn(), - replaceSession: vi.fn(), - search: vi.fn().mockReturnValue([ - { - session_id: 's1', - entry_id: 1, - kind: 'message', - name: 'message/user', - source_type: 'message', - source_id: 'm1', - source_seq: 0, - search_text: 'Redis compact marker', - summary_text: 'Redis compact marker', - refs_json: '{"messageId":"m1"}', - created_at: 100, - score: -2 - } - ]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const hits = service.search('s1', 'Redis compact', { limit: 5 }) - - expect(table.getMaxEntryId).toHaveBeenCalledWith('s1') - expect(table.getBySession).not.toHaveBeenCalled() - expect(projectionTable.search).toHaveBeenCalledWith( - 's1', - 'Redis compact', - expect.objectContaining({ limit: 5 }) - ) - expect(projectionTable.appendSession).not.toHaveBeenCalled() - expect(projectionTable.replaceSession).not.toHaveBeenCalled() - expect(hits[0]).toMatchObject({ - entryId: 1, - kind: 'message', - summary: 'Redis compact marker', - refs: { messageId: 'm1' }, - score: -2 - }) - expect(hits[0]).not.toHaveProperty('payload') - expect(hits[0]).not.toHaveProperty('meta') - }) - - it('falls back to effective tape search when projection search throws', () => { - const { table } = createTapeTableMock() - const projectionTable = { - isCurrent: vi.fn().mockReturnValue(true), - search: vi.fn(() => { - throw new Error('projection failed') - }) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'Redis fallback marker', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - - const hits = service.search('s1', 'Redis fallback', { limit: 5 }) - expect(hits).toHaveLength(1) - expect(hits[0]).toMatchObject({ - kind: 'message', - summary: expect.stringContaining('Redis fallback') - }) - expect(hits[0]).not.toHaveProperty('payload') - expect(hits[0]).not.toHaveProperty('meta') - }) - - it('appends tape projection rows when the previous projection is an effective prefix', () => { - const { table } = createTapeTableMock() - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'first redis', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm2', seq: 0 }, - payload: { - record: createRecord({ - id: 'm2', - content: JSON.stringify({ text: 'second vue', files: [], links: [] }), - createdAt: 110, - updatedAt: 110 - }) - }, - meta: { source: 'live', orderSeq: 2, role: 'user' }, - createdAt: 110 - }) - const projectionTable = { - isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => maxEntryId === 1), - getSessionMeta: vi.fn().mockReturnValue({ projectionVersion: 1, maxEntryId: 1 }), - getProjectedEntryIds: vi.fn().mockReturnValue([1]), - appendSession: vi.fn(), - replaceSession: vi.fn(), - search: vi.fn().mockReturnValue([]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.search('s1', 'vue', { limit: 5 }) - - expect(projectionTable.appendSession).toHaveBeenCalledTimes(1) - expect(projectionTable.appendSession.mock.calls[0][1].map((row: any) => row.entryId)).toEqual([ - 2 - ]) - expect(projectionTable.replaceSession).not.toHaveBeenCalled() - }) - - it('rebuilds tape projection when projected entry ids are not an effective prefix', () => { - const { table } = createTapeTableMock() - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'redis', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - const projectionTable = { - isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => maxEntryId === 0), - getSessionMeta: vi.fn().mockReturnValue({ projectionVersion: 1, maxEntryId: 0 }), - getProjectedEntryIds: vi.fn().mockReturnValue([99]), - appendSession: vi.fn(), - replaceSession: vi.fn(), - search: vi.fn().mockReturnValue([]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.search('s1', 'redis', { limit: 5 }) - - expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) - expect(projectionTable.appendSession).not.toHaveBeenCalled() - }) - - it('does not run LIKE fallback when FTS fills the tape projection search limit', () => { - const all = vi.fn((sql: string, _params: unknown[]) => - sql.includes('deepchat_tape_search_fts') - ? [ - { - session_id: 's1', - entry_id: 1, - kind: 'message', - name: 'message/user', - source_type: 'message', - source_id: 'm1', - source_seq: 0, - search_text: 'Redis TTL', - summary_text: 'Redis TTL', - refs_json: '{}', - created_at: 100, - score: -1 - } - ] - : [] - ) - const db = { - exec: vi.fn(() => { - throw new Error('unexpected FTS ensure') - }), - prepare: vi.fn((sql: string) => ({ - all: (...params: unknown[]) => all(sql, params), - get: (..._params: unknown[]) => { - if ( - sql.includes('deepchat_tape_search_projection_meta') || - sql.includes('deepchat_tape_search_fts_meta') - ) { - return { projection_version: 1, max_entry_id: 1 } - } - return undefined - }, - run: vi.fn() - })), - transaction: vi.fn((callback: () => void) => callback) - } - const projectionTable = new DeepChatTapeSearchProjectionTable(db as any) - ;(projectionTable as any).ftsReady = true - - const hits = projectionTable.search('s1', 'Redis', { limit: 1 }) - - expect(hits).toHaveLength(1) - expect(db.exec).not.toHaveBeenCalled() - const ftsCall = all.mock.calls.find(([sql]) => String(sql).includes('deepchat_tape_search_fts')) - expect(String(ftsCall?.[0])).toContain('deepchat_tape_search_fts.session_id = ?') - expect((ftsCall?.[1] as unknown[]).filter((param) => param === 's1')).toHaveLength(2) - expect( - vi.mocked(db.prepare).mock.calls.some(([sql]) => String(sql).includes('NULL AS score')) - ).toBe(false) - }) - - it('queries memory view manifests by agent without expanding session ids', () => { - const all = vi.fn().mockReturnValue([]) - const db = { - prepare: vi.fn((sql: string) => ({ - all: (...params: unknown[]) => all(sql, params) - })) - } - const table = new DeepChatTapeEntriesTable(db as any) - - table.listMemoryViewManifestAnchorsByAgent('agent-a', { - sessionId: 's-1', - messageId: 'msg-1', - limit: 7 - }) - - expect(all).toHaveBeenCalledWith( - expect.stringContaining('INNER JOIN new_sessions AS sessions'), - ['agent-a', 's-1', 'msg-1', 7] - ) - const sql = String(all.mock.calls[0][0]) - expect(sql).not.toContain(' IN (') - expect(sql).toContain('sessions.agent_id = ?') - expect(sql).toContain('tape.session_id = ?') - expect(sql).toContain("json_extract(tape.meta_json, '$.messageId') = ?") - }) - - it('keeps linked raw search candidate-bounded instead of materializing complete Tapes', () => { - const all = vi.fn().mockReturnValue([]) - const db = { - prepare: vi.fn((sql: string) => ({ - all: (...params: unknown[]) => all(sql, params) - })) - } - const table = new DeepChatTapeEntriesTable(db as any) - - table.searchEffectiveSourcesAtHeads( - [ - { sessionId: 'child-b', maxEntryId: 20 }, - { sessionId: 'child-a', maxEntryId: 10 } - ], - 'needle', - { limit: 5 } - ) - - const sql = String(all.mock.calls[0][0]) - const params = all.mock.calls[0][1] - expect(sql).toContain('FROM deepchat_tape_entries AS candidate') - expect(sql).toContain('candidate.entry_id <= source.max_entry_id') - expect(sql).toContain('FROM deepchat_tape_entries AS later_message') - expect(sql).toContain( - "typeof(json_extract(later_message.payload_json, '$.record.content')) = 'text'" - ) - expect(sql).not.toContain('bounded_rows AS') - expect(params[0]).toBe( - JSON.stringify([ - { sessionId: 'child-a', maxEntryId: 10 }, - { sessionId: 'child-b', maxEntryId: 20 } - ]) - ) - expect(params.at(-1)).toBe(5) - }) - - it('reads exact linked projections without invoking FTS recovery or writes', () => { - const run = vi.fn() - const exec = vi.fn() - const reads: Array<{ sql: string; params: unknown[] }> = [] - const prepare = vi.fn((sql: string) => ({ - all: (...params: unknown[]) => { - reads.push({ sql, params }) - if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { - return [{ session_id: 'child', max_entry_id: 2 }] - } - if (sql.includes('SELECT projection.*, NULL AS score')) { - return [ - { - session_id: 'child', - entry_id: 2, - kind: 'event', - name: 'child/result', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'projection needle', - summary_text: 'projection needle', - refs_json: '{}', - created_at: 100, - score: null - } - ] - } - return [] - }, - run - })) - const projectionTable = new DeepChatTapeSearchProjectionTable({ prepare, exec } as any) - - const result = projectionTable.searchSourcesReadOnly( - [{ sessionId: 'child', maxEntryId: 2 }], - 'projection needle', - { limit: 5 } - ) - - expect(result).toMatchObject({ - coveredSources: [{ sessionId: 'child', maxEntryId: 2 }], - rows: [{ session_id: 'child', entry_id: 2 }] - }) - expect(exec).not.toHaveBeenCalled() - expect(run).not.toHaveBeenCalled() - expect(prepare.mock.calls.map(([sql]) => String(sql)).join('\n')).not.toContain( - 'deepchat_tape_search_fts_meta' - ) - expect( - reads.find((read) => read.sql.includes('SELECT projection.*, NULL AS score'))?.params.at(-1) - ).toBe(5) - }) - - it('returns no ranked rows when linked projection coverage is only partial', () => { - const reads: string[] = [] - const prepare = vi.fn((sql: string) => ({ - all: () => { - reads.push(sql) - if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { - return [{ session_id: 'child-a', max_entry_id: 2 }] - } - return [] - } - })) - const projectionTable = new DeepChatTapeSearchProjectionTable({ - prepare, - exec: vi.fn() - } as any) - - const result = projectionTable.searchSourcesReadOnly( - [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 3 } - ], - 'projection needle', - { limit: 5 } - ) - - expect(result).toEqual({ - coveredSources: [{ sessionId: 'child-a', maxEntryId: 2 }], - rows: [] - }) - expect( - reads.some((sql) => sql.includes('FROM deepchat_tape_search_projection AS projection')) - ).toBe(false) - expect(reads.some((sql) => sql.includes('FROM deepchat_tape_search_fts'))).toBe(false) - }) - - it('uses one LIKE ranking when linked FTS freshness is only partial', () => { - const reads: Array<{ sql: string; params: unknown[] }> = [] - const prepare = vi.fn((sql: string) => ({ - all: (...params: unknown[]) => { - reads.push({ sql, params }) - if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { - return [ - { session_id: 'child-a', max_entry_id: 2 }, - { session_id: 'child-b', max_entry_id: 3 } - ] - } - if (sql.includes('deepchat_tape_search_fts_meta AS meta')) { - return [{ session_id: 'child-a', max_entry_id: 2 }] - } - if (sql.includes('SELECT projection.*, NULL AS score')) { - return [ - { - session_id: 'child-b', - entry_id: 3, - kind: 'event', - name: 'child/result', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'projection needle', - summary_text: 'projection needle', - refs_json: '{}', - created_at: 200, - score: null - } - ] - } - return [] - } - })) - const projectionTable = new DeepChatTapeSearchProjectionTable({ - prepare, - exec: vi.fn() - } as any) - ;(projectionTable as any).ftsReady = true - - const sources = [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 3 } - ] - const result = projectionTable.searchSourcesReadOnly(sources, 'projection needle', { limit: 5 }) - - expect(result.rows).toMatchObject([{ session_id: 'child-b', entry_id: 3, score: null }]) - expect(reads.some(({ sql }) => sql.includes('bm25(deepchat_tape_search_fts)'))).toBe(false) - expect( - reads.find(({ sql }) => sql.includes('SELECT projection.*, NULL AS score'))?.params[0] - ).toBe(JSON.stringify(sources)) - }) - - itIfSqlite( - `keeps projected and raw linked multi-term search aligned${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - table.createTable() - projectionTable.createTable() - table.ensureBootstrapAnchor('child') - const entry = table.appendEvent({ - sessionId: 'child', - name: 'child/result', - data: { text: 'alpha separated by several words before beta' }, - createdAt: 100 - }) - const sources = [{ sessionId: 'child', maxEntryId: entry.entry_id }] - projectionTable.replaceSession( - 'child', - [ - { - sessionId: 'child', - entryId: entry.entry_id, - kind: 'event', - name: 'child/result', - sourceType: null, - sourceId: null, - sourceSeq: null, - searchText: 'alpha separated by several words before beta', - summaryText: 'alpha separated by several words before beta', - refs: {}, - createdAt: 100 - } - ], - entry.entry_id - ) - - const projected = projectionTable.searchSourcesReadOnly(sources, 'alpha beta', { limit: 5 }) - const raw = table.searchEffectiveSourcesAtHeads(sources, 'alpha beta', { limit: 5 }) - - expect(projected.coveredSources).toEqual(sources) - expect(raw.map((row) => [row.session_id, row.entry_id])).toEqual( - projected.rows.map((row) => [row.session_id, row.entry_id]) - ) - expect(raw).toHaveLength(1) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `queries effective linked sources and context at frozen heads${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - table.ensureBootstrapAnchor('child-a') - table.ensureBootstrapAnchor('child-b') - table.appendEvent({ - sessionId: 'child-a', - name: 'child/result', - data: { text: 'native linked needle A' }, - createdAt: 100 - }) - table.appendEvent({ - sessionId: 'child-b', - name: 'child/result', - data: { text: 'native linked needle B' }, - createdAt: 200 - }) - table.appendEvent({ - sessionId: 'child-a', - name: 'child/late', - data: { text: 'native linked needle late' }, - createdAt: 300 - }) - - const sources = [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 2 } - ] - expect( - table.searchEffectiveSourcesAtHeads(sources, 'native linked needle', { limit: 1 }) - ).toMatchObject([{ session_id: 'child-b', entry_id: 2 }]) - expect( - table - .searchEffectiveSourcesAtHeads(sources, 'native linked needle', { limit: 10 }) - .map((row) => [row.session_id, row.entry_id]) - ).toEqual([ - ['child-b', 2], - ['child-a', 2] - ]) - expect( - table - .getEffectiveContextRowsAtHead({ sessionId: 'child-a', maxEntryId: 2 }, [2], { - before: 1, - after: 5, - limit: 10 - }) - .map((row) => row.entry_id) - ).toEqual([2, 1]) - - table.ensureBootstrapAnchor('child-message') - const original = createRecord({ - id: 'linked-message', - sessionId: 'child-message', - content: JSON.stringify({ text: 'old linked marker', files: [], links: [] }) - }) - const replacement = { - ...original, - content: JSON.stringify({ text: 'new linked marker', files: [], links: [] }), - updatedAt: 200 - } - appendMessageRecordToTape(table, original, 'live') - appendMessageReplacementToTape(table, replacement, 'native_edit') - const replacementHead = table.getMaxEntryId('child-message') - expect( - table.searchEffectiveSourcesAtHeads( - [{ sessionId: 'child-message', maxEntryId: replacementHead }], - 'old linked marker' - ) - ).toEqual([]) - const replacementHits = table.searchEffectiveSourcesAtHeads( - [{ sessionId: 'child-message', maxEntryId: replacementHead }], - 'new linked marker' - ) - expect(replacementHits).toHaveLength(1) - expect( - table - .getEffectiveContextRowsAtHead( - { sessionId: 'child-message', maxEntryId: replacementHead }, - [replacementHits[0].entry_id], - { before: 0, after: 0, limit: 5 } - ) - .map((row) => row.entry_id) - ).toEqual([replacementHits[0].entry_id]) - - table.append({ - sessionId: 'child-message', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: original.id, seq: 0 }, - provenanceKey: null, - payload: { - record: { - id: original.id, - sessionId: original.sessionId, - orderSeq: original.orderSeq, - role: original.role, - status: 'sent' - } - }, - meta: { source: 'malformed_import' } - }) - expect( - table.searchEffectiveSourcesAtHeads( - [ - { - sessionId: 'child-message', - maxEntryId: table.getMaxEntryId('child-message') - } - ], - 'new linked marker' - ) - ).toMatchObject([{ entry_id: replacementHits[0].entry_id }]) - - appendMessageRetractionToTape(table, replacement, 'native_delete') - expect( - table.searchEffectiveSourcesAtHeads( - [ - { - sessionId: 'child-message', - maxEntryId: table.getMaxEntryId('child-message') - } - ], - 'new linked marker' - ) - ).toEqual([]) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `searches exact frozen projections without repairing a later child tail${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - projectionTable.replaceSession( - 'child', - [ - { - sessionId: 'child', - entryId: 2, - kind: 'event', - name: 'child/result', - sourceType: null, - sourceId: null, - sourceSeq: null, - searchText: 'native frozen projection needle', - summaryText: 'native frozen projection needle', - refs: {}, - createdAt: 100 - } - ], - 2 - ) - const before = db - .prepare( - `SELECT projection_version, max_entry_id, updated_at - FROM deepchat_tape_search_projection_meta - WHERE session_id = ?` - ) - .get('child') - - const result = projectionTable.searchSourcesReadOnly( - [{ sessionId: 'child', maxEntryId: 2 }], - 'native frozen projection needle', - { limit: 5 } - ) - - expect(result.coveredSources).toEqual([{ sessionId: 'child', maxEntryId: 2 }]) - expect(result.rows).toMatchObject([{ session_id: 'child', entry_id: 2 }]) - expect( - projectionTable.searchSourcesReadOnly( - [{ sessionId: 'child', maxEntryId: 3 }], - 'native frozen projection needle', - { limit: 5 } - ) - ).toEqual({ rows: [], coveredSources: [] }) - expect( - db - .prepare( - `SELECT projection_version, max_entry_id, updated_at - FROM deepchat_tape_search_projection_meta - WHERE session_id = ?` - ) - .get('child') - ).toEqual(before) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `filters stale FTS rows through the base projection after restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 2, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'current', - sourceSeq: 0, - searchText: 'current Redis marker', - summaryText: 'current Redis marker', - refs: { messageId: 'current' }, - createdAt: 200 - } - ], - 2 - ) - db.prepare( - `INSERT INTO deepchat_tape_search_fts ( - search_text, - name, - session_id, - entry_id, - kind, - source_type, - source_id, - source_seq, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - 'stale removed marker', - 'message/user', - 's1', - 1, - 'message', - 'message', - 'old', - 0, - 'stale removed marker', - '{"messageId":"old"}', - 100 - ) - - const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - restartedProjectionTable.createTable() - - expect(restartedProjectionTable.isCurrent('s1', 2)).toBe(true) - expect(restartedProjectionTable.search('s1', 'stale removed marker', { limit: 5 })).toEqual( - [] - ) - expect( - restartedProjectionTable.search('s1', 'current Redis marker', { limit: 5 })[0] - ).toMatchObject({ - entry_id: 2, - refs_json: '{"messageId":"current"}' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `recovers same-entry stale FTS after a base-only projection write and restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm1', - sourceSeq: 0, - searchText: 'old durable marker', - summaryText: 'old durable marker', - refs: { messageId: 'm1' }, - createdAt: 100 - } - ], - 1 - ) - - projectionTable.disableFtsForTesting() - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm1', - sourceSeq: 0, - searchText: 'new durable marker', - summaryText: 'new durable marker', - refs: { messageId: 'm1' }, - createdAt: 100 - } - ], - 1 - ) - db.prepare( - `INSERT INTO deepchat_tape_search_fts ( - search_text, - name, - session_id, - entry_id, - kind, - source_type, - source_id, - source_seq, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - 'old durable marker', - 'message/user', - 's1', - 1, - 'message', - 'message', - 'm1', - 0, - 'old durable marker', - '{"messageId":"m1"}', - 100 - ) - - const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - restartedProjectionTable.createTable() - - expect(restartedProjectionTable.isCurrent('s1', 1)).toBe(true) - expect(restartedProjectionTable.search('s1', 'old durable marker', { limit: 5 })).toEqual( - [] - ) - expect( - restartedProjectionTable.search('s1', 'new durable marker', { limit: 5 })[0] - ).toMatchObject({ - entry_id: 1, - search_text: 'new durable marker' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `rebuilds FTS during append when previous FTS meta is missing after restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.disableFtsForTesting() - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'old', - sourceSeq: 0, - searchText: 'old append marker', - summaryText: 'old append marker', - refs: { messageId: 'old' }, - createdAt: 100 - } - ], - 1 - ) - - const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - restartedProjectionTable.createTable() - restartedProjectionTable.appendSession( - 's1', - [ - { - sessionId: 's1', - entryId: 2, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'new', - sourceSeq: 0, - searchText: 'new append marker', - summaryText: 'new append marker', - refs: { messageId: 'new' }, - createdAt: 200 - } - ], - 2 - ) - - expect(restartedProjectionTable.isCurrent('s1', 2)).toBe(true) - expect( - restartedProjectionTable.search('s1', 'old append marker', { limit: 1 })[0] - ).toMatchObject({ - entry_id: 1, - refs_json: '{"messageId":"old"}' - }) - expect( - restartedProjectionTable.search('s1', 'new append marker', { limit: 1 })[0] - ).toMatchObject({ - entry_id: 2, - refs_json: '{"messageId":"new"}' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `rebuilds migrated tape FTS when freshness meta is excluded${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'old', - sourceSeq: 0, - searchText: 'old migrated marker', - summaryText: 'old migrated marker', - refs: { messageId: 'old' }, - createdAt: 100 - } - ], - 1 - ) - db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run('s1') - db.prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?').run('s1') - - const migratedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - migratedProjectionTable.createTable() - migratedProjectionTable.appendSession( - 's1', - [ - { - sessionId: 's1', - entryId: 2, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'new', - sourceSeq: 0, - searchText: 'new migrated marker', - summaryText: 'new migrated marker', - refs: { messageId: 'new' }, - createdAt: 200 - } - ], - 2 - ) - - expect(migratedProjectionTable.isCurrent('s1', 2)).toBe(true) - expect( - migratedProjectionTable.search('s1', 'old migrated marker', { limit: 1 })[0] - ).toMatchObject({ - entry_id: 1, - refs_json: '{"messageId":"old"}' - }) - expect( - migratedProjectionTable.search('s1', 'new migrated marker', { limit: 1 })[0] - ).toMatchObject({ - entry_id: 2, - refs_json: '{"messageId":"new"}' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `keeps common-term FTS searches scoped and bounded on large session sets${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - - for (let index = 0; index < 180; index += 1) { - const sessionId = `s-${index}` - const rows = Array.from({ length: 8 }, (_, offset) => ({ - sessionId, - entryId: offset + 1, - kind: 'message' as const, - name: 'message/user', - sourceType: 'message' as const, - sourceId: `m-${index}-${offset}`, - sourceSeq: offset, - searchText: `sharedcommon marker session-${index} row-${offset}`, - summaryText: `sharedcommon marker session-${index} row-${offset}`, - refs: { messageId: `m-${index}-${offset}` }, - createdAt: index * 10 + offset - })) - projectionTable.replaceSession(sessionId, rows, rows.length) - } - - const planRows = db - .prepare( - `EXPLAIN QUERY PLAN - SELECT projection.session_id, - projection.entry_id, - projection.kind, - projection.name, - projection.source_type, - projection.source_id, - projection.source_seq, - projection.search_text, - projection.summary_text, - projection.refs_json, - projection.created_at, - bm25(deepchat_tape_search_fts) AS score - FROM deepchat_tape_search_fts - INNER JOIN deepchat_tape_search_projection AS projection - ON projection.session_id = deepchat_tape_search_fts.session_id - AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) - AND projection.search_text = deepchat_tape_search_fts.search_text - WHERE deepchat_tape_search_fts MATCH ? - AND deepchat_tape_search_fts.session_id = ? - AND projection.session_id = ? - ORDER BY score ASC, projection.entry_id DESC - LIMIT ?` - ) - .all('"sharedcommon"', 's-42', 's-42', 5) as Array<{ detail: string }> - const plan = planRows.map((row) => row.detail).join('\n') - - expect(plan).toMatch(/VIRTUAL TABLE INDEX/i) - expect(plan).toMatch(/SEARCH projection USING (?:COVERING )?INDEX/i) - expect(plan).not.toMatch(/\bSCAN projection\b/i) - - const startedAt = performance.now() - const hits = projectionTable.search('s-42', 'sharedcommon', { limit: 5 }) - const elapsedMs = performance.now() - startedAt - - expect(hits).toHaveLength(5) - expect(hits.every((hit) => hit.session_id === 's-42')).toBe(true) - expect(elapsedMs).toBeLessThan(1500) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `does not trust same-entry stale FTS text even when stale FTS meta is current${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm1', - sourceSeq: 0, - searchText: 'new guarded marker', - summaryText: 'new guarded marker', - refs: { messageId: 'm1' }, - createdAt: 100 - } - ], - 1 - ) - db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run('s1') - db.prepare( - `INSERT INTO deepchat_tape_search_fts ( - search_text, - name, - session_id, - entry_id, - kind, - source_type, - source_id, - source_seq, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - 'old guarded marker', - 'message/user', - 's1', - 1, - 'message', - 'message', - 'm1', - 0, - 'old guarded marker', - '{"messageId":"m1"}', - 100 - ) - - const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - restartedProjectionTable.createTable() - - expect(restartedProjectionTable.isCurrent('s1', 1)).toBe(true) - expect(restartedProjectionTable.search('s1', 'old guarded marker', { limit: 5 })).toEqual( - [] - ) - expect( - restartedProjectionTable.search('s1', 'new guarded marker', { limit: 5 })[0] - ).toMatchObject({ - entry_id: 1, - search_text: 'new guarded marker' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `does not mark a tape projection current when FTS DML fails${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.dropFtsForTesting() - ;(projectionTable as any).ftsReady = true - - expect(() => - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm1', - sourceSeq: 0, - searchText: 'Redis TTL', - summaryText: 'Redis TTL', - refs: { messageId: 'm1' }, - createdAt: 100 - } - ], - 1 - ) - ).toThrow() - expect(projectionTable.isCurrent('s1', 1)).toBe(false) - expect(projectionTable.getProjectedEntryIds('s1')).toEqual([]) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `filters memory view manifests by message in SQLite before limit${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - data: { ignored: true }, - meta: { messageId: 'msg-old' }, - createdAt: 999 - }) - for (let index = 0; index < 505; index += 1) { - table.appendAnchor({ - sessionId: 's1', - name: 'memory/view_assembled', - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: index, - selected: [`m-${index}`], - dropped: [], - queryHash: `hash-${index}` - }, - meta: { messageId: `msg-${index}` }, - createdAt: index - }) - } - - const rows = table.listMemoryViewManifestAnchorsBySessions(['s1'], { - limit: 1, - messageId: 'msg-0' - }) - - expect(rows).toHaveLength(1) - expect(rows[0]).toMatchObject({ - kind: 'anchor', - name: 'memory/view_assembled', - created_at: 0 - }) - expect(JSON.parse(rows[0].meta_json)).toEqual({ messageId: 'msg-0' }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `queries memory view manifests for large agents without expanding session parameters${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const sessionTable = new NewSessionsTable(db) - const tapeTable = new DeepChatTapeEntriesTable(db) - sessionTable.createTable() - tapeTable.createTable() - for (let index = 0; index < 1200; index += 1) { - const sessionId = `s-${index}` - db.prepare( - `INSERT INTO new_sessions ( - id, - agent_id, - title, - project_dir, - is_pinned, - is_draft, - active_skills, - disabled_agent_tools, - subagent_enabled, - session_kind, - parent_session_id, - subagent_meta_json, - created_at, - updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - sessionId, - 'agent-a', - `Session ${index}`, - null, - 0, - 0, - '[]', - '[]', - index % 2 === 0 ? 0 : 1, - index % 2 === 0 ? 'regular' : 'subagent', - null, - null, - index, - index - ) - tapeTable.appendAnchor({ - sessionId, - name: 'memory/view_assembled', - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: index, - selected: [`m-${index}`], - dropped: [], - queryHash: `hash-${index}` - }, - meta: { messageId: `msg-${index}` }, - createdAt: index - }) - } - db.prepare( - `INSERT INTO new_sessions ( - id, - agent_id, - title, - project_dir, - is_pinned, - is_draft, - active_skills, - disabled_agent_tools, - subagent_enabled, - session_kind, - parent_session_id, - subagent_meta_json, - created_at, - updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - 'other-session', - 'other-agent', - 'Other', - null, - 0, - 0, - '[]', - '[]', - 0, - 'regular', - null, - null, - 9999, - 9999 - ) - tapeTable.appendAnchor({ - sessionId: 'other-session', - name: 'memory/view_assembled', - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 9999, - selected: ['other'], - dropped: [], - queryHash: 'other' - }, - meta: { messageId: 'msg-0' }, - createdAt: 9999 - }) - - const rows = tapeTable.listMemoryViewManifestAnchorsByAgent('agent-a', { - messageId: 'msg-0', - limit: 1 - }) - - expect(rows).toHaveLength(1) - expect(rows[0]).toMatchObject({ - session_id: 's-0', - kind: 'anchor', - name: 'memory/view_assembled', - created_at: 0 - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `searches a SQLite tape projection and expands compact context without raw payloads${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - table.createTable() - projectionTable.createTable() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'u1', seq: 0 }, - payload: { - record: createRecord({ - id: 'u1', - content: JSON.stringify({ - text: 'Check Redis TTL with /usr/local/bin/deploy --flag and error 42.', - files: [], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.append({ - sessionId: 's1', - kind: 'tool_result', - name: 'shell', - source: { type: 'tool_result', id: 'u1:tc1', seq: 0 }, - payload: { - messageId: 'u1', - orderSeq: 2, - toolCallId: 'tc1', - exitStatus: 42, - response: 'Exit code 42 in /tmp/deploy.log' - }, - meta: { source: 'live', status: 'error' }, - createdAt: 110 - }) - - const pathHits = service.search('s1', '/usr/local/bin/deploy', { limit: 5 }) - expect(pathHits).toHaveLength(1) - expect(pathHits[0]).toMatchObject({ - kind: 'message', - summary: expect.stringContaining('Redis TTL'), - refs: { - messageId: 'u1', - role: 'user', - filePaths: expect.arrayContaining(['/usr/local/bin/deploy']) - } - }) - expect(pathHits[0]).not.toHaveProperty('payload') - expect(pathHits[0]).not.toHaveProperty('meta') - expect(service.search('s1', 'Redis TTL', { limit: 5 }).map((hit) => hit.entryId)).toContain( - pathHits[0].entryId - ) - const errorHits = service.search('s1', '42', { kinds: ['tool_result'], limit: 5 }) - expect(errorHits[0]).toMatchObject({ - refs: { - toolCallId: 'tc1', - exitStatus: 42 - } - }) - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) - - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'u2', seq: 0 }, - payload: { - record: createRecord({ - id: 'u2', - orderSeq: 3, - content: JSON.stringify({ text: 'zoxide marker 简洁', files: [], links: [] }), - createdAt: 120, - updatedAt: 120 - }) - }, - meta: { source: 'live', orderSeq: 3, role: 'user' }, - createdAt: 120 - }) - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(false) - const rebuiltHits = service.search('s1', '简洁', { limit: 5 }) - expect(rebuiltHits.map((hit) => hit.refs?.messageId)).toContain('u2') - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) - if (projectionTable.hasFtsReadyForTesting()) { - projectionTable.dropFtsForTesting() - ;(projectionTable as any).ftsReady = true - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'u3', seq: 0 }, - payload: { - record: createRecord({ - id: 'u3', - orderSeq: 4, - content: JSON.stringify({ - text: 'fts recovery marker', - files: [], - links: [] - }), - createdAt: 130, - updatedAt: 130 - }) - }, - meta: { source: 'live', orderSeq: 4, role: 'user' }, - createdAt: 130 - }) - const recoveryHits = service.search('s1', 'fts recovery marker', { limit: 5 }) - expect(recoveryHits.map((hit) => hit.refs?.messageId)).toContain('u3') - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(false) - expect(projectionTable.hasFtsReadyForTesting()).toBe(false) - - const restoredHits = service.search('s1', 'fts recovery marker', { limit: 5 }) - expect(restoredHits.map((hit) => hit.refs?.messageId)).toContain('u3') - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) - expect(projectionTable.hasFtsReadyForTesting()).toBe(true) - } - - const context = service.getContext('s1', [pathHits[0].entryId], { - before: 0, - after: 1, - limit: 2, - maxBytesPerEntry: 24, - maxTotalBytes: 24 - }) - expect(context.matchedEntryIds).toEqual([pathHits[0].entryId]) - expect(context.entries[0]).toMatchObject({ - entryId: pathHits[0].entryId, - summary: expect.stringContaining('Redis TTL'), - evidence: { - truncated: true - } - }) - expect(context.entries[0].evidence.bytes).toBeLessThanOrEqual(24) - expect(context.entries[0]).not.toHaveProperty('payload') - expect(context.entries[0]).not.toHaveProperty('meta') - const limitedContext = service.getContext( - 's1', - [pathHits[0].entryId, errorHits[0].entryId], - { - before: 0, - after: 0, - limit: 1 - } - ) - expect(limitedContext.entries.map((entry) => entry.entryId)).toEqual([pathHits[0].entryId]) - expect(limitedContext.matchedEntryIds).toEqual([pathHits[0].entryId]) - } finally { - db.close() - } - } - ) - - it('keeps legacy context builder output stable after tape backfill projection', () => { - const { table } = createTapeTableMock() - const records = [ - createRecord({ id: 'u1', orderSeq: 1 }), - createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { type: 'content', content: 'Tool finished', status: 'success', timestamp: 120 }, - { - type: 'tool_call', - status: 'success', - timestamp: 121, - tool_call: { - id: 'tc1', - name: 'example_tool', - params: '{"foo":"bar"}', - response: 'All good' - } - } - ]), - createdAt: 120, - updatedAt: 121 - }) - ] - const legacyMessageStore = { - getMessages: vi.fn().mockReturnValue(records) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const legacyContext = buildContext( - 's1', - { text: 'next', files: [] }, - 'System', - 10000, - 4096, - legacyMessageStore as any - ) - const tapeReady = service.ensureSessionTapeReady('s1', legacyMessageStore as any) - const tapeOnlyStore = { - getMessages: vi.fn(() => { - throw new Error('buildContext must use provided tape history records') - }) - } - const tapeContext = buildContext( - 's1', - { text: 'next', files: [] }, - 'System', - 10000, - 4096, - tapeOnlyStore as any, - false, - { - historyRecords: tapeReady.historyRecords - } - ) - - expect(tapeContext).toEqual(legacyContext) - expect(tapeOnlyStore.getMessages).not.toHaveBeenCalled() - }) - - it('rejects handoff anchors without a non-empty summary before writing Tape state', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - expect(() => service.handoff('s1', 'phase_done', { summary: ' ' })).toThrow( - 'Tape handoff requires a non-empty summary.' - ) - expect(() => service.handoff('s1', 'phase_done', { reason: 'phase complete' } as any)).toThrow( - 'Tape handoff requires a non-empty summary.' - ) - - expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() - expect(table.appendAnchor).not.toHaveBeenCalled() - expect(entries).toEqual([]) - }) - - it('migrates legacy session summary into a tape anchor during backfill', () => { - const { table, entries } = createTapeTableMock() - const messageStore = { - getMessages: vi.fn().mockReturnValue([ - createRecord({ id: 'u1', orderSeq: 1 }), - createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([{ type: 'content', content: 'answer', status: 'success' }]) - }) - ]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { - getSummaryState: vi.fn().mockReturnValue({ - summary_text: 'legacy compacted state', - summary_cursor_order_seq: 3, - summary_updated_at: 200 - }) - } - } as any) - - service.ensureSessionTapeReady('s1', messageStore as any) - - const summaryAnchor = entries.find((entry) => entry.name === 'compaction/migrated_summary') - expect(summaryAnchor).toMatchObject({ - kind: 'anchor', - source_type: 'summary', - source_id: 'legacy-summary', - created_at: 200 - }) - expect(JSON.parse(summaryAnchor.payload_json).state).toMatchObject({ - summary: 'legacy compacted state', - cursorOrderSeq: 3, - sourceMessageIds: ['u1', 'a1'] - }) - }) - - it('stores and lists view manifests as idempotent tape events', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user', - source: 'tape', - reason: 'selected_history' - } - ], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - - const first = service.appendViewManifest(manifest) - const second = service.appendViewManifest(manifest) - - expect(second.entry_id).toBe(first.entry_id) - expect(entries.filter((entry) => entry.name === 'view/assembled')).toHaveLength(1) - expect(JSON.parse(first.meta_json)).toMatchObject({ - policy: 'legacy_context_v1', - policyVersion: 1 - }) - expect(service.listViewManifestsByMessage('s1', 'a1')).toMatchObject([ - { - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - entryId: first.entry_id, - manifest: { - hashes: { - manifestHash: manifest.hashes.manifestHash - }, - policy: 'legacy_context_v1', - policyVersion: 1, - included: [ - { - messageId: 'u1', - entryId: sourceMaps.entryIdByMessageId.get('u1') - } - ] - } - } - ]) - }) - - it('indexes effective tool facts so tool-loop manifests reference real entries', () => { - const { table } = createTapeTableMock() - const assistantRecord = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } - } - ]) - }) - appendToolFactsToTape(table as any, assistantRecord, 'live', 'tool_loop') - - const service = createTapeService(table) - const sourceMaps = service.getViewManifestSourceMaps('s1') - expect(sourceMaps.toolCallEntryIdByToolId.get('tc1')).toBeGreaterThan(0) - expect(sourceMaps.toolResultEntryIdByToolId.get('tc1')).toBeGreaterThan(0) - - const refs = buildRequestRefs( - [ - { role: 'system', content: 'system' }, - { - role: 'assistant', - content: '', - tool_calls: [ - { id: 'tc1', type: 'function', function: { name: 'search', arguments: '{"q":"x"}' } } - ] - }, - { role: 'tool', content: 'result', tool_call_id: 'tc1' } - ], - sourceMaps - ) - expect(refs).toMatchObject([ - { role: 'system', source: 'synthetic' }, - { - role: 'assistant', - source: 'tape', - reason: 'tool_loop_message', - entryId: sourceMaps.toolCallEntryIdByToolId.get('tc1') - }, - { - role: 'tool', - source: 'tape', - reason: 'tool_loop_message', - entryId: sourceMaps.toolResultEntryIdByToolId.get('tc1') - } - ]) - }) - - it('scopes tool source maps to the in-flight message so reused tool ids do not collide', () => { - const { table } = createTapeTableMock() - const blocks = (response: string) => - JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response } - } - ]) - appendToolFactsToTape( - table as any, - createRecord({ id: 'a1', orderSeq: 2, role: 'assistant', content: blocks('first') }), - 'live', - 'tool_loop' - ) - appendToolFactsToTape( - table as any, - createRecord({ id: 'a2', orderSeq: 4, role: 'assistant', content: blocks('second') }), - 'live', - 'tool_loop' - ) - - const service = createTapeService(table) - const scopedToA1 = service.getViewManifestSourceMaps('s1', 'a1') - const scopedToA2 = service.getViewManifestSourceMaps('s1', 'a2') - - expect(scopedToA1.toolCallEntryIdByToolId.get('tc1')).toBeLessThan( - scopedToA2.toolCallEntryIdByToolId.get('tc1')! - ) - expect(scopedToA1.toolResultEntryIdByToolId.get('tc1')).not.toBe( - scopedToA2.toolResultEntryIdByToolId.get('tc1') - ) - }) - - it('exports tool_call and tool_result entries in a tool-loop replay slice', () => { - const { table } = createTapeTableMock() - const assistantRecord = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } - } - ]) - }) - appendToolFactsToTape(table as any, assistantRecord, 'live', 'tool_loop') - - const service = createTapeService(table) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const messages = [ - { role: 'system' as const, content: 'system' }, - { - role: 'assistant' as const, - content: '', - tool_calls: [ - { id: 'tc1', type: 'function' as const, function: { name: 'search', arguments: '{}' } } - ] - }, - { role: 'tool' as const, content: 'result', tool_call_id: 'tc1' } - ] - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: 1, - messages, - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: buildRequestRefs(messages, sourceMaps), - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - service.appendViewManifest(manifest) - - const slice = service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }) - const kinds = slice?.entries.map((entry) => entry.kind) ?? [] - expect(kinds).toContain('tool_call') - expect(kinds).toContain('tool_result') - }) - - it('filters malformed view manifest rows when listing by message', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { - type: 'runtime_event', - id: 'a1', - seq: 1 - }, - data: { - manifest: { - schemaVersion: 1, - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - included: 'not-an-array' - } - } - }) - - expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) - }) - - it('normalizes legacy manifests without hashVersion to hashVersion 1', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - const legacyManifest: Record = { ...manifest } - delete legacyManifest.hashVersion - - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 99 }, - data: { manifest: legacyManifest } - }) - - const [record] = service.listViewManifestsByMessage('s1', 'a1') - expect(record.manifest.hashVersion).toBe(1) - }) - - it('filters manifests whose hashVersion is not a number', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 99 }, - data: { manifest: { ...manifest, hashVersion: '2' } } - }) - - expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) - }) - - it('annotates read records with hash integrity without dropping tampered manifests', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const baseInput = { - sessionId: 's1', - taskType: 'chat' as const, - policy: 'legacy_context_v1' as const, - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - } - const validManifest = createTapeViewManifest({ ...baseInput, messageId: 'a1', requestSeq: 1 }) - service.appendViewManifest(validManifest) - - const tamperedManifest = createTapeViewManifest({ - ...baseInput, - messageId: 'a2', - requestSeq: 1 - }) - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a2', seq: 99 }, - data: { manifest: { ...tamperedManifest, latestEntryId: tamperedManifest.latestEntryId + 1 } } - }) - - const [validRecord] = service.listViewManifestsByMessage('s1', 'a1') - const [tamperedRecord] = service.listViewManifestsByMessage('s1', 'a2') - expect(validRecord.integrity).toBe('valid') - expect(tamperedRecord).toBeDefined() - expect(tamperedRecord.integrity).toBe('invalid') - }) - - it('binds reconstruction lineage to the latest reconstruction anchor including handoffs', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('s1') - table.appendAnchor({ - sessionId: 's1', - name: 'compaction/manual', - source: { type: 'summary', id: 's1', seq: 1 }, - state: {} - }) - table.appendAnchor({ - sessionId: 's1', - name: 'handoff/phase_done', - source: { type: 'handoff', id: 's1', seq: 2 }, - state: {} - }) - table.appendAnchor({ - sessionId: 's1', - name: 'fork/merge', - source: { type: 'fork', id: 'child', seq: 3 }, - state: {} - }) - - const sourceMaps = service.getViewManifestSourceMaps('s1') - const entryIdByName = (name: string) => - table.getBySession('s1').find((entry: any) => entry.name === name)?.entry_id - - expect(sourceMaps.anchorEntryIds).toHaveLength(4) - expect(sourceMaps.reconstructionAnchorEntryId).toBe(entryIdByName('handoff/phase_done')) - expect(sourceMaps.reconstructionAnchorEntryIds).toEqual([ - sourceMaps.reconstructionAnchorEntryId - ]) - expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain( - entryIdByName('compaction/manual') - ) - expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('fork/merge')) - }) - - it('keeps memory anchors off the reconstruction lineage', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('s1') - table.appendAnchor({ - sessionId: 's1', - name: 'compaction/manual', - source: { type: 'summary', id: 's1', seq: 1 }, - state: {} - }) - table.appendAnchor({ - sessionId: 's1', - name: 'memory/extract', - source: { type: 'runtime_event', id: 's1', seq: 2 }, - state: { memoryIds: ['m1'], count: 1, reason: 'episodic' } - }) - table.appendAnchor({ - sessionId: 's1', - name: 'memory/reflect', - source: { type: 'runtime_event', id: 's1', seq: 3 }, - state: { reflectionIds: ['r1'], sourceMemoryIds: ['m1'], count: 1 } - }) - - const sourceMaps = service.getViewManifestSourceMaps('s1') - const entryIdByName = (name: string) => - table.getBySession('s1').find((entry: any) => entry.name === name)?.entry_id - - // Memory anchors are recorded on the tape for observability... - expect(sourceMaps.anchorEntryIds).toContain(entryIdByName('memory/extract')) - expect(sourceMaps.anchorEntryIds).toContain(entryIdByName('memory/reflect')) - // ...but never own the reconstruction cursor; only the summary anchor does. - expect(sourceMaps.reconstructionAnchorEntryId).toBe(entryIdByName('compaction/manual')) - expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('memory/extract')) - expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('memory/reflect')) - }) - - it('bounds replay slices to the selected view instead of pre-cursor history', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.reconstructionAnchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user', - source: 'tape', - reason: 'selected_history' - } - ], - excluded: [], - summaryCursor: { - summaryCursorOrderSeq: 100, - preCursorOrderSeqMin: 1, - preCursorOrderSeqMax: 99, - preCursorCount: 99 - }, - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 100, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - service.appendViewManifest(manifest) - - const slice = service.exportReplaySlice('s1', 'a1') - - expect(slice?.refs.excludedEntryIds).toEqual([]) - expect(slice?.refs.anchorEntryIds).toEqual(sourceMaps.reconstructionAnchorEntryIds) - expect(slice?.refs.anchorEntryIds).toHaveLength(1) - expect(slice?.manifestRecord.manifest.excludedRanges).toEqual([ - { fromOrderSeq: 1, toOrderSeq: 99, count: 99, reason: 'before_summary_cursor' } - ]) - expect(slice?.entries).toHaveLength(3) - }) - - it('exports replay slices with metadata-only payloads by default', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [createTraceRow()]) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user', - source: 'tape', - reason: 'selected_history' - } - ], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: true, - assembledAt: 200 - }) - const manifestEntry = service.appendViewManifest(manifest) - - const nowSpy = vi.spyOn(Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000) - const slice = service.exportReplaySlice('s1', 'a1') - const secondSlice = service.exportReplaySlice('s1', 'a1') - nowSpy.mockRestore() - - expect(slice).toMatchObject({ - schemaVersion: 1, - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - mode: 'trace_bound', - refs: { - manifestEntryId: manifestEntry.entry_id, - includedEntryIds: [sourceMaps.entryIdByMessageId.get('u1')], - anchorEntryIds: sourceMaps.anchorEntryIds - }, - hashes: { - manifestHash: manifest.hashes.manifestHash - } - }) - expect(slice?.hashes.sliceHash).toHaveLength(64) - expect(secondSlice?.hashes.sliceHash).toBe(slice?.hashes.sliceHash) - expect(secondSlice?.createdAt).toBe(2000) - expect(slice?.trace?.bodyHash).toHaveLength(64) - expect(slice?.trace?.bodyJson).toBeUndefined() - expect(slice?.entries.some((entry) => entry.entryId === manifestEntry.entry_id)).toBe(true) - expect( - slice?.entries.every((entry) => entry.payload === undefined && entry.meta === undefined) - ).toBe(true) - }) - - it('exports explicit replay request sequences with opt-in payloads', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [ - createTraceRow({ id: 'trace-1', request_seq: 1 }), - createTraceRow({ - id: 'trace-2', - request_seq: 2, - body_json: '{"messages":[{"role":"tool","content":"done"}]}' - }) - ]) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const baseManifestInput: TapeViewManifestBuildInput = { - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat' as const, - policy: 'legacy_context_v1' as const, - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user', - source: 'tape', - reason: 'selected_history' - } - ], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: true, - assembledAt: 200 - } - const firstManifest = createTapeViewManifest(baseManifestInput) - const secondManifest = createTapeViewManifest({ - ...baseManifestInput, - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 250 - }) - service.appendViewManifest(firstManifest) - service.appendViewManifest(secondManifest) - - const latest = service.exportReplaySlice('s1', 'a1') - const first = service.exportReplaySlice('s1', 'a1', { - requestSeq: 1, - includeTapePayloads: true, - includeTracePayload: true - }) - - expect(latest?.requestSeq).toBe(2) - expect(first?.requestSeq).toBe(1) - expect(first?.trace?.bodyJson).toContain('"hello"') - expect(first?.entries.some((entry) => entry.payload?.record)).toBe(true) - expect(first?.entries.some((entry) => entry.meta?.source === 'backfill')).toBe(true) - }) - - it('binds each replay slice to its own request seq, ignoring sentinel gap traces', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [ - createTraceRow({ - id: 'trace-req-1', - request_seq: 1, - body_json: '{"messages":[{"role":"user","content":"first-request"}]}' - }), - createTraceRow({ - id: 'trace-gap', - request_seq: 0, - endpoint: 'deepchat://interleaved-reasoning-gap', - body_json: '{"providerId":"openai"}' - }), - createTraceRow({ - id: 'trace-req-2', - request_seq: 2, - body_json: '{"messages":[{"role":"tool","content":"second-request"}]}' - }) - ]) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const baseManifestInput = { - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat' as const, - policy: 'legacy_context_v1' as const, - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user' as const, - source: 'tape' as const, - reason: 'selected_history' as const - } - ], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: true, - assembledAt: 200 - } - service.appendViewManifest(createTapeViewManifest(baseManifestInput)) - service.appendViewManifest( - createTapeViewManifest({ - ...baseManifestInput, - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 250 - }) - ) - - const first = service.exportReplaySlice('s1', 'a1', { - requestSeq: 1, - includeTracePayload: true - }) - const second = service.exportReplaySlice('s1', 'a1', { - requestSeq: 2, - includeTracePayload: true - }) - - expect(first?.trace?.bodyJson).toContain('first-request') - expect(second?.trace?.bodyJson).toContain('second-request') - }) - - it('returns null when exporting a replay slice without a manifest', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [createTraceRow()]) - - expect(service.exportReplaySlice('s1', 'a1')).toBeNull() - }) - - it('rejects non-positive replay request sequences', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [createTraceRow()]) - - expect(() => service.exportReplaySlice('s1', 'a1', { requestSeq: 0 })).toThrow( - 'requestSeq must be a positive integer.' - ) - }) - - it('joins an exact manifest and trace into a causal observation slice', () => { - const { table } = createTapeTableMock() - const assistant = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: '[{"type":"content","content":"done","status":"success"}]', - status: 'sent', - metadata: '{"totalTokens":10}', - createdAt: 200, - updatedAt: 300 - }) - appendMessageRecordToTape(table as any, assistant, 'live') - const service = createTapeService( - table, - [createTraceRow(), createTraceRow({ id: 'other-session', session_id: 's2', request_seq: 9 })], - [createMessageRow()] - ) - service.appendViewManifest(createObservationManifest()) - - const slice = service.readCausalObservationSlice('s1', 'a1', { - currentRuntimeStatus: 'idle' - }) - - expect(slice).toMatchObject({ - schemaVersion: 1, - sessionId: 's1', - messageId: 'a1', - request: { - state: 'manifest_bound', - requestSeq: 1, - replay: { - requestSeq: 1, - mode: 'trace_bound', - trace: { id: 'trace-1', requestSeq: 1 } - } - }, - output: { - correlation: 'message_only', - terminalMessage: { - status: 'sent', - orderSeq: 2, - createdAt: 200, - updatedAt: 300 - } - }, - runtime: { scope: 'current_only', status: 'idle', eventHistory: 'not_persisted' } - }) - expect(slice.output.entries.map((entry) => entry.kind)).toContain('message') - expect(slice.output.terminalMessage?.contentHash).toHaveLength(64) - expect(slice.output.terminalMessage?.metadataHash).toHaveLength(64) - }) - - it('prefers an explicit causal request sequence and otherwise selects the latest raw sequence', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 100 })) - service.appendViewManifest( - createObservationManifest({ - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 200 - }) - ) - - expect(service.readCausalObservationSlice('s1', 'a1').request).toMatchObject({ - state: 'manifest_bound', - requestSeq: 2 - }) - expect(service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }).request).toMatchObject( - { state: 'manifest_bound', requestSeq: 1 } - ) - }) - - it('does not fall back to an older manifest when a later traced request has no manifest', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [ - createTraceRow({ id: 'trace-1', request_seq: 1 }), - createTraceRow({ id: 'trace-2', request_seq: 2 }) - ]) - service.appendViewManifest(createObservationManifest({ requestSeq: 1 })) - - expect(service.readCausalObservationSlice('s1', 'a1').request).toMatchObject({ - state: 'manifest_missing', - requestSeq: 2, - trace: { id: 'trace-2', requestSeq: 2 } - }) - }) - - it('returns a trace-only request without manufacturing a view manifest', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [createTraceRow({ request_seq: 4 })]) - - const request = service.readCausalObservationSlice('s1', 'a1').request - - expect(request).toMatchObject({ - state: 'manifest_missing', - requestSeq: 4, - trace: { requestSeq: 4 } - }) - expect( - request.state === 'manifest_missing' ? request.trace?.bodyJson : undefined - ).toBeUndefined() - }) - - it('distinguishes malformed manifests from hash-invalid but readable manifests', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'bad', seq: 2 }, - data: { manifest: { schemaVersion: 99, requestSeq: 2 } } - }) - const tamperedManifest = createObservationManifest({ messageId: 'a1', requestSeq: 1 }) - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 1 }, - data: { manifest: { ...tamperedManifest, latestEntryId: 1 } } - }) - - expect(service.readCausalObservationSlice('s1', 'bad').request).toEqual({ - state: 'manifest_malformed', - requestSeq: 2, - trace: null - }) - const invalid = service.readCausalObservationSlice('s1', 'a1').request - expect(invalid.state).toBe('manifest_bound') - expect(invalid.state === 'manifest_bound' ? invalid.replay.integrity : undefined).toBe( - 'invalid' - ) - }) - - it('ignores request sequence zero interleaved-reasoning sentinels', () => { - const { table } = createTapeTableMock() - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 0 }, - data: { manifest: { schemaVersion: 99, requestSeq: 0 } } - }) - const service = createTapeService(table, [ - createTraceRow({ - id: 'trace-gap', - request_seq: 0, - endpoint: 'deepchat://interleaved-reasoning-gap' - }) - ]) - - expect(service.readCausalObservationSlice('s1', 'a1').request).toEqual({ - state: 'request_unavailable', - requestSeq: null, - trace: null - }) - }) - - it('keeps multi-round assistant and tool output correlated at message scope only', () => { - const { table } = createTapeTableMock() - const assistant = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 220, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'result' - } - } - ]), - status: 'sent', - createdAt: 200, - updatedAt: 300 - }) - appendMessageRecordToTape(table as any, assistant, 'live') - const service = createTapeService(table, [], [createMessageRow({ content: assistant.content })]) - service.appendViewManifest(createObservationManifest({ requestSeq: 1 })) - service.appendViewManifest( - createObservationManifest({ - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null - }) - ) - - const firstOutput = service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }).output - const latestOutput = service.readCausalObservationSlice('s1', 'a1').output - - expect(latestOutput.correlation).toBe('message_only') - expect(latestOutput.entries.map((entry) => entry.kind)).toEqual([ - 'message', - 'tool_call', - 'tool_result' - ]) - expect(latestOutput.entries.map((entry) => entry.entryId)).toEqual( - firstOutput.entries.map((entry) => entry.entryId) - ) - }) - - it('does not infer a completed event from a paused pending assistant message', () => { - const { table } = createTapeTableMock() - const service = createTapeService( - table, - [], - [createMessageRow({ status: 'pending', content: '[{"type":"action","status":"pending"}]' })] - ) - - const slice = service.readCausalObservationSlice('s1', 'a1', { - currentRuntimeStatus: 'generating' - }) - - expect(slice.output.entries).toEqual([]) - expect(slice.output.terminalMessage).toBeNull() - expect(slice.runtime).toEqual({ - scope: 'current_only', - status: 'generating', - eventHistory: 'not_persisted' - }) - expect(JSON.stringify(slice)).not.toContain('completed') - }) - - it('reads an old message without bootstrapping or backfilling Tape', () => { - const { table, entries } = createTapeTableMock() - const messageInsert = vi.fn() - const traceInsert = vi.fn() - const projectionApply = vi.fn() - const projectionReplace = vi.fn() - const cursorWrite = vi.fn() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatMessageTracesTable: { - listByMessageId: vi.fn().mockReturnValue([]), - insert: traceInsert - }, - deepchatMessagesTable: { - get: vi.fn().mockReturnValue(createMessageRow()), - insert: messageInsert - }, - deepchatMemoryIngestionProjectionTable: { - applyAppendedEntry: projectionApply, - replaceSession: projectionReplace - }, - deepchatSessionsTable: { - getSummaryState: vi.fn().mockReturnValue(null), - updateMemoryCursorOrderSeq: cursorWrite - } - } as any) - - const slice = service.readCausalObservationSlice('s1', 'a1') - - expect(slice.request).toEqual({ - state: 'request_unavailable', - requestSeq: null, - trace: null - }) - expect(slice.output.terminalMessage?.status).toBe('sent') - expect(slice.runtime).toEqual({ - scope: 'unavailable', - status: null, - eventHistory: 'not_persisted' - }) - expect(entries).toEqual([]) - expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() - expect(table.append).not.toHaveBeenCalled() - expect(table.appendEvent).not.toHaveBeenCalled() - expect(messageInsert).not.toHaveBeenCalled() - expect(traceInsert).not.toHaveBeenCalled() - expect(projectionApply).not.toHaveBeenCalled() - expect(projectionReplace).not.toHaveBeenCalled() - expect(cursorWrite).not.toHaveBeenCalled() - }) - - it('keeps causal observations metadata-only by default and reuses replay payload opt-ins', () => { - const { table } = createTapeTableMock() - const assistant = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: 'tape-secret-output', - status: 'sent', - metadata: '{"secret":"tape-metadata"}', - createdAt: 200, - updatedAt: 300 - }) - appendMessageRecordToTape(table as any, assistant, 'live') - const service = createTapeService( - table, - [ - createTraceRow({ - headers_json: '{"authorization":"secret-header"}', - body_json: '{"prompt":"secret-request"}' - }) - ], - [ - createMessageRow({ - content: 'projection-only-content-secret', - metadata: '{"secret":"projection-only-metadata"}', - blocks_json: '["projection-only-blocks"]', - error: 'projection-only-error' - }) - ] - ) - service.appendViewManifest(createObservationManifest()) - - const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) - const metadataOnly = service.readCausalObservationSlice('s1', 'a1') - const withPayloads = service.readCausalObservationSlice('s1', 'a1', { - includeTapePayloads: true, - includeTracePayload: true - }) - now.mockRestore() - - expect(JSON.stringify(metadataOnly)).not.toContain('tape-secret-output') - expect(JSON.stringify(metadataOnly)).not.toContain('tape-metadata') - expect(JSON.stringify(metadataOnly)).not.toContain('secret-header') - expect(JSON.stringify(metadataOnly)).not.toContain('secret-request') - expect(JSON.stringify(metadataOnly)).not.toContain('projection-only') - expect(withPayloads.output.entries.some((entry) => entry.payload?.record)).toBe(true) - expect(withPayloads.request.state).toBe('manifest_bound') - expect( - withPayloads.request.state === 'manifest_bound' - ? withPayloads.request.replay.trace?.headersJson - : undefined - ).toContain('secret-header') - expect(JSON.stringify(withPayloads)).toContain('tape-secret-output') - expect(JSON.stringify(withPayloads)).not.toContain('projection-only') - expect(metadataOnly.output.terminalMessage).not.toHaveProperty('content') - expect(metadataOnly.output.terminalMessage).not.toHaveProperty('metadata') - expect(metadataOnly.output.terminalMessage).not.toHaveProperty('blocks') - expect(metadataOnly.output.terminalMessage).not.toHaveProperty('error') - expect(stripObservationPayloadOptIns(withPayloads)).toEqual( - stripObservationPayloadOptIns(metadataOnly) - ) - }) - - it('keeps storage and Memory seams unchanged across every causal observation read mode', () => { - const { table, entries } = createTapeTableMock() - const { final } = appendObservationIsolationFacts(table) - const traceRows = [ - createTraceRow({ id: 'trace-1', request_seq: 1 }), - createTraceRow({ id: 'trace-2', request_seq: 2 }), - createTraceRow({ id: 'trace-only', message_id: 'a-trace', request_seq: 7 }) - ] - const messageRows = [createMessageRow({ order_seq: 3 })] - const projectionState = { - meta: { session_id: 's1', projection_version: 1, max_entry_id: entries.length }, - range: [{ session_id: 's1', message_id: 'a1', order_seq: 3, entry_id: entries.length }] - } - const memoryCursorOrderSeq = 3 - const memoryProjectionWrites = createSpies([ - 'applyAppendedEntry', - 'replaceSession', - 'invalidateSession' - ]) - const memoryProjectionTable = { - ...memoryProjectionWrites, - readCurrentRange: vi.fn(() => structuredClone(projectionState.range)), - getSessionMeta: vi.fn(() => structuredClone(projectionState.meta)) - } - const cursorWrites = createSpies(['updateMemoryCursorOrderSeq', 'rewindMemoryCursorOrderSeq']) - const sessionTable = { - ...cursorWrites, - getSummaryState: vi.fn().mockReturnValue(null), - getMemoryCursorOrderSeq: vi.fn(() => memoryCursorOrderSeq) - } - const messageWrites = createSpies([ - 'insert', - 'updateContent', - 'updateStatus', - 'updateContentAndStatus' - ]) - const messageTable = { - ...messageWrites, - get: vi.fn((messageId: string) => messageRows.find((row) => row.id === messageId)) - } - const traceWrites = createSpies(['insert', 'deleteByMessageId', 'deleteBySessionId']) - const traceTable = { - ...traceWrites, - listByMessageId: vi.fn((messageId: string) => - traceRows.filter((row) => row.message_id === messageId) - ) - } - const { presenter, memoryPropertyAccess } = trackMemoryPropertyAccess({ - deepchatTapeEntriesTable: table, - deepchatMessageTracesTable: traceTable, - deepchatMessagesTable: messageTable, - deepchatSessionsTable: sessionTable, - deepchatMemoryIngestionProjectionTable: memoryProjectionTable - }) - const service = new SessionTape(presenter as any) - service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 210 })) - service.appendViewManifest( - createObservationManifest({ - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 310 - }) - ) - - for (const write of [ - table.ensureBootstrapAnchor, - table.append, - table.appendAnchor, - table.appendEvent, - table.deleteBySession - ]) { - write.mockClear() - } - const captureState = () => ({ - tapeRows: structuredClone(entries), - maxEntryId: table.getMaxEntryId('s1'), - entryOrder: entries.map((entry) => entry.entry_id), - maxMessageOrder: Math.max(0, ...service.getMessageRecords('s1').map((row) => row.orderSeq)), - messageRows: structuredClone(messageRows), - traceRows: structuredClone(traceRows), - viewManifestRows: structuredClone( - entries.filter((entry) => entry.kind === 'event' && entry.name === 'view/assembled') - ), - effectiveView: service.getMessageRecords('s1'), - replay: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }), - replayHash: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 })?.hashes.sliceHash, - projectionMeta: memoryProjectionTable.getSessionMeta(), - projectionRange: memoryProjectionTable.readCurrentRange(), - memoryCursorOrderSeq: sessionTable.getMemoryCursorOrderSeq() - }) - const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) - - try { - const before = captureState() - const { defaultObservation, repeatedObservation, explicitObservation, traceOnlyObservation } = - readObservationMatrix(service) - const after = captureState() - - expect(after).toEqual(before) - expect(repeatedObservation).toEqual(defaultObservation) - expect(defaultObservation.request).toMatchObject({ - state: 'manifest_bound', - requestSeq: 2 - }) - expect(explicitObservation.request).toMatchObject({ - state: 'manifest_bound', - requestSeq: 1 - }) - expect(traceOnlyObservation.request).toMatchObject({ - state: 'manifest_missing', - requestSeq: 7, - trace: { id: 'trace-only' } - }) - expect(defaultObservation.output).toMatchObject({ - correlation: 'message_only', - entries: [{ kind: 'message' }, { kind: 'tool_call' }, { kind: 'tool_result' }] - }) - expect(before.effectiveView.map((record) => record.id)).toEqual(['u1', 'a1']) - expect(JSON.parse(before.effectiveView[0].content).text).toBe('edited') - expect(before.effectiveView[1]).toMatchObject({ status: 'sent', content: final.content }) - } finally { - now.mockRestore() - } - - for (const write of [ - table.ensureBootstrapAnchor, - table.append, - table.appendAnchor, - table.appendEvent, - table.deleteBySession, - ...Object.values(messageWrites), - ...Object.values(traceWrites), - ...Object.values(memoryProjectionWrites), - ...Object.values(cursorWrites), - memoryPropertyAccess - ]) { - expect(write).not.toHaveBeenCalled() - } - }) - - itIfSqlite( - `preserves real Tape, message, trace, Memory projection and schema state while observing${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const memoryProjectionTable = new DeepChatMemoryIngestionProjectionTable(db) - const tapeTable = new DeepChatTapeEntriesTable(db, memoryProjectionTable) - const messageTable = new DeepChatMessagesTable(db) - const traceTable = new DeepChatMessageTracesTable(db) - const sessionTable = new DeepChatSessionsTable(db) - memoryProjectionTable.createTable() - tapeTable.createTable() - messageTable.createTable() - traceTable.createTable() - sessionTable.createTable() - sessionTable.create('s1', 'openai', 'gpt-4o', 'full_access') - sessionTable.updateMemoryCursorOrderSeq('s1', 3) - - appendObservationIsolationFacts(tapeTable) - messageTable.insert({ - id: 'a1', - sessionId: 's1', - orderSeq: 3, - role: 'assistant', - content: 'projection-only-content-secret', - status: 'sent', - metadata: '{"secret":"projection-only-metadata"}', - createdAt: 200, - updatedAt: 300 - }) - for (const trace of [ - { id: 'trace-1', messageId: 'a1', requestSeq: 1 }, - { id: 'trace-2', messageId: 'a1', requestSeq: 2 }, - { id: 'trace-only', messageId: 'a-trace', requestSeq: 7 } - ]) { - traceTable.insert({ - ...trace, - sessionId: 's1', - providerId: 'openai', - modelId: 'gpt-4o', - endpoint: 'https://api.openai.test/v1/chat/completions', - headersJson: `{"authorization":"${trace.id}-header"}`, - bodyJson: `{"prompt":"${trace.id}-body"}`, - truncated: false, - createdAt: 300 + trace.requestSeq - }) - } - - const service = new SessionTape({ - deepchatTapeEntriesTable: tapeTable, - deepchatMessagesTable: messageTable, - deepchatMessageTracesTable: traceTable, - deepchatSessionsTable: sessionTable, - deepchatMemoryIngestionProjectionTable: memoryProjectionTable - } as any) - service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 210 })) - service.appendViewManifest( - createObservationManifest({ - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 310 - }) - ) - const effectiveRecords = service.getMessageRecords('s1') - const sourceMaps = service.getViewManifestSourceMaps('s1') - memoryProjectionTable.replaceSession( - 's1', - effectiveRecords - .filter( - (record): record is ChatMessageRecord & { status: 'sent' | 'error' } => - record.status === 'sent' || record.status === 'error' - ) - .map((record) => ({ - sessionId: 's1', - messageId: record.id, - orderSeq: record.orderSeq, - entryId: sourceMaps.entryIdByMessageId.get(record.id)!, - role: record.role, - content: record.content, - status: record.status, - hadToolUse: record.id === 'a1' - })), - tapeTable.getMaxEntryId('s1') - ) - - const captureState = () => { - const tapeRows = tapeTable.getBySession('s1') - return { - tapeRows, - maxEntryId: tapeTable.getMaxEntryId('s1'), - entryOrder: tapeRows.map((row) => row.entry_id), - messageRow: messageTable.get('a1'), - traceRows: db - .prepare( - `SELECT * - FROM deepchat_message_traces - ORDER BY session_id, message_id, request_seq, id` - ) - .all(), - viewManifestRows: tapeRows.filter( - (row) => row.kind === 'event' && row.name === 'view/assembled' - ), - effectiveView: service.getMessageRecords('s1'), - replay: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }), - replayHash: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 })?.hashes.sliceHash, - projectionMeta: memoryProjectionTable.getSessionMeta('s1'), - projectionRange: memoryProjectionTable.readCurrentRange('s1', 0, 10), - memoryCursorOrderSeq: sessionTable.getMemoryCursorOrderSeq('s1'), - schema: db - .prepare( - `SELECT type, name, tbl_name, sql - FROM sqlite_master - WHERE name NOT LIKE 'sqlite_%' - ORDER BY type, name, tbl_name` - ) - .all() - } - } - const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) - - try { - const before = captureState() - readObservationMatrix(service) - const after = captureState() - - expect(after).toEqual(before) - } finally { - now.mockRestore() - } - } finally { - db.close() - } - } - ) - - it('keeps pending message records for resume but hides pending tool facts from search', () => { - const { table } = createTapeTableMock() - const pendingBlocks = [ - { - type: 'tool_call', - status: 'pending', - timestamp: 100, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'pending result' - } - } - ] - const messageStore = { - getMessages: vi.fn().mockReturnValue([ - createRecord({ - id: 'a1', - orderSeq: 1, - role: 'assistant', - status: 'pending', - content: JSON.stringify(pendingBlocks), - updatedAt: 100 - }) - ]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.ensureSessionTapeReady('s1', messageStore as any) - - expect(service.getMessageRecords('s1')).toMatchObject([{ id: 'a1', status: 'pending' }]) - expect(service.search('s1', 'pending result', { kinds: ['tool_result'] })).toEqual([]) - }) - - it('lets final assistant facts supersede earlier pending tape facts', () => { - const { table, entries } = createTapeTableMock() - const pendingBlocks = [ - { - type: 'tool_call', - status: 'pending', - timestamp: 100, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'pending result' - } - } - ] - const finalBlocks = [ - { - type: 'tool_call', - status: 'success', - timestamp: 200, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'final result' - } - } - ] - const messageStore = { - getMessages: vi - .fn() - .mockReturnValueOnce([ - createRecord({ - id: 'a1', - orderSeq: 1, - role: 'assistant', - status: 'pending', - content: JSON.stringify(pendingBlocks), - metadata: JSON.stringify({ totalTokens: 1 }), - updatedAt: 100 - }) - ]) - .mockReturnValue([ - createRecord({ - id: 'a1', - orderSeq: 1, - role: 'assistant', - status: 'sent', - content: JSON.stringify(finalBlocks), - metadata: JSON.stringify({ totalTokens: 7 }), - updatedAt: 200 - }) - ]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.ensureSessionTapeReady('s1', messageStore as any) - service.ensureSessionTapeReady('s1', messageStore as any) - - expect(service.getMessageRecords('s1')).toMatchObject([ - { - id: 'a1', - status: 'sent' - } - ]) - const effectiveRecord = service.getMessageRecords('s1')[0]! - expect(JSON.parse(effectiveRecord.content)[0].tool_call.response).toBe('final result') - expect( - entries.filter((entry) => entry.kind === 'message' && entry.name === 'message/assistant') - ).toHaveLength(2) - expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) - const finalToolResult = entries.filter((entry) => entry.kind === 'tool_result').at(-1)! - expect(JSON.parse(finalToolResult.payload_json).response).toBe('final result') - expect(service.info('s1').lastTokenUsage).toBe(7) - expect(service.search('s1', 'pending result', { kinds: ['tool_result'] })).toEqual([]) - expect(service.search('s1', 'final result', { kinds: ['tool_result'] })).toHaveLength(1) - }) - - it('keeps fork writes isolated until merge and discards fork entries on discard', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const fork = service.createFork('s1', 'fork-1') - service.appendForkMessageRecord(fork, createRecord({ id: 'fu1', sessionId: 'ignored' })) - - expect( - entries.some((entry) => entry.session_id === 's1' && entry.name === 'message/user') - ).toBe(false) - - const mergedCount = service.mergeFork('s1', 'fork-1') - - expect(mergedCount).toBe(1) - expect( - entries.some((entry) => entry.session_id === 's1' && entry.name === 'message/user') - ).toBe(true) - expect(entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/merge')).toBe( - true - ) - expect(entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/start')).toBe( - false - ) - - const discardFork = service.createFork('s1', 'fork-2') - service.appendForkMessageRecord(discardFork, createRecord({ id: 'fu2', sessionId: 'ignored' })) - service.discardFork('s1', 'fork-2') - - expect(entries.some((entry) => entry.session_id === discardFork.forkSessionId)).toBe(false) - expect( - entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') - ).toBe(true) - }) - - it('records exact fork heads and keeps retries bounded to the first merge', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('parent') - table.appendEvent({ sessionId: 'parent', name: 'parent/tail', data: { value: 1 } }) - const fork = service.createFork('parent', 'bounded') - table.appendEvent({ sessionId: 'parent', name: 'parent/later', data: { value: 2 } }) - const retriedFork = service.createFork('parent', 'bounded') - service.appendForkMessageRecord(fork, createRecord({ id: 'first', sessionId: 'ignored' })) - - const forkStart = entries.find( - (entry) => entry.session_id === fork.forkSessionId && entry.name === 'fork/start' - ) - expect(fork.parentHeadEntryId).toBe(2) - expect(retriedFork.parentHeadEntryId).toBe(2) - expect(JSON.parse(forkStart.payload_json).state.parentHeadEntryId).toBe(2) - - const getBoundedEntries = table.getBySessionUpToEntryId.getMockImplementation()! - table.getBySessionUpToEntryId.mockImplementationOnce( - (sessionId: string, maxEntryId: number) => { - table.appendEvent({ - sessionId: fork.forkSessionId, - name: 'fork/late-tail', - data: { value: 2 } - }) - return getBoundedEntries(sessionId, maxEntryId) - } - ) - - expect(service.mergeFork('parent', 'bounded')).toBe(1) - expect(service.mergeFork('parent', 'bounded')).toBe(1) - - const parentEntries = entries.filter((entry) => entry.session_id === 'parent') - expect(parentEntries.filter((entry) => entry.source_type === 'fork')).toHaveLength(2) - expect(parentEntries.some((entry) => entry.name === 'fork/late-tail')).toBe(false) - const receipt = parentEntries.find((entry) => entry.name === 'fork/merge')! - expect(JSON.parse(receipt.payload_json).data).toMatchObject({ - forkHeadEntryId: 3, - mergedCount: 1 - }) - }) - - it('rolls back mocked fork merge writes when a copied entry fails', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const fork = service.createFork('parent', 'mock-atomic') - service.appendForkMessageRecord( - fork, - createRecord({ id: 'first', orderSeq: 1, sessionId: 'ignored' }) - ) - service.appendForkMessageRecord( - fork, - createRecord({ id: 'second', orderSeq: 2, sessionId: 'ignored' }) - ) - - const append = table.append.getMockImplementation()! - let copiedEntryCount = 0 - table.append.mockImplementation((input: any) => { - if (input.sessionId === 'parent' && input.source?.type === 'fork') { - copiedEntryCount += 1 - if (copiedEntryCount === 2) { - throw new Error('injected merge failure') - } - } - return append(input) - }) - - expect(() => service.mergeFork('parent', 'mock-atomic')).toThrow('injected merge failure') - expect( - entries.filter( - (entry) => - entry.session_id === 'parent' && - (entry.source_type === 'fork' || entry.name === 'fork/merge') - ) - ).toEqual([]) - }) - - it('rolls back copied fork entries when the merge receipt fails', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const fork = service.createFork('parent', 'receipt-failure') - service.appendForkMessageRecord(fork, createRecord({ id: 'first', sessionId: 'ignored' })) - - const appendEvent = table.appendEvent.getMockImplementation()! - table.appendEvent.mockImplementation((input: any) => { - if (input.sessionId === 'parent' && input.name === 'fork/merge') { - throw new Error('injected receipt failure') - } - return appendEvent(input) - }) - - expect(() => service.mergeFork('parent', 'receipt-failure')).toThrow('injected receipt failure') - expect( - entries.filter( - (entry) => - entry.session_id === 'parent' && - (entry.source_type === 'fork' || entry.name === 'fork/merge') - ) - ).toEqual([]) - }) - - it('commits one idempotent receipt for an empty fork', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - service.createFork('parent', 'empty') - - expect(service.mergeFork('parent', 'empty')).toBe(0) - expect(service.mergeFork('parent', 'empty')).toBe(0) - const parentEntries = entries.filter((entry) => entry.session_id === 'parent') - expect(parentEntries).toHaveLength(1) - expect(parentEntries[0].name).toBe('fork/merge') - expect(JSON.parse(parentEntries[0].payload_json).data).toMatchObject({ - forkHeadEntryId: 2, - mergedCount: 0 - }) - }) - - it('rejects missing and discarded forks without committing an empty merge receipt', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - expect(() => service.mergeFork('parent', 'missing')).toThrow( - 'Fork missing does not exist or has been discarded.' - ) - - service.createFork('parent', 'discarded') - service.discardFork('parent', 'discarded') - expect(() => service.mergeFork('parent', 'discarded')).toThrow( - 'Fork discarded does not exist or has been discarded.' - ) - expect( - entries.filter((entry) => entry.session_id === 'parent' && entry.name === 'fork/merge') - ).toEqual([]) - }) - - it('returns an existing merge receipt after the merged fork Tape is cleaned up', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const fork = service.createFork('parent', 'cleanup-after-merge') - service.appendForkMessageRecord(fork, createRecord({ id: 'merged', sessionId: 'ignored' })) - - expect(service.mergeFork('parent', 'cleanup-after-merge')).toBe(1) - table.deleteBySession(fork.forkSessionId) - expect(service.mergeFork('parent', 'cleanup-after-merge')).toBe(1) - }) - - it('merges a legacy fork start that predates the parent head field', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const fork = service.createFork('parent', 'legacy-start') - const start = entries.find( - (entry) => entry.session_id === fork.forkSessionId && entry.name === 'fork/start' - )! - const payload = JSON.parse(start.payload_json) - delete payload.state.parentHeadEntryId - start.payload_json = JSON.stringify(payload) - service.appendForkMessageRecord(fork, createRecord({ id: 'legacy', sessionId: 'ignored' })) - - expect(service.mergeFork('parent', 'legacy-start')).toBe(1) - }) - - it('accepts a valid legacy fork merge receipt without a frozen head', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/merge', - source: { type: 'fork', id: 'legacy-receipt', seq: 0 }, - provenanceKey: 'fork:parent:legacy-receipt:merge:event', - data: { - forkId: 'legacy-receipt', - forkSessionId: 'parent::fork::legacy-receipt', - mergedCount: 2 - } - }) - - expect(service.mergeFork('parent', 'legacy-receipt')).toBe(2) - }) - - it('rejects a malformed stored fork merge receipt instead of reporting an empty merge', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/merge', - source: { type: 'fork', id: 'malformed-receipt', seq: 0 }, - provenanceKey: 'fork:parent:malformed-receipt:merge:event', - data: { - forkId: 'malformed-receipt', - forkSessionId: 'parent::fork::malformed-receipt', - mergedCount: 'two' - } - }) - - expect(() => service.mergeFork('parent', 'malformed-receipt')).toThrow( - 'Stored fork merge receipt is malformed' - ) - }) - - itIfSqlite('rolls back copied fork entries and the receipt when merge fails', () => { - const db = new DatabaseCtor(':memory:') - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('parent') - const fork = service.createFork('parent', 'atomic') - service.appendForkMessageRecord( - fork, - createRecord({ id: 'first', orderSeq: 1, sessionId: 'ignored' }) - ) - service.appendForkMessageRecord( - fork, - createRecord({ id: 'second', orderSeq: 2, sessionId: 'ignored' }) - ) - - const append = table.append.bind(table) - let copiedEntryCount = 0 - const appendSpy = vi.spyOn(table, 'append').mockImplementation((input) => { - if (input.sessionId === 'parent' && input.source?.type === 'fork') { - copiedEntryCount += 1 - if (copiedEntryCount === 2) { - throw new Error('injected merge failure') - } - } - return append(input) - }) - - expect(() => service.mergeFork('parent', 'atomic')).toThrow('injected merge failure') - expect( - table - .getBySession('parent') - .filter((entry) => entry.source_type === 'fork' || entry.name === 'fork/merge') - ).toEqual([]) - - appendSpy.mockRestore() - expect(service.mergeFork('parent', 'atomic')).toBe(2) - expect(service.mergeFork('parent', 'atomic')).toBe(2) - expect( - table.getBySession('parent').filter((entry) => entry.source_type === 'fork') - ).toHaveLength(3) - - db.close() - }) - - itIfSqlite('rejects missing and discarded forks in SQLite', () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - expect(() => service.mergeFork('parent', 'missing-native')).toThrow( - 'Fork missing-native does not exist or has been discarded.' - ) - service.createFork('parent', 'discarded-native') - service.discardFork('parent', 'discarded-native') - expect(() => service.mergeFork('parent', 'discarded-native')).toThrow( - 'Fork discarded-native does not exist or has been discarded.' - ) - expect(table.getBySession('parent').some((entry) => entry.name === 'fork/merge')).toBe(false) - } finally { - db.close() - } - }) - - it('cleans fork search projection on discard without blocking the discard event', () => { - const { table, entries } = createTapeTableMock() - const projectionTable = { - deleteBySession: vi.fn(() => { - throw new Error('projection cleanup failed') - }) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const fork = service.createFork('s1', 'fork-cleanup') - service.appendForkMessageRecord(fork, createRecord({ id: 'fu-cleanup', sessionId: 'ignored' })) - service.discardFork('s1', 'fork-cleanup') - - expect(table.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) - expect(projectionTable.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) - expect(entries.some((entry) => entry.session_id === fork.forkSessionId)).toBe(false) - expect( - entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') - ).toBe(true) - }) - - it('links a frozen subagent Tape without copying child entries and retries idempotently', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ sessionId: 'child', name: 'child/result', data: { text: 'done' } }) - const input = { - parentSessionId: 'parent', - childSessionId: 'child', - runId: 'run-1', - taskId: 'task-1', - slotId: 'reviewer', - taskTitle: 'Review', - outcome: 'completed' as const, - resultSummary: 'Done' - } - const first = service.linkSubagentTape(input) - table.appendEvent({ sessionId: 'child', name: 'child/late', data: { text: 'late' } }) - const retry = service.linkSubagentTape(input) - const normalizedRetry = service.linkSubagentTape({ - ...input, - slotId: ' reviewer ', - taskTitle: ' Review ', - resultSummary: ' Done ' - }) - - expect(first).toEqual({ - linkEntry: { sessionId: 'parent', entryId: 2 }, - childSessionId: 'child', - childHeadEntryId: 2, - childEntryCount: 2, - outcome: 'completed' - }) - expect(retry).toEqual(first) - expect(normalizedRetry).toEqual(first) - const links = entries.filter( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - ) - expect(links).toHaveLength(1) - expect(links[0]).toMatchObject({ - source_type: 'subagent', - source_id: 'child', - source_seq: 2 - }) - expect(JSON.parse(links[0].payload_json).data).toMatchObject({ - linkVersion: 2, - childTapeIdentity: expect.stringMatching(/^[a-f0-9]{64}$/) - }) - expect( - entries.some((entry) => entry.session_id === 'parent' && entry.name === 'child/result') - ).toBe(false) - expect(() => service.linkSubagentTape({ ...input, outcome: 'error' })).toThrow( - 'Subagent Tape link conflicts with finalized task run-1/task-1.' - ) - expect(() => service.linkSubagentTape({ ...input, slotId: 'writer' })).toThrow( - 'Subagent Tape link conflicts with finalized task run-1/task-1.' - ) - expect(() => service.linkSubagentTape({ ...input, taskTitle: 'Write' })).toThrow( - 'Subagent Tape link conflicts with finalized task run-1/task-1.' - ) - expect(() => service.linkSubagentTape({ ...input, resultSummary: 'Changed' })).toThrow( - 'Subagent Tape link conflicts with finalized task run-1/task-1.' - ) - - table.ensureBootstrapAnchor('child-2') - expect( - service.linkSubagentTape({ - ...input, - childSessionId: 'child-2', - outcome: 'error' - }) - ).toMatchObject({ - childSessionId: 'child-2', - outcome: 'error' - }) - expect( - entries.filter( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - ) - ).toHaveLength(2) - }) - - it('rejects a stored subagent link whose task identity no longer matches its provenance', () => { - const { table, entries } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - const input = createSubagentLinkInput('parent', 'child') - service.linkSubagentTape(input) - - const link = entries.find( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - ) - if (!link) { - throw new Error('Expected subagent Tape link fixture.') - } - const payload = JSON.parse(link.payload_json) as { - data?: Record - } - link.payload_json = JSON.stringify({ - ...payload, - data: { ...payload.data, runId: 'different-run' } - }) - - expect(() => service.linkSubagentTape(input)).toThrow( - /Stored subagent Tape link receipt is malformed/ - ) - expect(() => service.getContext('parent', [1], { sourceSessionId: 'child' })).toThrowError( - /not an authorized direct child/ - ) - }) - - it('keeps a version-one link readable while its original unmarked Tape remains present', () => { - const { table, entries } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/result', - data: { text: 'version one compatibility marker' } - }) - const input = createSubagentLinkInput('parent', 'child') - const receipt = service.linkSubagentTape(input) - const link = entries.find( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - )! - const payload = JSON.parse(link.payload_json) - payload.data.linkVersion = 1 - delete payload.data.childTapeIdentity - link.payload_json = JSON.stringify(payload) - entries.find((entry) => entry.session_id === 'child' && entry.entry_id === 1)!.meta_json = '{}' - - expect(service.linkSubagentTape(input)).toEqual(receipt) - expect( - service.search('parent', 'version one compatibility marker', { - scope: 'linked_subagents' - }) - ).toMatchObject([{ sessionId: 'child', entryId: 2 }]) - }) - - it('fails closed when a version-one link points at malformed incarnation metadata', () => { - const { table, entries } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/result', - data: { text: 'malformed incarnation metadata marker' } - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - const link = entries.find( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - )! - const payload = JSON.parse(link.payload_json) - payload.data.linkVersion = 1 - delete payload.data.childTapeIdentity - link.payload_json = JSON.stringify(payload) - entries.find((entry) => entry.session_id === 'child' && entry.entry_id === 1)!.meta_json = '{' - - expect(() => - service.search('parent', 'malformed incarnation metadata marker', { - scope: 'linked_subagents' - }) - ).toThrowError(/Linked Tape child is unavailable/) - }) - - it('searches direct linked children at frozen heads with one global limit', () => { - const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child-a', session_kind: 'subagent', parent_session_id: 'parent' }, - { id: 'child-b', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child-a') - table.ensureBootstrapAnchor('child-b') - table.appendEvent({ - sessionId: 'parent', - name: 'parent/note', - data: { text: 'shared needle from parent' }, - createdAt: 150 - }) - table.appendEvent({ - sessionId: 'child-a', - name: 'child/result', - data: { text: 'shared needle from A' }, - createdAt: 100 - }) - table.appendEvent({ - sessionId: 'child-b', - name: 'child/result', - data: { text: 'shared needle from B' }, - createdAt: 200 - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child-a')) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child-b')) - table.appendEvent({ - sessionId: 'child-a', - name: 'child/late', - data: { text: 'shared needle after cutoff' }, - createdAt: 300 - }) - - table.ensureBootstrapAnchor.mockClear() - table.append.mockClear() - const limited = service.search('parent', 'shared needle', { - scope: 'linked_subagents', - limit: 1 - }) - const all = service.search('parent', 'shared needle', { - scope: 'linked_subagents', - limit: 10 - }) - const combined = service.search('parent', 'shared needle', { - scope: 'current_and_linked', - limit: 10 - }) - - expect(limited).toMatchObject([{ sessionId: 'child-b', entryId: 2 }]) - expect(all.map((result) => [result.sessionId, result.entryId])).toEqual([ - ['child-b', 2], - ['child-a', 2] - ]) - expect(combined.map((result) => [result.sessionId, result.entryId])).toEqual([ - ['child-b', 2], - ['parent', 2], - ['child-a', 2] - ]) - expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( - [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 2 } - ], - 'shared needle', - expect.objectContaining({ limit: 1 }) - ) - expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() - expect(table.append).not.toHaveBeenCalled() - }) - - it('falls back all linked sources together when projection coverage is partial', () => { - const { table } = createTapeTableMock() - const projectionTable = { - searchSourcesReadOnly: vi.fn(() => ({ - coveredSources: [{ sessionId: 'child-a', maxEntryId: 2 }], - rows: [ - { - session_id: 'child-a', - entry_id: 2, - kind: 'event', - name: 'child/result', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'shared partial needle', - summary_text: 'shared partial needle from A', - refs_json: '{}', - created_at: 100, - score: -100 - } - ] - })) - } - const { service } = createLinkedTapeService( - table, - [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child-a', session_kind: 'subagent', parent_session_id: 'parent' }, - { id: 'child-b', session_kind: 'subagent', parent_session_id: 'parent' } - ], - projectionTable - ) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child-a') - table.ensureBootstrapAnchor('child-b') - table.appendEvent({ - sessionId: 'child-a', - name: 'child/result', - data: { text: 'shared partial needle from A' }, - createdAt: 100 - }) - table.appendEvent({ - sessionId: 'child-b', - name: 'child/result', - data: { text: 'shared partial needle from B' }, - createdAt: 200 - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child-a')) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child-b')) - table.searchEffectiveSourcesAtHeads.mockClear() - - const hits = service.search('parent', 'shared partial needle', { - scope: 'linked_subagents', - limit: 1 - }) - - expect(hits).toMatchObject([{ sessionId: 'child-b', entryId: 2 }]) - expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( - [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 2 } - ], - 'shared partial needle', - expect.objectContaining({ limit: 1 }) - ) - }) - - it('uses an exact frozen projection read-only after the child appends a late tail', () => { - const { table } = createTapeTableMock() - const projectionTable = { - searchSourcesReadOnly: vi.fn((sources: any[]) => ({ - coveredSources: sources, - rows: [ - { - session_id: 'child', - entry_id: 2, - kind: 'event', - name: 'child/result', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'frozen projection needle', - summary_text: 'frozen projection needle', - refs_json: '{}', - created_at: 100, - score: -1 - } - ] - })), - replaceSession: vi.fn(), - appendSession: vi.fn(), - getByEntryIds: vi.fn().mockReturnValue([]) - } - const { service } = createLinkedTapeService( - table, - [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ], - projectionTable - ) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/result', - data: { text: 'frozen projection needle' }, - createdAt: 100 - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - table.appendEvent({ - sessionId: 'child', - name: 'child/late', - data: { text: 'frozen projection needle late' }, - createdAt: 200 - }) - table.searchEffectiveSourcesAtHeads.mockClear() - - const hits = service.search('parent', 'frozen projection needle', { - scope: 'linked_subagents' - }) - - expect(projectionTable.searchSourcesReadOnly).toHaveBeenCalledWith( - [{ sessionId: 'child', maxEntryId: 2 }], - 'frozen projection needle', - expect.any(Object) - ) - expect(hits).toMatchObject([{ sessionId: 'child', entryId: 2 }]) - expect(table.searchEffectiveSourcesAtHeads).not.toHaveBeenCalled() - expect(projectionTable.replaceSession).not.toHaveBeenCalled() - expect(projectionTable.appendSession).not.toHaveBeenCalled() - }) - - it('deduplicates repeated child links at the newest finalized snapshot', () => { - const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/first', - data: { text: 'first snapshot marker' } - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - table.appendEvent({ - sessionId: 'child', - name: 'child/second', - data: { text: 'second snapshot marker' } - }) - service.linkSubagentTape({ - ...createSubagentLinkInput('parent', 'child'), - runId: 'run-child-second', - taskId: 'task-child-second' - }) - - expect( - service.search('parent', 'second snapshot marker', { scope: 'linked_subagents' }) - ).toMatchObject([{ sessionId: 'child', entryId: 3 }]) - expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( - [{ sessionId: 'child', maxEntryId: 3 }], - 'second snapshot marker', - expect.any(Object) - ) - }) - - it('expands linked context within one source and never crosses the frozen head', () => { - const { table } = createTapeTableMock() - const projectionTable = { - getByEntryIds: vi.fn(() => [ - { - session_id: 'child', - entry_id: 2, - kind: 'event', - name: 'child/target', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'stale projection text', - summary_text: 'stale projection summary', - refs_json: '{"stale":true}', - created_at: 100 - } - ]) - } - const { service } = createLinkedTapeService( - table, - [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ], - projectionTable - ) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/target', - data: { text: 'target evidence' }, - createdAt: 100 - }) - table.appendEvent({ - sessionId: 'child', - name: 'child/neighbor', - data: { text: 'neighbor evidence' }, - createdAt: 110 - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - table.appendEvent({ - sessionId: 'child', - name: 'child/late', - data: { text: 'late evidence' }, - createdAt: 120 - }) - table.append.mockClear() - - const context = service.getContext('parent', [2], { - sourceSessionId: 'child', - before: 0, - after: 5 - }) - - expect(context).toMatchObject({ - sessionId: 'parent', - sourceSessionId: 'child', - requestedEntryIds: [2], - matchedEntryIds: [2] - }) - expect(context.entries.map((entry) => entry.entryId)).toEqual([2, 3]) - expect(context.entries[0].summary).toContain('target evidence') - expect(context.entries[0].summary).not.toContain('stale projection') - expect(table.getEffectiveContextRowsAtHead).toHaveBeenCalledWith( - { sessionId: 'child', maxEntryId: 3 }, - [2], - expect.objectContaining({ before: 0, after: 5 }) - ) - expect(projectionTable.getByEntryIds).not.toHaveBeenCalled() - expect(table.append).not.toHaveBeenCalled() - }) - - it('rejects sibling and unlinked source ids even when a forged link event exists', () => { - const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'other-parent', session_kind: 'regular', parent_session_id: null }, - { id: 'sibling', session_kind: 'subagent', parent_session_id: 'other-parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('sibling') - service.linkSubagentTape(createSubagentLinkInput('parent', 'sibling')) - - let error: unknown - try { - service.getContext('parent', [1], { sourceSessionId: 'sibling' }) - } catch (caught) { - error = caught - } - expect(error).toMatchObject({ - code: 'linked_tape_unauthorized', - parentSessionId: 'parent', - sourceSessionId: 'sibling' - }) - }) - - it('reports a finalized linked Tape as unavailable after its durable session is deleted', () => { - const { table } = createTapeTableMock() - const { service, sessionById } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - sessionById.delete('child') - - let error: unknown - try { - service.search('parent', 'anything', { scope: 'linked_subagents' }) - } catch (caught) { - error = caught - } - expect(error).toMatchObject({ - code: 'linked_tape_unavailable', - parentSessionId: 'parent', - sourceSessionId: 'child' - }) - }) - - it('rejects a rebuilt child Tape until a new incarnation is explicitly linked', () => { - const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/original', - data: { text: 'original incarnation marker' } - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - - table.deleteBySession('child') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/rebuilt', - data: { text: 'rebuilt incarnation marker' } - }) - expect( - service.search('child', 'rebuilt incarnation marker', { scope: 'current' }) - ).toHaveLength(1) - expect(() => - service.search('parent', 'rebuilt incarnation marker', { scope: 'linked_subagents' }) - ).toThrowError(/Linked Tape child is unavailable/) - - service.linkSubagentTape({ - ...createSubagentLinkInput('parent', 'child'), - runId: 'run-rebuilt-child', - taskId: 'task-rebuilt-child' - }) - expect( - service.search('parent', 'rebuilt incarnation marker', { scope: 'linked_subagents' }) - ).toMatchObject([{ sessionId: 'child', entryId: 2 }]) - }) - - itIfSqlite('detects linked Tape replacement after entry ids are reused in SQLite', () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/original', - data: { text: 'native original incarnation' } - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - - table.deleteBySession('child') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/rebuilt', - data: { text: 'native rebuilt incarnation' } - }) - - expect(() => - service.search('parent', 'native rebuilt incarnation', { scope: 'linked_subagents' }) - ).toThrowError(/Linked Tape child is unavailable/) - } finally { - db.close() - } - }) - - it('reads legacy external merge links and keeps legacy discard audit-only', () => { - const { table, entries } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'legacy-child', session_kind: 'subagent', parent_session_id: 'parent' }, - { id: 'malformed-child', session_kind: 'subagent', parent_session_id: 'parent' }, - { id: 'discarded-child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('legacy-child') - table.ensureBootstrapAnchor('malformed-child') - table.ensureBootstrapAnchor('discarded-child') - const legacyBootstrap = entries.find( - (entry) => entry.session_id === 'legacy-child' && entry.entry_id === 1 - )! - legacyBootstrap.meta_json = '{}' - table.appendEvent({ - sessionId: 'legacy-child', - name: 'child/result', - data: { text: 'legacy link needle' } - }) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/merge', - source: { type: 'fork', id: 'legacy-child', seq: 0 }, - provenanceKey: 'fork:parent:legacy-child:external-merge:event', - data: { - forkId: 'legacy-child', - forkSessionId: 'legacy-child', - referencedEntryCount: 2, - status: 'completed' - } - }) - table.appendEvent({ - sessionId: 'malformed-child', - name: 'child/result', - data: { text: 'malformed legacy link needle' } - }) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/merge', - source: { type: 'fork', id: 'malformed-child', seq: 1 }, - provenanceKey: 'fork:parent:malformed-child:external-merge:event', - data: { - forkId: 'malformed-child', - forkSessionId: 'malformed-child', - referencedEntryCount: 2, - status: 'completed' - } - }) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/discard', - source: { type: 'fork', id: 'discarded-child', seq: 0 }, - provenanceKey: 'fork:parent:discarded-child:external-discard:event', - data: { - forkId: 'discarded-child', - forkSessionId: 'discarded-child', - status: 'cancelled' - } - }) - - expect( - service.search('parent', 'legacy link needle', { scope: 'linked_subagents' }) - ).toMatchObject([{ sessionId: 'legacy-child', entryId: 2 }]) - expect( - service.search('parent', 'malformed legacy link needle', { scope: 'linked_subagents' }) - ).toEqual([]) - let error: unknown - try { - service.getContext('parent', [1], { sourceSessionId: 'discarded-child' }) - } catch (caught) { - error = caught - } - expect(error).toMatchObject({ code: 'linked_tape_unauthorized' }) - }) - - it('uses effective message facts after replacement and retraction events', () => { - const { table, entries } = createTapeTableMock() - const original = createRecord({ id: 'u1', orderSeq: 1 }) - const messageStore = { - getMessages: vi.fn().mockReturnValue([original]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.ensureSessionTapeReady('s1', messageStore as any) - appendMessageReplacementToTape( - table as any, - createRecord({ - id: 'u1', - orderSeq: 1, - content: JSON.stringify({ - text: 'edited', - files: [], - links: [], - search: false, - think: false - }), - updatedAt: 300 - }), - 'test_edit' - ) - - expect(JSON.parse(service.getMessageRecords('s1')[0].content).text).toBe('edited') - expect(service.search('s1', 'hello', { kinds: ['message'] })).toEqual([]) - expect(service.search('s1', 'edited', { kinds: ['message'] })).toHaveLength(1) - expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) - - appendMessageRetractionToTape(table as any, service.getMessageRecords('s1')[0], 'test_delete') - - expect(service.getMessageRecords('s1')).toEqual([]) - expect(service.search('s1', 'edited', { kinds: ['message'] })).toEqual([]) - }) - - it('appends non-idempotent retractions without generated provenance keys', () => { - const { table, entries } = createTapeTableMock() - const record = createRecord({ id: 'u1' }) - - appendMessageRetractionToTape(table as any, record, 'first_delete') - appendMessageRetractionToTape(table as any, record, 'second_delete') - - const retractions = entries.filter((entry) => entry.name === 'message/retracted') - expect(retractions).toHaveLength(2) - expect(retractions.map((entry) => entry.provenance_key)).toEqual([null, null]) - }) -}) diff --git a/test/main/session/data/tapeFacts.test.ts b/test/main/session/data/tapeFacts.test.ts index 6dbfcf63c5..98e10024c6 100644 --- a/test/main/session/data/tapeFacts.test.ts +++ b/test/main/session/data/tapeFacts.test.ts @@ -2,7 +2,10 @@ import { describe, expect, it } from 'vitest' import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' import { appendMessageRecordToTape, appendToolFactsToTape } from '@/session/data/tapeFacts' import { buildEffectiveTapeView } from '@/session/data/tapeEffectiveView' -import { tapeToolRank } from '@/session/data/tables/deepchatTapeEffectiveSemantics' +import { + messageRecordHasFinalToolUse, + tapeToolRank +} from '@/session/data/tables/deepchatTapeEffectiveSemantics' import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' function createTable() { @@ -162,6 +165,26 @@ describe('appendToolFactsToTape', () => { expect(toolCallIds).toEqual(['done1']) }) + it('ignores malformed block elements without shifting valid tool provenance', () => { + const table = createTable() + const record = assistantRecord([ + null, + 'malformed', + toolCallBlock('success', 'valid', 'done') + ] as unknown as AssistantMessageBlock[]) + + expect(messageRecordHasFinalToolUse(record)).toBe(true) + expect(appendToolFactsToTape(table as any, record, 'live', 'tool_loop')).toBe(2) + expect(table.rows.filter((row) => row.kind === 'tool_call')).toMatchObject([ + { source_id: 'a1:valid', source_seq: 2 } + ]) + expect( + messageRecordHasFinalToolUse( + assistantRecord([null, 42] as unknown as AssistantMessageBlock[]) + ) + ).toBe(false) + }) + it('dedupes a mid-loop snapshot and the finalize write to one effective tool fact', () => { const table = createTable() const pending = assistantRecord([toolCallBlock('success', 'tc1', 'result')], { diff --git a/test/main/session/data/tapeFork.test.ts b/test/main/session/data/tapeFork.test.ts new file mode 100644 index 0000000000..be7da20c8a --- /dev/null +++ b/test/main/session/data/tapeFork.test.ts @@ -0,0 +1,459 @@ +import { + describe, + expect, + it, + vi, + SessionTape, + DeepChatTapeEntriesTable, + SqliteTapeLifecycleAdapter, + DatabaseCtor, + itIfSqlite, + createTapeTableMock, + createRecord +} from './tapeTestHarness' + +describe('SessionTape forks', () => { + it('keeps fork writes isolated until merge and discards fork entries on discard', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn() }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const fork = service.createFork('s1', 'fork-1') + service.appendForkMessageRecord(fork, createRecord({ id: 'fu1', sessionId: 'ignored' })) + + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'message/user') + ).toBe(false) + + const mergedCount = service.mergeFork('s1', 'fork-1') + + expect(mergedCount).toBe(1) + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'message/user') + ).toBe(true) + expect(entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/merge')).toBe( + true + ) + expect(entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/start')).toBe( + false + ) + + const discardFork = service.createFork('s1', 'fork-2') + service.appendForkMessageRecord(discardFork, createRecord({ id: 'fu2', sessionId: 'ignored' })) + service.discardFork('s1', 'fork-2') + + expect(entries.some((entry) => entry.session_id === discardFork.forkSessionId)).toBe(false) + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') + ).toBe(true) + }) + + it('records exact fork heads and keeps retries bounded to the first merge', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('parent') + table.appendEvent({ sessionId: 'parent', name: 'parent/tail', data: { value: 1 } }) + const fork = service.createFork('parent', 'bounded') + table.appendEvent({ sessionId: 'parent', name: 'parent/later', data: { value: 2 } }) + const retriedFork = service.createFork('parent', 'bounded') + service.appendForkMessageRecord(fork, createRecord({ id: 'first', sessionId: 'ignored' })) + + const forkStart = entries.find( + (entry) => entry.session_id === fork.forkSessionId && entry.name === 'fork/start' + ) + expect(fork.parentHeadEntryId).toBe(2) + expect(retriedFork.parentHeadEntryId).toBe(2) + expect(JSON.parse(forkStart.payload_json).state.parentHeadEntryId).toBe(2) + + const getBoundedEntries = table.getBySessionUpToEntryId.getMockImplementation()! + table.getBySessionUpToEntryId.mockImplementationOnce( + (sessionId: string, maxEntryId: number) => { + table.appendEvent({ + sessionId: fork.forkSessionId, + name: 'fork/late-tail', + data: { value: 2 } + }) + return getBoundedEntries(sessionId, maxEntryId) + } + ) + + expect(service.mergeFork('parent', 'bounded')).toBe(1) + expect(service.mergeFork('parent', 'bounded')).toBe(1) + expect(() => service.createFork('parent', 'bounded')).toThrow( + 'Fork bounded has already been merged and cannot be reused.' + ) + + const parentEntries = entries.filter((entry) => entry.session_id === 'parent') + expect(parentEntries.filter((entry) => entry.source_type === 'fork')).toHaveLength(2) + expect(parentEntries.some((entry) => entry.name === 'fork/late-tail')).toBe(false) + const receipt = parentEntries.find((entry) => entry.name === 'fork/merge')! + expect(JSON.parse(receipt.payload_json).data).toMatchObject({ + forkHeadEntryId: 3, + mergedCount: 1 + }) + }) + + it('rolls back mocked fork merge writes when a copied entry fails', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('parent', 'mock-atomic') + service.appendForkMessageRecord( + fork, + createRecord({ id: 'first', orderSeq: 1, sessionId: 'ignored' }) + ) + service.appendForkMessageRecord( + fork, + createRecord({ id: 'second', orderSeq: 2, sessionId: 'ignored' }) + ) + + const append = table.append.getMockImplementation()! + let copiedEntryCount = 0 + table.append.mockImplementation((input: any) => { + if (input.sessionId === 'parent' && input.source?.type === 'fork') { + copiedEntryCount += 1 + if (copiedEntryCount === 2) { + throw new Error('injected merge failure') + } + } + return append(input) + }) + + expect(() => service.mergeFork('parent', 'mock-atomic')).toThrow('injected merge failure') + expect( + entries.filter( + (entry) => + entry.session_id === 'parent' && + (entry.source_type === 'fork' || entry.name === 'fork/merge') + ) + ).toEqual([]) + }) + + it('rolls back copied fork entries when the merge receipt fails', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('parent', 'receipt-failure') + service.appendForkMessageRecord(fork, createRecord({ id: 'first', sessionId: 'ignored' })) + + const appendEvent = table.appendEvent.getMockImplementation()! + table.appendEvent.mockImplementation((input: any) => { + if (input.sessionId === 'parent' && input.name === 'fork/merge') { + throw new Error('injected receipt failure') + } + return appendEvent(input) + }) + + expect(() => service.mergeFork('parent', 'receipt-failure')).toThrow('injected receipt failure') + expect( + entries.filter( + (entry) => + entry.session_id === 'parent' && + (entry.source_type === 'fork' || entry.name === 'fork/merge') + ) + ).toEqual([]) + }) + + it('commits one idempotent receipt for an empty fork', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + service.createFork('parent', 'empty') + + expect(service.mergeFork('parent', 'empty')).toBe(0) + expect(service.mergeFork('parent', 'empty')).toBe(0) + const parentEntries = entries.filter((entry) => entry.session_id === 'parent') + expect(parentEntries).toHaveLength(1) + expect(parentEntries[0].name).toBe('fork/merge') + expect(JSON.parse(parentEntries[0].payload_json).data).toMatchObject({ + forkHeadEntryId: 2, + mergedCount: 0 + }) + }) + + it('rejects missing and discarded forks without committing an empty merge receipt', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn() }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + expect(() => service.mergeFork('parent', 'missing')).toThrow( + 'Fork missing does not exist or has been discarded.' + ) + + service.createFork('parent', 'discarded') + service.discardFork('parent', 'discarded') + expect(() => service.mergeFork('parent', 'discarded')).toThrow( + 'Fork discarded does not exist or has been discarded.' + ) + expect( + entries.filter((entry) => entry.session_id === 'parent' && entry.name === 'fork/merge') + ).toEqual([]) + }) + + it('returns an existing merge receipt after the merged fork Tape is cleaned up', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('parent', 'cleanup-after-merge') + service.appendForkMessageRecord(fork, createRecord({ id: 'merged', sessionId: 'ignored' })) + + expect(service.mergeFork('parent', 'cleanup-after-merge')).toBe(1) + table.deleteBySession(fork.forkSessionId) + expect(service.mergeFork('parent', 'cleanup-after-merge')).toBe(1) + }) + + it('merges a legacy fork start that predates the parent head field', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('parent', 'legacy-start') + const start = entries.find( + (entry) => entry.session_id === fork.forkSessionId && entry.name === 'fork/start' + )! + const payload = JSON.parse(start.payload_json) + delete payload.state.parentHeadEntryId + start.payload_json = JSON.stringify(payload) + service.appendForkMessageRecord(fork, createRecord({ id: 'legacy', sessionId: 'ignored' })) + + expect(service.mergeFork('parent', 'legacy-start')).toBe(1) + }) + + it('accepts a valid legacy fork merge receipt without a frozen head', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/merge', + source: { type: 'fork', id: 'legacy-receipt', seq: 0 }, + provenanceKey: 'fork:parent:legacy-receipt:merge:event', + data: { + forkId: 'legacy-receipt', + forkSessionId: 'parent::fork::legacy-receipt', + mergedCount: 2 + } + }) + + expect(service.mergeFork('parent', 'legacy-receipt')).toBe(2) + }) + + it('rejects a malformed stored fork merge receipt instead of reporting an empty merge', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/merge', + source: { type: 'fork', id: 'malformed-receipt', seq: 0 }, + provenanceKey: 'fork:parent:malformed-receipt:merge:event', + data: { + forkId: 'malformed-receipt', + forkSessionId: 'parent::fork::malformed-receipt', + mergedCount: 'two' + } + }) + + expect(() => service.mergeFork('parent', 'malformed-receipt')).toThrow( + 'Stored fork merge receipt is malformed' + ) + }) + + it('keeps external fork receipt identity fields authoritative over metadata', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.ensureBootstrapAnchor('external-child') + + const merge = service.recordExternalForkMerge('parent', 'external-child', 'external-child', { + forkId: 'spoofed', + forkSessionId: 'spoofed-session', + referencedEntryCount: 999, + taskId: 'task-1' + }) + const discard = service.recordExternalForkDiscard( + 'parent', + 'external-child', + 'external-child', + { + forkId: 'spoofed', + forkSessionId: 'spoofed-session', + taskId: 'task-1' + } + ) + + expect(JSON.parse(merge.payload_json).data).toEqual({ + forkId: 'external-child', + forkSessionId: 'external-child', + referencedEntryCount: 1, + taskId: 'task-1' + }) + expect(JSON.parse(discard.payload_json).data).toEqual({ + forkId: 'external-child', + forkSessionId: 'external-child', + taskId: 'task-1' + }) + }) + + itIfSqlite('rolls back copied fork entries and the receipt when merge fails', () => { + const db = new DatabaseCtor(':memory:') + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('parent') + const fork = service.createFork('parent', 'atomic') + service.appendForkMessageRecord( + fork, + createRecord({ id: 'first', orderSeq: 1, sessionId: 'ignored' }) + ) + service.appendForkMessageRecord( + fork, + createRecord({ id: 'second', orderSeq: 2, sessionId: 'ignored' }) + ) + + const append = table.append.bind(table) + let copiedEntryCount = 0 + const appendSpy = vi.spyOn(table, 'append').mockImplementation((input) => { + if (input.sessionId === 'parent' && input.source?.type === 'fork') { + copiedEntryCount += 1 + if (copiedEntryCount === 2) { + throw new Error('injected merge failure') + } + } + return append(input) + }) + + expect(() => service.mergeFork('parent', 'atomic')).toThrow('injected merge failure') + expect( + table + .getBySession('parent') + .filter((entry) => entry.source_type === 'fork' || entry.name === 'fork/merge') + ).toEqual([]) + + appendSpy.mockRestore() + expect(service.mergeFork('parent', 'atomic')).toBe(2) + expect(service.mergeFork('parent', 'atomic')).toBe(2) + expect( + table.getBySession('parent').filter((entry) => entry.source_type === 'fork') + ).toHaveLength(3) + + db.close() + }) + + itIfSqlite('rejects missing and discarded forks in SQLite', () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn() }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + expect(() => service.mergeFork('parent', 'missing-native')).toThrow( + 'Fork missing-native does not exist or has been discarded.' + ) + service.createFork('parent', 'discarded-native') + service.discardFork('parent', 'discarded-native') + expect(() => service.mergeFork('parent', 'discarded-native')).toThrow( + 'Fork discarded-native does not exist or has been discarded.' + ) + expect(table.getBySession('parent').some((entry) => entry.name === 'fork/merge')).toBe(false) + } finally { + db.close() + } + }) + + it('keeps a failed fork cleanup isolated and makes its discard receipt fail closed', () => { + const { table, entries } = createTapeTableMock() + const projectionTable = { + deleteBySession: vi.fn(() => { + throw new Error('projection cleanup failed') + }) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const fork = service.createFork('s1', 'fork-cleanup') + service.appendForkMessageRecord(fork, createRecord({ id: 'fu-cleanup', sessionId: 'ignored' })) + service.discardFork('s1', 'fork-cleanup') + + expect(table.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) + expect(projectionTable.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) + expect(entries.some((entry) => entry.session_id === fork.forkSessionId)).toBe(true) + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') + ).toBe(true) + expect(() => service.mergeFork('s1', 'fork-cleanup')).toThrow( + 'Fork fork-cleanup does not exist or has been discarded.' + ) + expect(() => service.createFork('s1', 'fork-cleanup')).toThrow( + 'Fork fork-cleanup has been discarded and cannot be reused.' + ) + }) + + it('restores fork entries when the discard receipt cannot be appended', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn() }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('s1', 'receipt-cleanup') + service.appendForkMessageRecord( + fork, + createRecord({ id: 'receipt-message', sessionId: 'ignored' }) + ) + const appendEvent = table.appendEvent.getMockImplementation()! + table.appendEvent.mockImplementation((input: any) => { + if (input.sessionId === 's1' && input.name === 'fork/discard') { + throw new Error('discard receipt failed') + } + return appendEvent(input) + }) + + expect(() => service.discardFork('s1', 'receipt-cleanup')).toThrow('discard receipt failed') + expect(entries.some((entry) => entry.session_id === fork.forkSessionId)).toBe(true) + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') + ).toBe(false) + }) +}) diff --git a/test/main/session/data/tapeLifecycle.test.ts b/test/main/session/data/tapeLifecycle.test.ts new file mode 100644 index 0000000000..87a3493837 --- /dev/null +++ b/test/main/session/data/tapeLifecycle.test.ts @@ -0,0 +1,429 @@ +import { + DatabaseCtor, + DeepChatTapeEntriesTable, + DeepChatTapeSearchProjectionTable, + describe, + expect, + it, + itIfSqlite, + SessionTape, + SqliteTapeLifecycleAdapter, + vi +} from './tapeTestHarness' +import { SessionTranscriptMutations } from '@/session/transcriptMutations' + +function createHarness() { + const calls: string[] = [] + const state = { entriesPresent: true, searchPresent: true, bootstrapCount: 0 } + const entryStore = { + runInTransaction: vi.fn((operation: () => unknown) => { + const snapshot = { ...state } + try { + return operation() + } catch (error) { + Object.assign(state, snapshot) + throw error + } + }), + ensureBootstrapAnchor: vi.fn((sessionId: string) => { + calls.push(`bootstrap:${sessionId}`) + state.entriesPresent = true + state.bootstrapCount += 1 + }) + } + const lifecycle = { + deleteBySession: vi.fn((sessionId: string) => { + calls.push(`entries:${sessionId}`) + state.entriesPresent = false + }) + } + const searchProjection = { + deleteBySession: vi.fn((sessionId: string) => { + calls.push(`search:${sessionId}`) + state.searchPresent = false + }) + } + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: lifecycle, + deepchatTapeSearchProjectionTable: searchProjection + } as any) + + return { calls, entryStore, lifecycle, searchProjection, state, tape } +} + +describe('SessionTape lifecycle administration', () => { + it('deletes entries before the search projection', () => { + const { calls, entryStore, tape } = createHarness() + + tape.deleteSessionTape('s1') + + expect(calls).toEqual(['entries:s1', 'search:s1']) + expect(entryStore.runInTransaction).toHaveBeenCalledOnce() + }) + + it('rolls back final Tape deletion when search cleanup fails', () => { + const { searchProjection, state, tape } = createHarness() + searchProjection.deleteBySession.mockImplementationOnce(() => { + throw new Error('search cleanup failed') + }) + + expect(() => tape.deleteSessionTape('s1')).toThrow('search cleanup failed') + expect(state).toEqual({ entriesPresent: true, searchPresent: true, bootstrapCount: 0 }) + }) + + it('rebuilds the bootstrap only after both destructive stores are cleared', () => { + const { calls, entryStore, tape } = createHarness() + + tape.resetSessionTape('s1') + + expect(calls).toEqual(['entries:s1', 'search:s1', 'bootstrap:s1']) + expect(entryStore.runInTransaction).toHaveBeenCalledOnce() + }) + + it('does not create a mixed-generation Tape when projection cleanup fails', () => { + const { entryStore, searchProjection, state, tape } = createHarness() + searchProjection.deleteBySession.mockImplementationOnce(() => { + throw new Error('search cleanup failed') + }) + + expect(() => tape.resetSessionTape('s1')).toThrow('search cleanup failed') + expect(entryStore.ensureBootstrapAnchor).not.toHaveBeenCalled() + expect(state).toEqual({ entriesPresent: true, searchPresent: true, bootstrapCount: 0 }) + }) + + it('rolls a reset back when the replacement bootstrap fails', () => { + const { entryStore, state, tape } = createHarness() + entryStore.ensureBootstrapAnchor.mockImplementationOnce(() => { + throw new Error('bootstrap failed') + }) + + expect(() => tape.resetSessionTape('s1')).toThrow('bootstrap failed') + expect(state).toEqual({ entriesPresent: true, searchPresent: true, bootstrapCount: 0 }) + }) + + itIfSqlite('rolls back a failed reset and creates a fresh incarnation only on retry', () => { + const db = new DatabaseCtor(':memory:') + try { + const entryStore = new DeepChatTapeEntriesTable(db) + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + entryStore.createTable() + searchProjection.createTable() + entryStore.ensureBootstrapAnchor('s1') + const oldBootstrap = entryStore.getBySession('s1')[0] + entryStore.appendEvent({ + sessionId: 's1', + name: 'old/generation', + data: { marker: 'old generation' } + }) + searchProjection.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'event', + name: 'old/generation', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'old generation', + summaryText: 'old generation', + refs: { generation: 'old' }, + createdAt: 100 + } + ], + 2 + ) + const deleteProjection = searchProjection.deleteBySession.bind(searchProjection) + const deleteProjectionSpy = vi + .spyOn(searchProjection, 'deleteBySession') + .mockImplementationOnce(() => { + throw new Error('search cleanup failed') + }) + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: searchProjection + } as any) + + expect(() => tape.resetSessionTape('s1')).toThrow('search cleanup failed') + expect(entryStore.getBySession('s1').map((entry) => entry.name)).toEqual([ + 'session/start', + 'old/generation' + ]) + expect(entryStore.getBySession('s1')[0].meta_json).toBe(oldBootstrap.meta_json) + expect(searchProjection.isCurrent('s1', 2)).toBe(true) + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([2]) + + deleteProjectionSpy.mockImplementation(deleteProjection) + tape.resetSessionTape('s1') + + const newEntries = entryStore.getBySession('s1') + expect(newEntries).toHaveLength(1) + expect(newEntries[0]).toMatchObject({ entry_id: 1, name: 'session/start' }) + expect(newEntries[0].meta_json).not.toBe(oldBootstrap.meta_json) + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([]) + expect(searchProjection.getSessionMeta('s1')).toBeNull() + } finally { + db.close() + } + }) + + itIfSqlite('rolls back partial search projection deletion', () => { + const db = new DatabaseCtor(':memory:') + try { + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + searchProjection.createTable() + searchProjection.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'event', + name: 'old/generation', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'old generation', + summaryText: 'old generation', + refs: { generation: 'old' }, + createdAt: 100 + } + ], + 1 + ) + db.exec(` + CREATE TRIGGER fail_projection_meta_delete + BEFORE DELETE ON deepchat_tape_search_projection_meta + WHEN old.session_id = 's1' + BEGIN + SELECT RAISE(ABORT, 'injected projection cleanup failure'); + END; + `) + + expect(() => searchProjection.deleteBySession('s1')).toThrow( + 'injected projection cleanup failure' + ) + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([1]) + expect(searchProjection.isCurrent('s1', 1)).toBe(true) + } finally { + db.close() + } + }) + + itIfSqlite('rolls back SQLite reset deletion when the replacement bootstrap fails', () => { + const db = new DatabaseCtor(':memory:') + try { + const entryStore = new DeepChatTapeEntriesTable(db) + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + entryStore.createTable() + searchProjection.createTable() + entryStore.ensureBootstrapAnchor('s1') + entryStore.appendEvent({ + sessionId: 's1', + name: 'old/generation', + data: { marker: 'old generation' } + }) + searchProjection.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'event', + name: 'old/generation', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'old generation', + summaryText: 'old generation', + refs: {}, + createdAt: 100 + } + ], + 2 + ) + const bootstrapSpy = vi + .spyOn(entryStore, 'ensureBootstrapAnchor') + .mockImplementationOnce(() => { + throw new Error('bootstrap failed') + }) + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: searchProjection + } as any) + + expect(() => tape.resetSessionTape('s1')).toThrow('bootstrap failed') + expect(entryStore.getBySession('s1').map((entry) => entry.name)).toEqual([ + 'session/start', + 'old/generation' + ]) + expect(searchProjection.isCurrent('s1', 2)).toBe(true) + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([2]) + bootstrapSpy.mockRestore() + } finally { + db.close() + } + }) + + itIfSqlite( + 'rolls transcript cleanup back with a failed Tape reset on one connection', + async () => { + const db = new DatabaseCtor(':memory:') + try { + db.exec(` + CREATE TABLE clear_pending (session_id TEXT NOT NULL); + CREATE TABLE clear_transcript (session_id TEXT NOT NULL); + INSERT INTO clear_pending (session_id) VALUES ('s1'); + INSERT INTO clear_transcript (session_id) VALUES ('s1'); + `) + const entryStore = new DeepChatTapeEntriesTable(db) + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + entryStore.createTable() + searchProjection.createTable() + entryStore.ensureBootstrapAnchor('s1') + entryStore.appendEvent({ + sessionId: 's1', + name: 'old/generation', + data: { marker: 'old generation' } + }) + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: searchProjection + } as any) + vi.spyOn(searchProjection, 'deleteBySession').mockImplementationOnce(() => { + throw new Error('search cleanup failed') + }) + const runtime = { + prepareClearMessages: vi.fn().mockResolvedValue(undefined), + finishClearMessages: vi.fn() + } + const mutations = new SessionTranscriptMutations({ + pendingInputs: { + deleteBySession: () => + db.prepare('DELETE FROM clear_pending WHERE session_id = ?').run('s1') + }, + transcript: { + deleteBySession: () => + db.prepare('DELETE FROM clear_transcript WHERE session_id = ?').run('s1') + }, + settings: { resetTape: () => tape.resetSessionTape('s1') }, + runtime, + runInTransaction: (operation) => db.transaction(operation)() + } as any) + + await expect(mutations.clearMessages('s1')).rejects.toThrow('search cleanup failed') + + expect(db.prepare('SELECT COUNT(*) AS count FROM clear_pending').get()).toEqual({ + count: 1 + }) + expect(db.prepare('SELECT COUNT(*) AS count FROM clear_transcript').get()).toEqual({ + count: 1 + }) + expect(entryStore.getBySession('s1').map((entry) => entry.name)).toEqual([ + 'session/start', + 'old/generation' + ]) + expect(runtime.finishClearMessages).not.toHaveBeenCalled() + } finally { + vi.restoreAllMocks() + db.close() + } + } + ) + + itIfSqlite('drops corrupt FTS state instead of blocking a Tape generation reset', () => { + const db = new DatabaseCtor(':memory:') + try { + const entryStore = new DeepChatTapeEntriesTable(db) + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + entryStore.createTable() + searchProjection.createTable() + if (!searchProjection.hasFtsReadyForTesting()) return + entryStore.ensureBootstrapAnchor('s1') + entryStore.appendEvent({ + sessionId: 's1', + name: 'old/generation', + data: { marker: 'private old generation' } + }) + searchProjection.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'event', + name: 'old/generation', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'private old generation', + summaryText: 'private old generation', + refs: {}, + createdAt: 100 + } + ], + 2 + ) + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: searchProjection + } as any) + const prepare = db.prepare.bind(db) + const prepareSpy = vi.spyOn(db, 'prepare').mockImplementation(((sql: string) => { + if ( + sql.trim().replace(/\s+/g, ' ') === + 'DELETE FROM deepchat_tape_search_fts WHERE session_id = ?' + ) { + throw new Error('injected corrupt FTS delete') + } + return prepare(sql) + }) as typeof db.prepare) + const exec = db.exec.bind(db) + const execSpy = vi.spyOn(db, 'exec').mockImplementation(((sql: string) => { + if (sql.trim() === 'DROP TABLE IF EXISTS deepchat_tape_search_fts') { + throw new Error('injected corrupt FTS drop') + } + return exec(sql) + }) as typeof db.exec) + + expect(() => tape.resetSessionTape('s1')).toThrow('injected corrupt FTS drop') + expect(entryStore.getBySession('s1').map((entry) => entry.name)).toEqual([ + 'session/start', + 'old/generation' + ]) + expect(searchProjection.isCurrent('s1', 2)).toBe(true) + execSpy.mockRestore() + + expect(() => tape.resetSessionTape('s1')).not.toThrow() + prepareSpy.mockRestore() + + expect(entryStore.getBySession('s1')).toMatchObject([{ entry_id: 1, name: 'session/start' }]) + expect(searchProjection.getSessionMeta('s1')).toBeNull() + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([]) + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'deepchat_tape_search_fts'" + ) + .get() + ).toBeUndefined() + expect(tape.search('s1', 'private old generation')).toEqual([]) + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'deepchat_tape_search_fts'" + ) + .get() + ).toEqual({ name: 'deepchat_tape_search_fts' }) + } finally { + vi.restoreAllMocks() + db.close() + } + }) +}) diff --git a/test/main/session/data/tapeLineage.test.ts b/test/main/session/data/tapeLineage.test.ts new file mode 100644 index 0000000000..da26bf7726 --- /dev/null +++ b/test/main/session/data/tapeLineage.test.ts @@ -0,0 +1,759 @@ +import { + describe, + expect, + it, + vi, + SessionTape, + DeepChatTapeEntriesTable, + DatabaseCtor, + itIfSqlite, + createTapeTableMock, + createLinkedTapeService, + createSubagentLinkInput, + SqliteTapeLifecycleAdapter +} from './tapeTestHarness' + +describe('SessionTape lineage', () => { + it('links a frozen subagent Tape without copying child entries and retries idempotently', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'child-2', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ sessionId: 'child', name: 'child/result', data: { text: 'done' } }) + const input = { + parentSessionId: 'parent', + childSessionId: 'child', + runId: 'run-1', + taskId: 'task-1', + slotId: 'reviewer', + taskTitle: 'Review', + outcome: 'completed' as const, + resultSummary: 'Done' + } + const first = service.linkSubagentTape(input) + table.appendEvent({ sessionId: 'child', name: 'child/late', data: { text: 'late' } }) + const retry = service.linkSubagentTape(input) + const normalizedRetry = service.linkSubagentTape({ + ...input, + slotId: ' reviewer ', + taskTitle: ' Review ', + resultSummary: ' Done ' + }) + + expect(first).toEqual({ + linkEntry: { sessionId: 'parent', entryId: 2 }, + childSessionId: 'child', + childHeadEntryId: 2, + childEntryCount: 2, + outcome: 'completed' + }) + expect(retry).toEqual(first) + expect(normalizedRetry).toEqual(first) + const links = entries.filter( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + ) + expect(links).toHaveLength(1) + expect(links[0]).toMatchObject({ + source_type: 'subagent', + source_id: 'child', + source_seq: 2 + }) + expect(JSON.parse(links[0].payload_json).data).toMatchObject({ + linkVersion: 2, + childTapeIdentity: expect.stringMatching(/^[a-f0-9]{64}$/) + }) + expect( + entries.some((entry) => entry.session_id === 'parent' && entry.name === 'child/result') + ).toBe(false) + expect(() => service.linkSubagentTape({ ...input, outcome: 'error' })).toThrow( + 'Subagent Tape link conflicts with finalized task run-1/task-1.' + ) + expect(() => service.linkSubagentTape({ ...input, slotId: 'writer' })).toThrow( + 'Subagent Tape link conflicts with finalized task run-1/task-1.' + ) + expect(() => service.linkSubagentTape({ ...input, taskTitle: 'Write' })).toThrow( + 'Subagent Tape link conflicts with finalized task run-1/task-1.' + ) + expect(() => service.linkSubagentTape({ ...input, resultSummary: 'Changed' })).toThrow( + 'Subagent Tape link conflicts with finalized task run-1/task-1.' + ) + + table.ensureBootstrapAnchor('child-2') + expect( + service.linkSubagentTape({ + ...input, + childSessionId: 'child-2', + outcome: 'error' + }) + ).toMatchObject({ + childSessionId: 'child-2', + outcome: 'error' + }) + expect( + entries.filter( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + ) + ).toHaveLength(2) + }) + + it('rejects a stored subagent link whose task identity no longer matches its provenance', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + const input = createSubagentLinkInput('parent', 'child') + service.linkSubagentTape(input) + + const link = entries.find( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + ) + if (!link) { + throw new Error('Expected subagent Tape link fixture.') + } + const payload = JSON.parse(link.payload_json) as { + data?: Record + } + link.payload_json = JSON.stringify({ + ...payload, + data: { ...payload.data, runId: 'different-run' } + }) + + expect(() => service.linkSubagentTape(input)).toThrow( + /Stored subagent Tape link receipt is malformed/ + ) + expect(() => service.getContext('parent', [1], { sourceSessionId: 'child' })).toThrowError( + /not an authorized direct child/ + ) + }) + + it('keeps a version-one link readable while its original unmarked Tape remains present', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/result', + data: { text: 'version one compatibility marker' } + }) + const input = createSubagentLinkInput('parent', 'child') + const receipt = service.linkSubagentTape(input) + const link = entries.find( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + )! + const payload = JSON.parse(link.payload_json) + payload.data.linkVersion = 1 + delete payload.data.childTapeIdentity + link.payload_json = JSON.stringify(payload) + entries.find((entry) => entry.session_id === 'child' && entry.entry_id === 1)!.meta_json = '{}' + + expect(service.linkSubagentTape(input)).toEqual(receipt) + expect( + service.search('parent', 'version one compatibility marker', { + scope: 'linked_subagents' + }) + ).toMatchObject([{ sessionId: 'child', entryId: 2 }]) + }) + + it('fails closed when a version-one link points at malformed incarnation metadata', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/result', + data: { text: 'malformed incarnation metadata marker' } + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + const link = entries.find( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + )! + const payload = JSON.parse(link.payload_json) + payload.data.linkVersion = 1 + delete payload.data.childTapeIdentity + link.payload_json = JSON.stringify(payload) + entries.find((entry) => entry.session_id === 'child' && entry.entry_id === 1)!.meta_json = '{' + + expect(() => + service.search('parent', 'malformed incarnation metadata marker', { + scope: 'linked_subagents' + }) + ).toThrowError(/Linked Tape child is unavailable/) + }) + + it('searches direct linked children at frozen heads with one global limit', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child-a', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'child-b', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child-a') + table.ensureBootstrapAnchor('child-b') + table.appendEvent({ + sessionId: 'parent', + name: 'parent/note', + data: { text: 'shared needle from parent' }, + createdAt: 150 + }) + table.appendEvent({ + sessionId: 'child-a', + name: 'child/result', + data: { text: 'shared needle from A' }, + createdAt: 100 + }) + table.appendEvent({ + sessionId: 'child-b', + name: 'child/result', + data: { text: 'shared needle from B' }, + createdAt: 200 + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child-a')) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child-b')) + table.appendEvent({ + sessionId: 'child-a', + name: 'child/late', + data: { text: 'shared needle after cutoff' }, + createdAt: 300 + }) + + table.ensureBootstrapAnchor.mockClear() + table.append.mockClear() + const limited = service.search('parent', 'shared needle', { + scope: 'linked_subagents', + limit: 1 + }) + const all = service.search('parent', 'shared needle', { + scope: 'linked_subagents', + limit: 10 + }) + const combined = service.search('parent', 'shared needle', { + scope: 'current_and_linked', + limit: 10 + }) + + expect(limited).toMatchObject([{ sessionId: 'child-b', entryId: 2 }]) + expect(all.map((result) => [result.sessionId, result.entryId])).toEqual([ + ['child-b', 2], + ['child-a', 2] + ]) + expect(combined.map((result) => [result.sessionId, result.entryId])).toEqual([ + ['child-b', 2], + ['parent', 2], + ['child-a', 2] + ]) + expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( + [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 2 } + ], + 'shared needle', + expect.objectContaining({ limit: 1 }) + ) + expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() + expect(table.append).not.toHaveBeenCalled() + }) + + it('falls back all linked sources together when projection coverage is partial', () => { + const { table } = createTapeTableMock() + const projectionTable = { + searchSourcesReadOnly: vi.fn(() => ({ + coveredSources: [{ sessionId: 'child-a', maxEntryId: 2 }], + rows: [ + { + session_id: 'child-a', + entry_id: 2, + kind: 'event', + name: 'child/result', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'shared partial needle', + summary_text: 'shared partial needle from A', + refs_json: '{}', + created_at: 100, + score: -100 + } + ] + })) + } + const { service } = createLinkedTapeService( + table, + [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child-a', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'child-b', session_kind: 'subagent', parent_session_id: 'parent' } + ], + projectionTable + ) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child-a') + table.ensureBootstrapAnchor('child-b') + table.appendEvent({ + sessionId: 'child-a', + name: 'child/result', + data: { text: 'shared partial needle from A' }, + createdAt: 100 + }) + table.appendEvent({ + sessionId: 'child-b', + name: 'child/result', + data: { text: 'shared partial needle from B' }, + createdAt: 200 + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child-a')) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child-b')) + table.searchEffectiveSourcesAtHeads.mockClear() + + const hits = service.search('parent', 'shared partial needle', { + scope: 'linked_subagents', + limit: 1 + }) + + expect(hits).toMatchObject([{ sessionId: 'child-b', entryId: 2 }]) + expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( + [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 2 } + ], + 'shared partial needle', + expect.objectContaining({ limit: 1 }) + ) + }) + + it('uses an exact frozen projection read-only after the child appends a late tail', () => { + const { table } = createTapeTableMock() + const projectionTable = { + searchSourcesReadOnly: vi.fn((sources: any[]) => ({ + coveredSources: sources, + rows: [ + { + session_id: 'child', + entry_id: 2, + kind: 'event', + name: 'child/result', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'frozen projection needle', + summary_text: 'frozen projection needle', + refs_json: '{}', + created_at: 100, + score: -1 + } + ] + })), + replaceSession: vi.fn(), + appendSession: vi.fn(), + getByEntryIds: vi.fn().mockReturnValue([]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) + } + const { service } = createLinkedTapeService( + table, + [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ], + projectionTable + ) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/result', + data: { text: 'frozen projection needle' }, + createdAt: 100 + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + table.appendEvent({ + sessionId: 'child', + name: 'child/late', + data: { text: 'frozen projection needle late' }, + createdAt: 200 + }) + table.searchEffectiveSourcesAtHeads.mockClear() + + const hits = service.search('parent', 'frozen projection needle', { + scope: 'linked_subagents' + }) + + expect(projectionTable.searchSourcesReadOnly).toHaveBeenCalledWith( + [{ sessionId: 'child', maxEntryId: 2 }], + 'frozen projection needle', + expect.any(Object) + ) + expect(hits).toMatchObject([{ sessionId: 'child', entryId: 2 }]) + expect(table.searchEffectiveSourcesAtHeads).not.toHaveBeenCalled() + expect(projectionTable.replaceSession).not.toHaveBeenCalled() + expect(projectionTable.appendSession).not.toHaveBeenCalled() + }) + + it('deduplicates repeated child links at the newest finalized snapshot', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/first', + data: { text: 'first snapshot marker' } + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + table.appendEvent({ + sessionId: 'child', + name: 'child/second', + data: { text: 'second snapshot marker' } + }) + service.linkSubagentTape({ + ...createSubagentLinkInput('parent', 'child'), + runId: 'run-child-second', + taskId: 'task-child-second' + }) + + expect( + service.search('parent', 'second snapshot marker', { scope: 'linked_subagents' }) + ).toMatchObject([{ sessionId: 'child', entryId: 3 }]) + expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( + [{ sessionId: 'child', maxEntryId: 3 }], + 'second snapshot marker', + expect.any(Object) + ) + }) + + it('expands linked context within one source and never crosses the frozen head', () => { + const { table } = createTapeTableMock() + const projectionTable = { + getByEntryIdsIfCurrent: vi.fn(() => [ + { + session_id: 'child', + entry_id: 2, + kind: 'event', + name: 'child/target', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'stale projection text', + summary_text: 'stale projection summary', + refs_json: '{"stale":true}', + created_at: 100 + } + ]) + } + const { service } = createLinkedTapeService( + table, + [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ], + projectionTable + ) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/target', + data: { text: 'target evidence' }, + createdAt: 100 + }) + table.appendEvent({ + sessionId: 'child', + name: 'child/neighbor', + data: { text: 'neighbor evidence' }, + createdAt: 110 + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + table.appendEvent({ + sessionId: 'child', + name: 'child/late', + data: { text: 'late evidence' }, + createdAt: 120 + }) + table.append.mockClear() + + const context = service.getContext('parent', [2], { + sourceSessionId: 'child', + before: 0, + after: 5 + }) + + expect(context).toMatchObject({ + sessionId: 'parent', + sourceSessionId: 'child', + requestedEntryIds: [2], + matchedEntryIds: [2] + }) + expect(context.entries.map((entry) => entry.entryId)).toEqual([2, 3]) + expect(context.entries[0].summary).toContain('target evidence') + expect(context.entries[0].summary).not.toContain('stale projection') + expect(table.getEffectiveContextRowsAtHead).toHaveBeenCalledWith( + { sessionId: 'child', maxEntryId: 3 }, + [2], + expect.objectContaining({ before: 0, after: 5 }) + ) + expect(projectionTable.getByEntryIdsIfCurrent).not.toHaveBeenCalled() + expect(table.append).not.toHaveBeenCalled() + }) + + it('rejects non-direct children at write time and after reparenting', () => { + const { table } = createTapeTableMock() + const { service, sessionById } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'other-parent', session_kind: 'regular', parent_session_id: null }, + { id: 'sibling', session_kind: 'subagent', parent_session_id: 'other-parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('sibling') + + expect(() => service.linkSubagentTape(createSubagentLinkInput('parent', 'sibling'))).toThrow( + 'Session sibling is not a direct subagent child of parent.' + ) + expect( + table + .getBySession('parent') + .some((entry: { name: string | null }) => entry.name === 'subagent/tape_linked') + ).toBe(false) + + sessionById.set('sibling', { + id: 'sibling', + session_kind: 'subagent', + parent_session_id: 'parent' + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'sibling')) + sessionById.set('sibling', { + id: 'sibling', + session_kind: 'subagent', + parent_session_id: 'other-parent' + }) + + let error: unknown + try { + service.getContext('parent', [1], { sourceSessionId: 'sibling' }) + } catch (caught) { + error = caught + } + expect(error).toMatchObject({ + code: 'linked_tape_unauthorized', + parentSessionId: 'parent', + sourceSessionId: 'sibling' + }) + }) + + it('rejects missing parents and non-subagent children before persisting a link', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'regular-child', session_kind: 'regular', parent_session_id: 'parent' }, + { id: 'orphan', session_kind: 'subagent', parent_session_id: 'missing-parent' } + ]) + table.ensureBootstrapAnchor('regular-child') + table.ensureBootstrapAnchor('orphan') + + expect(() => + service.linkSubagentTape(createSubagentLinkInput('parent', 'regular-child')) + ).toThrow('Session regular-child is not a direct subagent child of parent.') + expect(() => + service.linkSubagentTape(createSubagentLinkInput('missing-parent', 'orphan')) + ).toThrow('Session orphan is not a direct subagent child of missing-parent.') + expect( + ['parent', 'missing-parent'].some((sessionId) => + table + .getBySession(sessionId) + .some((entry: { name: string | null }) => entry.name === 'subagent/tape_linked') + ) + ).toBe(false) + }) + + it('reports a finalized linked Tape as unavailable after its durable session is deleted', () => { + const { table } = createTapeTableMock() + const { service, sessionById } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + sessionById.delete('child') + + let error: unknown + try { + service.search('parent', 'anything', { scope: 'linked_subagents' }) + } catch (caught) { + error = caught + } + expect(error).toMatchObject({ + code: 'linked_tape_unavailable', + parentSessionId: 'parent', + sourceSessionId: 'child' + }) + }) + + it('rejects a rebuilt child Tape until a new incarnation is explicitly linked', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/original', + data: { text: 'original incarnation marker' } + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + + table.deleteBySession('child') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/rebuilt', + data: { text: 'rebuilt incarnation marker' } + }) + expect( + service.search('child', 'rebuilt incarnation marker', { scope: 'current' }) + ).toHaveLength(1) + expect(() => + service.search('parent', 'rebuilt incarnation marker', { scope: 'linked_subagents' }) + ).toThrowError(/Linked Tape child is unavailable/) + + service.linkSubagentTape({ + ...createSubagentLinkInput('parent', 'child'), + runId: 'run-rebuilt-child', + taskId: 'task-rebuilt-child' + }) + expect( + service.search('parent', 'rebuilt incarnation marker', { scope: 'linked_subagents' }) + ).toMatchObject([{ sessionId: 'child', entryId: 2 }]) + }) + + itIfSqlite('detects linked Tape replacement after entry ids are reused in SQLite', () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const lifecycle = new SqliteTapeLifecycleAdapter(db) + table.createTable() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/original', + data: { text: 'native original incarnation' } + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + + lifecycle.deleteBySession('child') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/rebuilt', + data: { text: 'native rebuilt incarnation' } + }) + + expect(() => + service.search('parent', 'native rebuilt incarnation', { scope: 'linked_subagents' }) + ).toThrowError(/Linked Tape child is unavailable/) + } finally { + db.close() + } + }) + + it('reads legacy external merge links and keeps legacy discard audit-only', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'legacy-child', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'malformed-child', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'discarded-child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('legacy-child') + table.ensureBootstrapAnchor('malformed-child') + table.ensureBootstrapAnchor('discarded-child') + const legacyBootstrap = entries.find( + (entry) => entry.session_id === 'legacy-child' && entry.entry_id === 1 + )! + legacyBootstrap.meta_json = '{}' + table.appendEvent({ + sessionId: 'legacy-child', + name: 'child/result', + data: { text: 'legacy link needle' } + }) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/merge', + source: { type: 'fork', id: 'legacy-child', seq: 0 }, + provenanceKey: 'fork:parent:legacy-child:external-merge:event', + data: { + forkId: 'legacy-child', + forkSessionId: 'legacy-child', + referencedEntryCount: 2, + status: 'completed' + } + }) + table.appendEvent({ + sessionId: 'malformed-child', + name: 'child/result', + data: { text: 'malformed legacy link needle' } + }) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/merge', + source: { type: 'fork', id: 'malformed-child', seq: 1 }, + provenanceKey: 'fork:parent:malformed-child:external-merge:event', + data: { + forkId: 'malformed-child', + forkSessionId: 'malformed-child', + referencedEntryCount: 2, + status: 'completed' + } + }) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/discard', + source: { type: 'fork', id: 'discarded-child', seq: 0 }, + provenanceKey: 'fork:parent:discarded-child:external-discard:event', + data: { + forkId: 'discarded-child', + forkSessionId: 'discarded-child', + status: 'cancelled' + } + }) + + expect( + service.search('parent', 'legacy link needle', { scope: 'linked_subagents' }) + ).toMatchObject([{ sessionId: 'legacy-child', entryId: 2 }]) + expect( + service.search('parent', 'malformed legacy link needle', { scope: 'linked_subagents' }) + ).toEqual([]) + let error: unknown + try { + service.getContext('parent', [1], { sourceSessionId: 'discarded-child' }) + } catch (caught) { + error = caught + } + expect(error).toMatchObject({ code: 'linked_tape_unauthorized' }) + }) +}) diff --git a/test/main/session/data/tapeRecall.test.ts b/test/main/session/data/tapeRecall.test.ts new file mode 100644 index 0000000000..52840d8c2f --- /dev/null +++ b/test/main/session/data/tapeRecall.test.ts @@ -0,0 +1,2491 @@ +import { + performance, + describe, + expect, + it, + vi, + SessionTape, + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + DeepChatTapeEntriesTable, + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, + DeepChatTapeSearchProjectionTable, + NewSessionsTable, + DatabaseCtor, + sqliteAvailable, + sqliteSkipReason, + itIfSqlite, + createTapeTableMock, + createRecord, + createTapeService +} from './tapeTestHarness' + +describe('SessionTape recall', () => { + it('invalidates projections written before atomic generation transitions', () => { + expect(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION).toBeGreaterThan(2) + }) + + itIfSqlite('prunes legacy and metadata-orphaned search projections during initialization', () => { + const db = new DatabaseCtor(':memory:') + try { + const projection = new DeepChatTapeSearchProjectionTable(db) + projection.createTable() + const row = (sessionId: string, searchText: string) => ({ + sessionId, + entryId: 1, + kind: 'event' as const, + name: 'projection/test', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText, + summaryText: searchText, + refs: {}, + createdAt: 100 + }) + projection.replaceSession('legacy', [row('legacy', 'private legacy text')], 1, 2) + projection.replaceSession('current', [row('current', 'current text')], 1) + db.prepare( + `INSERT INTO deepchat_tape_search_projection ( + session_id, + entry_id, + kind, + name, + source_type, + source_id, + source_seq, + search_text, + summary_text, + refs_json, + created_at + ) + VALUES ('orphan', 1, 'event', 'projection/test', NULL, NULL, NULL, ?, ?, '{}', 100)` + ).run('private orphan text', 'private orphan text') + + const restarted = new DeepChatTapeSearchProjectionTable(db) + restarted.createTable() + + expect(restarted.getProjectedEntryIds('legacy')).toEqual([]) + expect(restarted.getSessionMeta('legacy')).toBeNull() + expect(restarted.getProjectedEntryIds('orphan')).toEqual([]) + expect(restarted.getProjectedEntryIds('current')).toEqual([1]) + expect(restarted.isCurrent('current', 1)).toBe(true) + const ftsExists = db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'deepchat_tape_search_fts'" + ) + .get() + if (ftsExists) { + expect( + db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_search_fts + WHERE session_id IN ('legacy', 'orphan')` + ) + .get() + ).toEqual({ count: 0 }) + expect( + db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_search_fts + WHERE session_id = 'current'` + ) + .get() + ).toEqual({ count: 1 }) + } + } finally { + db.close() + } + }) + + itIfSqlite('caches FTS capability detection per SQLite connection', () => { + const db = new DatabaseCtor(':memory:') + try { + const exec = db.exec.bind(db) + let probeStatementCount = 0 + const execSpy = vi.spyOn(db, 'exec').mockImplementation(((sql: string) => { + if (sql.includes('tape_search_fts_probe_')) probeStatementCount += 1 + return exec(sql) + }) as typeof db.exec) + + new DeepChatTapeSearchProjectionTable(db).createTable() + const firstDetectionCount = probeStatementCount + new DeepChatTapeSearchProjectionTable(db).createTable() + + expect(firstDetectionCount).toBeGreaterThan(0) + expect(probeStatementCount).toBe(firstDetectionCount) + execSpy.mockRestore() + } finally { + vi.restoreAllMocks() + db.close() + } + }) + + it('reports info, search, and handoff within one session scope', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([ + createRecord({ id: 'u1' }), + createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { type: 'content', content: 'answer', status: 'success', timestamp: 101 } + ]), + metadata: JSON.stringify({ totalTokens: 9 }), + createdAt: 101, + updatedAt: 101 + }) + ]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + service.handoff('s1', 'phase_done', { summary: ' done ' }) + const handoffAnchor = entries.find((entry) => entry.name === 'handoff/phase_done') + + expect(service.info('s1')).toMatchObject({ + sessionId: 's1', + anchors: 2, + lastAnchor: 'handoff/phase_done', + lastTokenUsage: 9, + migrationState: 'ready' + }) + expect(JSON.parse(handoffAnchor.payload_json).state).toMatchObject({ + summary: 'done', + cursorOrderSeq: 3, + range: { + fromOrderSeq: 1, + toOrderSeq: 2 + }, + sourceMessageIds: ['u1', 'a1'] + }) + expect(service.search('s1', 'hello')).toHaveLength(1) + expect( + service.search('s1', 'hello', { kinds: ['message'], start: '1970-01-01T00:00:00.000Z' }) + ).toHaveLength(1) + expect(service.search('s1', 'hello', { kinds: ['anchor'] })).toHaveLength(0) + expect(service.search('s1', 'hello', { end: '99' })).toHaveLength(0) + expect(() => service.search('s1', 'hello', { start: 'not-a-date' })).toThrow( + 'start must be an ISO date/time or millisecond timestamp.' + ) + expect(service.anchors('s1')).toMatchObject([ + { sessionId: 's1', name: 'session/start' }, + { sessionId: 's1', name: 'handoff/phase_done' } + ]) + expect(service.anchors('s1', { limit: 1 })).toMatchObject([ + { sessionId: 's1', name: 'handoff/phase_done' } + ]) + expect(service.search('s2', 'hello')).toHaveLength(0) + }) + + it('returns only effective message DTOs for a requested source span', () => { + const { table } = createTapeTableMock() + const first = table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { record: createRecord({ id: 'm1', orderSeq: 1 }) }, + meta: { source: 'live', orderSeq: 1, role: 'user' } + }) + const second = table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm2', seq: 0 }, + payload: { record: createRecord({ id: 'm2', orderSeq: 2 }) }, + meta: { source: 'live', orderSeq: 2, role: 'user' } + }) + table.appendEvent({ + sessionId: 's1', + name: 'message/retracted', + source: { type: 'message', id: 'm1', seq: 1 }, + data: { messageId: 'm1', reason: 'deleted' } + }) + const service = new SessionTape({ deepchatTapeEntriesTable: table } as any) + + const span = service.getEffectiveMessageSourceSpan('s1', [first.entry_id, second.entry_id]) + + expect(span).toHaveLength(1) + expect(span[0]).toEqual({ + entryId: second.entry_id, + record: { + role: 'user', + content: createRecord({ id: 'm2', orderSeq: 2 }).content, + orderSeq: 2 + } + }) + expect(span[0]).not.toHaveProperty('payload_json') + }) + + it('projects fallback tape search into compact results and bounded context', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'Run the dev server', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.append({ + sessionId: 's1', + kind: 'tool_result', + name: 'shell', + source: { type: 'tool_result', id: 'm1:tc1', seq: 0 }, + payload: { + messageId: 'm1', + orderSeq: 2, + toolCallId: 'tc1', + command: 'pnpm run dev', + exitStatus: 1, + response: 'Command failed with EADDRINUSE in /tmp/deepchat.log' + }, + meta: { source: 'live', status: 'error' }, + createdAt: 110 + }) + + const hits = service.search('s1', 'pnpm run dev', { limit: 5 }) + expect(hits).toHaveLength(1) + expect(hits[0]).toMatchObject({ + kind: 'tool_result', + summary: expect.stringContaining('EADDRINUSE'), + refs: { + toolCallId: 'tc1', + commands: expect.arrayContaining(['pnpm run dev']), + filePaths: expect.arrayContaining(['/tmp/deepchat.log']), + errorCodes: expect.arrayContaining(['EADDRINUSE']), + exitStatus: 1 + } + }) + expect(hits[0]).not.toHaveProperty('payload') + expect(hits[0]).not.toHaveProperty('meta') + + const context = service.getContext('s1', [hits[0].entryId], { + before: 0, + after: 0, + maxBytesPerEntry: 12, + maxTotalBytes: 12 + }) + expect(context.matchedEntryIds).toEqual([hits[0].entryId]) + expect(context.entries[0]).toMatchObject({ + entryId: hits[0].entryId, + evidence: { truncated: true } + }) + expect(context.entries[0].evidence.bytes).toBeLessThanOrEqual(12) + expect(context.entries[0]).not.toHaveProperty('payload') + expect(context.entries[0]).not.toHaveProperty('meta') + + const exhaustedContext = service.getContext('s1', [hits[0].entryId], { + before: 0, + after: 0, + maxTotalBytes: 0 + }) + expect(exhaustedContext.entries).toEqual([]) + expect(exhaustedContext.matchedEntryIds).toEqual([]) + }) + + it('projects user message attachment metadata into search text and refs', () => { + const { table } = createTapeTableMock() + const projectionTable = { + isCurrent: vi.fn().mockReturnValue(false), + getSessionMeta: vi.fn().mockReturnValue(null), + getProjectedEntryIds: vi.fn().mockReturnValue([]), + appendSession: vi.fn(), + replaceSession: vi.fn(), + search: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm-file', seq: 0 }, + payload: { + record: createRecord({ + id: 'm-file', + content: JSON.stringify({ + text: 'Please review the attachment', + files: [ + { + name: 'a.md', + path: '/tmp/a.md', + content: 'raw attachment body should not be projected', + metadata: { fileName: 'workspace-a.md' } + } + ], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + + service.search('s1', '/tmp/a.md', { limit: 5 }) + + expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) + const projectedRows = projectionTable.replaceSession.mock.calls[0][1] + expect(projectedRows[0]).toMatchObject({ + entryId: 1, + refs: { + filePaths: expect.arrayContaining(['/tmp/a.md']), + fileNames: expect.arrayContaining(['a.md', 'workspace-a.md']) + } + }) + expect(projectedRows[0].searchText).toContain('/tmp/a.md') + expect(projectedRows[0].searchText).toContain('a.md') + expect(projectedRows[0].searchText).toContain('workspace-a.md') + expect(projectedRows[0].searchText).not.toContain('raw attachment body should not be projected') + }) + + it('preserves relative file paths in projected refs', () => { + const { table } = createTapeTableMock() + let projectedRows: any[] = [] + const projectionTable = { + isCurrent: vi.fn().mockReturnValue(false), + getSessionMeta: vi.fn().mockReturnValue(null), + getProjectedEntryIds: vi.fn().mockReturnValue([]), + appendSession: vi.fn(), + replaceSession: vi.fn((_sessionId: string, rows: any[]) => { + projectedRows = rows + }), + search: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm-relative', seq: 0 }, + payload: { + record: createRecord({ + id: 'm-relative', + content: JSON.stringify({ + text: 'Touched src/main/index.ts, ./lib/util.ts, ../shared/types.ts, test/main/foo/bar.ts, /usr/local/bin/deploy, and https://example.com/not-a-file', + files: [], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + + service.search('s1', 'src/main/index.ts', { limit: 5 }) + + expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) + const filePaths = projectedRows[0].refs.filePaths + expect(filePaths).toEqual( + expect.arrayContaining([ + 'src/main/index.ts', + './lib/util.ts', + '../shared/types.ts', + 'test/main/foo/bar.ts', + '/usr/local/bin/deploy' + ]) + ) + expect(filePaths).not.toContain('/main/index.ts') + expect(filePaths).not.toContain('/lib/util.ts') + expect(filePaths).not.toContain('/main/foo/bar.ts') + expect(filePaths).not.toContain('example.com/not-a-file') + }) + + it('rebuilds old tape projection versions before attachment path search', () => { + const { table } = createTapeTableMock() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm-file', seq: 0 }, + payload: { + record: createRecord({ + id: 'm-file', + content: JSON.stringify({ + text: 'Please review the migrated attachment', + files: [ + { + name: 'legacy.md', + path: '/tmp/legacy.md', + content: 'raw migrated body must stay out of projection', + metadata: { fileName: 'legacy-workspace.md' } + } + ], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + let storedVersion = 1 + let storedMaxEntryId = 1 + let projectedRows: any[] = [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm-file', + sourceSeq: 0, + searchText: 'message/user Please review the migrated attachment', + summaryText: 'user: Please review the migrated attachment', + refs: { messageId: 'm-file' }, + createdAt: 100 + } + ] + const projectionTable = { + isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => { + return ( + storedVersion === DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION && + storedMaxEntryId === maxEntryId + ) + }), + getSessionMeta: vi.fn(() => ({ + projectionVersion: storedVersion, + maxEntryId: storedMaxEntryId + })), + getProjectedEntryIds: vi.fn().mockReturnValue([1]), + appendSession: vi.fn(), + replaceSession: vi.fn((_sessionId: string, rows: any[], maxEntryId: number) => { + projectedRows = rows + storedVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + storedMaxEntryId = maxEntryId + }), + search: vi.fn((_sessionId: string, query: string) => { + return projectedRows + .filter((row) => row.searchText.includes(query)) + .map((row) => ({ + session_id: row.sessionId, + entry_id: row.entryId, + kind: row.kind, + name: row.name, + source_type: row.sourceType, + source_id: row.sourceId, + source_seq: row.sourceSeq, + search_text: row.searchText, + summary_text: row.summaryText, + refs_json: JSON.stringify(row.refs), + created_at: row.createdAt, + score: 1 + })) + }) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const hits = service.search('s1', '/tmp/legacy.md', { limit: 5 }) + + expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) + expect(projectionTable.appendSession).not.toHaveBeenCalled() + expect(hits).toHaveLength(1) + expect(hits[0].refs).toMatchObject({ + filePaths: expect.arrayContaining(['/tmp/legacy.md']), + fileNames: expect.arrayContaining(['legacy.md', 'legacy-workspace.md']) + }) + expect(projectedRows[0].searchText).not.toContain( + 'raw migrated body must stay out of projection' + ) + }) + + it('prioritizes requested tape context entries before window entries under byte caps', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'before-1', seq: 0 }, + payload: { + record: createRecord({ + id: 'before-1', + content: JSON.stringify({ + text: 'before entry consumes the tiny byte budget first', + files: [], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'before-2', seq: 0 }, + payload: { + record: createRecord({ + id: 'before-2', + content: JSON.stringify({ + text: 'second before entry also appears earlier', + files: [], + links: [] + }), + createdAt: 110, + updatedAt: 110 + }) + }, + meta: { source: 'live', orderSeq: 2, role: 'user' }, + createdAt: 110 + }) + const target = table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'target', seq: 0 }, + payload: { + record: createRecord({ + id: 'target', + content: JSON.stringify({ + text: 'target-ok consumes the available budget', + files: [], + links: [] + }), + createdAt: 120, + updatedAt: 120 + }) + }, + meta: { source: 'live', orderSeq: 3, role: 'user' }, + createdAt: 120 + }) + + const context = service.getContext('s1', [target.entry_id], { + before: 2, + after: 0, + limit: 3, + maxBytesPerEntry: 18, + maxTotalBytes: 18 + }) + + expect(context.matchedEntryIds).toEqual([target.entry_id]) + expect(context.entries.map((entry) => entry.entryId)).toEqual([target.entry_id]) + expect(context.entries[0].evidence.text).toContain('target-ok') + }) + + it('falls back to the current Tape when the context projection is not current', () => { + const { table } = createTapeTableMock() + const entry = table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'current-message', seq: 0 }, + payload: { + record: createRecord({ + id: 'current-message', + content: JSON.stringify({ text: 'current generation context', files: [], links: [] }) + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' } + }) + const projectionTable = { + getByEntryIds: vi.fn(() => [ + { + session_id: 's1', + entry_id: entry.entry_id, + summary_text: 'old private context', + refs_json: '{"messageId":"old-message"}' + } + ]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const context = service.getContext('s1', [entry.entry_id], { before: 0, after: 0 }) + + expect(projectionTable.getByEntryIdsIfCurrent).toHaveBeenCalledWith('s1', entry.entry_id, [ + entry.entry_id + ]) + expect(projectionTable.getByEntryIds).not.toHaveBeenCalled() + expect(context.entries[0]).toMatchObject({ + summary: expect.stringContaining('current generation context'), + refs: { messageId: 'current-message' } + }) + expect(context.entries[0].summary).not.toContain('old private context') + }) + + itIfSqlite('ignores a pre-transition projection version at the same Tape head', () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + table.createTable() + projectionTable.createTable() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'current-message', seq: 0 }, + payload: { + record: createRecord({ + id: 'current-message', + content: JSON.stringify({ text: 'current generation context', files: [], links: [] }) + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' } + }) + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'old-message', + sourceSeq: 0, + searchText: 'old private context', + summaryText: 'old private context', + refs: { messageId: 'old-message' }, + createdAt: 100 + } + ], + 1, + 2 + ) + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const context = service.getContext('s1', [1], { before: 0, after: 0 }) + + expect(context.entries[0]).toMatchObject({ + entryId: 1, + summary: expect.stringContaining('current generation context'), + refs: { messageId: 'current-message' } + }) + expect(context.entries[0].summary).not.toContain('old private context') + } finally { + db.close() + } + }) + + itIfSqlite('ignores stale same-entry projection context when the Tape head has advanced', () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + table.createTable() + projectionTable.createTable() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'current-message', seq: 0 }, + payload: { + record: createRecord({ + id: 'current-message', + content: JSON.stringify({ + text: 'current generation context', + files: [], + links: [] + }) + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' } + }) + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'old-message', + sourceSeq: 0, + searchText: 'old private context', + summaryText: 'old private context', + refs: { messageId: 'old-message' }, + createdAt: 100 + } + ], + 0 + ) + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const context = service.getContext('s1', [1], { before: 0, after: 0 }) + + expect(context.entries[0]).toMatchObject({ + entryId: 1, + summary: expect.stringContaining('current generation context'), + refs: { messageId: 'current-message' } + }) + expect(context.entries[0].summary).not.toContain('old private context') + } finally { + db.close() + } + }) + + it('binds tape projection LIKE fallback params for single and multi-term queries', () => { + const all = vi.fn().mockReturnValue([]) + const db = { + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => all(sql, params), + get: vi.fn().mockReturnValue(undefined), + run: vi.fn() + })) + } + const projectionTable = new DeepChatTapeSearchProjectionTable(db as any) + ;(projectionTable as any).recoverSessionFts = vi.fn() + + projectionTable.search('s1', 'Redis', { limit: 5 }) + expect(all).toHaveBeenLastCalledWith( + expect.stringContaining('FROM deepchat_tape_search_projection'), + ['s1', '%Redis%', '%Redis%', '%Redis%', 5] + ) + + projectionTable.search('s1', 'Redis TTL', { limit: 5 }) + expect(all).toHaveBeenLastCalledWith( + expect.stringContaining('FROM deepchat_tape_search_projection'), + [ + 's1', + '%Redis TTL%', + '%Redis TTL%', + '%Redis TTL%', + '%Redis%', + '%Redis%', + '%Redis%', + '%TTL%', + '%TTL%', + '%TTL%', + 5 + ] + ) + + const oversizedQuery = Array.from({ length: 257 }, (_, index) => `term-${index}`).join(' ') + projectionTable.search('s1', oversizedQuery, { limit: 5 }) + expect(all).toHaveBeenLastCalledWith( + expect.stringContaining('FROM deepchat_tape_search_projection'), + ['s1', `%${oversizedQuery}%`, `%${oversizedQuery}%`, `%${oversizedQuery}%`, 5] + ) + }) + + it('checks projection version and head in the same context row query', () => { + const reads: Array<{ sql: string; params: unknown[] }> = [] + const projectionTable = new DeepChatTapeSearchProjectionTable({ + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => { + reads.push({ sql, params }) + return [] + } + })) + } as any) + + projectionTable.getByEntryIdsIfCurrent('s1', 7, [3, 3, 0], 42) + + expect(reads).toHaveLength(1) + expect(reads[0].sql).toContain('INNER JOIN deepchat_tape_search_projection_meta AS meta') + expect(reads[0].sql).toContain('meta.projection_version = ?') + expect(reads[0].sql).toContain('meta.max_entry_id = ?') + expect(reads[0].params).toEqual([42, 7, 's1', 3]) + }) + + it('uses current tape projection without loading full session rows', () => { + const { table } = createTapeTableMock() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'Redis compact marker', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.getBySession.mockClear() + const projectionTable = { + isCurrent: vi.fn().mockReturnValue(true), + getSessionMeta: vi.fn(), + getProjectedEntryIds: vi.fn(), + appendSession: vi.fn(), + replaceSession: vi.fn(), + search: vi.fn().mockReturnValue([ + { + session_id: 's1', + entry_id: 1, + kind: 'message', + name: 'message/user', + source_type: 'message', + source_id: 'm1', + source_seq: 0, + search_text: 'Redis compact marker', + summary_text: 'Redis compact marker', + refs_json: '{"messageId":"m1"}', + created_at: 100, + score: -2 + } + ]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const hits = service.search('s1', 'Redis compact', { limit: 5 }) + + expect(table.getMaxEntryId).toHaveBeenCalledWith('s1') + expect(table.getBySession).not.toHaveBeenCalled() + expect(projectionTable.search).toHaveBeenCalledWith( + 's1', + 'Redis compact', + expect.objectContaining({ limit: 5 }) + ) + expect(projectionTable.appendSession).not.toHaveBeenCalled() + expect(projectionTable.replaceSession).not.toHaveBeenCalled() + expect(hits[0]).toMatchObject({ + entryId: 1, + kind: 'message', + summary: 'Redis compact marker', + refs: { messageId: 'm1' }, + score: -2 + }) + expect(hits[0]).not.toHaveProperty('payload') + expect(hits[0]).not.toHaveProperty('meta') + }) + + it('falls back to effective tape search when projection search throws', () => { + const { table } = createTapeTableMock() + const projectionTable = { + isCurrent: vi.fn().mockReturnValue(true), + search: vi.fn(() => { + throw new Error('projection failed') + }) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'Redis fallback marker', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + + const hits = service.search('s1', 'Redis fallback', { limit: 5 }) + expect(hits).toHaveLength(1) + expect(hits[0]).toMatchObject({ + kind: 'message', + summary: expect.stringContaining('Redis fallback') + }) + expect(hits[0]).not.toHaveProperty('payload') + expect(hits[0]).not.toHaveProperty('meta') + }) + + it('appends tape projection rows when the previous projection is an effective prefix', () => { + const { table } = createTapeTableMock() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'first redis', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm2', seq: 0 }, + payload: { + record: createRecord({ + id: 'm2', + content: JSON.stringify({ text: 'second vue', files: [], links: [] }), + createdAt: 110, + updatedAt: 110 + }) + }, + meta: { source: 'live', orderSeq: 2, role: 'user' }, + createdAt: 110 + }) + const projectionTable = { + isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => maxEntryId === 1), + getSessionMeta: vi.fn().mockReturnValue({ projectionVersion: 1, maxEntryId: 1 }), + getProjectedEntryIds: vi.fn().mockReturnValue([1]), + appendSession: vi.fn(), + replaceSession: vi.fn(), + search: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.search('s1', 'vue', { limit: 5 }) + + expect(projectionTable.appendSession).toHaveBeenCalledTimes(1) + expect(projectionTable.appendSession.mock.calls[0][1].map((row: any) => row.entryId)).toEqual([ + 2 + ]) + expect(projectionTable.replaceSession).not.toHaveBeenCalled() + }) + + it('rebuilds tape projection when projected entry ids are not an effective prefix', () => { + const { table } = createTapeTableMock() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'redis', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + const projectionTable = { + isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => maxEntryId === 0), + getSessionMeta: vi.fn().mockReturnValue({ projectionVersion: 1, maxEntryId: 0 }), + getProjectedEntryIds: vi.fn().mockReturnValue([99]), + appendSession: vi.fn(), + replaceSession: vi.fn(), + search: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.search('s1', 'redis', { limit: 5 }) + + expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) + expect(projectionTable.appendSession).not.toHaveBeenCalled() + }) + + it('does not run LIKE fallback when FTS fills the tape projection search limit', () => { + const all = vi.fn((sql: string, _params: unknown[]) => + sql.includes('deepchat_tape_search_fts') + ? [ + { + session_id: 's1', + entry_id: 1, + kind: 'message', + name: 'message/user', + source_type: 'message', + source_id: 'm1', + source_seq: 0, + search_text: 'Redis TTL', + summary_text: 'Redis TTL', + refs_json: '{}', + created_at: 100, + score: -1 + } + ] + : [] + ) + const db = { + exec: vi.fn(() => { + throw new Error('unexpected FTS ensure') + }), + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => all(sql, params), + get: (..._params: unknown[]) => { + if ( + sql.includes('deepchat_tape_search_projection_meta') || + sql.includes('deepchat_tape_search_fts_meta') + ) { + return { projection_version: 1, max_entry_id: 1 } + } + return undefined + }, + run: vi.fn() + })), + transaction: vi.fn((callback: () => void) => callback) + } + const projectionTable = new DeepChatTapeSearchProjectionTable(db as any) + ;(projectionTable as any).ftsReady = true + + const hits = projectionTable.search('s1', 'Redis', { limit: 1 }) + + expect(hits).toHaveLength(1) + expect(db.exec).not.toHaveBeenCalled() + const ftsCall = all.mock.calls.find(([sql]) => String(sql).includes('deepchat_tape_search_fts')) + expect(String(ftsCall?.[0])).toContain('deepchat_tape_search_fts.session_id = ?') + expect((ftsCall?.[1] as unknown[]).filter((param) => param === 's1')).toHaveLength(2) + expect( + vi.mocked(db.prepare).mock.calls.some(([sql]) => String(sql).includes('NULL AS score')) + ).toBe(false) + }) + + it('invalidates a failed FTS query and rebuilds the derivative before the next search', () => { + const projectionRow = { + session_id: 's1', + entry_id: 1, + kind: 'message', + name: 'message/user', + source_type: 'message', + source_id: 'm1', + source_seq: 0, + search_text: 'Redis TTL', + summary_text: 'Redis TTL', + refs_json: '{}', + created_at: 100 + } + let ftsMetaPresent = true + let shouldFailFtsQuery = true + let likeQueryCount = 0 + let ftsQueryCount = 0 + const exec = vi.fn() + const prepare = vi.fn((sql: string) => ({ + all: (..._params: unknown[]) => { + if (sql.includes('bm25(deepchat_tape_search_fts)')) { + ftsQueryCount += 1 + if (shouldFailFtsQuery) { + shouldFailFtsQuery = false + throw new Error('injected corrupt FTS query') + } + return [{ ...projectionRow, score: -1 }] + } + if (sql.includes('NULL AS score')) { + likeQueryCount += 1 + return [{ ...projectionRow, score: null }] + } + if (sql.includes('SELECT *') && sql.includes('deepchat_tape_search_projection')) { + return [projectionRow] + } + return [] + }, + get: (..._params: unknown[]) => { + if (sql.includes('sqlite_master') && sql.includes('deepchat_tape_search_fts_meta')) { + return { name: 'deepchat_tape_search_fts_meta' } + } + if (sql.includes('FROM deepchat_tape_search_projection_meta')) { + return { projection_version: 1, max_entry_id: 1 } + } + if (sql.includes('FROM deepchat_tape_search_fts_meta')) { + return ftsMetaPresent ? { projection_version: 1, max_entry_id: 1 } : undefined + } + if (sql.includes('SELECT rowid')) { + return { rowid: 1 } + } + return undefined + }, + run: (..._params: unknown[]) => { + if (sql.includes('DELETE FROM deepchat_tape_search_fts_meta')) { + ftsMetaPresent = false + } + if (sql.includes('INSERT INTO deepchat_tape_search_fts_meta')) { + ftsMetaPresent = true + } + } + })) + const projectionTable = new DeepChatTapeSearchProjectionTable({ + exec, + prepare, + transaction: (callback: () => void) => callback + } as any) + ;(projectionTable as any).ftsReady = true + + expect(projectionTable.search('s1', 'Redis', { limit: 1 })).toMatchObject([ + { entry_id: 1, score: null } + ]) + expect(projectionTable.hasFtsReadyForTesting()).toBe(false) + expect(exec).toHaveBeenCalledWith('DROP TABLE IF EXISTS deepchat_tape_search_fts') + + expect(projectionTable.search('s1', 'Redis', { limit: 1 })).toMatchObject([ + { entry_id: 1, score: -1 } + ]) + expect(projectionTable.hasFtsReadyForTesting()).toBe(true) + expect(ftsQueryCount).toBe(2) + expect(likeQueryCount).toBe(1) + expect( + exec.mock.calls.some(([sql]) => String(sql).includes('CREATE VIRTUAL TABLE IF NOT EXISTS')) + ).toBe(true) + }) + + it('queries memory view manifests by agent without expanding session ids', () => { + const all = vi.fn().mockReturnValue([]) + const db = { + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => all(sql, params) + })) + } + const table = new DeepChatTapeEntriesTable(db as any) + + table.listMemoryViewManifestAnchorsByAgent('agent-a', { + sessionId: 's-1', + messageId: 'msg-1', + limit: 7 + }) + + expect(all).toHaveBeenCalledWith( + expect.stringContaining('INNER JOIN new_sessions AS sessions'), + ['agent-a', 's-1', 'msg-1', 7] + ) + const sql = String(all.mock.calls[0][0]) + expect(sql).not.toContain(' IN (') + expect(sql).toContain('sessions.agent_id = ?') + expect(sql).toContain('tape.session_id = ?') + expect(sql).toContain("json_extract(tape.meta_json, '$.messageId') = ?") + }) + + it('keeps linked raw search candidate-bounded instead of materializing complete Tapes', () => { + const all = vi.fn().mockReturnValue([]) + const db = { + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => all(sql, params) + })) + } + const table = new DeepChatTapeEntriesTable(db as any) + + table.searchEffectiveSourcesAtHeads( + [ + { sessionId: 'child-b', maxEntryId: 20 }, + { sessionId: 'child-a', maxEntryId: 10 } + ], + 'needle', + { limit: 5 } + ) + + const sql = String(all.mock.calls[0][0]) + const params = all.mock.calls[0][1] + expect(sql).toContain('FROM deepchat_tape_entries AS candidate') + expect(sql).toContain('candidate.entry_id <= source.max_entry_id') + expect(sql).toContain('FROM deepchat_tape_entries AS later_message') + expect(sql).toContain( + "typeof(json_extract(later_message.payload_json, '$.record.content')) = 'text'" + ) + expect(sql).not.toContain('bounded_rows AS') + expect(params[0]).toBe( + JSON.stringify([ + { sessionId: 'child-a', maxEntryId: 10 }, + { sessionId: 'child-b', maxEntryId: 20 } + ]) + ) + expect(params.at(-1)).toBe(5) + }) + + it('reads exact linked projections without invoking FTS recovery or writes', () => { + const run = vi.fn() + const exec = vi.fn() + const reads: Array<{ sql: string; params: unknown[] }> = [] + const prepare = vi.fn((sql: string) => ({ + all: (...params: unknown[]) => { + reads.push({ sql, params }) + if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { + return [{ session_id: 'child', max_entry_id: 2 }] + } + if (sql.includes('SELECT projection.*, NULL AS score')) { + return [ + { + session_id: 'child', + entry_id: 2, + kind: 'event', + name: 'child/result', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'projection needle', + summary_text: 'projection needle', + refs_json: '{}', + created_at: 100, + score: null + } + ] + } + return [] + }, + run + })) + const projectionTable = new DeepChatTapeSearchProjectionTable({ prepare, exec } as any) + + const result = projectionTable.searchSourcesReadOnly( + [{ sessionId: 'child', maxEntryId: 2 }], + 'projection needle', + { limit: 5 } + ) + + expect(result).toMatchObject({ + coveredSources: [{ sessionId: 'child', maxEntryId: 2 }], + rows: [{ session_id: 'child', entry_id: 2 }] + }) + expect(exec).not.toHaveBeenCalled() + expect(run).not.toHaveBeenCalled() + expect(prepare.mock.calls.map(([sql]) => String(sql)).join('\n')).not.toContain( + 'deepchat_tape_search_fts_meta' + ) + expect( + reads.find((read) => read.sql.includes('SELECT projection.*, NULL AS score'))?.params.at(-1) + ).toBe(5) + }) + + it('returns no ranked rows when linked projection coverage is only partial', () => { + const reads: string[] = [] + const prepare = vi.fn((sql: string) => ({ + all: () => { + reads.push(sql) + if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { + return [{ session_id: 'child-a', max_entry_id: 2 }] + } + return [] + } + })) + const projectionTable = new DeepChatTapeSearchProjectionTable({ + prepare, + exec: vi.fn() + } as any) + + const result = projectionTable.searchSourcesReadOnly( + [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 3 } + ], + 'projection needle', + { limit: 5 } + ) + + expect(result).toEqual({ + coveredSources: [{ sessionId: 'child-a', maxEntryId: 2 }], + rows: [] + }) + expect( + reads.some((sql) => sql.includes('FROM deepchat_tape_search_projection AS projection')) + ).toBe(false) + expect(reads.some((sql) => sql.includes('FROM deepchat_tape_search_fts'))).toBe(false) + }) + + it('uses one LIKE ranking when linked FTS freshness is only partial', () => { + const reads: Array<{ sql: string; params: unknown[] }> = [] + const prepare = vi.fn((sql: string) => ({ + all: (...params: unknown[]) => { + reads.push({ sql, params }) + if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { + return [ + { session_id: 'child-a', max_entry_id: 2 }, + { session_id: 'child-b', max_entry_id: 3 } + ] + } + if (sql.includes('deepchat_tape_search_fts_meta AS meta')) { + return [{ session_id: 'child-a', max_entry_id: 2 }] + } + if (sql.includes('SELECT projection.*, NULL AS score')) { + return [ + { + session_id: 'child-b', + entry_id: 3, + kind: 'event', + name: 'child/result', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'projection needle', + summary_text: 'projection needle', + refs_json: '{}', + created_at: 200, + score: null + } + ] + } + return [] + } + })) + const projectionTable = new DeepChatTapeSearchProjectionTable({ + prepare, + exec: vi.fn() + } as any) + ;(projectionTable as any).ftsReady = true + + const sources = [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 3 } + ] + const result = projectionTable.searchSourcesReadOnly(sources, 'projection needle', { limit: 5 }) + + expect(result.rows).toMatchObject([{ session_id: 'child-b', entry_id: 3, score: null }]) + expect(reads.some(({ sql }) => sql.includes('bm25(deepchat_tape_search_fts)'))).toBe(false) + expect( + reads.find(({ sql }) => sql.includes('SELECT projection.*, NULL AS score'))?.params[0] + ).toBe(JSON.stringify(sources)) + }) + + itIfSqlite( + `keeps projected and raw linked multi-term search aligned${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + table.createTable() + projectionTable.createTable() + table.ensureBootstrapAnchor('child') + const entry = table.appendEvent({ + sessionId: 'child', + name: 'child/result', + data: { text: 'alpha separated by several words before beta' }, + createdAt: 100 + }) + const sources = [{ sessionId: 'child', maxEntryId: entry.entry_id }] + projectionTable.replaceSession( + 'child', + [ + { + sessionId: 'child', + entryId: entry.entry_id, + kind: 'event', + name: 'child/result', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'alpha separated by several words before beta', + summaryText: 'alpha separated by several words before beta', + refs: {}, + createdAt: 100 + } + ], + entry.entry_id + ) + + const projected = projectionTable.searchSourcesReadOnly(sources, 'alpha beta', { limit: 5 }) + const raw = table.searchEffectiveSourcesAtHeads(sources, 'alpha beta', { limit: 5 }) + + expect(projected.coveredSources).toEqual(sources) + expect(raw.map((row) => [row.session_id, row.entry_id])).toEqual( + projected.rows.map((row) => [row.session_id, row.entry_id]) + ) + expect(raw).toHaveLength(1) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `queries effective linked sources and context at frozen heads${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + table.ensureBootstrapAnchor('child-a') + table.ensureBootstrapAnchor('child-b') + table.appendEvent({ + sessionId: 'child-a', + name: 'child/result', + data: { text: 'native linked needle A' }, + createdAt: 100 + }) + table.appendEvent({ + sessionId: 'child-b', + name: 'child/result', + data: { text: 'native linked needle B' }, + createdAt: 200 + }) + table.appendEvent({ + sessionId: 'child-a', + name: 'child/late', + data: { text: 'native linked needle late' }, + createdAt: 300 + }) + + const sources = [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 2 } + ] + expect( + table.searchEffectiveSourcesAtHeads(sources, 'native linked needle', { limit: 1 }) + ).toMatchObject([{ session_id: 'child-b', entry_id: 2 }]) + expect( + table + .searchEffectiveSourcesAtHeads(sources, 'native linked needle', { limit: 10 }) + .map((row) => [row.session_id, row.entry_id]) + ).toEqual([ + ['child-b', 2], + ['child-a', 2] + ]) + expect( + table + .getEffectiveContextRowsAtHead({ sessionId: 'child-a', maxEntryId: 2 }, [2], { + before: 1, + after: 5, + limit: 10 + }) + .map((row) => row.entry_id) + ).toEqual([2, 1]) + + table.ensureBootstrapAnchor('child-message') + const original = createRecord({ + id: 'linked-message', + sessionId: 'child-message', + content: JSON.stringify({ text: 'old linked marker', files: [], links: [] }) + }) + const replacement = { + ...original, + content: JSON.stringify({ text: 'new linked marker', files: [], links: [] }), + updatedAt: 200 + } + appendMessageRecordToTape(table, original, 'live') + appendMessageReplacementToTape(table, replacement, 'native_edit') + const replacementHead = table.getMaxEntryId('child-message') + expect( + table.searchEffectiveSourcesAtHeads( + [{ sessionId: 'child-message', maxEntryId: replacementHead }], + 'old linked marker' + ) + ).toEqual([]) + const replacementHits = table.searchEffectiveSourcesAtHeads( + [{ sessionId: 'child-message', maxEntryId: replacementHead }], + 'new linked marker' + ) + expect(replacementHits).toHaveLength(1) + expect( + table + .getEffectiveContextRowsAtHead( + { sessionId: 'child-message', maxEntryId: replacementHead }, + [replacementHits[0].entry_id], + { before: 0, after: 0, limit: 5 } + ) + .map((row) => row.entry_id) + ).toEqual([replacementHits[0].entry_id]) + + table.append({ + sessionId: 'child-message', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: original.id, seq: 0 }, + provenanceKey: null, + payload: { + record: { + id: original.id, + sessionId: original.sessionId, + orderSeq: original.orderSeq, + role: original.role, + status: 'sent' + } + }, + meta: { source: 'malformed_import' } + }) + expect( + table.searchEffectiveSourcesAtHeads( + [ + { + sessionId: 'child-message', + maxEntryId: table.getMaxEntryId('child-message') + } + ], + 'new linked marker' + ) + ).toMatchObject([{ entry_id: replacementHits[0].entry_id }]) + + appendMessageRetractionToTape(table, replacement, 'native_delete') + expect( + table.searchEffectiveSourcesAtHeads( + [ + { + sessionId: 'child-message', + maxEntryId: table.getMaxEntryId('child-message') + } + ], + 'new linked marker' + ) + ).toEqual([]) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `searches exact frozen projections without repairing a later child tail${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + projectionTable.replaceSession( + 'child', + [ + { + sessionId: 'child', + entryId: 2, + kind: 'event', + name: 'child/result', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'native frozen projection needle', + summaryText: 'native frozen projection needle', + refs: {}, + createdAt: 100 + } + ], + 2 + ) + const before = db + .prepare( + `SELECT projection_version, max_entry_id, updated_at + FROM deepchat_tape_search_projection_meta + WHERE session_id = ?` + ) + .get('child') + + const result = projectionTable.searchSourcesReadOnly( + [{ sessionId: 'child', maxEntryId: 2 }], + 'native frozen projection needle', + { limit: 5 } + ) + + expect(result.coveredSources).toEqual([{ sessionId: 'child', maxEntryId: 2 }]) + expect(result.rows).toMatchObject([{ session_id: 'child', entry_id: 2 }]) + expect( + projectionTable.searchSourcesReadOnly( + [{ sessionId: 'child', maxEntryId: 3 }], + 'native frozen projection needle', + { limit: 5 } + ) + ).toEqual({ rows: [], coveredSources: [] }) + expect( + db + .prepare( + `SELECT projection_version, max_entry_id, updated_at + FROM deepchat_tape_search_projection_meta + WHERE session_id = ?` + ) + .get('child') + ).toEqual(before) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `filters stale FTS rows through the base projection after restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'current', + sourceSeq: 0, + searchText: 'current Redis marker', + summaryText: 'current Redis marker', + refs: { messageId: 'current' }, + createdAt: 200 + } + ], + 2 + ) + db.prepare( + `INSERT INTO deepchat_tape_search_fts ( + search_text, + name, + session_id, + entry_id, + kind, + source_type, + source_id, + source_seq, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'stale removed marker', + 'message/user', + 's1', + 1, + 'message', + 'message', + 'old', + 0, + 'stale removed marker', + '{"messageId":"old"}', + 100 + ) + + const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + restartedProjectionTable.createTable() + + expect(restartedProjectionTable.isCurrent('s1', 2)).toBe(true) + expect(restartedProjectionTable.search('s1', 'stale removed marker', { limit: 5 })).toEqual( + [] + ) + expect( + restartedProjectionTable.search('s1', 'current Redis marker', { limit: 5 })[0] + ).toMatchObject({ + entry_id: 2, + refs_json: '{"messageId":"current"}' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `recovers same-entry stale FTS after a base-only projection write and restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm1', + sourceSeq: 0, + searchText: 'old durable marker', + summaryText: 'old durable marker', + refs: { messageId: 'm1' }, + createdAt: 100 + } + ], + 1 + ) + + projectionTable.disableFtsForTesting() + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm1', + sourceSeq: 0, + searchText: 'new durable marker', + summaryText: 'new durable marker', + refs: { messageId: 'm1' }, + createdAt: 100 + } + ], + 1 + ) + db.prepare( + `INSERT INTO deepchat_tape_search_fts ( + search_text, + name, + session_id, + entry_id, + kind, + source_type, + source_id, + source_seq, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'old durable marker', + 'message/user', + 's1', + 1, + 'message', + 'message', + 'm1', + 0, + 'old durable marker', + '{"messageId":"m1"}', + 100 + ) + + const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + restartedProjectionTable.createTable() + + expect(restartedProjectionTable.isCurrent('s1', 1)).toBe(true) + expect(restartedProjectionTable.search('s1', 'old durable marker', { limit: 5 })).toEqual( + [] + ) + expect( + restartedProjectionTable.search('s1', 'new durable marker', { limit: 5 })[0] + ).toMatchObject({ + entry_id: 1, + search_text: 'new durable marker' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `rebuilds FTS during append when previous FTS meta is missing after restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.disableFtsForTesting() + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'old', + sourceSeq: 0, + searchText: 'old append marker', + summaryText: 'old append marker', + refs: { messageId: 'old' }, + createdAt: 100 + } + ], + 1 + ) + + const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + restartedProjectionTable.createTable() + restartedProjectionTable.appendSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'new', + sourceSeq: 0, + searchText: 'new append marker', + summaryText: 'new append marker', + refs: { messageId: 'new' }, + createdAt: 200 + } + ], + 2 + ) + + expect(restartedProjectionTable.isCurrent('s1', 2)).toBe(true) + expect( + restartedProjectionTable.search('s1', 'old append marker', { limit: 1 })[0] + ).toMatchObject({ + entry_id: 1, + refs_json: '{"messageId":"old"}' + }) + expect( + restartedProjectionTable.search('s1', 'new append marker', { limit: 1 })[0] + ).toMatchObject({ + entry_id: 2, + refs_json: '{"messageId":"new"}' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `rebuilds migrated tape FTS when freshness meta is excluded${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'old', + sourceSeq: 0, + searchText: 'old migrated marker', + summaryText: 'old migrated marker', + refs: { messageId: 'old' }, + createdAt: 100 + } + ], + 1 + ) + db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run('s1') + db.prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?').run('s1') + + const migratedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + migratedProjectionTable.createTable() + migratedProjectionTable.appendSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'new', + sourceSeq: 0, + searchText: 'new migrated marker', + summaryText: 'new migrated marker', + refs: { messageId: 'new' }, + createdAt: 200 + } + ], + 2 + ) + + expect(migratedProjectionTable.isCurrent('s1', 2)).toBe(true) + expect( + migratedProjectionTable.search('s1', 'old migrated marker', { limit: 1 })[0] + ).toMatchObject({ + entry_id: 1, + refs_json: '{"messageId":"old"}' + }) + expect( + migratedProjectionTable.search('s1', 'new migrated marker', { limit: 1 })[0] + ).toMatchObject({ + entry_id: 2, + refs_json: '{"messageId":"new"}' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `keeps common-term FTS searches scoped and bounded on large session sets${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + + for (let index = 0; index < 180; index += 1) { + const sessionId = `s-${index}` + const rows = Array.from({ length: 8 }, (_, offset) => ({ + sessionId, + entryId: offset + 1, + kind: 'message' as const, + name: 'message/user', + sourceType: 'message' as const, + sourceId: `m-${index}-${offset}`, + sourceSeq: offset, + searchText: `sharedcommon marker session-${index} row-${offset}`, + summaryText: `sharedcommon marker session-${index} row-${offset}`, + refs: { messageId: `m-${index}-${offset}` }, + createdAt: index * 10 + offset + })) + projectionTable.replaceSession(sessionId, rows, rows.length) + } + + const planRows = db + .prepare( + `EXPLAIN QUERY PLAN + SELECT projection.session_id, + projection.entry_id, + projection.kind, + projection.name, + projection.source_type, + projection.source_id, + projection.source_seq, + projection.search_text, + projection.summary_text, + projection.refs_json, + projection.created_at, + bm25(deepchat_tape_search_fts) AS score + FROM deepchat_tape_search_fts + INNER JOIN deepchat_tape_search_projection AS projection + ON projection.session_id = deepchat_tape_search_fts.session_id + AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) + AND projection.search_text = deepchat_tape_search_fts.search_text + WHERE deepchat_tape_search_fts MATCH ? + AND deepchat_tape_search_fts.session_id = ? + AND projection.session_id = ? + ORDER BY score ASC, projection.entry_id DESC + LIMIT ?` + ) + .all('"sharedcommon"', 's-42', 's-42', 5) as Array<{ detail: string }> + const plan = planRows.map((row) => row.detail).join('\n') + + expect(plan).toMatch(/VIRTUAL TABLE INDEX/i) + expect(plan).toMatch(/SEARCH projection USING (?:COVERING )?INDEX/i) + expect(plan).not.toMatch(/\bSCAN projection\b/i) + + const startedAt = performance.now() + const hits = projectionTable.search('s-42', 'sharedcommon', { limit: 5 }) + const elapsedMs = performance.now() - startedAt + + expect(hits).toHaveLength(5) + expect(hits.every((hit) => hit.session_id === 's-42')).toBe(true) + expect(elapsedMs).toBeLessThan(1500) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `does not trust same-entry stale FTS text even when stale FTS meta is current${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm1', + sourceSeq: 0, + searchText: 'new guarded marker', + summaryText: 'new guarded marker', + refs: { messageId: 'm1' }, + createdAt: 100 + } + ], + 1 + ) + db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run('s1') + db.prepare( + `INSERT INTO deepchat_tape_search_fts ( + search_text, + name, + session_id, + entry_id, + kind, + source_type, + source_id, + source_seq, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'old guarded marker', + 'message/user', + 's1', + 1, + 'message', + 'message', + 'm1', + 0, + 'old guarded marker', + '{"messageId":"m1"}', + 100 + ) + + const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + restartedProjectionTable.createTable() + + expect(restartedProjectionTable.isCurrent('s1', 1)).toBe(true) + expect(restartedProjectionTable.search('s1', 'old guarded marker', { limit: 5 })).toEqual( + [] + ) + expect( + restartedProjectionTable.search('s1', 'new guarded marker', { limit: 5 })[0] + ).toMatchObject({ + entry_id: 1, + search_text: 'new guarded marker' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `does not mark a tape projection current when FTS DML fails${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.dropFtsForTesting() + ;(projectionTable as any).ftsReady = true + + expect(() => + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm1', + sourceSeq: 0, + searchText: 'Redis TTL', + summaryText: 'Redis TTL', + refs: { messageId: 'm1' }, + createdAt: 100 + } + ], + 1 + ) + ).toThrow() + expect(projectionTable.isCurrent('s1', 1)).toBe(false) + expect(projectionTable.getProjectedEntryIds('s1')).toEqual([]) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `filters memory view manifests by message in SQLite before limit${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + data: { ignored: true }, + meta: { messageId: 'msg-old' }, + createdAt: 999 + }) + for (let index = 0; index < 505; index += 1) { + table.appendAnchor({ + sessionId: 's1', + name: 'memory/view_assembled', + state: { + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: index, + selected: [`m-${index}`], + dropped: [], + queryHash: `hash-${index}` + }, + meta: { messageId: `msg-${index}` }, + createdAt: index + }) + } + + const rows = table.listMemoryViewManifestAnchorsBySessions(['s1'], { + limit: 1, + messageId: 'msg-0' + }) + + expect(rows).toHaveLength(1) + expect(rows[0]).toMatchObject({ + kind: 'anchor', + name: 'memory/view_assembled', + created_at: 0 + }) + expect(JSON.parse(rows[0].meta_json)).toEqual({ messageId: 'msg-0' }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `queries memory view manifests for large agents without expanding session parameters${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const sessionTable = new NewSessionsTable(db) + const tapeTable = new DeepChatTapeEntriesTable(db) + sessionTable.createTable() + tapeTable.createTable() + for (let index = 0; index < 1200; index += 1) { + const sessionId = `s-${index}` + db.prepare( + `INSERT INTO new_sessions ( + id, + agent_id, + title, + project_dir, + is_pinned, + is_draft, + active_skills, + disabled_agent_tools, + subagent_enabled, + session_kind, + parent_session_id, + subagent_meta_json, + created_at, + updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + sessionId, + 'agent-a', + `Session ${index}`, + null, + 0, + 0, + '[]', + '[]', + index % 2 === 0 ? 0 : 1, + index % 2 === 0 ? 'regular' : 'subagent', + null, + null, + index, + index + ) + tapeTable.appendAnchor({ + sessionId, + name: 'memory/view_assembled', + state: { + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: index, + selected: [`m-${index}`], + dropped: [], + queryHash: `hash-${index}` + }, + meta: { messageId: `msg-${index}` }, + createdAt: index + }) + } + db.prepare( + `INSERT INTO new_sessions ( + id, + agent_id, + title, + project_dir, + is_pinned, + is_draft, + active_skills, + disabled_agent_tools, + subagent_enabled, + session_kind, + parent_session_id, + subagent_meta_json, + created_at, + updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'other-session', + 'other-agent', + 'Other', + null, + 0, + 0, + '[]', + '[]', + 0, + 'regular', + null, + null, + 9999, + 9999 + ) + tapeTable.appendAnchor({ + sessionId: 'other-session', + name: 'memory/view_assembled', + state: { + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: 9999, + selected: ['other'], + dropped: [], + queryHash: 'other' + }, + meta: { messageId: 'msg-0' }, + createdAt: 9999 + }) + + const rows = tapeTable.listMemoryViewManifestAnchorsByAgent('agent-a', { + messageId: 'msg-0', + limit: 1 + }) + + expect(rows).toHaveLength(1) + expect(rows[0]).toMatchObject({ + session_id: 's-0', + kind: 'anchor', + name: 'memory/view_assembled', + created_at: 0 + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `searches a SQLite tape projection and expands compact context without raw payloads${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + table.createTable() + projectionTable.createTable() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'u1', seq: 0 }, + payload: { + record: createRecord({ + id: 'u1', + content: JSON.stringify({ + text: 'Check Redis TTL with /usr/local/bin/deploy --flag and error 42.', + files: [], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.append({ + sessionId: 's1', + kind: 'tool_result', + name: 'shell', + source: { type: 'tool_result', id: 'u1:tc1', seq: 0 }, + payload: { + messageId: 'u1', + orderSeq: 2, + toolCallId: 'tc1', + exitStatus: 42, + response: 'Exit code 42 in /tmp/deploy.log' + }, + meta: { source: 'live', status: 'error' }, + createdAt: 110 + }) + + const pathHits = service.search('s1', '/usr/local/bin/deploy', { limit: 5 }) + expect(pathHits).toHaveLength(1) + expect(pathHits[0]).toMatchObject({ + kind: 'message', + summary: expect.stringContaining('Redis TTL'), + refs: { + messageId: 'u1', + role: 'user', + filePaths: expect.arrayContaining(['/usr/local/bin/deploy']) + } + }) + expect(pathHits[0]).not.toHaveProperty('payload') + expect(pathHits[0]).not.toHaveProperty('meta') + expect(service.search('s1', 'Redis TTL', { limit: 5 }).map((hit) => hit.entryId)).toContain( + pathHits[0].entryId + ) + const errorHits = service.search('s1', '42', { kinds: ['tool_result'], limit: 5 }) + expect(errorHits[0]).toMatchObject({ + refs: { + toolCallId: 'tc1', + exitStatus: 42 + } + }) + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) + + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'u2', seq: 0 }, + payload: { + record: createRecord({ + id: 'u2', + orderSeq: 3, + content: JSON.stringify({ text: 'zoxide marker 简洁', files: [], links: [] }), + createdAt: 120, + updatedAt: 120 + }) + }, + meta: { source: 'live', orderSeq: 3, role: 'user' }, + createdAt: 120 + }) + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(false) + const rebuiltHits = service.search('s1', '简洁', { limit: 5 }) + expect(rebuiltHits.map((hit) => hit.refs?.messageId)).toContain('u2') + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) + if (projectionTable.hasFtsReadyForTesting()) { + projectionTable.dropFtsForTesting() + ;(projectionTable as any).ftsReady = true + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'u3', seq: 0 }, + payload: { + record: createRecord({ + id: 'u3', + orderSeq: 4, + content: JSON.stringify({ + text: 'fts recovery marker', + files: [], + links: [] + }), + createdAt: 130, + updatedAt: 130 + }) + }, + meta: { source: 'live', orderSeq: 4, role: 'user' }, + createdAt: 130 + }) + const recoveryHits = service.search('s1', 'fts recovery marker', { limit: 5 }) + expect(recoveryHits.map((hit) => hit.refs?.messageId)).toContain('u3') + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(false) + expect(projectionTable.hasFtsReadyForTesting()).toBe(false) + + const restoredHits = service.search('s1', 'fts recovery marker', { limit: 5 }) + expect(restoredHits.map((hit) => hit.refs?.messageId)).toContain('u3') + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) + expect(projectionTable.hasFtsReadyForTesting()).toBe(true) + } + + const context = service.getContext('s1', [pathHits[0].entryId], { + before: 0, + after: 1, + limit: 2, + maxBytesPerEntry: 24, + maxTotalBytes: 24 + }) + expect(context.matchedEntryIds).toEqual([pathHits[0].entryId]) + expect(context.entries[0]).toMatchObject({ + entryId: pathHits[0].entryId, + summary: expect.stringContaining('Redis TTL'), + evidence: { + truncated: true + } + }) + expect(context.entries[0].evidence.bytes).toBeLessThanOrEqual(24) + expect(context.entries[0]).not.toHaveProperty('payload') + expect(context.entries[0]).not.toHaveProperty('meta') + const limitedContext = service.getContext( + 's1', + [pathHits[0].entryId, errorHits[0].entryId], + { + before: 0, + after: 0, + limit: 1 + } + ) + expect(limitedContext.entries.map((entry) => entry.entryId)).toEqual([pathHits[0].entryId]) + expect(limitedContext.matchedEntryIds).toEqual([pathHits[0].entryId]) + } finally { + db.close() + } + } + ) +}) diff --git a/test/main/session/data/tapeReconciler.test.ts b/test/main/session/data/tapeReconciler.test.ts new file mode 100644 index 0000000000..28bf1077fd --- /dev/null +++ b/test/main/session/data/tapeReconciler.test.ts @@ -0,0 +1,399 @@ +import { + describe, + expect, + it, + vi, + buildContext, + toAppSessionId, + SessionTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + createTapeTableMock, + createRecord, + createTapeService +} from './tapeTestHarness' + +describe('SessionTape reconciliation and facts', () => { + it('keeps unkeyed idempotent harness appends distinct like the SQLite store', () => { + const { table, entries } = createTapeTableMock() + const input = { + sessionId: 's1', + kind: 'event', + name: 'unkeyed', + payload: { value: 1 }, + idempotent: true + } + + table.append(input) + table.append(input) + + expect(entries).toHaveLength(2) + expect(entries.map((entry) => entry.entry_id)).toEqual([1, 2]) + }) + + it('backfills message and tool facts idempotently before returning tape records', () => { + const { table, entries } = createTapeTableMock() + const assistantBlocks = [ + { + type: 'tool_call', + status: 'success', + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } + } + ] + const records = [ + createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify(assistantBlocks), + createdAt: 120, + updatedAt: 120 + }), + createRecord({ id: 'u1', orderSeq: 1 }) + ] + const messageStore = { + getMessages: vi.fn().mockReturnValue(records) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const first = service.ensureSessionTapeReady('s1', messageStore as any) + const second = service.ensureSessionTapeReady('s1', messageStore as any) + + expect(first.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) + expect(second.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) + expect(records.map((record) => record.id)).toEqual(['a1', 'u1']) + expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) + expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) + expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) + expect(entries.filter((entry) => entry.name === 'migration/backfill')).toHaveLength(1) + }) + + it('appends live tool facts through the stable recorder port idempotently', async () => { + const { table, entries } = createTapeTableMock() + const service = createTapeService(table) + const input = { + sessionId: toAppSessionId('s1'), + messageId: 'a1', + orderSeq: 2, + blockIndex: 0, + block: { + type: 'tool_call' as const, + status: 'success' as const, + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } + }, + provenance: { source: 'tool_call' as const, sourceId: 'a1:tc1', sequence: 0 } + } + + const first = await service.appendToolFact(input) + const second = await service.appendToolFact(input) + + expect(second).toEqual(first) + expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) + expect(JSON.parse(entries.find((entry) => entry.kind === 'tool_call').meta_json)).toEqual({ + source: 'live', + role: 'assistant', + status: 'success', + reason: 'tool_loop' + }) + }) + + it('keeps legacy context builder output stable after tape backfill projection', () => { + const { table } = createTapeTableMock() + const records = [ + createRecord({ id: 'u1', orderSeq: 1 }), + createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { type: 'content', content: 'Tool finished', status: 'success', timestamp: 120 }, + { + type: 'tool_call', + status: 'success', + timestamp: 121, + tool_call: { + id: 'tc1', + name: 'example_tool', + params: '{"foo":"bar"}', + response: 'All good' + } + } + ]), + createdAt: 120, + updatedAt: 121 + }) + ] + const legacyMessageStore = { + getMessages: vi.fn().mockReturnValue(records) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const legacyContext = buildContext( + 's1', + { text: 'next', files: [] }, + 'System', + 10000, + 4096, + legacyMessageStore as any + ) + const tapeReady = service.ensureSessionTapeReady('s1', legacyMessageStore as any) + const tapeOnlyStore = { + getMessages: vi.fn(() => { + throw new Error('buildContext must use provided tape history records') + }) + } + const tapeContext = buildContext( + 's1', + { text: 'next', files: [] }, + 'System', + 10000, + 4096, + tapeOnlyStore as any, + false, + { + historyRecords: tapeReady.historyRecords + } + ) + + expect(tapeContext).toEqual(legacyContext) + expect(tapeOnlyStore.getMessages).not.toHaveBeenCalled() + }) + + it('rejects handoff anchors without a non-empty summary before writing Tape state', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + expect(() => service.handoff('s1', 'phase_done', { summary: ' ' })).toThrow( + 'Tape handoff requires a non-empty summary.' + ) + expect(() => service.handoff('s1', 'phase_done', { reason: 'phase complete' } as any)).toThrow( + 'Tape handoff requires a non-empty summary.' + ) + + expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() + expect(table.appendAnchor).not.toHaveBeenCalled() + expect(entries).toEqual([]) + }) + + it('migrates legacy session summary into a tape anchor during backfill', () => { + const { table, entries } = createTapeTableMock() + const messageStore = { + getMessages: vi.fn().mockReturnValue([ + createRecord({ id: 'u1', orderSeq: 1 }), + createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([{ type: 'content', content: 'answer', status: 'success' }]) + }) + ]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { + getSummaryState: vi.fn().mockReturnValue({ + summary_text: 'legacy compacted state', + summary_cursor_order_seq: 3, + summary_updated_at: 200 + }) + } + } as any) + + service.ensureSessionTapeReady('s1', messageStore as any) + + const summaryAnchor = entries.find((entry) => entry.name === 'compaction/migrated_summary') + expect(summaryAnchor).toMatchObject({ + kind: 'anchor', + source_type: 'summary', + source_id: 'legacy-summary', + created_at: 200 + }) + expect(JSON.parse(summaryAnchor.payload_json).state).toMatchObject({ + summary: 'legacy compacted state', + cursorOrderSeq: 3, + sourceMessageIds: ['u1', 'a1'] + }) + }) + + it('keeps pending message records for resume but hides pending tool facts from search', () => { + const { table } = createTapeTableMock() + const pendingBlocks = [ + { + type: 'tool_call', + status: 'pending', + timestamp: 100, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'pending result' + } + } + ] + const messageStore = { + getMessages: vi.fn().mockReturnValue([ + createRecord({ + id: 'a1', + orderSeq: 1, + role: 'assistant', + status: 'pending', + content: JSON.stringify(pendingBlocks), + updatedAt: 100 + }) + ]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.ensureSessionTapeReady('s1', messageStore as any) + + expect(service.getMessageRecords('s1')).toMatchObject([{ id: 'a1', status: 'pending' }]) + expect(service.search('s1', 'pending result', { kinds: ['tool_result'] })).toEqual([]) + }) + + it('lets final assistant facts supersede earlier pending tape facts', () => { + const { table, entries } = createTapeTableMock() + const pendingBlocks = [ + { + type: 'tool_call', + status: 'pending', + timestamp: 100, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'pending result' + } + } + ] + const finalBlocks = [ + { + type: 'tool_call', + status: 'success', + timestamp: 200, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'final result' + } + } + ] + const messageStore = { + getMessages: vi + .fn() + .mockReturnValueOnce([ + createRecord({ + id: 'a1', + orderSeq: 1, + role: 'assistant', + status: 'pending', + content: JSON.stringify(pendingBlocks), + metadata: JSON.stringify({ totalTokens: 1 }), + updatedAt: 100 + }) + ]) + .mockReturnValue([ + createRecord({ + id: 'a1', + orderSeq: 1, + role: 'assistant', + status: 'sent', + content: JSON.stringify(finalBlocks), + metadata: JSON.stringify({ totalTokens: 7 }), + updatedAt: 200 + }) + ]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.ensureSessionTapeReady('s1', messageStore as any) + service.ensureSessionTapeReady('s1', messageStore as any) + + expect(service.getMessageRecords('s1')).toMatchObject([ + { + id: 'a1', + status: 'sent' + } + ]) + const effectiveRecord = service.getMessageRecords('s1')[0]! + expect(JSON.parse(effectiveRecord.content)[0].tool_call.response).toBe('final result') + expect( + entries.filter((entry) => entry.kind === 'message' && entry.name === 'message/assistant') + ).toHaveLength(2) + expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) + const finalToolResult = entries.filter((entry) => entry.kind === 'tool_result').at(-1)! + expect(JSON.parse(finalToolResult.payload_json).response).toBe('final result') + expect(service.info('s1').lastTokenUsage).toBe(7) + expect(service.search('s1', 'pending result', { kinds: ['tool_result'] })).toEqual([]) + expect(service.search('s1', 'final result', { kinds: ['tool_result'] })).toHaveLength(1) + }) + + it('uses effective message facts after replacement and retraction events', () => { + const { table, entries } = createTapeTableMock() + const original = createRecord({ id: 'u1', orderSeq: 1 }) + const messageStore = { + getMessages: vi.fn().mockReturnValue([original]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.ensureSessionTapeReady('s1', messageStore as any) + appendMessageReplacementToTape( + table as any, + createRecord({ + id: 'u1', + orderSeq: 1, + content: JSON.stringify({ + text: 'edited', + files: [], + links: [], + search: false, + think: false + }), + updatedAt: 300 + }), + 'test_edit' + ) + + expect(JSON.parse(service.getMessageRecords('s1')[0].content).text).toBe('edited') + expect(service.search('s1', 'hello', { kinds: ['message'] })).toEqual([]) + expect(service.search('s1', 'edited', { kinds: ['message'] })).toHaveLength(1) + expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) + + appendMessageRetractionToTape(table as any, service.getMessageRecords('s1')[0], 'test_delete') + + expect(service.getMessageRecords('s1')).toEqual([]) + expect(service.search('s1', 'edited', { kinds: ['message'] })).toEqual([]) + }) + + it('appends non-idempotent retractions without generated provenance keys', () => { + const { table, entries } = createTapeTableMock() + const record = createRecord({ id: 'u1' }) + + appendMessageRetractionToTape(table as any, record, 'first_delete') + appendMessageRetractionToTape(table as any, record, 'second_delete') + + const retractions = entries.filter((entry) => entry.name === 'message/retracted') + expect(retractions).toHaveLength(2) + expect(retractions.map((entry) => entry.provenance_key)).toEqual([null, null]) + }) +}) diff --git a/test/main/session/data/tapeTestHarness.ts b/test/main/session/data/tapeTestHarness.ts new file mode 100644 index 0000000000..5853059a3b --- /dev/null +++ b/test/main/session/data/tapeTestHarness.ts @@ -0,0 +1,654 @@ +import { performance } from 'node:perf_hooks' +import { describe, expect, it, vi } from 'vitest' +import { buildContext } from '@/agent/deepchat/runtime/contextBuilder' +import { toAppSessionId } from '@/agent/shared/agentSessionIds' +import { SessionTape } from '@/session/data/tape' +import { buildEffectiveTapeView, searchEffectiveTapeRows } from '@/session/data/tapeEffectiveView' +import { + createTapeViewManifest, + type TapeViewManifestBuildInput +} from '@/session/data/tapeViewManifest' +import { + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + appendToolFactsToTape +} from '@/session/data/tapeFacts' +import { buildRequestRefs } from '@/session/data/tapeViewManifest' +import { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' +import { SqliteTapeLifecycleAdapter } from '@/tape/infrastructure/sqlite/tapeLifecycleAdapter' +import { + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, + DeepChatTapeSearchProjectionTable +} from '@/session/data/tables/deepchatTapeSearchProjection' +import { DeepChatMemoryIngestionProjectionTable } from '@/memory/data/tables/deepchatMemoryIngestionProjection' +import { DeepChatMessagesTable } from '@/session/data/tables/deepchatMessages' +import { DeepChatMessageTracesTable } from '@/session/data/tables/deepchatMessageTraces' +import { DeepChatSessionsTable } from '@/session/data/tables/deepchatSessions' +import { NewSessionsTable } from '@/session/data/tables/newSessions' +import type { ChatMessageRecord } from '@shared/types/agent-interface' + +const sqliteModule = await import('better-sqlite3-multiple-ciphers').catch(() => null) +const Database = sqliteModule?.default +const DatabaseCtor = Database! +const sqliteSkipReason = 'skipped: better-sqlite3-multiple-ciphers is unavailable' +const requireNativeSqlite = process.env.DEEPCHAT_REQUIRE_NATIVE_SQLITE === '1' + +let sqliteAvailable = false +if (Database) { + try { + const smokeDb = new Database(':memory:') + smokeDb.close() + sqliteAvailable = true + } catch { + sqliteAvailable = false + } +} + +const itIfSqlite = sqliteAvailable + ? it + : requireNativeSqlite + ? (name: string, _test: () => unknown, timeout?: number) => + it( + name, + () => { + throw new Error(sqliteSkipReason) + }, + timeout + ) + : it.skip + +function createTapeTableMock() { + const entries: any[] = [] + let tapeIncarnationSequence = 0 + const table = { + ensureBootstrapAnchor: vi.fn((sessionId: string) => { + if ( + entries.some((entry) => entry.session_id === sessionId && entry.name === 'session/start') + ) { + return + } + table.appendAnchor({ + sessionId, + name: 'session/start', + source: { type: 'session', id: sessionId, seq: 0 }, + state: { owner: 'human' }, + meta: { tapeIncarnationId: `test-tape-${++tapeIncarnationSequence}` }, + idempotent: true + }) + }), + append: vi.fn((input: any) => { + const provenanceKey = + input.provenanceKey !== undefined + ? input.provenanceKey + : input.source + ? [ + input.source.type, + input.source.id, + input.source.seq ?? 0, + input.kind, + input.name ?? '' + ].join(':') + : null + const existing = + input.idempotent && provenanceKey + ? entries.find( + (entry) => + entry.session_id === input.sessionId && entry.provenance_key === provenanceKey + ) + : null + if (existing) { + return existing + } + const row = { + session_id: input.sessionId, + entry_id: + Math.max( + 0, + ...entries + .filter((entry) => entry.session_id === input.sessionId) + .map((entry) => entry.entry_id) + ) + 1, + kind: input.kind, + name: input.name ?? null, + source_type: input.source?.type ?? null, + source_id: input.source?.id ?? null, + source_seq: input.source?.seq ?? null, + provenance_key: provenanceKey, + payload_json: JSON.stringify(input.payload ?? {}), + meta_json: JSON.stringify(input.meta ?? {}), + created_at: input.createdAt ?? Date.now() + } + entries.push(row) + return row + }), + appendAnchor: vi.fn((input: any) => + table.append({ + ...input, + kind: 'anchor', + payload: { name: input.name, state: input.state } + }) + ), + appendEvent: vi.fn((input: any) => + table.append({ + ...input, + kind: 'event', + payload: { name: input.name, data: input.data } + }) + ), + runInTransaction: vi.fn((operation: () => unknown) => { + const snapshot = entries.map((entry) => ({ ...entry })) + try { + return operation() + } catch (error) { + entries.splice(0, entries.length, ...snapshot) + throw error + } + }), + getBySession: vi.fn((sessionId: string) => + entries.filter((entry) => entry.session_id === sessionId) + ), + listMemoryViewManifestAnchorsByAgent: vi.fn( + ( + _agentId: string, + _options?: { sessionId?: string; limit?: number; messageId?: string } + ): any[] => { + throw new Error('configure listMemoryViewManifestAnchorsByAgent for this test') + } + ), + getSubagentLineageEvents: vi.fn((sessionId: string) => + entries.filter( + (entry) => + entry.session_id === sessionId && + entry.kind === 'event' && + (entry.name === 'subagent/tape_linked' || entry.name === 'fork/merge') + ) + ), + getFirstEntriesBySessions: vi.fn((sessionIds: string[]) => + [...new Set(sessionIds)] + .flatMap((sessionId) => { + const first = entries + .filter((entry) => entry.session_id === sessionId) + .sort((left, right) => left.entry_id - right.entry_id)[0] + return first ? [first] : [] + }) + .sort((left, right) => left.session_id.localeCompare(right.session_id)) + ), + getBySessionUpToEntryId: vi.fn((sessionId: string, maxEntryId: number) => + entries.filter((entry) => entry.session_id === sessionId && entry.entry_id <= maxEntryId) + ), + getMaxEntryId: vi.fn((sessionId: string) => + Math.max( + 0, + ...entries.filter((entry) => entry.session_id === sessionId).map((entry) => entry.entry_id) + ) + ), + getMaxEntryIdsBySessions: vi.fn( + (sessionIds: string[]) => + new Map( + sessionIds.map((sessionId) => [ + sessionId, + Math.max( + 0, + ...entries + .filter((entry) => entry.session_id === sessionId) + .map((entry) => entry.entry_id) + ) + ]) + ) + ), + getLatestAnchor: vi.fn( + (sessionId: string) => + entries + .filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor') + .sort((left, right) => right.entry_id - left.entry_id)[0] + ), + getAnchors: vi.fn((sessionId: string, limit: number = 20) => + entries + .filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor') + .sort((left, right) => right.entry_id - left.entry_id) + .slice(0, Math.min(Math.max(Math.floor(limit), 1), 100)) + .reverse() + ), + getLatestSummaryAnchor: vi.fn( + (sessionId: string) => + entries + .filter( + (entry) => + entry.session_id === sessionId && + entry.kind === 'anchor' && + ['compaction/migrated_summary', 'compaction/manual', 'summary/reset'].includes( + entry.name + ) + ) + .sort((left, right) => right.entry_id - left.entry_id)[0] + ), + getByProvenanceKey: vi.fn((sessionId: string, provenanceKey: string) => + entries.find( + (entry) => entry.session_id === sessionId && entry.provenance_key === provenanceKey + ) + ), + countBySession: vi.fn( + (sessionId: string) => entries.filter((entry) => entry.session_id === sessionId).length + ), + countAnchorsBySession: vi.fn( + (sessionId: string) => + entries.filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor').length + ), + countEntriesAfter: vi.fn( + (sessionId: string, entryId: number) => + entries.filter((entry) => entry.session_id === sessionId && entry.entry_id > entryId).length + ), + search: vi.fn((sessionId: string, query: string, options: any = {}) => { + const normalizedQuery = query.trim() + if (!normalizedQuery) { + return [] + } + const limit = Number.isFinite(options.limit) ? Math.floor(options.limit) : 20 + return entries + .filter((entry) => entry.session_id === sessionId) + .filter( + (entry) => + entry.payload_json.includes(normalizedQuery) || + entry.meta_json.includes(normalizedQuery) || + entry.name?.includes(normalizedQuery) + ) + .filter((entry) => !options.kinds?.length || options.kinds.includes(entry.kind)) + .filter( + (entry) => + !Number.isFinite(options.startCreatedAt) || entry.created_at >= options.startCreatedAt + ) + .filter( + (entry) => + !Number.isFinite(options.endCreatedAt) || entry.created_at <= options.endCreatedAt + ) + .sort((left, right) => right.entry_id - left.entry_id) + .slice(0, Math.min(Math.max(limit, 1), 100)) + }), + searchEffectiveSourcesAtHeads: vi.fn((sources: any[], query: string, options: any = {}) => + sources + .flatMap((source) => + searchEffectiveTapeRows( + entries.filter( + (entry) => + entry.session_id === source.sessionId && entry.entry_id <= source.maxEntryId + ), + query, + { ...options, limit: 100 } + ) + ) + .sort( + (left, right) => + right.created_at - left.created_at || + left.session_id.localeCompare(right.session_id) || + right.entry_id - left.entry_id + ) + .slice(0, Math.min(Math.max(Math.floor(options.limit ?? 20), 1), 100)) + ), + getEffectiveContextRowsAtHead: vi.fn( + ( + source: any, + entryIds: number[], + options: { before: number; after: number; limit: number } + ) => { + const effectiveRows = buildEffectiveTapeView( + entries.filter( + (entry) => entry.session_id === source.sessionId && entry.entry_id <= source.maxEntryId + ), + { includePending: false } + ).rows + const indexesByEntryId = new Map( + effectiveRows.map((entry, index) => [entry.entry_id, index]) + ) + const indexes: number[] = [] + for (const entryId of entryIds) { + const index = indexesByEntryId.get(entryId) + if (index !== undefined) indexes.push(index) + } + for (const entryId of entryIds) { + const index = indexesByEntryId.get(entryId) + if (index === undefined) continue + for ( + let cursor = Math.max(0, index - options.before); + cursor <= Math.min(effectiveRows.length - 1, index + options.after); + cursor += 1 + ) { + if (cursor !== index) indexes.push(cursor) + } + } + return [...new Set(indexes)].slice(0, options.limit).map((index) => effectiveRows[index]) + } + ), + deleteBySession: vi.fn((sessionId: string) => { + for (let index = entries.length - 1; index >= 0; index -= 1) { + if (entries[index].session_id === sessionId) { + entries.splice(index, 1) + } + } + }) + } + return { table, entries } +} + +function createRecord(overrides: Partial): ChatMessageRecord { + return { + id: 'm1', + sessionId: 's1', + orderSeq: 1, + role: 'user', + content: JSON.stringify({ text: 'hello', files: [], links: [], search: false, think: false }), + status: 'sent', + isContextEdge: 0, + metadata: '{}', + traceCount: 0, + createdAt: 100, + updatedAt: 100, + ...overrides + } +} + +function createTraceRow(overrides: Record = {}) { + return { + id: 'trace-1', + message_id: 'a1', + session_id: 's1', + provider_id: 'openai', + model_id: 'gpt-4o', + request_seq: 1, + endpoint: 'https://api.openai.test/v1/chat/completions', + headers_json: '{"authorization":"[redacted]"}', + body_json: '{"messages":[{"role":"user","content":"hello"}]}', + truncated: 0, + created_at: 300, + ...overrides + } +} + +function createMessageRow(overrides: Record = {}) { + return { + id: 'a1', + session_id: 's1', + order_seq: 2, + role: 'assistant', + content: '[{"type":"content","content":"done","status":"success"}]', + status: 'sent', + is_context_edge: 0, + metadata: '{"totalTokens":10}', + created_at: 200, + updated_at: 300, + ...overrides + } +} + +function createObservationManifest( + overrides: Partial[0]> = {} +) { + return createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user', content: 'hello' }], + tools: [], + latestEntryId: 0, + anchorEntryIds: [], + included: [], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: true, + assembledAt: 200, + ...overrides + }) +} + +function createTapeService( + table: unknown, + traceRows: Array> = [], + messageRows: Array> = [] +) { + return new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: { + deleteBySession: vi.fn(), + isCurrent: vi.fn(() => { + throw new Error('projection unavailable') + }), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) + }, + deepchatMessageTracesTable: { + listByMessageId: vi.fn((messageId: string) => + traceRows.filter((row) => row.message_id === messageId) + ) + }, + deepchatMessagesTable: { + get: vi.fn((messageId: string) => messageRows.find((row) => row.id === messageId)) + }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) +} + +function createLinkedTapeService( + table: unknown, + sessions: Array<{ + id: string + session_kind: 'regular' | 'subagent' + parent_session_id: string | null + }>, + projectionTable?: unknown +) { + const sessionById = new Map(sessions.map((session) => [session.id, session])) + return { + service: new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: projectionTable, + newSessionsTable: { + get: vi.fn((sessionId: string) => sessionById.get(sessionId)), + getMany: vi.fn((sessionIds: string[]) => + sessionIds.flatMap((sessionId) => { + const session = sessionById.get(sessionId) + return session ? [session] : [] + }) + ) + }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any), + sessionById + } +} + +function createSubagentLinkInput(parentSessionId: string, childSessionId: string) { + return { + parentSessionId, + childSessionId, + runId: `run-${childSessionId}`, + taskId: `task-${childSessionId}`, + slotId: 'reviewer', + taskTitle: `Review ${childSessionId}`, + outcome: 'completed' as const, + resultSummary: 'Done' + } +} + +function appendObservationIsolationFacts(table: unknown) { + const original = createRecord({ id: 'u1', orderSeq: 1, createdAt: 100, updatedAt: 100 }) + const edited = createRecord({ + id: 'u1', + orderSeq: 1, + content: JSON.stringify({ + text: 'edited', + files: [], + links: [], + search: false, + think: false + }), + createdAt: 100, + updatedAt: 150 + }) + const retracted = createRecord({ id: 'u2', orderSeq: 2, createdAt: 160, updatedAt: 160 }) + const pending = createRecord({ + id: 'a1', + orderSeq: 3, + role: 'assistant', + status: 'pending', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'pending', + timestamp: 200, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}' } + } + ]), + createdAt: 200, + updatedAt: 200 + }) + const final = createRecord({ + id: 'a1', + orderSeq: 3, + role: 'assistant', + status: 'sent', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 300, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'tape-result-secret' + } + } + ]), + metadata: '{"totalTokens":12}', + createdAt: 200, + updatedAt: 300 + }) + + appendMessageRecordToTape(table as any, original, 'live') + appendMessageReplacementToTape(table as any, edited, 'test_edit') + appendMessageRecordToTape(table as any, retracted, 'live') + appendMessageRetractionToTape(table as any, retracted, 'test_delete') + appendMessageRecordToTape(table as any, pending, 'live') + appendMessageRecordToTape(table as any, final, 'live') + + return { edited, final } +} + +function stripObservationPayloadOptIns(value: T): T { + const copy = structuredClone(value) as any + const stripEntryPayloads = (entries: any[] | undefined) => { + for (const entry of entries ?? []) { + delete entry.payload + delete entry.meta + } + } + + if (copy.request?.state === 'manifest_bound') { + stripEntryPayloads(copy.request.replay.entries) + delete copy.request.replay.hashes.sliceHash + if (copy.request.replay.trace) { + delete copy.request.replay.trace.headersJson + delete copy.request.replay.trace.bodyJson + } + } else if (copy.request?.trace) { + delete copy.request.trace.headersJson + delete copy.request.trace.bodyJson + } + stripEntryPayloads(copy.output?.entries) + return copy +} + +function createSpies(names: string[]) { + return Object.fromEntries(names.map((name) => [name, vi.fn()])) as Record< + string, + ReturnType + > +} + +function trackMemoryPropertyAccess(target: T) { + const memoryPropertyAccess = vi.fn() + return { + memoryPropertyAccess, + presenter: new Proxy(target, { + get(value, property, receiver) { + if (typeof property === 'string' && /memory/i.test(property)) { + memoryPropertyAccess(property) + } + return Reflect.get(value, property, receiver) + } + }) + } +} + +function readObservationMatrix(service: SessionTape) { + return { + defaultObservation: service.readCausalObservationSlice('s1', 'a1'), + repeatedObservation: service.readCausalObservationSlice('s1', 'a1'), + explicitObservation: service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }), + optInObservation: service.readCausalObservationSlice('s1', 'a1', { + includeTapePayloads: true, + includeTracePayload: true + }), + traceOnlyObservation: service.readCausalObservationSlice('s1', 'a-trace') + } +} + +export { + performance, + describe, + expect, + it, + vi, + buildContext, + toAppSessionId, + SessionTape, + buildEffectiveTapeView, + searchEffectiveTapeRows, + createTapeViewManifest, + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + appendToolFactsToTape, + buildRequestRefs, + DeepChatTapeEntriesTable, + SqliteTapeLifecycleAdapter, + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, + DeepChatTapeSearchProjectionTable, + DeepChatMemoryIngestionProjectionTable, + DeepChatMessagesTable, + DeepChatMessageTracesTable, + DeepChatSessionsTable, + NewSessionsTable, + DatabaseCtor, + sqliteAvailable, + sqliteSkipReason, + itIfSqlite, + createTapeTableMock, + createRecord, + createTraceRow, + createMessageRow, + createObservationManifest, + createTapeService, + createLinkedTapeService, + createSubagentLinkInput, + appendObservationIsolationFacts, + stripObservationPayloadOptIns, + createSpies, + trackMemoryPropertyAccess, + readObservationMatrix +} diff --git a/test/main/session/data/tapeViewReplay.test.ts b/test/main/session/data/tapeViewReplay.test.ts new file mode 100644 index 0000000000..1af5777a55 --- /dev/null +++ b/test/main/session/data/tapeViewReplay.test.ts @@ -0,0 +1,1578 @@ +import type { TapeViewManifestBuildInput } from '@/tape/domain/viewManifest' +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import { + describe, + expect, + it, + vi, + SessionTape, + createTapeViewManifest, + appendMessageRecordToTape, + appendToolFactsToTape, + buildRequestRefs, + DeepChatTapeEntriesTable, + DeepChatMemoryIngestionProjectionTable, + DeepChatMessagesTable, + DeepChatMessageTracesTable, + DeepChatSessionsTable, + DatabaseCtor, + sqliteAvailable, + sqliteSkipReason, + itIfSqlite, + createTapeTableMock, + createRecord, + createTraceRow, + createMessageRow, + createObservationManifest, + createTapeService, + appendObservationIsolationFacts, + stripObservationPayloadOptIns, + createSpies, + trackMemoryPropertyAccess, + readObservationMatrix +} from './tapeTestHarness' + +describe('SessionTape view and replay', () => { + it('stores and lists view manifests as idempotent tape events', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user', + source: 'tape', + reason: 'selected_history' + } + ], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + + const first = service.appendViewManifest(manifest) + const second = service.appendViewManifest(manifest) + + expect(second.entry_id).toBe(first.entry_id) + expect(entries.filter((entry) => entry.name === 'view/assembled')).toHaveLength(1) + expect(JSON.parse(first.meta_json)).toMatchObject({ + policy: 'legacy_context_v1', + policyVersion: 1 + }) + expect(service.listViewManifestsByMessage('s1', 'a1')).toMatchObject([ + { + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + entryId: first.entry_id, + manifest: { + hashes: { + manifestHash: manifest.hashes.manifestHash + }, + policy: 'legacy_context_v1', + policyVersion: 1, + included: [ + { + messageId: 'u1', + entryId: sourceMaps.entryIdByMessageId.get('u1') + } + ] + } + } + ]) + }) + + it('projects memory view anchors into inspection DTOs', () => { + const { table, entries } = createTapeTableMock() + table.listMemoryViewManifestAnchorsByAgent = vi.fn( + (_agentId: string, options?: { messageId?: string }) => + entries.filter( + (entry) => + entry.kind === 'anchor' && + entry.name === 'memory/view_assembled' && + (!options?.messageId || JSON.parse(entry.meta_json).messageId === options.messageId) + ) + ) + const service = new SessionTape({ deepchatTapeEntriesTable: table } as any) + table.appendAnchor({ + sessionId: 's1', + name: 'memory/view_assembled', + state: { + policyVersion: 2, + tokenBudget: 1000, + estimatedTokens: 15, + selected: [ + 'm-string', + { id: 'm-object' }, + 'm-string', + { id: 'm-object' }, + { ignored: true }, + 3 + ], + dropped: ['m-dropped'], + queryHash: 'query-hash' + }, + meta: { messageId: 'msg-1' }, + createdAt: 300 + }) + + const manifests = service.listMemoryViewManifestsByAgent('agent-1', { + sessionId: 's1', + messageId: 'msg-1', + limit: 1 + }) + + expect(table.listMemoryViewManifestAnchorsByAgent).toHaveBeenCalledWith('agent-1', { + sessionId: 's1', + messageId: 'msg-1', + limit: 1 + }) + expect(manifests).toEqual([ + { + sessionId: 's1', + messageId: 'msg-1', + entryId: 1, + policyVersion: 2, + tokenBudget: 1000, + estimatedTokens: 15, + selectedCount: 6, + selectedIds: ['m-string', 'm-object'], + droppedCount: 1, + queryHash: 'query-hash', + createdAt: 300 + } + ]) + expect(manifests[0]).not.toHaveProperty('payload_json') + expect(manifests[0]).not.toHaveProperty('meta_json') + }) + + it('indexes effective tool facts so tool-loop manifests reference real entries', () => { + const { table } = createTapeTableMock() + const assistantRecord = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } + } + ]) + }) + appendToolFactsToTape(table as any, assistantRecord, 'live', 'tool_loop') + + const service = createTapeService(table) + const sourceMaps = service.getViewManifestSourceMaps('s1') + expect(sourceMaps.toolCallEntryIdByToolId.get('tc1')).toBeGreaterThan(0) + expect(sourceMaps.toolResultEntryIdByToolId.get('tc1')).toBeGreaterThan(0) + + const refs = buildRequestRefs( + [ + { role: 'system', content: 'system' }, + { + role: 'assistant', + content: '', + tool_calls: [ + { id: 'tc1', type: 'function', function: { name: 'search', arguments: '{"q":"x"}' } } + ] + }, + { role: 'tool', content: 'result', tool_call_id: 'tc1' } + ], + sourceMaps + ) + expect(refs).toMatchObject([ + { role: 'system', source: 'synthetic' }, + { + role: 'assistant', + source: 'tape', + reason: 'tool_loop_message', + entryId: sourceMaps.toolCallEntryIdByToolId.get('tc1') + }, + { + role: 'tool', + source: 'tape', + reason: 'tool_loop_message', + entryId: sourceMaps.toolResultEntryIdByToolId.get('tc1') + } + ]) + }) + + it('scopes tool source maps to the in-flight message so reused tool ids do not collide', () => { + const { table } = createTapeTableMock() + const blocks = (response: string) => + JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response } + } + ]) + appendToolFactsToTape( + table as any, + createRecord({ id: 'a1', orderSeq: 2, role: 'assistant', content: blocks('first') }), + 'live', + 'tool_loop' + ) + appendToolFactsToTape( + table as any, + createRecord({ id: 'a2', orderSeq: 4, role: 'assistant', content: blocks('second') }), + 'live', + 'tool_loop' + ) + + const service = createTapeService(table) + const scopedToA1 = service.getViewManifestSourceMaps('s1', 'a1') + const scopedToA2 = service.getViewManifestSourceMaps('s1', 'a2') + + expect(scopedToA1.toolCallEntryIdByToolId.get('tc1')).toBeLessThan( + scopedToA2.toolCallEntryIdByToolId.get('tc1')! + ) + expect(scopedToA1.toolResultEntryIdByToolId.get('tc1')).not.toBe( + scopedToA2.toolResultEntryIdByToolId.get('tc1') + ) + }) + + it('exports tool_call and tool_result entries in a tool-loop replay slice', () => { + const { table } = createTapeTableMock() + const assistantRecord = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } + } + ]) + }) + appendToolFactsToTape(table as any, assistantRecord, 'live', 'tool_loop') + + const service = createTapeService(table) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const messages = [ + { role: 'system' as const, content: 'system' }, + { + role: 'assistant' as const, + content: '', + tool_calls: [ + { id: 'tc1', type: 'function' as const, function: { name: 'search', arguments: '{}' } } + ] + }, + { role: 'tool' as const, content: 'result', tool_call_id: 'tc1' } + ] + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: 1, + messages, + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: buildRequestRefs(messages, sourceMaps), + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + service.appendViewManifest(manifest) + + const slice = service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }) + const kinds = slice?.entries.map((entry) => entry.kind) ?? [] + expect(kinds).toContain('tool_call') + expect(kinds).toContain('tool_result') + }) + + it('filters malformed view manifest rows when listing by message', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { + type: 'runtime_event', + id: 'a1', + seq: 1 + }, + data: { + manifest: { + schemaVersion: 1, + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + included: 'not-an-array' + } + } + }) + + expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) + }) + + it('rejects view manifests that disagree with their persisted source envelope', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 1 }, + data: { manifest: createObservationManifest({ messageId: 'other', requestSeq: 1 }) } + }) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 2 }, + data: { manifest: createObservationManifest({ messageId: 'a1', requestSeq: 1 }) } + }) + + expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) + }) + + it('normalizes legacy manifests without hashVersion to hashVersion 1', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + const legacyManifest: Record = { ...manifest } + delete legacyManifest.hashVersion + + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 1 }, + data: { manifest: legacyManifest } + }) + + const [record] = service.listViewManifestsByMessage('s1', 'a1') + expect(record.manifest.hashVersion).toBe(1) + }) + + it('filters manifests whose hashVersion is not a number', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 99 }, + data: { manifest: { ...manifest, hashVersion: '2' } } + }) + + expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) + }) + + it('annotates read records with hash integrity without dropping tampered manifests', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const baseInput = { + sessionId: 's1', + taskType: 'chat' as const, + policy: 'legacy_context_v1' as const, + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + } + const validManifest = createTapeViewManifest({ ...baseInput, messageId: 'a1', requestSeq: 1 }) + service.appendViewManifest(validManifest) + + const tamperedManifest = createTapeViewManifest({ + ...baseInput, + messageId: 'a2', + requestSeq: 1 + }) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a2', seq: 1 }, + data: { manifest: { ...tamperedManifest, latestEntryId: tamperedManifest.latestEntryId + 1 } } + }) + + const [validRecord] = service.listViewManifestsByMessage('s1', 'a1') + const [tamperedRecord] = service.listViewManifestsByMessage('s1', 'a2') + expect(validRecord.integrity).toBe('valid') + expect(tamperedRecord).toBeDefined() + expect(tamperedRecord.integrity).toBe('invalid') + }) + + it('binds reconstruction lineage to the latest reconstruction anchor including handoffs', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('s1') + table.appendAnchor({ + sessionId: 's1', + name: 'compaction/manual', + source: { type: 'summary', id: 's1', seq: 1 }, + state: {} + }) + table.appendAnchor({ + sessionId: 's1', + name: 'handoff/phase_done', + source: { type: 'handoff', id: 's1', seq: 2 }, + state: {} + }) + table.appendAnchor({ + sessionId: 's1', + name: 'fork/merge', + source: { type: 'fork', id: 'child', seq: 3 }, + state: {} + }) + + const sourceMaps = service.getViewManifestSourceMaps('s1') + const entryIdByName = (name: string) => + table.getBySession('s1').find((entry: any) => entry.name === name)?.entry_id + + expect(sourceMaps.anchorEntryIds).toHaveLength(4) + expect(sourceMaps.reconstructionAnchorEntryId).toBe(entryIdByName('handoff/phase_done')) + expect(sourceMaps.reconstructionAnchorEntryIds).toEqual([ + sourceMaps.reconstructionAnchorEntryId + ]) + expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain( + entryIdByName('compaction/manual') + ) + expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('fork/merge')) + }) + + it('keeps memory anchors off the reconstruction lineage', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('s1') + table.appendAnchor({ + sessionId: 's1', + name: 'compaction/manual', + source: { type: 'summary', id: 's1', seq: 1 }, + state: {} + }) + table.appendAnchor({ + sessionId: 's1', + name: 'memory/extract', + source: { type: 'runtime_event', id: 's1', seq: 2 }, + state: { memoryIds: ['m1'], count: 1, reason: 'episodic' } + }) + table.appendAnchor({ + sessionId: 's1', + name: 'memory/reflect', + source: { type: 'runtime_event', id: 's1', seq: 3 }, + state: { reflectionIds: ['r1'], sourceMemoryIds: ['m1'], count: 1 } + }) + + const sourceMaps = service.getViewManifestSourceMaps('s1') + const entryIdByName = (name: string) => + table.getBySession('s1').find((entry: any) => entry.name === name)?.entry_id + + // Memory anchors are recorded on the tape for observability... + expect(sourceMaps.anchorEntryIds).toContain(entryIdByName('memory/extract')) + expect(sourceMaps.anchorEntryIds).toContain(entryIdByName('memory/reflect')) + // ...but never own the reconstruction cursor; only the summary anchor does. + expect(sourceMaps.reconstructionAnchorEntryId).toBe(entryIdByName('compaction/manual')) + expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('memory/extract')) + expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('memory/reflect')) + }) + + it('bounds replay slices to the selected view instead of pre-cursor history', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.reconstructionAnchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user', + source: 'tape', + reason: 'selected_history' + } + ], + excluded: [], + summaryCursor: { + summaryCursorOrderSeq: 100, + preCursorOrderSeqMin: 1, + preCursorOrderSeqMax: 99, + preCursorCount: 99 + }, + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 100, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + service.appendViewManifest(manifest) + + const slice = service.exportReplaySlice('s1', 'a1') + + expect(slice?.refs.excludedEntryIds).toEqual([]) + expect(slice?.refs.anchorEntryIds).toEqual(sourceMaps.reconstructionAnchorEntryIds) + expect(slice?.refs.anchorEntryIds).toHaveLength(1) + expect(slice?.manifestRecord.manifest.excludedRanges).toEqual([ + { fromOrderSeq: 1, toOrderSeq: 99, count: 99, reason: 'before_summary_cursor' } + ]) + expect(slice?.entries).toHaveLength(3) + }) + + it('exports replay slices with metadata-only payloads by default', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [createTraceRow()]) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user', + source: 'tape', + reason: 'selected_history' + } + ], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: true, + assembledAt: 200 + }) + const manifestEntry = service.appendViewManifest(manifest) + + const nowSpy = vi.spyOn(Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000) + const slice = service.exportReplaySlice('s1', 'a1') + const secondSlice = service.exportReplaySlice('s1', 'a1') + nowSpy.mockRestore() + + expect(slice).toMatchObject({ + schemaVersion: 1, + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + mode: 'trace_bound', + refs: { + manifestEntryId: manifestEntry.entry_id, + includedEntryIds: [sourceMaps.entryIdByMessageId.get('u1')], + anchorEntryIds: sourceMaps.anchorEntryIds + }, + hashes: { + manifestHash: manifest.hashes.manifestHash + } + }) + expect(slice?.hashes.sliceHash).toHaveLength(64) + expect(secondSlice?.hashes.sliceHash).toBe(slice?.hashes.sliceHash) + expect(secondSlice?.createdAt).toBe(2000) + expect(slice?.trace?.bodyHash).toHaveLength(64) + expect(slice?.trace?.bodyJson).toBeUndefined() + expect(slice?.entries.some((entry) => entry.entryId === manifestEntry.entry_id)).toBe(true) + expect( + slice?.entries.every((entry) => entry.payload === undefined && entry.meta === undefined) + ).toBe(true) + }) + + it('exports explicit replay request sequences with opt-in payloads', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [ + createTraceRow({ id: 'trace-1', request_seq: 1 }), + createTraceRow({ + id: 'trace-2', + request_seq: 2, + body_json: '{"messages":[{"role":"tool","content":"done"}]}' + }) + ]) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const baseManifestInput: TapeViewManifestBuildInput = { + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat' as const, + policy: 'legacy_context_v1' as const, + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user', + source: 'tape', + reason: 'selected_history' + } + ], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: true, + assembledAt: 200 + } + const firstManifest = createTapeViewManifest(baseManifestInput) + const secondManifest = createTapeViewManifest({ + ...baseManifestInput, + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 250 + }) + service.appendViewManifest(firstManifest) + service.appendViewManifest(secondManifest) + + const latest = service.exportReplaySlice('s1', 'a1') + const first = service.exportReplaySlice('s1', 'a1', { + requestSeq: 1, + includeTapePayloads: true, + includeTracePayload: true + }) + + expect(latest?.requestSeq).toBe(2) + expect(first?.requestSeq).toBe(1) + expect(first?.trace?.bodyJson).toContain('"hello"') + expect(first?.entries.some((entry) => entry.payload?.record)).toBe(true) + expect(first?.entries.some((entry) => entry.meta?.source === 'backfill')).toBe(true) + }) + + it('binds each replay slice to its own request seq, ignoring sentinel gap traces', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [ + createTraceRow({ + id: 'trace-req-1', + request_seq: 1, + body_json: '{"messages":[{"role":"user","content":"first-request"}]}' + }), + createTraceRow({ + id: 'trace-gap', + request_seq: 0, + endpoint: 'deepchat://interleaved-reasoning-gap', + body_json: '{"providerId":"openai"}' + }), + createTraceRow({ + id: 'trace-req-2', + request_seq: 2, + body_json: '{"messages":[{"role":"tool","content":"second-request"}]}' + }) + ]) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const baseManifestInput = { + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat' as const, + policy: 'legacy_context_v1' as const, + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user' as const, + source: 'tape' as const, + reason: 'selected_history' as const + } + ], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: true, + assembledAt: 200 + } + service.appendViewManifest(createTapeViewManifest(baseManifestInput)) + service.appendViewManifest( + createTapeViewManifest({ + ...baseManifestInput, + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 250 + }) + ) + + const first = service.exportReplaySlice('s1', 'a1', { + requestSeq: 1, + includeTracePayload: true + }) + const second = service.exportReplaySlice('s1', 'a1', { + requestSeq: 2, + includeTracePayload: true + }) + + expect(first?.trace?.bodyJson).toContain('first-request') + expect(second?.trace?.bodyJson).toContain('second-request') + }) + + it('returns null when exporting a replay slice without a manifest', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [createTraceRow()]) + + expect(service.exportReplaySlice('s1', 'a1')).toBeNull() + }) + + it('rejects non-positive replay request sequences', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [createTraceRow()]) + + expect(() => service.exportReplaySlice('s1', 'a1', { requestSeq: 0 })).toThrow( + 'requestSeq must be a positive integer.' + ) + }) + + it('joins an exact manifest and trace into a causal observation slice', () => { + const { table } = createTapeTableMock() + const assistant = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: '[{"type":"content","content":"done","status":"success"}]', + status: 'sent', + metadata: '{"totalTokens":10}', + createdAt: 200, + updatedAt: 300 + }) + appendMessageRecordToTape(table as any, assistant, 'live') + const service = createTapeService( + table, + [createTraceRow(), createTraceRow({ id: 'other-session', session_id: 's2', request_seq: 9 })], + [createMessageRow()] + ) + service.appendViewManifest(createObservationManifest()) + + const slice = service.readCausalObservationSlice('s1', 'a1', { + currentRuntimeStatus: 'idle' + }) + + expect(slice).toMatchObject({ + schemaVersion: 1, + sessionId: 's1', + messageId: 'a1', + request: { + state: 'manifest_bound', + requestSeq: 1, + replay: { + requestSeq: 1, + mode: 'trace_bound', + trace: { id: 'trace-1', requestSeq: 1 } + } + }, + output: { + correlation: 'message_only', + terminalMessage: { + status: 'sent', + orderSeq: 2, + createdAt: 200, + updatedAt: 300 + } + }, + runtime: { scope: 'current_only', status: 'idle', eventHistory: 'not_persisted' } + }) + expect(slice.output.entries.map((entry) => entry.kind)).toContain('message') + expect(slice.output.terminalMessage?.contentHash).toHaveLength(64) + expect(slice.output.terminalMessage?.metadataHash).toHaveLength(64) + }) + + it('prefers an explicit causal request sequence and otherwise selects the latest raw sequence', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 100 })) + service.appendViewManifest( + createObservationManifest({ + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 200 + }) + ) + + expect(service.readCausalObservationSlice('s1', 'a1').request).toMatchObject({ + state: 'manifest_bound', + requestSeq: 2 + }) + expect(service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }).request).toMatchObject( + { state: 'manifest_bound', requestSeq: 1 } + ) + }) + + it('does not fall back to an older manifest when a later traced request has no manifest', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [ + createTraceRow({ id: 'trace-1', request_seq: 1 }), + createTraceRow({ id: 'trace-2', request_seq: 2 }) + ]) + service.appendViewManifest(createObservationManifest({ requestSeq: 1 })) + + expect(service.readCausalObservationSlice('s1', 'a1').request).toMatchObject({ + state: 'manifest_missing', + requestSeq: 2, + trace: { id: 'trace-2', requestSeq: 2 } + }) + }) + + it('returns a trace-only request without manufacturing a view manifest', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [createTraceRow({ request_seq: 4 })]) + + const request = service.readCausalObservationSlice('s1', 'a1').request + + expect(request).toMatchObject({ + state: 'manifest_missing', + requestSeq: 4, + trace: { requestSeq: 4 } + }) + expect( + request.state === 'manifest_missing' ? request.trace?.bodyJson : undefined + ).toBeUndefined() + }) + + it('distinguishes malformed manifests from hash-invalid but readable manifests', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'bad', seq: 2 }, + data: { manifest: { schemaVersion: 99, requestSeq: 2 } } + }) + const tamperedManifest = createObservationManifest({ messageId: 'a1', requestSeq: 1 }) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 1 }, + data: { manifest: { ...tamperedManifest, latestEntryId: 1 } } + }) + + expect(service.readCausalObservationSlice('s1', 'bad').request).toEqual({ + state: 'manifest_malformed', + requestSeq: 2, + trace: null + }) + const invalid = service.readCausalObservationSlice('s1', 'a1').request + expect(invalid.state).toBe('manifest_bound') + expect(invalid.state === 'manifest_bound' ? invalid.replay.integrity : undefined).toBe( + 'invalid' + ) + }) + + it('ignores request sequence zero interleaved-reasoning sentinels', () => { + const { table } = createTapeTableMock() + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 0 }, + data: { manifest: { schemaVersion: 99, requestSeq: 0 } } + }) + const service = createTapeService(table, [ + createTraceRow({ + id: 'trace-gap', + request_seq: 0, + endpoint: 'deepchat://interleaved-reasoning-gap' + }) + ]) + + expect(service.readCausalObservationSlice('s1', 'a1').request).toEqual({ + state: 'request_unavailable', + requestSeq: null, + trace: null + }) + }) + + it('keeps multi-round assistant and tool output correlated at message scope only', () => { + const { table } = createTapeTableMock() + const assistant = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 220, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'result' + } + } + ]), + status: 'sent', + createdAt: 200, + updatedAt: 300 + }) + appendMessageRecordToTape(table as any, assistant, 'live') + const service = createTapeService(table, [], [createMessageRow({ content: assistant.content })]) + service.appendViewManifest(createObservationManifest({ requestSeq: 1 })) + service.appendViewManifest( + createObservationManifest({ + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null + }) + ) + + const firstOutput = service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }).output + const latestOutput = service.readCausalObservationSlice('s1', 'a1').output + + expect(latestOutput.correlation).toBe('message_only') + expect(latestOutput.entries.map((entry) => entry.kind)).toEqual([ + 'message', + 'tool_call', + 'tool_result' + ]) + expect(latestOutput.entries.map((entry) => entry.entryId)).toEqual( + firstOutput.entries.map((entry) => entry.entryId) + ) + }) + + it('does not infer a completed event from a paused pending assistant message', () => { + const { table } = createTapeTableMock() + const service = createTapeService( + table, + [], + [createMessageRow({ status: 'pending', content: '[{"type":"action","status":"pending"}]' })] + ) + + const slice = service.readCausalObservationSlice('s1', 'a1', { + currentRuntimeStatus: 'generating' + }) + + expect(slice.output.entries).toEqual([]) + expect(slice.output.terminalMessage).toBeNull() + expect(slice.runtime).toEqual({ + scope: 'current_only', + status: 'generating', + eventHistory: 'not_persisted' + }) + expect(JSON.stringify(slice)).not.toContain('completed') + }) + + it('reads an old message without bootstrapping or backfilling Tape', () => { + const { table, entries } = createTapeTableMock() + const messageInsert = vi.fn() + const traceInsert = vi.fn() + const projectionApply = vi.fn() + const projectionReplace = vi.fn() + const cursorWrite = vi.fn() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatMessageTracesTable: { + listByMessageId: vi.fn().mockReturnValue([]), + insert: traceInsert + }, + deepchatMessagesTable: { + get: vi.fn().mockReturnValue(createMessageRow()), + insert: messageInsert + }, + deepchatMemoryIngestionProjectionTable: { + applyAppendedEntry: projectionApply, + replaceSession: projectionReplace + }, + deepchatSessionsTable: { + getSummaryState: vi.fn().mockReturnValue(null), + updateMemoryCursorOrderSeq: cursorWrite + } + } as any) + + const slice = service.readCausalObservationSlice('s1', 'a1') + + expect(slice.request).toEqual({ + state: 'request_unavailable', + requestSeq: null, + trace: null + }) + expect(slice.output.terminalMessage?.status).toBe('sent') + expect(slice.runtime).toEqual({ + scope: 'unavailable', + status: null, + eventHistory: 'not_persisted' + }) + expect(entries).toEqual([]) + expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() + expect(table.append).not.toHaveBeenCalled() + expect(table.appendEvent).not.toHaveBeenCalled() + expect(messageInsert).not.toHaveBeenCalled() + expect(traceInsert).not.toHaveBeenCalled() + expect(projectionApply).not.toHaveBeenCalled() + expect(projectionReplace).not.toHaveBeenCalled() + expect(cursorWrite).not.toHaveBeenCalled() + }) + + it('keeps causal observations metadata-only by default and reuses replay payload opt-ins', () => { + const { table } = createTapeTableMock() + const assistant = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: 'tape-secret-output', + status: 'sent', + metadata: '{"secret":"tape-metadata"}', + createdAt: 200, + updatedAt: 300 + }) + appendMessageRecordToTape(table as any, assistant, 'live') + const service = createTapeService( + table, + [ + createTraceRow({ + headers_json: '{"authorization":"secret-header"}', + body_json: '{"prompt":"secret-request"}' + }) + ], + [ + createMessageRow({ + content: 'projection-only-content-secret', + metadata: '{"secret":"projection-only-metadata"}', + blocks_json: '["projection-only-blocks"]', + error: 'projection-only-error' + }) + ] + ) + service.appendViewManifest(createObservationManifest()) + + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) + const metadataOnly = service.readCausalObservationSlice('s1', 'a1') + const withPayloads = service.readCausalObservationSlice('s1', 'a1', { + includeTapePayloads: true, + includeTracePayload: true + }) + now.mockRestore() + + expect(JSON.stringify(metadataOnly)).not.toContain('tape-secret-output') + expect(JSON.stringify(metadataOnly)).not.toContain('tape-metadata') + expect(JSON.stringify(metadataOnly)).not.toContain('secret-header') + expect(JSON.stringify(metadataOnly)).not.toContain('secret-request') + expect(JSON.stringify(metadataOnly)).not.toContain('projection-only') + expect(withPayloads.output.entries.some((entry) => entry.payload?.record)).toBe(true) + expect(withPayloads.request.state).toBe('manifest_bound') + expect( + withPayloads.request.state === 'manifest_bound' + ? withPayloads.request.replay.trace?.headersJson + : undefined + ).toContain('secret-header') + expect(JSON.stringify(withPayloads)).toContain('tape-secret-output') + expect(JSON.stringify(withPayloads)).not.toContain('projection-only') + expect(metadataOnly.output.terminalMessage).not.toHaveProperty('content') + expect(metadataOnly.output.terminalMessage).not.toHaveProperty('metadata') + expect(metadataOnly.output.terminalMessage).not.toHaveProperty('blocks') + expect(metadataOnly.output.terminalMessage).not.toHaveProperty('error') + expect(stripObservationPayloadOptIns(withPayloads)).toEqual( + stripObservationPayloadOptIns(metadataOnly) + ) + }) + + it('keeps storage and Memory seams unchanged across every causal observation read mode', () => { + const { table, entries } = createTapeTableMock() + const { final } = appendObservationIsolationFacts(table) + const traceRows = [ + createTraceRow({ id: 'trace-1', request_seq: 1 }), + createTraceRow({ id: 'trace-2', request_seq: 2 }), + createTraceRow({ id: 'trace-only', message_id: 'a-trace', request_seq: 7 }) + ] + const messageRows = [createMessageRow({ order_seq: 3 })] + const projectionState = { + meta: { session_id: 's1', projection_version: 1, max_entry_id: entries.length }, + range: [{ session_id: 's1', message_id: 'a1', order_seq: 3, entry_id: entries.length }] + } + const memoryCursorOrderSeq = 3 + const memoryProjectionWrites = createSpies([ + 'applyAppendedEntry', + 'replaceSession', + 'invalidateSession' + ]) + const memoryProjectionTable = { + ...memoryProjectionWrites, + readCurrentRange: vi.fn(() => structuredClone(projectionState.range)), + getSessionMeta: vi.fn(() => structuredClone(projectionState.meta)) + } + const cursorWrites = createSpies(['updateMemoryCursorOrderSeq', 'rewindMemoryCursorOrderSeq']) + const sessionTable = { + ...cursorWrites, + getSummaryState: vi.fn().mockReturnValue(null), + getMemoryCursorOrderSeq: vi.fn(() => memoryCursorOrderSeq) + } + const messageWrites = createSpies([ + 'insert', + 'updateContent', + 'updateStatus', + 'updateContentAndStatus' + ]) + const messageTable = { + ...messageWrites, + get: vi.fn((messageId: string) => messageRows.find((row) => row.id === messageId)) + } + const traceWrites = createSpies(['insert', 'deleteByMessageId', 'deleteBySessionId']) + const traceTable = { + ...traceWrites, + listByMessageId: vi.fn((messageId: string) => + traceRows.filter((row) => row.message_id === messageId) + ) + } + const { presenter, memoryPropertyAccess } = trackMemoryPropertyAccess({ + deepchatTapeEntriesTable: table, + deepchatMessageTracesTable: traceTable, + deepchatMessagesTable: messageTable, + deepchatSessionsTable: sessionTable, + deepchatMemoryIngestionProjectionTable: memoryProjectionTable + }) + const service = new SessionTape(presenter as any) + service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 210 })) + service.appendViewManifest( + createObservationManifest({ + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 310 + }) + ) + + for (const write of [ + table.ensureBootstrapAnchor, + table.append, + table.appendAnchor, + table.appendEvent, + table.deleteBySession + ]) { + write.mockClear() + } + const captureState = () => ({ + tapeRows: structuredClone(entries), + maxEntryId: table.getMaxEntryId('s1'), + entryOrder: entries.map((entry) => entry.entry_id), + maxMessageOrder: Math.max(0, ...service.getMessageRecords('s1').map((row) => row.orderSeq)), + messageRows: structuredClone(messageRows), + traceRows: structuredClone(traceRows), + viewManifestRows: structuredClone( + entries.filter((entry) => entry.kind === 'event' && entry.name === 'view/assembled') + ), + effectiveView: service.getMessageRecords('s1'), + replay: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }), + replayHash: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 })?.hashes.sliceHash, + projectionMeta: memoryProjectionTable.getSessionMeta(), + projectionRange: memoryProjectionTable.readCurrentRange(), + memoryCursorOrderSeq: sessionTable.getMemoryCursorOrderSeq() + }) + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) + + try { + const before = captureState() + const { defaultObservation, repeatedObservation, explicitObservation, traceOnlyObservation } = + readObservationMatrix(service) + const after = captureState() + + expect(after).toEqual(before) + expect(repeatedObservation).toEqual(defaultObservation) + expect(defaultObservation.request).toMatchObject({ + state: 'manifest_bound', + requestSeq: 2 + }) + expect(explicitObservation.request).toMatchObject({ + state: 'manifest_bound', + requestSeq: 1 + }) + expect(traceOnlyObservation.request).toMatchObject({ + state: 'manifest_missing', + requestSeq: 7, + trace: { id: 'trace-only' } + }) + expect(defaultObservation.output).toMatchObject({ + correlation: 'message_only', + entries: [{ kind: 'message' }, { kind: 'tool_call' }, { kind: 'tool_result' }] + }) + expect(before.effectiveView.map((record) => record.id)).toEqual(['u1', 'a1']) + expect(JSON.parse(before.effectiveView[0].content).text).toBe('edited') + expect(before.effectiveView[1]).toMatchObject({ status: 'sent', content: final.content }) + } finally { + now.mockRestore() + } + + for (const write of [ + table.ensureBootstrapAnchor, + table.append, + table.appendAnchor, + table.appendEvent, + table.deleteBySession, + ...Object.values(messageWrites), + ...Object.values(traceWrites), + ...Object.values(memoryProjectionWrites), + ...Object.values(cursorWrites), + memoryPropertyAccess + ]) { + expect(write).not.toHaveBeenCalled() + } + }) + + itIfSqlite( + `preserves real Tape, message, trace, Memory projection and schema state while observing${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const memoryProjectionTable = new DeepChatMemoryIngestionProjectionTable(db) + const tapeTable = new DeepChatTapeEntriesTable(db, memoryProjectionTable) + const messageTable = new DeepChatMessagesTable(db) + const traceTable = new DeepChatMessageTracesTable(db) + const sessionTable = new DeepChatSessionsTable(db) + memoryProjectionTable.createTable() + tapeTable.createTable() + messageTable.createTable() + traceTable.createTable() + sessionTable.createTable() + sessionTable.create('s1', 'openai', 'gpt-4o', 'full_access') + sessionTable.updateMemoryCursorOrderSeq('s1', 3) + + appendObservationIsolationFacts(tapeTable) + messageTable.insert({ + id: 'a1', + sessionId: 's1', + orderSeq: 3, + role: 'assistant', + content: 'projection-only-content-secret', + status: 'sent', + metadata: '{"secret":"projection-only-metadata"}', + createdAt: 200, + updatedAt: 300 + }) + for (const trace of [ + { id: 'trace-1', messageId: 'a1', requestSeq: 1 }, + { id: 'trace-2', messageId: 'a1', requestSeq: 2 }, + { id: 'trace-only', messageId: 'a-trace', requestSeq: 7 } + ]) { + traceTable.insert({ + ...trace, + sessionId: 's1', + providerId: 'openai', + modelId: 'gpt-4o', + endpoint: 'https://api.openai.test/v1/chat/completions', + headersJson: `{"authorization":"${trace.id}-header"}`, + bodyJson: `{"prompt":"${trace.id}-body"}`, + truncated: false, + createdAt: 300 + trace.requestSeq + }) + } + + const service = new SessionTape({ + deepchatTapeEntriesTable: tapeTable, + deepchatMessagesTable: messageTable, + deepchatMessageTracesTable: traceTable, + deepchatSessionsTable: sessionTable, + deepchatMemoryIngestionProjectionTable: memoryProjectionTable + } as any) + service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 210 })) + service.appendViewManifest( + createObservationManifest({ + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 310 + }) + ) + const effectiveRecords = service.getMessageRecords('s1') + const sourceMaps = service.getViewManifestSourceMaps('s1') + memoryProjectionTable.replaceSession( + 's1', + effectiveRecords + .filter( + (record): record is ChatMessageRecord & { status: 'sent' | 'error' } => + record.status === 'sent' || record.status === 'error' + ) + .map((record) => ({ + sessionId: 's1', + messageId: record.id, + orderSeq: record.orderSeq, + entryId: sourceMaps.entryIdByMessageId.get(record.id)!, + role: record.role, + content: record.content, + status: record.status, + hadToolUse: record.id === 'a1' + })), + tapeTable.getMaxEntryId('s1') + ) + + const captureState = () => { + const tapeRows = tapeTable.getBySession('s1') + return { + tapeRows, + maxEntryId: tapeTable.getMaxEntryId('s1'), + entryOrder: tapeRows.map((row) => row.entry_id), + messageRow: messageTable.get('a1'), + traceRows: db + .prepare( + `SELECT * + FROM deepchat_message_traces + ORDER BY session_id, message_id, request_seq, id` + ) + .all(), + viewManifestRows: tapeRows.filter( + (row) => row.kind === 'event' && row.name === 'view/assembled' + ), + effectiveView: service.getMessageRecords('s1'), + replay: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }), + replayHash: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 })?.hashes.sliceHash, + projectionMeta: memoryProjectionTable.getSessionMeta('s1'), + projectionRange: memoryProjectionTable.readCurrentRange('s1', 0, 10), + memoryCursorOrderSeq: sessionTable.getMemoryCursorOrderSeq('s1'), + schema: db + .prepare( + `SELECT type, name, tbl_name, sql + FROM sqlite_master + WHERE name NOT LIKE 'sqlite_%' + ORDER BY type, name, tbl_name` + ) + .all() + } + } + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) + + try { + const before = captureState() + readObservationMatrix(service) + const after = captureState() + + expect(after).toEqual(before) + } finally { + now.mockRestore() + } + } finally { + db.close() + } + } + ) +}) diff --git a/test/main/session/data/transcript.test.ts b/test/main/session/data/transcript.test.ts index f530befe1b..bf6252cfc9 100644 --- a/test/main/session/data/transcript.test.ts +++ b/test/main/session/data/transcript.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { SessionTranscript } from '@/session/data/transcript' +import { SessionTape } from '@/tape/application/sessionTape' import { cloneBlocksForRenderer } from '@/agent/deepchat/runtime/echo' import logger from '@shared/logger' @@ -142,7 +143,7 @@ describe('SessionTranscript', () => { beforeEach(() => { sqlitePresenter = createMockSqlitePresenter() - store = new SessionTranscript(sqlitePresenter) + store = new SessionTranscript(sqlitePresenter, new SessionTape(sqlitePresenter)) }) describe('createUserMessage', () => { @@ -832,6 +833,34 @@ describe('SessionTranscript', () => { expect(sqlitePresenter.deepchatMessageTracesTable.deleteByMessageIds).not.toHaveBeenCalled() expect(sqlitePresenter.deepchatMessagesTable.deleteFromOrderSeq).toHaveBeenCalledWith('s1', 2) }) + + it('defers projection deletion until every tape retraction has been appended', () => { + const appendEvent = vi + .fn() + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => { + throw new Error('second retraction failed') + }) + const transaction = vi.fn((operation: () => unknown) => () => operation()) + sqlitePresenter.getDatabase = vi.fn().mockReturnValue({ transaction }) + sqlitePresenter.deepchatTapeEntriesTable = { + ensureBootstrapAnchor: vi.fn(), + appendEvent + } + sqlitePresenter.deepchatMessagesTable.getBySession.mockReturnValue([ + createMessageRow({ id: 'm1', order_seq: 1 }), + createMessageRow({ id: 'm2', order_seq: 2 }), + createMessageRow({ id: 'm3', order_seq: 3 }) + ]) + + expect(() => store.deleteFromOrderSeq('s1', 2)).toThrow('second retraction failed') + + expect(transaction).toHaveBeenCalledOnce() + expect(appendEvent).toHaveBeenCalledTimes(2) + expect(sqlitePresenter.deepchatMessagesTable.deleteFromOrderSeq).not.toHaveBeenCalled() + expect(sqlitePresenter.deepchatSearchDocumentsTable.deleteByMessageIds).not.toHaveBeenCalled() + expect(sqlitePresenter.deepchatMessageTracesTable.deleteByMessageIds).not.toHaveBeenCalled() + }) }) describe('trace operations', () => { diff --git a/test/main/session/runtimeIntegration.test.ts b/test/main/session/runtimeIntegration.test.ts index 960fa5d2d5..6e723ed0c2 100644 --- a/test/main/session/runtimeIntegration.test.ts +++ b/test/main/session/runtimeIntegration.test.ts @@ -74,6 +74,7 @@ function createMockSqlitePresenter() { let messagesList: any[] = [] const tapeTable = { + runInTransaction: vi.fn((operation: () => unknown) => operation()), ensureBootstrapAnchor: vi.fn(), append: vi.fn((input: any) => { const row = { @@ -638,10 +639,12 @@ function createMockSqlitePresenter() { }) }, deepchatTapeEntriesTable: tapeTable, + tapeLifecycle: tapeTable, deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn(), isCurrent: vi.fn().mockReturnValue(false), - getByEntryIds: vi.fn().mockReturnValue([]) + getByEntryIds: vi.fn().mockReturnValue([]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) }, // Expose internal stores for assertion _sessionsStore: sessionsStore, @@ -789,7 +792,8 @@ function createTranscriptMutations( transcript: sessionData.transcript, settings: sessionData.settings, pendingInputs: sessionData.pendingInputs, - runtime + runtime, + runInTransaction: (operation) => sessionData.database.getDatabase().transaction(operation)() }) } @@ -1740,8 +1744,19 @@ describe('Integration: Session Tape boundary', () => { expect(search).toHaveBeenCalledTimes(2) expect(getContext).toHaveBeenCalledOnce() + ensureTapeReady.mockClear() + search.mockClear() + getContext.mockClear() + await sessionData.tape.searchTape('s1', 'needle') + expect(ensureTapeReady.mock.invocationCallOrder[0]).toBeLessThan( + search.mock.invocationCallOrder[0] + ) + await sessionData.tape.getTapeContext('s1', [2]) + expect(ensureTapeReady.mock.invocationCallOrder[1]).toBeLessThan( + getContext.mock.invocationCallOrder[0] + ) expect(ensureTapeReady).toHaveBeenCalledTimes(2) }) diff --git a/test/main/session/transcriptMutations.test.ts b/test/main/session/transcriptMutations.test.ts new file mode 100644 index 0000000000..b0e076f20d --- /dev/null +++ b/test/main/session/transcriptMutations.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it, vi } from 'vitest' +import { SessionTranscriptMutations } from '@/session/transcriptMutations' + +describe('SessionTranscriptMutations', () => { + it('rolls pending inputs and transcript deletion back when Tape reset fails', async () => { + const state = { pendingInputs: 1, transcriptMessages: 2, tapeEntries: 3 } + const runtime = { + prepareClearMessages: vi.fn().mockResolvedValue(undefined), + finishClearMessages: vi.fn() + } + const mutations = new SessionTranscriptMutations({ + pendingInputs: { + deleteBySession: vi.fn(() => { + state.pendingInputs = 0 + }) + }, + transcript: { + deleteBySession: vi.fn(() => { + state.transcriptMessages = 0 + }) + }, + settings: { + resetTape: vi.fn(() => { + state.tapeEntries = 0 + throw new Error('Tape reset failed') + }) + }, + runtime, + runInTransaction: (operation) => { + const snapshot = { ...state } + try { + return operation() + } catch (error) { + Object.assign(state, snapshot) + throw error + } + } + } as any) + + await expect(mutations.clearMessages('s1')).rejects.toThrow('Tape reset failed') + + expect(state).toEqual({ pendingInputs: 1, transcriptMessages: 2, tapeEntries: 3 }) + expect(runtime.prepareClearMessages).toHaveBeenCalledWith('s1') + expect(runtime.finishClearMessages).not.toHaveBeenCalled() + }) + + it('finishes runtime cleanup only after the shared transaction commits', async () => { + const calls: string[] = [] + const mutations = new SessionTranscriptMutations({ + pendingInputs: { deleteBySession: vi.fn(() => calls.push('pending')) }, + transcript: { deleteBySession: vi.fn(() => calls.push('transcript')) }, + settings: { resetTape: vi.fn(() => calls.push('tape')) }, + runtime: { + prepareClearMessages: vi.fn(async () => { + calls.push('prepare') + }), + finishClearMessages: vi.fn(() => calls.push('finish')) + }, + runInTransaction: (operation) => { + calls.push('transaction:start') + const result = operation() + calls.push('transaction:commit') + return result + } + } as any) + + await mutations.clearMessages('s1') + + expect(calls).toEqual([ + 'prepare', + 'transaction:start', + 'pending', + 'transcript', + 'tape', + 'transaction:commit', + 'finish' + ]) + }) +}) diff --git a/test/main/session/usageStatsService.test.ts b/test/main/session/usageStatsService.test.ts index ff5a1c7698..a1ba721c41 100644 --- a/test/main/session/usageStatsService.test.ts +++ b/test/main/session/usageStatsService.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { SessionTranscript } from '@/session/data/transcript' +import { SessionTape } from '@/tape/application/sessionTape' import { DASHBOARD_STATS_BACKFILL_KEY, type UsageStatsRecordInput } from '@/session/usageStats' import { UsageStatsService } from '@/session/usageStatsService' import type { PermissionMode } from '@shared/types/agent-interface' @@ -556,7 +557,7 @@ describe('UsageStatsService', () => { it('keeps a single stats row when live finalize updates a previously backfilled message', async () => { const { service, sqlitePresenter } = createService() - const messageStore = new SessionTranscript(sqlitePresenter) + const messageStore = new SessionTranscript(sqlitePresenter, new SessionTape(sqlitePresenter)) sqlitePresenter.deepchatSessionsTable.create('session-1', 'openai', 'gpt-4o', 'full_access') sqlitePresenter.deepchatMessagesTable.insert({ diff --git a/test/main/tape/layerBoundaries.test.ts b/test/main/tape/layerBoundaries.test.ts new file mode 100644 index 0000000000..5a9487188d --- /dev/null +++ b/test/main/tape/layerBoundaries.test.ts @@ -0,0 +1,643 @@ +import path from 'node:path' +import ts from 'typescript' +import { describe, expect, it, vi } from 'vitest' + +const MAIN_SOURCE_ROOT = path.resolve(process.cwd(), 'src/main') +const TAPE_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape') +const TAPE_DOMAIN_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape/domain') +const TAPE_SQLITE_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape/infrastructure/sqlite') +const TAPE_CAPABILITIES_MODULE = path.join(MAIN_SOURCE_ROOT, 'tape/ports/capabilities') +const TAPE_SESSION_FACADE_MODULE = path.join(MAIN_SOURCE_ROOT, 'tape/application/sessionTape') +const MEMORY_ROUTES_FILE = path.join(MAIN_SOURCE_ROOT, 'memory/routes.ts') +const TAPE_SQLITE_RELATIVE_ROOT = 'tape/infrastructure/sqlite/' +const TYPESCRIPT_SOURCE_EXTENSION = /\.[cm]?tsx?$/ + +interface LegacyTapeCompatibilityContract { + target: string + valueExports: readonly string[] + typeExports: readonly string[] + typeAliases?: Readonly> +} + +const LEGACY_TAPE_COMPATIBILITY_MODULES = new Map([ + [ + 'session/data/tape', + { + target: '@/tape/application/sessionTape', + valueExports: [ + 'AgentTapeViewError', + 'normalizeSubagentTapeLinkInput', + 'normalizeTapeHandoffState', + 'SessionTape' + ], + typeExports: [ + 'AgentTapeViewErrorCode', + 'TapeAnchorResult', + 'TapeBackfillResult', + 'TapeForkHandle', + 'TapeInfo', + 'TapeMigrationState', + 'TapeSearchResult' + ], + typeAliases: { TapeViewManifestSourceMaps: 'TapeViewManifestAssemblySources' } + } + ], + [ + 'session/data/tapeEffectiveView', + { + target: '@/tape/domain/effectiveView', + valueExports: [ + 'buildEffectiveTapeView', + 'getLastEffectiveTokenUsage', + 'searchEffectiveTapeRows' + ], + typeExports: ['EffectiveMessageEntry', 'EffectiveTapeView'] + } + ], + [ + 'session/data/tapeFacts', + { + target: '@/tape/application/factPersistence', + valueExports: [ + 'appendMessageRecordToTape', + 'appendMessageReplacementToTape', + 'appendMessageRetractionToTape', + 'appendTapeToolFact', + 'appendToolFactsToTape', + 'buildTapeToolFactInputs', + 'tapeEntriesToEffectiveMessageRecords', + 'tapeEntryToMessageRecord' + ], + typeExports: ['TapeFactSource'] + } + ], + [ + 'session/data/tapeViewManifest', + { + target: '@/tape/domain/viewManifest', + valueExports: [ + 'buildExcludedRefs', + 'buildIncludedRefs', + 'buildRequestRefs', + 'createTapeViewManifest', + 'hashJson', + 'isCompactionRecord', + 'resolveTapeViewManifestPolicy', + 'stableJsonStringify', + 'TAPE_VIEW_CONTEXT_BUILDER_VERSION', + 'TAPE_VIEW_MANIFEST_EVENT_NAME', + 'TAPE_VIEW_MANIFEST_HASH_VERSION', + 'verifyTapeViewManifestHash' + ], + typeExports: [ + 'ContextSummaryCursorMetadata', + 'TapeViewContextSelection', + 'TapeViewManifestBuildInput', + 'TapeViewManifestPolicyInput', + 'TapeViewManifestPolicyResult' + ], + typeAliases: { TapeViewManifestSourceMaps: 'TapeViewManifestLookupMaps' } + } + ], + [ + 'session/data/tables/deepchatTapeEffectiveSemantics', + { + target: '@/tape/domain/effectiveSemantics', + valueExports: [ + 'messageRecordHasFinalToolUse', + 'parseAssistantBlocks', + 'parseNestedTapeJsonObject', + 'parseTapeJsonObject', + 'readTapeMessageRetractionId', + 'readTapeToolIdentity', + 'readTapeToolStatus', + 'tapeEntryToMessageRecord', + 'tapeMessageRank', + 'tapeToolRank' + ], + typeExports: ['DeepChatTapeToolIdentity'] + } + ], + [ + 'session/data/tables/deepchatTapeEntries', + { + target: '@/tape/infrastructure/sqlite/tapeEntryStore', + valueExports: [ + 'buildDeepChatTapeFtsMatch', + 'buildDeepChatTapeLikeSearchPredicate', + 'DeepChatTapeEntriesTable', + 'normalizeDeepChatTapeReadSources', + 'serializeDeepChatTapeReadSources', + 'SUMMARY_ANCHOR_NAMES', + 'TAPE_INCARNATION_META_KEY' + ], + typeExports: [ + 'DeepChatTapeAppendInput', + 'DeepChatTapeEntryKind', + 'DeepChatTapeEntryRow', + 'DeepChatTapeMutationProjection', + 'DeepChatTapeReadSource', + 'DeepChatTapeSearchInput', + 'DeepChatTapeSourceInput', + 'DeepChatTapeSourceType' + ] + } + ], + [ + 'session/data/tables/deepchatTapeSearchProjection', + { + target: '@/tape/infrastructure/sqlite/tapeSearchProjectionStore', + valueExports: [ + 'DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION', + 'DeepChatTapeSearchProjectionTable' + ], + typeExports: [ + 'DeepChatTapeSearchProjectionInput', + 'DeepChatTapeSearchProjectionMeta', + 'DeepChatTapeSearchProjectionReadResult', + 'DeepChatTapeSearchProjectionResultRow', + 'DeepChatTapeSearchProjectionRow' + ] + } + ] +]) + +const CAPABILITY_SCOPED_CONSUMER_FILES = [ + 'agent/acp/compatibility/adapters.ts', + 'agent/acp/compatibility/dependencies.ts', + 'agent/deepchat/memory/memoryRuntimeCoordinator.ts', + 'agent/deepchat/runtime/deepChatLoopRunner.ts', + 'agent/deepchat/runtime/turnCoordinator.ts', + 'app/startupMigrations/legacyChatImportService.ts', + 'memory/routes.ts', + 'session/data/settings.ts', + 'session/data/transcript.ts' +].map((file) => path.join(MAIN_SOURCE_ROOT, file)) + +const FORBIDDEN_DOMAIN_SQLITE_IMPORTS = new Set([ + 'better-sqlite3', + 'better-sqlite3-multiple-ciphers', + 'bun:sqlite', + 'node:sqlite', + 'sql.js', + 'sqlite3' +]) +const FORBIDDEN_DOMAIN_LOGGING_IMPORTS = new Set([ + '@shared/logger', + 'electron-log', + 'loglevel', + 'pino', + 'winston' +]) + +const PHYSICAL_TAPE_STORAGE_PATTERN = + /\b(?:deepchat_tape_(?:entries|search_(?:projection(?:_meta)?|fts(?:_meta)?))|DeepChatTape(?:Entries|SearchProjection)Table|deepchatTape(?:Entries|SearchProjection)(?:Table)?)\b/ + +interface StorageBoundaryException { + physicalName?: string + sqliteImport?: string +} + +const ALLOWED_STORAGE_EXCEPTIONS = new Map([ + ['app/databaseSecurity.ts', { physicalName: 'database table-name security allowlist' }], + [ + 'app/startupMigrations/legacyChatImportService.ts', + { physicalName: 'migration-only full-table replacement and projection cleanup' } + ], + [ + 'data/schemaCatalog.ts', + { + physicalName: 'schema creation and migration registry', + sqliteImport: 'schema adapter construction' + } + ], + [ + 'data/sqliteCopyExclusions.ts', + { physicalName: 'SQLite virtual-table copy exclusion metadata' } + ], + [ + 'memory/data/tables/deepchatMemoryIngestionProjection.ts', + { physicalName: 'read-only single-statement Tape-head consistency check' } + ], + [ + 'session/data/database.ts', + { + physicalName: 'SQLite adapter compatibility getters', + sqliteImport: 'SQLite adapter composition' + } + ], + [ + 'session/data/tables/deepchatTapeEntries.ts', + { + physicalName: 'frozen legacy compatibility export', + sqliteImport: 'legacy import-path compatibility re-export' + } + ], + [ + 'session/data/tables/deepchatTapeSearchProjection.ts', + { + physicalName: 'frozen legacy compatibility export', + sqliteImport: 'legacy import-path compatibility re-export' + } + ], + ['tape/ports/application.ts', { physicalName: 'legacy database-shape compatibility adapter' }] +]) + +function listTypeScriptSources(root: string, fs: typeof import('node:fs')): string[] { + return fs + .readdirSync(root, { withFileTypes: true }) + .flatMap((entry) => { + const entryPath = path.join(root, entry.name) + if (entry.isDirectory()) return listTypeScriptSources(entryPath, fs) + if (entry.isFile() && TYPESCRIPT_SOURCE_EXTENSION.test(entry.name)) return [entryPath] + return [] + }) + .sort() +} + +function relativeToMain(file: string): string { + return path.relative(MAIN_SOURCE_ROOT, file).split(path.sep).join('/') +} + +function isInside(root: string, target: string): boolean { + const relativePath = path.relative(root, target) + return relativePath === '' || (!relativePath.startsWith('..') && !path.isAbsolute(relativePath)) +} + +function resolveMainImport(importingFile: string, specifier: string): string | null { + if (specifier.startsWith('@/')) { + return path.resolve(MAIN_SOURCE_ROOT, specifier.slice(2)) + } + if (specifier.startsWith('.')) { + return path.resolve(path.dirname(importingFile), specifier) + } + return null +} + +function withoutTypeScriptExtension(file: string): string { + return file.replace(TYPESCRIPT_SOURCE_EXTENSION, '') +} + +function matchesPackageOrSubpath(specifier: string, packages: ReadonlySet): boolean { + return [...packages].some( + (packageName) => specifier === packageName || specifier.startsWith(`${packageName}/`) + ) +} + +function getForbiddenDomainPackageCategory(specifier: string): string | null { + if ( + specifier === 'electron' || + specifier.startsWith('electron/') || + specifier.startsWith('@electron/') + ) { + return 'Electron runtime' + } + if ( + matchesPackageOrSubpath(specifier, FORBIDDEN_DOMAIN_SQLITE_IMPORTS) || + specifier.startsWith('@libsql/') + ) { + return 'SQLite runtime' + } + if (matchesPackageOrSubpath(specifier, FORBIDDEN_DOMAIN_LOGGING_IMPORTS)) { + return 'logging runtime' + } + return null +} + +function getDomainImportViolation(importingFile: string, specifier: string): string | null { + const forbiddenPackageCategory = getForbiddenDomainPackageCategory(specifier) + if (forbiddenPackageCategory) { + return `${forbiddenPackageCategory} import ${specifier}` + } + + const target = resolveMainImport(importingFile, specifier) + if (target && !isInside(TAPE_DOMAIN_ROOT, target)) { + return `main-process dependency ${specifier}` + } + return null +} + +function isLegacyTapeCompatibilityImport(importingFile: string, specifier: string): boolean { + const target = resolveMainImport(importingFile, specifier) + if (!target) return false + const relativeTarget = relativeToMain(withoutTypeScriptExtension(target)) + return LEGACY_TAPE_COMPATIBILITY_MODULES.has(relativeTarget) +} + +function isTapeModuleImport(importingFile: string, specifier: string): boolean { + const target = resolveMainImport(importingFile, specifier) + return Boolean(target && isInside(TAPE_ROOT, target)) +} + +function findConcreteTapeFacadeImportViolations(source: string, file: string): string[] { + return ts.preProcessFile(source, true, true).importedFiles.flatMap(({ fileName: specifier }) => { + const target = resolveMainImport(file, specifier) + return target && withoutTypeScriptExtension(target) === TAPE_SESSION_FACADE_MODULE + ? [`Concrete Tape facade import: ${specifier}`] + : [] + }) +} + +function findMemoryRouteTapeImportViolations(source: string, file: string): string[] { + const sourceFile = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true) + const tapeReferences = ts + .preProcessFile(source, true, true) + .importedFiles.filter(({ fileName }) => isTapeModuleImport(file, fileName)) + const staticTapeImports = sourceFile.statements.filter( + (statement): statement is ts.ImportDeclaration => + ts.isImportDeclaration(statement) && + ts.isStringLiteral(statement.moduleSpecifier) && + isTapeModuleImport(file, statement.moduleSpecifier.text) + ) + const violations = tapeReferences.flatMap(({ fileName: specifier }) => { + const target = resolveMainImport(file, specifier) + return !target || withoutTypeScriptExtension(target) !== TAPE_CAPABILITIES_MODULE + ? [`Tape import must use the inspection port: ${specifier}`] + : [] + }) + + if (tapeReferences.length !== staticTapeImports.length) { + violations.push('Tape references must use a static type-only import declaration') + } + + violations.push( + ...staticTapeImports.flatMap((statement) => { + const specifier = statement.moduleSpecifier.text + const importClause = statement.importClause + const namedBindings = importClause?.namedBindings + if ( + !importClause || + importClause.name || + !namedBindings || + !ts.isNamedImports(namedBindings) || + namedBindings.elements.length !== 1 + ) { + return [`Tape capabilities import must name only TapeInspectionReader: ${specifier}`] + } + + const [element] = namedBindings.elements + const importedName = element.propertyName?.text ?? element.name.text + const isTypeOnly = importClause.isTypeOnly || element.isTypeOnly + return importedName === 'TapeInspectionReader' && isTypeOnly + ? [] + : [`Memory routes may import only the TapeInspectionReader type: ${importedName}`] + }) + ) + + return [...new Set(violations)] +} + +function compatibilityExportDescriptors(contract: LegacyTapeCompatibilityContract): string[] { + return [ + ...contract.valueExports.map((name) => `value:${name}:${name}`), + ...contract.typeExports.map((name) => `type:${name}:${name}`), + ...Object.entries(contract.typeAliases ?? {}).map( + ([exportedName, importedName]) => `type:${importedName}:${exportedName}` + ) + ].sort() +} + +function isFrozenCompatibilityReexport( + source: string, + contract: LegacyTapeCompatibilityContract +): boolean { + if (!source.includes('@deprecated')) return false + const sourceFile = ts.createSourceFile('compatibility.ts', source, ts.ScriptTarget.Latest, true) + const descriptors = sourceFile.statements.flatMap((statement) => { + if ( + !ts.isExportDeclaration(statement) || + !statement.exportClause || + !ts.isNamedExports(statement.exportClause) || + !statement.moduleSpecifier || + !ts.isStringLiteral(statement.moduleSpecifier) || + statement.moduleSpecifier.text !== contract.target + ) { + return ['invalid'] + } + return statement.exportClause.elements.map((element) => { + const importedName = element.propertyName?.text ?? element.name.text + const exportedName = element.name.text + const kind = statement.isTypeOnly || element.isTypeOnly ? 'type' : 'value' + return `${kind}:${importedName}:${exportedName}` + }) + }) + return ( + sourceFile.statements.length > 0 && + JSON.stringify(descriptors.sort()) === JSON.stringify(compatibilityExportDescriptors(contract)) + ) +} + +describe('Tape layer boundaries', () => { + it('keeps the Tape domain independent from other main-process layers', async () => { + const fs = await vi.importActual('node:fs') + const violations = listTypeScriptSources(TAPE_DOMAIN_ROOT, fs).flatMap((file) => { + const source = fs.readFileSync(file, 'utf8') + const imports = ts.preProcessFile(source, true, true).importedFiles + + return imports.flatMap(({ fileName: specifier }) => { + const violation = getDomainImportViolation(file, specifier) + return violation ? [`${relativeToMain(file)}: ${violation}`] : [] + }) + }) + + expect(violations).toEqual([]) + }) + + it('keeps production code off legacy Tape compatibility imports', async () => { + const fs = await vi.importActual('node:fs') + const violations = listTypeScriptSources(MAIN_SOURCE_ROOT, fs).flatMap((file) => { + const source = fs.readFileSync(file, 'utf8') + return ts + .preProcessFile(source, true, true) + .importedFiles.flatMap(({ fileName: specifier }) => + isLegacyTapeCompatibilityImport(file, specifier) + ? [`${relativeToMain(file)} -> ${specifier}`] + : [] + ) + }) + + expect(violations).toEqual([]) + }) + + it('keeps legacy Tape compatibility modules on frozen deprecated export contracts', async () => { + const fs = await vi.importActual('node:fs') + const violations = [...LEGACY_TAPE_COMPATIBILITY_MODULES.entries()].flatMap( + ([relativeModule, contract]) => { + const file = path.join(MAIN_SOURCE_ROOT, `${relativeModule}.ts`) + const source = fs.readFileSync(file, 'utf8') + return isFrozenCompatibilityReexport(source, contract) + ? [] + : [`${relativeModule}.ts must match its frozen ${contract.target} export contract`] + } + ) + + expect(violations).toEqual([]) + }) + + it('keeps Memory routes on the Tape inspection DTO port', async () => { + const fs = await vi.importActual('node:fs') + const source = fs.readFileSync(MEMORY_ROUTES_FILE, 'utf8') + expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).toEqual([]) + }) + + it('keeps capability-scoped consumers off the concrete Tape facade', async () => { + const fs = await vi.importActual('node:fs') + const violations = CAPABILITY_SCOPED_CONSUMER_FILES.flatMap((file) => + findConcreteTapeFacadeImportViolations(fs.readFileSync(file, 'utf8'), file).map( + (violation) => `${relativeToMain(file)}: ${violation}` + ) + ) + + expect(violations).toEqual([]) + }) + + it.each([ + ['Session', '@/session/data/transcript'], + ['Agent', '@/agent/deepchat/runtime/process'], + ['Memory', '@/memory/routes'], + ['App', '@/app/composition'], + ['Tape ports', '@/tape/ports/capabilities'], + ['Tape SQLite infrastructure', '@/tape/infrastructure/sqlite/tapeEntryStore'], + ['bare SQLite', 'better-sqlite3'], + ['project SQLite driver', 'better-sqlite3-multiple-ciphers'], + ['Node SQLite', 'node:sqlite'], + ['Electron', 'electron'], + ['Electron subpath', 'electron/main'], + ['shared logging', '@shared/logger'], + ['Electron logging', 'electron-log'] + ])('detects forbidden %s imports in the Tape domain', (_category, specifier) => { + const importingFile = path.join(TAPE_DOMAIN_ROOT, 'negative-case.ts') + expect(getDomainImportViolation(importingFile, specifier)).not.toBeNull() + }) + + it.each([ + ['domain sibling', './entry'], + ['domain alias', '@/tape/domain/effectiveView'], + ['shared type', '@shared/types/tape-replay'], + ['Node crypto', 'node:crypto'] + ])('allows pure %s imports in the Tape domain', (_category, specifier) => { + const importingFile = path.join(TAPE_DOMAIN_ROOT, 'allowed-case.ts') + expect(getDomainImportViolation(importingFile, specifier)).toBeNull() + }) + + it.each([ + [path.join(MAIN_SOURCE_ROOT, 'agent/example.ts'), '@/session/data/tape'], + [path.join(MAIN_SOURCE_ROOT, 'session/data/index.ts'), './tapeFacts'], + [ + path.join(MAIN_SOURCE_ROOT, 'memory/example.ts'), + '@/session/data/tables/deepchatTapeEffectiveSemantics' + ], + [path.join(MAIN_SOURCE_ROOT, 'memory/example.ts'), '@/session/data/tables/deepchatTapeEntries'], + [ + path.join(MAIN_SOURCE_ROOT, 'app/example.ts'), + '@/session/data/tables/deepchatTapeSearchProjection' + ] + ])('detects legacy Tape compatibility import %s -> %s', (importingFile, specifier) => { + expect(isLegacyTapeCompatibilityImport(importingFile, specifier)).toBe(true) + }) + + it.each([ + [ + 'raw reader capability', + "import type { TapeRawEntryReader } from '@/tape/ports/capabilities'" + ], + [ + 'effective-view helper', + "import { buildEffectiveTapeView } from '@/tape/domain/effectiveView'" + ], + ['application facade', "import { SessionTape } from '@/tape/application/sessionTape'"], + ['inspection value import', "import { TapeInspectionReader } from '@/tape/ports/capabilities'"], + ['dynamic import', "void import('@/tape/application/sessionTape')"], + ['CommonJS require', "const tape = require('@/tape/domain/effectiveView')"], + ['type re-export', "export type { TapeInspectionReader } from '@/tape/ports/capabilities'"] + ])('detects Memory route Tape bypass through %s', (_category, source) => { + expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).not.toEqual([]) + }) + + it('allows Memory routes to import only the inspection reader type', () => { + const source = "import type { TapeInspectionReader } from '@/tape/ports/capabilities'" + expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).toEqual([]) + }) + + it('allows inline type syntax for the Memory inspection reader', () => { + const source = "import { type TapeInspectionReader } from '@/tape/ports/capabilities'" + expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).toEqual([]) + }) + + it.each([ + ['star export', "/** @deprecated */\nexport * from '@/tape/domain/effectiveView'"], + [ + 'extra export', + "/** @deprecated */\nexport { buildEffectiveTapeView, getLastEffectiveTokenUsage, searchEffectiveTapeRows, unexpected } from '@/tape/domain/effectiveView'\nexport type { EffectiveMessageEntry, EffectiveTapeView } from '@/tape/domain/effectiveView'" + ], + [ + 'missing export', + "/** @deprecated */\nexport { buildEffectiveTapeView } from '@/tape/domain/effectiveView'\nexport type { EffectiveMessageEntry, EffectiveTapeView } from '@/tape/domain/effectiveView'" + ], + [ + 'missing deprecation marker', + "export { buildEffectiveTapeView, getLastEffectiveTokenUsage, searchEffectiveTapeRows } from '@/tape/domain/effectiveView'\nexport type { EffectiveMessageEntry, EffectiveTapeView } from '@/tape/domain/effectiveView'" + ] + ])('rejects a legacy compatibility contract with a %s', (_case, source) => { + const contract = LEGACY_TAPE_COMPATIBILITY_MODULES.get('session/data/tapeEffectiveView')! + expect(isFrozenCompatibilityReexport(source, contract)).toBe(false) + }) + + it.each(['@/tape/application/sessionTape', '../../tape/application/sessionTape'])( + 'detects concrete Tape facade import %s in a capability-scoped consumer', + (specifier) => { + const file = path.join(MAIN_SOURCE_ROOT, 'session/data/consumer.ts') + const source = `import { SessionTape } from '${specifier}'` + expect(findConcreteTapeFacadeImportViolations(source, file)).not.toEqual([]) + } + ) + + it('allows physical Tape storage access only at explicit infrastructure boundaries', async () => { + const fs = await vi.importActual('node:fs') + const matchedExceptionCapabilities = new Set() + const violations = listTypeScriptSources(MAIN_SOURCE_ROOT, fs).flatMap((file) => { + const relativeFile = relativeToMain(file) + if (relativeFile.startsWith(TAPE_SQLITE_RELATIVE_ROOT)) return [] + + const source = fs.readFileSync(file, 'utf8') + const physicalName = source.match(PHYSICAL_TAPE_STORAGE_PATTERN)?.[0] + const sqliteImport = ts + .preProcessFile(source, true, true) + .importedFiles.map(({ fileName }) => ({ + fileName, + target: resolveMainImport(file, fileName) + })) + .find(({ target }) => target && isInside(TAPE_SQLITE_ROOT, target))?.fileName + const exception = ALLOWED_STORAGE_EXCEPTIONS.get(relativeFile) + const fileViolations: string[] = [] + + if (physicalName) { + if (exception?.physicalName) { + matchedExceptionCapabilities.add(`${relativeFile}:physicalName`) + } else { + fileViolations.push(`${relativeFile}: physical name ${physicalName}`) + } + } + if (sqliteImport) { + if (exception?.sqliteImport) { + matchedExceptionCapabilities.add(`${relativeFile}:sqliteImport`) + } else { + fileViolations.push(`${relativeFile}: SQLite import ${sqliteImport}`) + } + } + return fileViolations + }) + + const staleExceptions = [...ALLOWED_STORAGE_EXCEPTIONS.entries()].flatMap(([file, exception]) => + (Object.entries(exception) as Array<[keyof StorageBoundaryException, string]>).flatMap( + ([capability, reason]) => + matchedExceptionCapabilities.has(`${file}:${capability}`) + ? [] + : [`${file} (${capability}): ${reason}`] + ) + ) + + expect(violations).toEqual([]) + expect(staleExceptions).toEqual([]) + }) +}) diff --git a/test/memory-test-scope.json b/test/memory-test-scope.json index f0a41a447f..25630eaf77 100644 --- a/test/memory-test-scope.json +++ b/test/memory-test-scope.json @@ -51,7 +51,12 @@ "test/main/memory/agentMemoryAuditRetention.test.ts", "test/main/memory/agentMemoryPaginationNative.test.ts", "test/main/memory/agentMemoryTable.test.ts", - "test/main/session/data/tape.test.ts", + "test/main/session/data/tables/deepchatTapeEntriesTable.test.ts", + "test/main/session/data/tapeFork.test.ts", + "test/main/session/data/tapeLifecycle.test.ts", + "test/main/session/data/tapeLineage.test.ts", + "test/main/session/data/tapeRecall.test.ts", + "test/main/session/data/tapeViewReplay.test.ts", "test/main/memory/memoryNativeMigration.test.ts", "test/main/memory/memoryVectorStoreV2Native.test.ts", "test/main/memory/memoryUpdateNative.test.ts",