From 1d9b87a94d6daf45736a956baf2f83b58582f6a2 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 15:33:04 +0800 Subject: [PATCH 01/29] docs(tape): specify layering refactor --- docs/architecture/tape-layering/plan.md | 155 +++++++++++++++++++++++ docs/architecture/tape-layering/spec.md | 137 ++++++++++++++++++++ docs/architecture/tape-layering/tasks.md | 58 +++++++++ 3 files changed, 350 insertions(+) create mode 100644 docs/architecture/tape-layering/plan.md create mode 100644 docs/architecture/tape-layering/spec.md create mode 100644 docs/architecture/tape-layering/tasks.md diff --git a/docs/architecture/tape-layering/plan.md b/docs/architecture/tape-layering/plan.md new file mode 100644 index 0000000000..da70828d3c --- /dev/null +++ b/docs/architecture/tape-layering/plan.md @@ -0,0 +1,155 @@ +# Tape Layering Refactor Implementation Plan + +## Target Structure + +The implementation will create a top-level Tape subsystem: + +```text +src/main/tape/ + domain/ pure entry, fact, view, manifest, lineage, and replay logic + ports/ storage and consumer capability interfaces + application/ fact, reconciliation, recall, lineage, view/replay, and fork services + infrastructure/sqlite/ SQLite entry store, query SQL, search projection, and lifecycle adapter +``` + +Existing `src/main/session/data/tape*.ts` and table modules will become compatibility re-exports +where an old import path is still part of the current internal contract. New production imports +will target `@/tape/*`. + +## Domain and Port Design + +Tape entry rows, append inputs, source identities, entry references, fact provenance, and tool fact +inputs move out of Agent and table modules into Tape-owned types. Effective-view selection, +ViewManifest hashing, lineage validation, and replay value conversion remain pure. + +The primary ports are: + +- `TapeEntryStore`: append, anchor/event append helpers, bootstrap, and read/query operations. It + has no destructive method. +- `TapeSearchProjectionStore`: rebuildable search projection behavior. +- `TapeToolFactWriter`: the single `appendToolFact` capability used by the Agent loop. +- `TapeMessageFactWriter`: message, replacement, and retraction fact operations used by transcript. +- `TapeAnchorReader` and `TapeAnchorWriter`: narrow anchor capabilities for settings and Memory. +- `TapeRawEntryReader`: intentional raw-row access for effective-view rebuilding in Memory. +- `TapeInspectionReader`: effective source spans and Memory ViewManifest inspection queries. +- `TapeLifecycleAdmin`: Session-owned delete and reset operations across entries and projections. +- Explicit transcript and trace evidence read ports used by reconciliation and replay. + +One application object may implement multiple interfaces. Composition injects the narrow interface +at each call site. + +## Application Services + +The current `SessionTape` behavior is divided without changing method semantics: + +1. **TapeFactService** owns message, tool, event, anchor, handoff, and ViewManifest fact appends. +2. **TapeReconciler** owns bootstrap, legacy transcript backfill, and legacy summary-anchor repair. +3. **TapeRecallService** owns info, search, context windows, anchor listing, and authorized source + resolution needed by recall. +4. **TapeLineageService** owns link validation, frozen child heads, authorization, and lineage + receipts. +5. **TapeViewReplayService** owns ViewManifest source maps, manifest listing, replay exports, and + explicit trace-evidence reads. +6. **TapeForkService** owns fork creation, isolated writes, delta merge, discard, and external fork + receipts. + +`SessionTape` becomes a compatibility facade that constructs or receives these services and +forwards the existing methods. `SessionTapePort` in Session contracts remains unchanged. + +## SQLite Infrastructure + +The large entry table module is separated into Tape-owned row and append types, reusable effective +query SQL, a normal SQLite entry store, and a lifecycle adapter. Table names, indexes, SQL +predicates, provenance uniqueness, and payload serialization remain byte-compatible. + +Physical entry deletion is removed from the normal entry-store interface. `TapeLifecycleAdmin` +coordinates entry deletion, search projection deletion, and reset bootstrap. Startup legacy import +may continue to execute whole-database cleanup SQL because it rebuilds persisted state before +normal runtime composition. + +The Memory ingestion projection retains its current single SQL statement that compares +`MAX(deepchat_tape_entries.entry_id)` with the projection metadata head. Moving this comparison to +two independent port calls would introduce a freshness race and an extra query, so it is an +allowlisted read-only infrastructure dependency. + +## Composition and Data Flow + +Session data composition will create the entry store and Tape services before constructing +transcript and settings: + +```text +SQLite connection + -> Tape stores and services + -> Transcript with TapeMessageFactWriter + -> Settings with anchor and lifecycle capabilities + -> SessionTape facade and existing SessionTapePort adapter +``` + +Runtime composition passes `TapeToolFactWriter` to the loop, raw-row and anchor capabilities to +the Memory coordinator, and `TapeInspectionReader` to Memory routes. No application consumer gets +the concrete entry table. + +`ensureSessionTapeReady` remains at the current Session port boundary. Search and context requests +with linked-source scopes keep their existing conditional reconciliation behavior. + +## Transaction Boundaries + +- Transcript deletion and retry truncation append retractions inside the same SQLite transaction + that deletes projection rows. +- Summary compare-and-set appends its reconstruction anchor inside the same transaction that + updates summary state. +- Reset and Session deletion coordinate entry and search-projection cleanup through lifecycle + operations while preserving the current external ordering. +- Port implementations use the same connection provider and remain synchronous, so extracting a + service does not cross a transaction boundary. + +Contract tests will force failures between paired operations and verify rollback or unchanged +behavior where the current implementation is atomic. + +## Compatibility Strategy + +- Keep all shared DTOs and `SessionTapePort` signatures unchanged. +- Preserve old internal exported symbol names through compatibility re-exports. +- Preserve schema SQL, existing rows, canonical policy identifiers, hashes, source identities, + provenance keys, error messages where tested, and bounded query limits. +- Preserve projection failure fallback and best-effort fork projection cleanup. +- Keep trace evidence distinct from transcript projection in replay dependencies. + +## Test Strategy + +1. Record the current seven-file baseline: 120 passed and 26 native-SQLite-gated skipped tests. +2. Mechanically split the monolithic test suite by application-service boundary without changing + assertions or skip gates. +3. Add characterization coverage for reconciliation ordering, transaction atomicity, projection + fallback, and lifecycle reset. +4. Add contract coverage for append-only correction, frozen-head authorization, fork delta merge, + ViewManifest hashes, replay evidence, and projection rebuild equivalence. +5. Add a source-boundary test that rejects domain reverse imports and non-allowlisted table access. +6. Run the full main-process suite, Tape scale suite, type checks, formatting, i18n validation, and + lint before handoff. + +## Commit and Review Strategy + +Each implementation slice remains green and receives a local conventional commit. Before every +commit, review the complete unstaged and staged diff for hidden side effects, compatibility, +boundary cases, performance, security, misleading names, missing tests, and long-term maintenance +cost. Fix all findings and repeat validation before committing. + +The planned commits are: + +1. `docs(tape): specify layering refactor` +2. `test(tape): split behavior contracts` +3. `refactor(tape): establish domain ports` +4. `refactor(tape): split application services` +5. `refactor(tape): close storage bypasses` +6. `test(tape): enforce layer boundaries` +7. `docs(tape): refresh architecture map` + +No commit is pushed. The final review compares the complete branch with `dev`. + +## Rollback + +The work is organized into locally reviewable commits. Reverting must proceed in reverse order +because later composition and boundary changes depend on the earlier domain and port extraction. +The unchanged schema and compatibility re-exports allow a complete branch rollback without a data +migration. diff --git a/docs/architecture/tape-layering/spec.md b/docs/architecture/tape-layering/spec.md new file mode 100644 index 0000000000..eda8c5164f --- /dev/null +++ b/docs/architecture/tape-layering/spec.md @@ -0,0 +1,137 @@ +# Tape Layering Refactor Specification + +## Background + +DeepChat's Tape implementation has strong runtime semantics but weak module boundaries. The main +`SessionTape` implementation combines fact writing, migration and reconciliation, search and +context recall, ViewManifest and replay assembly, subagent lineage, and fork management in one +large module. The SQLite entry table also exposes destructive lifecycle operations beside normal +append and read operations. + +Several consumers bypass `SessionTape` and depend directly on the SQLite table. Transcript writes, +Memory ingestion, Memory management routes, Session settings, and startup migration each use a +different subset of Tape behavior, but the current table-shaped dependency gives them more +authority than they need. Tape types also flow in the wrong direction because the Tape layer +imports Agent loop port types. + +This refactor adopts Bub's useful dependency pattern—domain primitives, narrow store protocols, +application services, and independent view selection—without copying Bub's simpler schema or +reset semantics. DeepChat retains its stronger revision, retraction, ViewManifest, frozen-head, +and fork contracts. + +## Goals + +1. Establish a top-level `src/main/tape/` subsystem with explicit domain, port, application, and + SQLite infrastructure boundaries. +2. Split `SessionTape` along its existing cohesive behavior groups while retaining a compatibility + facade. +3. Replace raw table dependencies with the smallest capability required by each consumer. +4. Keep destructive reset and delete operations outside the append-only entry store contract. +5. Remove the Tape-to-Agent reverse dependency and delete unused `TapeRecorder` capabilities. +6. Preserve all persisted data, public IPC behavior, runtime ordering, transaction boundaries, + failure fallbacks, and performance characteristics. +7. Add enforceable dependency and behavioral contracts so the layering does not regress. + +## Data Families + +The refactor keeps three distinct data families: + +| Data family | Role | Authority | +| --- | --- | --- | +| Tape facts | Append-only execution facts, anchors, manifests, lineage, and fork receipts | Tape | +| Transcript projection | UI-oriented structured messages and a legacy backfill source | Session data | +| Trace evidence | Provider request and terminal execution evidence used by replay | Session trace storage | + +Replay may combine Tape facts with trace evidence through explicit read ports. This is not a reason +to treat trace evidence as transcript data or to move it into the Tape entry schema. + +## Required Invariants + +- Entries in an active Tape are append-only. Known facts are never updated in place. +- Corrections and deletions of projected messages are represented by appended replacement or + retraction facts. +- Anchors are reconstruction points and never imply deletion of earlier entries. +- Compaction changes the selected view, not the retained history. +- Fork merge appends only the fork delta and a merge receipt to the parent. +- Cross-Tape reads require an explicit direct-child lineage fact and remain bounded by the stored + child head. +- Search projections are rebuildable derivatives. Projection failures retain the existing bounded + effective-view fallback. +- Destructive Session cleanup is a lifecycle operation, not a normal Tape store operation. + +## Capability Boundaries + +| Consumer | Allowed capability | +| --- | --- | +| Agent loop | `TapeToolFactWriter` | +| Session transcript | `TapeMessageFactWriter` | +| Memory runtime | `TapeRawEntryReader` and `TapeAnchorWriter` | +| Session settings and compaction | `TapeAnchorReader`, `TapeAnchorWriter`, and `TapeLifecycleAdmin` | +| Memory management routes | `TapeInspectionReader` | +| Session IPC | Existing `SessionTapePort` facade | + +A single implementation may satisfy several ports, but each consumer receives only the structural +type it needs. + +## Direct Storage Access Inventory + +The implementation must account for every current physical-table access: + +- `session/data/tape.ts`: current core implementation; replaced by the new facade and services. +- `session/data/transcript.ts`: legitimate message fact producer; migrated to + `TapeMessageFactWriter` while preserving same-connection transactions. +- `session/data/settings.ts`: bootstrap, reconstruction-anchor reads, summary/reset anchors, and + destructive cleanup; migrated to anchor and lifecycle capabilities. +- `agent/deepchat/runtime/deepChatRuntimeCoordinator.ts`: Memory raw-row reads and anchor writes; + migrated to explicit read and write ports. +- `memory/routes.ts` and app composition: raw table object escape; replaced with domain-level + inspection queries. +- `memory/data/tables/deepchatMemoryIngestionProjection.ts`: one-statement freshness comparison + between Tape head and projection head; retained as an explicit read-only infrastructure + exception to preserve atomicity and query count. +- `app/startupMigrations/legacyChatImportService.ts`: destructive whole-database rebuild; retained + as an explicit startup-migration exception. +- Schema catalog and database security table-name lists: metadata, not runtime Tape access. + +## Acceptance Criteria + +1. `src/main/tape/domain/` does not import Agent, Session, Memory, App, or SQLite infrastructure. +2. Agent loop compilation depends on `TapeToolFactWriter`, not the broad `TapeRecorder` interface. +3. Runtime, transcript, settings, routes, and normal application composition do not receive a + `DeepChatTapeEntriesTable` instance. +4. `TapeEntryStore` exposes no reset or delete method. +5. Existing `SessionTapePort`, persisted schema, table names, entry payloads, View policy IDs, and + renderer contracts remain unchanged. +6. Transcript mutation plus Tape correction, and summary mutation plus anchor append, retain their + current transaction semantics. +7. `ensureSessionTapeReady` remains idempotent and runs at the same Session port boundaries. +8. Projection search fallback and non-blocking fork projection cleanup remain unchanged. +9. Baseline Tape tests remain green; the pre-refactor baseline is 120 passed and 26 + environment-gated skipped tests across seven files. +10. Tape scale coverage confirms bounded tail materialization and no added full-history query on + the Memory projection fast path. +11. Architecture tests reject forbidden imports and new physical-table bypasses. +12. No remote Git operations are performed as part of this work. + +## Constraints + +- Use synchronous ports where the current SQLite operation is synchronous; do not introduce + artificial asynchronous transaction boundaries. +- Preserve ordering, idempotency keys, hashes, error classes, and fallback logging semantics. +- Compatibility re-exports may remain at old module paths to control import churn. +- New SDD artifacts in this directory must use English prose. +- Every local commit requires a complete unstaged and staged diff review plus relevant validation. + +## Non-Goals + +- No database schema or data migration. +- No archive-on-reset behavior. +- No change to compaction, context selection, or ViewManifest policy. +- No redesign of transcript or trace storage. +- No renderer or IPC feature change. +- No GitHub issue, pull request, branch push, or other remote mutation. + +## Open Questions + +None. The implementation decisions required for this refactor are recorded in this specification +and the accompanying plan. diff --git a/docs/architecture/tape-layering/tasks.md b/docs/architecture/tape-layering/tasks.md new file mode 100644 index 0000000000..c05043fef1 --- /dev/null +++ b/docs/architecture/tape-layering/tasks.md @@ -0,0 +1,58 @@ +# Tape Layering Refactor Tasks + +## Specification and Baseline + +- [x] Record the pre-refactor Tape baseline: 120 passed and 26 environment-gated skipped tests. +- [x] Write the English `spec.md`, `plan.md`, and `tasks.md` artifacts. +- [x] Confirm the SDD artifacts contain no unresolved clarification markers or non-English prose. +- [x] Review and commit the SDD slice. + +## Behavior Characterization + +- [ ] Split the monolithic Tape suite by reconciliation, recall, lineage, view/replay, and fork + behavior. +- [ ] Preserve every existing assertion and environment skip gate during the mechanical split. +- [ ] Add transaction, reconciliation-order, and projection-fallback characterization coverage. +- [ ] Review and commit the characterization slice. + +## Domain and Ports + +- [ ] Move Tape-owned entry, source, provenance, and fact types out of Agent and SQLite modules. +- [ ] Move effective-view and ViewManifest pure logic into `src/main/tape/domain/`. +- [ ] Introduce normal storage and narrow consumer capability ports. +- [ ] Replace the broad `TapeRecorder` dependency with `TapeToolFactWriter`. +- [ ] Preserve old module paths through compatibility re-exports. +- [ ] Review and commit the domain and port slice. + +## Application Services + +- [ ] Extract fact, reconciliation, recall, lineage, view/replay, and fork services. +- [ ] Convert `SessionTape` into a compatibility facade. +- [ ] Preserve `SessionTapePort` and current reconciliation timing. +- [ ] Review and commit the application-service slice. + +## Infrastructure and Bypass Closure + +- [ ] Separate normal SQLite entry operations from destructive lifecycle operations. +- [ ] Inject message fact capabilities into transcript without changing transaction boundaries. +- [ ] Inject anchor and lifecycle capabilities into Session settings. +- [ ] Replace Memory runtime and route table access with explicit capabilities. +- [ ] Document and allowlist startup migration and Memory projection infrastructure exceptions. +- [ ] Add lifecycle, transaction, projection, and authorization tests. +- [ ] Review and commit the storage-boundary slice. + +## Architecture Enforcement + +- [ ] Add domain dependency-direction tests. +- [ ] Add a physical Tape table access guard with a narrow explicit allowlist. +- [ ] Run the Tape contract and scale suites. +- [ ] Review and commit the architecture-guard slice. + +## Documentation and Final Validation + +- [ ] Update Tape, Session, and Memory architecture references. +- [ ] Run the full main-process test suite and Memory performance suite. +- [ ] Run full type checks, formatting, i18n validation, and lint. +- [ ] Review the complete `dev...HEAD` diff and fix every finding. +- [ ] Complete the task checklist and commit the final documentation slice. +- [ ] Confirm the working tree is clean and the branch has not been pushed. From e4eeefc0156beecd56b4d6cf5953b58e08366ead Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 15:39:35 +0800 Subject: [PATCH 02/29] test(tape): split behavior contracts --- docs/architecture/tape-layering/tasks.md | 8 +- test/main/session/data/settings.test.ts | 44 + test/main/session/data/tape.test.ts | 5531 ----------------- test/main/session/data/tapeFork.test.ts | 376 ++ test/main/session/data/tapeLineage.test.ts | 710 +++ test/main/session/data/tapeRecall.test.ts | 2077 +++++++ test/main/session/data/tapeReconciler.test.ts | 381 ++ test/main/session/data/tapeTestHarness.ts | 638 ++ test/main/session/data/tapeViewReplay.test.ts | 1493 +++++ test/main/session/data/transcript.test.ts | 28 + test/main/session/runtimeIntegration.test.ts | 11 + 11 files changed, 5762 insertions(+), 5535 deletions(-) delete mode 100644 test/main/session/data/tape.test.ts create mode 100644 test/main/session/data/tapeFork.test.ts create mode 100644 test/main/session/data/tapeLineage.test.ts create mode 100644 test/main/session/data/tapeRecall.test.ts create mode 100644 test/main/session/data/tapeReconciler.test.ts create mode 100644 test/main/session/data/tapeTestHarness.ts create mode 100644 test/main/session/data/tapeViewReplay.test.ts diff --git a/docs/architecture/tape-layering/tasks.md b/docs/architecture/tape-layering/tasks.md index c05043fef1..7ef5780a9b 100644 --- a/docs/architecture/tape-layering/tasks.md +++ b/docs/architecture/tape-layering/tasks.md @@ -9,11 +9,11 @@ ## Behavior Characterization -- [ ] Split the monolithic Tape suite by reconciliation, recall, lineage, view/replay, and fork +- [x] Split the monolithic Tape suite by reconciliation, recall, lineage, view/replay, and fork behavior. -- [ ] Preserve every existing assertion and environment skip gate during the mechanical split. -- [ ] Add transaction, reconciliation-order, and projection-fallback characterization coverage. -- [ ] Review and commit the characterization slice. +- [x] Preserve every existing assertion and environment skip gate during the mechanical split. +- [x] Add transaction, reconciliation-order, and projection-fallback characterization coverage. +- [x] Review and commit the characterization slice. ## Domain and Ports diff --git a/test/main/session/data/settings.test.ts b/test/main/session/data/settings.test.ts index 357e85514e..2773200a3e 100644 --- a/test/main/session/data/settings.test.ts +++ b/test/main/session/data/settings.test.ts @@ -303,6 +303,50 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { sqlitePresenter.close() }) + it('rolls back summary state when the matching tape anchor cannot be appended', () => { + const { sqlitePresenter, store } = createStore() + const originalState = { + summaryText: 'stable summary', + summaryCursorOrderSeq: 3, + summaryUpdatedAt: 50 + } + + store.create('s1', 'openai', 'gpt-4o') + store.updateSummaryState('s1', originalState) + sqlitePresenter.getDatabase().exec(` + CREATE TRIGGER fail_compaction_anchor + BEFORE INSERT ON deepchat_tape_entries + WHEN NEW.name = 'compaction/failing' + BEGIN + SELECT RAISE(ABORT, 'forced tape anchor failure'); + END; + `) + + expect(() => + store.compareAndSetSummaryState( + 's1', + originalState, + { + summaryText: 'must roll back', + summaryCursorOrderSeq: 6, + summaryUpdatedAt: 100 + }, + { + name: 'compaction/failing', + state: { + summary: 'must roll back', + cursorOrderSeq: 6 + } + } + ) + ).toThrow('forced tape anchor failure') + + expect(store.getSummaryState('s1')).toEqual(originalState) + expect(sqlitePresenter.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toBeUndefined() + + sqlitePresenter.close() + }) + it('uses reset anchors to invalidate older compaction anchors', () => { const { sqlitePresenter, store } = createStore() diff --git a/test/main/session/data/tape.test.ts b/test/main/session/data/tape.test.ts deleted file mode 100644 index cb5521f7cb..0000000000 --- a/test/main/session/data/tape.test.ts +++ /dev/null @@ -1,5531 +0,0 @@ -import { performance } from 'node:perf_hooks' -import { describe, expect, it, vi } from 'vitest' -import { buildContext } from '@/agent/deepchat/runtime/contextBuilder' -import { toAppSessionId } from '@/agent/shared/agentSessionIds' -import { SessionTape } from '@/session/data/tape' -import { buildEffectiveTapeView, searchEffectiveTapeRows } from '@/session/data/tapeEffectiveView' -import { - createTapeViewManifest, - type TapeViewManifestBuildInput -} from '@/session/data/tapeViewManifest' -import { - appendMessageRecordToTape, - appendMessageReplacementToTape, - appendMessageRetractionToTape, - appendToolFactsToTape -} from '@/session/data/tapeFacts' -import { buildRequestRefs } from '@/session/data/tapeViewManifest' -import { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' -import { - DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, - DeepChatTapeSearchProjectionTable -} from '@/session/data/tables/deepchatTapeSearchProjection' -import { DeepChatMemoryIngestionProjectionTable } from '@/memory/data/tables/deepchatMemoryIngestionProjection' -import { DeepChatMessagesTable } from '@/session/data/tables/deepchatMessages' -import { DeepChatMessageTracesTable } from '@/session/data/tables/deepchatMessageTraces' -import { DeepChatSessionsTable } from '@/session/data/tables/deepchatSessions' -import { NewSessionsTable } from '@/session/data/tables/newSessions' -import type { ChatMessageRecord } from '@shared/types/agent-interface' - -const sqliteModule = await import('better-sqlite3-multiple-ciphers').catch(() => null) -const Database = sqliteModule?.default -const DatabaseCtor = Database! -const sqliteSkipReason = 'skipped: better-sqlite3-multiple-ciphers is unavailable' -const requireNativeSqlite = process.env.DEEPCHAT_REQUIRE_NATIVE_SQLITE === '1' - -let sqliteAvailable = false -if (Database) { - try { - const smokeDb = new Database(':memory:') - smokeDb.close() - sqliteAvailable = true - } catch { - sqliteAvailable = false - } -} - -const itIfSqlite = sqliteAvailable - ? it - : requireNativeSqlite - ? (name: string, _test: () => unknown, timeout?: number) => - it( - name, - () => { - throw new Error(sqliteSkipReason) - }, - timeout - ) - : it.skip - -function createTapeTableMock() { - const entries: any[] = [] - let tapeIncarnationSequence = 0 - const table = { - ensureBootstrapAnchor: vi.fn((sessionId: string) => { - if ( - entries.some((entry) => entry.session_id === sessionId && entry.name === 'session/start') - ) { - return - } - table.appendAnchor({ - sessionId, - name: 'session/start', - source: { type: 'session', id: sessionId, seq: 0 }, - state: { owner: 'human' }, - meta: { tapeIncarnationId: `test-tape-${++tapeIncarnationSequence}` }, - idempotent: true - }) - }), - append: vi.fn((input: any) => { - const provenanceKey = - input.provenanceKey !== undefined - ? input.provenanceKey - : input.source - ? [ - input.source.type, - input.source.id, - input.source.seq ?? 0, - input.kind, - input.name ?? '' - ].join(':') - : null - const existing = input.idempotent - ? entries.find( - (entry) => - entry.session_id === input.sessionId && entry.provenance_key === provenanceKey - ) - : null - if (existing) { - return existing - } - const row = { - session_id: input.sessionId, - entry_id: - Math.max( - 0, - ...entries - .filter((entry) => entry.session_id === input.sessionId) - .map((entry) => entry.entry_id) - ) + 1, - kind: input.kind, - name: input.name ?? null, - source_type: input.source?.type ?? null, - source_id: input.source?.id ?? null, - source_seq: input.source?.seq ?? null, - provenance_key: provenanceKey, - payload_json: JSON.stringify(input.payload ?? {}), - meta_json: JSON.stringify(input.meta ?? {}), - created_at: input.createdAt ?? Date.now() - } - entries.push(row) - return row - }), - appendAnchor: vi.fn((input: any) => - table.append({ - ...input, - kind: 'anchor', - payload: { name: input.name, state: input.state } - }) - ), - appendEvent: vi.fn((input: any) => - table.append({ - ...input, - kind: 'event', - payload: { name: input.name, data: input.data } - }) - ), - runInTransaction: vi.fn((operation: () => unknown) => { - const snapshot = entries.map((entry) => ({ ...entry })) - try { - return operation() - } catch (error) { - entries.splice(0, entries.length, ...snapshot) - throw error - } - }), - getBySession: vi.fn((sessionId: string) => - entries.filter((entry) => entry.session_id === sessionId) - ), - getSubagentLineageEvents: vi.fn((sessionId: string) => - entries.filter( - (entry) => - entry.session_id === sessionId && - entry.kind === 'event' && - (entry.name === 'subagent/tape_linked' || entry.name === 'fork/merge') - ) - ), - getFirstEntriesBySessions: vi.fn((sessionIds: string[]) => - [...new Set(sessionIds)] - .flatMap((sessionId) => { - const first = entries - .filter((entry) => entry.session_id === sessionId) - .sort((left, right) => left.entry_id - right.entry_id)[0] - return first ? [first] : [] - }) - .sort((left, right) => left.session_id.localeCompare(right.session_id)) - ), - getBySessionUpToEntryId: vi.fn((sessionId: string, maxEntryId: number) => - entries.filter((entry) => entry.session_id === sessionId && entry.entry_id <= maxEntryId) - ), - getMaxEntryId: vi.fn((sessionId: string) => - Math.max( - 0, - ...entries.filter((entry) => entry.session_id === sessionId).map((entry) => entry.entry_id) - ) - ), - getMaxEntryIdsBySessions: vi.fn( - (sessionIds: string[]) => - new Map( - sessionIds.map((sessionId) => [ - sessionId, - Math.max( - 0, - ...entries - .filter((entry) => entry.session_id === sessionId) - .map((entry) => entry.entry_id) - ) - ]) - ) - ), - getLatestAnchor: vi.fn( - (sessionId: string) => - entries - .filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor') - .sort((left, right) => right.entry_id - left.entry_id)[0] - ), - getAnchors: vi.fn((sessionId: string, limit: number = 20) => - entries - .filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor') - .sort((left, right) => right.entry_id - left.entry_id) - .slice(0, Math.min(Math.max(Math.floor(limit), 1), 100)) - .reverse() - ), - getLatestSummaryAnchor: vi.fn( - (sessionId: string) => - entries - .filter( - (entry) => - entry.session_id === sessionId && - entry.kind === 'anchor' && - ['compaction/migrated_summary', 'compaction/manual', 'summary/reset'].includes( - entry.name - ) - ) - .sort((left, right) => right.entry_id - left.entry_id)[0] - ), - getByProvenanceKey: vi.fn((sessionId: string, provenanceKey: string) => - entries.find( - (entry) => entry.session_id === sessionId && entry.provenance_key === provenanceKey - ) - ), - countBySession: vi.fn( - (sessionId: string) => entries.filter((entry) => entry.session_id === sessionId).length - ), - countAnchorsBySession: vi.fn( - (sessionId: string) => - entries.filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor').length - ), - countEntriesAfter: vi.fn( - (sessionId: string, entryId: number) => - entries.filter((entry) => entry.session_id === sessionId && entry.entry_id > entryId).length - ), - search: vi.fn((sessionId: string, query: string, options: any = {}) => { - const normalizedQuery = query.trim() - if (!normalizedQuery) { - return [] - } - const limit = Number.isFinite(options.limit) ? Math.floor(options.limit) : 20 - return entries - .filter((entry) => entry.session_id === sessionId) - .filter( - (entry) => - entry.payload_json.includes(normalizedQuery) || - entry.meta_json.includes(normalizedQuery) || - entry.name?.includes(normalizedQuery) - ) - .filter((entry) => !options.kinds?.length || options.kinds.includes(entry.kind)) - .filter( - (entry) => - !Number.isFinite(options.startCreatedAt) || entry.created_at >= options.startCreatedAt - ) - .filter( - (entry) => - !Number.isFinite(options.endCreatedAt) || entry.created_at <= options.endCreatedAt - ) - .sort((left, right) => right.entry_id - left.entry_id) - .slice(0, Math.min(Math.max(limit, 1), 100)) - }), - searchEffectiveSourcesAtHeads: vi.fn((sources: any[], query: string, options: any = {}) => - sources - .flatMap((source) => - searchEffectiveTapeRows( - entries.filter( - (entry) => - entry.session_id === source.sessionId && entry.entry_id <= source.maxEntryId - ), - query, - { ...options, limit: 100 } - ) - ) - .sort( - (left, right) => - right.created_at - left.created_at || - left.session_id.localeCompare(right.session_id) || - right.entry_id - left.entry_id - ) - .slice(0, Math.min(Math.max(Math.floor(options.limit ?? 20), 1), 100)) - ), - getEffectiveContextRowsAtHead: vi.fn( - ( - source: any, - entryIds: number[], - options: { before: number; after: number; limit: number } - ) => { - const effectiveRows = buildEffectiveTapeView( - entries.filter( - (entry) => entry.session_id === source.sessionId && entry.entry_id <= source.maxEntryId - ), - { includePending: false } - ).rows - const indexesByEntryId = new Map( - effectiveRows.map((entry, index) => [entry.entry_id, index]) - ) - const indexes: number[] = [] - for (const entryId of entryIds) { - const index = indexesByEntryId.get(entryId) - if (index !== undefined) indexes.push(index) - } - for (const entryId of entryIds) { - const index = indexesByEntryId.get(entryId) - if (index === undefined) continue - for ( - let cursor = Math.max(0, index - options.before); - cursor <= Math.min(effectiveRows.length - 1, index + options.after); - cursor += 1 - ) { - if (cursor !== index) indexes.push(cursor) - } - } - return [...new Set(indexes)].slice(0, options.limit).map((index) => effectiveRows[index]) - } - ), - deleteBySession: vi.fn((sessionId: string) => { - for (let index = entries.length - 1; index >= 0; index -= 1) { - if (entries[index].session_id === sessionId) { - entries.splice(index, 1) - } - } - }) - } - return { table, entries } -} - -function createRecord(overrides: Partial): ChatMessageRecord { - return { - id: 'm1', - sessionId: 's1', - orderSeq: 1, - role: 'user', - content: JSON.stringify({ text: 'hello', files: [], links: [], search: false, think: false }), - status: 'sent', - isContextEdge: 0, - metadata: '{}', - traceCount: 0, - createdAt: 100, - updatedAt: 100, - ...overrides - } -} - -function createTraceRow(overrides: Record = {}) { - return { - id: 'trace-1', - message_id: 'a1', - session_id: 's1', - provider_id: 'openai', - model_id: 'gpt-4o', - request_seq: 1, - endpoint: 'https://api.openai.test/v1/chat/completions', - headers_json: '{"authorization":"[redacted]"}', - body_json: '{"messages":[{"role":"user","content":"hello"}]}', - truncated: 0, - created_at: 300, - ...overrides - } -} - -function createMessageRow(overrides: Record = {}) { - return { - id: 'a1', - session_id: 's1', - order_seq: 2, - role: 'assistant', - content: '[{"type":"content","content":"done","status":"success"}]', - status: 'sent', - is_context_edge: 0, - metadata: '{"totalTokens":10}', - created_at: 200, - updated_at: 300, - ...overrides - } -} - -function createObservationManifest( - overrides: Partial[0]> = {} -) { - return createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user', content: 'hello' }], - tools: [], - latestEntryId: 0, - anchorEntryIds: [], - included: [], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: true, - assembledAt: 200, - ...overrides - }) -} - -function createTapeService( - table: unknown, - traceRows: Array> = [], - messageRows: Array> = [] -) { - return new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: { - deleteBySession: vi.fn(), - getByEntryIds: vi.fn().mockReturnValue([]) - }, - deepchatMessageTracesTable: { - listByMessageId: vi.fn((messageId: string) => - traceRows.filter((row) => row.message_id === messageId) - ) - }, - deepchatMessagesTable: { - get: vi.fn((messageId: string) => messageRows.find((row) => row.id === messageId)) - }, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) -} - -function createLinkedTapeService( - table: unknown, - sessions: Array<{ - id: string - session_kind: 'regular' | 'subagent' - parent_session_id: string | null - }>, - projectionTable?: unknown -) { - const sessionById = new Map(sessions.map((session) => [session.id, session])) - return { - service: new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - newSessionsTable: { - get: vi.fn((sessionId: string) => sessionById.get(sessionId)), - getMany: vi.fn((sessionIds: string[]) => - sessionIds.flatMap((sessionId) => { - const session = sessionById.get(sessionId) - return session ? [session] : [] - }) - ) - }, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any), - sessionById - } -} - -function createSubagentLinkInput(parentSessionId: string, childSessionId: string) { - return { - parentSessionId, - childSessionId, - runId: `run-${childSessionId}`, - taskId: `task-${childSessionId}`, - slotId: 'reviewer', - taskTitle: `Review ${childSessionId}`, - outcome: 'completed' as const, - resultSummary: 'Done' - } -} - -function appendObservationIsolationFacts(table: unknown) { - const original = createRecord({ id: 'u1', orderSeq: 1, createdAt: 100, updatedAt: 100 }) - const edited = createRecord({ - id: 'u1', - orderSeq: 1, - content: JSON.stringify({ - text: 'edited', - files: [], - links: [], - search: false, - think: false - }), - createdAt: 100, - updatedAt: 150 - }) - const retracted = createRecord({ id: 'u2', orderSeq: 2, createdAt: 160, updatedAt: 160 }) - const pending = createRecord({ - id: 'a1', - orderSeq: 3, - role: 'assistant', - status: 'pending', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'pending', - timestamp: 200, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}' } - } - ]), - createdAt: 200, - updatedAt: 200 - }) - const final = createRecord({ - id: 'a1', - orderSeq: 3, - role: 'assistant', - status: 'sent', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 300, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'tape-result-secret' - } - } - ]), - metadata: '{"totalTokens":12}', - createdAt: 200, - updatedAt: 300 - }) - - appendMessageRecordToTape(table as any, original, 'live') - appendMessageReplacementToTape(table as any, edited, 'test_edit') - appendMessageRecordToTape(table as any, retracted, 'live') - appendMessageRetractionToTape(table as any, retracted, 'test_delete') - appendMessageRecordToTape(table as any, pending, 'live') - appendMessageRecordToTape(table as any, final, 'live') - - return { edited, final } -} - -function stripObservationPayloadOptIns(value: T): T { - const copy = structuredClone(value) as any - const stripEntryPayloads = (entries: any[] | undefined) => { - for (const entry of entries ?? []) { - delete entry.payload - delete entry.meta - } - } - - if (copy.request?.state === 'manifest_bound') { - stripEntryPayloads(copy.request.replay.entries) - delete copy.request.replay.hashes.sliceHash - if (copy.request.replay.trace) { - delete copy.request.replay.trace.headersJson - delete copy.request.replay.trace.bodyJson - } - } else if (copy.request?.trace) { - delete copy.request.trace.headersJson - delete copy.request.trace.bodyJson - } - stripEntryPayloads(copy.output?.entries) - return copy -} - -function createSpies(names: string[]) { - return Object.fromEntries(names.map((name) => [name, vi.fn()])) as Record< - string, - ReturnType - > -} - -function trackMemoryPropertyAccess(target: T) { - const memoryPropertyAccess = vi.fn() - return { - memoryPropertyAccess, - presenter: new Proxy(target, { - get(value, property, receiver) { - if (typeof property === 'string' && /memory/i.test(property)) { - memoryPropertyAccess(property) - } - return Reflect.get(value, property, receiver) - } - }) - } -} - -function readObservationMatrix(service: SessionTape) { - return { - defaultObservation: service.readCausalObservationSlice('s1', 'a1'), - repeatedObservation: service.readCausalObservationSlice('s1', 'a1'), - explicitObservation: service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }), - optInObservation: service.readCausalObservationSlice('s1', 'a1', { - includeTapePayloads: true, - includeTracePayload: true - }), - traceOnlyObservation: service.readCausalObservationSlice('s1', 'a-trace') - } -} - -describe('SessionTape', () => { - it('backfills message and tool facts idempotently before returning tape records', () => { - const { table, entries } = createTapeTableMock() - const assistantBlocks = [ - { - type: 'tool_call', - status: 'success', - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } - } - ] - const records = [ - createRecord({ id: 'u1', orderSeq: 1 }), - createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify(assistantBlocks), - createdAt: 120, - updatedAt: 120 - }) - ] - const messageStore = { - getMessages: vi.fn().mockReturnValue(records) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const first = service.ensureSessionTapeReady('s1', messageStore as any) - const second = service.ensureSessionTapeReady('s1', messageStore as any) - - expect(first.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) - expect(second.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) - expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) - expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) - expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) - expect(entries.filter((entry) => entry.name === 'migration/backfill')).toHaveLength(1) - }) - - it('appends live tool facts through the stable recorder port idempotently', async () => { - const { table, entries } = createTapeTableMock() - const service = createTapeService(table) - const input = { - sessionId: toAppSessionId('s1'), - messageId: 'a1', - orderSeq: 2, - blockIndex: 0, - block: { - type: 'tool_call' as const, - status: 'success' as const, - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } - }, - provenance: { source: 'tool_call' as const, sourceId: 'a1:tc1', sequence: 0 } - } - - const first = await service.appendToolFact(input) - const second = await service.appendToolFact(input) - - expect(second).toEqual(first) - expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) - expect(JSON.parse(entries.find((entry) => entry.kind === 'tool_call').meta_json)).toEqual({ - source: 'live', - role: 'assistant', - status: 'success', - reason: 'tool_loop' - }) - }) - - it('reports info, search, and handoff within one session scope', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([ - createRecord({ id: 'u1' }), - createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { type: 'content', content: 'answer', status: 'success', timestamp: 101 } - ]), - metadata: JSON.stringify({ totalTokens: 9 }), - createdAt: 101, - updatedAt: 101 - }) - ]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - service.handoff('s1', 'phase_done', { summary: ' done ' }) - const handoffAnchor = entries.find((entry) => entry.name === 'handoff/phase_done') - - expect(service.info('s1')).toMatchObject({ - sessionId: 's1', - anchors: 2, - lastAnchor: 'handoff/phase_done', - lastTokenUsage: 9, - migrationState: 'ready' - }) - expect(JSON.parse(handoffAnchor.payload_json).state).toMatchObject({ - summary: 'done', - cursorOrderSeq: 3, - range: { - fromOrderSeq: 1, - toOrderSeq: 2 - }, - sourceMessageIds: ['u1', 'a1'] - }) - expect(service.search('s1', 'hello')).toHaveLength(1) - expect( - service.search('s1', 'hello', { kinds: ['message'], start: '1970-01-01T00:00:00.000Z' }) - ).toHaveLength(1) - expect(service.search('s1', 'hello', { kinds: ['anchor'] })).toHaveLength(0) - expect(service.search('s1', 'hello', { end: '99' })).toHaveLength(0) - expect(() => service.search('s1', 'hello', { start: 'not-a-date' })).toThrow( - 'start must be an ISO date/time or millisecond timestamp.' - ) - expect(service.anchors('s1')).toMatchObject([ - { sessionId: 's1', name: 'session/start' }, - { sessionId: 's1', name: 'handoff/phase_done' } - ]) - expect(service.anchors('s1', { limit: 1 })).toMatchObject([ - { sessionId: 's1', name: 'handoff/phase_done' } - ]) - expect(service.search('s2', 'hello')).toHaveLength(0) - }) - - it('projects fallback tape search into compact results and bounded context', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'Run the dev server', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.append({ - sessionId: 's1', - kind: 'tool_result', - name: 'shell', - source: { type: 'tool_result', id: 'm1:tc1', seq: 0 }, - payload: { - messageId: 'm1', - orderSeq: 2, - toolCallId: 'tc1', - command: 'pnpm run dev', - exitStatus: 1, - response: 'Command failed with EADDRINUSE in /tmp/deepchat.log' - }, - meta: { source: 'live', status: 'error' }, - createdAt: 110 - }) - - const hits = service.search('s1', 'pnpm run dev', { limit: 5 }) - expect(hits).toHaveLength(1) - expect(hits[0]).toMatchObject({ - kind: 'tool_result', - summary: expect.stringContaining('EADDRINUSE'), - refs: { - toolCallId: 'tc1', - commands: expect.arrayContaining(['pnpm run dev']), - filePaths: expect.arrayContaining(['/tmp/deepchat.log']), - errorCodes: expect.arrayContaining(['EADDRINUSE']), - exitStatus: 1 - } - }) - expect(hits[0]).not.toHaveProperty('payload') - expect(hits[0]).not.toHaveProperty('meta') - - const context = service.getContext('s1', [hits[0].entryId], { - before: 0, - after: 0, - maxBytesPerEntry: 12, - maxTotalBytes: 12 - }) - expect(context.matchedEntryIds).toEqual([hits[0].entryId]) - expect(context.entries[0]).toMatchObject({ - entryId: hits[0].entryId, - evidence: { truncated: true } - }) - expect(context.entries[0].evidence.bytes).toBeLessThanOrEqual(12) - expect(context.entries[0]).not.toHaveProperty('payload') - expect(context.entries[0]).not.toHaveProperty('meta') - - const exhaustedContext = service.getContext('s1', [hits[0].entryId], { - before: 0, - after: 0, - maxTotalBytes: 0 - }) - expect(exhaustedContext.entries).toEqual([]) - expect(exhaustedContext.matchedEntryIds).toEqual([]) - }) - - it('projects user message attachment metadata into search text and refs', () => { - const { table } = createTapeTableMock() - const projectionTable = { - isCurrent: vi.fn().mockReturnValue(false), - getSessionMeta: vi.fn().mockReturnValue(null), - getProjectedEntryIds: vi.fn().mockReturnValue([]), - appendSession: vi.fn(), - replaceSession: vi.fn(), - search: vi.fn().mockReturnValue([]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm-file', seq: 0 }, - payload: { - record: createRecord({ - id: 'm-file', - content: JSON.stringify({ - text: 'Please review the attachment', - files: [ - { - name: 'a.md', - path: '/tmp/a.md', - content: 'raw attachment body should not be projected', - metadata: { fileName: 'workspace-a.md' } - } - ], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - - service.search('s1', '/tmp/a.md', { limit: 5 }) - - expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) - const projectedRows = projectionTable.replaceSession.mock.calls[0][1] - expect(projectedRows[0]).toMatchObject({ - entryId: 1, - refs: { - filePaths: expect.arrayContaining(['/tmp/a.md']), - fileNames: expect.arrayContaining(['a.md', 'workspace-a.md']) - } - }) - expect(projectedRows[0].searchText).toContain('/tmp/a.md') - expect(projectedRows[0].searchText).toContain('a.md') - expect(projectedRows[0].searchText).toContain('workspace-a.md') - expect(projectedRows[0].searchText).not.toContain('raw attachment body should not be projected') - }) - - it('preserves relative file paths in projected refs', () => { - const { table } = createTapeTableMock() - let projectedRows: any[] = [] - const projectionTable = { - isCurrent: vi.fn().mockReturnValue(false), - getSessionMeta: vi.fn().mockReturnValue(null), - getProjectedEntryIds: vi.fn().mockReturnValue([]), - appendSession: vi.fn(), - replaceSession: vi.fn((_sessionId: string, rows: any[]) => { - projectedRows = rows - }), - search: vi.fn().mockReturnValue([]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm-relative', seq: 0 }, - payload: { - record: createRecord({ - id: 'm-relative', - content: JSON.stringify({ - text: 'Touched src/main/index.ts, ./lib/util.ts, ../shared/types.ts, test/main/foo/bar.ts, /usr/local/bin/deploy, and https://example.com/not-a-file', - files: [], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - - service.search('s1', 'src/main/index.ts', { limit: 5 }) - - expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) - const filePaths = projectedRows[0].refs.filePaths - expect(filePaths).toEqual( - expect.arrayContaining([ - 'src/main/index.ts', - './lib/util.ts', - '../shared/types.ts', - 'test/main/foo/bar.ts', - '/usr/local/bin/deploy' - ]) - ) - expect(filePaths).not.toContain('/main/index.ts') - expect(filePaths).not.toContain('/lib/util.ts') - expect(filePaths).not.toContain('/main/foo/bar.ts') - expect(filePaths).not.toContain('example.com/not-a-file') - }) - - it('rebuilds old tape projection versions before attachment path search', () => { - const { table } = createTapeTableMock() - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm-file', seq: 0 }, - payload: { - record: createRecord({ - id: 'm-file', - content: JSON.stringify({ - text: 'Please review the migrated attachment', - files: [ - { - name: 'legacy.md', - path: '/tmp/legacy.md', - content: 'raw migrated body must stay out of projection', - metadata: { fileName: 'legacy-workspace.md' } - } - ], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - let storedVersion = 1 - let storedMaxEntryId = 1 - let projectedRows: any[] = [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm-file', - sourceSeq: 0, - searchText: 'message/user Please review the migrated attachment', - summaryText: 'user: Please review the migrated attachment', - refs: { messageId: 'm-file' }, - createdAt: 100 - } - ] - const projectionTable = { - isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => { - return ( - storedVersion === DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION && - storedMaxEntryId === maxEntryId - ) - }), - getSessionMeta: vi.fn(() => ({ - projectionVersion: storedVersion, - maxEntryId: storedMaxEntryId - })), - getProjectedEntryIds: vi.fn().mockReturnValue([1]), - appendSession: vi.fn(), - replaceSession: vi.fn((_sessionId: string, rows: any[], maxEntryId: number) => { - projectedRows = rows - storedVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - storedMaxEntryId = maxEntryId - }), - search: vi.fn((_sessionId: string, query: string) => { - return projectedRows - .filter((row) => row.searchText.includes(query)) - .map((row) => ({ - session_id: row.sessionId, - entry_id: row.entryId, - kind: row.kind, - name: row.name, - source_type: row.sourceType, - source_id: row.sourceId, - source_seq: row.sourceSeq, - search_text: row.searchText, - summary_text: row.summaryText, - refs_json: JSON.stringify(row.refs), - created_at: row.createdAt, - score: 1 - })) - }) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const hits = service.search('s1', '/tmp/legacy.md', { limit: 5 }) - - expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) - expect(projectionTable.appendSession).not.toHaveBeenCalled() - expect(hits).toHaveLength(1) - expect(hits[0].refs).toMatchObject({ - filePaths: expect.arrayContaining(['/tmp/legacy.md']), - fileNames: expect.arrayContaining(['legacy.md', 'legacy-workspace.md']) - }) - expect(projectedRows[0].searchText).not.toContain( - 'raw migrated body must stay out of projection' - ) - }) - - it('prioritizes requested tape context entries before window entries under byte caps', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'before-1', seq: 0 }, - payload: { - record: createRecord({ - id: 'before-1', - content: JSON.stringify({ - text: 'before entry consumes the tiny byte budget first', - files: [], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'before-2', seq: 0 }, - payload: { - record: createRecord({ - id: 'before-2', - content: JSON.stringify({ - text: 'second before entry also appears earlier', - files: [], - links: [] - }), - createdAt: 110, - updatedAt: 110 - }) - }, - meta: { source: 'live', orderSeq: 2, role: 'user' }, - createdAt: 110 - }) - const target = table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'target', seq: 0 }, - payload: { - record: createRecord({ - id: 'target', - content: JSON.stringify({ - text: 'target-ok consumes the available budget', - files: [], - links: [] - }), - createdAt: 120, - updatedAt: 120 - }) - }, - meta: { source: 'live', orderSeq: 3, role: 'user' }, - createdAt: 120 - }) - - const context = service.getContext('s1', [target.entry_id], { - before: 2, - after: 0, - limit: 3, - maxBytesPerEntry: 18, - maxTotalBytes: 18 - }) - - expect(context.matchedEntryIds).toEqual([target.entry_id]) - expect(context.entries.map((entry) => entry.entryId)).toEqual([target.entry_id]) - expect(context.entries[0].evidence.text).toContain('target-ok') - }) - - it('binds tape projection LIKE fallback params for single and multi-term queries', () => { - const all = vi.fn().mockReturnValue([]) - const db = { - prepare: vi.fn((sql: string) => ({ - all: (...params: unknown[]) => all(sql, params), - get: vi.fn().mockReturnValue(undefined), - run: vi.fn() - })) - } - const projectionTable = new DeepChatTapeSearchProjectionTable(db as any) - ;(projectionTable as any).recoverSessionFts = vi.fn() - - projectionTable.search('s1', 'Redis', { limit: 5 }) - expect(all).toHaveBeenLastCalledWith( - expect.stringContaining('FROM deepchat_tape_search_projection'), - ['s1', '%Redis%', '%Redis%', '%Redis%', 5] - ) - - projectionTable.search('s1', 'Redis TTL', { limit: 5 }) - expect(all).toHaveBeenLastCalledWith( - expect.stringContaining('FROM deepchat_tape_search_projection'), - [ - 's1', - '%Redis TTL%', - '%Redis TTL%', - '%Redis TTL%', - '%Redis%', - '%Redis%', - '%Redis%', - '%TTL%', - '%TTL%', - '%TTL%', - 5 - ] - ) - - const oversizedQuery = Array.from({ length: 257 }, (_, index) => `term-${index}`).join(' ') - projectionTable.search('s1', oversizedQuery, { limit: 5 }) - expect(all).toHaveBeenLastCalledWith( - expect.stringContaining('FROM deepchat_tape_search_projection'), - ['s1', `%${oversizedQuery}%`, `%${oversizedQuery}%`, `%${oversizedQuery}%`, 5] - ) - }) - - it('uses current tape projection without loading full session rows', () => { - const { table } = createTapeTableMock() - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'Redis compact marker', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.getBySession.mockClear() - const projectionTable = { - isCurrent: vi.fn().mockReturnValue(true), - getSessionMeta: vi.fn(), - getProjectedEntryIds: vi.fn(), - appendSession: vi.fn(), - replaceSession: vi.fn(), - search: vi.fn().mockReturnValue([ - { - session_id: 's1', - entry_id: 1, - kind: 'message', - name: 'message/user', - source_type: 'message', - source_id: 'm1', - source_seq: 0, - search_text: 'Redis compact marker', - summary_text: 'Redis compact marker', - refs_json: '{"messageId":"m1"}', - created_at: 100, - score: -2 - } - ]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const hits = service.search('s1', 'Redis compact', { limit: 5 }) - - expect(table.getMaxEntryId).toHaveBeenCalledWith('s1') - expect(table.getBySession).not.toHaveBeenCalled() - expect(projectionTable.search).toHaveBeenCalledWith( - 's1', - 'Redis compact', - expect.objectContaining({ limit: 5 }) - ) - expect(projectionTable.appendSession).not.toHaveBeenCalled() - expect(projectionTable.replaceSession).not.toHaveBeenCalled() - expect(hits[0]).toMatchObject({ - entryId: 1, - kind: 'message', - summary: 'Redis compact marker', - refs: { messageId: 'm1' }, - score: -2 - }) - expect(hits[0]).not.toHaveProperty('payload') - expect(hits[0]).not.toHaveProperty('meta') - }) - - it('falls back to effective tape search when projection search throws', () => { - const { table } = createTapeTableMock() - const projectionTable = { - isCurrent: vi.fn().mockReturnValue(true), - search: vi.fn(() => { - throw new Error('projection failed') - }) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'Redis fallback marker', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - - const hits = service.search('s1', 'Redis fallback', { limit: 5 }) - expect(hits).toHaveLength(1) - expect(hits[0]).toMatchObject({ - kind: 'message', - summary: expect.stringContaining('Redis fallback') - }) - expect(hits[0]).not.toHaveProperty('payload') - expect(hits[0]).not.toHaveProperty('meta') - }) - - it('appends tape projection rows when the previous projection is an effective prefix', () => { - const { table } = createTapeTableMock() - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'first redis', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm2', seq: 0 }, - payload: { - record: createRecord({ - id: 'm2', - content: JSON.stringify({ text: 'second vue', files: [], links: [] }), - createdAt: 110, - updatedAt: 110 - }) - }, - meta: { source: 'live', orderSeq: 2, role: 'user' }, - createdAt: 110 - }) - const projectionTable = { - isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => maxEntryId === 1), - getSessionMeta: vi.fn().mockReturnValue({ projectionVersion: 1, maxEntryId: 1 }), - getProjectedEntryIds: vi.fn().mockReturnValue([1]), - appendSession: vi.fn(), - replaceSession: vi.fn(), - search: vi.fn().mockReturnValue([]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.search('s1', 'vue', { limit: 5 }) - - expect(projectionTable.appendSession).toHaveBeenCalledTimes(1) - expect(projectionTable.appendSession.mock.calls[0][1].map((row: any) => row.entryId)).toEqual([ - 2 - ]) - expect(projectionTable.replaceSession).not.toHaveBeenCalled() - }) - - it('rebuilds tape projection when projected entry ids are not an effective prefix', () => { - const { table } = createTapeTableMock() - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'redis', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - const projectionTable = { - isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => maxEntryId === 0), - getSessionMeta: vi.fn().mockReturnValue({ projectionVersion: 1, maxEntryId: 0 }), - getProjectedEntryIds: vi.fn().mockReturnValue([99]), - appendSession: vi.fn(), - replaceSession: vi.fn(), - search: vi.fn().mockReturnValue([]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.search('s1', 'redis', { limit: 5 }) - - expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) - expect(projectionTable.appendSession).not.toHaveBeenCalled() - }) - - it('does not run LIKE fallback when FTS fills the tape projection search limit', () => { - const all = vi.fn((sql: string, _params: unknown[]) => - sql.includes('deepchat_tape_search_fts') - ? [ - { - session_id: 's1', - entry_id: 1, - kind: 'message', - name: 'message/user', - source_type: 'message', - source_id: 'm1', - source_seq: 0, - search_text: 'Redis TTL', - summary_text: 'Redis TTL', - refs_json: '{}', - created_at: 100, - score: -1 - } - ] - : [] - ) - const db = { - exec: vi.fn(() => { - throw new Error('unexpected FTS ensure') - }), - prepare: vi.fn((sql: string) => ({ - all: (...params: unknown[]) => all(sql, params), - get: (..._params: unknown[]) => { - if ( - sql.includes('deepchat_tape_search_projection_meta') || - sql.includes('deepchat_tape_search_fts_meta') - ) { - return { projection_version: 1, max_entry_id: 1 } - } - return undefined - }, - run: vi.fn() - })), - transaction: vi.fn((callback: () => void) => callback) - } - const projectionTable = new DeepChatTapeSearchProjectionTable(db as any) - ;(projectionTable as any).ftsReady = true - - const hits = projectionTable.search('s1', 'Redis', { limit: 1 }) - - expect(hits).toHaveLength(1) - expect(db.exec).not.toHaveBeenCalled() - const ftsCall = all.mock.calls.find(([sql]) => String(sql).includes('deepchat_tape_search_fts')) - expect(String(ftsCall?.[0])).toContain('deepchat_tape_search_fts.session_id = ?') - expect((ftsCall?.[1] as unknown[]).filter((param) => param === 's1')).toHaveLength(2) - expect( - vi.mocked(db.prepare).mock.calls.some(([sql]) => String(sql).includes('NULL AS score')) - ).toBe(false) - }) - - it('queries memory view manifests by agent without expanding session ids', () => { - const all = vi.fn().mockReturnValue([]) - const db = { - prepare: vi.fn((sql: string) => ({ - all: (...params: unknown[]) => all(sql, params) - })) - } - const table = new DeepChatTapeEntriesTable(db as any) - - table.listMemoryViewManifestAnchorsByAgent('agent-a', { - sessionId: 's-1', - messageId: 'msg-1', - limit: 7 - }) - - expect(all).toHaveBeenCalledWith( - expect.stringContaining('INNER JOIN new_sessions AS sessions'), - ['agent-a', 's-1', 'msg-1', 7] - ) - const sql = String(all.mock.calls[0][0]) - expect(sql).not.toContain(' IN (') - expect(sql).toContain('sessions.agent_id = ?') - expect(sql).toContain('tape.session_id = ?') - expect(sql).toContain("json_extract(tape.meta_json, '$.messageId') = ?") - }) - - it('keeps linked raw search candidate-bounded instead of materializing complete Tapes', () => { - const all = vi.fn().mockReturnValue([]) - const db = { - prepare: vi.fn((sql: string) => ({ - all: (...params: unknown[]) => all(sql, params) - })) - } - const table = new DeepChatTapeEntriesTable(db as any) - - table.searchEffectiveSourcesAtHeads( - [ - { sessionId: 'child-b', maxEntryId: 20 }, - { sessionId: 'child-a', maxEntryId: 10 } - ], - 'needle', - { limit: 5 } - ) - - const sql = String(all.mock.calls[0][0]) - const params = all.mock.calls[0][1] - expect(sql).toContain('FROM deepchat_tape_entries AS candidate') - expect(sql).toContain('candidate.entry_id <= source.max_entry_id') - expect(sql).toContain('FROM deepchat_tape_entries AS later_message') - expect(sql).toContain( - "typeof(json_extract(later_message.payload_json, '$.record.content')) = 'text'" - ) - expect(sql).not.toContain('bounded_rows AS') - expect(params[0]).toBe( - JSON.stringify([ - { sessionId: 'child-a', maxEntryId: 10 }, - { sessionId: 'child-b', maxEntryId: 20 } - ]) - ) - expect(params.at(-1)).toBe(5) - }) - - it('reads exact linked projections without invoking FTS recovery or writes', () => { - const run = vi.fn() - const exec = vi.fn() - const reads: Array<{ sql: string; params: unknown[] }> = [] - const prepare = vi.fn((sql: string) => ({ - all: (...params: unknown[]) => { - reads.push({ sql, params }) - if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { - return [{ session_id: 'child', max_entry_id: 2 }] - } - if (sql.includes('SELECT projection.*, NULL AS score')) { - return [ - { - session_id: 'child', - entry_id: 2, - kind: 'event', - name: 'child/result', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'projection needle', - summary_text: 'projection needle', - refs_json: '{}', - created_at: 100, - score: null - } - ] - } - return [] - }, - run - })) - const projectionTable = new DeepChatTapeSearchProjectionTable({ prepare, exec } as any) - - const result = projectionTable.searchSourcesReadOnly( - [{ sessionId: 'child', maxEntryId: 2 }], - 'projection needle', - { limit: 5 } - ) - - expect(result).toMatchObject({ - coveredSources: [{ sessionId: 'child', maxEntryId: 2 }], - rows: [{ session_id: 'child', entry_id: 2 }] - }) - expect(exec).not.toHaveBeenCalled() - expect(run).not.toHaveBeenCalled() - expect(prepare.mock.calls.map(([sql]) => String(sql)).join('\n')).not.toContain( - 'deepchat_tape_search_fts_meta' - ) - expect( - reads.find((read) => read.sql.includes('SELECT projection.*, NULL AS score'))?.params.at(-1) - ).toBe(5) - }) - - it('returns no ranked rows when linked projection coverage is only partial', () => { - const reads: string[] = [] - const prepare = vi.fn((sql: string) => ({ - all: () => { - reads.push(sql) - if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { - return [{ session_id: 'child-a', max_entry_id: 2 }] - } - return [] - } - })) - const projectionTable = new DeepChatTapeSearchProjectionTable({ - prepare, - exec: vi.fn() - } as any) - - const result = projectionTable.searchSourcesReadOnly( - [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 3 } - ], - 'projection needle', - { limit: 5 } - ) - - expect(result).toEqual({ - coveredSources: [{ sessionId: 'child-a', maxEntryId: 2 }], - rows: [] - }) - expect( - reads.some((sql) => sql.includes('FROM deepchat_tape_search_projection AS projection')) - ).toBe(false) - expect(reads.some((sql) => sql.includes('FROM deepchat_tape_search_fts'))).toBe(false) - }) - - it('uses one LIKE ranking when linked FTS freshness is only partial', () => { - const reads: Array<{ sql: string; params: unknown[] }> = [] - const prepare = vi.fn((sql: string) => ({ - all: (...params: unknown[]) => { - reads.push({ sql, params }) - if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { - return [ - { session_id: 'child-a', max_entry_id: 2 }, - { session_id: 'child-b', max_entry_id: 3 } - ] - } - if (sql.includes('deepchat_tape_search_fts_meta AS meta')) { - return [{ session_id: 'child-a', max_entry_id: 2 }] - } - if (sql.includes('SELECT projection.*, NULL AS score')) { - return [ - { - session_id: 'child-b', - entry_id: 3, - kind: 'event', - name: 'child/result', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'projection needle', - summary_text: 'projection needle', - refs_json: '{}', - created_at: 200, - score: null - } - ] - } - return [] - } - })) - const projectionTable = new DeepChatTapeSearchProjectionTable({ - prepare, - exec: vi.fn() - } as any) - ;(projectionTable as any).ftsReady = true - - const sources = [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 3 } - ] - const result = projectionTable.searchSourcesReadOnly(sources, 'projection needle', { limit: 5 }) - - expect(result.rows).toMatchObject([{ session_id: 'child-b', entry_id: 3, score: null }]) - expect(reads.some(({ sql }) => sql.includes('bm25(deepchat_tape_search_fts)'))).toBe(false) - expect( - reads.find(({ sql }) => sql.includes('SELECT projection.*, NULL AS score'))?.params[0] - ).toBe(JSON.stringify(sources)) - }) - - itIfSqlite( - `keeps projected and raw linked multi-term search aligned${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - table.createTable() - projectionTable.createTable() - table.ensureBootstrapAnchor('child') - const entry = table.appendEvent({ - sessionId: 'child', - name: 'child/result', - data: { text: 'alpha separated by several words before beta' }, - createdAt: 100 - }) - const sources = [{ sessionId: 'child', maxEntryId: entry.entry_id }] - projectionTable.replaceSession( - 'child', - [ - { - sessionId: 'child', - entryId: entry.entry_id, - kind: 'event', - name: 'child/result', - sourceType: null, - sourceId: null, - sourceSeq: null, - searchText: 'alpha separated by several words before beta', - summaryText: 'alpha separated by several words before beta', - refs: {}, - createdAt: 100 - } - ], - entry.entry_id - ) - - const projected = projectionTable.searchSourcesReadOnly(sources, 'alpha beta', { limit: 5 }) - const raw = table.searchEffectiveSourcesAtHeads(sources, 'alpha beta', { limit: 5 }) - - expect(projected.coveredSources).toEqual(sources) - expect(raw.map((row) => [row.session_id, row.entry_id])).toEqual( - projected.rows.map((row) => [row.session_id, row.entry_id]) - ) - expect(raw).toHaveLength(1) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `queries effective linked sources and context at frozen heads${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - table.ensureBootstrapAnchor('child-a') - table.ensureBootstrapAnchor('child-b') - table.appendEvent({ - sessionId: 'child-a', - name: 'child/result', - data: { text: 'native linked needle A' }, - createdAt: 100 - }) - table.appendEvent({ - sessionId: 'child-b', - name: 'child/result', - data: { text: 'native linked needle B' }, - createdAt: 200 - }) - table.appendEvent({ - sessionId: 'child-a', - name: 'child/late', - data: { text: 'native linked needle late' }, - createdAt: 300 - }) - - const sources = [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 2 } - ] - expect( - table.searchEffectiveSourcesAtHeads(sources, 'native linked needle', { limit: 1 }) - ).toMatchObject([{ session_id: 'child-b', entry_id: 2 }]) - expect( - table - .searchEffectiveSourcesAtHeads(sources, 'native linked needle', { limit: 10 }) - .map((row) => [row.session_id, row.entry_id]) - ).toEqual([ - ['child-b', 2], - ['child-a', 2] - ]) - expect( - table - .getEffectiveContextRowsAtHead({ sessionId: 'child-a', maxEntryId: 2 }, [2], { - before: 1, - after: 5, - limit: 10 - }) - .map((row) => row.entry_id) - ).toEqual([2, 1]) - - table.ensureBootstrapAnchor('child-message') - const original = createRecord({ - id: 'linked-message', - sessionId: 'child-message', - content: JSON.stringify({ text: 'old linked marker', files: [], links: [] }) - }) - const replacement = { - ...original, - content: JSON.stringify({ text: 'new linked marker', files: [], links: [] }), - updatedAt: 200 - } - appendMessageRecordToTape(table, original, 'live') - appendMessageReplacementToTape(table, replacement, 'native_edit') - const replacementHead = table.getMaxEntryId('child-message') - expect( - table.searchEffectiveSourcesAtHeads( - [{ sessionId: 'child-message', maxEntryId: replacementHead }], - 'old linked marker' - ) - ).toEqual([]) - const replacementHits = table.searchEffectiveSourcesAtHeads( - [{ sessionId: 'child-message', maxEntryId: replacementHead }], - 'new linked marker' - ) - expect(replacementHits).toHaveLength(1) - expect( - table - .getEffectiveContextRowsAtHead( - { sessionId: 'child-message', maxEntryId: replacementHead }, - [replacementHits[0].entry_id], - { before: 0, after: 0, limit: 5 } - ) - .map((row) => row.entry_id) - ).toEqual([replacementHits[0].entry_id]) - - table.append({ - sessionId: 'child-message', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: original.id, seq: 0 }, - provenanceKey: null, - payload: { - record: { - id: original.id, - sessionId: original.sessionId, - orderSeq: original.orderSeq, - role: original.role, - status: 'sent' - } - }, - meta: { source: 'malformed_import' } - }) - expect( - table.searchEffectiveSourcesAtHeads( - [ - { - sessionId: 'child-message', - maxEntryId: table.getMaxEntryId('child-message') - } - ], - 'new linked marker' - ) - ).toMatchObject([{ entry_id: replacementHits[0].entry_id }]) - - appendMessageRetractionToTape(table, replacement, 'native_delete') - expect( - table.searchEffectiveSourcesAtHeads( - [ - { - sessionId: 'child-message', - maxEntryId: table.getMaxEntryId('child-message') - } - ], - 'new linked marker' - ) - ).toEqual([]) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `searches exact frozen projections without repairing a later child tail${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - projectionTable.replaceSession( - 'child', - [ - { - sessionId: 'child', - entryId: 2, - kind: 'event', - name: 'child/result', - sourceType: null, - sourceId: null, - sourceSeq: null, - searchText: 'native frozen projection needle', - summaryText: 'native frozen projection needle', - refs: {}, - createdAt: 100 - } - ], - 2 - ) - const before = db - .prepare( - `SELECT projection_version, max_entry_id, updated_at - FROM deepchat_tape_search_projection_meta - WHERE session_id = ?` - ) - .get('child') - - const result = projectionTable.searchSourcesReadOnly( - [{ sessionId: 'child', maxEntryId: 2 }], - 'native frozen projection needle', - { limit: 5 } - ) - - expect(result.coveredSources).toEqual([{ sessionId: 'child', maxEntryId: 2 }]) - expect(result.rows).toMatchObject([{ session_id: 'child', entry_id: 2 }]) - expect( - projectionTable.searchSourcesReadOnly( - [{ sessionId: 'child', maxEntryId: 3 }], - 'native frozen projection needle', - { limit: 5 } - ) - ).toEqual({ rows: [], coveredSources: [] }) - expect( - db - .prepare( - `SELECT projection_version, max_entry_id, updated_at - FROM deepchat_tape_search_projection_meta - WHERE session_id = ?` - ) - .get('child') - ).toEqual(before) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `filters stale FTS rows through the base projection after restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 2, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'current', - sourceSeq: 0, - searchText: 'current Redis marker', - summaryText: 'current Redis marker', - refs: { messageId: 'current' }, - createdAt: 200 - } - ], - 2 - ) - db.prepare( - `INSERT INTO deepchat_tape_search_fts ( - search_text, - name, - session_id, - entry_id, - kind, - source_type, - source_id, - source_seq, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - 'stale removed marker', - 'message/user', - 's1', - 1, - 'message', - 'message', - 'old', - 0, - 'stale removed marker', - '{"messageId":"old"}', - 100 - ) - - const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - restartedProjectionTable.createTable() - - expect(restartedProjectionTable.isCurrent('s1', 2)).toBe(true) - expect(restartedProjectionTable.search('s1', 'stale removed marker', { limit: 5 })).toEqual( - [] - ) - expect( - restartedProjectionTable.search('s1', 'current Redis marker', { limit: 5 })[0] - ).toMatchObject({ - entry_id: 2, - refs_json: '{"messageId":"current"}' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `recovers same-entry stale FTS after a base-only projection write and restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm1', - sourceSeq: 0, - searchText: 'old durable marker', - summaryText: 'old durable marker', - refs: { messageId: 'm1' }, - createdAt: 100 - } - ], - 1 - ) - - projectionTable.disableFtsForTesting() - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm1', - sourceSeq: 0, - searchText: 'new durable marker', - summaryText: 'new durable marker', - refs: { messageId: 'm1' }, - createdAt: 100 - } - ], - 1 - ) - db.prepare( - `INSERT INTO deepchat_tape_search_fts ( - search_text, - name, - session_id, - entry_id, - kind, - source_type, - source_id, - source_seq, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - 'old durable marker', - 'message/user', - 's1', - 1, - 'message', - 'message', - 'm1', - 0, - 'old durable marker', - '{"messageId":"m1"}', - 100 - ) - - const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - restartedProjectionTable.createTable() - - expect(restartedProjectionTable.isCurrent('s1', 1)).toBe(true) - expect(restartedProjectionTable.search('s1', 'old durable marker', { limit: 5 })).toEqual( - [] - ) - expect( - restartedProjectionTable.search('s1', 'new durable marker', { limit: 5 })[0] - ).toMatchObject({ - entry_id: 1, - search_text: 'new durable marker' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `rebuilds FTS during append when previous FTS meta is missing after restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.disableFtsForTesting() - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'old', - sourceSeq: 0, - searchText: 'old append marker', - summaryText: 'old append marker', - refs: { messageId: 'old' }, - createdAt: 100 - } - ], - 1 - ) - - const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - restartedProjectionTable.createTable() - restartedProjectionTable.appendSession( - 's1', - [ - { - sessionId: 's1', - entryId: 2, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'new', - sourceSeq: 0, - searchText: 'new append marker', - summaryText: 'new append marker', - refs: { messageId: 'new' }, - createdAt: 200 - } - ], - 2 - ) - - expect(restartedProjectionTable.isCurrent('s1', 2)).toBe(true) - expect( - restartedProjectionTable.search('s1', 'old append marker', { limit: 1 })[0] - ).toMatchObject({ - entry_id: 1, - refs_json: '{"messageId":"old"}' - }) - expect( - restartedProjectionTable.search('s1', 'new append marker', { limit: 1 })[0] - ).toMatchObject({ - entry_id: 2, - refs_json: '{"messageId":"new"}' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `rebuilds migrated tape FTS when freshness meta is excluded${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'old', - sourceSeq: 0, - searchText: 'old migrated marker', - summaryText: 'old migrated marker', - refs: { messageId: 'old' }, - createdAt: 100 - } - ], - 1 - ) - db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run('s1') - db.prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?').run('s1') - - const migratedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - migratedProjectionTable.createTable() - migratedProjectionTable.appendSession( - 's1', - [ - { - sessionId: 's1', - entryId: 2, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'new', - sourceSeq: 0, - searchText: 'new migrated marker', - summaryText: 'new migrated marker', - refs: { messageId: 'new' }, - createdAt: 200 - } - ], - 2 - ) - - expect(migratedProjectionTable.isCurrent('s1', 2)).toBe(true) - expect( - migratedProjectionTable.search('s1', 'old migrated marker', { limit: 1 })[0] - ).toMatchObject({ - entry_id: 1, - refs_json: '{"messageId":"old"}' - }) - expect( - migratedProjectionTable.search('s1', 'new migrated marker', { limit: 1 })[0] - ).toMatchObject({ - entry_id: 2, - refs_json: '{"messageId":"new"}' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `keeps common-term FTS searches scoped and bounded on large session sets${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - - for (let index = 0; index < 180; index += 1) { - const sessionId = `s-${index}` - const rows = Array.from({ length: 8 }, (_, offset) => ({ - sessionId, - entryId: offset + 1, - kind: 'message' as const, - name: 'message/user', - sourceType: 'message' as const, - sourceId: `m-${index}-${offset}`, - sourceSeq: offset, - searchText: `sharedcommon marker session-${index} row-${offset}`, - summaryText: `sharedcommon marker session-${index} row-${offset}`, - refs: { messageId: `m-${index}-${offset}` }, - createdAt: index * 10 + offset - })) - projectionTable.replaceSession(sessionId, rows, rows.length) - } - - const planRows = db - .prepare( - `EXPLAIN QUERY PLAN - SELECT projection.session_id, - projection.entry_id, - projection.kind, - projection.name, - projection.source_type, - projection.source_id, - projection.source_seq, - projection.search_text, - projection.summary_text, - projection.refs_json, - projection.created_at, - bm25(deepchat_tape_search_fts) AS score - FROM deepchat_tape_search_fts - INNER JOIN deepchat_tape_search_projection AS projection - ON projection.session_id = deepchat_tape_search_fts.session_id - AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) - AND projection.search_text = deepchat_tape_search_fts.search_text - WHERE deepchat_tape_search_fts MATCH ? - AND deepchat_tape_search_fts.session_id = ? - AND projection.session_id = ? - ORDER BY score ASC, projection.entry_id DESC - LIMIT ?` - ) - .all('"sharedcommon"', 's-42', 's-42', 5) as Array<{ detail: string }> - const plan = planRows.map((row) => row.detail).join('\n') - - expect(plan).toMatch(/VIRTUAL TABLE INDEX/i) - expect(plan).toMatch(/SEARCH projection USING (?:COVERING )?INDEX/i) - expect(plan).not.toMatch(/\bSCAN projection\b/i) - - const startedAt = performance.now() - const hits = projectionTable.search('s-42', 'sharedcommon', { limit: 5 }) - const elapsedMs = performance.now() - startedAt - - expect(hits).toHaveLength(5) - expect(hits.every((hit) => hit.session_id === 's-42')).toBe(true) - expect(elapsedMs).toBeLessThan(1500) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `does not trust same-entry stale FTS text even when stale FTS meta is current${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm1', - sourceSeq: 0, - searchText: 'new guarded marker', - summaryText: 'new guarded marker', - refs: { messageId: 'm1' }, - createdAt: 100 - } - ], - 1 - ) - db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run('s1') - db.prepare( - `INSERT INTO deepchat_tape_search_fts ( - search_text, - name, - session_id, - entry_id, - kind, - source_type, - source_id, - source_seq, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - 'old guarded marker', - 'message/user', - 's1', - 1, - 'message', - 'message', - 'm1', - 0, - 'old guarded marker', - '{"messageId":"m1"}', - 100 - ) - - const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - restartedProjectionTable.createTable() - - expect(restartedProjectionTable.isCurrent('s1', 1)).toBe(true) - expect(restartedProjectionTable.search('s1', 'old guarded marker', { limit: 5 })).toEqual( - [] - ) - expect( - restartedProjectionTable.search('s1', 'new guarded marker', { limit: 5 })[0] - ).toMatchObject({ - entry_id: 1, - search_text: 'new guarded marker' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `does not mark a tape projection current when FTS DML fails${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.dropFtsForTesting() - ;(projectionTable as any).ftsReady = true - - expect(() => - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm1', - sourceSeq: 0, - searchText: 'Redis TTL', - summaryText: 'Redis TTL', - refs: { messageId: 'm1' }, - createdAt: 100 - } - ], - 1 - ) - ).toThrow() - expect(projectionTable.isCurrent('s1', 1)).toBe(false) - expect(projectionTable.getProjectedEntryIds('s1')).toEqual([]) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `filters memory view manifests by message in SQLite before limit${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - data: { ignored: true }, - meta: { messageId: 'msg-old' }, - createdAt: 999 - }) - for (let index = 0; index < 505; index += 1) { - table.appendAnchor({ - sessionId: 's1', - name: 'memory/view_assembled', - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: index, - selected: [`m-${index}`], - dropped: [], - queryHash: `hash-${index}` - }, - meta: { messageId: `msg-${index}` }, - createdAt: index - }) - } - - const rows = table.listMemoryViewManifestAnchorsBySessions(['s1'], { - limit: 1, - messageId: 'msg-0' - }) - - expect(rows).toHaveLength(1) - expect(rows[0]).toMatchObject({ - kind: 'anchor', - name: 'memory/view_assembled', - created_at: 0 - }) - expect(JSON.parse(rows[0].meta_json)).toEqual({ messageId: 'msg-0' }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `queries memory view manifests for large agents without expanding session parameters${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const sessionTable = new NewSessionsTable(db) - const tapeTable = new DeepChatTapeEntriesTable(db) - sessionTable.createTable() - tapeTable.createTable() - for (let index = 0; index < 1200; index += 1) { - const sessionId = `s-${index}` - db.prepare( - `INSERT INTO new_sessions ( - id, - agent_id, - title, - project_dir, - is_pinned, - is_draft, - active_skills, - disabled_agent_tools, - subagent_enabled, - session_kind, - parent_session_id, - subagent_meta_json, - created_at, - updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - sessionId, - 'agent-a', - `Session ${index}`, - null, - 0, - 0, - '[]', - '[]', - index % 2 === 0 ? 0 : 1, - index % 2 === 0 ? 'regular' : 'subagent', - null, - null, - index, - index - ) - tapeTable.appendAnchor({ - sessionId, - name: 'memory/view_assembled', - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: index, - selected: [`m-${index}`], - dropped: [], - queryHash: `hash-${index}` - }, - meta: { messageId: `msg-${index}` }, - createdAt: index - }) - } - db.prepare( - `INSERT INTO new_sessions ( - id, - agent_id, - title, - project_dir, - is_pinned, - is_draft, - active_skills, - disabled_agent_tools, - subagent_enabled, - session_kind, - parent_session_id, - subagent_meta_json, - created_at, - updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - 'other-session', - 'other-agent', - 'Other', - null, - 0, - 0, - '[]', - '[]', - 0, - 'regular', - null, - null, - 9999, - 9999 - ) - tapeTable.appendAnchor({ - sessionId: 'other-session', - name: 'memory/view_assembled', - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 9999, - selected: ['other'], - dropped: [], - queryHash: 'other' - }, - meta: { messageId: 'msg-0' }, - createdAt: 9999 - }) - - const rows = tapeTable.listMemoryViewManifestAnchorsByAgent('agent-a', { - messageId: 'msg-0', - limit: 1 - }) - - expect(rows).toHaveLength(1) - expect(rows[0]).toMatchObject({ - session_id: 's-0', - kind: 'anchor', - name: 'memory/view_assembled', - created_at: 0 - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `searches a SQLite tape projection and expands compact context without raw payloads${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - table.createTable() - projectionTable.createTable() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'u1', seq: 0 }, - payload: { - record: createRecord({ - id: 'u1', - content: JSON.stringify({ - text: 'Check Redis TTL with /usr/local/bin/deploy --flag and error 42.', - files: [], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.append({ - sessionId: 's1', - kind: 'tool_result', - name: 'shell', - source: { type: 'tool_result', id: 'u1:tc1', seq: 0 }, - payload: { - messageId: 'u1', - orderSeq: 2, - toolCallId: 'tc1', - exitStatus: 42, - response: 'Exit code 42 in /tmp/deploy.log' - }, - meta: { source: 'live', status: 'error' }, - createdAt: 110 - }) - - const pathHits = service.search('s1', '/usr/local/bin/deploy', { limit: 5 }) - expect(pathHits).toHaveLength(1) - expect(pathHits[0]).toMatchObject({ - kind: 'message', - summary: expect.stringContaining('Redis TTL'), - refs: { - messageId: 'u1', - role: 'user', - filePaths: expect.arrayContaining(['/usr/local/bin/deploy']) - } - }) - expect(pathHits[0]).not.toHaveProperty('payload') - expect(pathHits[0]).not.toHaveProperty('meta') - expect(service.search('s1', 'Redis TTL', { limit: 5 }).map((hit) => hit.entryId)).toContain( - pathHits[0].entryId - ) - const errorHits = service.search('s1', '42', { kinds: ['tool_result'], limit: 5 }) - expect(errorHits[0]).toMatchObject({ - refs: { - toolCallId: 'tc1', - exitStatus: 42 - } - }) - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) - - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'u2', seq: 0 }, - payload: { - record: createRecord({ - id: 'u2', - orderSeq: 3, - content: JSON.stringify({ text: 'zoxide marker 简洁', files: [], links: [] }), - createdAt: 120, - updatedAt: 120 - }) - }, - meta: { source: 'live', orderSeq: 3, role: 'user' }, - createdAt: 120 - }) - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(false) - const rebuiltHits = service.search('s1', '简洁', { limit: 5 }) - expect(rebuiltHits.map((hit) => hit.refs?.messageId)).toContain('u2') - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) - if (projectionTable.hasFtsReadyForTesting()) { - projectionTable.dropFtsForTesting() - ;(projectionTable as any).ftsReady = true - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'u3', seq: 0 }, - payload: { - record: createRecord({ - id: 'u3', - orderSeq: 4, - content: JSON.stringify({ - text: 'fts recovery marker', - files: [], - links: [] - }), - createdAt: 130, - updatedAt: 130 - }) - }, - meta: { source: 'live', orderSeq: 4, role: 'user' }, - createdAt: 130 - }) - const recoveryHits = service.search('s1', 'fts recovery marker', { limit: 5 }) - expect(recoveryHits.map((hit) => hit.refs?.messageId)).toContain('u3') - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(false) - expect(projectionTable.hasFtsReadyForTesting()).toBe(false) - - const restoredHits = service.search('s1', 'fts recovery marker', { limit: 5 }) - expect(restoredHits.map((hit) => hit.refs?.messageId)).toContain('u3') - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) - expect(projectionTable.hasFtsReadyForTesting()).toBe(true) - } - - const context = service.getContext('s1', [pathHits[0].entryId], { - before: 0, - after: 1, - limit: 2, - maxBytesPerEntry: 24, - maxTotalBytes: 24 - }) - expect(context.matchedEntryIds).toEqual([pathHits[0].entryId]) - expect(context.entries[0]).toMatchObject({ - entryId: pathHits[0].entryId, - summary: expect.stringContaining('Redis TTL'), - evidence: { - truncated: true - } - }) - expect(context.entries[0].evidence.bytes).toBeLessThanOrEqual(24) - expect(context.entries[0]).not.toHaveProperty('payload') - expect(context.entries[0]).not.toHaveProperty('meta') - const limitedContext = service.getContext( - 's1', - [pathHits[0].entryId, errorHits[0].entryId], - { - before: 0, - after: 0, - limit: 1 - } - ) - expect(limitedContext.entries.map((entry) => entry.entryId)).toEqual([pathHits[0].entryId]) - expect(limitedContext.matchedEntryIds).toEqual([pathHits[0].entryId]) - } finally { - db.close() - } - } - ) - - it('keeps legacy context builder output stable after tape backfill projection', () => { - const { table } = createTapeTableMock() - const records = [ - createRecord({ id: 'u1', orderSeq: 1 }), - createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { type: 'content', content: 'Tool finished', status: 'success', timestamp: 120 }, - { - type: 'tool_call', - status: 'success', - timestamp: 121, - tool_call: { - id: 'tc1', - name: 'example_tool', - params: '{"foo":"bar"}', - response: 'All good' - } - } - ]), - createdAt: 120, - updatedAt: 121 - }) - ] - const legacyMessageStore = { - getMessages: vi.fn().mockReturnValue(records) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const legacyContext = buildContext( - 's1', - { text: 'next', files: [] }, - 'System', - 10000, - 4096, - legacyMessageStore as any - ) - const tapeReady = service.ensureSessionTapeReady('s1', legacyMessageStore as any) - const tapeOnlyStore = { - getMessages: vi.fn(() => { - throw new Error('buildContext must use provided tape history records') - }) - } - const tapeContext = buildContext( - 's1', - { text: 'next', files: [] }, - 'System', - 10000, - 4096, - tapeOnlyStore as any, - false, - { - historyRecords: tapeReady.historyRecords - } - ) - - expect(tapeContext).toEqual(legacyContext) - expect(tapeOnlyStore.getMessages).not.toHaveBeenCalled() - }) - - it('rejects handoff anchors without a non-empty summary before writing Tape state', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - expect(() => service.handoff('s1', 'phase_done', { summary: ' ' })).toThrow( - 'Tape handoff requires a non-empty summary.' - ) - expect(() => service.handoff('s1', 'phase_done', { reason: 'phase complete' } as any)).toThrow( - 'Tape handoff requires a non-empty summary.' - ) - - expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() - expect(table.appendAnchor).not.toHaveBeenCalled() - expect(entries).toEqual([]) - }) - - it('migrates legacy session summary into a tape anchor during backfill', () => { - const { table, entries } = createTapeTableMock() - const messageStore = { - getMessages: vi.fn().mockReturnValue([ - createRecord({ id: 'u1', orderSeq: 1 }), - createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([{ type: 'content', content: 'answer', status: 'success' }]) - }) - ]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { - getSummaryState: vi.fn().mockReturnValue({ - summary_text: 'legacy compacted state', - summary_cursor_order_seq: 3, - summary_updated_at: 200 - }) - } - } as any) - - service.ensureSessionTapeReady('s1', messageStore as any) - - const summaryAnchor = entries.find((entry) => entry.name === 'compaction/migrated_summary') - expect(summaryAnchor).toMatchObject({ - kind: 'anchor', - source_type: 'summary', - source_id: 'legacy-summary', - created_at: 200 - }) - expect(JSON.parse(summaryAnchor.payload_json).state).toMatchObject({ - summary: 'legacy compacted state', - cursorOrderSeq: 3, - sourceMessageIds: ['u1', 'a1'] - }) - }) - - it('stores and lists view manifests as idempotent tape events', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user', - source: 'tape', - reason: 'selected_history' - } - ], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - - const first = service.appendViewManifest(manifest) - const second = service.appendViewManifest(manifest) - - expect(second.entry_id).toBe(first.entry_id) - expect(entries.filter((entry) => entry.name === 'view/assembled')).toHaveLength(1) - expect(JSON.parse(first.meta_json)).toMatchObject({ - policy: 'legacy_context_v1', - policyVersion: 1 - }) - expect(service.listViewManifestsByMessage('s1', 'a1')).toMatchObject([ - { - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - entryId: first.entry_id, - manifest: { - hashes: { - manifestHash: manifest.hashes.manifestHash - }, - policy: 'legacy_context_v1', - policyVersion: 1, - included: [ - { - messageId: 'u1', - entryId: sourceMaps.entryIdByMessageId.get('u1') - } - ] - } - } - ]) - }) - - it('indexes effective tool facts so tool-loop manifests reference real entries', () => { - const { table } = createTapeTableMock() - const assistantRecord = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } - } - ]) - }) - appendToolFactsToTape(table as any, assistantRecord, 'live', 'tool_loop') - - const service = createTapeService(table) - const sourceMaps = service.getViewManifestSourceMaps('s1') - expect(sourceMaps.toolCallEntryIdByToolId.get('tc1')).toBeGreaterThan(0) - expect(sourceMaps.toolResultEntryIdByToolId.get('tc1')).toBeGreaterThan(0) - - const refs = buildRequestRefs( - [ - { role: 'system', content: 'system' }, - { - role: 'assistant', - content: '', - tool_calls: [ - { id: 'tc1', type: 'function', function: { name: 'search', arguments: '{"q":"x"}' } } - ] - }, - { role: 'tool', content: 'result', tool_call_id: 'tc1' } - ], - sourceMaps - ) - expect(refs).toMatchObject([ - { role: 'system', source: 'synthetic' }, - { - role: 'assistant', - source: 'tape', - reason: 'tool_loop_message', - entryId: sourceMaps.toolCallEntryIdByToolId.get('tc1') - }, - { - role: 'tool', - source: 'tape', - reason: 'tool_loop_message', - entryId: sourceMaps.toolResultEntryIdByToolId.get('tc1') - } - ]) - }) - - it('scopes tool source maps to the in-flight message so reused tool ids do not collide', () => { - const { table } = createTapeTableMock() - const blocks = (response: string) => - JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response } - } - ]) - appendToolFactsToTape( - table as any, - createRecord({ id: 'a1', orderSeq: 2, role: 'assistant', content: blocks('first') }), - 'live', - 'tool_loop' - ) - appendToolFactsToTape( - table as any, - createRecord({ id: 'a2', orderSeq: 4, role: 'assistant', content: blocks('second') }), - 'live', - 'tool_loop' - ) - - const service = createTapeService(table) - const scopedToA1 = service.getViewManifestSourceMaps('s1', 'a1') - const scopedToA2 = service.getViewManifestSourceMaps('s1', 'a2') - - expect(scopedToA1.toolCallEntryIdByToolId.get('tc1')).toBeLessThan( - scopedToA2.toolCallEntryIdByToolId.get('tc1')! - ) - expect(scopedToA1.toolResultEntryIdByToolId.get('tc1')).not.toBe( - scopedToA2.toolResultEntryIdByToolId.get('tc1') - ) - }) - - it('exports tool_call and tool_result entries in a tool-loop replay slice', () => { - const { table } = createTapeTableMock() - const assistantRecord = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } - } - ]) - }) - appendToolFactsToTape(table as any, assistantRecord, 'live', 'tool_loop') - - const service = createTapeService(table) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const messages = [ - { role: 'system' as const, content: 'system' }, - { - role: 'assistant' as const, - content: '', - tool_calls: [ - { id: 'tc1', type: 'function' as const, function: { name: 'search', arguments: '{}' } } - ] - }, - { role: 'tool' as const, content: 'result', tool_call_id: 'tc1' } - ] - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: 1, - messages, - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: buildRequestRefs(messages, sourceMaps), - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - service.appendViewManifest(manifest) - - const slice = service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }) - const kinds = slice?.entries.map((entry) => entry.kind) ?? [] - expect(kinds).toContain('tool_call') - expect(kinds).toContain('tool_result') - }) - - it('filters malformed view manifest rows when listing by message', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { - type: 'runtime_event', - id: 'a1', - seq: 1 - }, - data: { - manifest: { - schemaVersion: 1, - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - included: 'not-an-array' - } - } - }) - - expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) - }) - - it('normalizes legacy manifests without hashVersion to hashVersion 1', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - const legacyManifest: Record = { ...manifest } - delete legacyManifest.hashVersion - - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 99 }, - data: { manifest: legacyManifest } - }) - - const [record] = service.listViewManifestsByMessage('s1', 'a1') - expect(record.manifest.hashVersion).toBe(1) - }) - - it('filters manifests whose hashVersion is not a number', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 99 }, - data: { manifest: { ...manifest, hashVersion: '2' } } - }) - - expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) - }) - - it('annotates read records with hash integrity without dropping tampered manifests', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const baseInput = { - sessionId: 's1', - taskType: 'chat' as const, - policy: 'legacy_context_v1' as const, - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - } - const validManifest = createTapeViewManifest({ ...baseInput, messageId: 'a1', requestSeq: 1 }) - service.appendViewManifest(validManifest) - - const tamperedManifest = createTapeViewManifest({ - ...baseInput, - messageId: 'a2', - requestSeq: 1 - }) - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a2', seq: 99 }, - data: { manifest: { ...tamperedManifest, latestEntryId: tamperedManifest.latestEntryId + 1 } } - }) - - const [validRecord] = service.listViewManifestsByMessage('s1', 'a1') - const [tamperedRecord] = service.listViewManifestsByMessage('s1', 'a2') - expect(validRecord.integrity).toBe('valid') - expect(tamperedRecord).toBeDefined() - expect(tamperedRecord.integrity).toBe('invalid') - }) - - it('binds reconstruction lineage to the latest reconstruction anchor including handoffs', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('s1') - table.appendAnchor({ - sessionId: 's1', - name: 'compaction/manual', - source: { type: 'summary', id: 's1', seq: 1 }, - state: {} - }) - table.appendAnchor({ - sessionId: 's1', - name: 'handoff/phase_done', - source: { type: 'handoff', id: 's1', seq: 2 }, - state: {} - }) - table.appendAnchor({ - sessionId: 's1', - name: 'fork/merge', - source: { type: 'fork', id: 'child', seq: 3 }, - state: {} - }) - - const sourceMaps = service.getViewManifestSourceMaps('s1') - const entryIdByName = (name: string) => - table.getBySession('s1').find((entry: any) => entry.name === name)?.entry_id - - expect(sourceMaps.anchorEntryIds).toHaveLength(4) - expect(sourceMaps.reconstructionAnchorEntryId).toBe(entryIdByName('handoff/phase_done')) - expect(sourceMaps.reconstructionAnchorEntryIds).toEqual([ - sourceMaps.reconstructionAnchorEntryId - ]) - expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain( - entryIdByName('compaction/manual') - ) - expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('fork/merge')) - }) - - it('keeps memory anchors off the reconstruction lineage', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('s1') - table.appendAnchor({ - sessionId: 's1', - name: 'compaction/manual', - source: { type: 'summary', id: 's1', seq: 1 }, - state: {} - }) - table.appendAnchor({ - sessionId: 's1', - name: 'memory/extract', - source: { type: 'runtime_event', id: 's1', seq: 2 }, - state: { memoryIds: ['m1'], count: 1, reason: 'episodic' } - }) - table.appendAnchor({ - sessionId: 's1', - name: 'memory/reflect', - source: { type: 'runtime_event', id: 's1', seq: 3 }, - state: { reflectionIds: ['r1'], sourceMemoryIds: ['m1'], count: 1 } - }) - - const sourceMaps = service.getViewManifestSourceMaps('s1') - const entryIdByName = (name: string) => - table.getBySession('s1').find((entry: any) => entry.name === name)?.entry_id - - // Memory anchors are recorded on the tape for observability... - expect(sourceMaps.anchorEntryIds).toContain(entryIdByName('memory/extract')) - expect(sourceMaps.anchorEntryIds).toContain(entryIdByName('memory/reflect')) - // ...but never own the reconstruction cursor; only the summary anchor does. - expect(sourceMaps.reconstructionAnchorEntryId).toBe(entryIdByName('compaction/manual')) - expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('memory/extract')) - expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('memory/reflect')) - }) - - it('bounds replay slices to the selected view instead of pre-cursor history', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.reconstructionAnchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user', - source: 'tape', - reason: 'selected_history' - } - ], - excluded: [], - summaryCursor: { - summaryCursorOrderSeq: 100, - preCursorOrderSeqMin: 1, - preCursorOrderSeqMax: 99, - preCursorCount: 99 - }, - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 100, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - service.appendViewManifest(manifest) - - const slice = service.exportReplaySlice('s1', 'a1') - - expect(slice?.refs.excludedEntryIds).toEqual([]) - expect(slice?.refs.anchorEntryIds).toEqual(sourceMaps.reconstructionAnchorEntryIds) - expect(slice?.refs.anchorEntryIds).toHaveLength(1) - expect(slice?.manifestRecord.manifest.excludedRanges).toEqual([ - { fromOrderSeq: 1, toOrderSeq: 99, count: 99, reason: 'before_summary_cursor' } - ]) - expect(slice?.entries).toHaveLength(3) - }) - - it('exports replay slices with metadata-only payloads by default', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [createTraceRow()]) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user', - source: 'tape', - reason: 'selected_history' - } - ], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: true, - assembledAt: 200 - }) - const manifestEntry = service.appendViewManifest(manifest) - - const nowSpy = vi.spyOn(Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000) - const slice = service.exportReplaySlice('s1', 'a1') - const secondSlice = service.exportReplaySlice('s1', 'a1') - nowSpy.mockRestore() - - expect(slice).toMatchObject({ - schemaVersion: 1, - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - mode: 'trace_bound', - refs: { - manifestEntryId: manifestEntry.entry_id, - includedEntryIds: [sourceMaps.entryIdByMessageId.get('u1')], - anchorEntryIds: sourceMaps.anchorEntryIds - }, - hashes: { - manifestHash: manifest.hashes.manifestHash - } - }) - expect(slice?.hashes.sliceHash).toHaveLength(64) - expect(secondSlice?.hashes.sliceHash).toBe(slice?.hashes.sliceHash) - expect(secondSlice?.createdAt).toBe(2000) - expect(slice?.trace?.bodyHash).toHaveLength(64) - expect(slice?.trace?.bodyJson).toBeUndefined() - expect(slice?.entries.some((entry) => entry.entryId === manifestEntry.entry_id)).toBe(true) - expect( - slice?.entries.every((entry) => entry.payload === undefined && entry.meta === undefined) - ).toBe(true) - }) - - it('exports explicit replay request sequences with opt-in payloads', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [ - createTraceRow({ id: 'trace-1', request_seq: 1 }), - createTraceRow({ - id: 'trace-2', - request_seq: 2, - body_json: '{"messages":[{"role":"tool","content":"done"}]}' - }) - ]) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const baseManifestInput: TapeViewManifestBuildInput = { - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat' as const, - policy: 'legacy_context_v1' as const, - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user', - source: 'tape', - reason: 'selected_history' - } - ], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: true, - assembledAt: 200 - } - const firstManifest = createTapeViewManifest(baseManifestInput) - const secondManifest = createTapeViewManifest({ - ...baseManifestInput, - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 250 - }) - service.appendViewManifest(firstManifest) - service.appendViewManifest(secondManifest) - - const latest = service.exportReplaySlice('s1', 'a1') - const first = service.exportReplaySlice('s1', 'a1', { - requestSeq: 1, - includeTapePayloads: true, - includeTracePayload: true - }) - - expect(latest?.requestSeq).toBe(2) - expect(first?.requestSeq).toBe(1) - expect(first?.trace?.bodyJson).toContain('"hello"') - expect(first?.entries.some((entry) => entry.payload?.record)).toBe(true) - expect(first?.entries.some((entry) => entry.meta?.source === 'backfill')).toBe(true) - }) - - it('binds each replay slice to its own request seq, ignoring sentinel gap traces', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [ - createTraceRow({ - id: 'trace-req-1', - request_seq: 1, - body_json: '{"messages":[{"role":"user","content":"first-request"}]}' - }), - createTraceRow({ - id: 'trace-gap', - request_seq: 0, - endpoint: 'deepchat://interleaved-reasoning-gap', - body_json: '{"providerId":"openai"}' - }), - createTraceRow({ - id: 'trace-req-2', - request_seq: 2, - body_json: '{"messages":[{"role":"tool","content":"second-request"}]}' - }) - ]) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const baseManifestInput = { - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat' as const, - policy: 'legacy_context_v1' as const, - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user' as const, - source: 'tape' as const, - reason: 'selected_history' as const - } - ], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: true, - assembledAt: 200 - } - service.appendViewManifest(createTapeViewManifest(baseManifestInput)) - service.appendViewManifest( - createTapeViewManifest({ - ...baseManifestInput, - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 250 - }) - ) - - const first = service.exportReplaySlice('s1', 'a1', { - requestSeq: 1, - includeTracePayload: true - }) - const second = service.exportReplaySlice('s1', 'a1', { - requestSeq: 2, - includeTracePayload: true - }) - - expect(first?.trace?.bodyJson).toContain('first-request') - expect(second?.trace?.bodyJson).toContain('second-request') - }) - - it('returns null when exporting a replay slice without a manifest', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [createTraceRow()]) - - expect(service.exportReplaySlice('s1', 'a1')).toBeNull() - }) - - it('rejects non-positive replay request sequences', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [createTraceRow()]) - - expect(() => service.exportReplaySlice('s1', 'a1', { requestSeq: 0 })).toThrow( - 'requestSeq must be a positive integer.' - ) - }) - - it('joins an exact manifest and trace into a causal observation slice', () => { - const { table } = createTapeTableMock() - const assistant = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: '[{"type":"content","content":"done","status":"success"}]', - status: 'sent', - metadata: '{"totalTokens":10}', - createdAt: 200, - updatedAt: 300 - }) - appendMessageRecordToTape(table as any, assistant, 'live') - const service = createTapeService( - table, - [createTraceRow(), createTraceRow({ id: 'other-session', session_id: 's2', request_seq: 9 })], - [createMessageRow()] - ) - service.appendViewManifest(createObservationManifest()) - - const slice = service.readCausalObservationSlice('s1', 'a1', { - currentRuntimeStatus: 'idle' - }) - - expect(slice).toMatchObject({ - schemaVersion: 1, - sessionId: 's1', - messageId: 'a1', - request: { - state: 'manifest_bound', - requestSeq: 1, - replay: { - requestSeq: 1, - mode: 'trace_bound', - trace: { id: 'trace-1', requestSeq: 1 } - } - }, - output: { - correlation: 'message_only', - terminalMessage: { - status: 'sent', - orderSeq: 2, - createdAt: 200, - updatedAt: 300 - } - }, - runtime: { scope: 'current_only', status: 'idle', eventHistory: 'not_persisted' } - }) - expect(slice.output.entries.map((entry) => entry.kind)).toContain('message') - expect(slice.output.terminalMessage?.contentHash).toHaveLength(64) - expect(slice.output.terminalMessage?.metadataHash).toHaveLength(64) - }) - - it('prefers an explicit causal request sequence and otherwise selects the latest raw sequence', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 100 })) - service.appendViewManifest( - createObservationManifest({ - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 200 - }) - ) - - expect(service.readCausalObservationSlice('s1', 'a1').request).toMatchObject({ - state: 'manifest_bound', - requestSeq: 2 - }) - expect(service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }).request).toMatchObject( - { state: 'manifest_bound', requestSeq: 1 } - ) - }) - - it('does not fall back to an older manifest when a later traced request has no manifest', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [ - createTraceRow({ id: 'trace-1', request_seq: 1 }), - createTraceRow({ id: 'trace-2', request_seq: 2 }) - ]) - service.appendViewManifest(createObservationManifest({ requestSeq: 1 })) - - expect(service.readCausalObservationSlice('s1', 'a1').request).toMatchObject({ - state: 'manifest_missing', - requestSeq: 2, - trace: { id: 'trace-2', requestSeq: 2 } - }) - }) - - it('returns a trace-only request without manufacturing a view manifest', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [createTraceRow({ request_seq: 4 })]) - - const request = service.readCausalObservationSlice('s1', 'a1').request - - expect(request).toMatchObject({ - state: 'manifest_missing', - requestSeq: 4, - trace: { requestSeq: 4 } - }) - expect( - request.state === 'manifest_missing' ? request.trace?.bodyJson : undefined - ).toBeUndefined() - }) - - it('distinguishes malformed manifests from hash-invalid but readable manifests', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'bad', seq: 2 }, - data: { manifest: { schemaVersion: 99, requestSeq: 2 } } - }) - const tamperedManifest = createObservationManifest({ messageId: 'a1', requestSeq: 1 }) - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 1 }, - data: { manifest: { ...tamperedManifest, latestEntryId: 1 } } - }) - - expect(service.readCausalObservationSlice('s1', 'bad').request).toEqual({ - state: 'manifest_malformed', - requestSeq: 2, - trace: null - }) - const invalid = service.readCausalObservationSlice('s1', 'a1').request - expect(invalid.state).toBe('manifest_bound') - expect(invalid.state === 'manifest_bound' ? invalid.replay.integrity : undefined).toBe( - 'invalid' - ) - }) - - it('ignores request sequence zero interleaved-reasoning sentinels', () => { - const { table } = createTapeTableMock() - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 0 }, - data: { manifest: { schemaVersion: 99, requestSeq: 0 } } - }) - const service = createTapeService(table, [ - createTraceRow({ - id: 'trace-gap', - request_seq: 0, - endpoint: 'deepchat://interleaved-reasoning-gap' - }) - ]) - - expect(service.readCausalObservationSlice('s1', 'a1').request).toEqual({ - state: 'request_unavailable', - requestSeq: null, - trace: null - }) - }) - - it('keeps multi-round assistant and tool output correlated at message scope only', () => { - const { table } = createTapeTableMock() - const assistant = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 220, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'result' - } - } - ]), - status: 'sent', - createdAt: 200, - updatedAt: 300 - }) - appendMessageRecordToTape(table as any, assistant, 'live') - const service = createTapeService(table, [], [createMessageRow({ content: assistant.content })]) - service.appendViewManifest(createObservationManifest({ requestSeq: 1 })) - service.appendViewManifest( - createObservationManifest({ - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null - }) - ) - - const firstOutput = service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }).output - const latestOutput = service.readCausalObservationSlice('s1', 'a1').output - - expect(latestOutput.correlation).toBe('message_only') - expect(latestOutput.entries.map((entry) => entry.kind)).toEqual([ - 'message', - 'tool_call', - 'tool_result' - ]) - expect(latestOutput.entries.map((entry) => entry.entryId)).toEqual( - firstOutput.entries.map((entry) => entry.entryId) - ) - }) - - it('does not infer a completed event from a paused pending assistant message', () => { - const { table } = createTapeTableMock() - const service = createTapeService( - table, - [], - [createMessageRow({ status: 'pending', content: '[{"type":"action","status":"pending"}]' })] - ) - - const slice = service.readCausalObservationSlice('s1', 'a1', { - currentRuntimeStatus: 'generating' - }) - - expect(slice.output.entries).toEqual([]) - expect(slice.output.terminalMessage).toBeNull() - expect(slice.runtime).toEqual({ - scope: 'current_only', - status: 'generating', - eventHistory: 'not_persisted' - }) - expect(JSON.stringify(slice)).not.toContain('completed') - }) - - it('reads an old message without bootstrapping or backfilling Tape', () => { - const { table, entries } = createTapeTableMock() - const messageInsert = vi.fn() - const traceInsert = vi.fn() - const projectionApply = vi.fn() - const projectionReplace = vi.fn() - const cursorWrite = vi.fn() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatMessageTracesTable: { - listByMessageId: vi.fn().mockReturnValue([]), - insert: traceInsert - }, - deepchatMessagesTable: { - get: vi.fn().mockReturnValue(createMessageRow()), - insert: messageInsert - }, - deepchatMemoryIngestionProjectionTable: { - applyAppendedEntry: projectionApply, - replaceSession: projectionReplace - }, - deepchatSessionsTable: { - getSummaryState: vi.fn().mockReturnValue(null), - updateMemoryCursorOrderSeq: cursorWrite - } - } as any) - - const slice = service.readCausalObservationSlice('s1', 'a1') - - expect(slice.request).toEqual({ - state: 'request_unavailable', - requestSeq: null, - trace: null - }) - expect(slice.output.terminalMessage?.status).toBe('sent') - expect(slice.runtime).toEqual({ - scope: 'unavailable', - status: null, - eventHistory: 'not_persisted' - }) - expect(entries).toEqual([]) - expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() - expect(table.append).not.toHaveBeenCalled() - expect(table.appendEvent).not.toHaveBeenCalled() - expect(messageInsert).not.toHaveBeenCalled() - expect(traceInsert).not.toHaveBeenCalled() - expect(projectionApply).not.toHaveBeenCalled() - expect(projectionReplace).not.toHaveBeenCalled() - expect(cursorWrite).not.toHaveBeenCalled() - }) - - it('keeps causal observations metadata-only by default and reuses replay payload opt-ins', () => { - const { table } = createTapeTableMock() - const assistant = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: 'tape-secret-output', - status: 'sent', - metadata: '{"secret":"tape-metadata"}', - createdAt: 200, - updatedAt: 300 - }) - appendMessageRecordToTape(table as any, assistant, 'live') - const service = createTapeService( - table, - [ - createTraceRow({ - headers_json: '{"authorization":"secret-header"}', - body_json: '{"prompt":"secret-request"}' - }) - ], - [ - createMessageRow({ - content: 'projection-only-content-secret', - metadata: '{"secret":"projection-only-metadata"}', - blocks_json: '["projection-only-blocks"]', - error: 'projection-only-error' - }) - ] - ) - service.appendViewManifest(createObservationManifest()) - - const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) - const metadataOnly = service.readCausalObservationSlice('s1', 'a1') - const withPayloads = service.readCausalObservationSlice('s1', 'a1', { - includeTapePayloads: true, - includeTracePayload: true - }) - now.mockRestore() - - expect(JSON.stringify(metadataOnly)).not.toContain('tape-secret-output') - expect(JSON.stringify(metadataOnly)).not.toContain('tape-metadata') - expect(JSON.stringify(metadataOnly)).not.toContain('secret-header') - expect(JSON.stringify(metadataOnly)).not.toContain('secret-request') - expect(JSON.stringify(metadataOnly)).not.toContain('projection-only') - expect(withPayloads.output.entries.some((entry) => entry.payload?.record)).toBe(true) - expect(withPayloads.request.state).toBe('manifest_bound') - expect( - withPayloads.request.state === 'manifest_bound' - ? withPayloads.request.replay.trace?.headersJson - : undefined - ).toContain('secret-header') - expect(JSON.stringify(withPayloads)).toContain('tape-secret-output') - expect(JSON.stringify(withPayloads)).not.toContain('projection-only') - expect(metadataOnly.output.terminalMessage).not.toHaveProperty('content') - expect(metadataOnly.output.terminalMessage).not.toHaveProperty('metadata') - expect(metadataOnly.output.terminalMessage).not.toHaveProperty('blocks') - expect(metadataOnly.output.terminalMessage).not.toHaveProperty('error') - expect(stripObservationPayloadOptIns(withPayloads)).toEqual( - stripObservationPayloadOptIns(metadataOnly) - ) - }) - - it('keeps storage and Memory seams unchanged across every causal observation read mode', () => { - const { table, entries } = createTapeTableMock() - const { final } = appendObservationIsolationFacts(table) - const traceRows = [ - createTraceRow({ id: 'trace-1', request_seq: 1 }), - createTraceRow({ id: 'trace-2', request_seq: 2 }), - createTraceRow({ id: 'trace-only', message_id: 'a-trace', request_seq: 7 }) - ] - const messageRows = [createMessageRow({ order_seq: 3 })] - const projectionState = { - meta: { session_id: 's1', projection_version: 1, max_entry_id: entries.length }, - range: [{ session_id: 's1', message_id: 'a1', order_seq: 3, entry_id: entries.length }] - } - const memoryCursorOrderSeq = 3 - const memoryProjectionWrites = createSpies([ - 'applyAppendedEntry', - 'replaceSession', - 'invalidateSession' - ]) - const memoryProjectionTable = { - ...memoryProjectionWrites, - readCurrentRange: vi.fn(() => structuredClone(projectionState.range)), - getSessionMeta: vi.fn(() => structuredClone(projectionState.meta)) - } - const cursorWrites = createSpies(['updateMemoryCursorOrderSeq', 'rewindMemoryCursorOrderSeq']) - const sessionTable = { - ...cursorWrites, - getSummaryState: vi.fn().mockReturnValue(null), - getMemoryCursorOrderSeq: vi.fn(() => memoryCursorOrderSeq) - } - const messageWrites = createSpies([ - 'insert', - 'updateContent', - 'updateStatus', - 'updateContentAndStatus' - ]) - const messageTable = { - ...messageWrites, - get: vi.fn((messageId: string) => messageRows.find((row) => row.id === messageId)) - } - const traceWrites = createSpies(['insert', 'deleteByMessageId', 'deleteBySessionId']) - const traceTable = { - ...traceWrites, - listByMessageId: vi.fn((messageId: string) => - traceRows.filter((row) => row.message_id === messageId) - ) - } - const { presenter, memoryPropertyAccess } = trackMemoryPropertyAccess({ - deepchatTapeEntriesTable: table, - deepchatMessageTracesTable: traceTable, - deepchatMessagesTable: messageTable, - deepchatSessionsTable: sessionTable, - deepchatMemoryIngestionProjectionTable: memoryProjectionTable - }) - const service = new SessionTape(presenter as any) - service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 210 })) - service.appendViewManifest( - createObservationManifest({ - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 310 - }) - ) - - for (const write of [ - table.ensureBootstrapAnchor, - table.append, - table.appendAnchor, - table.appendEvent, - table.deleteBySession - ]) { - write.mockClear() - } - const captureState = () => ({ - tapeRows: structuredClone(entries), - maxEntryId: table.getMaxEntryId('s1'), - entryOrder: entries.map((entry) => entry.entry_id), - maxMessageOrder: Math.max(0, ...service.getMessageRecords('s1').map((row) => row.orderSeq)), - messageRows: structuredClone(messageRows), - traceRows: structuredClone(traceRows), - viewManifestRows: structuredClone( - entries.filter((entry) => entry.kind === 'event' && entry.name === 'view/assembled') - ), - effectiveView: service.getMessageRecords('s1'), - replay: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }), - replayHash: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 })?.hashes.sliceHash, - projectionMeta: memoryProjectionTable.getSessionMeta(), - projectionRange: memoryProjectionTable.readCurrentRange(), - memoryCursorOrderSeq: sessionTable.getMemoryCursorOrderSeq() - }) - const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) - - try { - const before = captureState() - const { defaultObservation, repeatedObservation, explicitObservation, traceOnlyObservation } = - readObservationMatrix(service) - const after = captureState() - - expect(after).toEqual(before) - expect(repeatedObservation).toEqual(defaultObservation) - expect(defaultObservation.request).toMatchObject({ - state: 'manifest_bound', - requestSeq: 2 - }) - expect(explicitObservation.request).toMatchObject({ - state: 'manifest_bound', - requestSeq: 1 - }) - expect(traceOnlyObservation.request).toMatchObject({ - state: 'manifest_missing', - requestSeq: 7, - trace: { id: 'trace-only' } - }) - expect(defaultObservation.output).toMatchObject({ - correlation: 'message_only', - entries: [{ kind: 'message' }, { kind: 'tool_call' }, { kind: 'tool_result' }] - }) - expect(before.effectiveView.map((record) => record.id)).toEqual(['u1', 'a1']) - expect(JSON.parse(before.effectiveView[0].content).text).toBe('edited') - expect(before.effectiveView[1]).toMatchObject({ status: 'sent', content: final.content }) - } finally { - now.mockRestore() - } - - for (const write of [ - table.ensureBootstrapAnchor, - table.append, - table.appendAnchor, - table.appendEvent, - table.deleteBySession, - ...Object.values(messageWrites), - ...Object.values(traceWrites), - ...Object.values(memoryProjectionWrites), - ...Object.values(cursorWrites), - memoryPropertyAccess - ]) { - expect(write).not.toHaveBeenCalled() - } - }) - - itIfSqlite( - `preserves real Tape, message, trace, Memory projection and schema state while observing${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const memoryProjectionTable = new DeepChatMemoryIngestionProjectionTable(db) - const tapeTable = new DeepChatTapeEntriesTable(db, memoryProjectionTable) - const messageTable = new DeepChatMessagesTable(db) - const traceTable = new DeepChatMessageTracesTable(db) - const sessionTable = new DeepChatSessionsTable(db) - memoryProjectionTable.createTable() - tapeTable.createTable() - messageTable.createTable() - traceTable.createTable() - sessionTable.createTable() - sessionTable.create('s1', 'openai', 'gpt-4o', 'full_access') - sessionTable.updateMemoryCursorOrderSeq('s1', 3) - - appendObservationIsolationFacts(tapeTable) - messageTable.insert({ - id: 'a1', - sessionId: 's1', - orderSeq: 3, - role: 'assistant', - content: 'projection-only-content-secret', - status: 'sent', - metadata: '{"secret":"projection-only-metadata"}', - createdAt: 200, - updatedAt: 300 - }) - for (const trace of [ - { id: 'trace-1', messageId: 'a1', requestSeq: 1 }, - { id: 'trace-2', messageId: 'a1', requestSeq: 2 }, - { id: 'trace-only', messageId: 'a-trace', requestSeq: 7 } - ]) { - traceTable.insert({ - ...trace, - sessionId: 's1', - providerId: 'openai', - modelId: 'gpt-4o', - endpoint: 'https://api.openai.test/v1/chat/completions', - headersJson: `{"authorization":"${trace.id}-header"}`, - bodyJson: `{"prompt":"${trace.id}-body"}`, - truncated: false, - createdAt: 300 + trace.requestSeq - }) - } - - const service = new SessionTape({ - deepchatTapeEntriesTable: tapeTable, - deepchatMessagesTable: messageTable, - deepchatMessageTracesTable: traceTable, - deepchatSessionsTable: sessionTable, - deepchatMemoryIngestionProjectionTable: memoryProjectionTable - } as any) - service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 210 })) - service.appendViewManifest( - createObservationManifest({ - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 310 - }) - ) - const effectiveRecords = service.getMessageRecords('s1') - const sourceMaps = service.getViewManifestSourceMaps('s1') - memoryProjectionTable.replaceSession( - 's1', - effectiveRecords - .filter( - (record): record is ChatMessageRecord & { status: 'sent' | 'error' } => - record.status === 'sent' || record.status === 'error' - ) - .map((record) => ({ - sessionId: 's1', - messageId: record.id, - orderSeq: record.orderSeq, - entryId: sourceMaps.entryIdByMessageId.get(record.id)!, - role: record.role, - content: record.content, - status: record.status, - hadToolUse: record.id === 'a1' - })), - tapeTable.getMaxEntryId('s1') - ) - - const captureState = () => { - const tapeRows = tapeTable.getBySession('s1') - return { - tapeRows, - maxEntryId: tapeTable.getMaxEntryId('s1'), - entryOrder: tapeRows.map((row) => row.entry_id), - messageRow: messageTable.get('a1'), - traceRows: db - .prepare( - `SELECT * - FROM deepchat_message_traces - ORDER BY session_id, message_id, request_seq, id` - ) - .all(), - viewManifestRows: tapeRows.filter( - (row) => row.kind === 'event' && row.name === 'view/assembled' - ), - effectiveView: service.getMessageRecords('s1'), - replay: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }), - replayHash: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 })?.hashes.sliceHash, - projectionMeta: memoryProjectionTable.getSessionMeta('s1'), - projectionRange: memoryProjectionTable.readCurrentRange('s1', 0, 10), - memoryCursorOrderSeq: sessionTable.getMemoryCursorOrderSeq('s1'), - schema: db - .prepare( - `SELECT type, name, tbl_name, sql - FROM sqlite_master - WHERE name NOT LIKE 'sqlite_%' - ORDER BY type, name, tbl_name` - ) - .all() - } - } - const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) - - try { - const before = captureState() - readObservationMatrix(service) - const after = captureState() - - expect(after).toEqual(before) - } finally { - now.mockRestore() - } - } finally { - db.close() - } - } - ) - - it('keeps pending message records for resume but hides pending tool facts from search', () => { - const { table } = createTapeTableMock() - const pendingBlocks = [ - { - type: 'tool_call', - status: 'pending', - timestamp: 100, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'pending result' - } - } - ] - const messageStore = { - getMessages: vi.fn().mockReturnValue([ - createRecord({ - id: 'a1', - orderSeq: 1, - role: 'assistant', - status: 'pending', - content: JSON.stringify(pendingBlocks), - updatedAt: 100 - }) - ]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.ensureSessionTapeReady('s1', messageStore as any) - - expect(service.getMessageRecords('s1')).toMatchObject([{ id: 'a1', status: 'pending' }]) - expect(service.search('s1', 'pending result', { kinds: ['tool_result'] })).toEqual([]) - }) - - it('lets final assistant facts supersede earlier pending tape facts', () => { - const { table, entries } = createTapeTableMock() - const pendingBlocks = [ - { - type: 'tool_call', - status: 'pending', - timestamp: 100, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'pending result' - } - } - ] - const finalBlocks = [ - { - type: 'tool_call', - status: 'success', - timestamp: 200, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'final result' - } - } - ] - const messageStore = { - getMessages: vi - .fn() - .mockReturnValueOnce([ - createRecord({ - id: 'a1', - orderSeq: 1, - role: 'assistant', - status: 'pending', - content: JSON.stringify(pendingBlocks), - metadata: JSON.stringify({ totalTokens: 1 }), - updatedAt: 100 - }) - ]) - .mockReturnValue([ - createRecord({ - id: 'a1', - orderSeq: 1, - role: 'assistant', - status: 'sent', - content: JSON.stringify(finalBlocks), - metadata: JSON.stringify({ totalTokens: 7 }), - updatedAt: 200 - }) - ]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.ensureSessionTapeReady('s1', messageStore as any) - service.ensureSessionTapeReady('s1', messageStore as any) - - expect(service.getMessageRecords('s1')).toMatchObject([ - { - id: 'a1', - status: 'sent' - } - ]) - const effectiveRecord = service.getMessageRecords('s1')[0]! - expect(JSON.parse(effectiveRecord.content)[0].tool_call.response).toBe('final result') - expect( - entries.filter((entry) => entry.kind === 'message' && entry.name === 'message/assistant') - ).toHaveLength(2) - expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) - const finalToolResult = entries.filter((entry) => entry.kind === 'tool_result').at(-1)! - expect(JSON.parse(finalToolResult.payload_json).response).toBe('final result') - expect(service.info('s1').lastTokenUsage).toBe(7) - expect(service.search('s1', 'pending result', { kinds: ['tool_result'] })).toEqual([]) - expect(service.search('s1', 'final result', { kinds: ['tool_result'] })).toHaveLength(1) - }) - - it('keeps fork writes isolated until merge and discards fork entries on discard', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const fork = service.createFork('s1', 'fork-1') - service.appendForkMessageRecord(fork, createRecord({ id: 'fu1', sessionId: 'ignored' })) - - expect( - entries.some((entry) => entry.session_id === 's1' && entry.name === 'message/user') - ).toBe(false) - - const mergedCount = service.mergeFork('s1', 'fork-1') - - expect(mergedCount).toBe(1) - expect( - entries.some((entry) => entry.session_id === 's1' && entry.name === 'message/user') - ).toBe(true) - expect(entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/merge')).toBe( - true - ) - expect(entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/start')).toBe( - false - ) - - const discardFork = service.createFork('s1', 'fork-2') - service.appendForkMessageRecord(discardFork, createRecord({ id: 'fu2', sessionId: 'ignored' })) - service.discardFork('s1', 'fork-2') - - expect(entries.some((entry) => entry.session_id === discardFork.forkSessionId)).toBe(false) - expect( - entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') - ).toBe(true) - }) - - it('records exact fork heads and keeps retries bounded to the first merge', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('parent') - table.appendEvent({ sessionId: 'parent', name: 'parent/tail', data: { value: 1 } }) - const fork = service.createFork('parent', 'bounded') - table.appendEvent({ sessionId: 'parent', name: 'parent/later', data: { value: 2 } }) - const retriedFork = service.createFork('parent', 'bounded') - service.appendForkMessageRecord(fork, createRecord({ id: 'first', sessionId: 'ignored' })) - - const forkStart = entries.find( - (entry) => entry.session_id === fork.forkSessionId && entry.name === 'fork/start' - ) - expect(fork.parentHeadEntryId).toBe(2) - expect(retriedFork.parentHeadEntryId).toBe(2) - expect(JSON.parse(forkStart.payload_json).state.parentHeadEntryId).toBe(2) - - const getBoundedEntries = table.getBySessionUpToEntryId.getMockImplementation()! - table.getBySessionUpToEntryId.mockImplementationOnce( - (sessionId: string, maxEntryId: number) => { - table.appendEvent({ - sessionId: fork.forkSessionId, - name: 'fork/late-tail', - data: { value: 2 } - }) - return getBoundedEntries(sessionId, maxEntryId) - } - ) - - expect(service.mergeFork('parent', 'bounded')).toBe(1) - expect(service.mergeFork('parent', 'bounded')).toBe(1) - - const parentEntries = entries.filter((entry) => entry.session_id === 'parent') - expect(parentEntries.filter((entry) => entry.source_type === 'fork')).toHaveLength(2) - expect(parentEntries.some((entry) => entry.name === 'fork/late-tail')).toBe(false) - const receipt = parentEntries.find((entry) => entry.name === 'fork/merge')! - expect(JSON.parse(receipt.payload_json).data).toMatchObject({ - forkHeadEntryId: 3, - mergedCount: 1 - }) - }) - - it('rolls back mocked fork merge writes when a copied entry fails', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const fork = service.createFork('parent', 'mock-atomic') - service.appendForkMessageRecord( - fork, - createRecord({ id: 'first', orderSeq: 1, sessionId: 'ignored' }) - ) - service.appendForkMessageRecord( - fork, - createRecord({ id: 'second', orderSeq: 2, sessionId: 'ignored' }) - ) - - const append = table.append.getMockImplementation()! - let copiedEntryCount = 0 - table.append.mockImplementation((input: any) => { - if (input.sessionId === 'parent' && input.source?.type === 'fork') { - copiedEntryCount += 1 - if (copiedEntryCount === 2) { - throw new Error('injected merge failure') - } - } - return append(input) - }) - - expect(() => service.mergeFork('parent', 'mock-atomic')).toThrow('injected merge failure') - expect( - entries.filter( - (entry) => - entry.session_id === 'parent' && - (entry.source_type === 'fork' || entry.name === 'fork/merge') - ) - ).toEqual([]) - }) - - it('rolls back copied fork entries when the merge receipt fails', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const fork = service.createFork('parent', 'receipt-failure') - service.appendForkMessageRecord(fork, createRecord({ id: 'first', sessionId: 'ignored' })) - - const appendEvent = table.appendEvent.getMockImplementation()! - table.appendEvent.mockImplementation((input: any) => { - if (input.sessionId === 'parent' && input.name === 'fork/merge') { - throw new Error('injected receipt failure') - } - return appendEvent(input) - }) - - expect(() => service.mergeFork('parent', 'receipt-failure')).toThrow('injected receipt failure') - expect( - entries.filter( - (entry) => - entry.session_id === 'parent' && - (entry.source_type === 'fork' || entry.name === 'fork/merge') - ) - ).toEqual([]) - }) - - it('commits one idempotent receipt for an empty fork', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - service.createFork('parent', 'empty') - - expect(service.mergeFork('parent', 'empty')).toBe(0) - expect(service.mergeFork('parent', 'empty')).toBe(0) - const parentEntries = entries.filter((entry) => entry.session_id === 'parent') - expect(parentEntries).toHaveLength(1) - expect(parentEntries[0].name).toBe('fork/merge') - expect(JSON.parse(parentEntries[0].payload_json).data).toMatchObject({ - forkHeadEntryId: 2, - mergedCount: 0 - }) - }) - - it('rejects missing and discarded forks without committing an empty merge receipt', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - expect(() => service.mergeFork('parent', 'missing')).toThrow( - 'Fork missing does not exist or has been discarded.' - ) - - service.createFork('parent', 'discarded') - service.discardFork('parent', 'discarded') - expect(() => service.mergeFork('parent', 'discarded')).toThrow( - 'Fork discarded does not exist or has been discarded.' - ) - expect( - entries.filter((entry) => entry.session_id === 'parent' && entry.name === 'fork/merge') - ).toEqual([]) - }) - - it('returns an existing merge receipt after the merged fork Tape is cleaned up', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const fork = service.createFork('parent', 'cleanup-after-merge') - service.appendForkMessageRecord(fork, createRecord({ id: 'merged', sessionId: 'ignored' })) - - expect(service.mergeFork('parent', 'cleanup-after-merge')).toBe(1) - table.deleteBySession(fork.forkSessionId) - expect(service.mergeFork('parent', 'cleanup-after-merge')).toBe(1) - }) - - it('merges a legacy fork start that predates the parent head field', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const fork = service.createFork('parent', 'legacy-start') - const start = entries.find( - (entry) => entry.session_id === fork.forkSessionId && entry.name === 'fork/start' - )! - const payload = JSON.parse(start.payload_json) - delete payload.state.parentHeadEntryId - start.payload_json = JSON.stringify(payload) - service.appendForkMessageRecord(fork, createRecord({ id: 'legacy', sessionId: 'ignored' })) - - expect(service.mergeFork('parent', 'legacy-start')).toBe(1) - }) - - it('accepts a valid legacy fork merge receipt without a frozen head', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/merge', - source: { type: 'fork', id: 'legacy-receipt', seq: 0 }, - provenanceKey: 'fork:parent:legacy-receipt:merge:event', - data: { - forkId: 'legacy-receipt', - forkSessionId: 'parent::fork::legacy-receipt', - mergedCount: 2 - } - }) - - expect(service.mergeFork('parent', 'legacy-receipt')).toBe(2) - }) - - it('rejects a malformed stored fork merge receipt instead of reporting an empty merge', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/merge', - source: { type: 'fork', id: 'malformed-receipt', seq: 0 }, - provenanceKey: 'fork:parent:malformed-receipt:merge:event', - data: { - forkId: 'malformed-receipt', - forkSessionId: 'parent::fork::malformed-receipt', - mergedCount: 'two' - } - }) - - expect(() => service.mergeFork('parent', 'malformed-receipt')).toThrow( - 'Stored fork merge receipt is malformed' - ) - }) - - itIfSqlite('rolls back copied fork entries and the receipt when merge fails', () => { - const db = new DatabaseCtor(':memory:') - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('parent') - const fork = service.createFork('parent', 'atomic') - service.appendForkMessageRecord( - fork, - createRecord({ id: 'first', orderSeq: 1, sessionId: 'ignored' }) - ) - service.appendForkMessageRecord( - fork, - createRecord({ id: 'second', orderSeq: 2, sessionId: 'ignored' }) - ) - - const append = table.append.bind(table) - let copiedEntryCount = 0 - const appendSpy = vi.spyOn(table, 'append').mockImplementation((input) => { - if (input.sessionId === 'parent' && input.source?.type === 'fork') { - copiedEntryCount += 1 - if (copiedEntryCount === 2) { - throw new Error('injected merge failure') - } - } - return append(input) - }) - - expect(() => service.mergeFork('parent', 'atomic')).toThrow('injected merge failure') - expect( - table - .getBySession('parent') - .filter((entry) => entry.source_type === 'fork' || entry.name === 'fork/merge') - ).toEqual([]) - - appendSpy.mockRestore() - expect(service.mergeFork('parent', 'atomic')).toBe(2) - expect(service.mergeFork('parent', 'atomic')).toBe(2) - expect( - table.getBySession('parent').filter((entry) => entry.source_type === 'fork') - ).toHaveLength(3) - - db.close() - }) - - itIfSqlite('rejects missing and discarded forks in SQLite', () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - expect(() => service.mergeFork('parent', 'missing-native')).toThrow( - 'Fork missing-native does not exist or has been discarded.' - ) - service.createFork('parent', 'discarded-native') - service.discardFork('parent', 'discarded-native') - expect(() => service.mergeFork('parent', 'discarded-native')).toThrow( - 'Fork discarded-native does not exist or has been discarded.' - ) - expect(table.getBySession('parent').some((entry) => entry.name === 'fork/merge')).toBe(false) - } finally { - db.close() - } - }) - - it('cleans fork search projection on discard without blocking the discard event', () => { - const { table, entries } = createTapeTableMock() - const projectionTable = { - deleteBySession: vi.fn(() => { - throw new Error('projection cleanup failed') - }) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const fork = service.createFork('s1', 'fork-cleanup') - service.appendForkMessageRecord(fork, createRecord({ id: 'fu-cleanup', sessionId: 'ignored' })) - service.discardFork('s1', 'fork-cleanup') - - expect(table.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) - expect(projectionTable.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) - expect(entries.some((entry) => entry.session_id === fork.forkSessionId)).toBe(false) - expect( - entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') - ).toBe(true) - }) - - it('links a frozen subagent Tape without copying child entries and retries idempotently', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ sessionId: 'child', name: 'child/result', data: { text: 'done' } }) - const input = { - parentSessionId: 'parent', - childSessionId: 'child', - runId: 'run-1', - taskId: 'task-1', - slotId: 'reviewer', - taskTitle: 'Review', - outcome: 'completed' as const, - resultSummary: 'Done' - } - const first = service.linkSubagentTape(input) - table.appendEvent({ sessionId: 'child', name: 'child/late', data: { text: 'late' } }) - const retry = service.linkSubagentTape(input) - const normalizedRetry = service.linkSubagentTape({ - ...input, - slotId: ' reviewer ', - taskTitle: ' Review ', - resultSummary: ' Done ' - }) - - expect(first).toEqual({ - linkEntry: { sessionId: 'parent', entryId: 2 }, - childSessionId: 'child', - childHeadEntryId: 2, - childEntryCount: 2, - outcome: 'completed' - }) - expect(retry).toEqual(first) - expect(normalizedRetry).toEqual(first) - const links = entries.filter( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - ) - expect(links).toHaveLength(1) - expect(links[0]).toMatchObject({ - source_type: 'subagent', - source_id: 'child', - source_seq: 2 - }) - expect(JSON.parse(links[0].payload_json).data).toMatchObject({ - linkVersion: 2, - childTapeIdentity: expect.stringMatching(/^[a-f0-9]{64}$/) - }) - expect( - entries.some((entry) => entry.session_id === 'parent' && entry.name === 'child/result') - ).toBe(false) - expect(() => service.linkSubagentTape({ ...input, outcome: 'error' })).toThrow( - 'Subagent Tape link conflicts with finalized task run-1/task-1.' - ) - expect(() => service.linkSubagentTape({ ...input, slotId: 'writer' })).toThrow( - 'Subagent Tape link conflicts with finalized task run-1/task-1.' - ) - expect(() => service.linkSubagentTape({ ...input, taskTitle: 'Write' })).toThrow( - 'Subagent Tape link conflicts with finalized task run-1/task-1.' - ) - expect(() => service.linkSubagentTape({ ...input, resultSummary: 'Changed' })).toThrow( - 'Subagent Tape link conflicts with finalized task run-1/task-1.' - ) - - table.ensureBootstrapAnchor('child-2') - expect( - service.linkSubagentTape({ - ...input, - childSessionId: 'child-2', - outcome: 'error' - }) - ).toMatchObject({ - childSessionId: 'child-2', - outcome: 'error' - }) - expect( - entries.filter( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - ) - ).toHaveLength(2) - }) - - it('rejects a stored subagent link whose task identity no longer matches its provenance', () => { - const { table, entries } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - const input = createSubagentLinkInput('parent', 'child') - service.linkSubagentTape(input) - - const link = entries.find( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - ) - if (!link) { - throw new Error('Expected subagent Tape link fixture.') - } - const payload = JSON.parse(link.payload_json) as { - data?: Record - } - link.payload_json = JSON.stringify({ - ...payload, - data: { ...payload.data, runId: 'different-run' } - }) - - expect(() => service.linkSubagentTape(input)).toThrow( - /Stored subagent Tape link receipt is malformed/ - ) - expect(() => service.getContext('parent', [1], { sourceSessionId: 'child' })).toThrowError( - /not an authorized direct child/ - ) - }) - - it('keeps a version-one link readable while its original unmarked Tape remains present', () => { - const { table, entries } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/result', - data: { text: 'version one compatibility marker' } - }) - const input = createSubagentLinkInput('parent', 'child') - const receipt = service.linkSubagentTape(input) - const link = entries.find( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - )! - const payload = JSON.parse(link.payload_json) - payload.data.linkVersion = 1 - delete payload.data.childTapeIdentity - link.payload_json = JSON.stringify(payload) - entries.find((entry) => entry.session_id === 'child' && entry.entry_id === 1)!.meta_json = '{}' - - expect(service.linkSubagentTape(input)).toEqual(receipt) - expect( - service.search('parent', 'version one compatibility marker', { - scope: 'linked_subagents' - }) - ).toMatchObject([{ sessionId: 'child', entryId: 2 }]) - }) - - it('fails closed when a version-one link points at malformed incarnation metadata', () => { - const { table, entries } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/result', - data: { text: 'malformed incarnation metadata marker' } - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - const link = entries.find( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - )! - const payload = JSON.parse(link.payload_json) - payload.data.linkVersion = 1 - delete payload.data.childTapeIdentity - link.payload_json = JSON.stringify(payload) - entries.find((entry) => entry.session_id === 'child' && entry.entry_id === 1)!.meta_json = '{' - - expect(() => - service.search('parent', 'malformed incarnation metadata marker', { - scope: 'linked_subagents' - }) - ).toThrowError(/Linked Tape child is unavailable/) - }) - - it('searches direct linked children at frozen heads with one global limit', () => { - const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child-a', session_kind: 'subagent', parent_session_id: 'parent' }, - { id: 'child-b', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child-a') - table.ensureBootstrapAnchor('child-b') - table.appendEvent({ - sessionId: 'parent', - name: 'parent/note', - data: { text: 'shared needle from parent' }, - createdAt: 150 - }) - table.appendEvent({ - sessionId: 'child-a', - name: 'child/result', - data: { text: 'shared needle from A' }, - createdAt: 100 - }) - table.appendEvent({ - sessionId: 'child-b', - name: 'child/result', - data: { text: 'shared needle from B' }, - createdAt: 200 - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child-a')) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child-b')) - table.appendEvent({ - sessionId: 'child-a', - name: 'child/late', - data: { text: 'shared needle after cutoff' }, - createdAt: 300 - }) - - table.ensureBootstrapAnchor.mockClear() - table.append.mockClear() - const limited = service.search('parent', 'shared needle', { - scope: 'linked_subagents', - limit: 1 - }) - const all = service.search('parent', 'shared needle', { - scope: 'linked_subagents', - limit: 10 - }) - const combined = service.search('parent', 'shared needle', { - scope: 'current_and_linked', - limit: 10 - }) - - expect(limited).toMatchObject([{ sessionId: 'child-b', entryId: 2 }]) - expect(all.map((result) => [result.sessionId, result.entryId])).toEqual([ - ['child-b', 2], - ['child-a', 2] - ]) - expect(combined.map((result) => [result.sessionId, result.entryId])).toEqual([ - ['child-b', 2], - ['parent', 2], - ['child-a', 2] - ]) - expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( - [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 2 } - ], - 'shared needle', - expect.objectContaining({ limit: 1 }) - ) - expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() - expect(table.append).not.toHaveBeenCalled() - }) - - it('falls back all linked sources together when projection coverage is partial', () => { - const { table } = createTapeTableMock() - const projectionTable = { - searchSourcesReadOnly: vi.fn(() => ({ - coveredSources: [{ sessionId: 'child-a', maxEntryId: 2 }], - rows: [ - { - session_id: 'child-a', - entry_id: 2, - kind: 'event', - name: 'child/result', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'shared partial needle', - summary_text: 'shared partial needle from A', - refs_json: '{}', - created_at: 100, - score: -100 - } - ] - })) - } - const { service } = createLinkedTapeService( - table, - [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child-a', session_kind: 'subagent', parent_session_id: 'parent' }, - { id: 'child-b', session_kind: 'subagent', parent_session_id: 'parent' } - ], - projectionTable - ) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child-a') - table.ensureBootstrapAnchor('child-b') - table.appendEvent({ - sessionId: 'child-a', - name: 'child/result', - data: { text: 'shared partial needle from A' }, - createdAt: 100 - }) - table.appendEvent({ - sessionId: 'child-b', - name: 'child/result', - data: { text: 'shared partial needle from B' }, - createdAt: 200 - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child-a')) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child-b')) - table.searchEffectiveSourcesAtHeads.mockClear() - - const hits = service.search('parent', 'shared partial needle', { - scope: 'linked_subagents', - limit: 1 - }) - - expect(hits).toMatchObject([{ sessionId: 'child-b', entryId: 2 }]) - expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( - [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 2 } - ], - 'shared partial needle', - expect.objectContaining({ limit: 1 }) - ) - }) - - it('uses an exact frozen projection read-only after the child appends a late tail', () => { - const { table } = createTapeTableMock() - const projectionTable = { - searchSourcesReadOnly: vi.fn((sources: any[]) => ({ - coveredSources: sources, - rows: [ - { - session_id: 'child', - entry_id: 2, - kind: 'event', - name: 'child/result', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'frozen projection needle', - summary_text: 'frozen projection needle', - refs_json: '{}', - created_at: 100, - score: -1 - } - ] - })), - replaceSession: vi.fn(), - appendSession: vi.fn(), - getByEntryIds: vi.fn().mockReturnValue([]) - } - const { service } = createLinkedTapeService( - table, - [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ], - projectionTable - ) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/result', - data: { text: 'frozen projection needle' }, - createdAt: 100 - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - table.appendEvent({ - sessionId: 'child', - name: 'child/late', - data: { text: 'frozen projection needle late' }, - createdAt: 200 - }) - table.searchEffectiveSourcesAtHeads.mockClear() - - const hits = service.search('parent', 'frozen projection needle', { - scope: 'linked_subagents' - }) - - expect(projectionTable.searchSourcesReadOnly).toHaveBeenCalledWith( - [{ sessionId: 'child', maxEntryId: 2 }], - 'frozen projection needle', - expect.any(Object) - ) - expect(hits).toMatchObject([{ sessionId: 'child', entryId: 2 }]) - expect(table.searchEffectiveSourcesAtHeads).not.toHaveBeenCalled() - expect(projectionTable.replaceSession).not.toHaveBeenCalled() - expect(projectionTable.appendSession).not.toHaveBeenCalled() - }) - - it('deduplicates repeated child links at the newest finalized snapshot', () => { - const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/first', - data: { text: 'first snapshot marker' } - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - table.appendEvent({ - sessionId: 'child', - name: 'child/second', - data: { text: 'second snapshot marker' } - }) - service.linkSubagentTape({ - ...createSubagentLinkInput('parent', 'child'), - runId: 'run-child-second', - taskId: 'task-child-second' - }) - - expect( - service.search('parent', 'second snapshot marker', { scope: 'linked_subagents' }) - ).toMatchObject([{ sessionId: 'child', entryId: 3 }]) - expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( - [{ sessionId: 'child', maxEntryId: 3 }], - 'second snapshot marker', - expect.any(Object) - ) - }) - - it('expands linked context within one source and never crosses the frozen head', () => { - const { table } = createTapeTableMock() - const projectionTable = { - getByEntryIds: vi.fn(() => [ - { - session_id: 'child', - entry_id: 2, - kind: 'event', - name: 'child/target', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'stale projection text', - summary_text: 'stale projection summary', - refs_json: '{"stale":true}', - created_at: 100 - } - ]) - } - const { service } = createLinkedTapeService( - table, - [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ], - projectionTable - ) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/target', - data: { text: 'target evidence' }, - createdAt: 100 - }) - table.appendEvent({ - sessionId: 'child', - name: 'child/neighbor', - data: { text: 'neighbor evidence' }, - createdAt: 110 - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - table.appendEvent({ - sessionId: 'child', - name: 'child/late', - data: { text: 'late evidence' }, - createdAt: 120 - }) - table.append.mockClear() - - const context = service.getContext('parent', [2], { - sourceSessionId: 'child', - before: 0, - after: 5 - }) - - expect(context).toMatchObject({ - sessionId: 'parent', - sourceSessionId: 'child', - requestedEntryIds: [2], - matchedEntryIds: [2] - }) - expect(context.entries.map((entry) => entry.entryId)).toEqual([2, 3]) - expect(context.entries[0].summary).toContain('target evidence') - expect(context.entries[0].summary).not.toContain('stale projection') - expect(table.getEffectiveContextRowsAtHead).toHaveBeenCalledWith( - { sessionId: 'child', maxEntryId: 3 }, - [2], - expect.objectContaining({ before: 0, after: 5 }) - ) - expect(projectionTable.getByEntryIds).not.toHaveBeenCalled() - expect(table.append).not.toHaveBeenCalled() - }) - - it('rejects sibling and unlinked source ids even when a forged link event exists', () => { - const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'other-parent', session_kind: 'regular', parent_session_id: null }, - { id: 'sibling', session_kind: 'subagent', parent_session_id: 'other-parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('sibling') - service.linkSubagentTape(createSubagentLinkInput('parent', 'sibling')) - - let error: unknown - try { - service.getContext('parent', [1], { sourceSessionId: 'sibling' }) - } catch (caught) { - error = caught - } - expect(error).toMatchObject({ - code: 'linked_tape_unauthorized', - parentSessionId: 'parent', - sourceSessionId: 'sibling' - }) - }) - - it('reports a finalized linked Tape as unavailable after its durable session is deleted', () => { - const { table } = createTapeTableMock() - const { service, sessionById } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - sessionById.delete('child') - - let error: unknown - try { - service.search('parent', 'anything', { scope: 'linked_subagents' }) - } catch (caught) { - error = caught - } - expect(error).toMatchObject({ - code: 'linked_tape_unavailable', - parentSessionId: 'parent', - sourceSessionId: 'child' - }) - }) - - it('rejects a rebuilt child Tape until a new incarnation is explicitly linked', () => { - const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/original', - data: { text: 'original incarnation marker' } - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - - table.deleteBySession('child') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/rebuilt', - data: { text: 'rebuilt incarnation marker' } - }) - expect( - service.search('child', 'rebuilt incarnation marker', { scope: 'current' }) - ).toHaveLength(1) - expect(() => - service.search('parent', 'rebuilt incarnation marker', { scope: 'linked_subagents' }) - ).toThrowError(/Linked Tape child is unavailable/) - - service.linkSubagentTape({ - ...createSubagentLinkInput('parent', 'child'), - runId: 'run-rebuilt-child', - taskId: 'task-rebuilt-child' - }) - expect( - service.search('parent', 'rebuilt incarnation marker', { scope: 'linked_subagents' }) - ).toMatchObject([{ sessionId: 'child', entryId: 2 }]) - }) - - itIfSqlite('detects linked Tape replacement after entry ids are reused in SQLite', () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/original', - data: { text: 'native original incarnation' } - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - - table.deleteBySession('child') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/rebuilt', - data: { text: 'native rebuilt incarnation' } - }) - - expect(() => - service.search('parent', 'native rebuilt incarnation', { scope: 'linked_subagents' }) - ).toThrowError(/Linked Tape child is unavailable/) - } finally { - db.close() - } - }) - - it('reads legacy external merge links and keeps legacy discard audit-only', () => { - const { table, entries } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'legacy-child', session_kind: 'subagent', parent_session_id: 'parent' }, - { id: 'malformed-child', session_kind: 'subagent', parent_session_id: 'parent' }, - { id: 'discarded-child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('legacy-child') - table.ensureBootstrapAnchor('malformed-child') - table.ensureBootstrapAnchor('discarded-child') - const legacyBootstrap = entries.find( - (entry) => entry.session_id === 'legacy-child' && entry.entry_id === 1 - )! - legacyBootstrap.meta_json = '{}' - table.appendEvent({ - sessionId: 'legacy-child', - name: 'child/result', - data: { text: 'legacy link needle' } - }) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/merge', - source: { type: 'fork', id: 'legacy-child', seq: 0 }, - provenanceKey: 'fork:parent:legacy-child:external-merge:event', - data: { - forkId: 'legacy-child', - forkSessionId: 'legacy-child', - referencedEntryCount: 2, - status: 'completed' - } - }) - table.appendEvent({ - sessionId: 'malformed-child', - name: 'child/result', - data: { text: 'malformed legacy link needle' } - }) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/merge', - source: { type: 'fork', id: 'malformed-child', seq: 1 }, - provenanceKey: 'fork:parent:malformed-child:external-merge:event', - data: { - forkId: 'malformed-child', - forkSessionId: 'malformed-child', - referencedEntryCount: 2, - status: 'completed' - } - }) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/discard', - source: { type: 'fork', id: 'discarded-child', seq: 0 }, - provenanceKey: 'fork:parent:discarded-child:external-discard:event', - data: { - forkId: 'discarded-child', - forkSessionId: 'discarded-child', - status: 'cancelled' - } - }) - - expect( - service.search('parent', 'legacy link needle', { scope: 'linked_subagents' }) - ).toMatchObject([{ sessionId: 'legacy-child', entryId: 2 }]) - expect( - service.search('parent', 'malformed legacy link needle', { scope: 'linked_subagents' }) - ).toEqual([]) - let error: unknown - try { - service.getContext('parent', [1], { sourceSessionId: 'discarded-child' }) - } catch (caught) { - error = caught - } - expect(error).toMatchObject({ code: 'linked_tape_unauthorized' }) - }) - - it('uses effective message facts after replacement and retraction events', () => { - const { table, entries } = createTapeTableMock() - const original = createRecord({ id: 'u1', orderSeq: 1 }) - const messageStore = { - getMessages: vi.fn().mockReturnValue([original]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.ensureSessionTapeReady('s1', messageStore as any) - appendMessageReplacementToTape( - table as any, - createRecord({ - id: 'u1', - orderSeq: 1, - content: JSON.stringify({ - text: 'edited', - files: [], - links: [], - search: false, - think: false - }), - updatedAt: 300 - }), - 'test_edit' - ) - - expect(JSON.parse(service.getMessageRecords('s1')[0].content).text).toBe('edited') - expect(service.search('s1', 'hello', { kinds: ['message'] })).toEqual([]) - expect(service.search('s1', 'edited', { kinds: ['message'] })).toHaveLength(1) - expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) - - appendMessageRetractionToTape(table as any, service.getMessageRecords('s1')[0], 'test_delete') - - expect(service.getMessageRecords('s1')).toEqual([]) - expect(service.search('s1', 'edited', { kinds: ['message'] })).toEqual([]) - }) - - it('appends non-idempotent retractions without generated provenance keys', () => { - const { table, entries } = createTapeTableMock() - const record = createRecord({ id: 'u1' }) - - appendMessageRetractionToTape(table as any, record, 'first_delete') - appendMessageRetractionToTape(table as any, record, 'second_delete') - - const retractions = entries.filter((entry) => entry.name === 'message/retracted') - expect(retractions).toHaveLength(2) - expect(retractions.map((entry) => entry.provenance_key)).toEqual([null, null]) - }) -}) diff --git a/test/main/session/data/tapeFork.test.ts b/test/main/session/data/tapeFork.test.ts new file mode 100644 index 0000000000..27dfb7d3b0 --- /dev/null +++ b/test/main/session/data/tapeFork.test.ts @@ -0,0 +1,376 @@ +import { + describe, + expect, + it, + vi, + SessionTape, + DeepChatTapeEntriesTable, + DatabaseCtor, + itIfSqlite, + createTapeTableMock, + createRecord +} from './tapeTestHarness' + +describe('SessionTape forks', () => { + it('keeps fork writes isolated until merge and discards fork entries on discard', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const fork = service.createFork('s1', 'fork-1') + service.appendForkMessageRecord(fork, createRecord({ id: 'fu1', sessionId: 'ignored' })) + + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'message/user') + ).toBe(false) + + const mergedCount = service.mergeFork('s1', 'fork-1') + + expect(mergedCount).toBe(1) + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'message/user') + ).toBe(true) + expect(entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/merge')).toBe( + true + ) + expect(entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/start')).toBe( + false + ) + + const discardFork = service.createFork('s1', 'fork-2') + service.appendForkMessageRecord(discardFork, createRecord({ id: 'fu2', sessionId: 'ignored' })) + service.discardFork('s1', 'fork-2') + + expect(entries.some((entry) => entry.session_id === discardFork.forkSessionId)).toBe(false) + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') + ).toBe(true) + }) + + it('records exact fork heads and keeps retries bounded to the first merge', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('parent') + table.appendEvent({ sessionId: 'parent', name: 'parent/tail', data: { value: 1 } }) + const fork = service.createFork('parent', 'bounded') + table.appendEvent({ sessionId: 'parent', name: 'parent/later', data: { value: 2 } }) + const retriedFork = service.createFork('parent', 'bounded') + service.appendForkMessageRecord(fork, createRecord({ id: 'first', sessionId: 'ignored' })) + + const forkStart = entries.find( + (entry) => entry.session_id === fork.forkSessionId && entry.name === 'fork/start' + ) + expect(fork.parentHeadEntryId).toBe(2) + expect(retriedFork.parentHeadEntryId).toBe(2) + expect(JSON.parse(forkStart.payload_json).state.parentHeadEntryId).toBe(2) + + const getBoundedEntries = table.getBySessionUpToEntryId.getMockImplementation()! + table.getBySessionUpToEntryId.mockImplementationOnce( + (sessionId: string, maxEntryId: number) => { + table.appendEvent({ + sessionId: fork.forkSessionId, + name: 'fork/late-tail', + data: { value: 2 } + }) + return getBoundedEntries(sessionId, maxEntryId) + } + ) + + expect(service.mergeFork('parent', 'bounded')).toBe(1) + expect(service.mergeFork('parent', 'bounded')).toBe(1) + + const parentEntries = entries.filter((entry) => entry.session_id === 'parent') + expect(parentEntries.filter((entry) => entry.source_type === 'fork')).toHaveLength(2) + expect(parentEntries.some((entry) => entry.name === 'fork/late-tail')).toBe(false) + const receipt = parentEntries.find((entry) => entry.name === 'fork/merge')! + expect(JSON.parse(receipt.payload_json).data).toMatchObject({ + forkHeadEntryId: 3, + mergedCount: 1 + }) + }) + + it('rolls back mocked fork merge writes when a copied entry fails', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('parent', 'mock-atomic') + service.appendForkMessageRecord( + fork, + createRecord({ id: 'first', orderSeq: 1, sessionId: 'ignored' }) + ) + service.appendForkMessageRecord( + fork, + createRecord({ id: 'second', orderSeq: 2, sessionId: 'ignored' }) + ) + + const append = table.append.getMockImplementation()! + let copiedEntryCount = 0 + table.append.mockImplementation((input: any) => { + if (input.sessionId === 'parent' && input.source?.type === 'fork') { + copiedEntryCount += 1 + if (copiedEntryCount === 2) { + throw new Error('injected merge failure') + } + } + return append(input) + }) + + expect(() => service.mergeFork('parent', 'mock-atomic')).toThrow('injected merge failure') + expect( + entries.filter( + (entry) => + entry.session_id === 'parent' && + (entry.source_type === 'fork' || entry.name === 'fork/merge') + ) + ).toEqual([]) + }) + + it('rolls back copied fork entries when the merge receipt fails', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('parent', 'receipt-failure') + service.appendForkMessageRecord(fork, createRecord({ id: 'first', sessionId: 'ignored' })) + + const appendEvent = table.appendEvent.getMockImplementation()! + table.appendEvent.mockImplementation((input: any) => { + if (input.sessionId === 'parent' && input.name === 'fork/merge') { + throw new Error('injected receipt failure') + } + return appendEvent(input) + }) + + expect(() => service.mergeFork('parent', 'receipt-failure')).toThrow('injected receipt failure') + expect( + entries.filter( + (entry) => + entry.session_id === 'parent' && + (entry.source_type === 'fork' || entry.name === 'fork/merge') + ) + ).toEqual([]) + }) + + it('commits one idempotent receipt for an empty fork', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + service.createFork('parent', 'empty') + + expect(service.mergeFork('parent', 'empty')).toBe(0) + expect(service.mergeFork('parent', 'empty')).toBe(0) + const parentEntries = entries.filter((entry) => entry.session_id === 'parent') + expect(parentEntries).toHaveLength(1) + expect(parentEntries[0].name).toBe('fork/merge') + expect(JSON.parse(parentEntries[0].payload_json).data).toMatchObject({ + forkHeadEntryId: 2, + mergedCount: 0 + }) + }) + + it('rejects missing and discarded forks without committing an empty merge receipt', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + expect(() => service.mergeFork('parent', 'missing')).toThrow( + 'Fork missing does not exist or has been discarded.' + ) + + service.createFork('parent', 'discarded') + service.discardFork('parent', 'discarded') + expect(() => service.mergeFork('parent', 'discarded')).toThrow( + 'Fork discarded does not exist or has been discarded.' + ) + expect( + entries.filter((entry) => entry.session_id === 'parent' && entry.name === 'fork/merge') + ).toEqual([]) + }) + + it('returns an existing merge receipt after the merged fork Tape is cleaned up', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('parent', 'cleanup-after-merge') + service.appendForkMessageRecord(fork, createRecord({ id: 'merged', sessionId: 'ignored' })) + + expect(service.mergeFork('parent', 'cleanup-after-merge')).toBe(1) + table.deleteBySession(fork.forkSessionId) + expect(service.mergeFork('parent', 'cleanup-after-merge')).toBe(1) + }) + + it('merges a legacy fork start that predates the parent head field', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('parent', 'legacy-start') + const start = entries.find( + (entry) => entry.session_id === fork.forkSessionId && entry.name === 'fork/start' + )! + const payload = JSON.parse(start.payload_json) + delete payload.state.parentHeadEntryId + start.payload_json = JSON.stringify(payload) + service.appendForkMessageRecord(fork, createRecord({ id: 'legacy', sessionId: 'ignored' })) + + expect(service.mergeFork('parent', 'legacy-start')).toBe(1) + }) + + it('accepts a valid legacy fork merge receipt without a frozen head', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/merge', + source: { type: 'fork', id: 'legacy-receipt', seq: 0 }, + provenanceKey: 'fork:parent:legacy-receipt:merge:event', + data: { + forkId: 'legacy-receipt', + forkSessionId: 'parent::fork::legacy-receipt', + mergedCount: 2 + } + }) + + expect(service.mergeFork('parent', 'legacy-receipt')).toBe(2) + }) + + it('rejects a malformed stored fork merge receipt instead of reporting an empty merge', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/merge', + source: { type: 'fork', id: 'malformed-receipt', seq: 0 }, + provenanceKey: 'fork:parent:malformed-receipt:merge:event', + data: { + forkId: 'malformed-receipt', + forkSessionId: 'parent::fork::malformed-receipt', + mergedCount: 'two' + } + }) + + expect(() => service.mergeFork('parent', 'malformed-receipt')).toThrow( + 'Stored fork merge receipt is malformed' + ) + }) + + itIfSqlite('rolls back copied fork entries and the receipt when merge fails', () => { + const db = new DatabaseCtor(':memory:') + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('parent') + const fork = service.createFork('parent', 'atomic') + service.appendForkMessageRecord( + fork, + createRecord({ id: 'first', orderSeq: 1, sessionId: 'ignored' }) + ) + service.appendForkMessageRecord( + fork, + createRecord({ id: 'second', orderSeq: 2, sessionId: 'ignored' }) + ) + + const append = table.append.bind(table) + let copiedEntryCount = 0 + const appendSpy = vi.spyOn(table, 'append').mockImplementation((input) => { + if (input.sessionId === 'parent' && input.source?.type === 'fork') { + copiedEntryCount += 1 + if (copiedEntryCount === 2) { + throw new Error('injected merge failure') + } + } + return append(input) + }) + + expect(() => service.mergeFork('parent', 'atomic')).toThrow('injected merge failure') + expect( + table + .getBySession('parent') + .filter((entry) => entry.source_type === 'fork' || entry.name === 'fork/merge') + ).toEqual([]) + + appendSpy.mockRestore() + expect(service.mergeFork('parent', 'atomic')).toBe(2) + expect(service.mergeFork('parent', 'atomic')).toBe(2) + expect( + table.getBySession('parent').filter((entry) => entry.source_type === 'fork') + ).toHaveLength(3) + + db.close() + }) + + itIfSqlite('rejects missing and discarded forks in SQLite', () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + expect(() => service.mergeFork('parent', 'missing-native')).toThrow( + 'Fork missing-native does not exist or has been discarded.' + ) + service.createFork('parent', 'discarded-native') + service.discardFork('parent', 'discarded-native') + expect(() => service.mergeFork('parent', 'discarded-native')).toThrow( + 'Fork discarded-native does not exist or has been discarded.' + ) + expect(table.getBySession('parent').some((entry) => entry.name === 'fork/merge')).toBe(false) + } finally { + db.close() + } + }) + + it('cleans fork search projection on discard without blocking the discard event', () => { + const { table, entries } = createTapeTableMock() + const projectionTable = { + deleteBySession: vi.fn(() => { + throw new Error('projection cleanup failed') + }) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const fork = service.createFork('s1', 'fork-cleanup') + service.appendForkMessageRecord(fork, createRecord({ id: 'fu-cleanup', sessionId: 'ignored' })) + service.discardFork('s1', 'fork-cleanup') + + expect(table.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) + expect(projectionTable.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) + expect(entries.some((entry) => entry.session_id === fork.forkSessionId)).toBe(false) + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') + ).toBe(true) + }) +}) diff --git a/test/main/session/data/tapeLineage.test.ts b/test/main/session/data/tapeLineage.test.ts new file mode 100644 index 0000000000..19dbfe2d3e --- /dev/null +++ b/test/main/session/data/tapeLineage.test.ts @@ -0,0 +1,710 @@ +import { + describe, + expect, + it, + vi, + SessionTape, + DeepChatTapeEntriesTable, + DatabaseCtor, + itIfSqlite, + createTapeTableMock, + createLinkedTapeService, + createSubagentLinkInput +} from './tapeTestHarness' + +describe('SessionTape lineage', () => { + it('links a frozen subagent Tape without copying child entries and retries idempotently', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ sessionId: 'child', name: 'child/result', data: { text: 'done' } }) + const input = { + parentSessionId: 'parent', + childSessionId: 'child', + runId: 'run-1', + taskId: 'task-1', + slotId: 'reviewer', + taskTitle: 'Review', + outcome: 'completed' as const, + resultSummary: 'Done' + } + const first = service.linkSubagentTape(input) + table.appendEvent({ sessionId: 'child', name: 'child/late', data: { text: 'late' } }) + const retry = service.linkSubagentTape(input) + const normalizedRetry = service.linkSubagentTape({ + ...input, + slotId: ' reviewer ', + taskTitle: ' Review ', + resultSummary: ' Done ' + }) + + expect(first).toEqual({ + linkEntry: { sessionId: 'parent', entryId: 2 }, + childSessionId: 'child', + childHeadEntryId: 2, + childEntryCount: 2, + outcome: 'completed' + }) + expect(retry).toEqual(first) + expect(normalizedRetry).toEqual(first) + const links = entries.filter( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + ) + expect(links).toHaveLength(1) + expect(links[0]).toMatchObject({ + source_type: 'subagent', + source_id: 'child', + source_seq: 2 + }) + expect(JSON.parse(links[0].payload_json).data).toMatchObject({ + linkVersion: 2, + childTapeIdentity: expect.stringMatching(/^[a-f0-9]{64}$/) + }) + expect( + entries.some((entry) => entry.session_id === 'parent' && entry.name === 'child/result') + ).toBe(false) + expect(() => service.linkSubagentTape({ ...input, outcome: 'error' })).toThrow( + 'Subagent Tape link conflicts with finalized task run-1/task-1.' + ) + expect(() => service.linkSubagentTape({ ...input, slotId: 'writer' })).toThrow( + 'Subagent Tape link conflicts with finalized task run-1/task-1.' + ) + expect(() => service.linkSubagentTape({ ...input, taskTitle: 'Write' })).toThrow( + 'Subagent Tape link conflicts with finalized task run-1/task-1.' + ) + expect(() => service.linkSubagentTape({ ...input, resultSummary: 'Changed' })).toThrow( + 'Subagent Tape link conflicts with finalized task run-1/task-1.' + ) + + table.ensureBootstrapAnchor('child-2') + expect( + service.linkSubagentTape({ + ...input, + childSessionId: 'child-2', + outcome: 'error' + }) + ).toMatchObject({ + childSessionId: 'child-2', + outcome: 'error' + }) + expect( + entries.filter( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + ) + ).toHaveLength(2) + }) + + it('rejects a stored subagent link whose task identity no longer matches its provenance', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + const input = createSubagentLinkInput('parent', 'child') + service.linkSubagentTape(input) + + const link = entries.find( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + ) + if (!link) { + throw new Error('Expected subagent Tape link fixture.') + } + const payload = JSON.parse(link.payload_json) as { + data?: Record + } + link.payload_json = JSON.stringify({ + ...payload, + data: { ...payload.data, runId: 'different-run' } + }) + + expect(() => service.linkSubagentTape(input)).toThrow( + /Stored subagent Tape link receipt is malformed/ + ) + expect(() => service.getContext('parent', [1], { sourceSessionId: 'child' })).toThrowError( + /not an authorized direct child/ + ) + }) + + it('keeps a version-one link readable while its original unmarked Tape remains present', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/result', + data: { text: 'version one compatibility marker' } + }) + const input = createSubagentLinkInput('parent', 'child') + const receipt = service.linkSubagentTape(input) + const link = entries.find( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + )! + const payload = JSON.parse(link.payload_json) + payload.data.linkVersion = 1 + delete payload.data.childTapeIdentity + link.payload_json = JSON.stringify(payload) + entries.find((entry) => entry.session_id === 'child' && entry.entry_id === 1)!.meta_json = '{}' + + expect(service.linkSubagentTape(input)).toEqual(receipt) + expect( + service.search('parent', 'version one compatibility marker', { + scope: 'linked_subagents' + }) + ).toMatchObject([{ sessionId: 'child', entryId: 2 }]) + }) + + it('fails closed when a version-one link points at malformed incarnation metadata', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/result', + data: { text: 'malformed incarnation metadata marker' } + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + const link = entries.find( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + )! + const payload = JSON.parse(link.payload_json) + payload.data.linkVersion = 1 + delete payload.data.childTapeIdentity + link.payload_json = JSON.stringify(payload) + entries.find((entry) => entry.session_id === 'child' && entry.entry_id === 1)!.meta_json = '{' + + expect(() => + service.search('parent', 'malformed incarnation metadata marker', { + scope: 'linked_subagents' + }) + ).toThrowError(/Linked Tape child is unavailable/) + }) + + it('searches direct linked children at frozen heads with one global limit', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child-a', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'child-b', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child-a') + table.ensureBootstrapAnchor('child-b') + table.appendEvent({ + sessionId: 'parent', + name: 'parent/note', + data: { text: 'shared needle from parent' }, + createdAt: 150 + }) + table.appendEvent({ + sessionId: 'child-a', + name: 'child/result', + data: { text: 'shared needle from A' }, + createdAt: 100 + }) + table.appendEvent({ + sessionId: 'child-b', + name: 'child/result', + data: { text: 'shared needle from B' }, + createdAt: 200 + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child-a')) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child-b')) + table.appendEvent({ + sessionId: 'child-a', + name: 'child/late', + data: { text: 'shared needle after cutoff' }, + createdAt: 300 + }) + + table.ensureBootstrapAnchor.mockClear() + table.append.mockClear() + const limited = service.search('parent', 'shared needle', { + scope: 'linked_subagents', + limit: 1 + }) + const all = service.search('parent', 'shared needle', { + scope: 'linked_subagents', + limit: 10 + }) + const combined = service.search('parent', 'shared needle', { + scope: 'current_and_linked', + limit: 10 + }) + + expect(limited).toMatchObject([{ sessionId: 'child-b', entryId: 2 }]) + expect(all.map((result) => [result.sessionId, result.entryId])).toEqual([ + ['child-b', 2], + ['child-a', 2] + ]) + expect(combined.map((result) => [result.sessionId, result.entryId])).toEqual([ + ['child-b', 2], + ['parent', 2], + ['child-a', 2] + ]) + expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( + [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 2 } + ], + 'shared needle', + expect.objectContaining({ limit: 1 }) + ) + expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() + expect(table.append).not.toHaveBeenCalled() + }) + + it('falls back all linked sources together when projection coverage is partial', () => { + const { table } = createTapeTableMock() + const projectionTable = { + searchSourcesReadOnly: vi.fn(() => ({ + coveredSources: [{ sessionId: 'child-a', maxEntryId: 2 }], + rows: [ + { + session_id: 'child-a', + entry_id: 2, + kind: 'event', + name: 'child/result', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'shared partial needle', + summary_text: 'shared partial needle from A', + refs_json: '{}', + created_at: 100, + score: -100 + } + ] + })) + } + const { service } = createLinkedTapeService( + table, + [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child-a', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'child-b', session_kind: 'subagent', parent_session_id: 'parent' } + ], + projectionTable + ) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child-a') + table.ensureBootstrapAnchor('child-b') + table.appendEvent({ + sessionId: 'child-a', + name: 'child/result', + data: { text: 'shared partial needle from A' }, + createdAt: 100 + }) + table.appendEvent({ + sessionId: 'child-b', + name: 'child/result', + data: { text: 'shared partial needle from B' }, + createdAt: 200 + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child-a')) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child-b')) + table.searchEffectiveSourcesAtHeads.mockClear() + + const hits = service.search('parent', 'shared partial needle', { + scope: 'linked_subagents', + limit: 1 + }) + + expect(hits).toMatchObject([{ sessionId: 'child-b', entryId: 2 }]) + expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( + [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 2 } + ], + 'shared partial needle', + expect.objectContaining({ limit: 1 }) + ) + }) + + it('uses an exact frozen projection read-only after the child appends a late tail', () => { + const { table } = createTapeTableMock() + const projectionTable = { + searchSourcesReadOnly: vi.fn((sources: any[]) => ({ + coveredSources: sources, + rows: [ + { + session_id: 'child', + entry_id: 2, + kind: 'event', + name: 'child/result', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'frozen projection needle', + summary_text: 'frozen projection needle', + refs_json: '{}', + created_at: 100, + score: -1 + } + ] + })), + replaceSession: vi.fn(), + appendSession: vi.fn(), + getByEntryIds: vi.fn().mockReturnValue([]) + } + const { service } = createLinkedTapeService( + table, + [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ], + projectionTable + ) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/result', + data: { text: 'frozen projection needle' }, + createdAt: 100 + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + table.appendEvent({ + sessionId: 'child', + name: 'child/late', + data: { text: 'frozen projection needle late' }, + createdAt: 200 + }) + table.searchEffectiveSourcesAtHeads.mockClear() + + const hits = service.search('parent', 'frozen projection needle', { + scope: 'linked_subagents' + }) + + expect(projectionTable.searchSourcesReadOnly).toHaveBeenCalledWith( + [{ sessionId: 'child', maxEntryId: 2 }], + 'frozen projection needle', + expect.any(Object) + ) + expect(hits).toMatchObject([{ sessionId: 'child', entryId: 2 }]) + expect(table.searchEffectiveSourcesAtHeads).not.toHaveBeenCalled() + expect(projectionTable.replaceSession).not.toHaveBeenCalled() + expect(projectionTable.appendSession).not.toHaveBeenCalled() + }) + + it('deduplicates repeated child links at the newest finalized snapshot', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/first', + data: { text: 'first snapshot marker' } + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + table.appendEvent({ + sessionId: 'child', + name: 'child/second', + data: { text: 'second snapshot marker' } + }) + service.linkSubagentTape({ + ...createSubagentLinkInput('parent', 'child'), + runId: 'run-child-second', + taskId: 'task-child-second' + }) + + expect( + service.search('parent', 'second snapshot marker', { scope: 'linked_subagents' }) + ).toMatchObject([{ sessionId: 'child', entryId: 3 }]) + expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( + [{ sessionId: 'child', maxEntryId: 3 }], + 'second snapshot marker', + expect.any(Object) + ) + }) + + it('expands linked context within one source and never crosses the frozen head', () => { + const { table } = createTapeTableMock() + const projectionTable = { + getByEntryIds: vi.fn(() => [ + { + session_id: 'child', + entry_id: 2, + kind: 'event', + name: 'child/target', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'stale projection text', + summary_text: 'stale projection summary', + refs_json: '{"stale":true}', + created_at: 100 + } + ]) + } + const { service } = createLinkedTapeService( + table, + [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ], + projectionTable + ) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/target', + data: { text: 'target evidence' }, + createdAt: 100 + }) + table.appendEvent({ + sessionId: 'child', + name: 'child/neighbor', + data: { text: 'neighbor evidence' }, + createdAt: 110 + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + table.appendEvent({ + sessionId: 'child', + name: 'child/late', + data: { text: 'late evidence' }, + createdAt: 120 + }) + table.append.mockClear() + + const context = service.getContext('parent', [2], { + sourceSessionId: 'child', + before: 0, + after: 5 + }) + + expect(context).toMatchObject({ + sessionId: 'parent', + sourceSessionId: 'child', + requestedEntryIds: [2], + matchedEntryIds: [2] + }) + expect(context.entries.map((entry) => entry.entryId)).toEqual([2, 3]) + expect(context.entries[0].summary).toContain('target evidence') + expect(context.entries[0].summary).not.toContain('stale projection') + expect(table.getEffectiveContextRowsAtHead).toHaveBeenCalledWith( + { sessionId: 'child', maxEntryId: 3 }, + [2], + expect.objectContaining({ before: 0, after: 5 }) + ) + expect(projectionTable.getByEntryIds).not.toHaveBeenCalled() + expect(table.append).not.toHaveBeenCalled() + }) + + it('rejects sibling and unlinked source ids even when a forged link event exists', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'other-parent', session_kind: 'regular', parent_session_id: null }, + { id: 'sibling', session_kind: 'subagent', parent_session_id: 'other-parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('sibling') + service.linkSubagentTape(createSubagentLinkInput('parent', 'sibling')) + + let error: unknown + try { + service.getContext('parent', [1], { sourceSessionId: 'sibling' }) + } catch (caught) { + error = caught + } + expect(error).toMatchObject({ + code: 'linked_tape_unauthorized', + parentSessionId: 'parent', + sourceSessionId: 'sibling' + }) + }) + + it('reports a finalized linked Tape as unavailable after its durable session is deleted', () => { + const { table } = createTapeTableMock() + const { service, sessionById } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + sessionById.delete('child') + + let error: unknown + try { + service.search('parent', 'anything', { scope: 'linked_subagents' }) + } catch (caught) { + error = caught + } + expect(error).toMatchObject({ + code: 'linked_tape_unavailable', + parentSessionId: 'parent', + sourceSessionId: 'child' + }) + }) + + it('rejects a rebuilt child Tape until a new incarnation is explicitly linked', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/original', + data: { text: 'original incarnation marker' } + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + + table.deleteBySession('child') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/rebuilt', + data: { text: 'rebuilt incarnation marker' } + }) + expect( + service.search('child', 'rebuilt incarnation marker', { scope: 'current' }) + ).toHaveLength(1) + expect(() => + service.search('parent', 'rebuilt incarnation marker', { scope: 'linked_subagents' }) + ).toThrowError(/Linked Tape child is unavailable/) + + service.linkSubagentTape({ + ...createSubagentLinkInput('parent', 'child'), + runId: 'run-rebuilt-child', + taskId: 'task-rebuilt-child' + }) + expect( + service.search('parent', 'rebuilt incarnation marker', { scope: 'linked_subagents' }) + ).toMatchObject([{ sessionId: 'child', entryId: 2 }]) + }) + + itIfSqlite('detects linked Tape replacement after entry ids are reused in SQLite', () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/original', + data: { text: 'native original incarnation' } + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + + table.deleteBySession('child') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/rebuilt', + data: { text: 'native rebuilt incarnation' } + }) + + expect(() => + service.search('parent', 'native rebuilt incarnation', { scope: 'linked_subagents' }) + ).toThrowError(/Linked Tape child is unavailable/) + } finally { + db.close() + } + }) + + it('reads legacy external merge links and keeps legacy discard audit-only', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'legacy-child', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'malformed-child', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'discarded-child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('legacy-child') + table.ensureBootstrapAnchor('malformed-child') + table.ensureBootstrapAnchor('discarded-child') + const legacyBootstrap = entries.find( + (entry) => entry.session_id === 'legacy-child' && entry.entry_id === 1 + )! + legacyBootstrap.meta_json = '{}' + table.appendEvent({ + sessionId: 'legacy-child', + name: 'child/result', + data: { text: 'legacy link needle' } + }) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/merge', + source: { type: 'fork', id: 'legacy-child', seq: 0 }, + provenanceKey: 'fork:parent:legacy-child:external-merge:event', + data: { + forkId: 'legacy-child', + forkSessionId: 'legacy-child', + referencedEntryCount: 2, + status: 'completed' + } + }) + table.appendEvent({ + sessionId: 'malformed-child', + name: 'child/result', + data: { text: 'malformed legacy link needle' } + }) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/merge', + source: { type: 'fork', id: 'malformed-child', seq: 1 }, + provenanceKey: 'fork:parent:malformed-child:external-merge:event', + data: { + forkId: 'malformed-child', + forkSessionId: 'malformed-child', + referencedEntryCount: 2, + status: 'completed' + } + }) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/discard', + source: { type: 'fork', id: 'discarded-child', seq: 0 }, + provenanceKey: 'fork:parent:discarded-child:external-discard:event', + data: { + forkId: 'discarded-child', + forkSessionId: 'discarded-child', + status: 'cancelled' + } + }) + + expect( + service.search('parent', 'legacy link needle', { scope: 'linked_subagents' }) + ).toMatchObject([{ sessionId: 'legacy-child', entryId: 2 }]) + expect( + service.search('parent', 'malformed legacy link needle', { scope: 'linked_subagents' }) + ).toEqual([]) + let error: unknown + try { + service.getContext('parent', [1], { sourceSessionId: 'discarded-child' }) + } catch (caught) { + error = caught + } + expect(error).toMatchObject({ code: 'linked_tape_unauthorized' }) + }) +}) diff --git a/test/main/session/data/tapeRecall.test.ts b/test/main/session/data/tapeRecall.test.ts new file mode 100644 index 0000000000..247b8cfdcd --- /dev/null +++ b/test/main/session/data/tapeRecall.test.ts @@ -0,0 +1,2077 @@ +import { + performance, + describe, + expect, + it, + vi, + SessionTape, + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + DeepChatTapeEntriesTable, + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, + DeepChatTapeSearchProjectionTable, + NewSessionsTable, + DatabaseCtor, + sqliteAvailable, + sqliteSkipReason, + itIfSqlite, + createTapeTableMock, + createRecord, + createTapeService +} from './tapeTestHarness' + +describe('SessionTape recall', () => { + it('reports info, search, and handoff within one session scope', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([ + createRecord({ id: 'u1' }), + createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { type: 'content', content: 'answer', status: 'success', timestamp: 101 } + ]), + metadata: JSON.stringify({ totalTokens: 9 }), + createdAt: 101, + updatedAt: 101 + }) + ]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + service.handoff('s1', 'phase_done', { summary: ' done ' }) + const handoffAnchor = entries.find((entry) => entry.name === 'handoff/phase_done') + + expect(service.info('s1')).toMatchObject({ + sessionId: 's1', + anchors: 2, + lastAnchor: 'handoff/phase_done', + lastTokenUsage: 9, + migrationState: 'ready' + }) + expect(JSON.parse(handoffAnchor.payload_json).state).toMatchObject({ + summary: 'done', + cursorOrderSeq: 3, + range: { + fromOrderSeq: 1, + toOrderSeq: 2 + }, + sourceMessageIds: ['u1', 'a1'] + }) + expect(service.search('s1', 'hello')).toHaveLength(1) + expect( + service.search('s1', 'hello', { kinds: ['message'], start: '1970-01-01T00:00:00.000Z' }) + ).toHaveLength(1) + expect(service.search('s1', 'hello', { kinds: ['anchor'] })).toHaveLength(0) + expect(service.search('s1', 'hello', { end: '99' })).toHaveLength(0) + expect(() => service.search('s1', 'hello', { start: 'not-a-date' })).toThrow( + 'start must be an ISO date/time or millisecond timestamp.' + ) + expect(service.anchors('s1')).toMatchObject([ + { sessionId: 's1', name: 'session/start' }, + { sessionId: 's1', name: 'handoff/phase_done' } + ]) + expect(service.anchors('s1', { limit: 1 })).toMatchObject([ + { sessionId: 's1', name: 'handoff/phase_done' } + ]) + expect(service.search('s2', 'hello')).toHaveLength(0) + }) + + it('projects fallback tape search into compact results and bounded context', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'Run the dev server', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.append({ + sessionId: 's1', + kind: 'tool_result', + name: 'shell', + source: { type: 'tool_result', id: 'm1:tc1', seq: 0 }, + payload: { + messageId: 'm1', + orderSeq: 2, + toolCallId: 'tc1', + command: 'pnpm run dev', + exitStatus: 1, + response: 'Command failed with EADDRINUSE in /tmp/deepchat.log' + }, + meta: { source: 'live', status: 'error' }, + createdAt: 110 + }) + + const hits = service.search('s1', 'pnpm run dev', { limit: 5 }) + expect(hits).toHaveLength(1) + expect(hits[0]).toMatchObject({ + kind: 'tool_result', + summary: expect.stringContaining('EADDRINUSE'), + refs: { + toolCallId: 'tc1', + commands: expect.arrayContaining(['pnpm run dev']), + filePaths: expect.arrayContaining(['/tmp/deepchat.log']), + errorCodes: expect.arrayContaining(['EADDRINUSE']), + exitStatus: 1 + } + }) + expect(hits[0]).not.toHaveProperty('payload') + expect(hits[0]).not.toHaveProperty('meta') + + const context = service.getContext('s1', [hits[0].entryId], { + before: 0, + after: 0, + maxBytesPerEntry: 12, + maxTotalBytes: 12 + }) + expect(context.matchedEntryIds).toEqual([hits[0].entryId]) + expect(context.entries[0]).toMatchObject({ + entryId: hits[0].entryId, + evidence: { truncated: true } + }) + expect(context.entries[0].evidence.bytes).toBeLessThanOrEqual(12) + expect(context.entries[0]).not.toHaveProperty('payload') + expect(context.entries[0]).not.toHaveProperty('meta') + + const exhaustedContext = service.getContext('s1', [hits[0].entryId], { + before: 0, + after: 0, + maxTotalBytes: 0 + }) + expect(exhaustedContext.entries).toEqual([]) + expect(exhaustedContext.matchedEntryIds).toEqual([]) + }) + + it('projects user message attachment metadata into search text and refs', () => { + const { table } = createTapeTableMock() + const projectionTable = { + isCurrent: vi.fn().mockReturnValue(false), + getSessionMeta: vi.fn().mockReturnValue(null), + getProjectedEntryIds: vi.fn().mockReturnValue([]), + appendSession: vi.fn(), + replaceSession: vi.fn(), + search: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm-file', seq: 0 }, + payload: { + record: createRecord({ + id: 'm-file', + content: JSON.stringify({ + text: 'Please review the attachment', + files: [ + { + name: 'a.md', + path: '/tmp/a.md', + content: 'raw attachment body should not be projected', + metadata: { fileName: 'workspace-a.md' } + } + ], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + + service.search('s1', '/tmp/a.md', { limit: 5 }) + + expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) + const projectedRows = projectionTable.replaceSession.mock.calls[0][1] + expect(projectedRows[0]).toMatchObject({ + entryId: 1, + refs: { + filePaths: expect.arrayContaining(['/tmp/a.md']), + fileNames: expect.arrayContaining(['a.md', 'workspace-a.md']) + } + }) + expect(projectedRows[0].searchText).toContain('/tmp/a.md') + expect(projectedRows[0].searchText).toContain('a.md') + expect(projectedRows[0].searchText).toContain('workspace-a.md') + expect(projectedRows[0].searchText).not.toContain('raw attachment body should not be projected') + }) + + it('preserves relative file paths in projected refs', () => { + const { table } = createTapeTableMock() + let projectedRows: any[] = [] + const projectionTable = { + isCurrent: vi.fn().mockReturnValue(false), + getSessionMeta: vi.fn().mockReturnValue(null), + getProjectedEntryIds: vi.fn().mockReturnValue([]), + appendSession: vi.fn(), + replaceSession: vi.fn((_sessionId: string, rows: any[]) => { + projectedRows = rows + }), + search: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm-relative', seq: 0 }, + payload: { + record: createRecord({ + id: 'm-relative', + content: JSON.stringify({ + text: 'Touched src/main/index.ts, ./lib/util.ts, ../shared/types.ts, test/main/foo/bar.ts, /usr/local/bin/deploy, and https://example.com/not-a-file', + files: [], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + + service.search('s1', 'src/main/index.ts', { limit: 5 }) + + expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) + const filePaths = projectedRows[0].refs.filePaths + expect(filePaths).toEqual( + expect.arrayContaining([ + 'src/main/index.ts', + './lib/util.ts', + '../shared/types.ts', + 'test/main/foo/bar.ts', + '/usr/local/bin/deploy' + ]) + ) + expect(filePaths).not.toContain('/main/index.ts') + expect(filePaths).not.toContain('/lib/util.ts') + expect(filePaths).not.toContain('/main/foo/bar.ts') + expect(filePaths).not.toContain('example.com/not-a-file') + }) + + it('rebuilds old tape projection versions before attachment path search', () => { + const { table } = createTapeTableMock() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm-file', seq: 0 }, + payload: { + record: createRecord({ + id: 'm-file', + content: JSON.stringify({ + text: 'Please review the migrated attachment', + files: [ + { + name: 'legacy.md', + path: '/tmp/legacy.md', + content: 'raw migrated body must stay out of projection', + metadata: { fileName: 'legacy-workspace.md' } + } + ], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + let storedVersion = 1 + let storedMaxEntryId = 1 + let projectedRows: any[] = [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm-file', + sourceSeq: 0, + searchText: 'message/user Please review the migrated attachment', + summaryText: 'user: Please review the migrated attachment', + refs: { messageId: 'm-file' }, + createdAt: 100 + } + ] + const projectionTable = { + isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => { + return ( + storedVersion === DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION && + storedMaxEntryId === maxEntryId + ) + }), + getSessionMeta: vi.fn(() => ({ + projectionVersion: storedVersion, + maxEntryId: storedMaxEntryId + })), + getProjectedEntryIds: vi.fn().mockReturnValue([1]), + appendSession: vi.fn(), + replaceSession: vi.fn((_sessionId: string, rows: any[], maxEntryId: number) => { + projectedRows = rows + storedVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + storedMaxEntryId = maxEntryId + }), + search: vi.fn((_sessionId: string, query: string) => { + return projectedRows + .filter((row) => row.searchText.includes(query)) + .map((row) => ({ + session_id: row.sessionId, + entry_id: row.entryId, + kind: row.kind, + name: row.name, + source_type: row.sourceType, + source_id: row.sourceId, + source_seq: row.sourceSeq, + search_text: row.searchText, + summary_text: row.summaryText, + refs_json: JSON.stringify(row.refs), + created_at: row.createdAt, + score: 1 + })) + }) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const hits = service.search('s1', '/tmp/legacy.md', { limit: 5 }) + + expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) + expect(projectionTable.appendSession).not.toHaveBeenCalled() + expect(hits).toHaveLength(1) + expect(hits[0].refs).toMatchObject({ + filePaths: expect.arrayContaining(['/tmp/legacy.md']), + fileNames: expect.arrayContaining(['legacy.md', 'legacy-workspace.md']) + }) + expect(projectedRows[0].searchText).not.toContain( + 'raw migrated body must stay out of projection' + ) + }) + + it('prioritizes requested tape context entries before window entries under byte caps', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'before-1', seq: 0 }, + payload: { + record: createRecord({ + id: 'before-1', + content: JSON.stringify({ + text: 'before entry consumes the tiny byte budget first', + files: [], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'before-2', seq: 0 }, + payload: { + record: createRecord({ + id: 'before-2', + content: JSON.stringify({ + text: 'second before entry also appears earlier', + files: [], + links: [] + }), + createdAt: 110, + updatedAt: 110 + }) + }, + meta: { source: 'live', orderSeq: 2, role: 'user' }, + createdAt: 110 + }) + const target = table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'target', seq: 0 }, + payload: { + record: createRecord({ + id: 'target', + content: JSON.stringify({ + text: 'target-ok consumes the available budget', + files: [], + links: [] + }), + createdAt: 120, + updatedAt: 120 + }) + }, + meta: { source: 'live', orderSeq: 3, role: 'user' }, + createdAt: 120 + }) + + const context = service.getContext('s1', [target.entry_id], { + before: 2, + after: 0, + limit: 3, + maxBytesPerEntry: 18, + maxTotalBytes: 18 + }) + + expect(context.matchedEntryIds).toEqual([target.entry_id]) + expect(context.entries.map((entry) => entry.entryId)).toEqual([target.entry_id]) + expect(context.entries[0].evidence.text).toContain('target-ok') + }) + + it('binds tape projection LIKE fallback params for single and multi-term queries', () => { + const all = vi.fn().mockReturnValue([]) + const db = { + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => all(sql, params), + get: vi.fn().mockReturnValue(undefined), + run: vi.fn() + })) + } + const projectionTable = new DeepChatTapeSearchProjectionTable(db as any) + ;(projectionTable as any).recoverSessionFts = vi.fn() + + projectionTable.search('s1', 'Redis', { limit: 5 }) + expect(all).toHaveBeenLastCalledWith( + expect.stringContaining('FROM deepchat_tape_search_projection'), + ['s1', '%Redis%', '%Redis%', '%Redis%', 5] + ) + + projectionTable.search('s1', 'Redis TTL', { limit: 5 }) + expect(all).toHaveBeenLastCalledWith( + expect.stringContaining('FROM deepchat_tape_search_projection'), + [ + 's1', + '%Redis TTL%', + '%Redis TTL%', + '%Redis TTL%', + '%Redis%', + '%Redis%', + '%Redis%', + '%TTL%', + '%TTL%', + '%TTL%', + 5 + ] + ) + + const oversizedQuery = Array.from({ length: 257 }, (_, index) => `term-${index}`).join(' ') + projectionTable.search('s1', oversizedQuery, { limit: 5 }) + expect(all).toHaveBeenLastCalledWith( + expect.stringContaining('FROM deepchat_tape_search_projection'), + ['s1', `%${oversizedQuery}%`, `%${oversizedQuery}%`, `%${oversizedQuery}%`, 5] + ) + }) + + it('uses current tape projection without loading full session rows', () => { + const { table } = createTapeTableMock() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'Redis compact marker', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.getBySession.mockClear() + const projectionTable = { + isCurrent: vi.fn().mockReturnValue(true), + getSessionMeta: vi.fn(), + getProjectedEntryIds: vi.fn(), + appendSession: vi.fn(), + replaceSession: vi.fn(), + search: vi.fn().mockReturnValue([ + { + session_id: 's1', + entry_id: 1, + kind: 'message', + name: 'message/user', + source_type: 'message', + source_id: 'm1', + source_seq: 0, + search_text: 'Redis compact marker', + summary_text: 'Redis compact marker', + refs_json: '{"messageId":"m1"}', + created_at: 100, + score: -2 + } + ]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const hits = service.search('s1', 'Redis compact', { limit: 5 }) + + expect(table.getMaxEntryId).toHaveBeenCalledWith('s1') + expect(table.getBySession).not.toHaveBeenCalled() + expect(projectionTable.search).toHaveBeenCalledWith( + 's1', + 'Redis compact', + expect.objectContaining({ limit: 5 }) + ) + expect(projectionTable.appendSession).not.toHaveBeenCalled() + expect(projectionTable.replaceSession).not.toHaveBeenCalled() + expect(hits[0]).toMatchObject({ + entryId: 1, + kind: 'message', + summary: 'Redis compact marker', + refs: { messageId: 'm1' }, + score: -2 + }) + expect(hits[0]).not.toHaveProperty('payload') + expect(hits[0]).not.toHaveProperty('meta') + }) + + it('falls back to effective tape search when projection search throws', () => { + const { table } = createTapeTableMock() + const projectionTable = { + isCurrent: vi.fn().mockReturnValue(true), + search: vi.fn(() => { + throw new Error('projection failed') + }) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'Redis fallback marker', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + + const hits = service.search('s1', 'Redis fallback', { limit: 5 }) + expect(hits).toHaveLength(1) + expect(hits[0]).toMatchObject({ + kind: 'message', + summary: expect.stringContaining('Redis fallback') + }) + expect(hits[0]).not.toHaveProperty('payload') + expect(hits[0]).not.toHaveProperty('meta') + }) + + it('appends tape projection rows when the previous projection is an effective prefix', () => { + const { table } = createTapeTableMock() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'first redis', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm2', seq: 0 }, + payload: { + record: createRecord({ + id: 'm2', + content: JSON.stringify({ text: 'second vue', files: [], links: [] }), + createdAt: 110, + updatedAt: 110 + }) + }, + meta: { source: 'live', orderSeq: 2, role: 'user' }, + createdAt: 110 + }) + const projectionTable = { + isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => maxEntryId === 1), + getSessionMeta: vi.fn().mockReturnValue({ projectionVersion: 1, maxEntryId: 1 }), + getProjectedEntryIds: vi.fn().mockReturnValue([1]), + appendSession: vi.fn(), + replaceSession: vi.fn(), + search: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.search('s1', 'vue', { limit: 5 }) + + expect(projectionTable.appendSession).toHaveBeenCalledTimes(1) + expect(projectionTable.appendSession.mock.calls[0][1].map((row: any) => row.entryId)).toEqual([ + 2 + ]) + expect(projectionTable.replaceSession).not.toHaveBeenCalled() + }) + + it('rebuilds tape projection when projected entry ids are not an effective prefix', () => { + const { table } = createTapeTableMock() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'redis', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + const projectionTable = { + isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => maxEntryId === 0), + getSessionMeta: vi.fn().mockReturnValue({ projectionVersion: 1, maxEntryId: 0 }), + getProjectedEntryIds: vi.fn().mockReturnValue([99]), + appendSession: vi.fn(), + replaceSession: vi.fn(), + search: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.search('s1', 'redis', { limit: 5 }) + + expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) + expect(projectionTable.appendSession).not.toHaveBeenCalled() + }) + + it('does not run LIKE fallback when FTS fills the tape projection search limit', () => { + const all = vi.fn((sql: string, _params: unknown[]) => + sql.includes('deepchat_tape_search_fts') + ? [ + { + session_id: 's1', + entry_id: 1, + kind: 'message', + name: 'message/user', + source_type: 'message', + source_id: 'm1', + source_seq: 0, + search_text: 'Redis TTL', + summary_text: 'Redis TTL', + refs_json: '{}', + created_at: 100, + score: -1 + } + ] + : [] + ) + const db = { + exec: vi.fn(() => { + throw new Error('unexpected FTS ensure') + }), + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => all(sql, params), + get: (..._params: unknown[]) => { + if ( + sql.includes('deepchat_tape_search_projection_meta') || + sql.includes('deepchat_tape_search_fts_meta') + ) { + return { projection_version: 1, max_entry_id: 1 } + } + return undefined + }, + run: vi.fn() + })), + transaction: vi.fn((callback: () => void) => callback) + } + const projectionTable = new DeepChatTapeSearchProjectionTable(db as any) + ;(projectionTable as any).ftsReady = true + + const hits = projectionTable.search('s1', 'Redis', { limit: 1 }) + + expect(hits).toHaveLength(1) + expect(db.exec).not.toHaveBeenCalled() + const ftsCall = all.mock.calls.find(([sql]) => String(sql).includes('deepchat_tape_search_fts')) + expect(String(ftsCall?.[0])).toContain('deepchat_tape_search_fts.session_id = ?') + expect((ftsCall?.[1] as unknown[]).filter((param) => param === 's1')).toHaveLength(2) + expect( + vi.mocked(db.prepare).mock.calls.some(([sql]) => String(sql).includes('NULL AS score')) + ).toBe(false) + }) + + it('queries memory view manifests by agent without expanding session ids', () => { + const all = vi.fn().mockReturnValue([]) + const db = { + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => all(sql, params) + })) + } + const table = new DeepChatTapeEntriesTable(db as any) + + table.listMemoryViewManifestAnchorsByAgent('agent-a', { + sessionId: 's-1', + messageId: 'msg-1', + limit: 7 + }) + + expect(all).toHaveBeenCalledWith( + expect.stringContaining('INNER JOIN new_sessions AS sessions'), + ['agent-a', 's-1', 'msg-1', 7] + ) + const sql = String(all.mock.calls[0][0]) + expect(sql).not.toContain(' IN (') + expect(sql).toContain('sessions.agent_id = ?') + expect(sql).toContain('tape.session_id = ?') + expect(sql).toContain("json_extract(tape.meta_json, '$.messageId') = ?") + }) + + it('keeps linked raw search candidate-bounded instead of materializing complete Tapes', () => { + const all = vi.fn().mockReturnValue([]) + const db = { + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => all(sql, params) + })) + } + const table = new DeepChatTapeEntriesTable(db as any) + + table.searchEffectiveSourcesAtHeads( + [ + { sessionId: 'child-b', maxEntryId: 20 }, + { sessionId: 'child-a', maxEntryId: 10 } + ], + 'needle', + { limit: 5 } + ) + + const sql = String(all.mock.calls[0][0]) + const params = all.mock.calls[0][1] + expect(sql).toContain('FROM deepchat_tape_entries AS candidate') + expect(sql).toContain('candidate.entry_id <= source.max_entry_id') + expect(sql).toContain('FROM deepchat_tape_entries AS later_message') + expect(sql).toContain( + "typeof(json_extract(later_message.payload_json, '$.record.content')) = 'text'" + ) + expect(sql).not.toContain('bounded_rows AS') + expect(params[0]).toBe( + JSON.stringify([ + { sessionId: 'child-a', maxEntryId: 10 }, + { sessionId: 'child-b', maxEntryId: 20 } + ]) + ) + expect(params.at(-1)).toBe(5) + }) + + it('reads exact linked projections without invoking FTS recovery or writes', () => { + const run = vi.fn() + const exec = vi.fn() + const reads: Array<{ sql: string; params: unknown[] }> = [] + const prepare = vi.fn((sql: string) => ({ + all: (...params: unknown[]) => { + reads.push({ sql, params }) + if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { + return [{ session_id: 'child', max_entry_id: 2 }] + } + if (sql.includes('SELECT projection.*, NULL AS score')) { + return [ + { + session_id: 'child', + entry_id: 2, + kind: 'event', + name: 'child/result', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'projection needle', + summary_text: 'projection needle', + refs_json: '{}', + created_at: 100, + score: null + } + ] + } + return [] + }, + run + })) + const projectionTable = new DeepChatTapeSearchProjectionTable({ prepare, exec } as any) + + const result = projectionTable.searchSourcesReadOnly( + [{ sessionId: 'child', maxEntryId: 2 }], + 'projection needle', + { limit: 5 } + ) + + expect(result).toMatchObject({ + coveredSources: [{ sessionId: 'child', maxEntryId: 2 }], + rows: [{ session_id: 'child', entry_id: 2 }] + }) + expect(exec).not.toHaveBeenCalled() + expect(run).not.toHaveBeenCalled() + expect(prepare.mock.calls.map(([sql]) => String(sql)).join('\n')).not.toContain( + 'deepchat_tape_search_fts_meta' + ) + expect( + reads.find((read) => read.sql.includes('SELECT projection.*, NULL AS score'))?.params.at(-1) + ).toBe(5) + }) + + it('returns no ranked rows when linked projection coverage is only partial', () => { + const reads: string[] = [] + const prepare = vi.fn((sql: string) => ({ + all: () => { + reads.push(sql) + if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { + return [{ session_id: 'child-a', max_entry_id: 2 }] + } + return [] + } + })) + const projectionTable = new DeepChatTapeSearchProjectionTable({ + prepare, + exec: vi.fn() + } as any) + + const result = projectionTable.searchSourcesReadOnly( + [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 3 } + ], + 'projection needle', + { limit: 5 } + ) + + expect(result).toEqual({ + coveredSources: [{ sessionId: 'child-a', maxEntryId: 2 }], + rows: [] + }) + expect( + reads.some((sql) => sql.includes('FROM deepchat_tape_search_projection AS projection')) + ).toBe(false) + expect(reads.some((sql) => sql.includes('FROM deepchat_tape_search_fts'))).toBe(false) + }) + + it('uses one LIKE ranking when linked FTS freshness is only partial', () => { + const reads: Array<{ sql: string; params: unknown[] }> = [] + const prepare = vi.fn((sql: string) => ({ + all: (...params: unknown[]) => { + reads.push({ sql, params }) + if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { + return [ + { session_id: 'child-a', max_entry_id: 2 }, + { session_id: 'child-b', max_entry_id: 3 } + ] + } + if (sql.includes('deepchat_tape_search_fts_meta AS meta')) { + return [{ session_id: 'child-a', max_entry_id: 2 }] + } + if (sql.includes('SELECT projection.*, NULL AS score')) { + return [ + { + session_id: 'child-b', + entry_id: 3, + kind: 'event', + name: 'child/result', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'projection needle', + summary_text: 'projection needle', + refs_json: '{}', + created_at: 200, + score: null + } + ] + } + return [] + } + })) + const projectionTable = new DeepChatTapeSearchProjectionTable({ + prepare, + exec: vi.fn() + } as any) + ;(projectionTable as any).ftsReady = true + + const sources = [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 3 } + ] + const result = projectionTable.searchSourcesReadOnly(sources, 'projection needle', { limit: 5 }) + + expect(result.rows).toMatchObject([{ session_id: 'child-b', entry_id: 3, score: null }]) + expect(reads.some(({ sql }) => sql.includes('bm25(deepchat_tape_search_fts)'))).toBe(false) + expect( + reads.find(({ sql }) => sql.includes('SELECT projection.*, NULL AS score'))?.params[0] + ).toBe(JSON.stringify(sources)) + }) + + itIfSqlite( + `keeps projected and raw linked multi-term search aligned${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + table.createTable() + projectionTable.createTable() + table.ensureBootstrapAnchor('child') + const entry = table.appendEvent({ + sessionId: 'child', + name: 'child/result', + data: { text: 'alpha separated by several words before beta' }, + createdAt: 100 + }) + const sources = [{ sessionId: 'child', maxEntryId: entry.entry_id }] + projectionTable.replaceSession( + 'child', + [ + { + sessionId: 'child', + entryId: entry.entry_id, + kind: 'event', + name: 'child/result', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'alpha separated by several words before beta', + summaryText: 'alpha separated by several words before beta', + refs: {}, + createdAt: 100 + } + ], + entry.entry_id + ) + + const projected = projectionTable.searchSourcesReadOnly(sources, 'alpha beta', { limit: 5 }) + const raw = table.searchEffectiveSourcesAtHeads(sources, 'alpha beta', { limit: 5 }) + + expect(projected.coveredSources).toEqual(sources) + expect(raw.map((row) => [row.session_id, row.entry_id])).toEqual( + projected.rows.map((row) => [row.session_id, row.entry_id]) + ) + expect(raw).toHaveLength(1) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `queries effective linked sources and context at frozen heads${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + table.ensureBootstrapAnchor('child-a') + table.ensureBootstrapAnchor('child-b') + table.appendEvent({ + sessionId: 'child-a', + name: 'child/result', + data: { text: 'native linked needle A' }, + createdAt: 100 + }) + table.appendEvent({ + sessionId: 'child-b', + name: 'child/result', + data: { text: 'native linked needle B' }, + createdAt: 200 + }) + table.appendEvent({ + sessionId: 'child-a', + name: 'child/late', + data: { text: 'native linked needle late' }, + createdAt: 300 + }) + + const sources = [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 2 } + ] + expect( + table.searchEffectiveSourcesAtHeads(sources, 'native linked needle', { limit: 1 }) + ).toMatchObject([{ session_id: 'child-b', entry_id: 2 }]) + expect( + table + .searchEffectiveSourcesAtHeads(sources, 'native linked needle', { limit: 10 }) + .map((row) => [row.session_id, row.entry_id]) + ).toEqual([ + ['child-b', 2], + ['child-a', 2] + ]) + expect( + table + .getEffectiveContextRowsAtHead({ sessionId: 'child-a', maxEntryId: 2 }, [2], { + before: 1, + after: 5, + limit: 10 + }) + .map((row) => row.entry_id) + ).toEqual([2, 1]) + + table.ensureBootstrapAnchor('child-message') + const original = createRecord({ + id: 'linked-message', + sessionId: 'child-message', + content: JSON.stringify({ text: 'old linked marker', files: [], links: [] }) + }) + const replacement = { + ...original, + content: JSON.stringify({ text: 'new linked marker', files: [], links: [] }), + updatedAt: 200 + } + appendMessageRecordToTape(table, original, 'live') + appendMessageReplacementToTape(table, replacement, 'native_edit') + const replacementHead = table.getMaxEntryId('child-message') + expect( + table.searchEffectiveSourcesAtHeads( + [{ sessionId: 'child-message', maxEntryId: replacementHead }], + 'old linked marker' + ) + ).toEqual([]) + const replacementHits = table.searchEffectiveSourcesAtHeads( + [{ sessionId: 'child-message', maxEntryId: replacementHead }], + 'new linked marker' + ) + expect(replacementHits).toHaveLength(1) + expect( + table + .getEffectiveContextRowsAtHead( + { sessionId: 'child-message', maxEntryId: replacementHead }, + [replacementHits[0].entry_id], + { before: 0, after: 0, limit: 5 } + ) + .map((row) => row.entry_id) + ).toEqual([replacementHits[0].entry_id]) + + table.append({ + sessionId: 'child-message', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: original.id, seq: 0 }, + provenanceKey: null, + payload: { + record: { + id: original.id, + sessionId: original.sessionId, + orderSeq: original.orderSeq, + role: original.role, + status: 'sent' + } + }, + meta: { source: 'malformed_import' } + }) + expect( + table.searchEffectiveSourcesAtHeads( + [ + { + sessionId: 'child-message', + maxEntryId: table.getMaxEntryId('child-message') + } + ], + 'new linked marker' + ) + ).toMatchObject([{ entry_id: replacementHits[0].entry_id }]) + + appendMessageRetractionToTape(table, replacement, 'native_delete') + expect( + table.searchEffectiveSourcesAtHeads( + [ + { + sessionId: 'child-message', + maxEntryId: table.getMaxEntryId('child-message') + } + ], + 'new linked marker' + ) + ).toEqual([]) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `searches exact frozen projections without repairing a later child tail${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + projectionTable.replaceSession( + 'child', + [ + { + sessionId: 'child', + entryId: 2, + kind: 'event', + name: 'child/result', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'native frozen projection needle', + summaryText: 'native frozen projection needle', + refs: {}, + createdAt: 100 + } + ], + 2 + ) + const before = db + .prepare( + `SELECT projection_version, max_entry_id, updated_at + FROM deepchat_tape_search_projection_meta + WHERE session_id = ?` + ) + .get('child') + + const result = projectionTable.searchSourcesReadOnly( + [{ sessionId: 'child', maxEntryId: 2 }], + 'native frozen projection needle', + { limit: 5 } + ) + + expect(result.coveredSources).toEqual([{ sessionId: 'child', maxEntryId: 2 }]) + expect(result.rows).toMatchObject([{ session_id: 'child', entry_id: 2 }]) + expect( + projectionTable.searchSourcesReadOnly( + [{ sessionId: 'child', maxEntryId: 3 }], + 'native frozen projection needle', + { limit: 5 } + ) + ).toEqual({ rows: [], coveredSources: [] }) + expect( + db + .prepare( + `SELECT projection_version, max_entry_id, updated_at + FROM deepchat_tape_search_projection_meta + WHERE session_id = ?` + ) + .get('child') + ).toEqual(before) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `filters stale FTS rows through the base projection after restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'current', + sourceSeq: 0, + searchText: 'current Redis marker', + summaryText: 'current Redis marker', + refs: { messageId: 'current' }, + createdAt: 200 + } + ], + 2 + ) + db.prepare( + `INSERT INTO deepchat_tape_search_fts ( + search_text, + name, + session_id, + entry_id, + kind, + source_type, + source_id, + source_seq, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'stale removed marker', + 'message/user', + 's1', + 1, + 'message', + 'message', + 'old', + 0, + 'stale removed marker', + '{"messageId":"old"}', + 100 + ) + + const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + restartedProjectionTable.createTable() + + expect(restartedProjectionTable.isCurrent('s1', 2)).toBe(true) + expect(restartedProjectionTable.search('s1', 'stale removed marker', { limit: 5 })).toEqual( + [] + ) + expect( + restartedProjectionTable.search('s1', 'current Redis marker', { limit: 5 })[0] + ).toMatchObject({ + entry_id: 2, + refs_json: '{"messageId":"current"}' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `recovers same-entry stale FTS after a base-only projection write and restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm1', + sourceSeq: 0, + searchText: 'old durable marker', + summaryText: 'old durable marker', + refs: { messageId: 'm1' }, + createdAt: 100 + } + ], + 1 + ) + + projectionTable.disableFtsForTesting() + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm1', + sourceSeq: 0, + searchText: 'new durable marker', + summaryText: 'new durable marker', + refs: { messageId: 'm1' }, + createdAt: 100 + } + ], + 1 + ) + db.prepare( + `INSERT INTO deepchat_tape_search_fts ( + search_text, + name, + session_id, + entry_id, + kind, + source_type, + source_id, + source_seq, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'old durable marker', + 'message/user', + 's1', + 1, + 'message', + 'message', + 'm1', + 0, + 'old durable marker', + '{"messageId":"m1"}', + 100 + ) + + const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + restartedProjectionTable.createTable() + + expect(restartedProjectionTable.isCurrent('s1', 1)).toBe(true) + expect(restartedProjectionTable.search('s1', 'old durable marker', { limit: 5 })).toEqual( + [] + ) + expect( + restartedProjectionTable.search('s1', 'new durable marker', { limit: 5 })[0] + ).toMatchObject({ + entry_id: 1, + search_text: 'new durable marker' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `rebuilds FTS during append when previous FTS meta is missing after restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.disableFtsForTesting() + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'old', + sourceSeq: 0, + searchText: 'old append marker', + summaryText: 'old append marker', + refs: { messageId: 'old' }, + createdAt: 100 + } + ], + 1 + ) + + const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + restartedProjectionTable.createTable() + restartedProjectionTable.appendSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'new', + sourceSeq: 0, + searchText: 'new append marker', + summaryText: 'new append marker', + refs: { messageId: 'new' }, + createdAt: 200 + } + ], + 2 + ) + + expect(restartedProjectionTable.isCurrent('s1', 2)).toBe(true) + expect( + restartedProjectionTable.search('s1', 'old append marker', { limit: 1 })[0] + ).toMatchObject({ + entry_id: 1, + refs_json: '{"messageId":"old"}' + }) + expect( + restartedProjectionTable.search('s1', 'new append marker', { limit: 1 })[0] + ).toMatchObject({ + entry_id: 2, + refs_json: '{"messageId":"new"}' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `rebuilds migrated tape FTS when freshness meta is excluded${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'old', + sourceSeq: 0, + searchText: 'old migrated marker', + summaryText: 'old migrated marker', + refs: { messageId: 'old' }, + createdAt: 100 + } + ], + 1 + ) + db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run('s1') + db.prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?').run('s1') + + const migratedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + migratedProjectionTable.createTable() + migratedProjectionTable.appendSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'new', + sourceSeq: 0, + searchText: 'new migrated marker', + summaryText: 'new migrated marker', + refs: { messageId: 'new' }, + createdAt: 200 + } + ], + 2 + ) + + expect(migratedProjectionTable.isCurrent('s1', 2)).toBe(true) + expect( + migratedProjectionTable.search('s1', 'old migrated marker', { limit: 1 })[0] + ).toMatchObject({ + entry_id: 1, + refs_json: '{"messageId":"old"}' + }) + expect( + migratedProjectionTable.search('s1', 'new migrated marker', { limit: 1 })[0] + ).toMatchObject({ + entry_id: 2, + refs_json: '{"messageId":"new"}' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `keeps common-term FTS searches scoped and bounded on large session sets${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + + for (let index = 0; index < 180; index += 1) { + const sessionId = `s-${index}` + const rows = Array.from({ length: 8 }, (_, offset) => ({ + sessionId, + entryId: offset + 1, + kind: 'message' as const, + name: 'message/user', + sourceType: 'message' as const, + sourceId: `m-${index}-${offset}`, + sourceSeq: offset, + searchText: `sharedcommon marker session-${index} row-${offset}`, + summaryText: `sharedcommon marker session-${index} row-${offset}`, + refs: { messageId: `m-${index}-${offset}` }, + createdAt: index * 10 + offset + })) + projectionTable.replaceSession(sessionId, rows, rows.length) + } + + const planRows = db + .prepare( + `EXPLAIN QUERY PLAN + SELECT projection.session_id, + projection.entry_id, + projection.kind, + projection.name, + projection.source_type, + projection.source_id, + projection.source_seq, + projection.search_text, + projection.summary_text, + projection.refs_json, + projection.created_at, + bm25(deepchat_tape_search_fts) AS score + FROM deepchat_tape_search_fts + INNER JOIN deepchat_tape_search_projection AS projection + ON projection.session_id = deepchat_tape_search_fts.session_id + AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) + AND projection.search_text = deepchat_tape_search_fts.search_text + WHERE deepchat_tape_search_fts MATCH ? + AND deepchat_tape_search_fts.session_id = ? + AND projection.session_id = ? + ORDER BY score ASC, projection.entry_id DESC + LIMIT ?` + ) + .all('"sharedcommon"', 's-42', 's-42', 5) as Array<{ detail: string }> + const plan = planRows.map((row) => row.detail).join('\n') + + expect(plan).toMatch(/VIRTUAL TABLE INDEX/i) + expect(plan).toMatch(/SEARCH projection USING (?:COVERING )?INDEX/i) + expect(plan).not.toMatch(/\bSCAN projection\b/i) + + const startedAt = performance.now() + const hits = projectionTable.search('s-42', 'sharedcommon', { limit: 5 }) + const elapsedMs = performance.now() - startedAt + + expect(hits).toHaveLength(5) + expect(hits.every((hit) => hit.session_id === 's-42')).toBe(true) + expect(elapsedMs).toBeLessThan(1500) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `does not trust same-entry stale FTS text even when stale FTS meta is current${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm1', + sourceSeq: 0, + searchText: 'new guarded marker', + summaryText: 'new guarded marker', + refs: { messageId: 'm1' }, + createdAt: 100 + } + ], + 1 + ) + db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run('s1') + db.prepare( + `INSERT INTO deepchat_tape_search_fts ( + search_text, + name, + session_id, + entry_id, + kind, + source_type, + source_id, + source_seq, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'old guarded marker', + 'message/user', + 's1', + 1, + 'message', + 'message', + 'm1', + 0, + 'old guarded marker', + '{"messageId":"m1"}', + 100 + ) + + const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + restartedProjectionTable.createTable() + + expect(restartedProjectionTable.isCurrent('s1', 1)).toBe(true) + expect(restartedProjectionTable.search('s1', 'old guarded marker', { limit: 5 })).toEqual( + [] + ) + expect( + restartedProjectionTable.search('s1', 'new guarded marker', { limit: 5 })[0] + ).toMatchObject({ + entry_id: 1, + search_text: 'new guarded marker' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `does not mark a tape projection current when FTS DML fails${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.dropFtsForTesting() + ;(projectionTable as any).ftsReady = true + + expect(() => + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm1', + sourceSeq: 0, + searchText: 'Redis TTL', + summaryText: 'Redis TTL', + refs: { messageId: 'm1' }, + createdAt: 100 + } + ], + 1 + ) + ).toThrow() + expect(projectionTable.isCurrent('s1', 1)).toBe(false) + expect(projectionTable.getProjectedEntryIds('s1')).toEqual([]) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `filters memory view manifests by message in SQLite before limit${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + data: { ignored: true }, + meta: { messageId: 'msg-old' }, + createdAt: 999 + }) + for (let index = 0; index < 505; index += 1) { + table.appendAnchor({ + sessionId: 's1', + name: 'memory/view_assembled', + state: { + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: index, + selected: [`m-${index}`], + dropped: [], + queryHash: `hash-${index}` + }, + meta: { messageId: `msg-${index}` }, + createdAt: index + }) + } + + const rows = table.listMemoryViewManifestAnchorsBySessions(['s1'], { + limit: 1, + messageId: 'msg-0' + }) + + expect(rows).toHaveLength(1) + expect(rows[0]).toMatchObject({ + kind: 'anchor', + name: 'memory/view_assembled', + created_at: 0 + }) + expect(JSON.parse(rows[0].meta_json)).toEqual({ messageId: 'msg-0' }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `queries memory view manifests for large agents without expanding session parameters${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const sessionTable = new NewSessionsTable(db) + const tapeTable = new DeepChatTapeEntriesTable(db) + sessionTable.createTable() + tapeTable.createTable() + for (let index = 0; index < 1200; index += 1) { + const sessionId = `s-${index}` + db.prepare( + `INSERT INTO new_sessions ( + id, + agent_id, + title, + project_dir, + is_pinned, + is_draft, + active_skills, + disabled_agent_tools, + subagent_enabled, + session_kind, + parent_session_id, + subagent_meta_json, + created_at, + updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + sessionId, + 'agent-a', + `Session ${index}`, + null, + 0, + 0, + '[]', + '[]', + index % 2 === 0 ? 0 : 1, + index % 2 === 0 ? 'regular' : 'subagent', + null, + null, + index, + index + ) + tapeTable.appendAnchor({ + sessionId, + name: 'memory/view_assembled', + state: { + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: index, + selected: [`m-${index}`], + dropped: [], + queryHash: `hash-${index}` + }, + meta: { messageId: `msg-${index}` }, + createdAt: index + }) + } + db.prepare( + `INSERT INTO new_sessions ( + id, + agent_id, + title, + project_dir, + is_pinned, + is_draft, + active_skills, + disabled_agent_tools, + subagent_enabled, + session_kind, + parent_session_id, + subagent_meta_json, + created_at, + updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'other-session', + 'other-agent', + 'Other', + null, + 0, + 0, + '[]', + '[]', + 0, + 'regular', + null, + null, + 9999, + 9999 + ) + tapeTable.appendAnchor({ + sessionId: 'other-session', + name: 'memory/view_assembled', + state: { + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: 9999, + selected: ['other'], + dropped: [], + queryHash: 'other' + }, + meta: { messageId: 'msg-0' }, + createdAt: 9999 + }) + + const rows = tapeTable.listMemoryViewManifestAnchorsByAgent('agent-a', { + messageId: 'msg-0', + limit: 1 + }) + + expect(rows).toHaveLength(1) + expect(rows[0]).toMatchObject({ + session_id: 's-0', + kind: 'anchor', + name: 'memory/view_assembled', + created_at: 0 + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `searches a SQLite tape projection and expands compact context without raw payloads${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + table.createTable() + projectionTable.createTable() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'u1', seq: 0 }, + payload: { + record: createRecord({ + id: 'u1', + content: JSON.stringify({ + text: 'Check Redis TTL with /usr/local/bin/deploy --flag and error 42.', + files: [], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.append({ + sessionId: 's1', + kind: 'tool_result', + name: 'shell', + source: { type: 'tool_result', id: 'u1:tc1', seq: 0 }, + payload: { + messageId: 'u1', + orderSeq: 2, + toolCallId: 'tc1', + exitStatus: 42, + response: 'Exit code 42 in /tmp/deploy.log' + }, + meta: { source: 'live', status: 'error' }, + createdAt: 110 + }) + + const pathHits = service.search('s1', '/usr/local/bin/deploy', { limit: 5 }) + expect(pathHits).toHaveLength(1) + expect(pathHits[0]).toMatchObject({ + kind: 'message', + summary: expect.stringContaining('Redis TTL'), + refs: { + messageId: 'u1', + role: 'user', + filePaths: expect.arrayContaining(['/usr/local/bin/deploy']) + } + }) + expect(pathHits[0]).not.toHaveProperty('payload') + expect(pathHits[0]).not.toHaveProperty('meta') + expect(service.search('s1', 'Redis TTL', { limit: 5 }).map((hit) => hit.entryId)).toContain( + pathHits[0].entryId + ) + const errorHits = service.search('s1', '42', { kinds: ['tool_result'], limit: 5 }) + expect(errorHits[0]).toMatchObject({ + refs: { + toolCallId: 'tc1', + exitStatus: 42 + } + }) + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) + + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'u2', seq: 0 }, + payload: { + record: createRecord({ + id: 'u2', + orderSeq: 3, + content: JSON.stringify({ text: 'zoxide marker 简洁', files: [], links: [] }), + createdAt: 120, + updatedAt: 120 + }) + }, + meta: { source: 'live', orderSeq: 3, role: 'user' }, + createdAt: 120 + }) + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(false) + const rebuiltHits = service.search('s1', '简洁', { limit: 5 }) + expect(rebuiltHits.map((hit) => hit.refs?.messageId)).toContain('u2') + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) + if (projectionTable.hasFtsReadyForTesting()) { + projectionTable.dropFtsForTesting() + ;(projectionTable as any).ftsReady = true + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'u3', seq: 0 }, + payload: { + record: createRecord({ + id: 'u3', + orderSeq: 4, + content: JSON.stringify({ + text: 'fts recovery marker', + files: [], + links: [] + }), + createdAt: 130, + updatedAt: 130 + }) + }, + meta: { source: 'live', orderSeq: 4, role: 'user' }, + createdAt: 130 + }) + const recoveryHits = service.search('s1', 'fts recovery marker', { limit: 5 }) + expect(recoveryHits.map((hit) => hit.refs?.messageId)).toContain('u3') + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(false) + expect(projectionTable.hasFtsReadyForTesting()).toBe(false) + + const restoredHits = service.search('s1', 'fts recovery marker', { limit: 5 }) + expect(restoredHits.map((hit) => hit.refs?.messageId)).toContain('u3') + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) + expect(projectionTable.hasFtsReadyForTesting()).toBe(true) + } + + const context = service.getContext('s1', [pathHits[0].entryId], { + before: 0, + after: 1, + limit: 2, + maxBytesPerEntry: 24, + maxTotalBytes: 24 + }) + expect(context.matchedEntryIds).toEqual([pathHits[0].entryId]) + expect(context.entries[0]).toMatchObject({ + entryId: pathHits[0].entryId, + summary: expect.stringContaining('Redis TTL'), + evidence: { + truncated: true + } + }) + expect(context.entries[0].evidence.bytes).toBeLessThanOrEqual(24) + expect(context.entries[0]).not.toHaveProperty('payload') + expect(context.entries[0]).not.toHaveProperty('meta') + const limitedContext = service.getContext( + 's1', + [pathHits[0].entryId, errorHits[0].entryId], + { + before: 0, + after: 0, + limit: 1 + } + ) + expect(limitedContext.entries.map((entry) => entry.entryId)).toEqual([pathHits[0].entryId]) + expect(limitedContext.matchedEntryIds).toEqual([pathHits[0].entryId]) + } finally { + db.close() + } + } + ) +}) diff --git a/test/main/session/data/tapeReconciler.test.ts b/test/main/session/data/tapeReconciler.test.ts new file mode 100644 index 0000000000..8dfa450bf1 --- /dev/null +++ b/test/main/session/data/tapeReconciler.test.ts @@ -0,0 +1,381 @@ +import { + describe, + expect, + it, + vi, + buildContext, + toAppSessionId, + SessionTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + createTapeTableMock, + createRecord, + createTapeService +} from './tapeTestHarness' + +describe('SessionTape reconciliation and facts', () => { + it('backfills message and tool facts idempotently before returning tape records', () => { + const { table, entries } = createTapeTableMock() + const assistantBlocks = [ + { + type: 'tool_call', + status: 'success', + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } + } + ] + const records = [ + createRecord({ id: 'u1', orderSeq: 1 }), + createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify(assistantBlocks), + createdAt: 120, + updatedAt: 120 + }) + ] + const messageStore = { + getMessages: vi.fn().mockReturnValue(records) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const first = service.ensureSessionTapeReady('s1', messageStore as any) + const second = service.ensureSessionTapeReady('s1', messageStore as any) + + expect(first.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) + expect(second.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) + expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) + expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) + expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) + expect(entries.filter((entry) => entry.name === 'migration/backfill')).toHaveLength(1) + }) + + it('appends live tool facts through the stable recorder port idempotently', async () => { + const { table, entries } = createTapeTableMock() + const service = createTapeService(table) + const input = { + sessionId: toAppSessionId('s1'), + messageId: 'a1', + orderSeq: 2, + blockIndex: 0, + block: { + type: 'tool_call' as const, + status: 'success' as const, + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } + }, + provenance: { source: 'tool_call' as const, sourceId: 'a1:tc1', sequence: 0 } + } + + const first = await service.appendToolFact(input) + const second = await service.appendToolFact(input) + + expect(second).toEqual(first) + expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) + expect(JSON.parse(entries.find((entry) => entry.kind === 'tool_call').meta_json)).toEqual({ + source: 'live', + role: 'assistant', + status: 'success', + reason: 'tool_loop' + }) + }) + + it('keeps legacy context builder output stable after tape backfill projection', () => { + const { table } = createTapeTableMock() + const records = [ + createRecord({ id: 'u1', orderSeq: 1 }), + createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { type: 'content', content: 'Tool finished', status: 'success', timestamp: 120 }, + { + type: 'tool_call', + status: 'success', + timestamp: 121, + tool_call: { + id: 'tc1', + name: 'example_tool', + params: '{"foo":"bar"}', + response: 'All good' + } + } + ]), + createdAt: 120, + updatedAt: 121 + }) + ] + const legacyMessageStore = { + getMessages: vi.fn().mockReturnValue(records) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const legacyContext = buildContext( + 's1', + { text: 'next', files: [] }, + 'System', + 10000, + 4096, + legacyMessageStore as any + ) + const tapeReady = service.ensureSessionTapeReady('s1', legacyMessageStore as any) + const tapeOnlyStore = { + getMessages: vi.fn(() => { + throw new Error('buildContext must use provided tape history records') + }) + } + const tapeContext = buildContext( + 's1', + { text: 'next', files: [] }, + 'System', + 10000, + 4096, + tapeOnlyStore as any, + false, + { + historyRecords: tapeReady.historyRecords + } + ) + + expect(tapeContext).toEqual(legacyContext) + expect(tapeOnlyStore.getMessages).not.toHaveBeenCalled() + }) + + it('rejects handoff anchors without a non-empty summary before writing Tape state', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + expect(() => service.handoff('s1', 'phase_done', { summary: ' ' })).toThrow( + 'Tape handoff requires a non-empty summary.' + ) + expect(() => service.handoff('s1', 'phase_done', { reason: 'phase complete' } as any)).toThrow( + 'Tape handoff requires a non-empty summary.' + ) + + expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() + expect(table.appendAnchor).not.toHaveBeenCalled() + expect(entries).toEqual([]) + }) + + it('migrates legacy session summary into a tape anchor during backfill', () => { + const { table, entries } = createTapeTableMock() + const messageStore = { + getMessages: vi.fn().mockReturnValue([ + createRecord({ id: 'u1', orderSeq: 1 }), + createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([{ type: 'content', content: 'answer', status: 'success' }]) + }) + ]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { + getSummaryState: vi.fn().mockReturnValue({ + summary_text: 'legacy compacted state', + summary_cursor_order_seq: 3, + summary_updated_at: 200 + }) + } + } as any) + + service.ensureSessionTapeReady('s1', messageStore as any) + + const summaryAnchor = entries.find((entry) => entry.name === 'compaction/migrated_summary') + expect(summaryAnchor).toMatchObject({ + kind: 'anchor', + source_type: 'summary', + source_id: 'legacy-summary', + created_at: 200 + }) + expect(JSON.parse(summaryAnchor.payload_json).state).toMatchObject({ + summary: 'legacy compacted state', + cursorOrderSeq: 3, + sourceMessageIds: ['u1', 'a1'] + }) + }) + + it('keeps pending message records for resume but hides pending tool facts from search', () => { + const { table } = createTapeTableMock() + const pendingBlocks = [ + { + type: 'tool_call', + status: 'pending', + timestamp: 100, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'pending result' + } + } + ] + const messageStore = { + getMessages: vi.fn().mockReturnValue([ + createRecord({ + id: 'a1', + orderSeq: 1, + role: 'assistant', + status: 'pending', + content: JSON.stringify(pendingBlocks), + updatedAt: 100 + }) + ]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.ensureSessionTapeReady('s1', messageStore as any) + + expect(service.getMessageRecords('s1')).toMatchObject([{ id: 'a1', status: 'pending' }]) + expect(service.search('s1', 'pending result', { kinds: ['tool_result'] })).toEqual([]) + }) + + it('lets final assistant facts supersede earlier pending tape facts', () => { + const { table, entries } = createTapeTableMock() + const pendingBlocks = [ + { + type: 'tool_call', + status: 'pending', + timestamp: 100, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'pending result' + } + } + ] + const finalBlocks = [ + { + type: 'tool_call', + status: 'success', + timestamp: 200, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'final result' + } + } + ] + const messageStore = { + getMessages: vi + .fn() + .mockReturnValueOnce([ + createRecord({ + id: 'a1', + orderSeq: 1, + role: 'assistant', + status: 'pending', + content: JSON.stringify(pendingBlocks), + metadata: JSON.stringify({ totalTokens: 1 }), + updatedAt: 100 + }) + ]) + .mockReturnValue([ + createRecord({ + id: 'a1', + orderSeq: 1, + role: 'assistant', + status: 'sent', + content: JSON.stringify(finalBlocks), + metadata: JSON.stringify({ totalTokens: 7 }), + updatedAt: 200 + }) + ]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.ensureSessionTapeReady('s1', messageStore as any) + service.ensureSessionTapeReady('s1', messageStore as any) + + expect(service.getMessageRecords('s1')).toMatchObject([ + { + id: 'a1', + status: 'sent' + } + ]) + const effectiveRecord = service.getMessageRecords('s1')[0]! + expect(JSON.parse(effectiveRecord.content)[0].tool_call.response).toBe('final result') + expect( + entries.filter((entry) => entry.kind === 'message' && entry.name === 'message/assistant') + ).toHaveLength(2) + expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) + const finalToolResult = entries.filter((entry) => entry.kind === 'tool_result').at(-1)! + expect(JSON.parse(finalToolResult.payload_json).response).toBe('final result') + expect(service.info('s1').lastTokenUsage).toBe(7) + expect(service.search('s1', 'pending result', { kinds: ['tool_result'] })).toEqual([]) + expect(service.search('s1', 'final result', { kinds: ['tool_result'] })).toHaveLength(1) + }) + + it('uses effective message facts after replacement and retraction events', () => { + const { table, entries } = createTapeTableMock() + const original = createRecord({ id: 'u1', orderSeq: 1 }) + const messageStore = { + getMessages: vi.fn().mockReturnValue([original]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.ensureSessionTapeReady('s1', messageStore as any) + appendMessageReplacementToTape( + table as any, + createRecord({ + id: 'u1', + orderSeq: 1, + content: JSON.stringify({ + text: 'edited', + files: [], + links: [], + search: false, + think: false + }), + updatedAt: 300 + }), + 'test_edit' + ) + + expect(JSON.parse(service.getMessageRecords('s1')[0].content).text).toBe('edited') + expect(service.search('s1', 'hello', { kinds: ['message'] })).toEqual([]) + expect(service.search('s1', 'edited', { kinds: ['message'] })).toHaveLength(1) + expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) + + appendMessageRetractionToTape(table as any, service.getMessageRecords('s1')[0], 'test_delete') + + expect(service.getMessageRecords('s1')).toEqual([]) + expect(service.search('s1', 'edited', { kinds: ['message'] })).toEqual([]) + }) + + it('appends non-idempotent retractions without generated provenance keys', () => { + const { table, entries } = createTapeTableMock() + const record = createRecord({ id: 'u1' }) + + appendMessageRetractionToTape(table as any, record, 'first_delete') + appendMessageRetractionToTape(table as any, record, 'second_delete') + + const retractions = entries.filter((entry) => entry.name === 'message/retracted') + expect(retractions).toHaveLength(2) + expect(retractions.map((entry) => entry.provenance_key)).toEqual([null, null]) + }) +}) diff --git a/test/main/session/data/tapeTestHarness.ts b/test/main/session/data/tapeTestHarness.ts new file mode 100644 index 0000000000..0aed0aad2d --- /dev/null +++ b/test/main/session/data/tapeTestHarness.ts @@ -0,0 +1,638 @@ +import { performance } from 'node:perf_hooks' +import { describe, expect, it, vi } from 'vitest' +import { buildContext } from '@/agent/deepchat/runtime/contextBuilder' +import { toAppSessionId } from '@/agent/shared/agentSessionIds' +import { SessionTape } from '@/session/data/tape' +import { buildEffectiveTapeView, searchEffectiveTapeRows } from '@/session/data/tapeEffectiveView' +import { + createTapeViewManifest, + type TapeViewManifestBuildInput +} from '@/session/data/tapeViewManifest' +import { + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + appendToolFactsToTape +} from '@/session/data/tapeFacts' +import { buildRequestRefs } from '@/session/data/tapeViewManifest' +import { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' +import { + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, + DeepChatTapeSearchProjectionTable +} from '@/session/data/tables/deepchatTapeSearchProjection' +import { DeepChatMemoryIngestionProjectionTable } from '@/memory/data/tables/deepchatMemoryIngestionProjection' +import { DeepChatMessagesTable } from '@/session/data/tables/deepchatMessages' +import { DeepChatMessageTracesTable } from '@/session/data/tables/deepchatMessageTraces' +import { DeepChatSessionsTable } from '@/session/data/tables/deepchatSessions' +import { NewSessionsTable } from '@/session/data/tables/newSessions' +import type { ChatMessageRecord } from '@shared/types/agent-interface' + +const sqliteModule = await import('better-sqlite3-multiple-ciphers').catch(() => null) +const Database = sqliteModule?.default +const DatabaseCtor = Database! +const sqliteSkipReason = 'skipped: better-sqlite3-multiple-ciphers is unavailable' +const requireNativeSqlite = process.env.DEEPCHAT_REQUIRE_NATIVE_SQLITE === '1' + +let sqliteAvailable = false +if (Database) { + try { + const smokeDb = new Database(':memory:') + smokeDb.close() + sqliteAvailable = true + } catch { + sqliteAvailable = false + } +} + +const itIfSqlite = sqliteAvailable + ? it + : requireNativeSqlite + ? (name: string, _test: () => unknown, timeout?: number) => + it( + name, + () => { + throw new Error(sqliteSkipReason) + }, + timeout + ) + : it.skip + +function createTapeTableMock() { + const entries: any[] = [] + let tapeIncarnationSequence = 0 + const table = { + ensureBootstrapAnchor: vi.fn((sessionId: string) => { + if ( + entries.some((entry) => entry.session_id === sessionId && entry.name === 'session/start') + ) { + return + } + table.appendAnchor({ + sessionId, + name: 'session/start', + source: { type: 'session', id: sessionId, seq: 0 }, + state: { owner: 'human' }, + meta: { tapeIncarnationId: `test-tape-${++tapeIncarnationSequence}` }, + idempotent: true + }) + }), + append: vi.fn((input: any) => { + const provenanceKey = + input.provenanceKey !== undefined + ? input.provenanceKey + : input.source + ? [ + input.source.type, + input.source.id, + input.source.seq ?? 0, + input.kind, + input.name ?? '' + ].join(':') + : null + const existing = input.idempotent + ? entries.find( + (entry) => + entry.session_id === input.sessionId && entry.provenance_key === provenanceKey + ) + : null + if (existing) { + return existing + } + const row = { + session_id: input.sessionId, + entry_id: + Math.max( + 0, + ...entries + .filter((entry) => entry.session_id === input.sessionId) + .map((entry) => entry.entry_id) + ) + 1, + kind: input.kind, + name: input.name ?? null, + source_type: input.source?.type ?? null, + source_id: input.source?.id ?? null, + source_seq: input.source?.seq ?? null, + provenance_key: provenanceKey, + payload_json: JSON.stringify(input.payload ?? {}), + meta_json: JSON.stringify(input.meta ?? {}), + created_at: input.createdAt ?? Date.now() + } + entries.push(row) + return row + }), + appendAnchor: vi.fn((input: any) => + table.append({ + ...input, + kind: 'anchor', + payload: { name: input.name, state: input.state } + }) + ), + appendEvent: vi.fn((input: any) => + table.append({ + ...input, + kind: 'event', + payload: { name: input.name, data: input.data } + }) + ), + runInTransaction: vi.fn((operation: () => unknown) => { + const snapshot = entries.map((entry) => ({ ...entry })) + try { + return operation() + } catch (error) { + entries.splice(0, entries.length, ...snapshot) + throw error + } + }), + getBySession: vi.fn((sessionId: string) => + entries.filter((entry) => entry.session_id === sessionId) + ), + getSubagentLineageEvents: vi.fn((sessionId: string) => + entries.filter( + (entry) => + entry.session_id === sessionId && + entry.kind === 'event' && + (entry.name === 'subagent/tape_linked' || entry.name === 'fork/merge') + ) + ), + getFirstEntriesBySessions: vi.fn((sessionIds: string[]) => + [...new Set(sessionIds)] + .flatMap((sessionId) => { + const first = entries + .filter((entry) => entry.session_id === sessionId) + .sort((left, right) => left.entry_id - right.entry_id)[0] + return first ? [first] : [] + }) + .sort((left, right) => left.session_id.localeCompare(right.session_id)) + ), + getBySessionUpToEntryId: vi.fn((sessionId: string, maxEntryId: number) => + entries.filter((entry) => entry.session_id === sessionId && entry.entry_id <= maxEntryId) + ), + getMaxEntryId: vi.fn((sessionId: string) => + Math.max( + 0, + ...entries.filter((entry) => entry.session_id === sessionId).map((entry) => entry.entry_id) + ) + ), + getMaxEntryIdsBySessions: vi.fn( + (sessionIds: string[]) => + new Map( + sessionIds.map((sessionId) => [ + sessionId, + Math.max( + 0, + ...entries + .filter((entry) => entry.session_id === sessionId) + .map((entry) => entry.entry_id) + ) + ]) + ) + ), + getLatestAnchor: vi.fn( + (sessionId: string) => + entries + .filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor') + .sort((left, right) => right.entry_id - left.entry_id)[0] + ), + getAnchors: vi.fn((sessionId: string, limit: number = 20) => + entries + .filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor') + .sort((left, right) => right.entry_id - left.entry_id) + .slice(0, Math.min(Math.max(Math.floor(limit), 1), 100)) + .reverse() + ), + getLatestSummaryAnchor: vi.fn( + (sessionId: string) => + entries + .filter( + (entry) => + entry.session_id === sessionId && + entry.kind === 'anchor' && + ['compaction/migrated_summary', 'compaction/manual', 'summary/reset'].includes( + entry.name + ) + ) + .sort((left, right) => right.entry_id - left.entry_id)[0] + ), + getByProvenanceKey: vi.fn((sessionId: string, provenanceKey: string) => + entries.find( + (entry) => entry.session_id === sessionId && entry.provenance_key === provenanceKey + ) + ), + countBySession: vi.fn( + (sessionId: string) => entries.filter((entry) => entry.session_id === sessionId).length + ), + countAnchorsBySession: vi.fn( + (sessionId: string) => + entries.filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor').length + ), + countEntriesAfter: vi.fn( + (sessionId: string, entryId: number) => + entries.filter((entry) => entry.session_id === sessionId && entry.entry_id > entryId).length + ), + search: vi.fn((sessionId: string, query: string, options: any = {}) => { + const normalizedQuery = query.trim() + if (!normalizedQuery) { + return [] + } + const limit = Number.isFinite(options.limit) ? Math.floor(options.limit) : 20 + return entries + .filter((entry) => entry.session_id === sessionId) + .filter( + (entry) => + entry.payload_json.includes(normalizedQuery) || + entry.meta_json.includes(normalizedQuery) || + entry.name?.includes(normalizedQuery) + ) + .filter((entry) => !options.kinds?.length || options.kinds.includes(entry.kind)) + .filter( + (entry) => + !Number.isFinite(options.startCreatedAt) || entry.created_at >= options.startCreatedAt + ) + .filter( + (entry) => + !Number.isFinite(options.endCreatedAt) || entry.created_at <= options.endCreatedAt + ) + .sort((left, right) => right.entry_id - left.entry_id) + .slice(0, Math.min(Math.max(limit, 1), 100)) + }), + searchEffectiveSourcesAtHeads: vi.fn((sources: any[], query: string, options: any = {}) => + sources + .flatMap((source) => + searchEffectiveTapeRows( + entries.filter( + (entry) => + entry.session_id === source.sessionId && entry.entry_id <= source.maxEntryId + ), + query, + { ...options, limit: 100 } + ) + ) + .sort( + (left, right) => + right.created_at - left.created_at || + left.session_id.localeCompare(right.session_id) || + right.entry_id - left.entry_id + ) + .slice(0, Math.min(Math.max(Math.floor(options.limit ?? 20), 1), 100)) + ), + getEffectiveContextRowsAtHead: vi.fn( + ( + source: any, + entryIds: number[], + options: { before: number; after: number; limit: number } + ) => { + const effectiveRows = buildEffectiveTapeView( + entries.filter( + (entry) => entry.session_id === source.sessionId && entry.entry_id <= source.maxEntryId + ), + { includePending: false } + ).rows + const indexesByEntryId = new Map( + effectiveRows.map((entry, index) => [entry.entry_id, index]) + ) + const indexes: number[] = [] + for (const entryId of entryIds) { + const index = indexesByEntryId.get(entryId) + if (index !== undefined) indexes.push(index) + } + for (const entryId of entryIds) { + const index = indexesByEntryId.get(entryId) + if (index === undefined) continue + for ( + let cursor = Math.max(0, index - options.before); + cursor <= Math.min(effectiveRows.length - 1, index + options.after); + cursor += 1 + ) { + if (cursor !== index) indexes.push(cursor) + } + } + return [...new Set(indexes)].slice(0, options.limit).map((index) => effectiveRows[index]) + } + ), + deleteBySession: vi.fn((sessionId: string) => { + for (let index = entries.length - 1; index >= 0; index -= 1) { + if (entries[index].session_id === sessionId) { + entries.splice(index, 1) + } + } + }) + } + return { table, entries } +} + +function createRecord(overrides: Partial): ChatMessageRecord { + return { + id: 'm1', + sessionId: 's1', + orderSeq: 1, + role: 'user', + content: JSON.stringify({ text: 'hello', files: [], links: [], search: false, think: false }), + status: 'sent', + isContextEdge: 0, + metadata: '{}', + traceCount: 0, + createdAt: 100, + updatedAt: 100, + ...overrides + } +} + +function createTraceRow(overrides: Record = {}) { + return { + id: 'trace-1', + message_id: 'a1', + session_id: 's1', + provider_id: 'openai', + model_id: 'gpt-4o', + request_seq: 1, + endpoint: 'https://api.openai.test/v1/chat/completions', + headers_json: '{"authorization":"[redacted]"}', + body_json: '{"messages":[{"role":"user","content":"hello"}]}', + truncated: 0, + created_at: 300, + ...overrides + } +} + +function createMessageRow(overrides: Record = {}) { + return { + id: 'a1', + session_id: 's1', + order_seq: 2, + role: 'assistant', + content: '[{"type":"content","content":"done","status":"success"}]', + status: 'sent', + is_context_edge: 0, + metadata: '{"totalTokens":10}', + created_at: 200, + updated_at: 300, + ...overrides + } +} + +function createObservationManifest( + overrides: Partial[0]> = {} +) { + return createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user', content: 'hello' }], + tools: [], + latestEntryId: 0, + anchorEntryIds: [], + included: [], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: true, + assembledAt: 200, + ...overrides + }) +} + +function createTapeService( + table: unknown, + traceRows: Array> = [], + messageRows: Array> = [] +) { + return new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: { + deleteBySession: vi.fn(), + getByEntryIds: vi.fn().mockReturnValue([]) + }, + deepchatMessageTracesTable: { + listByMessageId: vi.fn((messageId: string) => + traceRows.filter((row) => row.message_id === messageId) + ) + }, + deepchatMessagesTable: { + get: vi.fn((messageId: string) => messageRows.find((row) => row.id === messageId)) + }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) +} + +function createLinkedTapeService( + table: unknown, + sessions: Array<{ + id: string + session_kind: 'regular' | 'subagent' + parent_session_id: string | null + }>, + projectionTable?: unknown +) { + const sessionById = new Map(sessions.map((session) => [session.id, session])) + return { + service: new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + newSessionsTable: { + get: vi.fn((sessionId: string) => sessionById.get(sessionId)), + getMany: vi.fn((sessionIds: string[]) => + sessionIds.flatMap((sessionId) => { + const session = sessionById.get(sessionId) + return session ? [session] : [] + }) + ) + }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any), + sessionById + } +} + +function createSubagentLinkInput(parentSessionId: string, childSessionId: string) { + return { + parentSessionId, + childSessionId, + runId: `run-${childSessionId}`, + taskId: `task-${childSessionId}`, + slotId: 'reviewer', + taskTitle: `Review ${childSessionId}`, + outcome: 'completed' as const, + resultSummary: 'Done' + } +} + +function appendObservationIsolationFacts(table: unknown) { + const original = createRecord({ id: 'u1', orderSeq: 1, createdAt: 100, updatedAt: 100 }) + const edited = createRecord({ + id: 'u1', + orderSeq: 1, + content: JSON.stringify({ + text: 'edited', + files: [], + links: [], + search: false, + think: false + }), + createdAt: 100, + updatedAt: 150 + }) + const retracted = createRecord({ id: 'u2', orderSeq: 2, createdAt: 160, updatedAt: 160 }) + const pending = createRecord({ + id: 'a1', + orderSeq: 3, + role: 'assistant', + status: 'pending', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'pending', + timestamp: 200, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}' } + } + ]), + createdAt: 200, + updatedAt: 200 + }) + const final = createRecord({ + id: 'a1', + orderSeq: 3, + role: 'assistant', + status: 'sent', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 300, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'tape-result-secret' + } + } + ]), + metadata: '{"totalTokens":12}', + createdAt: 200, + updatedAt: 300 + }) + + appendMessageRecordToTape(table as any, original, 'live') + appendMessageReplacementToTape(table as any, edited, 'test_edit') + appendMessageRecordToTape(table as any, retracted, 'live') + appendMessageRetractionToTape(table as any, retracted, 'test_delete') + appendMessageRecordToTape(table as any, pending, 'live') + appendMessageRecordToTape(table as any, final, 'live') + + return { edited, final } +} + +function stripObservationPayloadOptIns(value: T): T { + const copy = structuredClone(value) as any + const stripEntryPayloads = (entries: any[] | undefined) => { + for (const entry of entries ?? []) { + delete entry.payload + delete entry.meta + } + } + + if (copy.request?.state === 'manifest_bound') { + stripEntryPayloads(copy.request.replay.entries) + delete copy.request.replay.hashes.sliceHash + if (copy.request.replay.trace) { + delete copy.request.replay.trace.headersJson + delete copy.request.replay.trace.bodyJson + } + } else if (copy.request?.trace) { + delete copy.request.trace.headersJson + delete copy.request.trace.bodyJson + } + stripEntryPayloads(copy.output?.entries) + return copy +} + +function createSpies(names: string[]) { + return Object.fromEntries(names.map((name) => [name, vi.fn()])) as Record< + string, + ReturnType + > +} + +function trackMemoryPropertyAccess(target: T) { + const memoryPropertyAccess = vi.fn() + return { + memoryPropertyAccess, + presenter: new Proxy(target, { + get(value, property, receiver) { + if (typeof property === 'string' && /memory/i.test(property)) { + memoryPropertyAccess(property) + } + return Reflect.get(value, property, receiver) + } + }) + } +} + +function readObservationMatrix(service: SessionTape) { + return { + defaultObservation: service.readCausalObservationSlice('s1', 'a1'), + repeatedObservation: service.readCausalObservationSlice('s1', 'a1'), + explicitObservation: service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }), + optInObservation: service.readCausalObservationSlice('s1', 'a1', { + includeTapePayloads: true, + includeTracePayload: true + }), + traceOnlyObservation: service.readCausalObservationSlice('s1', 'a-trace') + } +} + +export { + performance, + describe, + expect, + it, + vi, + buildContext, + toAppSessionId, + SessionTape, + buildEffectiveTapeView, + searchEffectiveTapeRows, + createTapeViewManifest, + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + appendToolFactsToTape, + buildRequestRefs, + DeepChatTapeEntriesTable, + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, + DeepChatTapeSearchProjectionTable, + DeepChatMemoryIngestionProjectionTable, + DeepChatMessagesTable, + DeepChatMessageTracesTable, + DeepChatSessionsTable, + NewSessionsTable, + DatabaseCtor, + sqliteAvailable, + sqliteSkipReason, + itIfSqlite, + createTapeTableMock, + createRecord, + createTraceRow, + createMessageRow, + createObservationManifest, + createTapeService, + createLinkedTapeService, + createSubagentLinkInput, + appendObservationIsolationFacts, + stripObservationPayloadOptIns, + createSpies, + trackMemoryPropertyAccess, + readObservationMatrix +} diff --git a/test/main/session/data/tapeViewReplay.test.ts b/test/main/session/data/tapeViewReplay.test.ts new file mode 100644 index 0000000000..126e8e7804 --- /dev/null +++ b/test/main/session/data/tapeViewReplay.test.ts @@ -0,0 +1,1493 @@ +import { + describe, + expect, + it, + vi, + SessionTape, + createTapeViewManifest, + appendMessageRecordToTape, + appendToolFactsToTape, + buildRequestRefs, + DeepChatTapeEntriesTable, + DeepChatMemoryIngestionProjectionTable, + DeepChatMessagesTable, + DeepChatMessageTracesTable, + DeepChatSessionsTable, + DatabaseCtor, + sqliteAvailable, + sqliteSkipReason, + itIfSqlite, + createTapeTableMock, + createRecord, + createTraceRow, + createMessageRow, + createObservationManifest, + createTapeService, + appendObservationIsolationFacts, + stripObservationPayloadOptIns, + createSpies, + trackMemoryPropertyAccess, + readObservationMatrix +} from './tapeTestHarness' + +describe('SessionTape view and replay', () => { + it('stores and lists view manifests as idempotent tape events', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user', + source: 'tape', + reason: 'selected_history' + } + ], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + + const first = service.appendViewManifest(manifest) + const second = service.appendViewManifest(manifest) + + expect(second.entry_id).toBe(first.entry_id) + expect(entries.filter((entry) => entry.name === 'view/assembled')).toHaveLength(1) + expect(JSON.parse(first.meta_json)).toMatchObject({ + policy: 'legacy_context_v1', + policyVersion: 1 + }) + expect(service.listViewManifestsByMessage('s1', 'a1')).toMatchObject([ + { + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + entryId: first.entry_id, + manifest: { + hashes: { + manifestHash: manifest.hashes.manifestHash + }, + policy: 'legacy_context_v1', + policyVersion: 1, + included: [ + { + messageId: 'u1', + entryId: sourceMaps.entryIdByMessageId.get('u1') + } + ] + } + } + ]) + }) + + it('indexes effective tool facts so tool-loop manifests reference real entries', () => { + const { table } = createTapeTableMock() + const assistantRecord = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } + } + ]) + }) + appendToolFactsToTape(table as any, assistantRecord, 'live', 'tool_loop') + + const service = createTapeService(table) + const sourceMaps = service.getViewManifestSourceMaps('s1') + expect(sourceMaps.toolCallEntryIdByToolId.get('tc1')).toBeGreaterThan(0) + expect(sourceMaps.toolResultEntryIdByToolId.get('tc1')).toBeGreaterThan(0) + + const refs = buildRequestRefs( + [ + { role: 'system', content: 'system' }, + { + role: 'assistant', + content: '', + tool_calls: [ + { id: 'tc1', type: 'function', function: { name: 'search', arguments: '{"q":"x"}' } } + ] + }, + { role: 'tool', content: 'result', tool_call_id: 'tc1' } + ], + sourceMaps + ) + expect(refs).toMatchObject([ + { role: 'system', source: 'synthetic' }, + { + role: 'assistant', + source: 'tape', + reason: 'tool_loop_message', + entryId: sourceMaps.toolCallEntryIdByToolId.get('tc1') + }, + { + role: 'tool', + source: 'tape', + reason: 'tool_loop_message', + entryId: sourceMaps.toolResultEntryIdByToolId.get('tc1') + } + ]) + }) + + it('scopes tool source maps to the in-flight message so reused tool ids do not collide', () => { + const { table } = createTapeTableMock() + const blocks = (response: string) => + JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response } + } + ]) + appendToolFactsToTape( + table as any, + createRecord({ id: 'a1', orderSeq: 2, role: 'assistant', content: blocks('first') }), + 'live', + 'tool_loop' + ) + appendToolFactsToTape( + table as any, + createRecord({ id: 'a2', orderSeq: 4, role: 'assistant', content: blocks('second') }), + 'live', + 'tool_loop' + ) + + const service = createTapeService(table) + const scopedToA1 = service.getViewManifestSourceMaps('s1', 'a1') + const scopedToA2 = service.getViewManifestSourceMaps('s1', 'a2') + + expect(scopedToA1.toolCallEntryIdByToolId.get('tc1')).toBeLessThan( + scopedToA2.toolCallEntryIdByToolId.get('tc1')! + ) + expect(scopedToA1.toolResultEntryIdByToolId.get('tc1')).not.toBe( + scopedToA2.toolResultEntryIdByToolId.get('tc1') + ) + }) + + it('exports tool_call and tool_result entries in a tool-loop replay slice', () => { + const { table } = createTapeTableMock() + const assistantRecord = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } + } + ]) + }) + appendToolFactsToTape(table as any, assistantRecord, 'live', 'tool_loop') + + const service = createTapeService(table) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const messages = [ + { role: 'system' as const, content: 'system' }, + { + role: 'assistant' as const, + content: '', + tool_calls: [ + { id: 'tc1', type: 'function' as const, function: { name: 'search', arguments: '{}' } } + ] + }, + { role: 'tool' as const, content: 'result', tool_call_id: 'tc1' } + ] + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: 1, + messages, + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: buildRequestRefs(messages, sourceMaps), + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + service.appendViewManifest(manifest) + + const slice = service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }) + const kinds = slice?.entries.map((entry) => entry.kind) ?? [] + expect(kinds).toContain('tool_call') + expect(kinds).toContain('tool_result') + }) + + it('filters malformed view manifest rows when listing by message', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { + type: 'runtime_event', + id: 'a1', + seq: 1 + }, + data: { + manifest: { + schemaVersion: 1, + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + included: 'not-an-array' + } + } + }) + + expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) + }) + + it('normalizes legacy manifests without hashVersion to hashVersion 1', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + const legacyManifest: Record = { ...manifest } + delete legacyManifest.hashVersion + + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 99 }, + data: { manifest: legacyManifest } + }) + + const [record] = service.listViewManifestsByMessage('s1', 'a1') + expect(record.manifest.hashVersion).toBe(1) + }) + + it('filters manifests whose hashVersion is not a number', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 99 }, + data: { manifest: { ...manifest, hashVersion: '2' } } + }) + + expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) + }) + + it('annotates read records with hash integrity without dropping tampered manifests', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const baseInput = { + sessionId: 's1', + taskType: 'chat' as const, + policy: 'legacy_context_v1' as const, + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + } + const validManifest = createTapeViewManifest({ ...baseInput, messageId: 'a1', requestSeq: 1 }) + service.appendViewManifest(validManifest) + + const tamperedManifest = createTapeViewManifest({ + ...baseInput, + messageId: 'a2', + requestSeq: 1 + }) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a2', seq: 99 }, + data: { manifest: { ...tamperedManifest, latestEntryId: tamperedManifest.latestEntryId + 1 } } + }) + + const [validRecord] = service.listViewManifestsByMessage('s1', 'a1') + const [tamperedRecord] = service.listViewManifestsByMessage('s1', 'a2') + expect(validRecord.integrity).toBe('valid') + expect(tamperedRecord).toBeDefined() + expect(tamperedRecord.integrity).toBe('invalid') + }) + + it('binds reconstruction lineage to the latest reconstruction anchor including handoffs', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('s1') + table.appendAnchor({ + sessionId: 's1', + name: 'compaction/manual', + source: { type: 'summary', id: 's1', seq: 1 }, + state: {} + }) + table.appendAnchor({ + sessionId: 's1', + name: 'handoff/phase_done', + source: { type: 'handoff', id: 's1', seq: 2 }, + state: {} + }) + table.appendAnchor({ + sessionId: 's1', + name: 'fork/merge', + source: { type: 'fork', id: 'child', seq: 3 }, + state: {} + }) + + const sourceMaps = service.getViewManifestSourceMaps('s1') + const entryIdByName = (name: string) => + table.getBySession('s1').find((entry: any) => entry.name === name)?.entry_id + + expect(sourceMaps.anchorEntryIds).toHaveLength(4) + expect(sourceMaps.reconstructionAnchorEntryId).toBe(entryIdByName('handoff/phase_done')) + expect(sourceMaps.reconstructionAnchorEntryIds).toEqual([ + sourceMaps.reconstructionAnchorEntryId + ]) + expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain( + entryIdByName('compaction/manual') + ) + expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('fork/merge')) + }) + + it('keeps memory anchors off the reconstruction lineage', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('s1') + table.appendAnchor({ + sessionId: 's1', + name: 'compaction/manual', + source: { type: 'summary', id: 's1', seq: 1 }, + state: {} + }) + table.appendAnchor({ + sessionId: 's1', + name: 'memory/extract', + source: { type: 'runtime_event', id: 's1', seq: 2 }, + state: { memoryIds: ['m1'], count: 1, reason: 'episodic' } + }) + table.appendAnchor({ + sessionId: 's1', + name: 'memory/reflect', + source: { type: 'runtime_event', id: 's1', seq: 3 }, + state: { reflectionIds: ['r1'], sourceMemoryIds: ['m1'], count: 1 } + }) + + const sourceMaps = service.getViewManifestSourceMaps('s1') + const entryIdByName = (name: string) => + table.getBySession('s1').find((entry: any) => entry.name === name)?.entry_id + + // Memory anchors are recorded on the tape for observability... + expect(sourceMaps.anchorEntryIds).toContain(entryIdByName('memory/extract')) + expect(sourceMaps.anchorEntryIds).toContain(entryIdByName('memory/reflect')) + // ...but never own the reconstruction cursor; only the summary anchor does. + expect(sourceMaps.reconstructionAnchorEntryId).toBe(entryIdByName('compaction/manual')) + expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('memory/extract')) + expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('memory/reflect')) + }) + + it('bounds replay slices to the selected view instead of pre-cursor history', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.reconstructionAnchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user', + source: 'tape', + reason: 'selected_history' + } + ], + excluded: [], + summaryCursor: { + summaryCursorOrderSeq: 100, + preCursorOrderSeqMin: 1, + preCursorOrderSeqMax: 99, + preCursorCount: 99 + }, + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 100, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + service.appendViewManifest(manifest) + + const slice = service.exportReplaySlice('s1', 'a1') + + expect(slice?.refs.excludedEntryIds).toEqual([]) + expect(slice?.refs.anchorEntryIds).toEqual(sourceMaps.reconstructionAnchorEntryIds) + expect(slice?.refs.anchorEntryIds).toHaveLength(1) + expect(slice?.manifestRecord.manifest.excludedRanges).toEqual([ + { fromOrderSeq: 1, toOrderSeq: 99, count: 99, reason: 'before_summary_cursor' } + ]) + expect(slice?.entries).toHaveLength(3) + }) + + it('exports replay slices with metadata-only payloads by default', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [createTraceRow()]) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user', + source: 'tape', + reason: 'selected_history' + } + ], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: true, + assembledAt: 200 + }) + const manifestEntry = service.appendViewManifest(manifest) + + const nowSpy = vi.spyOn(Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000) + const slice = service.exportReplaySlice('s1', 'a1') + const secondSlice = service.exportReplaySlice('s1', 'a1') + nowSpy.mockRestore() + + expect(slice).toMatchObject({ + schemaVersion: 1, + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + mode: 'trace_bound', + refs: { + manifestEntryId: manifestEntry.entry_id, + includedEntryIds: [sourceMaps.entryIdByMessageId.get('u1')], + anchorEntryIds: sourceMaps.anchorEntryIds + }, + hashes: { + manifestHash: manifest.hashes.manifestHash + } + }) + expect(slice?.hashes.sliceHash).toHaveLength(64) + expect(secondSlice?.hashes.sliceHash).toBe(slice?.hashes.sliceHash) + expect(secondSlice?.createdAt).toBe(2000) + expect(slice?.trace?.bodyHash).toHaveLength(64) + expect(slice?.trace?.bodyJson).toBeUndefined() + expect(slice?.entries.some((entry) => entry.entryId === manifestEntry.entry_id)).toBe(true) + expect( + slice?.entries.every((entry) => entry.payload === undefined && entry.meta === undefined) + ).toBe(true) + }) + + it('exports explicit replay request sequences with opt-in payloads', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [ + createTraceRow({ id: 'trace-1', request_seq: 1 }), + createTraceRow({ + id: 'trace-2', + request_seq: 2, + body_json: '{"messages":[{"role":"tool","content":"done"}]}' + }) + ]) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const baseManifestInput: TapeViewManifestBuildInput = { + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat' as const, + policy: 'legacy_context_v1' as const, + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user', + source: 'tape', + reason: 'selected_history' + } + ], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: true, + assembledAt: 200 + } + const firstManifest = createTapeViewManifest(baseManifestInput) + const secondManifest = createTapeViewManifest({ + ...baseManifestInput, + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 250 + }) + service.appendViewManifest(firstManifest) + service.appendViewManifest(secondManifest) + + const latest = service.exportReplaySlice('s1', 'a1') + const first = service.exportReplaySlice('s1', 'a1', { + requestSeq: 1, + includeTapePayloads: true, + includeTracePayload: true + }) + + expect(latest?.requestSeq).toBe(2) + expect(first?.requestSeq).toBe(1) + expect(first?.trace?.bodyJson).toContain('"hello"') + expect(first?.entries.some((entry) => entry.payload?.record)).toBe(true) + expect(first?.entries.some((entry) => entry.meta?.source === 'backfill')).toBe(true) + }) + + it('binds each replay slice to its own request seq, ignoring sentinel gap traces', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [ + createTraceRow({ + id: 'trace-req-1', + request_seq: 1, + body_json: '{"messages":[{"role":"user","content":"first-request"}]}' + }), + createTraceRow({ + id: 'trace-gap', + request_seq: 0, + endpoint: 'deepchat://interleaved-reasoning-gap', + body_json: '{"providerId":"openai"}' + }), + createTraceRow({ + id: 'trace-req-2', + request_seq: 2, + body_json: '{"messages":[{"role":"tool","content":"second-request"}]}' + }) + ]) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const baseManifestInput = { + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat' as const, + policy: 'legacy_context_v1' as const, + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user' as const, + source: 'tape' as const, + reason: 'selected_history' as const + } + ], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: true, + assembledAt: 200 + } + service.appendViewManifest(createTapeViewManifest(baseManifestInput)) + service.appendViewManifest( + createTapeViewManifest({ + ...baseManifestInput, + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 250 + }) + ) + + const first = service.exportReplaySlice('s1', 'a1', { + requestSeq: 1, + includeTracePayload: true + }) + const second = service.exportReplaySlice('s1', 'a1', { + requestSeq: 2, + includeTracePayload: true + }) + + expect(first?.trace?.bodyJson).toContain('first-request') + expect(second?.trace?.bodyJson).toContain('second-request') + }) + + it('returns null when exporting a replay slice without a manifest', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [createTraceRow()]) + + expect(service.exportReplaySlice('s1', 'a1')).toBeNull() + }) + + it('rejects non-positive replay request sequences', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [createTraceRow()]) + + expect(() => service.exportReplaySlice('s1', 'a1', { requestSeq: 0 })).toThrow( + 'requestSeq must be a positive integer.' + ) + }) + + it('joins an exact manifest and trace into a causal observation slice', () => { + const { table } = createTapeTableMock() + const assistant = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: '[{"type":"content","content":"done","status":"success"}]', + status: 'sent', + metadata: '{"totalTokens":10}', + createdAt: 200, + updatedAt: 300 + }) + appendMessageRecordToTape(table as any, assistant, 'live') + const service = createTapeService( + table, + [createTraceRow(), createTraceRow({ id: 'other-session', session_id: 's2', request_seq: 9 })], + [createMessageRow()] + ) + service.appendViewManifest(createObservationManifest()) + + const slice = service.readCausalObservationSlice('s1', 'a1', { + currentRuntimeStatus: 'idle' + }) + + expect(slice).toMatchObject({ + schemaVersion: 1, + sessionId: 's1', + messageId: 'a1', + request: { + state: 'manifest_bound', + requestSeq: 1, + replay: { + requestSeq: 1, + mode: 'trace_bound', + trace: { id: 'trace-1', requestSeq: 1 } + } + }, + output: { + correlation: 'message_only', + terminalMessage: { + status: 'sent', + orderSeq: 2, + createdAt: 200, + updatedAt: 300 + } + }, + runtime: { scope: 'current_only', status: 'idle', eventHistory: 'not_persisted' } + }) + expect(slice.output.entries.map((entry) => entry.kind)).toContain('message') + expect(slice.output.terminalMessage?.contentHash).toHaveLength(64) + expect(slice.output.terminalMessage?.metadataHash).toHaveLength(64) + }) + + it('prefers an explicit causal request sequence and otherwise selects the latest raw sequence', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 100 })) + service.appendViewManifest( + createObservationManifest({ + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 200 + }) + ) + + expect(service.readCausalObservationSlice('s1', 'a1').request).toMatchObject({ + state: 'manifest_bound', + requestSeq: 2 + }) + expect(service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }).request).toMatchObject( + { state: 'manifest_bound', requestSeq: 1 } + ) + }) + + it('does not fall back to an older manifest when a later traced request has no manifest', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [ + createTraceRow({ id: 'trace-1', request_seq: 1 }), + createTraceRow({ id: 'trace-2', request_seq: 2 }) + ]) + service.appendViewManifest(createObservationManifest({ requestSeq: 1 })) + + expect(service.readCausalObservationSlice('s1', 'a1').request).toMatchObject({ + state: 'manifest_missing', + requestSeq: 2, + trace: { id: 'trace-2', requestSeq: 2 } + }) + }) + + it('returns a trace-only request without manufacturing a view manifest', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [createTraceRow({ request_seq: 4 })]) + + const request = service.readCausalObservationSlice('s1', 'a1').request + + expect(request).toMatchObject({ + state: 'manifest_missing', + requestSeq: 4, + trace: { requestSeq: 4 } + }) + expect( + request.state === 'manifest_missing' ? request.trace?.bodyJson : undefined + ).toBeUndefined() + }) + + it('distinguishes malformed manifests from hash-invalid but readable manifests', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'bad', seq: 2 }, + data: { manifest: { schemaVersion: 99, requestSeq: 2 } } + }) + const tamperedManifest = createObservationManifest({ messageId: 'a1', requestSeq: 1 }) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 1 }, + data: { manifest: { ...tamperedManifest, latestEntryId: 1 } } + }) + + expect(service.readCausalObservationSlice('s1', 'bad').request).toEqual({ + state: 'manifest_malformed', + requestSeq: 2, + trace: null + }) + const invalid = service.readCausalObservationSlice('s1', 'a1').request + expect(invalid.state).toBe('manifest_bound') + expect(invalid.state === 'manifest_bound' ? invalid.replay.integrity : undefined).toBe( + 'invalid' + ) + }) + + it('ignores request sequence zero interleaved-reasoning sentinels', () => { + const { table } = createTapeTableMock() + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 0 }, + data: { manifest: { schemaVersion: 99, requestSeq: 0 } } + }) + const service = createTapeService(table, [ + createTraceRow({ + id: 'trace-gap', + request_seq: 0, + endpoint: 'deepchat://interleaved-reasoning-gap' + }) + ]) + + expect(service.readCausalObservationSlice('s1', 'a1').request).toEqual({ + state: 'request_unavailable', + requestSeq: null, + trace: null + }) + }) + + it('keeps multi-round assistant and tool output correlated at message scope only', () => { + const { table } = createTapeTableMock() + const assistant = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 220, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'result' + } + } + ]), + status: 'sent', + createdAt: 200, + updatedAt: 300 + }) + appendMessageRecordToTape(table as any, assistant, 'live') + const service = createTapeService(table, [], [createMessageRow({ content: assistant.content })]) + service.appendViewManifest(createObservationManifest({ requestSeq: 1 })) + service.appendViewManifest( + createObservationManifest({ + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null + }) + ) + + const firstOutput = service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }).output + const latestOutput = service.readCausalObservationSlice('s1', 'a1').output + + expect(latestOutput.correlation).toBe('message_only') + expect(latestOutput.entries.map((entry) => entry.kind)).toEqual([ + 'message', + 'tool_call', + 'tool_result' + ]) + expect(latestOutput.entries.map((entry) => entry.entryId)).toEqual( + firstOutput.entries.map((entry) => entry.entryId) + ) + }) + + it('does not infer a completed event from a paused pending assistant message', () => { + const { table } = createTapeTableMock() + const service = createTapeService( + table, + [], + [createMessageRow({ status: 'pending', content: '[{"type":"action","status":"pending"}]' })] + ) + + const slice = service.readCausalObservationSlice('s1', 'a1', { + currentRuntimeStatus: 'generating' + }) + + expect(slice.output.entries).toEqual([]) + expect(slice.output.terminalMessage).toBeNull() + expect(slice.runtime).toEqual({ + scope: 'current_only', + status: 'generating', + eventHistory: 'not_persisted' + }) + expect(JSON.stringify(slice)).not.toContain('completed') + }) + + it('reads an old message without bootstrapping or backfilling Tape', () => { + const { table, entries } = createTapeTableMock() + const messageInsert = vi.fn() + const traceInsert = vi.fn() + const projectionApply = vi.fn() + const projectionReplace = vi.fn() + const cursorWrite = vi.fn() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatMessageTracesTable: { + listByMessageId: vi.fn().mockReturnValue([]), + insert: traceInsert + }, + deepchatMessagesTable: { + get: vi.fn().mockReturnValue(createMessageRow()), + insert: messageInsert + }, + deepchatMemoryIngestionProjectionTable: { + applyAppendedEntry: projectionApply, + replaceSession: projectionReplace + }, + deepchatSessionsTable: { + getSummaryState: vi.fn().mockReturnValue(null), + updateMemoryCursorOrderSeq: cursorWrite + } + } as any) + + const slice = service.readCausalObservationSlice('s1', 'a1') + + expect(slice.request).toEqual({ + state: 'request_unavailable', + requestSeq: null, + trace: null + }) + expect(slice.output.terminalMessage?.status).toBe('sent') + expect(slice.runtime).toEqual({ + scope: 'unavailable', + status: null, + eventHistory: 'not_persisted' + }) + expect(entries).toEqual([]) + expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() + expect(table.append).not.toHaveBeenCalled() + expect(table.appendEvent).not.toHaveBeenCalled() + expect(messageInsert).not.toHaveBeenCalled() + expect(traceInsert).not.toHaveBeenCalled() + expect(projectionApply).not.toHaveBeenCalled() + expect(projectionReplace).not.toHaveBeenCalled() + expect(cursorWrite).not.toHaveBeenCalled() + }) + + it('keeps causal observations metadata-only by default and reuses replay payload opt-ins', () => { + const { table } = createTapeTableMock() + const assistant = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: 'tape-secret-output', + status: 'sent', + metadata: '{"secret":"tape-metadata"}', + createdAt: 200, + updatedAt: 300 + }) + appendMessageRecordToTape(table as any, assistant, 'live') + const service = createTapeService( + table, + [ + createTraceRow({ + headers_json: '{"authorization":"secret-header"}', + body_json: '{"prompt":"secret-request"}' + }) + ], + [ + createMessageRow({ + content: 'projection-only-content-secret', + metadata: '{"secret":"projection-only-metadata"}', + blocks_json: '["projection-only-blocks"]', + error: 'projection-only-error' + }) + ] + ) + service.appendViewManifest(createObservationManifest()) + + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) + const metadataOnly = service.readCausalObservationSlice('s1', 'a1') + const withPayloads = service.readCausalObservationSlice('s1', 'a1', { + includeTapePayloads: true, + includeTracePayload: true + }) + now.mockRestore() + + expect(JSON.stringify(metadataOnly)).not.toContain('tape-secret-output') + expect(JSON.stringify(metadataOnly)).not.toContain('tape-metadata') + expect(JSON.stringify(metadataOnly)).not.toContain('secret-header') + expect(JSON.stringify(metadataOnly)).not.toContain('secret-request') + expect(JSON.stringify(metadataOnly)).not.toContain('projection-only') + expect(withPayloads.output.entries.some((entry) => entry.payload?.record)).toBe(true) + expect(withPayloads.request.state).toBe('manifest_bound') + expect( + withPayloads.request.state === 'manifest_bound' + ? withPayloads.request.replay.trace?.headersJson + : undefined + ).toContain('secret-header') + expect(JSON.stringify(withPayloads)).toContain('tape-secret-output') + expect(JSON.stringify(withPayloads)).not.toContain('projection-only') + expect(metadataOnly.output.terminalMessage).not.toHaveProperty('content') + expect(metadataOnly.output.terminalMessage).not.toHaveProperty('metadata') + expect(metadataOnly.output.terminalMessage).not.toHaveProperty('blocks') + expect(metadataOnly.output.terminalMessage).not.toHaveProperty('error') + expect(stripObservationPayloadOptIns(withPayloads)).toEqual( + stripObservationPayloadOptIns(metadataOnly) + ) + }) + + it('keeps storage and Memory seams unchanged across every causal observation read mode', () => { + const { table, entries } = createTapeTableMock() + const { final } = appendObservationIsolationFacts(table) + const traceRows = [ + createTraceRow({ id: 'trace-1', request_seq: 1 }), + createTraceRow({ id: 'trace-2', request_seq: 2 }), + createTraceRow({ id: 'trace-only', message_id: 'a-trace', request_seq: 7 }) + ] + const messageRows = [createMessageRow({ order_seq: 3 })] + const projectionState = { + meta: { session_id: 's1', projection_version: 1, max_entry_id: entries.length }, + range: [{ session_id: 's1', message_id: 'a1', order_seq: 3, entry_id: entries.length }] + } + const memoryCursorOrderSeq = 3 + const memoryProjectionWrites = createSpies([ + 'applyAppendedEntry', + 'replaceSession', + 'invalidateSession' + ]) + const memoryProjectionTable = { + ...memoryProjectionWrites, + readCurrentRange: vi.fn(() => structuredClone(projectionState.range)), + getSessionMeta: vi.fn(() => structuredClone(projectionState.meta)) + } + const cursorWrites = createSpies(['updateMemoryCursorOrderSeq', 'rewindMemoryCursorOrderSeq']) + const sessionTable = { + ...cursorWrites, + getSummaryState: vi.fn().mockReturnValue(null), + getMemoryCursorOrderSeq: vi.fn(() => memoryCursorOrderSeq) + } + const messageWrites = createSpies([ + 'insert', + 'updateContent', + 'updateStatus', + 'updateContentAndStatus' + ]) + const messageTable = { + ...messageWrites, + get: vi.fn((messageId: string) => messageRows.find((row) => row.id === messageId)) + } + const traceWrites = createSpies(['insert', 'deleteByMessageId', 'deleteBySessionId']) + const traceTable = { + ...traceWrites, + listByMessageId: vi.fn((messageId: string) => + traceRows.filter((row) => row.message_id === messageId) + ) + } + const { presenter, memoryPropertyAccess } = trackMemoryPropertyAccess({ + deepchatTapeEntriesTable: table, + deepchatMessageTracesTable: traceTable, + deepchatMessagesTable: messageTable, + deepchatSessionsTable: sessionTable, + deepchatMemoryIngestionProjectionTable: memoryProjectionTable + }) + const service = new SessionTape(presenter as any) + service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 210 })) + service.appendViewManifest( + createObservationManifest({ + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 310 + }) + ) + + for (const write of [ + table.ensureBootstrapAnchor, + table.append, + table.appendAnchor, + table.appendEvent, + table.deleteBySession + ]) { + write.mockClear() + } + const captureState = () => ({ + tapeRows: structuredClone(entries), + maxEntryId: table.getMaxEntryId('s1'), + entryOrder: entries.map((entry) => entry.entry_id), + maxMessageOrder: Math.max(0, ...service.getMessageRecords('s1').map((row) => row.orderSeq)), + messageRows: structuredClone(messageRows), + traceRows: structuredClone(traceRows), + viewManifestRows: structuredClone( + entries.filter((entry) => entry.kind === 'event' && entry.name === 'view/assembled') + ), + effectiveView: service.getMessageRecords('s1'), + replay: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }), + replayHash: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 })?.hashes.sliceHash, + projectionMeta: memoryProjectionTable.getSessionMeta(), + projectionRange: memoryProjectionTable.readCurrentRange(), + memoryCursorOrderSeq: sessionTable.getMemoryCursorOrderSeq() + }) + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) + + try { + const before = captureState() + const { defaultObservation, repeatedObservation, explicitObservation, traceOnlyObservation } = + readObservationMatrix(service) + const after = captureState() + + expect(after).toEqual(before) + expect(repeatedObservation).toEqual(defaultObservation) + expect(defaultObservation.request).toMatchObject({ + state: 'manifest_bound', + requestSeq: 2 + }) + expect(explicitObservation.request).toMatchObject({ + state: 'manifest_bound', + requestSeq: 1 + }) + expect(traceOnlyObservation.request).toMatchObject({ + state: 'manifest_missing', + requestSeq: 7, + trace: { id: 'trace-only' } + }) + expect(defaultObservation.output).toMatchObject({ + correlation: 'message_only', + entries: [{ kind: 'message' }, { kind: 'tool_call' }, { kind: 'tool_result' }] + }) + expect(before.effectiveView.map((record) => record.id)).toEqual(['u1', 'a1']) + expect(JSON.parse(before.effectiveView[0].content).text).toBe('edited') + expect(before.effectiveView[1]).toMatchObject({ status: 'sent', content: final.content }) + } finally { + now.mockRestore() + } + + for (const write of [ + table.ensureBootstrapAnchor, + table.append, + table.appendAnchor, + table.appendEvent, + table.deleteBySession, + ...Object.values(messageWrites), + ...Object.values(traceWrites), + ...Object.values(memoryProjectionWrites), + ...Object.values(cursorWrites), + memoryPropertyAccess + ]) { + expect(write).not.toHaveBeenCalled() + } + }) + + itIfSqlite( + `preserves real Tape, message, trace, Memory projection and schema state while observing${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const memoryProjectionTable = new DeepChatMemoryIngestionProjectionTable(db) + const tapeTable = new DeepChatTapeEntriesTable(db, memoryProjectionTable) + const messageTable = new DeepChatMessagesTable(db) + const traceTable = new DeepChatMessageTracesTable(db) + const sessionTable = new DeepChatSessionsTable(db) + memoryProjectionTable.createTable() + tapeTable.createTable() + messageTable.createTable() + traceTable.createTable() + sessionTable.createTable() + sessionTable.create('s1', 'openai', 'gpt-4o', 'full_access') + sessionTable.updateMemoryCursorOrderSeq('s1', 3) + + appendObservationIsolationFacts(tapeTable) + messageTable.insert({ + id: 'a1', + sessionId: 's1', + orderSeq: 3, + role: 'assistant', + content: 'projection-only-content-secret', + status: 'sent', + metadata: '{"secret":"projection-only-metadata"}', + createdAt: 200, + updatedAt: 300 + }) + for (const trace of [ + { id: 'trace-1', messageId: 'a1', requestSeq: 1 }, + { id: 'trace-2', messageId: 'a1', requestSeq: 2 }, + { id: 'trace-only', messageId: 'a-trace', requestSeq: 7 } + ]) { + traceTable.insert({ + ...trace, + sessionId: 's1', + providerId: 'openai', + modelId: 'gpt-4o', + endpoint: 'https://api.openai.test/v1/chat/completions', + headersJson: `{"authorization":"${trace.id}-header"}`, + bodyJson: `{"prompt":"${trace.id}-body"}`, + truncated: false, + createdAt: 300 + trace.requestSeq + }) + } + + const service = new SessionTape({ + deepchatTapeEntriesTable: tapeTable, + deepchatMessagesTable: messageTable, + deepchatMessageTracesTable: traceTable, + deepchatSessionsTable: sessionTable, + deepchatMemoryIngestionProjectionTable: memoryProjectionTable + } as any) + service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 210 })) + service.appendViewManifest( + createObservationManifest({ + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 310 + }) + ) + const effectiveRecords = service.getMessageRecords('s1') + const sourceMaps = service.getViewManifestSourceMaps('s1') + memoryProjectionTable.replaceSession( + 's1', + effectiveRecords + .filter( + (record): record is ChatMessageRecord & { status: 'sent' | 'error' } => + record.status === 'sent' || record.status === 'error' + ) + .map((record) => ({ + sessionId: 's1', + messageId: record.id, + orderSeq: record.orderSeq, + entryId: sourceMaps.entryIdByMessageId.get(record.id)!, + role: record.role, + content: record.content, + status: record.status, + hadToolUse: record.id === 'a1' + })), + tapeTable.getMaxEntryId('s1') + ) + + const captureState = () => { + const tapeRows = tapeTable.getBySession('s1') + return { + tapeRows, + maxEntryId: tapeTable.getMaxEntryId('s1'), + entryOrder: tapeRows.map((row) => row.entry_id), + messageRow: messageTable.get('a1'), + traceRows: db + .prepare( + `SELECT * + FROM deepchat_message_traces + ORDER BY session_id, message_id, request_seq, id` + ) + .all(), + viewManifestRows: tapeRows.filter( + (row) => row.kind === 'event' && row.name === 'view/assembled' + ), + effectiveView: service.getMessageRecords('s1'), + replay: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }), + replayHash: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 })?.hashes.sliceHash, + projectionMeta: memoryProjectionTable.getSessionMeta('s1'), + projectionRange: memoryProjectionTable.readCurrentRange('s1', 0, 10), + memoryCursorOrderSeq: sessionTable.getMemoryCursorOrderSeq('s1'), + schema: db + .prepare( + `SELECT type, name, tbl_name, sql + FROM sqlite_master + WHERE name NOT LIKE 'sqlite_%' + ORDER BY type, name, tbl_name` + ) + .all() + } + } + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) + + try { + const before = captureState() + readObservationMatrix(service) + const after = captureState() + + expect(after).toEqual(before) + } finally { + now.mockRestore() + } + } finally { + db.close() + } + } + ) +}) diff --git a/test/main/session/data/transcript.test.ts b/test/main/session/data/transcript.test.ts index f530befe1b..42e5e55072 100644 --- a/test/main/session/data/transcript.test.ts +++ b/test/main/session/data/transcript.test.ts @@ -832,6 +832,34 @@ describe('SessionTranscript', () => { expect(sqlitePresenter.deepchatMessageTracesTable.deleteByMessageIds).not.toHaveBeenCalled() expect(sqlitePresenter.deepchatMessagesTable.deleteFromOrderSeq).toHaveBeenCalledWith('s1', 2) }) + + it('defers projection deletion until every tape retraction has been appended', () => { + const appendEvent = vi + .fn() + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => { + throw new Error('second retraction failed') + }) + const transaction = vi.fn((operation: () => unknown) => () => operation()) + sqlitePresenter.getDatabase = vi.fn().mockReturnValue({ transaction }) + sqlitePresenter.deepchatTapeEntriesTable = { + ensureBootstrapAnchor: vi.fn(), + appendEvent + } + sqlitePresenter.deepchatMessagesTable.getBySession.mockReturnValue([ + createMessageRow({ id: 'm1', order_seq: 1 }), + createMessageRow({ id: 'm2', order_seq: 2 }), + createMessageRow({ id: 'm3', order_seq: 3 }) + ]) + + expect(() => store.deleteFromOrderSeq('s1', 2)).toThrow('second retraction failed') + + expect(transaction).toHaveBeenCalledOnce() + expect(appendEvent).toHaveBeenCalledTimes(2) + expect(sqlitePresenter.deepchatMessagesTable.deleteFromOrderSeq).not.toHaveBeenCalled() + expect(sqlitePresenter.deepchatSearchDocumentsTable.deleteByMessageIds).not.toHaveBeenCalled() + expect(sqlitePresenter.deepchatMessageTracesTable.deleteByMessageIds).not.toHaveBeenCalled() + }) }) describe('trace operations', () => { diff --git a/test/main/session/runtimeIntegration.test.ts b/test/main/session/runtimeIntegration.test.ts index 960fa5d2d5..fcfc6f412f 100644 --- a/test/main/session/runtimeIntegration.test.ts +++ b/test/main/session/runtimeIntegration.test.ts @@ -1740,8 +1740,19 @@ describe('Integration: Session Tape boundary', () => { expect(search).toHaveBeenCalledTimes(2) expect(getContext).toHaveBeenCalledOnce() + ensureTapeReady.mockClear() + search.mockClear() + getContext.mockClear() + await sessionData.tape.searchTape('s1', 'needle') + expect(ensureTapeReady.mock.invocationCallOrder[0]).toBeLessThan( + search.mock.invocationCallOrder[0] + ) + await sessionData.tape.getTapeContext('s1', [2]) + expect(ensureTapeReady.mock.invocationCallOrder[1]).toBeLessThan( + getContext.mock.invocationCallOrder[0] + ) expect(ensureTapeReady).toHaveBeenCalledTimes(2) }) From bf9786c2e75a488fd9aa240767650a74d195db1f Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 15:50:02 +0800 Subject: [PATCH 03/29] refactor(tape): establish domain ports --- docs/architecture/tape-layering/tasks.md | 12 +- src/main/agent/deepchat/loop/ports.ts | 51 +-- .../memory/memoryRuntimeCoordinator.ts | 4 +- .../agent/deepchat/runtime/contextBuilder.ts | 2 +- .../deepchat/runtime/deepChatLoopRunner.ts | 4 +- src/main/agent/deepchat/runtime/process.ts | 2 +- src/main/agent/deepchat/runtime/types.ts | 4 +- .../deepchatMemoryIngestionProjection.ts | 10 +- src/main/memory/routes.ts | 40 +- src/main/session/data/database.ts | 8 +- src/main/session/data/index.ts | 6 +- src/main/session/data/settings.ts | 2 +- .../tables/deepchatTapeEffectiveSemantics.ts | 162 +------- .../data/tables/deepchatTapeEntries.ts | 169 +++----- .../tables/deepchatTapeSearchProjection.ts | 10 +- src/main/session/data/tape.ts | 11 +- src/main/session/data/tapeEffectiveView.ts | 256 +----------- src/main/session/data/tapeFacts.ts | 32 +- src/main/session/data/tapeViewManifest.ts | 363 +----------------- src/main/tape/domain/effectiveSemantics.ts | 161 ++++++++ src/main/tape/domain/effectiveView.ts | 251 ++++++++++++ src/main/tape/domain/entry.ts | 118 ++++++ src/main/tape/domain/facts.ts | 29 ++ src/main/tape/domain/viewManifest.ts | 363 ++++++++++++++++++ src/main/tape/ports/capabilities.ts | 43 +++ src/main/tape/ports/storage.ts | 72 ++++ .../agent/deepchat/runtime/process.test.ts | 4 +- test/main/evals/nativeAgent/harness.ts | 2 +- 28 files changed, 1150 insertions(+), 1041 deletions(-) create mode 100644 src/main/tape/domain/effectiveSemantics.ts create mode 100644 src/main/tape/domain/effectiveView.ts create mode 100644 src/main/tape/domain/entry.ts create mode 100644 src/main/tape/domain/facts.ts create mode 100644 src/main/tape/domain/viewManifest.ts create mode 100644 src/main/tape/ports/capabilities.ts create mode 100644 src/main/tape/ports/storage.ts diff --git a/docs/architecture/tape-layering/tasks.md b/docs/architecture/tape-layering/tasks.md index 7ef5780a9b..374fb4ff69 100644 --- a/docs/architecture/tape-layering/tasks.md +++ b/docs/architecture/tape-layering/tasks.md @@ -17,12 +17,12 @@ ## Domain and Ports -- [ ] Move Tape-owned entry, source, provenance, and fact types out of Agent and SQLite modules. -- [ ] Move effective-view and ViewManifest pure logic into `src/main/tape/domain/`. -- [ ] Introduce normal storage and narrow consumer capability ports. -- [ ] Replace the broad `TapeRecorder` dependency with `TapeToolFactWriter`. -- [ ] Preserve old module paths through compatibility re-exports. -- [ ] Review and commit the domain and port slice. +- [x] Move Tape-owned entry, source, provenance, and fact types out of Agent and SQLite modules. +- [x] Move effective-view and ViewManifest pure logic into `src/main/tape/domain/`. +- [x] Introduce normal storage and narrow consumer capability ports. +- [x] Replace the broad `TapeRecorder` dependency with `TapeToolFactWriter`. +- [x] Preserve old module paths through compatibility re-exports. +- [x] Review and commit the domain and port slice. ## Application Services diff --git a/src/main/agent/deepchat/loop/ports.ts b/src/main/agent/deepchat/loop/ports.ts index 69a939dce1..42bb831725 100644 --- a/src/main/agent/deepchat/loop/ports.ts +++ b/src/main/agent/deepchat/loop/ports.ts @@ -1,11 +1,10 @@ import type { AppSessionId } from '@/agent/shared/agentSessionIds' -import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' +import type { AssistantMessageBlock } from '@shared/types/agent-interface' import type { ChatMessage } from '@shared/types/core/chat-message' import type { LLMCoreStreamEvent } from '@shared/types/core/llm-events' import type { MCPToolCall, MCPToolDefinition, MCPToolResponse } from '@shared/types/core/mcp' import type { ToolCallOptions, ToolPermissionPreCheckResult } from '@shared/types/tool' import type { ModelConfig } from '@shared/types/provider' -import type { DeepChatTapeViewManifest } from '@shared/types/tape-view-manifest' import type { MemorySessionHandle } from '@/agent/deepchat/memory/memoryPromptContributor' export interface ProviderRequest { @@ -162,54 +161,6 @@ export interface ToolResultPort { }): Promise } -export interface TapeEntryRef { - sessionId: AppSessionId - entryId: number -} - -export interface TapeHead { - sessionId: AppSessionId - latestEntryId: number -} - -export interface TapeFactProvenance { - source: 'message' | 'tool_call' | 'tool_result' | 'runtime_event' - sourceId: string - sequence: number -} - -export interface TapeToolFactInput { - sessionId: AppSessionId - messageId: string - orderSeq: number - blockIndex: number - block: AssistantMessageBlock - provenance: TapeFactProvenance -} - -export interface TapeAnchorInput { - sessionId: AppSessionId - name: string - state: Readonly> - meta: Readonly> - provenance: TapeFactProvenance -} - -export interface TapeEffectiveView { - sessionId: AppSessionId - records: ChatMessageRecord[] -} - -export interface TapeRecorder { - ensureSession(input: { sessionId: AppSessionId }): Promise - appendUserMessage(input: { record: ChatMessageRecord }): Promise - appendViewManifest(manifest: DeepChatTapeViewManifest): Promise - appendAssistantFact(input: { record: ChatMessageRecord }): Promise - appendToolFact(input: TapeToolFactInput): Promise - appendAnchor(input: TapeAnchorInput): Promise - readEffectiveView(input: { sessionId: AppSessionId }): Promise -} - export interface OutputSink { update(input: { runId: string diff --git a/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts b/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts index 75fecf0aa7..c3c90d5b7f 100644 --- a/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts +++ b/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts @@ -4,13 +4,13 @@ import { appendMemorySectionWithManifest } from '@/memory/injection' import type { MemoryExecutionToken, MemoryRuntimePort } from '@/memory/injection' import { BUILTIN_DEEPCHAT_AGENT_ID } from '@/agent/repository' import { withSoftDeadline } from '@/memory/core/asyncDeadline' -import { buildEffectiveTapeView } from '@/session/data/tapeEffectiveView' +import { buildEffectiveTapeView } from '@/tape/domain/effectiveView' import type { DeepChatMemoryIngestionCurrentRange, DeepChatMemoryIngestionProjectionInput, DeepChatMemoryIngestionProjectionRow } from '@/memory/data/tables/deepchatMemoryIngestionProjection' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' import { MEMORY_EXTRACTION_CHUNKS_PER_QUEUE_TASK, buildMemoryExtractionChunks, diff --git a/src/main/agent/deepchat/runtime/contextBuilder.ts b/src/main/agent/deepchat/runtime/contextBuilder.ts index 705316c019..cd2deb0e7d 100644 --- a/src/main/agent/deepchat/runtime/contextBuilder.ts +++ b/src/main/agent/deepchat/runtime/contextBuilder.ts @@ -14,7 +14,7 @@ import { estimateMessageTokens, estimateMessagesTokens } from '@shared/utils/messageTokens' -import { isCompactionRecord } from '@/session/data/tapeViewManifest' +import { isCompactionRecord } from '@/tape/domain/viewManifest' export { estimateMessagesTokens } from '@shared/utils/messageTokens' diff --git a/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts b/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts index 99257feccd..9be85d5f23 100644 --- a/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts +++ b/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts @@ -66,7 +66,7 @@ import { createTapeViewManifest, resolveTapeViewManifestPolicy, type TapeViewContextSelection -} from '@/session/data/tapeViewManifest' +} from '@/tape/domain/viewManifest' import type { SessionTape } from '@/session/data/tape' import type { AgentTraceSettingsPort } from '@/agent/traceSettings' import type { DeepChatToolResolver } from '@/agent/deepchat/runtime/toolResolver' @@ -754,7 +754,7 @@ export class DeepChatLoopRunner { }, io: { messageStore: this.ports.messageStore, - tapeRecorder: this.ports.tapeService, + tapeToolFactWriter: this.ports.tapeService, publishEvent: this.ports.publishEvent, publishSessionUpdate: this.ports.publishSessionUpdate } diff --git a/src/main/agent/deepchat/runtime/process.ts b/src/main/agent/deepchat/runtime/process.ts index 8f0c6679f5..e9ee0e67ef 100644 --- a/src/main/agent/deepchat/runtime/process.ts +++ b/src/main/agent/deepchat/runtime/process.ts @@ -739,7 +739,7 @@ export async function processStream(params: ProcessParams): Promise & { - tapeRecorder: Pick + tapeToolFactWriter: TapeToolFactWriter } export interface ProcessControlCollaborators { diff --git a/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts b/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts index 557247e3ba..40dd788de5 100644 --- a/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts +++ b/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts @@ -1,6 +1,7 @@ import type Database from 'better-sqlite3-multiple-ciphers' import { BaseTable } from '@/data/baseTable' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeMutationProjection } from '@/tape/ports/storage' import type { MemoryPerfObserver } from '../../../memory/ports' import { readTapeMessageRetractionId, @@ -9,7 +10,7 @@ import { tapeEntryToMessageRecord, tapeMessageRank, tapeToolRank -} from '@/session/data/tables/deepchatTapeEffectiveSemantics' +} from '@/tape/domain/effectiveSemantics' export const DEEPCHAT_MEMORY_INGESTION_PROJECTION_VERSION = 1 @@ -92,7 +93,10 @@ const MEMORY_INGESTION_PROJECTION_SQL = ` ); ` -export class DeepChatMemoryIngestionProjectionTable extends BaseTable { +export class DeepChatMemoryIngestionProjectionTable + extends BaseTable + implements TapeMutationProjection +{ constructor( db: Database.Database, private readonly perfObserver?: MemoryPerfObserver diff --git a/src/main/memory/routes.ts b/src/main/memory/routes.ts index 055b5a20c5..7c7fd8b381 100644 --- a/src/main/memory/routes.ts +++ b/src/main/memory/routes.ts @@ -40,7 +40,9 @@ import { type MemoryUpdateResult } from '@shared/contracts/routes/memory.routes' import { createRouteMap, type DeepchatRouteMap } from '@/routes/routeRegistry' -import { buildEffectiveTapeView } from '@/session/data/tapeEffectiveView' +import { buildEffectiveTapeView } from '@/tape/domain/effectiveView' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeInspectionReader } from '@/tape/ports/capabilities' import type { MemoryConflictPair, MemoryConflictResolution, @@ -195,32 +197,7 @@ function deriveSelectedMemoryIds(value: unknown): string[] | null { return [...ids] } -interface MemoryTapeEntryRow { - session_id: string - entry_id: number - kind: 'event' | 'anchor' | 'message' | 'tool_call' | 'tool_result' - name: string | null - source_type: - | 'session' - | 'message' - | 'assistant_block' - | 'tool_call' - | 'tool_result' - | 'runtime_event' - | 'migration' - | 'summary' - | 'fork' - | 'subagent' - | null - source_id: string | null - source_seq: number | null - provenance_key: string | null - payload_json: string - meta_json: string - created_at: number -} - -function toMemoryViewManifestDto(row: MemoryTapeEntryRow) { +function toMemoryViewManifestDto(row: DeepChatTapeEntryRow) { const payload = parseJsonRecord(row.payload_json) const meta = parseJsonRecord(row.meta_json) const manifest = @@ -248,13 +225,6 @@ function toMemoryViewManifestDto(row: MemoryTapeEntryRow) { } } -type MemoryTapeEntries = { - getBySession(sessionId: string): MemoryTapeEntryRow[] - listMemoryViewManifestAnchorsByAgent( - agentId: string, - options: { sessionId?: string; limit?: number; messageId?: string } - ): MemoryTapeEntryRow[] -} type MemoryAuditEntries = { listByAgent(agentId: string, options: MemoryAuditListOptions): AgentMemoryAuditRow[] } @@ -316,7 +286,7 @@ interface MemoryRouteService { export function createMemoryRoutes(deps: { memoryService: MemoryRouteService getAgentType(agentId: string): Promise - getTapeEntries(): MemoryTapeEntries + getTapeEntries(): TapeInspectionReader getAuditEntries(): MemoryAuditEntries }): DeepchatRouteMap { const { memoryService } = deps diff --git a/src/main/session/data/database.ts b/src/main/session/data/database.ts index 9e6d48024b..14efffc10f 100644 --- a/src/main/session/data/database.ts +++ b/src/main/session/data/database.ts @@ -15,10 +15,8 @@ import { DeepChatMessageSearchResultsTable } from './tables/deepchatMessageSearc import { DeepChatSearchDocumentsTable } from './tables/deepchatSearchDocuments' import { DeepChatPendingInputsTable } from './tables/deepchatPendingInputs' import { DeepChatUsageStatsTable } from './tables/deepchatUsageStats' -import { - DeepChatTapeEntriesTable, - type DeepChatTapeMutationProjection -} from './tables/deepchatTapeEntries' +import { DeepChatTapeEntriesTable } from './tables/deepchatTapeEntries' +import type { TapeMutationProjection } from '@/tape/ports/storage' import { DeepChatTapeSearchProjectionTable } from './tables/deepchatTapeSearchProjection' import { DeepChatSessionMetadataTable } from './tables/deepchatSessionMetadata' import { NewSessionActiveSkillsTable } from './tables/newSessionActiveSkills' @@ -27,7 +25,7 @@ import { NewSessionDisabledAgentToolsTable } from './tables/newSessionDisabledAg export class SessionDatabase { constructor( private readonly connection: DatabaseConnectionProvider, - private readonly getTapeMutationProjection?: () => DeepChatTapeMutationProjection + private readonly getTapeMutationProjection?: () => TapeMutationProjection ) {} getDatabase() { diff --git a/src/main/session/data/index.ts b/src/main/session/data/index.ts index bf528a77a3..fc2d05aca4 100644 --- a/src/main/session/data/index.ts +++ b/src/main/session/data/index.ts @@ -1,5 +1,6 @@ import type { DatabaseConnectionProvider } from '@/data/databaseConnection' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeMutationProjection } from '@/tape/ports/storage' import type { SessionTapePort } from './contracts' import { SessionPendingInputStore } from './pendingInputStore' import { SessionPendingInputs } from './pendingInputs' @@ -7,11 +8,10 @@ import { SessionSettingsStore } from './settings' import { normalizeTapeHandoffState, SessionTape } from './tape' import { SessionTranscript } from './transcript' import { SessionDatabase } from './database' -import type { DeepChatTapeMutationProjection } from './tables/deepchatTapeEntries' export function createSessionData( connection: DatabaseConnectionProvider, - getTapeMutationProjection: (() => DeepChatTapeMutationProjection) | undefined, + getTapeMutationProjection: (() => TapeMutationProjection) | undefined, events: SessionDataEvents ) { const database = new SessionDatabase(connection, getTapeMutationProjection) diff --git a/src/main/session/data/settings.ts b/src/main/session/data/settings.ts index 344edcd8b3..7b308d3b52 100644 --- a/src/main/session/data/settings.ts +++ b/src/main/session/data/settings.ts @@ -1,7 +1,7 @@ import { SessionDatabase } from './database' import type { PermissionMode, SessionGenerationSettings } from '@shared/types/agent-interface' import type { DeepChatSessionSummaryRow } from '@/session/data/tables/deepchatSessions' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' export type SessionSummaryState = { summaryText: string | null diff --git a/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts b/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts index 932234b62f..7a707b39b9 100644 --- a/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts +++ b/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts @@ -1,161 +1 @@ -import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' -import type { DeepChatTapeEntryRow } from './deepchatTapeEntries' - -const TERMINAL_TAPE_TOOL_STATUSES = new Set(['success', 'error']) - -export interface DeepChatTapeToolIdentity { - key: string - messageId: string -} - -export function parseTapeJsonObject(raw: string): Record { - try { - const parsed = JSON.parse(raw) as unknown - if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { - return parsed as Record - } - } catch {} - return {} -} - -export function parseNestedTapeJsonObject(value: unknown): Record { - if (typeof value === 'string') { - return parseTapeJsonObject(value) - } - if (value && typeof value === 'object' && !Array.isArray(value)) { - return value as Record - } - return {} -} - -export function parseAssistantBlocks(rawContent: string): AssistantMessageBlock[] { - try { - const parsed = JSON.parse(rawContent) as unknown - return Array.isArray(parsed) ? (parsed as AssistantMessageBlock[]) : [] - } catch { - return [] - } -} - -export function messageRecordHasFinalToolUse(record: ChatMessageRecord): boolean { - if (record.role !== 'assistant' || (record.status !== 'sent' && record.status !== 'error')) { - return false - } - const blocks = parseAssistantBlocks(record.content) - const pendingInteractionToolIds = new Set( - blocks.flatMap((block) => - block.type === 'action' && - (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && - block.status === 'pending' && - typeof block.tool_call?.id === 'string' - ? [block.tool_call.id] - : [] - ) - ) - return blocks.some( - (block) => - block.type === 'tool_call' && - (block.status === 'success' || block.status === 'error') && - typeof block.tool_call?.id === 'string' && - !pendingInteractionToolIds.has(block.tool_call.id) - ) -} - -function isMessageStatus(value: unknown): value is ChatMessageRecord['status'] { - return value === 'pending' || value === 'sent' || value === 'error' -} - -export function tapeEntryToMessageRecord(row: DeepChatTapeEntryRow): ChatMessageRecord | null { - if (row.kind !== 'message') { - return null - } - - const payload = parseTapeJsonObject(row.payload_json) - const record = payload.record - if (!record || typeof record !== 'object' || Array.isArray(record)) { - return null - } - - const candidate = record as Partial - if ( - typeof candidate.id !== 'string' || - typeof candidate.sessionId !== 'string' || - typeof candidate.orderSeq !== 'number' || - (candidate.role !== 'user' && candidate.role !== 'assistant') || - typeof candidate.content !== 'string' - ) { - return null - } - - return { - id: candidate.id, - sessionId: candidate.sessionId, - orderSeq: candidate.orderSeq, - role: candidate.role, - content: candidate.content, - status: isMessageStatus(candidate.status) ? candidate.status : 'sent', - isContextEdge: typeof candidate.isContextEdge === 'number' ? candidate.isContextEdge : 0, - metadata: typeof candidate.metadata === 'string' ? candidate.metadata : '{}', - traceCount: typeof candidate.traceCount === 'number' ? candidate.traceCount : 0, - createdAt: typeof candidate.createdAt === 'number' ? candidate.createdAt : row.created_at, - updatedAt: typeof candidate.updatedAt === 'number' ? candidate.updatedAt : row.created_at - } -} - -export function tapeMessageRank(record: ChatMessageRecord, includePending: boolean): number { - if (record.status === 'sent' || record.status === 'error') { - return 2 - } - return includePending && record.status === 'pending' ? 1 : 0 -} - -export function readTapeMessageRetractionId(row: DeepChatTapeEntryRow): string | null { - if (row.kind !== 'event' || row.name !== 'message/retracted') { - return null - } - - const payload = parseTapeJsonObject(row.payload_json) - const data = parseNestedTapeJsonObject(payload.data) - return typeof data.messageId === 'string' ? data.messageId : null -} - -export function readTapeToolStatus(row: DeepChatTapeEntryRow): string | null { - const meta = parseTapeJsonObject(row.meta_json) - return typeof meta.status === 'string' ? meta.status : null -} - -export function tapeToolRank(row: DeepChatTapeEntryRow, includePending: boolean): number { - const status = readTapeToolStatus(row) - if (status === 'pending') { - return includePending ? 1 : 0 - } - return status !== null && TERMINAL_TAPE_TOOL_STATUSES.has(status) ? 2 : 0 -} - -export function readTapeToolIdentity(row: DeepChatTapeEntryRow): DeepChatTapeToolIdentity | null { - if (row.kind !== 'tool_call' && row.kind !== 'tool_result') { - return null - } - - const payload = parseTapeJsonObject(row.payload_json) - const messageId = payload.messageId - if (typeof messageId !== 'string' || messageId.length === 0) { - return null - } - - let toolCallId: unknown - if (row.kind === 'tool_call') { - toolCallId = parseNestedTapeJsonObject(payload.toolCall).id - } else { - toolCallId = payload.toolCallId - } - - if (typeof toolCallId !== 'string' || toolCallId.length === 0) { - return null - } - - return { - key: `${row.kind}:${messageId}:${toolCallId}`, - messageId - } -} +export * from '@/tape/domain/effectiveSemantics' diff --git a/src/main/session/data/tables/deepchatTapeEntries.ts b/src/main/session/data/tables/deepchatTapeEntries.ts index 2987280341..018a5ce2ef 100644 --- a/src/main/session/data/tables/deepchatTapeEntries.ts +++ b/src/main/session/data/tables/deepchatTapeEntries.ts @@ -2,82 +2,45 @@ import Database from 'better-sqlite3-multiple-ciphers' import logger from '@shared/logger' import { BaseTable } from '@/data/baseTable' import { randomUUID } from 'crypto' - -export type DeepChatTapeEntryKind = 'event' | 'anchor' | 'message' | 'tool_call' | 'tool_result' - -export const TAPE_INCARNATION_META_KEY = 'tapeIncarnationId' - -export type DeepChatTapeSourceType = - | 'session' - | 'message' - | 'assistant_block' - | 'tool_call' - | 'tool_result' - | 'runtime_event' - | 'migration' - | 'summary' - | 'fork' - | 'subagent' - -export interface DeepChatTapeEntryRow { - session_id: string - entry_id: number - kind: DeepChatTapeEntryKind - name: string | null - source_type: DeepChatTapeSourceType | null - source_id: string | null - source_seq: number | null - provenance_key: string | null - payload_json: string - meta_json: string - created_at: number -} - -export interface DeepChatTapeSourceInput { - type: DeepChatTapeSourceType - id: string - seq?: number | null -} - -export interface DeepChatTapeAppendInput { - sessionId: string - kind: DeepChatTapeEntryKind - name?: string | null - source?: DeepChatTapeSourceInput | null - provenanceKey?: string | null - payload: Record - meta?: Record - createdAt?: number - idempotent?: boolean -} - -export interface DeepChatTapeSearchInput { - limit?: number - kinds?: DeepChatTapeEntryKind[] - startCreatedAt?: number - endCreatedAt?: number -} - -export interface DeepChatTapeReadSource { - sessionId: string - maxEntryId: number -} - -export interface DeepChatTapeMutationProjection { - applyAppendedEntry(row: DeepChatTapeEntryRow, previousSessionMaxEntryId: number): boolean - invalidateSession(sessionId: string): void - deleteBySession(sessionId: string): void -} - -export const SUMMARY_ANCHOR_NAMES = [ - 'compaction/auto', - 'compaction/manual', - 'compaction/context_pressure', - 'compaction/resume', - 'compaction/migrated_summary', - 'auto_handoff/context_overflow', - 'summary/reset' -] as const +import { + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources, + SUMMARY_ANCHOR_NAMES, + TAPE_INCARNATION_META_KEY, + type DeepChatTapeAppendInput, + type DeepChatTapeEntryRow, + type DeepChatTapeReadSource, + type DeepChatTapeSearchInput, + type TapeAnchorAppendInput, + type TapeEventAppendInput +} from '@/tape/domain/entry' +import type { + TapeBootstrapStore, + TapeEntryLifecycleStore, + TapeEntryStore, + TapeMutationProjection, + TapeTransactionRunner +} from '@/tape/ports/storage' + +export { + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources, + SUMMARY_ANCHOR_NAMES, + TAPE_INCARNATION_META_KEY +} from '@/tape/domain/entry' +export type { + DeepChatTapeAppendInput, + DeepChatTapeEntryKind, + DeepChatTapeEntryRow, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + DeepChatTapeSourceInput, + DeepChatTapeSourceType, + TapeAnchorAppendInput, + TapeEventAppendInput +} from '@/tape/domain/entry' + +export type DeepChatTapeMutationProjection = TapeMutationProjection const RECONSTRUCTION_ANCHOR_NAMES = SUMMARY_ANCHOR_NAMES @@ -161,33 +124,6 @@ export function buildDeepChatTapeLikeSearchPredicate( } } -export function normalizeDeepChatTapeReadSources( - sources: readonly DeepChatTapeReadSource[] -): DeepChatTapeReadSource[] { - const maxEntryIdBySession = new Map() - for (const source of sources) { - const sessionId = source.sessionId.trim() - if (!sessionId || !Number.isSafeInteger(source.maxEntryId) || source.maxEntryId < 0) { - continue - } - maxEntryIdBySession.set( - sessionId, - Math.max(maxEntryIdBySession.get(sessionId) ?? 0, source.maxEntryId) - ) - } - return [...maxEntryIdBySession.entries()] - .map(([sessionId, maxEntryId]) => ({ sessionId, maxEntryId })) - .sort((left, right) => - left.sessionId < right.sessionId ? -1 : left.sessionId > right.sessionId ? 1 : 0 - ) -} - -export function serializeDeepChatTapeReadSources( - sources: readonly DeepChatTapeReadSource[] -): string { - return JSON.stringify(normalizeDeepChatTapeReadSources(sources)) -} - const AUTHORIZED_TAPE_SOURCES_CTE_SQL = ` authorized_sources(session_id, max_entry_id) AS ( SELECT @@ -447,7 +383,10 @@ const EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL = ` ) ` -export class DeepChatTapeEntriesTable extends BaseTable { +export class DeepChatTapeEntriesTable + extends BaseTable + implements TapeEntryStore, TapeTransactionRunner, TapeBootstrapStore, TapeEntryLifecycleStore +{ constructor( db: Database.Database, private readonly mutationProjection?: DeepChatTapeMutationProjection @@ -583,16 +522,7 @@ export class DeepChatTapeEntriesTable extends BaseTable { return append() } - appendAnchor(input: { - sessionId: string - name: string - state: Record - meta?: Record - source?: DeepChatTapeSourceInput | null - provenanceKey?: string | null - createdAt?: number - idempotent?: boolean - }): DeepChatTapeEntryRow { + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow { return this.append({ sessionId: input.sessionId, kind: 'anchor', @@ -609,16 +539,7 @@ export class DeepChatTapeEntriesTable extends BaseTable { }) } - appendEvent(input: { - sessionId: string - name: string - data: Record - meta?: Record - source?: DeepChatTapeSourceInput | null - provenanceKey?: string | null - createdAt?: number - idempotent?: boolean - }): DeepChatTapeEntryRow { + appendEvent(input: TapeEventAppendInput): DeepChatTapeEntryRow { return this.append({ sessionId: input.sessionId, kind: 'event', diff --git a/src/main/session/data/tables/deepchatTapeSearchProjection.ts b/src/main/session/data/tables/deepchatTapeSearchProjection.ts index 46660b03f3..70910cafcb 100644 --- a/src/main/session/data/tables/deepchatTapeSearchProjection.ts +++ b/src/main/session/data/tables/deepchatTapeSearchProjection.ts @@ -2,16 +2,18 @@ import Database from 'better-sqlite3-multiple-ciphers' import { BaseTable } from '@/data/baseTable' import { buildDeepChatTapeFtsMatch, - buildDeepChatTapeLikeSearchPredicate, + buildDeepChatTapeLikeSearchPredicate +} from './deepchatTapeEntries' +import { normalizeDeepChatTapeReadSources, serializeDeepChatTapeReadSources -} from './deepchatTapeEntries' +} from '@/tape/domain/entry' import type { - DeepChatTapeReadSource, DeepChatTapeEntryKind, + DeepChatTapeReadSource, DeepChatTapeSearchInput, DeepChatTapeSourceType -} from './deepchatTapeEntries' +} from '@/tape/domain/entry' export const DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION = 2 diff --git a/src/main/session/data/tape.ts b/src/main/session/data/tape.ts index a0908c9d42..7266e7c72e 100644 --- a/src/main/session/data/tape.ts +++ b/src/main/session/data/tape.ts @@ -37,7 +37,7 @@ import { type DeepChatTapeReadSource, type DeepChatTapeEntryRow, type DeepChatTapeSearchInput -} from '@/session/data/tables/deepchatTapeEntries' +} from '@/tape/domain/entry' import type { DeepChatTapeSearchProjectionInput, DeepChatTapeSearchProjectionResultRow, @@ -45,17 +45,18 @@ import type { } from '@/session/data/tables/deepchatTapeSearchProjection' import type { DeepChatMessageTraceRow } from '@/session/data/tables/deepchatMessageTraces' import { appendMessageRecordToTape, appendTapeToolFact } from '@/session/data/tapeFacts' -import type { TapeEntryRef, TapeRecorder, TapeToolFactInput } from '@/agent/deepchat/loop/ports' +import type { TapeEntryRef, TapeToolFactInput } from '@/tape/domain/facts' +import type { TapeToolFactWriter } from '@/tape/ports/capabilities' import { buildEffectiveTapeView, getLastEffectiveTokenUsage, searchEffectiveTapeRows -} from '@/session/data/tapeEffectiveView' +} from '@/tape/domain/effectiveView' import { hashJson, TAPE_VIEW_MANIFEST_EVENT_NAME, verifyTapeViewManifestHash -} from '@/session/data/tapeViewManifest' +} from '@/tape/domain/viewManifest' export type TapeMigrationState = 'none' | 'ready' @@ -1268,7 +1269,7 @@ function withReplaySliceHash( } } -export class SessionTape implements Pick { +export class SessionTape implements TapeToolFactWriter { constructor(private readonly database: SessionDatabase) {} private get table(): SessionDatabase['deepchatTapeEntriesTable'] { diff --git a/src/main/session/data/tapeEffectiveView.ts b/src/main/session/data/tapeEffectiveView.ts index b49135e2bc..0cb25c20b6 100644 --- a/src/main/session/data/tapeEffectiveView.ts +++ b/src/main/session/data/tapeEffectiveView.ts @@ -1,255 +1 @@ -import type { ChatMessageRecord } from '@shared/types/agent-interface' -import type { - DeepChatTapeEntryKind, - DeepChatTapeEntryRow, - DeepChatTapeSearchInput -} from '@/session/data/tables/deepchatTapeEntries' -import { - parseNestedTapeJsonObject, - readTapeMessageRetractionId, - readTapeToolIdentity, - tapeEntryToMessageRecord, - tapeMessageRank, - tapeToolRank -} from '@/session/data/tables/deepchatTapeEffectiveSemantics' - -export interface EffectiveMessageEntry { - entryId: number - record: ChatMessageRecord -} - -export interface EffectiveTapeView { - rows: DeepChatTapeEntryRow[] - messageRecords: ChatMessageRecord[] - /** Effective messages paired with their tape entry_id, ordered by orderSeq (for lineage). */ - messageEntries: EffectiveMessageEntry[] -} - -interface EffectiveTapeViewOptions { - includePending?: boolean - includeAuditEvents?: boolean -} - -type EffectiveMessageCandidate = { - row: DeepChatTapeEntryRow - record: ChatMessageRecord -} - -function compareSqliteBinaryText(left: string, right: string): number { - return Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')) -} - -function toNonNegativeInteger(value: unknown): number | null { - if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { - return null - } - return Math.floor(value) -} - -function readTokenUsage(metadata: Record): number | null { - const totalTokens = toNonNegativeInteger(metadata.totalTokens ?? metadata.total_tokens) - if (totalTokens !== null) { - return totalTokens - } - - const inputTokens = toNonNegativeInteger(metadata.inputTokens ?? metadata.input_tokens) - const outputTokens = toNonNegativeInteger(metadata.outputTokens ?? metadata.output_tokens) - if (inputTokens !== null || outputTokens !== null) { - return (inputTokens ?? 0) + (outputTokens ?? 0) - } - - return null -} - -function shouldReplaceMessage( - current: EffectiveMessageCandidate | undefined, - next: EffectiveMessageCandidate, - includePending: boolean -): boolean { - if (!current) { - return true - } - - const currentRank = tapeMessageRank(current.record, includePending) - const nextRank = tapeMessageRank(next.record, includePending) - if (nextRank > currentRank) { - return true - } - if (nextRank < currentRank) { - return false - } - return next.row.entry_id > current.row.entry_id -} - -function isAuditEvent(row: DeepChatTapeEntryRow): boolean { - return ( - row.name === 'message/retracted' || - row.name === 'message/compaction_indicator' || - row.name === 'migration/backfill' - ) -} - -function shouldReplaceToolRow( - current: DeepChatTapeEntryRow | undefined, - next: DeepChatTapeEntryRow, - includePending: boolean -): boolean { - if (!current) { - return true - } - - const currentRank = tapeToolRank(current, includePending) - const nextRank = tapeToolRank(next, includePending) - if (nextRank > currentRank) { - return true - } - if (nextRank < currentRank) { - return false - } - return next.entry_id > current.entry_id -} - -function matchesKinds( - row: DeepChatTapeEntryRow, - kinds: DeepChatTapeEntryKind[] | undefined -): boolean { - return !kinds?.length || kinds.includes(row.kind) -} - -function matchesCreatedAt(row: DeepChatTapeEntryRow, options: DeepChatTapeSearchInput): boolean { - if ( - Number.isFinite(options.startCreatedAt) && - row.created_at < (options.startCreatedAt as number) - ) { - return false - } - if (Number.isFinite(options.endCreatedAt) && row.created_at > (options.endCreatedAt as number)) { - return false - } - return true -} - -function matchesQuery(row: DeepChatTapeEntryRow, normalizedQuery: string): boolean { - const haystack = `${row.payload_json}\n${row.meta_json}\n${row.name ?? ''}`.toLowerCase() - return haystack.includes(normalizedQuery) -} - -export function buildEffectiveTapeView( - rows: DeepChatTapeEntryRow[], - options: EffectiveTapeViewOptions = {} -): EffectiveTapeView { - const includePending = options.includePending === true - const includeAuditEvents = options.includeAuditEvents === true - const messageCandidates = new Map() - const retractedMessageIds = new Set() - const toolRows = new Map() - const anchorRows: DeepChatTapeEntryRow[] = [] - const eventRows: DeepChatTapeEntryRow[] = [] - - for (const row of [...rows].sort((left, right) => left.entry_id - right.entry_id)) { - if (row.kind === 'anchor') { - anchorRows.push(row) - continue - } - - if (row.kind === 'event') { - const retractedMessageId = readTapeMessageRetractionId(row) - if (retractedMessageId) { - messageCandidates.delete(retractedMessageId) - retractedMessageIds.add(retractedMessageId) - } - if (includeAuditEvents || !isAuditEvent(row)) { - eventRows.push(row) - } - continue - } - - if (row.kind === 'message') { - const record = tapeEntryToMessageRecord(row) - if (!record) { - continue - } - const rank = tapeMessageRank(record, includePending) - if (rank === 0) { - continue - } - const candidate = { row, record } - if (shouldReplaceMessage(messageCandidates.get(record.id), candidate, includePending)) { - messageCandidates.set(record.id, candidate) - retractedMessageIds.delete(record.id) - } - continue - } - - const identity = readTapeToolIdentity(row) - if (!identity || tapeToolRank(row, includePending) === 0) { - continue - } - const current = toolRows.get(identity.key)?.row - if (shouldReplaceToolRow(current, row, includePending)) { - toolRows.set(identity.key, { row, messageId: identity.messageId }) - } - } - - const messageRows = [...messageCandidates.values()] - .filter((candidate) => !retractedMessageIds.has(candidate.record.id)) - .sort( - (left, right) => - left.record.orderSeq - right.record.orderSeq || - compareSqliteBinaryText(left.record.id, right.record.id) - ) - const effectiveMessageIds = new Set(messageRows.map((candidate) => candidate.record.id)) - const effectiveToolRows = [...toolRows.values()] - .filter((candidate) => effectiveMessageIds.has(candidate.messageId)) - .map((candidate) => candidate.row) - const effectiveRows = [ - ...anchorRows, - ...eventRows, - ...messageRows.map((candidate) => candidate.row), - ...effectiveToolRows - ].sort((left, right) => left.entry_id - right.entry_id) - - return { - rows: effectiveRows, - messageRecords: messageRows.map((candidate) => candidate.record), - messageEntries: messageRows.map((candidate) => ({ - entryId: candidate.row.entry_id, - record: candidate.record - })) - } -} - -export function searchEffectiveTapeRows( - rows: DeepChatTapeEntryRow[], - query: string, - options: DeepChatTapeSearchInput = {} -): DeepChatTapeEntryRow[] { - const normalizedQuery = query.trim().toLowerCase() - if (!normalizedQuery) { - return [] - } - - const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 - const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) - return buildEffectiveTapeView(rows, { includePending: false }) - .rows.filter((row) => matchesKinds(row, options.kinds)) - .filter((row) => matchesCreatedAt(row, options)) - .filter((row) => matchesQuery(row, normalizedQuery)) - .sort((left, right) => right.entry_id - left.entry_id) - .slice(0, cappedLimit) -} - -export function getLastEffectiveTokenUsage(rows: DeepChatTapeEntryRow[]): number | null { - const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows - for (let index = effectiveRows.length - 1; index >= 0; index -= 1) { - const record = tapeEntryToMessageRecord(effectiveRows[index]) - if (!record || record.role !== 'assistant') { - continue - } - const usage = readTokenUsage(parseNestedTapeJsonObject(record.metadata)) - if (usage !== null) { - return usage - } - } - return null -} +export * from '@/tape/domain/effectiveView' diff --git a/src/main/session/data/tapeFacts.ts b/src/main/session/data/tapeFacts.ts index e8e4028124..186ccde9d2 100644 --- a/src/main/session/data/tapeFacts.ts +++ b/src/main/session/data/tapeFacts.ts @@ -1,15 +1,15 @@ import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' -import type { TapeToolFactInput } from '@/agent/deepchat/loop/ports' -import { toAppSessionId } from '@/agent/shared/agentSessionIds' -import type { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' -import { buildEffectiveTapeView } from './tapeEffectiveView' -import { hashJson } from './tapeViewManifest' -import { parseAssistantBlocks } from '@/session/data/tables/deepchatTapeEffectiveSemantics' +import { toTapeSessionId, type TapeFactSource, type TapeToolFactInput } from '@/tape/domain/facts' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeBootstrapStore, TapeEntryStore } from '@/tape/ports/storage' +import { buildEffectiveTapeView } from '@/tape/domain/effectiveView' +import { parseAssistantBlocks } from '@/tape/domain/effectiveSemantics' +import { hashJson } from '@/tape/domain/viewManifest' -export { tapeEntryToMessageRecord } from '@/session/data/tables/deepchatTapeEffectiveSemantics' +export { tapeEntryToMessageRecord } from '@/tape/domain/effectiveSemantics' +export type { TapeFactSource } from '@/tape/domain/facts' -export type TapeFactSource = 'live' | 'backfill' | 'repair' +type TapeFactStore = Pick & TapeBootstrapStore function readCompactionStatus(record: ChatMessageRecord): string | null { try { @@ -81,7 +81,7 @@ export function buildTapeToolFactInputs(record: ChatMessageRecord): TapeToolFact const factBlock = block.timestamp === undefined ? { ...block, timestamp: record.updatedAt } : block inputs.push({ - sessionId: toAppSessionId(record.sessionId), + sessionId: toTapeSessionId(record.sessionId), messageId: record.id, orderSeq: record.orderSeq, blockIndex, @@ -90,7 +90,7 @@ export function buildTapeToolFactInputs(record: ChatMessageRecord): TapeToolFact }) if (typeof block.tool_call.response === 'string' && block.tool_call.response.length > 0) { inputs.push({ - sessionId: toAppSessionId(record.sessionId), + sessionId: toTapeSessionId(record.sessionId), messageId: record.id, orderSeq: record.orderSeq, blockIndex, @@ -103,7 +103,7 @@ export function buildTapeToolFactInputs(record: ChatMessageRecord): TapeToolFact } export function appendTapeToolFact( - table: DeepChatTapeEntriesTable, + table: TapeFactStore, input: TapeToolFactInput, source: TapeFactSource, reason?: string @@ -181,7 +181,7 @@ export function appendTapeToolFact( } export function appendToolFactsToTape( - table: DeepChatTapeEntriesTable, + table: TapeFactStore, record: ChatMessageRecord, source: TapeFactSource, reason?: string @@ -197,7 +197,7 @@ export function appendToolFactsToTape( } export function appendMessageRecordToTape( - table: DeepChatTapeEntriesTable, + table: TapeFactStore, record: ChatMessageRecord, source: TapeFactSource ): number { @@ -269,7 +269,7 @@ export function appendMessageRecordToTape( } export function appendMessageReplacementToTape( - table: DeepChatTapeEntriesTable, + table: TapeFactStore, record: ChatMessageRecord, reason: string ): number { @@ -315,7 +315,7 @@ export function appendMessageReplacementToTape( } export function appendMessageRetractionToTape( - table: DeepChatTapeEntriesTable, + table: TapeFactStore, record: ChatMessageRecord, reason: string ): number { diff --git a/src/main/session/data/tapeViewManifest.ts b/src/main/session/data/tapeViewManifest.ts index 95794a955d..4b7cec35ef 100644 --- a/src/main/session/data/tapeViewManifest.ts +++ b/src/main/session/data/tapeViewManifest.ts @@ -1,362 +1 @@ -import { createHash } from 'crypto' -import type { ChatMessage } from '@shared/types/core/chat-message' -import type { MCPToolDefinition } from '@shared/types/core/mcp' -import type { ChatMessageRecord } from '@shared/types/agent-interface' -import type { - DeepChatTapeViewEntryRef, - DeepChatTapeViewExcludedRange, - DeepChatTapeViewExcludedRef, - DeepChatTapeViewManifest, - DeepChatTapeViewManifestIntegrity, - DeepChatTapeViewPolicy, - DeepChatTapeViewTaskType, - DeepChatTapeViewTokenBudget -} from '@shared/types/tape-view-manifest' -import { estimateMessagesTokens } from '@shared/utils/messageTokens' - -export type ContextSummaryCursorMetadata = { - summaryCursorOrderSeq: number - preCursorOrderSeqMin: number | null - preCursorOrderSeqMax: number | null - preCursorCount: number -} - -export function isCompactionRecord(record: ChatMessageRecord): boolean { - try { - const metadata = JSON.parse(record.metadata) as { messageType?: string } - return metadata.messageType === 'compaction' - } catch { - return false - } -} - -export const TAPE_VIEW_MANIFEST_EVENT_NAME = 'view/assembled' -export const TAPE_VIEW_CONTEXT_BUILDER_VERSION = 'legacy-v1' as const - -export type TapeViewManifestSourceMaps = { - entryIdByMessageId?: Map - toolCallEntryIdByToolId?: Map - toolResultEntryIdByToolId?: Map -} - -export type TapeViewManifestBuildInput = { - sessionId: string - messageId: string - requestSeq: number - taskType: DeepChatTapeViewTaskType - policy: DeepChatTapeViewPolicy - policyVersion?: number | null - messages: ChatMessage[] - tools: MCPToolDefinition[] - latestEntryId: number - anchorEntryIds: number[] - reconstructionAnchorEntryId?: number | null - included: DeepChatTapeViewEntryRef[] - excluded: DeepChatTapeViewExcludedRef[] - summaryCursor?: ContextSummaryCursorMetadata - tokenBudget: Omit - providerId: string - modelId: string - summaryCursorOrderSeq: number - supportsVision: boolean - supportsAudioInput: boolean - traceDebugEnabled: boolean - assembledAt?: number -} - -export type TapeViewManifestPolicyInput = { - recoveredFromContextPressure: boolean - isInitialViewRequest: boolean - viewPolicy?: DeepChatTapeViewPolicy - viewPolicyVersion?: number | null -} - -export type TapeViewManifestPolicyResult = { - policy: DeepChatTapeViewPolicy - policyVersion: number | null -} - -export type TapeViewContextSelection = { - includedRecords: Array<{ - record: ChatMessageRecord - reason: DeepChatTapeViewEntryRef['reason'] - }> - excludedRecords: Array<{ - record: ChatMessageRecord - reason: DeepChatTapeViewExcludedRef['reason'] - }> - summaryCursor?: ContextSummaryCursorMetadata - includesSystemPrompt: boolean - newUserMessageId?: string | null -} - -export function resolveTapeViewManifestPolicy( - input: TapeViewManifestPolicyInput -): TapeViewManifestPolicyResult { - if (input.recoveredFromContextPressure) { - return { - policy: 'context_pressure_recovery_shadow', - policyVersion: null - } - } - - if (input.isInitialViewRequest && input.viewPolicy) { - return { - policy: input.viewPolicy, - policyVersion: input.viewPolicyVersion ?? null - } - } - - return { - policy: 'tool_loop_shadow', - policyVersion: null - } -} - -function normalizeForStableJson(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map(normalizeForStableJson) - } - - if (!value || typeof value !== 'object') { - return value - } - - const record = value as Record - return Object.keys(record) - .sort() - .reduce>((result, key) => { - const nested = record[key] - if (nested !== undefined) { - result[key] = normalizeForStableJson(nested) - } - return result - }, {}) -} - -export function stableJsonStringify(value: unknown): string { - return JSON.stringify(normalizeForStableJson(value)) -} - -export function hashJson(value: unknown): string { - return createHash('sha256').update(stableJsonStringify(value)).digest('hex') -} - -export const TAPE_VIEW_MANIFEST_HASH_VERSION = 2 - -function buildManifestHash(manifest: DeepChatTapeViewManifest): string { - const hashable: Record = { ...manifest } - delete hashable.assembledAt - delete hashable.viewId - hashable.hashes = { - promptHash: manifest.hashes.promptHash, - toolDefinitionsHash: manifest.hashes.toolDefinitionsHash - } - return hashJson(hashable) -} - -function buildExcludedRanges( - summaryCursor?: ContextSummaryCursorMetadata -): DeepChatTapeViewExcludedRange[] { - if ( - !summaryCursor || - summaryCursor.preCursorCount === 0 || - summaryCursor.preCursorOrderSeqMin === null || - summaryCursor.preCursorOrderSeqMax === null - ) { - return [] - } - return [ - { - fromOrderSeq: summaryCursor.preCursorOrderSeqMin, - toOrderSeq: summaryCursor.preCursorOrderSeqMax, - count: summaryCursor.preCursorCount, - reason: 'before_summary_cursor' - } - ] -} - -export function verifyTapeViewManifestHash( - manifest: DeepChatTapeViewManifest -): DeepChatTapeViewManifestIntegrity { - if (manifest.hashVersion !== TAPE_VIEW_MANIFEST_HASH_VERSION) { - return 'unverified' - } - return buildManifestHash(manifest) === manifest.hashes.manifestHash ? 'valid' : 'invalid' -} - -export function createTapeViewManifest( - input: TapeViewManifestBuildInput -): DeepChatTapeViewManifest { - const assembledAt = input.assembledAt ?? Date.now() - const excludedRanges = buildExcludedRanges(input.summaryCursor) - const draft: DeepChatTapeViewManifest = { - schemaVersion: 2, - hashVersion: TAPE_VIEW_MANIFEST_HASH_VERSION, - viewId: '', - sessionId: input.sessionId, - messageId: input.messageId, - requestSeq: input.requestSeq, - taskType: input.taskType, - policy: input.policy, - policyVersion: input.policyVersion ?? null, - contextBuilderVersion: TAPE_VIEW_CONTEXT_BUILDER_VERSION, - latestEntryId: input.latestEntryId, - anchorEntryIds: [...input.anchorEntryIds], - ...(input.reconstructionAnchorEntryId !== undefined - ? { reconstructionAnchorEntryId: input.reconstructionAnchorEntryId } - : {}), - included: input.included.map((entry) => ({ ...entry })), - excluded: input.excluded.map((entry) => ({ ...entry })), - ...(excludedRanges.length > 0 ? { excludedRanges } : {}), - tokenBudget: { - ...input.tokenBudget, - estimatedPromptTokens: estimateMessagesTokens(input.messages) - }, - hashes: { - promptHash: hashJson(input.messages), - toolDefinitionsHash: hashJson(input.tools), - manifestHash: '' - }, - meta: { - providerId: input.providerId, - modelId: input.modelId, - summaryCursorOrderSeq: input.summaryCursorOrderSeq, - supportsVision: input.supportsVision, - supportsAudioInput: input.supportsAudioInput, - traceDebugEnabled: input.traceDebugEnabled - }, - assembledAt - } - - const manifestHash = buildManifestHash(draft) - return { - ...draft, - viewId: `view_${manifestHash.slice(0, 16)}`, - hashes: { ...draft.hashes, manifestHash } - } -} - -export function buildIncludedRefs( - selection: TapeViewContextSelection, - sourceMaps: TapeViewManifestSourceMaps = {} -): DeepChatTapeViewEntryRef[] { - const refs: DeepChatTapeViewEntryRef[] = [] - - if (selection.includesSystemPrompt) { - refs.push({ - entryId: null, - messageId: null, - orderSeq: null, - role: 'system', - source: 'synthetic', - reason: 'system_prompt' - }) - } - - for (const item of selection.includedRecords) { - refs.push({ - entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, - messageId: item.record.id, - orderSeq: item.record.orderSeq, - role: item.record.role, - source: sourceMaps.entryIdByMessageId?.has(item.record.id) ? 'tape' : 'synthetic', - reason: item.reason - }) - } - - if (selection.newUserMessageId) { - refs.push({ - entryId: sourceMaps.entryIdByMessageId?.get(selection.newUserMessageId) ?? null, - messageId: selection.newUserMessageId, - orderSeq: null, - role: 'user', - source: sourceMaps.entryIdByMessageId?.has(selection.newUserMessageId) ? 'tape' : 'synthetic', - reason: 'new_user_input' - }) - } - - return refs -} - -export function buildExcludedRefs( - selection: TapeViewContextSelection, - sourceMaps: TapeViewManifestSourceMaps = {} -): DeepChatTapeViewExcludedRef[] { - return selection.excludedRecords.map((item) => ({ - entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, - messageId: item.record.id, - orderSeq: item.record.orderSeq, - reason: item.reason - })) -} - -export function buildRequestRefs( - messages: ChatMessage[], - sourceMaps: TapeViewManifestSourceMaps = {} -): DeepChatTapeViewEntryRef[] { - const lastToolCallIndex = new Map() - const lastToolResultIndex = new Map() - messages.forEach((message, index) => { - if (message.role === 'assistant' && message.tool_calls?.length) { - for (const toolCall of message.tool_calls) { - lastToolCallIndex.set(toolCall.id, index) - } - } else if (message.role === 'tool' && message.tool_call_id) { - lastToolResultIndex.set(message.tool_call_id, index) - } - }) - - const refs: DeepChatTapeViewEntryRef[] = [] - messages.forEach((message, index) => { - if (message.role === 'assistant' && message.tool_calls?.length) { - for (const toolCall of message.tool_calls) { - const entryId = - lastToolCallIndex.get(toolCall.id) === index - ? (sourceMaps.toolCallEntryIdByToolId?.get(toolCall.id) ?? null) - : null - refs.push({ - entryId, - messageId: null, - orderSeq: null, - role: 'assistant', - source: entryId === null ? 'synthetic' : 'tape', - reason: 'tool_loop_message' - }) - } - return - } - - if (message.role === 'tool' && message.tool_call_id) { - const entryId = - lastToolResultIndex.get(message.tool_call_id) === index - ? (sourceMaps.toolResultEntryIdByToolId?.get(message.tool_call_id) ?? null) - : null - refs.push({ - entryId, - messageId: null, - orderSeq: null, - role: 'tool', - source: entryId === null ? 'synthetic' : 'tape', - reason: 'tool_loop_message' - }) - return - } - - refs.push({ - entryId: null, - messageId: null, - orderSeq: null, - role: message.role, - source: 'synthetic', - reason: - message.role === 'system' - ? 'system_prompt' - : message.role === 'tool' - ? 'tool_loop_message' - : 'selected_history' - }) - }) - - return refs -} +export * from '@/tape/domain/viewManifest' diff --git a/src/main/tape/domain/effectiveSemantics.ts b/src/main/tape/domain/effectiveSemantics.ts new file mode 100644 index 0000000000..058d96de25 --- /dev/null +++ b/src/main/tape/domain/effectiveSemantics.ts @@ -0,0 +1,161 @@ +import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryRow } from './entry' + +const TERMINAL_TAPE_TOOL_STATUSES = new Set(['success', 'error']) + +export interface DeepChatTapeToolIdentity { + key: string + messageId: string +} + +export function parseTapeJsonObject(raw: string): Record { + try { + const parsed = JSON.parse(raw) as unknown + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record + } + } catch {} + return {} +} + +export function parseNestedTapeJsonObject(value: unknown): Record { + if (typeof value === 'string') { + return parseTapeJsonObject(value) + } + if (value && typeof value === 'object' && !Array.isArray(value)) { + return value as Record + } + return {} +} + +export function parseAssistantBlocks(rawContent: string): AssistantMessageBlock[] { + try { + const parsed = JSON.parse(rawContent) as unknown + return Array.isArray(parsed) ? (parsed as AssistantMessageBlock[]) : [] + } catch { + return [] + } +} + +export function messageRecordHasFinalToolUse(record: ChatMessageRecord): boolean { + if (record.role !== 'assistant' || (record.status !== 'sent' && record.status !== 'error')) { + return false + } + const blocks = parseAssistantBlocks(record.content) + const pendingInteractionToolIds = new Set( + blocks.flatMap((block) => + block.type === 'action' && + (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && + block.status === 'pending' && + typeof block.tool_call?.id === 'string' + ? [block.tool_call.id] + : [] + ) + ) + return blocks.some( + (block) => + block.type === 'tool_call' && + (block.status === 'success' || block.status === 'error') && + typeof block.tool_call?.id === 'string' && + !pendingInteractionToolIds.has(block.tool_call.id) + ) +} + +function isMessageStatus(value: unknown): value is ChatMessageRecord['status'] { + return value === 'pending' || value === 'sent' || value === 'error' +} + +export function tapeEntryToMessageRecord(row: DeepChatTapeEntryRow): ChatMessageRecord | null { + if (row.kind !== 'message') { + return null + } + + const payload = parseTapeJsonObject(row.payload_json) + const record = payload.record + if (!record || typeof record !== 'object' || Array.isArray(record)) { + return null + } + + const candidate = record as Partial + if ( + typeof candidate.id !== 'string' || + typeof candidate.sessionId !== 'string' || + typeof candidate.orderSeq !== 'number' || + (candidate.role !== 'user' && candidate.role !== 'assistant') || + typeof candidate.content !== 'string' + ) { + return null + } + + return { + id: candidate.id, + sessionId: candidate.sessionId, + orderSeq: candidate.orderSeq, + role: candidate.role, + content: candidate.content, + status: isMessageStatus(candidate.status) ? candidate.status : 'sent', + isContextEdge: typeof candidate.isContextEdge === 'number' ? candidate.isContextEdge : 0, + metadata: typeof candidate.metadata === 'string' ? candidate.metadata : '{}', + traceCount: typeof candidate.traceCount === 'number' ? candidate.traceCount : 0, + createdAt: typeof candidate.createdAt === 'number' ? candidate.createdAt : row.created_at, + updatedAt: typeof candidate.updatedAt === 'number' ? candidate.updatedAt : row.created_at + } +} + +export function tapeMessageRank(record: ChatMessageRecord, includePending: boolean): number { + if (record.status === 'sent' || record.status === 'error') { + return 2 + } + return includePending && record.status === 'pending' ? 1 : 0 +} + +export function readTapeMessageRetractionId(row: DeepChatTapeEntryRow): string | null { + if (row.kind !== 'event' || row.name !== 'message/retracted') { + return null + } + + const payload = parseTapeJsonObject(row.payload_json) + const data = parseNestedTapeJsonObject(payload.data) + return typeof data.messageId === 'string' ? data.messageId : null +} + +export function readTapeToolStatus(row: DeepChatTapeEntryRow): string | null { + const meta = parseTapeJsonObject(row.meta_json) + return typeof meta.status === 'string' ? meta.status : null +} + +export function tapeToolRank(row: DeepChatTapeEntryRow, includePending: boolean): number { + const status = readTapeToolStatus(row) + if (status === 'pending') { + return includePending ? 1 : 0 + } + return status !== null && TERMINAL_TAPE_TOOL_STATUSES.has(status) ? 2 : 0 +} + +export function readTapeToolIdentity(row: DeepChatTapeEntryRow): DeepChatTapeToolIdentity | null { + if (row.kind !== 'tool_call' && row.kind !== 'tool_result') { + return null + } + + const payload = parseTapeJsonObject(row.payload_json) + const messageId = payload.messageId + if (typeof messageId !== 'string' || messageId.length === 0) { + return null + } + + let toolCallId: unknown + if (row.kind === 'tool_call') { + toolCallId = parseNestedTapeJsonObject(payload.toolCall).id + } else { + toolCallId = payload.toolCallId + } + + if (typeof toolCallId !== 'string' || toolCallId.length === 0) { + return null + } + + return { + key: `${row.kind}:${messageId}:${toolCallId}`, + messageId + } +} diff --git a/src/main/tape/domain/effectiveView.ts b/src/main/tape/domain/effectiveView.ts new file mode 100644 index 0000000000..377341ace1 --- /dev/null +++ b/src/main/tape/domain/effectiveView.ts @@ -0,0 +1,251 @@ +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryKind, DeepChatTapeEntryRow, DeepChatTapeSearchInput } from './entry' +import { + parseNestedTapeJsonObject, + readTapeMessageRetractionId, + readTapeToolIdentity, + tapeEntryToMessageRecord, + tapeMessageRank, + tapeToolRank +} from './effectiveSemantics' + +export interface EffectiveMessageEntry { + entryId: number + record: ChatMessageRecord +} + +export interface EffectiveTapeView { + rows: DeepChatTapeEntryRow[] + messageRecords: ChatMessageRecord[] + /** Effective messages paired with their tape entry_id, ordered by orderSeq (for lineage). */ + messageEntries: EffectiveMessageEntry[] +} + +interface EffectiveTapeViewOptions { + includePending?: boolean + includeAuditEvents?: boolean +} + +type EffectiveMessageCandidate = { + row: DeepChatTapeEntryRow + record: ChatMessageRecord +} + +function compareSqliteBinaryText(left: string, right: string): number { + return Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')) +} + +function toNonNegativeInteger(value: unknown): number | null { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + return null + } + return Math.floor(value) +} + +function readTokenUsage(metadata: Record): number | null { + const totalTokens = toNonNegativeInteger(metadata.totalTokens ?? metadata.total_tokens) + if (totalTokens !== null) { + return totalTokens + } + + const inputTokens = toNonNegativeInteger(metadata.inputTokens ?? metadata.input_tokens) + const outputTokens = toNonNegativeInteger(metadata.outputTokens ?? metadata.output_tokens) + if (inputTokens !== null || outputTokens !== null) { + return (inputTokens ?? 0) + (outputTokens ?? 0) + } + + return null +} + +function shouldReplaceMessage( + current: EffectiveMessageCandidate | undefined, + next: EffectiveMessageCandidate, + includePending: boolean +): boolean { + if (!current) { + return true + } + + const currentRank = tapeMessageRank(current.record, includePending) + const nextRank = tapeMessageRank(next.record, includePending) + if (nextRank > currentRank) { + return true + } + if (nextRank < currentRank) { + return false + } + return next.row.entry_id > current.row.entry_id +} + +function isAuditEvent(row: DeepChatTapeEntryRow): boolean { + return ( + row.name === 'message/retracted' || + row.name === 'message/compaction_indicator' || + row.name === 'migration/backfill' + ) +} + +function shouldReplaceToolRow( + current: DeepChatTapeEntryRow | undefined, + next: DeepChatTapeEntryRow, + includePending: boolean +): boolean { + if (!current) { + return true + } + + const currentRank = tapeToolRank(current, includePending) + const nextRank = tapeToolRank(next, includePending) + if (nextRank > currentRank) { + return true + } + if (nextRank < currentRank) { + return false + } + return next.entry_id > current.entry_id +} + +function matchesKinds( + row: DeepChatTapeEntryRow, + kinds: DeepChatTapeEntryKind[] | undefined +): boolean { + return !kinds?.length || kinds.includes(row.kind) +} + +function matchesCreatedAt(row: DeepChatTapeEntryRow, options: DeepChatTapeSearchInput): boolean { + if ( + Number.isFinite(options.startCreatedAt) && + row.created_at < (options.startCreatedAt as number) + ) { + return false + } + if (Number.isFinite(options.endCreatedAt) && row.created_at > (options.endCreatedAt as number)) { + return false + } + return true +} + +function matchesQuery(row: DeepChatTapeEntryRow, normalizedQuery: string): boolean { + const haystack = `${row.payload_json}\n${row.meta_json}\n${row.name ?? ''}`.toLowerCase() + return haystack.includes(normalizedQuery) +} + +export function buildEffectiveTapeView( + rows: DeepChatTapeEntryRow[], + options: EffectiveTapeViewOptions = {} +): EffectiveTapeView { + const includePending = options.includePending === true + const includeAuditEvents = options.includeAuditEvents === true + const messageCandidates = new Map() + const retractedMessageIds = new Set() + const toolRows = new Map() + const anchorRows: DeepChatTapeEntryRow[] = [] + const eventRows: DeepChatTapeEntryRow[] = [] + + for (const row of [...rows].sort((left, right) => left.entry_id - right.entry_id)) { + if (row.kind === 'anchor') { + anchorRows.push(row) + continue + } + + if (row.kind === 'event') { + const retractedMessageId = readTapeMessageRetractionId(row) + if (retractedMessageId) { + messageCandidates.delete(retractedMessageId) + retractedMessageIds.add(retractedMessageId) + } + if (includeAuditEvents || !isAuditEvent(row)) { + eventRows.push(row) + } + continue + } + + if (row.kind === 'message') { + const record = tapeEntryToMessageRecord(row) + if (!record) { + continue + } + const rank = tapeMessageRank(record, includePending) + if (rank === 0) { + continue + } + const candidate = { row, record } + if (shouldReplaceMessage(messageCandidates.get(record.id), candidate, includePending)) { + messageCandidates.set(record.id, candidate) + retractedMessageIds.delete(record.id) + } + continue + } + + const identity = readTapeToolIdentity(row) + if (!identity || tapeToolRank(row, includePending) === 0) { + continue + } + const current = toolRows.get(identity.key)?.row + if (shouldReplaceToolRow(current, row, includePending)) { + toolRows.set(identity.key, { row, messageId: identity.messageId }) + } + } + + const messageRows = [...messageCandidates.values()] + .filter((candidate) => !retractedMessageIds.has(candidate.record.id)) + .sort( + (left, right) => + left.record.orderSeq - right.record.orderSeq || + compareSqliteBinaryText(left.record.id, right.record.id) + ) + const effectiveMessageIds = new Set(messageRows.map((candidate) => candidate.record.id)) + const effectiveToolRows = [...toolRows.values()] + .filter((candidate) => effectiveMessageIds.has(candidate.messageId)) + .map((candidate) => candidate.row) + const effectiveRows = [ + ...anchorRows, + ...eventRows, + ...messageRows.map((candidate) => candidate.row), + ...effectiveToolRows + ].sort((left, right) => left.entry_id - right.entry_id) + + return { + rows: effectiveRows, + messageRecords: messageRows.map((candidate) => candidate.record), + messageEntries: messageRows.map((candidate) => ({ + entryId: candidate.row.entry_id, + record: candidate.record + })) + } +} + +export function searchEffectiveTapeRows( + rows: DeepChatTapeEntryRow[], + query: string, + options: DeepChatTapeSearchInput = {} +): DeepChatTapeEntryRow[] { + const normalizedQuery = query.trim().toLowerCase() + if (!normalizedQuery) { + return [] + } + + const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 + const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) + return buildEffectiveTapeView(rows, { includePending: false }) + .rows.filter((row) => matchesKinds(row, options.kinds)) + .filter((row) => matchesCreatedAt(row, options)) + .filter((row) => matchesQuery(row, normalizedQuery)) + .sort((left, right) => right.entry_id - left.entry_id) + .slice(0, cappedLimit) +} + +export function getLastEffectiveTokenUsage(rows: DeepChatTapeEntryRow[]): number | null { + const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows + for (let index = effectiveRows.length - 1; index >= 0; index -= 1) { + const record = tapeEntryToMessageRecord(effectiveRows[index]) + if (!record || record.role !== 'assistant') { + continue + } + const usage = readTokenUsage(parseNestedTapeJsonObject(record.metadata)) + if (usage !== null) { + return usage + } + } + return null +} diff --git a/src/main/tape/domain/entry.ts b/src/main/tape/domain/entry.ts new file mode 100644 index 0000000000..2bfbaf30e3 --- /dev/null +++ b/src/main/tape/domain/entry.ts @@ -0,0 +1,118 @@ +export type DeepChatTapeEntryKind = 'event' | 'anchor' | 'message' | 'tool_call' | 'tool_result' + +export const TAPE_INCARNATION_META_KEY = 'tapeIncarnationId' + +export type DeepChatTapeSourceType = + | 'session' + | 'message' + | 'assistant_block' + | 'tool_call' + | 'tool_result' + | 'runtime_event' + | 'migration' + | 'summary' + | 'fork' + | 'subagent' + +export interface DeepChatTapeEntryRow { + session_id: string + entry_id: number + kind: DeepChatTapeEntryKind + name: string | null + source_type: DeepChatTapeSourceType | null + source_id: string | null + source_seq: number | null + provenance_key: string | null + payload_json: string + meta_json: string + created_at: number +} + +export interface DeepChatTapeSourceInput { + type: DeepChatTapeSourceType + id: string + seq?: number | null +} + +export interface DeepChatTapeAppendInput { + sessionId: string + kind: DeepChatTapeEntryKind + name?: string | null + source?: DeepChatTapeSourceInput | null + provenanceKey?: string | null + payload: Record + meta?: Record + createdAt?: number + idempotent?: boolean +} + +export interface TapeAnchorAppendInput { + sessionId: string + name: string + state: Record + meta?: Record + source?: DeepChatTapeSourceInput | null + provenanceKey?: string | null + createdAt?: number + idempotent?: boolean +} + +export interface TapeEventAppendInput { + sessionId: string + name: string + data: Record + meta?: Record + source?: DeepChatTapeSourceInput | null + provenanceKey?: string | null + createdAt?: number + idempotent?: boolean +} + +export interface DeepChatTapeSearchInput { + limit?: number + kinds?: DeepChatTapeEntryKind[] + startCreatedAt?: number + endCreatedAt?: number +} + +export interface DeepChatTapeReadSource { + sessionId: string + maxEntryId: number +} + +export const SUMMARY_ANCHOR_NAMES = [ + 'compaction/auto', + 'compaction/manual', + 'compaction/context_pressure', + 'compaction/resume', + 'compaction/migrated_summary', + 'auto_handoff/context_overflow', + 'summary/reset' +] as const + +export function normalizeDeepChatTapeReadSources( + sources: readonly DeepChatTapeReadSource[] +): DeepChatTapeReadSource[] { + const maxEntryIdBySession = new Map() + for (const source of sources) { + const sessionId = source.sessionId.trim() + if (!sessionId || !Number.isSafeInteger(source.maxEntryId) || source.maxEntryId < 0) { + continue + } + maxEntryIdBySession.set( + sessionId, + Math.max(maxEntryIdBySession.get(sessionId) ?? 0, source.maxEntryId) + ) + } + return [...maxEntryIdBySession.entries()] + .map(([sessionId, maxEntryId]) => ({ sessionId, maxEntryId })) + .sort((left, right) => + left.sessionId < right.sessionId ? -1 : left.sessionId > right.sessionId ? 1 : 0 + ) +} + +export function serializeDeepChatTapeReadSources( + sources: readonly DeepChatTapeReadSource[] +): string { + return JSON.stringify(normalizeDeepChatTapeReadSources(sources)) +} diff --git a/src/main/tape/domain/facts.ts b/src/main/tape/domain/facts.ts new file mode 100644 index 0000000000..2c30b6abbe --- /dev/null +++ b/src/main/tape/domain/facts.ts @@ -0,0 +1,29 @@ +import type { AssistantMessageBlock } from '@shared/types/agent-interface' + +declare const tapeSessionIdBrand: unique symbol + +export type TapeSessionId = string & { readonly [tapeSessionIdBrand]: 'TapeSessionId' } + +export const toTapeSessionId = (value: string): TapeSessionId => value as TapeSessionId + +export interface TapeEntryRef { + sessionId: TapeSessionId + entryId: number +} + +export interface TapeFactProvenance { + source: 'message' | 'tool_call' | 'tool_result' | 'runtime_event' + sourceId: string + sequence: number +} + +export interface TapeToolFactInput { + sessionId: TapeSessionId + messageId: string + orderSeq: number + blockIndex: number + block: AssistantMessageBlock + provenance: TapeFactProvenance +} + +export type TapeFactSource = 'live' | 'backfill' | 'repair' diff --git a/src/main/tape/domain/viewManifest.ts b/src/main/tape/domain/viewManifest.ts new file mode 100644 index 0000000000..8aacdac00e --- /dev/null +++ b/src/main/tape/domain/viewManifest.ts @@ -0,0 +1,363 @@ +import { createHash } from 'crypto' +import type { ChatMessage } from '@shared/types/core/chat-message' +import type { MCPToolDefinition } from '@shared/types/core/mcp' +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { + DeepChatTapeViewEntryRef, + DeepChatTapeViewExcludedRange, + DeepChatTapeViewExcludedRef, + DeepChatTapeViewManifest, + DeepChatTapeViewManifestIntegrity, + DeepChatTapeViewPolicy, + DeepChatTapeViewTaskType, + DeepChatTapeViewTokenBudget +} from '@shared/types/tape-view-manifest' +import { estimateMessagesTokens } from '@shared/utils/messageTokens' + +export type ContextSummaryCursorMetadata = { + summaryCursorOrderSeq: number + preCursorOrderSeqMin: number | null + preCursorOrderSeqMax: number | null + preCursorCount: number +} + +export function isCompactionRecord(record: ChatMessageRecord): boolean { + try { + const metadata = JSON.parse(record.metadata) as { messageType?: string } + return metadata.messageType === 'compaction' + } catch { + return false + } +} + +/** Stable event name persisted for deterministic view reconstruction. */ +export const TAPE_VIEW_MANIFEST_EVENT_NAME = 'view/assembled' +export const TAPE_VIEW_CONTEXT_BUILDER_VERSION = 'legacy-v1' as const + +export type TapeViewManifestSourceMaps = { + entryIdByMessageId?: Map + toolCallEntryIdByToolId?: Map + toolResultEntryIdByToolId?: Map +} + +export type TapeViewManifestBuildInput = { + sessionId: string + messageId: string + requestSeq: number + taskType: DeepChatTapeViewTaskType + policy: DeepChatTapeViewPolicy + policyVersion?: number | null + messages: ChatMessage[] + tools: MCPToolDefinition[] + latestEntryId: number + anchorEntryIds: number[] + reconstructionAnchorEntryId?: number | null + included: DeepChatTapeViewEntryRef[] + excluded: DeepChatTapeViewExcludedRef[] + summaryCursor?: ContextSummaryCursorMetadata + tokenBudget: Omit + providerId: string + modelId: string + summaryCursorOrderSeq: number + supportsVision: boolean + supportsAudioInput: boolean + traceDebugEnabled: boolean + assembledAt?: number +} + +export type TapeViewManifestPolicyInput = { + recoveredFromContextPressure: boolean + isInitialViewRequest: boolean + viewPolicy?: DeepChatTapeViewPolicy + viewPolicyVersion?: number | null +} + +export type TapeViewManifestPolicyResult = { + policy: DeepChatTapeViewPolicy + policyVersion: number | null +} + +export type TapeViewContextSelection = { + includedRecords: Array<{ + record: ChatMessageRecord + reason: DeepChatTapeViewEntryRef['reason'] + }> + excludedRecords: Array<{ + record: ChatMessageRecord + reason: DeepChatTapeViewExcludedRef['reason'] + }> + summaryCursor?: ContextSummaryCursorMetadata + includesSystemPrompt: boolean + newUserMessageId?: string | null +} + +export function resolveTapeViewManifestPolicy( + input: TapeViewManifestPolicyInput +): TapeViewManifestPolicyResult { + if (input.recoveredFromContextPressure) { + return { + policy: 'context_pressure_recovery_shadow', + policyVersion: null + } + } + + if (input.isInitialViewRequest && input.viewPolicy) { + return { + policy: input.viewPolicy, + policyVersion: input.viewPolicyVersion ?? null + } + } + + return { + policy: 'tool_loop_shadow', + policyVersion: null + } +} + +function normalizeForStableJson(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(normalizeForStableJson) + } + + if (!value || typeof value !== 'object') { + return value + } + + const record = value as Record + return Object.keys(record) + .sort() + .reduce>((result, key) => { + const nested = record[key] + if (nested !== undefined) { + result[key] = normalizeForStableJson(nested) + } + return result + }, {}) +} + +export function stableJsonStringify(value: unknown): string { + return JSON.stringify(normalizeForStableJson(value)) +} + +export function hashJson(value: unknown): string { + return createHash('sha256').update(stableJsonStringify(value)).digest('hex') +} + +export const TAPE_VIEW_MANIFEST_HASH_VERSION = 2 + +function buildManifestHash(manifest: DeepChatTapeViewManifest): string { + const hashable: Record = { ...manifest } + delete hashable.assembledAt + delete hashable.viewId + hashable.hashes = { + promptHash: manifest.hashes.promptHash, + toolDefinitionsHash: manifest.hashes.toolDefinitionsHash + } + return hashJson(hashable) +} + +function buildExcludedRanges( + summaryCursor?: ContextSummaryCursorMetadata +): DeepChatTapeViewExcludedRange[] { + if ( + !summaryCursor || + summaryCursor.preCursorCount === 0 || + summaryCursor.preCursorOrderSeqMin === null || + summaryCursor.preCursorOrderSeqMax === null + ) { + return [] + } + return [ + { + fromOrderSeq: summaryCursor.preCursorOrderSeqMin, + toOrderSeq: summaryCursor.preCursorOrderSeqMax, + count: summaryCursor.preCursorCount, + reason: 'before_summary_cursor' + } + ] +} + +export function verifyTapeViewManifestHash( + manifest: DeepChatTapeViewManifest +): DeepChatTapeViewManifestIntegrity { + if (manifest.hashVersion !== TAPE_VIEW_MANIFEST_HASH_VERSION) { + return 'unverified' + } + return buildManifestHash(manifest) === manifest.hashes.manifestHash ? 'valid' : 'invalid' +} + +export function createTapeViewManifest( + input: TapeViewManifestBuildInput +): DeepChatTapeViewManifest { + const assembledAt = input.assembledAt ?? Date.now() + const excludedRanges = buildExcludedRanges(input.summaryCursor) + const draft: DeepChatTapeViewManifest = { + schemaVersion: 2, + hashVersion: TAPE_VIEW_MANIFEST_HASH_VERSION, + viewId: '', + sessionId: input.sessionId, + messageId: input.messageId, + requestSeq: input.requestSeq, + taskType: input.taskType, + policy: input.policy, + policyVersion: input.policyVersion ?? null, + contextBuilderVersion: TAPE_VIEW_CONTEXT_BUILDER_VERSION, + latestEntryId: input.latestEntryId, + anchorEntryIds: [...input.anchorEntryIds], + ...(input.reconstructionAnchorEntryId !== undefined + ? { reconstructionAnchorEntryId: input.reconstructionAnchorEntryId } + : {}), + included: input.included.map((entry) => ({ ...entry })), + excluded: input.excluded.map((entry) => ({ ...entry })), + ...(excludedRanges.length > 0 ? { excludedRanges } : {}), + tokenBudget: { + ...input.tokenBudget, + estimatedPromptTokens: estimateMessagesTokens(input.messages) + }, + hashes: { + promptHash: hashJson(input.messages), + toolDefinitionsHash: hashJson(input.tools), + manifestHash: '' + }, + meta: { + providerId: input.providerId, + modelId: input.modelId, + summaryCursorOrderSeq: input.summaryCursorOrderSeq, + supportsVision: input.supportsVision, + supportsAudioInput: input.supportsAudioInput, + traceDebugEnabled: input.traceDebugEnabled + }, + assembledAt + } + + const manifestHash = buildManifestHash(draft) + return { + ...draft, + viewId: `view_${manifestHash.slice(0, 16)}`, + hashes: { ...draft.hashes, manifestHash } + } +} + +export function buildIncludedRefs( + selection: TapeViewContextSelection, + sourceMaps: TapeViewManifestSourceMaps = {} +): DeepChatTapeViewEntryRef[] { + const refs: DeepChatTapeViewEntryRef[] = [] + + if (selection.includesSystemPrompt) { + refs.push({ + entryId: null, + messageId: null, + orderSeq: null, + role: 'system', + source: 'synthetic', + reason: 'system_prompt' + }) + } + + for (const item of selection.includedRecords) { + refs.push({ + entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, + messageId: item.record.id, + orderSeq: item.record.orderSeq, + role: item.record.role, + source: sourceMaps.entryIdByMessageId?.has(item.record.id) ? 'tape' : 'synthetic', + reason: item.reason + }) + } + + if (selection.newUserMessageId) { + refs.push({ + entryId: sourceMaps.entryIdByMessageId?.get(selection.newUserMessageId) ?? null, + messageId: selection.newUserMessageId, + orderSeq: null, + role: 'user', + source: sourceMaps.entryIdByMessageId?.has(selection.newUserMessageId) ? 'tape' : 'synthetic', + reason: 'new_user_input' + }) + } + + return refs +} + +export function buildExcludedRefs( + selection: TapeViewContextSelection, + sourceMaps: TapeViewManifestSourceMaps = {} +): DeepChatTapeViewExcludedRef[] { + return selection.excludedRecords.map((item) => ({ + entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, + messageId: item.record.id, + orderSeq: item.record.orderSeq, + reason: item.reason + })) +} + +export function buildRequestRefs( + messages: ChatMessage[], + sourceMaps: TapeViewManifestSourceMaps = {} +): DeepChatTapeViewEntryRef[] { + const lastToolCallIndex = new Map() + const lastToolResultIndex = new Map() + messages.forEach((message, index) => { + if (message.role === 'assistant' && message.tool_calls?.length) { + for (const toolCall of message.tool_calls) { + lastToolCallIndex.set(toolCall.id, index) + } + } else if (message.role === 'tool' && message.tool_call_id) { + lastToolResultIndex.set(message.tool_call_id, index) + } + }) + + const refs: DeepChatTapeViewEntryRef[] = [] + messages.forEach((message, index) => { + if (message.role === 'assistant' && message.tool_calls?.length) { + for (const toolCall of message.tool_calls) { + const entryId = + lastToolCallIndex.get(toolCall.id) === index + ? (sourceMaps.toolCallEntryIdByToolId?.get(toolCall.id) ?? null) + : null + refs.push({ + entryId, + messageId: null, + orderSeq: null, + role: 'assistant', + source: entryId === null ? 'synthetic' : 'tape', + reason: 'tool_loop_message' + }) + } + return + } + + if (message.role === 'tool' && message.tool_call_id) { + const entryId = + lastToolResultIndex.get(message.tool_call_id) === index + ? (sourceMaps.toolResultEntryIdByToolId?.get(message.tool_call_id) ?? null) + : null + refs.push({ + entryId, + messageId: null, + orderSeq: null, + role: 'tool', + source: entryId === null ? 'synthetic' : 'tape', + reason: 'tool_loop_message' + }) + return + } + + refs.push({ + entryId: null, + messageId: null, + orderSeq: null, + role: message.role, + source: 'synthetic', + reason: + message.role === 'system' + ? 'system_prompt' + : message.role === 'tool' + ? 'tool_loop_message' + : 'selected_history' + }) + }) + + return refs +} diff --git a/src/main/tape/ports/capabilities.ts b/src/main/tape/ports/capabilities.ts new file mode 100644 index 0000000000..8851136905 --- /dev/null +++ b/src/main/tape/ports/capabilities.ts @@ -0,0 +1,43 @@ +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryRow, TapeAnchorAppendInput } from '../domain/entry' +import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' + +export interface TapeToolFactWriter { + appendToolFact(input: TapeToolFactInput): Promise +} + +export interface TapeMessageFactWriter { + appendMessageRecord(record: ChatMessageRecord): number + appendMessageReplacement(record: ChatMessageRecord, reason: string): number + appendMessageRetraction(record: ChatMessageRecord, reason: string): number +} + +export interface TapeRawEntryReader { + getBySession(sessionId: string): DeepChatTapeEntryRow[] + getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] + getMaxEntryId(sessionId: string): number +} + +export interface TapeAnchorReader { + getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined + getAnchors(sessionId: string, limit?: number): DeepChatTapeEntryRow[] + getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined + getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined +} + +export interface TapeAnchorWriter { + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow +} + +export interface TapeInspectionReader { + getBySession(sessionId: string): DeepChatTapeEntryRow[] + listMemoryViewManifestAnchorsByAgent( + agentId: string, + options?: { sessionId?: string; limit?: number; messageId?: string } + ): DeepChatTapeEntryRow[] +} + +export interface TapeLifecycleAdmin { + deleteSessionTape(sessionId: string): void + resetSessionTape(sessionId: string): void +} diff --git a/src/main/tape/ports/storage.ts b/src/main/tape/ports/storage.ts new file mode 100644 index 0000000000..ee1b274fbb --- /dev/null +++ b/src/main/tape/ports/storage.ts @@ -0,0 +1,72 @@ +import type { + DeepChatTapeAppendInput, + DeepChatTapeEntryRow, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + TapeAnchorAppendInput, + TapeEventAppendInput +} from '../domain/entry' + +export interface TapeMutationProjection { + applyAppendedEntry(row: DeepChatTapeEntryRow, previousSessionMaxEntryId: number): boolean + invalidateSession(sessionId: string): void + deleteBySession(sessionId: string): void +} + +/** Append/read/query persistence only. Physical deletion belongs to TapeEntryLifecycleStore. */ +export interface TapeEntryStore { + append(input: DeepChatTapeAppendInput): DeepChatTapeEntryRow + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow + appendEvent(input: TapeEventAppendInput): DeepChatTapeEntryRow + getBySession(sessionId: string): DeepChatTapeEntryRow[] + getSubagentLineageEvents(sessionId: string): DeepChatTapeEntryRow[] + getFirstEntriesBySessions(sessionIds: string[]): DeepChatTapeEntryRow[] + getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] + listMemoryViewManifestAnchorsBySessions( + sessionIds: string[], + optionsOrLimit?: number | { limit?: number; messageId?: string } + ): DeepChatTapeEntryRow[] + listMemoryViewManifestAnchorsByAgent( + agentId: string, + options?: { sessionId?: string; limit?: number; messageId?: string } + ): DeepChatTapeEntryRow[] + getEntriesAfter(sessionId: string, entryId: number): DeepChatTapeEntryRow[] + getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined + getAnchors(sessionId: string, limit?: number): DeepChatTapeEntryRow[] + getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined + getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined + getByProvenanceKey(sessionId: string, provenanceKey: string): DeepChatTapeEntryRow | undefined + getMaxEntryId(sessionId: string): number + getMaxEntryIdsBySessions(sessionIds: string[]): Map + countAnchorsBySession(sessionId: string): number + countEntriesAfter(sessionId: string, entryId: number): number + countBySession(sessionId: string): number + search( + sessionId: string, + query: string, + options?: DeepChatTapeSearchInput + ): DeepChatTapeEntryRow[] + searchEffectiveSourcesAtHeads( + sources: readonly DeepChatTapeReadSource[], + query: string, + options?: DeepChatTapeSearchInput + ): DeepChatTapeEntryRow[] + getEffectiveContextRowsAtHead( + source: DeepChatTapeReadSource, + entryIds: number[], + options: { before: number; after: number; limit: number } + ): DeepChatTapeEntryRow[] +} + +export interface TapeTransactionRunner { + runInTransaction(operation: () => T): T +} + +/** Transitional bootstrap capability until bootstrap orchestration lives in application services. */ +export interface TapeBootstrapStore { + ensureBootstrapAnchor(sessionId: string): void +} + +export interface TapeEntryLifecycleStore { + deleteBySession(sessionId: string): void +} diff --git a/test/main/agent/deepchat/runtime/process.test.ts b/test/main/agent/deepchat/runtime/process.test.ts index 6c1b7a5c90..db31730680 100644 --- a/test/main/agent/deepchat/runtime/process.test.ts +++ b/test/main/agent/deepchat/runtime/process.test.ts @@ -241,7 +241,7 @@ describe('processStream', () => { permissionMode: 'full_access', io: { messageStore, - tapeRecorder, + tapeToolFactWriter: tapeRecorder, publishEvent: publishDeepchatEventMock, publishSessionUpdate: vi.fn() }, @@ -487,7 +487,7 @@ describe('processStream', () => { ]) }) - it('keeps the tool loop fail-open when TapeRecorder rejects a fact', async () => { + it('keeps the tool loop fail-open when TapeToolFactWriter rejects a fact', async () => { tapeRecorder.appendToolFact.mockRejectedValue(new Error('tape unavailable')) const coreStream = createToolThenCompleteStream('action') diff --git a/test/main/evals/nativeAgent/harness.ts b/test/main/evals/nativeAgent/harness.ts index 7b8ce7fc05..3656bd979c 100644 --- a/test/main/evals/nativeAgent/harness.ts +++ b/test/main/evals/nativeAgent/harness.ts @@ -499,7 +499,7 @@ export async function runNativeAgentEvalScenario( messageStore, publishEvent: vi.fn(), publishSessionUpdate: vi.fn(), - tapeRecorder: { + tapeToolFactWriter: { appendToolFact: async () => ({ sessionId, entryId: 1 }) } } From 2ee6d2164b1336f8a98277ed59365e0c00fe2420 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 16:03:47 +0800 Subject: [PATCH 04/29] refactor(tape): split application services --- docs/architecture/tape-layering/tasks.md | 8 +- .../tables/deepchatTapeSearchProjection.ts | 64 +- src/main/session/data/tape.ts | 2633 +---------------- src/main/session/data/tapeFacts.ts | 353 +-- src/main/tape/application/common.ts | 64 + src/main/tape/application/contracts.ts | 53 + src/main/tape/application/factPersistence.ts | 352 +++ src/main/tape/application/factService.ts | 44 + src/main/tape/application/forkService.ts | 444 +++ src/main/tape/application/lineageService.ts | 438 +++ src/main/tape/application/recallProjection.ts | 466 +++ src/main/tape/application/recallService.ts | 561 ++++ .../tape/application/reconcilerService.ts | 127 + src/main/tape/application/sessionTape.ts | 201 ++ .../tape/application/viewReplayService.ts | 621 ++++ src/main/tape/ports/application.ts | 173 ++ 16 files changed, 3567 insertions(+), 3035 deletions(-) create mode 100644 src/main/tape/application/common.ts create mode 100644 src/main/tape/application/contracts.ts create mode 100644 src/main/tape/application/factPersistence.ts create mode 100644 src/main/tape/application/factService.ts create mode 100644 src/main/tape/application/forkService.ts create mode 100644 src/main/tape/application/lineageService.ts create mode 100644 src/main/tape/application/recallProjection.ts create mode 100644 src/main/tape/application/recallService.ts create mode 100644 src/main/tape/application/reconcilerService.ts create mode 100644 src/main/tape/application/sessionTape.ts create mode 100644 src/main/tape/application/viewReplayService.ts create mode 100644 src/main/tape/ports/application.ts diff --git a/docs/architecture/tape-layering/tasks.md b/docs/architecture/tape-layering/tasks.md index 374fb4ff69..9c9f2acc56 100644 --- a/docs/architecture/tape-layering/tasks.md +++ b/docs/architecture/tape-layering/tasks.md @@ -26,10 +26,10 @@ ## Application Services -- [ ] Extract fact, reconciliation, recall, lineage, view/replay, and fork services. -- [ ] Convert `SessionTape` into a compatibility facade. -- [ ] Preserve `SessionTapePort` and current reconciliation timing. -- [ ] Review and commit the application-service slice. +- [x] Extract fact, reconciliation, recall, lineage, view/replay, and fork services. +- [x] Convert `SessionTape` into a compatibility facade. +- [x] Preserve `SessionTapePort` and current reconciliation timing. +- [x] Review and commit the application-service slice. ## Infrastructure and Bypass Closure diff --git a/src/main/session/data/tables/deepchatTapeSearchProjection.ts b/src/main/session/data/tables/deepchatTapeSearchProjection.ts index 70910cafcb..75ca28ce79 100644 --- a/src/main/session/data/tables/deepchatTapeSearchProjection.ts +++ b/src/main/session/data/tables/deepchatTapeSearchProjection.ts @@ -8,56 +8,23 @@ import { normalizeDeepChatTapeReadSources, serializeDeepChatTapeReadSources } from '@/tape/domain/entry' +import type { DeepChatTapeReadSource, DeepChatTapeSearchInput } from '@/tape/domain/entry' import type { - DeepChatTapeEntryKind, - DeepChatTapeReadSource, - DeepChatTapeSearchInput, - DeepChatTapeSourceType -} from '@/tape/domain/entry' + TapeSearchProjectionInput, + TapeSearchProjectionMeta, + TapeSearchProjectionReadResult, + TapeSearchProjectionResultRow, + TapeSearchProjectionRow, + TapeSearchProjectionStore +} from '@/tape/ports/application' export const DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION = 2 -export interface DeepChatTapeSearchProjectionInput { - sessionId: string - entryId: number - kind: DeepChatTapeEntryKind - name: string | null - sourceType: DeepChatTapeSourceType | null - sourceId: string | null - sourceSeq: number | null - searchText: string - summaryText: string - refs: Record - createdAt: number -} - -export interface DeepChatTapeSearchProjectionRow { - session_id: string - entry_id: number - kind: DeepChatTapeEntryKind - name: string | null - source_type: DeepChatTapeSourceType | null - source_id: string | null - source_seq: number | null - search_text: string - summary_text: string - refs_json: string - created_at: number -} - -export interface DeepChatTapeSearchProjectionResultRow extends DeepChatTapeSearchProjectionRow { - score: number | null -} - -export interface DeepChatTapeSearchProjectionMeta { - projectionVersion: number - maxEntryId: number -} - -export interface DeepChatTapeSearchProjectionReadResult { - rows: DeepChatTapeSearchProjectionResultRow[] - coveredSources: DeepChatTapeReadSource[] -} +export type DeepChatTapeSearchProjectionInput = TapeSearchProjectionInput +export type DeepChatTapeSearchProjectionRow = TapeSearchProjectionRow +export type DeepChatTapeSearchProjectionResultRow = TapeSearchProjectionResultRow +export type DeepChatTapeSearchProjectionMeta = TapeSearchProjectionMeta +export type DeepChatTapeSearchProjectionReadResult = TapeSearchProjectionReadResult type FtsCapability = { available: boolean; tokenizer: 'trigram' | 'unicode61' } @@ -87,7 +54,10 @@ function parseRefs(value: string): Record { } } -export class DeepChatTapeSearchProjectionTable extends BaseTable { +export class DeepChatTapeSearchProjectionTable + extends BaseTable + implements TapeSearchProjectionStore +{ private ftsCapability: FtsCapability | undefined private ftsReady = false diff --git a/src/main/session/data/tape.ts b/src/main/session/data/tape.ts index 7266e7c72e..38e1db43db 100644 --- a/src/main/session/data/tape.ts +++ b/src/main/session/data/tape.ts @@ -1,2632 +1 @@ -import { SessionDatabase } from './database' -import { nanoid } from 'nanoid' -import { createHash } from 'crypto' -import logger from 'electron-log' -import type { - AgentTapeAnchorResult, - AgentTapeAnchorsOptions, - AgentTapeContextEntry, - AgentTapeContextOptions, - AgentTapeContextResult, - AgentTapeHandoffState, - AgentTapeSearchOptions, - AgentTapeViewScope, - ChatMessageRecord, - SubagentTapeLinkInput, - SubagentTapeLinkOutcome, - SubagentTapeLinkReceipt -} from '@shared/types/agent-interface' -import type { - DeepChatTapeViewExcludedRange, - DeepChatTapeViewManifest, - DeepChatTapeViewManifestRecord -} from '@shared/types/tape-view-manifest' -import type { - DeepChatCausalObservationReadOptions, - DeepChatCausalObservationRequest, - DeepChatCausalObservationSlice, - DeepChatTapeReplayEntrySnapshot, - DeepChatTapeReplayExportOptions, - DeepChatTapeReplaySlice, - DeepChatTapeReplayTraceSnapshot -} from '@shared/types/tape-replay' -import type { SessionTranscript } from '@/session/data/transcript' -import { - SUMMARY_ANCHOR_NAMES, - TAPE_INCARNATION_META_KEY, - type DeepChatTapeReadSource, - type DeepChatTapeEntryRow, - type DeepChatTapeSearchInput -} from '@/tape/domain/entry' -import type { - DeepChatTapeSearchProjectionInput, - DeepChatTapeSearchProjectionResultRow, - DeepChatTapeSearchProjectionRow -} from '@/session/data/tables/deepchatTapeSearchProjection' -import type { DeepChatMessageTraceRow } from '@/session/data/tables/deepchatMessageTraces' -import { appendMessageRecordToTape, appendTapeToolFact } from '@/session/data/tapeFacts' -import type { TapeEntryRef, TapeToolFactInput } from '@/tape/domain/facts' -import type { TapeToolFactWriter } from '@/tape/ports/capabilities' -import { - buildEffectiveTapeView, - getLastEffectiveTokenUsage, - searchEffectiveTapeRows -} from '@/tape/domain/effectiveView' -import { - hashJson, - TAPE_VIEW_MANIFEST_EVENT_NAME, - verifyTapeViewManifestHash -} from '@/tape/domain/viewManifest' - -export type TapeMigrationState = 'none' | 'ready' - -export type TapeBackfillResult = { - sessionId: string - migrationState: TapeMigrationState - messageCount: number - maxOrderSeq: number - appendedFactCount: number - historyRecords: ChatMessageRecord[] -} - -export type TapeInfo = { - sessionId: string - entries: number - anchors: number - lastAnchor: string | null - lastAnchorEntryId: number | null - entriesSinceLastAnchor: number - lastTokenUsage: number | null - migrationState: TapeMigrationState -} - -export type TapeSearchResult = { - sessionId: string - entryId: number - kind: string - name: string | null - createdAt: number - summary?: string - refs?: Record - score?: number -} - -export type TapeAnchorResult = AgentTapeAnchorResult - -export type TapeForkHandle = { - parentSessionId: string - forkId: string - forkSessionId: string - parentHeadEntryId: number -} - -export type TapeViewManifestSourceMaps = { - latestEntryId: number - anchorEntryIds: number[] - reconstructionAnchorEntryIds: number[] - reconstructionAnchorEntryId: number | null - entryIdByMessageId: Map - toolCallEntryIdByToolId: Map - toolResultEntryIdByToolId: Map -} - -const BOOTSTRAP_ANCHOR_NAME = 'session/start' -const DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY = 2048 -const DEFAULT_CONTEXT_MAX_TOTAL_BYTES = 16384 -const MAX_CONTEXT_MAX_BYTES_PER_ENTRY = 8192 -const MAX_CONTEXT_MAX_TOTAL_BYTES = 65536 - -// Mirrors getLatestReconstructionAnchor: the anchor set that owns the summary -// cursor and prompt-visible reconstruction state (summaries, handoffs). -function isReconstructionAnchorName(name: string | null): boolean { - if (name === null) { - return false - } - return ( - (SUMMARY_ANCHOR_NAMES as readonly string[]).includes(name) || - name.startsWith('handoff/') || - name.startsWith('auto_handoff/') - ) -} - -function parseJsonObject(raw: string): Record { - try { - const parsed = JSON.parse(raw) as unknown - if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { - return parsed as Record - } - } catch {} - return {} -} - -function parseJsonValue(raw: string): unknown { - try { - return JSON.parse(raw) as unknown - } catch { - return null - } -} - -const SUBAGENT_TAPE_LINK_EVENT_NAME = 'subagent/tape_linked' -const SUBAGENT_TAPE_LINK_VERSION = 2 -const TAPE_IDENTITY_PATTERN = /^[a-f0-9]{64}$/ -const SUBAGENT_TAPE_LINK_OUTCOMES = new Set([ - 'completed', - 'error', - 'cancelled' -]) - -type SubagentTapeLinkSnapshot = { - linkEntryId: number - childSessionId: string - childHeadEntryId: number - childEntryCount: number - outcome: SubagentTapeLinkOutcome - childTapeIdentity: string | null -} - -type ParsedSubagentTapeLink = { - snapshot: SubagentTapeLinkSnapshot - frozenInput: SubagentTapeLinkInput -} - -type LinkedTapeSourceResolution = { - sources: DeepChatTapeReadSource[] - unavailableSourceIds: Set -} - -export type AgentTapeViewErrorCode = - | 'current_tape_unavailable' - | 'linked_tape_unavailable' - | 'linked_tape_unauthorized' - -export class AgentTapeViewError extends Error { - readonly name = 'AgentTapeViewError' - - constructor( - readonly code: AgentTapeViewErrorCode, - readonly parentSessionId: string, - readonly sourceSessionId: string, - message: string - ) { - super(message) - } -} - -export function normalizeSubagentTapeLinkInput( - input: SubagentTapeLinkInput -): SubagentTapeLinkInput { - const normalized = { - parentSessionId: input.parentSessionId.trim(), - childSessionId: input.childSessionId.trim(), - runId: input.runId.trim(), - taskId: input.taskId.trim(), - slotId: input.slotId.trim(), - taskTitle: compactText(input.taskTitle, 500), - outcome: input.outcome, - resultSummary: input.resultSummary?.trim() ? compactText(input.resultSummary, 2000) : null - } - for (const [name, value] of Object.entries(normalized)) { - if (name === 'resultSummary' || name === 'outcome') continue - if (typeof value !== 'string' || !value) { - throw new Error(`Subagent Tape link ${name} is required.`) - } - } - if (normalized.parentSessionId === normalized.childSessionId) { - throw new Error('Subagent Tape link child must differ from its parent.') - } - if (!SUBAGENT_TAPE_LINK_OUTCOMES.has(normalized.outcome)) { - throw new Error(`Invalid subagent Tape link outcome: ${String(normalized.outcome)}`) - } - return normalized -} - -function isUnmarkedLegacyTape(row: DeepChatTapeEntryRow): boolean { - const meta = parseJsonValue(row.meta_json) - return ( - meta !== null && - typeof meta === 'object' && - !Array.isArray(meta) && - !Object.prototype.hasOwnProperty.call(meta, TAPE_INCARNATION_META_KEY) - ) -} - -function computeTapeIdentity(row: DeepChatTapeEntryRow): string { - return createHash('sha256') - .update( - JSON.stringify([ - row.session_id, - row.entry_id, - row.kind, - row.name, - row.source_type, - row.source_id, - row.source_seq, - row.provenance_key, - row.payload_json, - row.meta_json, - row.created_at - ]) - ) - .digest('hex') -} - -function subagentTapeLinkProvenanceKey(input: SubagentTapeLinkInput): string { - // This version belongs to the stable task-identity key, independently of the evolving event - // payload's linkVersion. - const identityHash = createHash('sha256') - .update( - JSON.stringify([input.parentSessionId, input.childSessionId, input.runId, input.taskId]) - ) - .digest('hex') - return `subagent:tape-link:v1:${identityHash}` -} - -function parseSubagentTapeLink(row: DeepChatTapeEntryRow): ParsedSubagentTapeLink | null { - const payload = parseJsonObject(row.payload_json) - const data = - payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) - ? (payload.data as Record) - : {} - const childSessionId = data.childSessionId - const childHeadEntryId = data.childHeadEntryId - const childEntryCount = data.childEntryCount - const outcome = data.outcome - const runId = data.runId - const taskId = data.taskId - const slotId = data.slotId - const taskTitle = data.taskTitle - const resultSummary = data.resultSummary - const linkVersion = data.linkVersion - const childTapeIdentity = data.childTapeIdentity - const hasValidLinkVersion = - (linkVersion === 1 && childTapeIdentity === undefined) || - (linkVersion === SUBAGENT_TAPE_LINK_VERSION && - typeof childTapeIdentity === 'string' && - TAPE_IDENTITY_PATTERN.test(childTapeIdentity)) - if ( - row.kind !== 'event' || - row.name !== SUBAGENT_TAPE_LINK_EVENT_NAME || - typeof childSessionId !== 'string' || - !childSessionId || - typeof childHeadEntryId !== 'number' || - !Number.isSafeInteger(childHeadEntryId) || - childHeadEntryId < 0 || - typeof childEntryCount !== 'number' || - !Number.isSafeInteger(childEntryCount) || - childEntryCount < 0 || - typeof outcome !== 'string' || - !SUBAGENT_TAPE_LINK_OUTCOMES.has(outcome as SubagentTapeLinkOutcome) || - typeof runId !== 'string' || - typeof taskId !== 'string' || - typeof slotId !== 'string' || - typeof taskTitle !== 'string' || - (resultSummary !== null && typeof resultSummary !== 'string') || - !hasValidLinkVersion || - row.source_type !== 'subagent' || - row.source_id !== childSessionId || - row.source_seq !== childHeadEntryId || - childEntryCount > childHeadEntryId - ) { - return null - } - - let normalizedInput: SubagentTapeLinkInput - try { - normalizedInput = normalizeSubagentTapeLinkInput({ - parentSessionId: row.session_id, - childSessionId, - runId, - taskId, - slotId, - taskTitle, - outcome: outcome as SubagentTapeLinkOutcome, - resultSummary - }) - } catch { - return null - } - if ( - normalizedInput.parentSessionId !== row.session_id || - normalizedInput.childSessionId !== childSessionId || - normalizedInput.runId !== runId || - normalizedInput.taskId !== taskId || - normalizedInput.slotId !== slotId || - normalizedInput.taskTitle !== taskTitle || - normalizedInput.resultSummary !== resultSummary || - row.provenance_key !== subagentTapeLinkProvenanceKey(normalizedInput) - ) { - return null - } - - return { - snapshot: { - linkEntryId: row.entry_id, - childSessionId, - childHeadEntryId, - childEntryCount, - outcome: outcome as SubagentTapeLinkOutcome, - childTapeIdentity: - linkVersion === SUBAGENT_TAPE_LINK_VERSION ? (childTapeIdentity as string) : null - }, - frozenInput: normalizedInput - } -} - -function parseSubagentTapeLinkSnapshot(row: DeepChatTapeEntryRow): SubagentTapeLinkSnapshot | null { - return parseSubagentTapeLink(row)?.snapshot ?? null -} - -function parseLegacyExternalTapeLinkSnapshot( - row: DeepChatTapeEntryRow -): SubagentTapeLinkSnapshot | null { - if (row.kind !== 'event' || row.name !== 'fork/merge' || row.source_type !== 'fork') { - return null - } - const payload = parseJsonObject(row.payload_json) - const data = - payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) - ? (payload.data as Record) - : {} - const childSessionId = data.forkSessionId - const forkId = data.forkId - const referencedEntryCount = data.referencedEntryCount - if ( - typeof childSessionId !== 'string' || - !childSessionId || - forkId !== childSessionId || - row.source_id !== childSessionId || - row.source_seq !== 0 || - row.provenance_key !== `fork:${row.session_id}:${childSessionId}:external-merge:event` || - typeof referencedEntryCount !== 'number' || - !Number.isSafeInteger(referencedEntryCount) || - referencedEntryCount <= 0 - ) { - return null - } - return { - linkEntryId: row.entry_id, - childSessionId, - childHeadEntryId: referencedEntryCount, - childEntryCount: referencedEntryCount, - outcome: 'completed', - childTapeIdentity: null - } -} - -function readForkMergeReceiptCount( - row: DeepChatTapeEntryRow, - parentSessionId: string, - forkId: string, - forkSessionIdValue: string -): number { - const payload = parseJsonObject(row.payload_json) - const data = - payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) - ? (payload.data as Record) - : {} - const mergedCount = data.mergedCount - const forkHeadEntryId = data.forkHeadEntryId - const hasValidLegacyOrCurrentHead = - forkHeadEntryId === undefined || - (typeof forkHeadEntryId === 'number' && - Number.isSafeInteger(forkHeadEntryId) && - forkHeadEntryId >= 0) - if ( - row.session_id !== parentSessionId || - row.kind !== 'event' || - row.name !== 'fork/merge' || - row.source_type !== 'fork' || - row.source_id !== forkId || - row.source_seq !== 0 || - row.provenance_key !== `fork:${parentSessionId}:${forkId}:merge:event` || - data.forkId !== forkId || - data.forkSessionId !== forkSessionIdValue || - typeof mergedCount !== 'number' || - !Number.isSafeInteger(mergedCount) || - mergedCount < 0 || - !hasValidLegacyOrCurrentHead || - (typeof forkHeadEntryId === 'number' && mergedCount > forkHeadEntryId) - ) { - throw new Error(`Stored fork merge receipt is malformed: ${row.entry_id}`) - } - return mergedCount -} - -function assertValidForkStart( - row: DeepChatTapeEntryRow | undefined, - parentSessionId: string, - forkId: string, - forkSessionIdValue: string -): void { - if (!row) { - throw new Error(`Fork ${forkId} does not exist or has been discarded.`) - } - const payload = parseJsonObject(row.payload_json) - const state = - payload.state && typeof payload.state === 'object' && !Array.isArray(payload.state) - ? (payload.state as Record) - : {} - const parentHeadEntryId = state.parentHeadEntryId - const hasValidLegacyOrCurrentHead = - parentHeadEntryId === undefined || - (typeof parentHeadEntryId === 'number' && - Number.isSafeInteger(parentHeadEntryId) && - parentHeadEntryId >= 0) - if ( - row.session_id !== forkSessionIdValue || - row.kind !== 'anchor' || - row.name !== 'fork/start' || - row.source_type !== 'fork' || - row.source_id !== forkId || - row.source_seq !== 0 || - row.provenance_key !== `fork:${parentSessionId}:${forkId}:start` || - state.parentSessionId !== parentSessionId || - !hasValidLegacyOrCurrentHead - ) { - throw new Error(`Stored fork start is malformed: ${row.entry_id}`) - } -} - -function toSubagentTapeLinkReceipt(row: DeepChatTapeEntryRow): SubagentTapeLinkReceipt { - const snapshot = parseSubagentTapeLinkSnapshot(row) - if (!snapshot) { - throw new Error(`Stored subagent Tape link receipt is malformed: ${row.entry_id}`) - } - return { - linkEntry: { - sessionId: row.session_id, - entryId: snapshot.linkEntryId - }, - childSessionId: snapshot.childSessionId, - childHeadEntryId: snapshot.childHeadEntryId, - childEntryCount: snapshot.childEntryCount, - outcome: snapshot.outcome - } -} - -function assertSubagentTapeLinkMatchesInput( - row: DeepChatTapeEntryRow, - input: SubagentTapeLinkInput -): void { - const parsed = parseSubagentTapeLink(row) - if (!parsed) { - throw new Error(`Stored subagent Tape link receipt is malformed: ${row.entry_id}`) - } - const storedInput = parsed.frozenInput - const storedKeys = Object.keys(storedInput) as Array - if ( - storedKeys.length !== Object.keys(input).length || - storedKeys.some((key) => storedInput[key] !== input[key]) - ) { - throw new Error( - `Subagent Tape link conflicts with finalized task ${input.runId}/${input.taskId}.` - ) - } -} - -function compactText(value: string, maxLength = 1000): string { - const normalized = value.replace(/\s+/g, ' ').trim() - if (normalized.length <= maxLength) return normalized - return `${normalized.slice(0, Math.max(0, maxLength - 3)).trimEnd()}...` -} - -function stringifyForSummary(value: unknown, maxLength = 1000): string { - if (typeof value === 'string') return compactText(value, maxLength) - if (value === null || value === undefined) return '' - try { - return compactText(JSON.stringify(value), maxLength) - } catch { - return compactText(String(value), maxLength) - } -} - -function truncateToUtf8Bytes(text: string, maxBytes: number): { text: string; truncated: boolean } { - const normalized = text.trim() - if (maxBytes <= 0) { - return { text: '', truncated: normalized.length > 0 } - } - if (maxBytes < 3) { - return { text: '', truncated: normalized.length > 0 } - } - if (Buffer.byteLength(normalized, 'utf8') <= maxBytes) { - return { text: normalized, truncated: false } - } - let bytes = 0 - let output = '' - for (const character of normalized) { - const nextBytes = Buffer.byteLength(character, 'utf8') - if (bytes + nextBytes > Math.max(0, maxBytes - 3)) break - output += character - bytes += nextBytes - } - return { text: `${output.trimEnd()}...`, truncated: true } -} - -function normalizeContextByteLimit( - value: number | undefined, - fallback: number, - max: number -): number { - if (!Number.isFinite(value)) return fallback - return Math.min(Math.max(Math.floor(value as number), 0), max) -} - -function uniqueStrings(values: string[], limit = 10): string[] { - const seen = new Set() - const result: string[] = [] - for (const value of values) { - const normalized = value.trim() - if (!normalized || seen.has(normalized)) continue - seen.add(normalized) - result.push(normalized) - if (result.length >= limit) break - } - return result -} - -function extractFilePaths(text: string): string[] { - const matches = [ - ...text.matchAll( - /(?:^|[\s"'`([{<])((?:[A-Za-z]:\\|\/|\.{1,2}\/|[\w.-]+\/)[^\s"'`<>{}(),;!?]+(?:[/\\][^\s"'`<>{}(),;!?]+)*)/g - ) - ].map((match) => match[1].replace(/[.:]+$/g, '')) - return uniqueStrings(matches ?? []) -} - -function extractErrorCodes(text: string): string[] { - const matches = text.match(/\b(?:E[A-Z0-9_]{3,}|[A-Z][A-Z0-9_]*Error)\b/g) - return uniqueStrings(matches ?? []) -} - -function collectKeyedStrings( - value: unknown, - keys: Set, - output: string[] = [], - depth = 0 -): string[] { - if (depth > 4 || output.length >= 10 || !value || typeof value !== 'object') return output - if (Array.isArray(value)) { - for (const item of value) collectKeyedStrings(item, keys, output, depth + 1) - return output - } - for (const [key, nested] of Object.entries(value as Record)) { - if (keys.has(key) && typeof nested === 'string' && nested.trim()) { - output.push(compactText(nested, 500)) - if (output.length >= 10) return output - } - collectKeyedStrings(nested, keys, output, depth + 1) - if (output.length >= 10) return output - } - return output -} - -function collectUserMessageAttachmentRefs(files: unknown): { - searchText: string[] - filePaths: string[] - fileNames: string[] -} { - const searchText: string[] = [] - const filePaths: string[] = [] - const fileNames: string[] = [] - if (!Array.isArray(files)) { - return { searchText, filePaths, fileNames } - } - for (const file of files) { - if (!isRecordObject(file)) continue - const path = typeof file.path === 'string' ? file.path : '' - const name = typeof file.name === 'string' ? file.name : '' - const metadata = isRecordObject(file.metadata) ? file.metadata : null - const metadataFileName = - metadata && typeof metadata.fileName === 'string' ? metadata.fileName : '' - if (path) { - filePaths.push(compactText(path, 500)) - searchText.push(compactText(path, 500)) - } - for (const value of [name, metadataFileName]) { - if (!value) continue - fileNames.push(compactText(value, 500)) - searchText.push(compactText(value, 500)) - } - } - return { - searchText: uniqueStrings(searchText, 20), - filePaths: uniqueStrings(filePaths, 20), - fileNames: uniqueStrings(fileNames, 20) - } -} - -type UserMessageProjectionText = { - text: string - attachmentRefs: { - searchText: string[] - filePaths: string[] - fileNames: string[] - } -} - -function emptyUserMessageAttachmentRefs(): UserMessageProjectionText['attachmentRefs'] { - return { searchText: [], filePaths: [], fileNames: [] } -} - -function parseUserMessageProjectionText(content: string): UserMessageProjectionText { - const parsed = parseJsonValue(content) - if (isRecordObject(parsed) && typeof parsed.text === 'string') { - const attachmentRefs = collectUserMessageAttachmentRefs(parsed.files) - const parts = [parsed.text] - if (Array.isArray(parsed.files) && parsed.files.length > 0) { - parts.push(`files:${parsed.files.length}`) - parts.push(...attachmentRefs.searchText) - } - if (Array.isArray(parsed.links) && parsed.links.length > 0) { - parts.push(`links:${parsed.links.length}`) - } - return { text: parts.join(' '), attachmentRefs } - } - return { text: content, attachmentRefs: emptyUserMessageAttachmentRefs() } -} - -function getUserMessageProjectionText( - row: DeepChatTapeEntryRow, - payload: Record -): UserMessageProjectionText | null { - if (row.kind !== 'message' || !isRecordObject(payload.record)) return null - const record = payload.record - const role = typeof record.role === 'string' ? record.role : 'message' - if (role === 'assistant') return null - const content = typeof record.content === 'string' ? record.content : '' - return parseUserMessageProjectionText(content) -} - -function collectUserMessageAttachmentRefsFromPayload(payload: Record): { - searchText: string[] - filePaths: string[] - fileNames: string[] -} { - if (!isRecordObject(payload.record)) { - return { searchText: [], filePaths: [], fileNames: [] } - } - const content = typeof payload.record.content === 'string' ? payload.record.content : '' - const parsed = parseJsonValue(content) - return isRecordObject(parsed) - ? collectUserMessageAttachmentRefs(parsed.files) - : { searchText: [], filePaths: [], fileNames: [] } -} - -function readUserMessageText(content: string, parsed?: UserMessageProjectionText): string { - return parsed?.text ?? parseUserMessageProjectionText(content).text -} - -function readAssistantMessageText(content: string): string { - const parsed = parseJsonValue(content) - if (!Array.isArray(parsed)) return content - const parts: string[] = [] - for (const block of parsed) { - if (!isRecordObject(block)) continue - if (typeof block.content === 'string' && block.content.trim()) { - parts.push(block.content) - continue - } - const toolCall = block.tool_call - if (isRecordObject(toolCall)) { - const name = typeof toolCall.name === 'string' ? toolCall.name : 'unknown' - const params = typeof toolCall.params === 'string' ? toolCall.params : '' - const response = typeof toolCall.response === 'string' ? toolCall.response : '' - parts.push(`tool ${name} ${params} ${response}`.trim()) - } - } - return parts.join(' ') -} - -function summarizeTapeRow( - row: DeepChatTapeEntryRow, - payload: Record, - userMessage?: UserMessageProjectionText | null -): string { - if (row.kind === 'message') { - const record = payload.record - if (isRecordObject(record)) { - const role = typeof record.role === 'string' ? record.role : 'message' - const content = typeof record.content === 'string' ? record.content : '' - const text = - role === 'assistant' - ? readAssistantMessageText(content) - : readUserMessageText(content, userMessage ?? undefined) - return compactText(`${role}: ${text}`, 1200) - } - } - - if (row.kind === 'tool_call') { - const toolCall = payload.toolCall - if (isRecordObject(toolCall)) { - const name = typeof toolCall.name === 'string' ? toolCall.name : (row.name ?? 'unknown') - const params = typeof toolCall.params === 'string' ? toolCall.params : '' - return compactText(`tool_call ${name}: ${params}`, 1200) - } - } - - if (row.kind === 'tool_result') { - const response = typeof payload.response === 'string' ? payload.response : payload - return compactText( - `tool_result ${row.name ?? 'unknown'}: ${stringifyForSummary(response)}`, - 1200 - ) - } - - if (row.kind === 'anchor') { - const state = isRecordObject(payload.state) ? payload.state : payload - const summary = typeof state.summary === 'string' ? state.summary : stringifyForSummary(state) - return compactText(`anchor ${row.name ?? 'unknown'}: ${summary}`, 1200) - } - - if (row.kind === 'event') { - const data = isRecordObject(payload.data) ? payload.data : payload - return compactText(`event ${row.name ?? 'unknown'}: ${stringifyForSummary(data)}`, 1200) - } - - return compactText(`${row.kind} ${row.name ?? ''}`.trim(), 1200) -} - -function buildTapeRowEvidenceText( - row: DeepChatTapeEntryRow, - payload: Record, - meta: Record, - userMessage?: UserMessageProjectionText | null -): string { - const parts: string[] = [] - if (row.kind === 'message' && isRecordObject(payload.record)) { - const record = payload.record - const content = typeof record.content === 'string' ? record.content : '' - const role = typeof record.role === 'string' ? record.role : 'message' - parts.push( - role === 'assistant' - ? readAssistantMessageText(content) - : readUserMessageText(content, userMessage ?? undefined) - ) - } else if (row.kind === 'tool_call' && isRecordObject(payload.toolCall)) { - const toolCall = payload.toolCall - parts.push(stringifyForSummary(toolCall.name, 200)) - parts.push(stringifyForSummary(toolCall.params, 3000)) - } else if (row.kind === 'tool_result') { - parts.push(stringifyForSummary(payload.response ?? payload, 4000)) - } else if (row.kind === 'anchor') { - parts.push(stringifyForSummary(isRecordObject(payload.state) ? payload.state : payload, 4000)) - } else if (row.kind === 'event') { - parts.push(stringifyForSummary(isRecordObject(payload.data) ? payload.data : payload, 4000)) - } else { - parts.push(stringifyForSummary(payload, 4000)) - } - if (typeof meta.status === 'string') parts.push(`status:${meta.status}`) - return compactText(parts.filter(Boolean).join('\n'), 5000) -} - -function setRef(target: Record, key: string, value: unknown): void { - if (value !== null && value !== undefined && value !== '') { - target[key] = value - } -} - -function enrichTapeRowRefs( - refs: Record, - payload: Record, - meta: Record, - evidenceText: string, - userMessage?: UserMessageProjectionText | null -): void { - const attachmentRefs = - userMessage?.attachmentRefs ?? collectUserMessageAttachmentRefsFromPayload(payload) - const filePaths = uniqueStrings( - [...extractFilePaths(evidenceText), ...attachmentRefs.filePaths], - 20 - ) - const errorCodes = extractErrorCodes(evidenceText) - const commands = uniqueStrings( - [ - ...collectKeyedStrings(payload, new Set(['command', 'cmd', 'script', 'shellCommand'])), - ...collectKeyedStrings(meta, new Set(['command', 'cmd', 'script', 'shellCommand'])) - ], - 10 - ) - setRef(refs, 'filePaths', filePaths.length ? filePaths : null) - setRef(refs, 'fileNames', attachmentRefs.fileNames.length ? attachmentRefs.fileNames : null) - setRef(refs, 'commands', commands.length ? commands : null) - setRef(refs, 'errorCodes', errorCodes.length ? errorCodes : null) - for (const key of ['exitCode', 'exitStatus', 'code']) { - const value = payload[key] ?? meta[key] - if (typeof value === 'number' || typeof value === 'string') { - setRef(refs, key, value) - } - } -} - -function buildTapeRowRefs( - row: DeepChatTapeEntryRow, - payload: Record, - meta: Record, - userMessage?: UserMessageProjectionText | null, - evidenceText?: string -): Record { - const refs: Record = {} - setRef(refs, 'sourceType', row.source_type) - setRef(refs, 'sourceId', row.source_id) - setRef(refs, 'sourceSeq', row.source_seq) - setRef(refs, 'status', typeof meta.status === 'string' ? meta.status : null) - - if (row.kind === 'message' && isRecordObject(payload.record)) { - const record = payload.record - setRef(refs, 'messageId', record.id) - setRef(refs, 'orderSeq', record.orderSeq) - setRef(refs, 'role', record.role) - setRef(refs, 'messageStatus', record.status) - } else if (row.kind === 'tool_call' && isRecordObject(payload.toolCall)) { - const toolCall = payload.toolCall - setRef(refs, 'messageId', payload.messageId) - setRef(refs, 'orderSeq', payload.orderSeq) - setRef(refs, 'toolCallId', toolCall.id) - setRef(refs, 'toolName', toolCall.name ?? row.name) - setRef(refs, 'serverName', toolCall.serverName) - } else if (row.kind === 'tool_result') { - setRef(refs, 'messageId', payload.messageId) - setRef(refs, 'orderSeq', payload.orderSeq) - setRef(refs, 'toolCallId', payload.toolCallId) - setRef(refs, 'toolName', row.name) - } else if (row.kind === 'anchor') { - setRef(refs, 'anchorName', row.name) - } else if (row.kind === 'event') { - setRef(refs, 'eventName', row.name) - } - - enrichTapeRowRefs( - refs, - payload, - meta, - evidenceText ?? buildTapeRowEvidenceText(row, payload, meta, userMessage), - userMessage - ) - return refs -} - -function parseProjectionRefs(raw: string): Record { - const parsed = parseJsonObject(raw) - return parsed -} - -function normalizeContextWindowValue(value: number | undefined, fallback: number): number { - if (!Number.isFinite(value)) return fallback - return Math.min(Math.max(Math.floor(value as number), 0), 20) -} - -function normalizeContextLimit(value: number | undefined): number { - if (!Number.isFinite(value)) return 50 - return Math.min(Math.max(Math.floor(value as number), 1), 100) -} - -function readToolFactStatus(row: DeepChatTapeEntryRow): string | null { - const status = parseJsonObject(row.meta_json).status - return typeof status === 'string' ? status : null -} - -function readToolFactToolCallId(row: DeepChatTapeEntryRow): string | null { - const payload = parseJsonObject(row.payload_json) - if (row.kind === 'tool_call') { - const toolCall = payload.toolCall - if (toolCall && typeof toolCall === 'object' && !Array.isArray(toolCall)) { - const id = (toolCall as Record).id - return typeof id === 'string' && id.length > 0 ? id : null - } - return null - } - const toolCallId = payload.toolCallId - return typeof toolCallId === 'string' && toolCallId.length > 0 ? toolCallId : null -} - -function readToolFactMessageId(row: DeepChatTapeEntryRow): string | null { - const messageId = parseJsonObject(row.payload_json).messageId - return typeof messageId === 'string' && messageId.length > 0 ? messageId : null -} - -function parseSearchBoundary(value: string | undefined, name: string): number | undefined { - const trimmed = value?.trim() - if (!trimmed) { - return undefined - } - - const numericValue = Number(trimmed) - if (Number.isFinite(numericValue)) { - return numericValue - } - - const parsedDate = Date.parse(trimmed) - if (Number.isFinite(parsedDate)) { - return parsedDate - } - - throw new Error(`${name} must be an ISO date/time or millisecond timestamp.`) -} - -function toTapeSearchInput(options: AgentTapeSearchOptions | undefined): DeepChatTapeSearchInput { - return { - limit: options?.limit, - kinds: options?.kinds, - startCreatedAt: parseSearchBoundary(options?.start, 'start'), - endCreatedAt: parseSearchBoundary(options?.end, 'end') - } -} - -function normalizeTapeViewScope(scope: AgentTapeViewScope | undefined): AgentTapeViewScope { - if (scope === undefined || scope === 'current') return 'current' - if (scope === 'linked_subagents' || scope === 'current_and_linked') return scope - throw new Error(`Invalid Tape view scope: ${String(scope)}`) -} - -function normalizeTapeSearchLimit(value: number | undefined): number { - if (!Number.isFinite(value)) return 20 - return Math.min(Math.max(Math.floor(value as number), 1), 100) -} - -function compareTapeSearchResults(left: TapeSearchResult, right: TapeSearchResult): number { - const leftHasScore = typeof left.score === 'number' && Number.isFinite(left.score) - const rightHasScore = typeof right.score === 'number' && Number.isFinite(right.score) - if (leftHasScore && rightHasScore && left.score !== right.score) { - return (left.score as number) - (right.score as number) - } - if (leftHasScore !== rightHasScore) { - return leftHasScore ? -1 : 1 - } - if (left.createdAt !== right.createdAt) { - return right.createdAt - left.createdAt - } - if (left.sessionId !== right.sessionId) { - return left.sessionId < right.sessionId ? -1 : 1 - } - return right.entryId - left.entryId -} - -function migrationProvenanceKey(sessionId: string): string { - return `migration:${sessionId}:message-backfill:v1` -} - -function legacySummaryProvenanceKey(sessionId: string): string { - return `summary:${sessionId}:legacy-summary:v1` -} - -function normalizeHandoffName(name: string): string { - const trimmed = name.trim() - if (!trimmed) { - return 'handoff/manual' - } - if (trimmed.startsWith('handoff/') || trimmed.startsWith('auto_handoff/')) { - return trimmed - } - return `handoff/${trimmed}` -} - -function normalizePositiveInteger(value: unknown): number | null { - if (typeof value === 'number' && Number.isFinite(value)) { - return Math.max(1, Math.floor(value)) - } - return null -} - -function hasOwnKey(value: Record, key: string): boolean { - return Object.prototype.hasOwnProperty.call(value, key) -} - -function buildOrderSeqRange(records: ChatMessageRecord[]): Record | null { - if (records.length === 0) { - return null - } - - return { - fromOrderSeq: records[0].orderSeq, - toOrderSeq: records[records.length - 1].orderSeq - } -} - -function enrichHandoffState( - state: Record, - historyRecords: ChatMessageRecord[] -): Record { - const maxOrderSeq = historyRecords.reduce( - (currentMax, record) => Math.max(currentMax, record.orderSeq), - 0 - ) - const cursorOrderSeq = - normalizePositiveInteger(state.cursorOrderSeq ?? state.summaryCursorOrderSeq) ?? maxOrderSeq + 1 - const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) - const enrichedState: Record = { - ...state, - cursorOrderSeq - } - - if (!hasOwnKey(enrichedState, 'range')) { - enrichedState.range = buildOrderSeqRange(sourceRecords) - } - - const sourceMessageIds = enrichedState.sourceMessageIds - if (!Array.isArray(sourceMessageIds) || sourceMessageIds.some((id) => typeof id !== 'string')) { - enrichedState.sourceMessageIds = sourceRecords.map((record) => record.id) - } - - return enrichedState -} - -export function normalizeTapeHandoffState(state: unknown): AgentTapeHandoffState { - if (!state || typeof state !== 'object' || Array.isArray(state)) { - throw new Error('Tape handoff requires a non-empty summary.') - } - - const summary = (state as Record).summary - if (typeof summary !== 'string' || !summary.trim()) { - throw new Error('Tape handoff requires a non-empty summary.') - } - - return { - ...(state as Record), - summary: summary.trim() - } -} - -function forkSessionId(parentSessionId: string, forkId: string): string { - return `${parentSessionId}::fork::${forkId}` -} - -function isEntryIdPrefix(prefix: number[], values: number[]): boolean { - if (prefix.length > values.length) return false - for (let index = 0; index < prefix.length; index += 1) { - if (prefix[index] !== values[index]) return false - } - return true -} - -function hashString(value: string): string { - return createHash('sha256').update(value).digest('hex') -} - -function isPositiveInteger(value: number): boolean { - return Number.isInteger(value) && value > 0 -} - -function collectEntryIds(values: Array): number[] { - return [...new Set(values.filter((value): value is number => typeof value === 'number'))].sort( - (left, right) => left - right - ) -} - -const VIEW_POLICIES = new Set([ - 'legacy_context_v1', - 'legacy_context_shadow', - 'resume_shadow', - 'tool_loop_shadow', - 'context_pressure_recovery_shadow' -]) - -const VIEW_ENTRY_REASONS = new Set([ - 'system_prompt', - 'selected_history', - 'new_user_input', - 'resume_target', - 'tool_loop_message' -]) - -const VIEW_EXCLUDED_REASONS = new Set([ - 'before_summary_cursor', - 'compaction_indicator', - 'pending_not_context_history', - 'out_of_budget', - 'empty_after_formatting', - 'superseded', - 'retracted' -]) - -function isRecordObject(value: unknown): value is Record { - return Boolean(value && typeof value === 'object' && !Array.isArray(value)) -} - -function isNullableString(value: unknown): value is string | null { - return value === null || typeof value === 'string' -} - -function isNullableNumber(value: unknown): value is number | null { - return value === null || typeof value === 'number' -} - -function isViewEntryRef(value: unknown): value is DeepChatTapeViewManifest['included'][number] { - if (!isRecordObject(value)) { - return false - } - - return ( - isNullableNumber(value.entryId) && - isNullableString(value.messageId) && - isNullableNumber(value.orderSeq) && - (value.role === 'system' || - value.role === 'user' || - value.role === 'assistant' || - value.role === 'tool' || - value.role === null) && - (value.source === 'tape' || value.source === 'synthetic') && - typeof value.reason === 'string' && - VIEW_ENTRY_REASONS.has(value.reason) - ) -} - -function isViewExcludedRef(value: unknown): value is DeepChatTapeViewManifest['excluded'][number] { - if (!isRecordObject(value)) { - return false - } - - return ( - isNullableNumber(value.entryId) && - isNullableString(value.messageId) && - isNullableNumber(value.orderSeq) && - typeof value.reason === 'string' && - VIEW_EXCLUDED_REASONS.has(value.reason) - ) -} - -function isViewExcludedRange(value: unknown): value is DeepChatTapeViewExcludedRange { - if (!isRecordObject(value)) { - return false - } - - return ( - typeof value.fromOrderSeq === 'number' && - typeof value.toOrderSeq === 'number' && - typeof value.count === 'number' && - typeof value.reason === 'string' && - VIEW_EXCLUDED_REASONS.has(value.reason) - ) -} - -function hasNumberFields(value: unknown, fields: string[]): value is Record { - if (!isRecordObject(value)) { - return false - } - - return fields.every((field) => typeof value[field] === 'number') -} - -function hasStringFields(value: unknown, fields: string[]): value is Record { - if (!isRecordObject(value)) { - return false - } - - return fields.every((field) => typeof value[field] === 'string') -} - -function isViewManifestMeta(value: unknown): value is DeepChatTapeViewManifest['meta'] { - if (!isRecordObject(value)) { - return false - } - - return ( - typeof value.providerId === 'string' && - typeof value.modelId === 'string' && - typeof value.summaryCursorOrderSeq === 'number' && - typeof value.supportsVision === 'boolean' && - typeof value.supportsAudioInput === 'boolean' && - typeof value.traceDebugEnabled === 'boolean' - ) -} - -function isViewManifest(value: unknown, sessionId: string): value is DeepChatTapeViewManifest { - if (!isRecordObject(value)) { - return false - } - - return ( - (value.schemaVersion === 1 || value.schemaVersion === 2) && - typeof value.hashVersion === 'number' && - value.sessionId === sessionId && - typeof value.viewId === 'string' && - typeof value.messageId === 'string' && - typeof value.requestSeq === 'number' && - (value.taskType === 'chat' || value.taskType === 'resume' || value.taskType === 'tool_loop') && - typeof value.policy === 'string' && - VIEW_POLICIES.has(value.policy) && - (typeof value.policyVersion === 'number' || value.policyVersion === null) && - value.contextBuilderVersion === 'legacy-v1' && - typeof value.latestEntryId === 'number' && - Array.isArray(value.anchorEntryIds) && - value.anchorEntryIds.every((entryId) => typeof entryId === 'number') && - (value.reconstructionAnchorEntryId === undefined || - isNullableNumber(value.reconstructionAnchorEntryId)) && - (value.excludedRanges === undefined || - (Array.isArray(value.excludedRanges) && value.excludedRanges.every(isViewExcludedRange))) && - Array.isArray(value.included) && - value.included.every(isViewEntryRef) && - Array.isArray(value.excluded) && - value.excluded.every(isViewExcludedRef) && - hasNumberFields(value.tokenBudget, [ - 'contextLength', - 'requestedMaxTokens', - 'effectiveMaxTokens', - 'reserveTokens', - 'toolReserveTokens', - 'estimatedPromptTokens' - ]) && - hasStringFields(value.hashes, ['promptHash', 'toolDefinitionsHash', 'manifestHash']) && - isViewManifestMeta(value.meta) && - typeof value.assembledAt === 'number' - ) -} - -function withReplaySliceHash( - slice: Omit & { - hashes: Omit & { sliceHash: '' } - } -): DeepChatTapeReplaySlice { - const sliceForHash = { ...slice } as Partial - delete sliceForHash.createdAt - delete sliceForHash.integrity - return { - ...slice, - hashes: { - ...slice.hashes, - sliceHash: hashJson(sliceForHash) - } - } -} - -export class SessionTape implements TapeToolFactWriter { - constructor(private readonly database: SessionDatabase) {} - - private get table(): SessionDatabase['deepchatTapeEntriesTable'] { - return this.database.deepchatTapeEntriesTable - } - - private get searchProjectionTable(): SessionDatabase['deepchatTapeSearchProjectionTable'] { - return this.database.deepchatTapeSearchProjectionTable - } - - ensureSessionTapeReady(sessionId: string, messageStore: SessionTranscript): TapeBackfillResult { - const table = this.table - const historyRecords = messageStore - .getMessages(sessionId) - .sort((left, right) => left.orderSeq - right.orderSeq) - const maxOrderSeq = historyRecords.reduce( - (currentMax, record) => Math.max(currentMax, record.orderSeq), - 0 - ) - - table.ensureBootstrapAnchor(sessionId) - - let appendedFactCount = 0 - for (const record of historyRecords) { - appendedFactCount += appendMessageRecordToTape(table, record, 'backfill') - } - - this.backfillLegacySummaryAnchor(sessionId, historyRecords) - - table.appendEvent({ - sessionId, - name: 'migration/backfill', - source: { - type: 'migration', - id: 'message-backfill', - seq: 1 - }, - provenanceKey: migrationProvenanceKey(sessionId), - data: { - source: 'deepchat_messages', - messageCount: historyRecords.length, - maxOrderSeq - }, - idempotent: true - }) - - return { - sessionId, - migrationState: 'ready', - messageCount: historyRecords.length, - maxOrderSeq, - appendedFactCount, - historyRecords: this.getMessageRecords(sessionId) - } - } - - appendMessageRecord(record: ChatMessageRecord): number { - return appendMessageRecordToTape(this.table, record, 'live') - } - - async appendToolFact(input: TapeToolFactInput): Promise { - const row = appendTapeToolFact(this.table, input, 'live', 'tool_loop') - if (!row) throw new Error('Tape tool fact was not appendable.') - return { sessionId: input.sessionId, entryId: row.entry_id } - } - - getMessageRecords(sessionId: string): ChatMessageRecord[] { - return buildEffectiveTapeView(this.table.getBySession(sessionId), { includePending: true }) - .messageRecords - } - - info(sessionId: string): TapeInfo { - const table = this.table - const lastAnchor = table.getLatestAnchor(sessionId) - const rows = table.getBySession(sessionId) - return { - sessionId, - entries: table.countBySession(sessionId), - anchors: table.countAnchorsBySession(sessionId), - lastAnchor: lastAnchor?.name ?? null, - lastAnchorEntryId: lastAnchor?.entry_id ?? null, - entriesSinceLastAnchor: lastAnchor - ? table.countEntriesAfter(sessionId, lastAnchor.entry_id) - : 0, - lastTokenUsage: getLastEffectiveTokenUsage(rows), - migrationState: table.getByProvenanceKey(sessionId, migrationProvenanceKey(sessionId)) - ? 'ready' - : 'none' - } - } - - search(sessionId: string, query: string, options?: AgentTapeSearchOptions): TapeSearchResult[] { - const scope = normalizeTapeViewScope(options?.scope) - if (!query.trim()) { - return [] - } - if (scope === 'current') { - return this.searchCurrentTape(sessionId, query, options) - } - - const resolution = this.resolveLinkedTapeSources(sessionId) - if (resolution.unavailableSourceIds.size > 0) { - const sourceSessionId = [...resolution.unavailableSourceIds].sort()[0] - throw new AgentTapeViewError( - 'linked_tape_unavailable', - sessionId, - sourceSessionId, - `Linked Tape ${sourceSessionId} is unavailable.` - ) - } - const sources = [...resolution.sources] - if (scope === 'current_and_linked') { - sources.push({ sessionId, maxEntryId: this.table?.getMaxEntryId(sessionId) ?? 0 }) - } - return this.searchTapeSourcesReadOnly(sources, query, options) - } - - private searchCurrentTape( - sessionId: string, - query: string, - options?: AgentTapeSearchOptions - ): TapeSearchResult[] { - const table = this.table - const searchInput = toTapeSearchInput(options) - const projectionTable = this.searchProjectionTable - let skipProjectionSearch = false - - if (projectionTable) { - try { - const maxEntryId = table.getMaxEntryId(sessionId) - if (projectionTable.isCurrent(sessionId, maxEntryId)) { - return projectionTable - .search(sessionId, query, searchInput) - .map((row) => this.toProjectedSearchResult(row, undefined)) - } - } catch (error) { - skipProjectionSearch = true - logger.warn( - `[Tape] projection fast-path search failed; falling back to effective search: ${String(error)}` - ) - } - } - - const rows = table.getBySession(sessionId) - const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows - const preparedProjectionTable = skipProjectionSearch - ? null - : this.ensureSearchProjection(sessionId, rows, effectiveRows) - if (!preparedProjectionTable) { - return searchEffectiveTapeRows(rows, query, searchInput).map((row) => - this.toSearchResult(row) - ) - } - - const rowByEntryId = new Map(effectiveRows.map((row) => [row.entry_id, row])) - try { - return preparedProjectionTable - .search(sessionId, query, searchInput) - .map((row) => this.toProjectedSearchResult(row, rowByEntryId.get(row.entry_id))) - } catch (error) { - logger.warn( - `[Tape] projection search failed; falling back to effective search: ${String(error)}` - ) - return searchEffectiveTapeRows(rows, query, searchInput).map((row) => - this.toSearchResult(row) - ) - } - } - - getContext( - sessionId: string, - entryIds: number[], - options: AgentTapeContextOptions = {} - ): AgentTapeContextResult { - const sourceSessionId = options.sourceSessionId?.trim() || sessionId - if (sourceSessionId !== sessionId) { - return this.getLinkedTapeContext(sessionId, sourceSessionId, entryIds, options) - } - - const requestedEntryIds = [ - ...new Set(entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0)) - ].sort((left, right) => left - right) - const table = this.table - if (!table || requestedEntryIds.length === 0) { - return { - sessionId, - sourceSessionId, - requestedEntryIds, - matchedEntryIds: [], - entries: [] - } - } - - const rows = table.getBySession(sessionId) - const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows - const indexByEntryId = new Map(effectiveRows.map((row, index) => [row.entry_id, index])) - const before = normalizeContextWindowValue(options.before, 2) - const after = normalizeContextWindowValue(options.after, 2) - const limit = normalizeContextLimit(options.limit) - const maxBytesPerEntry = normalizeContextByteLimit( - options.maxBytesPerEntry, - DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY, - MAX_CONTEXT_MAX_BYTES_PER_ENTRY - ) - const maxTotalBytes = normalizeContextByteLimit( - options.maxTotalBytes, - DEFAULT_CONTEXT_MAX_TOTAL_BYTES, - MAX_CONTEXT_MAX_TOTAL_BYTES - ) - const selectedIndexes = new Set() - const requestedIndexes: number[] = [] - const windowIndexes: number[] = [] - - for (const entryId of requestedEntryIds) { - const index = indexByEntryId.get(entryId) - if (index === undefined) continue - requestedIndexes.push(index) - for ( - let cursor = Math.max(0, index - before); - cursor <= Math.min(effectiveRows.length - 1, index + after); - cursor += 1 - ) { - if (cursor === index) continue - windowIndexes.push(cursor) - } - } - - for (const index of requestedIndexes) { - if (selectedIndexes.size >= limit) break - selectedIndexes.add(index) - } - for (const index of windowIndexes) { - if (selectedIndexes.size >= limit) break - selectedIndexes.add(index) - } - - const selectedRows = [...selectedIndexes] - .sort((left, right) => left - right) - .map((index) => effectiveRows[index]) - let projectionRows = new Map() - try { - projectionRows = new Map( - this.searchProjectionTable - .getByEntryIds( - sessionId, - selectedRows.map((row) => row.entry_id) - ) - .map((row) => [row.entry_id, row]) - ) - } catch { - projectionRows = new Map() - } - let usedBytes = 0 - const entries: AgentTapeContextEntry[] = [] - const priorityIndexes = [...requestedIndexes, ...windowIndexes].filter( - (index, offset, indexes) => { - return selectedIndexes.has(index) && indexes.indexOf(index) === offset - } - ) - for (const index of priorityIndexes) { - const row = effectiveRows[index] - const remaining = Math.max(0, maxTotalBytes - usedBytes) - if (remaining <= 0) break - const maxEntryBytes = Math.min(maxBytesPerEntry, remaining) - if (maxEntryBytes <= 0) break - const entry = this.toContextEntry(row, projectionRows.get(row.entry_id), maxEntryBytes) - if (entry.evidence.bytes <= 0) continue - usedBytes += entry.evidence.bytes - entries.push(entry) - } - entries.sort((left, right) => left.entryId - right.entryId) - const returnedEntryIds = new Set(entries.map((entry) => entry.entryId)) - - return { - sessionId, - sourceSessionId, - requestedEntryIds, - matchedEntryIds: requestedEntryIds.filter((entryId) => returnedEntryIds.has(entryId)), - entries - } - } - - private resolveLinkedTapeSources(parentSessionId: string): LinkedTapeSourceResolution { - const table = this.table - const sessionTable = this.database.newSessionsTable - if (!table || !sessionTable?.get(parentSessionId)) { - throw new AgentTapeViewError( - 'current_tape_unavailable', - parentSessionId, - parentSessionId, - `Current Tape ${parentSessionId} is unavailable.` - ) - } - - const parsedSnapshots = table - .getSubagentLineageEvents(parentSessionId) - .map((row) => parseSubagentTapeLinkSnapshot(row) ?? parseLegacyExternalTapeLinkSnapshot(row)) - .filter((snapshot): snapshot is SubagentTapeLinkSnapshot => snapshot !== null) - const latestSnapshotByChild = new Map() - for (const snapshot of parsedSnapshots) { - const current = latestSnapshotByChild.get(snapshot.childSessionId) - if (!current || snapshot.linkEntryId > current.linkEntryId) { - latestSnapshotByChild.set(snapshot.childSessionId, snapshot) - } - } - const snapshots = [...latestSnapshotByChild.values()] - const childSessionIds = [...new Set(snapshots.map((snapshot) => snapshot.childSessionId))] - const childById = new Map(sessionTable.getMany(childSessionIds).map((row) => [row.id, row])) - const unavailableSourceIds = new Set() - const snapshotBySource = new Map() - - for (const snapshot of snapshots) { - const child = childById.get(snapshot.childSessionId) - if (!child) { - unavailableSourceIds.add(snapshot.childSessionId) - continue - } - if (child.session_kind !== 'subagent' || child.parent_session_id !== parentSessionId) { - continue - } - snapshotBySource.set(snapshot.childSessionId, snapshot) - } - - const authorizedSourceIds = [...snapshotBySource.keys()] - const firstEntryBySource = new Map( - table.getFirstEntriesBySessions(authorizedSourceIds).map((row) => [row.session_id, row]) - ) - const liveHeads = table.getMaxEntryIdsBySessions(authorizedSourceIds) - const availableSources: DeepChatTapeReadSource[] = [] - for (const [sourceSessionId, snapshot] of snapshotBySource) { - const firstEntry = firstEntryBySource.get(sourceSessionId) - const identityMatches = snapshot.childTapeIdentity - ? firstEntry !== undefined && computeTapeIdentity(firstEntry) === snapshot.childTapeIdentity - : firstEntry !== undefined && isUnmarkedLegacyTape(firstEntry) - if (!identityMatches || (liveHeads.get(sourceSessionId) ?? 0) < snapshot.childHeadEntryId) { - unavailableSourceIds.add(sourceSessionId) - continue - } - availableSources.push({ - sessionId: sourceSessionId, - maxEntryId: snapshot.childHeadEntryId - }) - } - - return { - sources: availableSources.sort((left, right) => - left.sessionId < right.sessionId ? -1 : left.sessionId > right.sessionId ? 1 : 0 - ), - unavailableSourceIds - } - } - - private searchTapeSourcesReadOnly( - sources: DeepChatTapeReadSource[], - query: string, - options: AgentTapeSearchOptions | undefined - ): TapeSearchResult[] { - const table = this.table - if (!table || !query.trim() || sources.length === 0) { - return [] - } - const searchInput = toTapeSearchInput(options) - const limit = normalizeTapeSearchLimit(options?.limit) - const results: TapeSearchResult[] = [] - let uncoveredSources = sources - - try { - const projected = this.searchProjectionTable?.searchSourcesReadOnly( - sources, - query, - searchInput - ) - if (projected) { - const coveredHeadBySource = new Map( - projected.coveredSources.map((source) => [source.sessionId, source.maxEntryId]) - ) - const hasCompleteProjection = - coveredHeadBySource.size === sources.length && - sources.every((source) => coveredHeadBySource.get(source.sessionId) === source.maxEntryId) - if (hasCompleteProjection) { - results.push(...projected.rows.map((row) => this.toProjectedSearchResult(row, undefined))) - uncoveredSources = [] - } - } - } catch (error) { - logger.warn( - `[Tape] linked projection search failed; using read-only Tape fallback: ${String(error)}` - ) - uncoveredSources = sources - } - - if (uncoveredSources.length > 0) { - results.push( - ...table - .searchEffectiveSourcesAtHeads(uncoveredSources, query, searchInput) - .map((row) => this.toSearchResult(row)) - ) - } - - const seen = new Set() - return results - .sort(compareTapeSearchResults) - .filter((result) => { - const key = `${result.sessionId}:${result.entryId}` - if (seen.has(key)) return false - seen.add(key) - return true - }) - .slice(0, limit) - } - - private getLinkedTapeContext( - parentSessionId: string, - sourceSessionId: string, - entryIds: number[], - options: AgentTapeContextOptions - ): AgentTapeContextResult { - const resolution = this.resolveLinkedTapeSources(parentSessionId) - if (resolution.unavailableSourceIds.has(sourceSessionId)) { - throw new AgentTapeViewError( - 'linked_tape_unavailable', - parentSessionId, - sourceSessionId, - `Linked Tape ${sourceSessionId} is unavailable.` - ) - } - const source = resolution.sources.find((candidate) => candidate.sessionId === sourceSessionId) - if (!source) { - throw new AgentTapeViewError( - 'linked_tape_unauthorized', - parentSessionId, - sourceSessionId, - `Tape ${sourceSessionId} is not an authorized direct child of ${parentSessionId}.` - ) - } - - const requestedEntryIds = [ - ...new Set(entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0)) - ].sort((left, right) => left - right) - const table = this.table - if (!table || requestedEntryIds.length === 0) { - return { - sessionId: parentSessionId, - sourceSessionId, - requestedEntryIds, - matchedEntryIds: [], - entries: [] - } - } - - const before = normalizeContextWindowValue(options.before, 2) - const after = normalizeContextWindowValue(options.after, 2) - const limit = normalizeContextLimit(options.limit) - const maxBytesPerEntry = normalizeContextByteLimit( - options.maxBytesPerEntry, - DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY, - MAX_CONTEXT_MAX_BYTES_PER_ENTRY - ) - const maxTotalBytes = normalizeContextByteLimit( - options.maxTotalBytes, - DEFAULT_CONTEXT_MAX_TOTAL_BYTES, - MAX_CONTEXT_MAX_TOTAL_BYTES - ) - const rows = table.getEffectiveContextRowsAtHead(source, requestedEntryIds, { - before, - after, - limit - }) - - let usedBytes = 0 - const entries: AgentTapeContextEntry[] = [] - for (const row of rows) { - const remaining = Math.max(0, maxTotalBytes - usedBytes) - if (remaining <= 0) break - const maxEntryBytes = Math.min(maxBytesPerEntry, remaining) - const entry = this.toContextEntry(row, undefined, maxEntryBytes) - if (entry.evidence.bytes <= 0) continue - usedBytes += entry.evidence.bytes - entries.push(entry) - } - entries.sort((left, right) => left.entryId - right.entryId) - const returnedEntryIds = new Set(entries.map((entry) => entry.entryId)) - - return { - sessionId: parentSessionId, - sourceSessionId, - requestedEntryIds, - matchedEntryIds: requestedEntryIds.filter((entryId) => returnedEntryIds.has(entryId)), - entries - } - } - - anchors(sessionId: string, options: AgentTapeAnchorsOptions = {}): TapeAnchorResult[] { - return this.table.getAnchors(sessionId, options.limit).map((row) => this.toAnchorResult(row)) - } - - getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestSourceMaps { - const table = this.table - const rows = table.getBySession(sessionId) - const entryIdByMessageId = new Map() - const toolCallEntryIdByToolId = new Map() - const toolResultEntryIdByToolId = new Map() - let latestEntryId = 0 - const anchorEntryIds: number[] = [] - let reconstructionAnchorEntryId: number | null = null - let bootstrapAnchorEntryId: number | null = null - - for (const row of rows) { - latestEntryId = Math.max(latestEntryId, row.entry_id) - if (row.kind === 'anchor') { - anchorEntryIds.push(row.entry_id) - if (isReconstructionAnchorName(row.name)) { - if (reconstructionAnchorEntryId === null || row.entry_id > reconstructionAnchorEntryId) { - reconstructionAnchorEntryId = row.entry_id - } - } else if (row.name === BOOTSTRAP_ANCHOR_NAME) { - bootstrapAnchorEntryId = row.entry_id - } - continue - } - if (row.kind === 'message' && row.source_type === 'message' && row.source_id) { - entryIdByMessageId.set(row.source_id, row.entry_id) - continue - } - if (row.kind === 'tool_call' || row.kind === 'tool_result') { - if (messageId && readToolFactMessageId(row) !== messageId) { - continue - } - const toolCallId = readToolFactToolCallId(row) - if (!toolCallId || readToolFactStatus(row) === 'pending') { - continue - } - const target = - row.kind === 'tool_call' ? toolCallEntryIdByToolId : toolResultEntryIdByToolId - target.set(toolCallId, row.entry_id) - } - } - - const reconstructionAnchorEntryIds = - reconstructionAnchorEntryId !== null - ? [reconstructionAnchorEntryId] - : bootstrapAnchorEntryId !== null - ? [bootstrapAnchorEntryId] - : [] - - return { - latestEntryId, - anchorEntryIds, - reconstructionAnchorEntryIds, - reconstructionAnchorEntryId, - entryIdByMessageId, - toolCallEntryIdByToolId, - toolResultEntryIdByToolId - } - } - - appendViewManifest(manifest: DeepChatTapeViewManifest): DeepChatTapeEntryRow { - const table = this.table - table.ensureBootstrapAnchor(manifest.sessionId) - return table.appendEvent({ - sessionId: manifest.sessionId, - name: TAPE_VIEW_MANIFEST_EVENT_NAME, - source: { - type: 'runtime_event', - id: manifest.messageId, - seq: manifest.requestSeq - }, - provenanceKey: `view:${manifest.sessionId}:${manifest.messageId}:${manifest.requestSeq}:${manifest.hashes.manifestHash}`, - data: { - manifest - }, - meta: { - viewId: manifest.viewId, - requestSeq: manifest.requestSeq, - taskType: manifest.taskType, - policy: manifest.policy, - policyVersion: manifest.policyVersion - }, - createdAt: manifest.assembledAt, - idempotent: true - }) - } - - listViewManifestsByMessage( - sessionId: string, - messageId: string - ): DeepChatTapeViewManifestRecord[] { - const table = this.table - return table - .getBySession(sessionId) - .filter( - (row) => - row.kind === 'event' && - row.name === TAPE_VIEW_MANIFEST_EVENT_NAME && - row.source_type === 'runtime_event' && - row.source_id === messageId - ) - .map((row) => this.toViewManifestRecord(row)) - .filter((record): record is DeepChatTapeViewManifestRecord => Boolean(record)) - .sort((left, right) => right.requestSeq - left.requestSeq || right.entryId - left.entryId) - } - - exportReplaySlice( - sessionId: string, - messageId: string, - options: DeepChatTapeReplayExportOptions = {} - ): DeepChatTapeReplaySlice | null { - if (options.requestSeq !== undefined && !isPositiveInteger(options.requestSeq)) { - throw new Error('requestSeq must be a positive integer.') - } - - const manifests = this.listViewManifestsByMessage(sessionId, messageId) - const manifestRecord = - options.requestSeq === undefined - ? manifests[0] - : manifests.find((record) => record.requestSeq === options.requestSeq) - if (!manifestRecord) { - return null - } - - return this.buildReplaySlice(sessionId, messageId, manifestRecord, options) - } - - readCausalObservationSlice( - sessionId: string, - messageId: string, - options: DeepChatCausalObservationReadOptions = {} - ): DeepChatCausalObservationSlice { - if (options.requestSeq !== undefined && !isPositiveInteger(options.requestSeq)) { - throw new Error('requestSeq must be a positive integer.') - } - - const rows = this.table.getBySession(sessionId) - const manifestRows = rows.filter( - (row) => - row.kind === 'event' && - row.name === TAPE_VIEW_MANIFEST_EVENT_NAME && - row.source_type === 'runtime_event' && - row.source_id === messageId - ) - const traces = this.database.deepchatMessageTracesTable - .listByMessageId(messageId) - .filter( - (row) => - row.session_id === sessionId && - row.message_id === messageId && - isPositiveInteger(row.request_seq) - ) - - const requestSeq = - options.requestSeq ?? - [...manifestRows.map((row) => row.source_seq), ...traces.map((row) => row.request_seq)] - .filter((value): value is number => typeof value === 'number' && isPositiveInteger(value)) - .reduce((latest, value) => Math.max(latest ?? value, value), null) - - let request: DeepChatCausalObservationRequest - if (requestSeq === null) { - request = { state: 'request_unavailable', requestSeq: null, trace: null } - } else { - const selectedManifestRows = manifestRows.filter((row) => row.source_seq === requestSeq) - const manifestRecord = selectedManifestRows - .map((row) => this.toViewManifestRecord(row)) - .find((record) => record?.messageId === messageId && record.requestSeq === requestSeq) - const trace = traces.find((row) => row.request_seq === requestSeq) ?? null - - if (manifestRecord) { - request = { - state: 'manifest_bound', - requestSeq, - replay: this.buildReplaySlice(sessionId, messageId, manifestRecord, options) - } - } else { - request = { - state: selectedManifestRows.length > 0 ? 'manifest_malformed' : 'manifest_missing', - requestSeq, - trace: trace - ? this.toReplayTraceSnapshot(trace, options.includeTracePayload === true) - : null - } - } - } - - const outputEntries = buildEffectiveTapeView(rows, { includePending: false }) - .rows.filter( - (row) => - (row.kind === 'message' && - row.source_type === 'message' && - row.source_id === messageId) || - ((row.kind === 'tool_call' || row.kind === 'tool_result') && - readToolFactMessageId(row) === messageId) - ) - .map((row) => this.toReplayEntrySnapshot(row, options.includeTapePayloads === true)) - const message = this.database.deepchatMessagesTable.get(messageId) - const terminalMessage = - message?.session_id === sessionId && - message.role === 'assistant' && - (message.status === 'sent' || message.status === 'error') - ? { - status: message.status, - orderSeq: message.order_seq, - createdAt: message.created_at, - updatedAt: message.updated_at, - contentHash: hashString(message.content), - metadataHash: hashString(message.metadata) - } - : null - - return { - schemaVersion: 1, - sessionId, - messageId, - request, - output: { - correlation: 'message_only', - entries: outputEntries, - terminalMessage - }, - runtime: - options.currentRuntimeStatus === undefined - ? { scope: 'unavailable', status: null, eventHistory: 'not_persisted' } - : { - scope: 'current_only', - status: options.currentRuntimeStatus, - eventHistory: 'not_persisted' - } - } - } - - private buildReplaySlice( - sessionId: string, - messageId: string, - manifestRecord: DeepChatTapeViewManifestRecord, - options: DeepChatTapeReplayExportOptions - ): DeepChatTapeReplaySlice { - const table = this.table - const manifest = manifestRecord.manifest - const includedEntryIds = collectEntryIds(manifest.included.map((ref) => ref.entryId)) - const excludedEntryIds = collectEntryIds(manifest.excluded.map((ref) => ref.entryId)) - const anchorEntryIds = collectEntryIds(manifest.anchorEntryIds) - const selectedEntryIds = new Set([ - manifestRecord.entryId, - ...includedEntryIds, - ...excludedEntryIds, - ...anchorEntryIds - ]) - const entries = table - .getBySession(sessionId) - .filter((row) => selectedEntryIds.has(row.entry_id)) - .map((row) => this.toReplayEntrySnapshot(row, options.includeTapePayloads === true)) - - const trace = this.findReplayTrace(sessionId, messageId, manifestRecord.requestSeq) - const createdAt = Date.now() - const sliceBase: Omit & { - hashes: Omit & { sliceHash: '' } - } = { - schemaVersion: 1 as const, - sliceId: `replay_${hashJson({ - sessionId, - messageId, - requestSeq: manifestRecord.requestSeq, - manifestHash: manifest.hashes.manifestHash - }).slice(0, 16)}`, - sessionId, - messageId, - requestSeq: manifestRecord.requestSeq, - mode: trace ? 'trace_bound' : 'manifest_only', - manifestRecord, - trace: trace ? this.toReplayTraceSnapshot(trace, options.includeTracePayload === true) : null, - entries, - refs: { - manifestEntryId: manifestRecord.entryId, - includedEntryIds, - excludedEntryIds, - anchorEntryIds - }, - hashes: { - manifestHash: manifest.hashes.manifestHash, - sliceHash: '' - }, - integrity: manifestRecord.integrity, - createdAt - } - - return withReplaySliceHash(sliceBase) - } - - handoff( - sessionId: string, - name: string, - state: AgentTapeHandoffState, - meta: Record = {} - ): DeepChatTapeEntryRow { - const normalizedState = normalizeTapeHandoffState(state) - const table = this.table - table.ensureBootstrapAnchor(sessionId) - const handoffState = enrichHandoffState(normalizedState, this.getMessageRecords(sessionId)) - return table.appendAnchor({ - sessionId, - name: normalizeHandoffName(name), - source: { - type: 'runtime_event', - id: `handoff:${Date.now()}`, - seq: 0 - }, - state: handoffState, - meta: { - ...meta, - handoff: true - } - }) - } - - handoffResult( - sessionId: string, - name: string, - state: AgentTapeHandoffState, - meta: Record = {} - ): TapeAnchorResult { - return this.toAnchorResult(this.handoff(sessionId, name, state, meta)) - } - - createFork(parentSessionId: string, forkId: string = nanoid()): TapeForkHandle { - const table = this.table - const forkIdValue = forkId.trim() || nanoid() - const forkSessionIdValue = forkSessionId(parentSessionId, forkIdValue) - const parentHeadEntryId = table.getMaxEntryId(parentSessionId) - table.ensureBootstrapAnchor(forkSessionIdValue) - const parentAnchor = table.getLatestAnchor(parentSessionId) - const forkStart = table.appendAnchor({ - sessionId: forkSessionIdValue, - name: 'fork/start', - source: { - type: 'fork', - id: forkIdValue, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkIdValue}:start`, - state: { - parentSessionId, - parentHeadEntryId, - parentLastAnchorEntryId: parentAnchor?.entry_id ?? null, - parentLastAnchorName: parentAnchor?.name ?? null - }, - idempotent: true - }) - const forkStartPayload = parseJsonObject(forkStart.payload_json) - const persistedState = - forkStartPayload.state && - typeof forkStartPayload.state === 'object' && - !Array.isArray(forkStartPayload.state) - ? (forkStartPayload.state as Record) - : {} - const persistedParentHeadEntryId = persistedState.parentHeadEntryId - return { - parentSessionId, - forkId: forkIdValue, - forkSessionId: forkSessionIdValue, - parentHeadEntryId: - typeof persistedParentHeadEntryId === 'number' && - Number.isSafeInteger(persistedParentHeadEntryId) && - persistedParentHeadEntryId >= 0 - ? persistedParentHeadEntryId - : parentHeadEntryId - } - } - - appendForkMessageRecord(handle: TapeForkHandle, record: ChatMessageRecord): number { - return appendMessageRecordToTape( - this.table, - { - ...record, - sessionId: handle.forkSessionId - }, - 'live' - ) - } - - mergeFork(parentSessionId: string, forkId: string): number { - const table = this.table - const forkSessionIdValue = forkSessionId(parentSessionId, forkId) - const mergeProvenanceKey = `fork:${parentSessionId}:${forkId}:merge:event` - - return table.runInTransaction(() => { - const existingReceipt = table.getByProvenanceKey(parentSessionId, mergeProvenanceKey) - if (existingReceipt) { - return readForkMergeReceiptCount( - existingReceipt, - parentSessionId, - forkId, - forkSessionIdValue - ) - } - - assertValidForkStart( - table.getByProvenanceKey(forkSessionIdValue, `fork:${parentSessionId}:${forkId}:start`), - parentSessionId, - forkId, - forkSessionIdValue - ) - - const forkHeadEntryId = table.getMaxEntryId(forkSessionIdValue) - const forkEntries = table - .getBySessionUpToEntryId(forkSessionIdValue, forkHeadEntryId) - .filter( - (entry) => - !( - entry.kind === 'anchor' && - (entry.name === 'session/start' || entry.name === 'fork/start') - ) - ) - - for (const entry of forkEntries) { - table.append({ - sessionId: parentSessionId, - kind: entry.kind, - name: entry.name, - source: { - type: 'fork', - id: forkId, - seq: entry.entry_id - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:merge:${entry.entry_id}`, - payload: parseJsonObject(entry.payload_json), - meta: { - ...parseJsonObject(entry.meta_json), - forkId, - forkSessionId: forkSessionIdValue, - mergedFromEntryId: entry.entry_id - }, - createdAt: entry.created_at, - idempotent: true - }) - } - - table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/merge', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: mergeProvenanceKey, - data: { - forkId, - forkSessionId: forkSessionIdValue, - forkHeadEntryId, - mergedCount: forkEntries.length - }, - idempotent: true - }) - - return forkEntries.length - }) - } - - discardFork(parentSessionId: string, forkId: string): void { - const table = this.table - const forkSessionIdValue = forkSessionId(parentSessionId, forkId) - table.deleteBySession(forkSessionIdValue) - try { - this.searchProjectionTable.deleteBySession(forkSessionIdValue) - } catch (error) { - logger.warn(`[Tape] failed to delete fork search projection: ${String(error)}`) - } - table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/discard', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:discard:event`, - data: { - forkId, - forkSessionId: forkSessionIdValue - }, - idempotent: true - }) - } - - recordExternalForkMerge( - parentSessionId: string, - forkSessionIdValue: string, - forkId: string, - meta: Record = {} - ): DeepChatTapeEntryRow { - const table = this.table - const referencedEntryCount = table.countBySession(forkSessionIdValue) - return table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/merge', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:external-merge:event`, - data: { - forkId, - forkSessionId: forkSessionIdValue, - referencedEntryCount, - ...meta - }, - idempotent: true - }) - } - - recordExternalForkDiscard( - parentSessionId: string, - forkSessionIdValue: string, - forkId: string, - meta: Record = {} - ): DeepChatTapeEntryRow { - const table = this.table - return table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/discard', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:external-discard:event`, - data: { - forkId, - forkSessionId: forkSessionIdValue, - ...meta - }, - idempotent: true - }) - } - - linkSubagentTape(input: SubagentTapeLinkInput): SubagentTapeLinkReceipt { - const table = this.table - const normalized = normalizeSubagentTapeLinkInput(input) - const provenanceKey = subagentTapeLinkProvenanceKey(normalized) - return table.runInTransaction(() => { - const existing = table.getByProvenanceKey(normalized.parentSessionId, provenanceKey) - if (existing) { - assertSubagentTapeLinkMatchesInput(existing, normalized) - const receipt = toSubagentTapeLinkReceipt(existing) - return receipt - } - - const childFirstEntry = table.getFirstEntriesBySessions([normalized.childSessionId])[0] - if (!childFirstEntry || childFirstEntry.session_id !== normalized.childSessionId) { - throw new Error(`Subagent Tape ${normalized.childSessionId} is unavailable.`) - } - const childTapeIdentity = computeTapeIdentity(childFirstEntry) - const childHeadEntryId = table.getMaxEntryId(normalized.childSessionId) - const childEntryCount = table.countBySession(normalized.childSessionId) - const row = table.appendEvent({ - sessionId: normalized.parentSessionId, - name: SUBAGENT_TAPE_LINK_EVENT_NAME, - source: { - type: 'subagent', - id: normalized.childSessionId, - seq: childHeadEntryId - }, - provenanceKey, - data: { - linkVersion: SUBAGENT_TAPE_LINK_VERSION, - childSessionId: normalized.childSessionId, - childHeadEntryId, - childEntryCount, - childTapeIdentity, - runId: normalized.runId, - taskId: normalized.taskId, - slotId: normalized.slotId, - taskTitle: normalized.taskTitle, - outcome: normalized.outcome, - resultSummary: normalized.resultSummary - }, - idempotent: true - }) - assertSubagentTapeLinkMatchesInput(row, normalized) - const receipt = toSubagentTapeLinkReceipt(row) - return receipt - }) - } - - private backfillLegacySummaryAnchor( - sessionId: string, - historyRecords: ChatMessageRecord[] - ): void { - const table = this.table - if (table.getLatestSummaryAnchor(sessionId)) { - return - } - - const legacyState = this.database.deepchatSessionsTable.getSummaryState(sessionId) - if (!legacyState) { - return - } - - const summary = legacyState.summary_text?.trim() - if (!summary) { - return - } - - const cursorOrderSeq = Math.max(1, legacyState.summary_cursor_order_seq ?? 1) - const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) - table.appendAnchor({ - sessionId, - name: 'compaction/migrated_summary', - source: { - type: 'summary', - id: 'legacy-summary', - seq: 1 - }, - provenanceKey: legacySummaryProvenanceKey(sessionId), - state: { - summary, - cursorOrderSeq, - range: - sourceRecords.length > 0 - ? { - fromOrderSeq: sourceRecords[0].orderSeq, - toOrderSeq: sourceRecords[sourceRecords.length - 1].orderSeq - } - : null, - sourceMessageIds: sourceRecords.map((record) => record.id), - migratedFrom: 'deepchat_sessions.summary_text' - }, - idempotent: true, - createdAt: legacyState.summary_updated_at ?? undefined - }) - } - - private ensureSearchProjection( - sessionId: string, - rows: DeepChatTapeEntryRow[], - effectiveRows: DeepChatTapeEntryRow[] - ): SessionDatabase['deepchatTapeSearchProjectionTable'] | null { - const projectionTable = this.searchProjectionTable - const maxEntryId = rows.reduce((max, row) => Math.max(max, row.entry_id), 0) - try { - if (!projectionTable.isCurrent(sessionId, maxEntryId)) { - const meta = projectionTable.getSessionMeta(sessionId) - const projectedEntryIds = projectionTable.getProjectedEntryIds(sessionId) - const effectiveEntryIds = effectiveRows.map((row) => row.entry_id) - const canAppend = - !!meta && - projectionTable.isCurrent(sessionId, meta.maxEntryId) && - meta.maxEntryId <= maxEntryId && - isEntryIdPrefix(projectedEntryIds, effectiveEntryIds) - if (canAppend) { - projectionTable.appendSession( - sessionId, - effectiveRows.slice(projectedEntryIds.length).map((row) => this.toProjectionInput(row)), - maxEntryId - ) - } else { - projectionTable.replaceSession( - sessionId, - effectiveRows.map((row) => this.toProjectionInput(row)), - maxEntryId - ) - } - } - return projectionTable - } catch { - return null - } - } - - private toProjectionInput(row: DeepChatTapeEntryRow): DeepChatTapeSearchProjectionInput { - const payload = parseJsonObject(row.payload_json) - const meta = parseJsonObject(row.meta_json) - const userMessage = getUserMessageProjectionText(row, payload) - const summaryText = summarizeTapeRow(row, payload, userMessage) - const evidenceText = buildTapeRowEvidenceText(row, payload, meta, userMessage) - const refs = buildTapeRowRefs(row, payload, meta, userMessage, evidenceText) - const searchText = [ - row.kind, - row.name ?? '', - summaryText, - evidenceText, - Object.values(refs) - .map((value) => stringifyForSummary(value)) - .join(' ') - ] - .filter(Boolean) - .join('\n') - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - sourceType: row.source_type, - sourceId: row.source_id, - sourceSeq: row.source_seq, - searchText, - summaryText, - refs, - createdAt: row.created_at - } - } - - private toProjectedSearchResult( - row: DeepChatTapeSearchProjectionResultRow, - _sourceRow: DeepChatTapeEntryRow | undefined - ): TapeSearchResult { - const score = - typeof row.score === 'number' && Number.isFinite(row.score) ? row.score : undefined - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - createdAt: row.created_at, - summary: row.summary_text, - refs: parseProjectionRefs(row.refs_json), - ...(score === undefined ? {} : { score }) - } - } - - private toContextEntry( - row: DeepChatTapeEntryRow, - projectionRow: DeepChatTapeSearchProjectionRow | undefined, - maxBytes: number - ): AgentTapeContextEntry { - const fallbackProjection = projectionRow ? null : this.toProjectionInput(row) - const payload = parseJsonObject(row.payload_json) - const meta = parseJsonObject(row.meta_json) - const evidenceSource = buildTapeRowEvidenceText(row, payload, meta) - const clipped = truncateToUtf8Bytes(evidenceSource, maxBytes) - const bytes = Buffer.byteLength(clipped.text, 'utf8') - return { - entryId: row.entry_id, - kind: row.kind, - name: row.name, - summary: projectionRow?.summary_text ?? fallbackProjection?.summaryText ?? '', - refs: projectionRow - ? parseProjectionRefs(projectionRow.refs_json) - : (fallbackProjection?.refs ?? {}), - evidence: { - text: clipped.text, - truncated: clipped.truncated, - bytes - }, - createdAt: row.created_at - } - } - - private toSearchResult(row: DeepChatTapeEntryRow): TapeSearchResult { - const projection = this.toProjectionInput(row) - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - createdAt: row.created_at, - summary: projection.summaryText, - refs: projection.refs - } - } - - private toAnchorResult(row: DeepChatTapeEntryRow): TapeAnchorResult { - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - payload: parseJsonObject(row.payload_json), - meta: parseJsonObject(row.meta_json), - createdAt: row.created_at - } - } - - private toViewManifestRecord(row: DeepChatTapeEntryRow): DeepChatTapeViewManifestRecord | null { - const payload = parseJsonObject(row.payload_json) - const data = payload.data - const rawManifest = - data && typeof data === 'object' && !Array.isArray(data) - ? (data as Record).manifest - : undefined - const manifest = - isRecordObject(rawManifest) && rawManifest.hashVersion === undefined - ? { ...rawManifest, hashVersion: 1 } - : rawManifest - if (!isViewManifest(manifest, row.session_id)) { - return null - } - - return { - sessionId: row.session_id, - messageId: manifest.messageId, - requestSeq: manifest.requestSeq, - entryId: row.entry_id, - createdAt: row.created_at, - integrity: verifyTapeViewManifestHash(manifest), - manifest - } - } - - private findReplayTrace( - sessionId: string, - messageId: string, - requestSeq: number - ): DeepChatMessageTraceRow | null { - const traceTable = this.database.deepchatMessageTracesTable - return ( - traceTable - .listByMessageId(messageId) - .find((row) => row.session_id === sessionId && row.request_seq === requestSeq) ?? null - ) - } - - private toReplayEntrySnapshot( - row: DeepChatTapeEntryRow, - includePayloads: boolean - ): DeepChatTapeReplayEntrySnapshot { - const snapshot: DeepChatTapeReplayEntrySnapshot = { - entryId: row.entry_id, - kind: row.kind, - name: row.name, - sourceType: row.source_type, - sourceId: row.source_id, - sourceSeq: row.source_seq, - provenanceKey: row.provenance_key, - payloadHash: hashString(row.payload_json), - metaHash: hashString(row.meta_json), - createdAt: row.created_at - } - - if (includePayloads) { - snapshot.payload = parseJsonObject(row.payload_json) - snapshot.meta = parseJsonObject(row.meta_json) - } - - return snapshot - } - - private toReplayTraceSnapshot( - row: DeepChatMessageTraceRow, - includePayload: boolean - ): DeepChatTapeReplayTraceSnapshot { - const snapshot: DeepChatTapeReplayTraceSnapshot = { - id: row.id, - requestSeq: row.request_seq, - providerId: row.provider_id, - modelId: row.model_id, - endpoint: row.endpoint, - headersHash: hashString(row.headers_json), - bodyHash: hashString(row.body_json), - truncated: row.truncated === 1, - createdAt: row.created_at - } - - if (includePayload) { - snapshot.headersJson = row.headers_json - snapshot.bodyJson = row.body_json - } - - return snapshot - } -} +export * from '@/tape/application/sessionTape' diff --git a/src/main/session/data/tapeFacts.ts b/src/main/session/data/tapeFacts.ts index 186ccde9d2..1b31870d9a 100644 --- a/src/main/session/data/tapeFacts.ts +++ b/src/main/session/data/tapeFacts.ts @@ -1,352 +1 @@ -import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' -import { toTapeSessionId, type TapeFactSource, type TapeToolFactInput } from '@/tape/domain/facts' -import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' -import type { TapeBootstrapStore, TapeEntryStore } from '@/tape/ports/storage' -import { buildEffectiveTapeView } from '@/tape/domain/effectiveView' -import { parseAssistantBlocks } from '@/tape/domain/effectiveSemantics' -import { hashJson } from '@/tape/domain/viewManifest' - -export { tapeEntryToMessageRecord } from '@/tape/domain/effectiveSemantics' -export type { TapeFactSource } from '@/tape/domain/facts' - -type TapeFactStore = Pick & TapeBootstrapStore - -function readCompactionStatus(record: ChatMessageRecord): string | null { - try { - const parsed = JSON.parse(record.metadata) as { - messageType?: string - compactionStatus?: unknown - } - if (parsed.messageType !== 'compaction') { - return null - } - return typeof parsed.compactionStatus === 'string' ? parsed.compactionStatus : record.status - } catch { - return null - } -} - -function shouldUseRevisionProvenance(record: ChatMessageRecord, source: TapeFactSource): boolean { - return source === 'repair' || record.status !== 'sent' -} - -function buildMessageProvenanceKey( - record: ChatMessageRecord, - source: TapeFactSource -): string | undefined { - if (!shouldUseRevisionProvenance(record, source)) { - return undefined - } - return `message:${record.id}:revision:${record.status}:${record.updatedAt}` -} - -function buildToolFactProvenanceKey( - kind: 'tool_call' | 'tool_result', - messageId: string, - toolCallId: string, - payload: Record -): string { - return `${kind}:${messageId}:${toolCallId}:${hashJson(payload)}` -} - -function collectPendingInteractionToolIds(blocks: AssistantMessageBlock[]): Set { - const ids = new Set() - for (const block of blocks) { - if ( - block.type === 'action' && - (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && - block.status === 'pending' && - typeof block.tool_call?.id === 'string' && - block.tool_call.id.length > 0 - ) { - ids.add(block.tool_call.id) - } - } - return ids -} - -export function buildTapeToolFactInputs(record: ChatMessageRecord): TapeToolFactInput[] { - if (record.role !== 'assistant') return [] - - const inputs: TapeToolFactInput[] = [] - const blocks = parseAssistantBlocks(record.content) - const pendingInteractionToolIds = collectPendingInteractionToolIds(blocks) - blocks.forEach((block, blockIndex) => { - if (block.type !== 'tool_call' || !block.tool_call) return - if (block.status !== 'success' && block.status !== 'error') return - - const toolCallId = block.tool_call.id - if (!toolCallId || pendingInteractionToolIds.has(toolCallId)) return - const sourceId = `${record.id}:${toolCallId}` - const factBlock = - block.timestamp === undefined ? { ...block, timestamp: record.updatedAt } : block - inputs.push({ - sessionId: toTapeSessionId(record.sessionId), - messageId: record.id, - orderSeq: record.orderSeq, - blockIndex, - block: factBlock, - provenance: { source: 'tool_call', sourceId, sequence: blockIndex } - }) - if (typeof block.tool_call.response === 'string' && block.tool_call.response.length > 0) { - inputs.push({ - sessionId: toTapeSessionId(record.sessionId), - messageId: record.id, - orderSeq: record.orderSeq, - blockIndex, - block: factBlock, - provenance: { source: 'tool_result', sourceId, sequence: blockIndex } - }) - } - }) - return inputs -} - -export function appendTapeToolFact( - table: TapeFactStore, - input: TapeToolFactInput, - source: TapeFactSource, - reason?: string -): DeepChatTapeEntryRow | null { - const block = input.block - const toolCall = block.type === 'tool_call' ? block.tool_call : undefined - if ( - !toolCall?.id || - (block.status !== 'success' && block.status !== 'error') || - (input.provenance.source !== 'tool_call' && input.provenance.source !== 'tool_result') - ) { - return null - } - - table.ensureBootstrapAnchor(input.sessionId) - const meta = reason - ? { source, role: 'assistant', status: block.status, reason } - : { source, role: 'assistant', status: block.status } - if (input.provenance.source === 'tool_call') { - const payload = { - messageId: input.messageId, - orderSeq: input.orderSeq, - toolCall: { - id: toolCall.id, - name: toolCall.name, - params: toolCall.params, - serverName: toolCall.server_name, - serverIcons: toolCall.server_icons, - serverDescription: toolCall.server_description - } - } - return table.append({ - sessionId: input.sessionId, - kind: 'tool_call', - name: toolCall.name || 'unknown', - source: { - type: input.provenance.source, - id: input.provenance.sourceId, - seq: input.provenance.sequence - }, - provenanceKey: buildToolFactProvenanceKey('tool_call', input.messageId, toolCall.id, payload), - payload, - meta, - createdAt: block.timestamp, - idempotent: true - }) - } - - if (typeof toolCall.response !== 'string' || toolCall.response.length === 0) return null - const payload = { - messageId: input.messageId, - orderSeq: input.orderSeq, - toolCallId: toolCall.id, - response: toolCall.response, - rtkApplied: toolCall.rtkApplied, - rtkMode: toolCall.rtkMode, - rtkFallbackReason: toolCall.rtkFallbackReason, - imagePreviews: toolCall.imagePreviews - } - return table.append({ - sessionId: input.sessionId, - kind: 'tool_result', - name: toolCall.name || 'unknown', - source: { - type: input.provenance.source, - id: input.provenance.sourceId, - seq: input.provenance.sequence - }, - provenanceKey: buildToolFactProvenanceKey('tool_result', input.messageId, toolCall.id, payload), - payload, - meta, - createdAt: block.timestamp, - idempotent: true - }) -} - -export function appendToolFactsToTape( - table: TapeFactStore, - record: ChatMessageRecord, - source: TapeFactSource, - reason?: string -): number { - if (record.role !== 'assistant') { - return 0 - } - - return buildTapeToolFactInputs(record).reduce( - (appended, input) => appended + (appendTapeToolFact(table, input, source, reason) ? 1 : 0), - 0 - ) -} - -export function appendMessageRecordToTape( - table: TapeFactStore, - record: ChatMessageRecord, - source: TapeFactSource -): number { - table.ensureBootstrapAnchor(record.sessionId) - - const compactionStatus = readCompactionStatus(record) - if (compactionStatus) { - table.appendEvent({ - sessionId: record.sessionId, - name: 'message/compaction_indicator', - source: { - type: 'message', - id: record.id, - seq: record.updatedAt - }, - provenanceKey: `message:${record.id}:compaction_indicator:${compactionStatus}:${record.updatedAt}`, - data: { - messageId: record.id, - orderSeq: record.orderSeq, - status: compactionStatus, - metadata: record.metadata - }, - meta: { - source, - status: compactionStatus - }, - createdAt: record.updatedAt, - idempotent: true - }) - return 1 - } - - table.append({ - sessionId: record.sessionId, - kind: 'message', - name: `message/${record.role}`, - source: { - type: 'message', - id: record.id, - seq: 0 - }, - provenanceKey: buildMessageProvenanceKey(record, source), - payload: { - record: { - id: record.id, - sessionId: record.sessionId, - orderSeq: record.orderSeq, - role: record.role, - content: record.content, - status: record.status, - isContextEdge: record.isContextEdge, - metadata: record.metadata, - traceCount: record.traceCount, - createdAt: record.createdAt, - updatedAt: record.updatedAt - } - }, - meta: { - source, - orderSeq: record.orderSeq, - role: record.role, - status: record.status - }, - createdAt: record.createdAt, - idempotent: true - }) - - return 1 + appendToolFactsToTape(table, record, source) -} - -export function appendMessageReplacementToTape( - table: TapeFactStore, - record: ChatMessageRecord, - reason: string -): number { - table.ensureBootstrapAnchor(record.sessionId) - table.append({ - sessionId: record.sessionId, - kind: 'message', - name: `message/${record.role}`, - source: { - type: 'message', - id: record.id, - seq: record.updatedAt - }, - provenanceKey: `message:${record.id}:revision:${record.updatedAt}`, - payload: { - record: { - id: record.id, - sessionId: record.sessionId, - orderSeq: record.orderSeq, - role: record.role, - content: record.content, - status: record.status, - isContextEdge: record.isContextEdge, - metadata: record.metadata, - traceCount: record.traceCount, - createdAt: record.createdAt, - updatedAt: record.updatedAt - } - }, - meta: { - source: 'live', - correction: true, - reason, - orderSeq: record.orderSeq, - role: record.role, - status: record.status - }, - createdAt: record.updatedAt, - idempotent: true - }) - - return 1 + appendToolFactsToTape(table, record, 'repair') -} - -export function appendMessageRetractionToTape( - table: TapeFactStore, - record: ChatMessageRecord, - reason: string -): number { - table.ensureBootstrapAnchor(record.sessionId) - table.appendEvent({ - sessionId: record.sessionId, - name: 'message/retracted', - source: { - type: 'message', - id: record.id, - seq: Date.now() - }, - provenanceKey: null, - data: { - messageId: record.id, - orderSeq: record.orderSeq, - role: record.role, - reason - }, - meta: { - source: 'live', - correction: true - }, - idempotent: false - }) - - return 1 -} - -export function tapeEntriesToEffectiveMessageRecords( - rows: DeepChatTapeEntryRow[] -): ChatMessageRecord[] { - return buildEffectiveTapeView(rows, { includePending: true }).messageRecords -} +export * from '@/tape/application/factPersistence' diff --git a/src/main/tape/application/common.ts b/src/main/tape/application/common.ts new file mode 100644 index 0000000000..13c6bbcd5c --- /dev/null +++ b/src/main/tape/application/common.ts @@ -0,0 +1,64 @@ +import { createHash } from 'crypto' +import type { DeepChatTapeReplaySlice } from '@shared/types/tape-replay' + +export function parseJsonObject(raw: string): Record { + try { + const parsed = JSON.parse(raw) as unknown + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record + } + } catch {} + return {} +} + +export function parseJsonValue(raw: string): unknown { + try { + return JSON.parse(raw) as unknown + } catch { + return null + } +} + +export function hashString(value: string): string { + return createHash('sha256').update(value).digest('hex') +} + +export function isPositiveInteger(value: number): boolean { + return Number.isInteger(value) && value > 0 +} + +export function collectEntryIds(values: Array): number[] { + return [...new Set(values.filter((value): value is number => typeof value === 'number'))].sort( + (left, right) => left - right + ) +} + +export function isEntryIdPrefix(prefix: number[], values: number[]): boolean { + if (prefix.length > values.length) return false + for (let index = 0; index < prefix.length; index += 1) { + if (prefix[index] !== values[index]) return false + } + return true +} + +export function migrationProvenanceKey(sessionId: string): string { + return `migration:${sessionId}:message-backfill:v1` +} + +export function withReplaySliceHash( + slice: Omit & { + hashes: Omit & { sliceHash: '' } + }, + hashJson: (value: unknown) => string +): DeepChatTapeReplaySlice { + const sliceForHash = { ...slice } as Partial + delete sliceForHash.createdAt + delete sliceForHash.integrity + return { + ...slice, + hashes: { + ...slice.hashes, + sliceHash: hashJson(sliceForHash) + } + } +} diff --git a/src/main/tape/application/contracts.ts b/src/main/tape/application/contracts.ts new file mode 100644 index 0000000000..504fba28ba --- /dev/null +++ b/src/main/tape/application/contracts.ts @@ -0,0 +1,53 @@ +import type { AgentTapeAnchorResult, ChatMessageRecord } from '@shared/types/agent-interface' + +export type TapeMigrationState = 'none' | 'ready' + +export type TapeBackfillResult = { + sessionId: string + migrationState: TapeMigrationState + messageCount: number + maxOrderSeq: number + appendedFactCount: number + historyRecords: ChatMessageRecord[] +} + +export type TapeInfo = { + sessionId: string + entries: number + anchors: number + lastAnchor: string | null + lastAnchorEntryId: number | null + entriesSinceLastAnchor: number + lastTokenUsage: number | null + migrationState: TapeMigrationState +} + +export type TapeSearchResult = { + sessionId: string + entryId: number + kind: string + name: string | null + createdAt: number + summary?: string + refs?: Record + score?: number +} + +export type TapeAnchorResult = AgentTapeAnchorResult + +export type TapeForkHandle = { + parentSessionId: string + forkId: string + forkSessionId: string + parentHeadEntryId: number +} + +export type TapeViewManifestSourceMaps = { + latestEntryId: number + anchorEntryIds: number[] + reconstructionAnchorEntryIds: number[] + reconstructionAnchorEntryId: number | null + entryIdByMessageId: Map + toolCallEntryIdByToolId: Map + toolResultEntryIdByToolId: Map +} diff --git a/src/main/tape/application/factPersistence.ts b/src/main/tape/application/factPersistence.ts new file mode 100644 index 0000000000..186ccde9d2 --- /dev/null +++ b/src/main/tape/application/factPersistence.ts @@ -0,0 +1,352 @@ +import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' +import { toTapeSessionId, type TapeFactSource, type TapeToolFactInput } from '@/tape/domain/facts' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeBootstrapStore, TapeEntryStore } from '@/tape/ports/storage' +import { buildEffectiveTapeView } from '@/tape/domain/effectiveView' +import { parseAssistantBlocks } from '@/tape/domain/effectiveSemantics' +import { hashJson } from '@/tape/domain/viewManifest' + +export { tapeEntryToMessageRecord } from '@/tape/domain/effectiveSemantics' +export type { TapeFactSource } from '@/tape/domain/facts' + +type TapeFactStore = Pick & TapeBootstrapStore + +function readCompactionStatus(record: ChatMessageRecord): string | null { + try { + const parsed = JSON.parse(record.metadata) as { + messageType?: string + compactionStatus?: unknown + } + if (parsed.messageType !== 'compaction') { + return null + } + return typeof parsed.compactionStatus === 'string' ? parsed.compactionStatus : record.status + } catch { + return null + } +} + +function shouldUseRevisionProvenance(record: ChatMessageRecord, source: TapeFactSource): boolean { + return source === 'repair' || record.status !== 'sent' +} + +function buildMessageProvenanceKey( + record: ChatMessageRecord, + source: TapeFactSource +): string | undefined { + if (!shouldUseRevisionProvenance(record, source)) { + return undefined + } + return `message:${record.id}:revision:${record.status}:${record.updatedAt}` +} + +function buildToolFactProvenanceKey( + kind: 'tool_call' | 'tool_result', + messageId: string, + toolCallId: string, + payload: Record +): string { + return `${kind}:${messageId}:${toolCallId}:${hashJson(payload)}` +} + +function collectPendingInteractionToolIds(blocks: AssistantMessageBlock[]): Set { + const ids = new Set() + for (const block of blocks) { + if ( + block.type === 'action' && + (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && + block.status === 'pending' && + typeof block.tool_call?.id === 'string' && + block.tool_call.id.length > 0 + ) { + ids.add(block.tool_call.id) + } + } + return ids +} + +export function buildTapeToolFactInputs(record: ChatMessageRecord): TapeToolFactInput[] { + if (record.role !== 'assistant') return [] + + const inputs: TapeToolFactInput[] = [] + const blocks = parseAssistantBlocks(record.content) + const pendingInteractionToolIds = collectPendingInteractionToolIds(blocks) + blocks.forEach((block, blockIndex) => { + if (block.type !== 'tool_call' || !block.tool_call) return + if (block.status !== 'success' && block.status !== 'error') return + + const toolCallId = block.tool_call.id + if (!toolCallId || pendingInteractionToolIds.has(toolCallId)) return + const sourceId = `${record.id}:${toolCallId}` + const factBlock = + block.timestamp === undefined ? { ...block, timestamp: record.updatedAt } : block + inputs.push({ + sessionId: toTapeSessionId(record.sessionId), + messageId: record.id, + orderSeq: record.orderSeq, + blockIndex, + block: factBlock, + provenance: { source: 'tool_call', sourceId, sequence: blockIndex } + }) + if (typeof block.tool_call.response === 'string' && block.tool_call.response.length > 0) { + inputs.push({ + sessionId: toTapeSessionId(record.sessionId), + messageId: record.id, + orderSeq: record.orderSeq, + blockIndex, + block: factBlock, + provenance: { source: 'tool_result', sourceId, sequence: blockIndex } + }) + } + }) + return inputs +} + +export function appendTapeToolFact( + table: TapeFactStore, + input: TapeToolFactInput, + source: TapeFactSource, + reason?: string +): DeepChatTapeEntryRow | null { + const block = input.block + const toolCall = block.type === 'tool_call' ? block.tool_call : undefined + if ( + !toolCall?.id || + (block.status !== 'success' && block.status !== 'error') || + (input.provenance.source !== 'tool_call' && input.provenance.source !== 'tool_result') + ) { + return null + } + + table.ensureBootstrapAnchor(input.sessionId) + const meta = reason + ? { source, role: 'assistant', status: block.status, reason } + : { source, role: 'assistant', status: block.status } + if (input.provenance.source === 'tool_call') { + const payload = { + messageId: input.messageId, + orderSeq: input.orderSeq, + toolCall: { + id: toolCall.id, + name: toolCall.name, + params: toolCall.params, + serverName: toolCall.server_name, + serverIcons: toolCall.server_icons, + serverDescription: toolCall.server_description + } + } + return table.append({ + sessionId: input.sessionId, + kind: 'tool_call', + name: toolCall.name || 'unknown', + source: { + type: input.provenance.source, + id: input.provenance.sourceId, + seq: input.provenance.sequence + }, + provenanceKey: buildToolFactProvenanceKey('tool_call', input.messageId, toolCall.id, payload), + payload, + meta, + createdAt: block.timestamp, + idempotent: true + }) + } + + if (typeof toolCall.response !== 'string' || toolCall.response.length === 0) return null + const payload = { + messageId: input.messageId, + orderSeq: input.orderSeq, + toolCallId: toolCall.id, + response: toolCall.response, + rtkApplied: toolCall.rtkApplied, + rtkMode: toolCall.rtkMode, + rtkFallbackReason: toolCall.rtkFallbackReason, + imagePreviews: toolCall.imagePreviews + } + return table.append({ + sessionId: input.sessionId, + kind: 'tool_result', + name: toolCall.name || 'unknown', + source: { + type: input.provenance.source, + id: input.provenance.sourceId, + seq: input.provenance.sequence + }, + provenanceKey: buildToolFactProvenanceKey('tool_result', input.messageId, toolCall.id, payload), + payload, + meta, + createdAt: block.timestamp, + idempotent: true + }) +} + +export function appendToolFactsToTape( + table: TapeFactStore, + record: ChatMessageRecord, + source: TapeFactSource, + reason?: string +): number { + if (record.role !== 'assistant') { + return 0 + } + + return buildTapeToolFactInputs(record).reduce( + (appended, input) => appended + (appendTapeToolFact(table, input, source, reason) ? 1 : 0), + 0 + ) +} + +export function appendMessageRecordToTape( + table: TapeFactStore, + record: ChatMessageRecord, + source: TapeFactSource +): number { + table.ensureBootstrapAnchor(record.sessionId) + + const compactionStatus = readCompactionStatus(record) + if (compactionStatus) { + table.appendEvent({ + sessionId: record.sessionId, + name: 'message/compaction_indicator', + source: { + type: 'message', + id: record.id, + seq: record.updatedAt + }, + provenanceKey: `message:${record.id}:compaction_indicator:${compactionStatus}:${record.updatedAt}`, + data: { + messageId: record.id, + orderSeq: record.orderSeq, + status: compactionStatus, + metadata: record.metadata + }, + meta: { + source, + status: compactionStatus + }, + createdAt: record.updatedAt, + idempotent: true + }) + return 1 + } + + table.append({ + sessionId: record.sessionId, + kind: 'message', + name: `message/${record.role}`, + source: { + type: 'message', + id: record.id, + seq: 0 + }, + provenanceKey: buildMessageProvenanceKey(record, source), + payload: { + record: { + id: record.id, + sessionId: record.sessionId, + orderSeq: record.orderSeq, + role: record.role, + content: record.content, + status: record.status, + isContextEdge: record.isContextEdge, + metadata: record.metadata, + traceCount: record.traceCount, + createdAt: record.createdAt, + updatedAt: record.updatedAt + } + }, + meta: { + source, + orderSeq: record.orderSeq, + role: record.role, + status: record.status + }, + createdAt: record.createdAt, + idempotent: true + }) + + return 1 + appendToolFactsToTape(table, record, source) +} + +export function appendMessageReplacementToTape( + table: TapeFactStore, + record: ChatMessageRecord, + reason: string +): number { + table.ensureBootstrapAnchor(record.sessionId) + table.append({ + sessionId: record.sessionId, + kind: 'message', + name: `message/${record.role}`, + source: { + type: 'message', + id: record.id, + seq: record.updatedAt + }, + provenanceKey: `message:${record.id}:revision:${record.updatedAt}`, + payload: { + record: { + id: record.id, + sessionId: record.sessionId, + orderSeq: record.orderSeq, + role: record.role, + content: record.content, + status: record.status, + isContextEdge: record.isContextEdge, + metadata: record.metadata, + traceCount: record.traceCount, + createdAt: record.createdAt, + updatedAt: record.updatedAt + } + }, + meta: { + source: 'live', + correction: true, + reason, + orderSeq: record.orderSeq, + role: record.role, + status: record.status + }, + createdAt: record.updatedAt, + idempotent: true + }) + + return 1 + appendToolFactsToTape(table, record, 'repair') +} + +export function appendMessageRetractionToTape( + table: TapeFactStore, + record: ChatMessageRecord, + reason: string +): number { + table.ensureBootstrapAnchor(record.sessionId) + table.appendEvent({ + sessionId: record.sessionId, + name: 'message/retracted', + source: { + type: 'message', + id: record.id, + seq: Date.now() + }, + provenanceKey: null, + data: { + messageId: record.id, + orderSeq: record.orderSeq, + role: record.role, + reason + }, + meta: { + source: 'live', + correction: true + }, + idempotent: false + }) + + return 1 +} + +export function tapeEntriesToEffectiveMessageRecords( + rows: DeepChatTapeEntryRow[] +): ChatMessageRecord[] { + return buildEffectiveTapeView(rows, { includePending: true }).messageRecords +} diff --git a/src/main/tape/application/factService.ts b/src/main/tape/application/factService.ts new file mode 100644 index 0000000000..05e782ff2a --- /dev/null +++ b/src/main/tape/application/factService.ts @@ -0,0 +1,44 @@ +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' +import { buildEffectiveTapeView } from '../domain/effectiveView' +import type { TapeToolFactWriter, TapeMessageFactWriter } from '../ports/capabilities' +import type { TapeApplicationProviders } from '../ports/application' +import { + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + appendTapeToolFact +} from './factPersistence' + +type TapeFactProviders = Pick + +export class TapeFactService implements TapeToolFactWriter, TapeMessageFactWriter { + constructor(private readonly providers: TapeFactProviders) {} + + private get table() { + return this.providers.getEntryStore() + } + + appendMessageRecord(record: ChatMessageRecord): number { + return appendMessageRecordToTape(this.table, record, 'live') + } + + appendMessageReplacement(record: ChatMessageRecord, reason: string): number { + return appendMessageReplacementToTape(this.table, record, reason) + } + + appendMessageRetraction(record: ChatMessageRecord, reason: string): number { + return appendMessageRetractionToTape(this.table, record, reason) + } + + async appendToolFact(input: TapeToolFactInput): Promise { + const row = appendTapeToolFact(this.table, input, 'live', 'tool_loop') + if (!row) throw new Error('Tape tool fact was not appendable.') + return { sessionId: input.sessionId, entryId: row.entry_id } + } + + getMessageRecords(sessionId: string): ChatMessageRecord[] { + return buildEffectiveTapeView(this.table.getBySession(sessionId), { includePending: true }) + .messageRecords + } +} diff --git a/src/main/tape/application/forkService.ts b/src/main/tape/application/forkService.ts new file mode 100644 index 0000000000..1b71f29f2e --- /dev/null +++ b/src/main/tape/application/forkService.ts @@ -0,0 +1,444 @@ +import { nanoid } from 'nanoid' +import logger from 'electron-log' +import type { AgentTapeHandoffState, ChatMessageRecord } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryRow } from '../domain/entry' +import type { TapeApplicationProviders } from '../ports/application' +import { appendMessageRecordToTape } from './factPersistence' +import { parseJsonObject } from './common' +import type { TapeAnchorResult, TapeForkHandle } from './contracts' +import type { TapeFactService } from './factService' + +type TapeForkProviders = Pick + +function readForkMergeReceiptCount( + row: DeepChatTapeEntryRow, + parentSessionId: string, + forkId: string, + forkSessionIdValue: string +): number { + const payload = parseJsonObject(row.payload_json) + const data = + payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) + ? (payload.data as Record) + : {} + const mergedCount = data.mergedCount + const forkHeadEntryId = data.forkHeadEntryId + const hasValidLegacyOrCurrentHead = + forkHeadEntryId === undefined || + (typeof forkHeadEntryId === 'number' && + Number.isSafeInteger(forkHeadEntryId) && + forkHeadEntryId >= 0) + if ( + row.session_id !== parentSessionId || + row.kind !== 'event' || + row.name !== 'fork/merge' || + row.source_type !== 'fork' || + row.source_id !== forkId || + row.source_seq !== 0 || + row.provenance_key !== `fork:${parentSessionId}:${forkId}:merge:event` || + data.forkId !== forkId || + data.forkSessionId !== forkSessionIdValue || + typeof mergedCount !== 'number' || + !Number.isSafeInteger(mergedCount) || + mergedCount < 0 || + !hasValidLegacyOrCurrentHead || + (typeof forkHeadEntryId === 'number' && mergedCount > forkHeadEntryId) + ) { + throw new Error(`Stored fork merge receipt is malformed: ${row.entry_id}`) + } + return mergedCount +} + +function assertValidForkStart( + row: DeepChatTapeEntryRow | undefined, + parentSessionId: string, + forkId: string, + forkSessionIdValue: string +): void { + if (!row) { + throw new Error(`Fork ${forkId} does not exist or has been discarded.`) + } + const payload = parseJsonObject(row.payload_json) + const state = + payload.state && typeof payload.state === 'object' && !Array.isArray(payload.state) + ? (payload.state as Record) + : {} + const parentHeadEntryId = state.parentHeadEntryId + const hasValidLegacyOrCurrentHead = + parentHeadEntryId === undefined || + (typeof parentHeadEntryId === 'number' && + Number.isSafeInteger(parentHeadEntryId) && + parentHeadEntryId >= 0) + if ( + row.session_id !== forkSessionIdValue || + row.kind !== 'anchor' || + row.name !== 'fork/start' || + row.source_type !== 'fork' || + row.source_id !== forkId || + row.source_seq !== 0 || + row.provenance_key !== `fork:${parentSessionId}:${forkId}:start` || + state.parentSessionId !== parentSessionId || + !hasValidLegacyOrCurrentHead + ) { + throw new Error(`Stored fork start is malformed: ${row.entry_id}`) + } +} + +function normalizeHandoffName(name: string): string { + const trimmed = name.trim() + if (!trimmed) { + return 'handoff/manual' + } + if (trimmed.startsWith('handoff/') || trimmed.startsWith('auto_handoff/')) { + return trimmed + } + return `handoff/${trimmed}` +} + +function normalizePositiveInteger(value: unknown): number | null { + if (typeof value === 'number' && Number.isFinite(value)) { + return Math.max(1, Math.floor(value)) + } + return null +} + +function hasOwnKey(value: Record, key: string): boolean { + return Object.prototype.hasOwnProperty.call(value, key) +} + +function buildOrderSeqRange(records: ChatMessageRecord[]): Record | null { + if (records.length === 0) { + return null + } + + return { + fromOrderSeq: records[0].orderSeq, + toOrderSeq: records[records.length - 1].orderSeq + } +} + +function enrichHandoffState( + state: Record, + historyRecords: ChatMessageRecord[] +): Record { + const maxOrderSeq = historyRecords.reduce( + (currentMax, record) => Math.max(currentMax, record.orderSeq), + 0 + ) + const cursorOrderSeq = + normalizePositiveInteger(state.cursorOrderSeq ?? state.summaryCursorOrderSeq) ?? maxOrderSeq + 1 + const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) + const enrichedState: Record = { + ...state, + cursorOrderSeq + } + + if (!hasOwnKey(enrichedState, 'range')) { + enrichedState.range = buildOrderSeqRange(sourceRecords) + } + + const sourceMessageIds = enrichedState.sourceMessageIds + if (!Array.isArray(sourceMessageIds) || sourceMessageIds.some((id) => typeof id !== 'string')) { + enrichedState.sourceMessageIds = sourceRecords.map((record) => record.id) + } + + return enrichedState +} + +export function normalizeTapeHandoffState(state: unknown): AgentTapeHandoffState { + if (!state || typeof state !== 'object' || Array.isArray(state)) { + throw new Error('Tape handoff requires a non-empty summary.') + } + + const summary = (state as Record).summary + if (typeof summary !== 'string' || !summary.trim()) { + throw new Error('Tape handoff requires a non-empty summary.') + } + + return { + ...(state as Record), + summary: summary.trim() + } +} + +function forkSessionId(parentSessionId: string, forkId: string): string { + return `${parentSessionId}::fork::${forkId}` +} + +export class TapeForkService { + constructor( + private readonly providers: TapeForkProviders, + private readonly facts: TapeFactService + ) {} + + private get table() { + return this.providers.getForkStore() + } + + private get searchProjectionTable() { + return this.providers.getSearchProjectionStore() + } + + private toAnchorResult(row: DeepChatTapeEntryRow): TapeAnchorResult { + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + payload: parseJsonObject(row.payload_json), + meta: parseJsonObject(row.meta_json), + createdAt: row.created_at + } + } + + handoff( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): DeepChatTapeEntryRow { + const normalizedState = normalizeTapeHandoffState(state) + const table = this.table + table.ensureBootstrapAnchor(sessionId) + const handoffState = enrichHandoffState( + normalizedState, + this.facts.getMessageRecords(sessionId) + ) + return table.appendAnchor({ + sessionId, + name: normalizeHandoffName(name), + source: { + type: 'runtime_event', + id: `handoff:${Date.now()}`, + seq: 0 + }, + state: handoffState, + meta: { + ...meta, + handoff: true + } + }) + } + + handoffResult( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): TapeAnchorResult { + return this.toAnchorResult(this.handoff(sessionId, name, state, meta)) + } + + createFork(parentSessionId: string, forkId: string = nanoid()): TapeForkHandle { + const table = this.table + const forkIdValue = forkId.trim() || nanoid() + const forkSessionIdValue = forkSessionId(parentSessionId, forkIdValue) + const parentHeadEntryId = table.getMaxEntryId(parentSessionId) + table.ensureBootstrapAnchor(forkSessionIdValue) + const parentAnchor = table.getLatestAnchor(parentSessionId) + const forkStart = table.appendAnchor({ + sessionId: forkSessionIdValue, + name: 'fork/start', + source: { + type: 'fork', + id: forkIdValue, + seq: 0 + }, + provenanceKey: `fork:${parentSessionId}:${forkIdValue}:start`, + state: { + parentSessionId, + parentHeadEntryId, + parentLastAnchorEntryId: parentAnchor?.entry_id ?? null, + parentLastAnchorName: parentAnchor?.name ?? null + }, + idempotent: true + }) + const forkStartPayload = parseJsonObject(forkStart.payload_json) + const persistedState = + forkStartPayload.state && + typeof forkStartPayload.state === 'object' && + !Array.isArray(forkStartPayload.state) + ? (forkStartPayload.state as Record) + : {} + const persistedParentHeadEntryId = persistedState.parentHeadEntryId + return { + parentSessionId, + forkId: forkIdValue, + forkSessionId: forkSessionIdValue, + parentHeadEntryId: + typeof persistedParentHeadEntryId === 'number' && + Number.isSafeInteger(persistedParentHeadEntryId) && + persistedParentHeadEntryId >= 0 + ? persistedParentHeadEntryId + : parentHeadEntryId + } + } + + appendForkMessageRecord(handle: TapeForkHandle, record: ChatMessageRecord): number { + return appendMessageRecordToTape( + this.table, + { + ...record, + sessionId: handle.forkSessionId + }, + 'live' + ) + } + + mergeFork(parentSessionId: string, forkId: string): number { + const table = this.table + const forkSessionIdValue = forkSessionId(parentSessionId, forkId) + const mergeProvenanceKey = `fork:${parentSessionId}:${forkId}:merge:event` + + return table.runInTransaction(() => { + const existingReceipt = table.getByProvenanceKey(parentSessionId, mergeProvenanceKey) + if (existingReceipt) { + return readForkMergeReceiptCount( + existingReceipt, + parentSessionId, + forkId, + forkSessionIdValue + ) + } + + assertValidForkStart( + table.getByProvenanceKey(forkSessionIdValue, `fork:${parentSessionId}:${forkId}:start`), + parentSessionId, + forkId, + forkSessionIdValue + ) + + const forkHeadEntryId = table.getMaxEntryId(forkSessionIdValue) + const forkEntries = table + .getBySessionUpToEntryId(forkSessionIdValue, forkHeadEntryId) + .filter( + (entry) => + !( + entry.kind === 'anchor' && + (entry.name === 'session/start' || entry.name === 'fork/start') + ) + ) + + for (const entry of forkEntries) { + table.append({ + sessionId: parentSessionId, + kind: entry.kind, + name: entry.name, + source: { + type: 'fork', + id: forkId, + seq: entry.entry_id + }, + provenanceKey: `fork:${parentSessionId}:${forkId}:merge:${entry.entry_id}`, + payload: parseJsonObject(entry.payload_json), + meta: { + ...parseJsonObject(entry.meta_json), + forkId, + forkSessionId: forkSessionIdValue, + mergedFromEntryId: entry.entry_id + }, + createdAt: entry.created_at, + idempotent: true + }) + } + + table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/merge', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: mergeProvenanceKey, + data: { + forkId, + forkSessionId: forkSessionIdValue, + forkHeadEntryId, + mergedCount: forkEntries.length + }, + idempotent: true + }) + + return forkEntries.length + }) + } + + discardFork(parentSessionId: string, forkId: string): void { + const table = this.table + const forkSessionIdValue = forkSessionId(parentSessionId, forkId) + table.deleteBySession(forkSessionIdValue) + try { + this.searchProjectionTable.deleteBySession(forkSessionIdValue) + } catch (error) { + logger.warn(`[Tape] failed to delete fork search projection: ${String(error)}`) + } + table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/discard', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: `fork:${parentSessionId}:${forkId}:discard:event`, + data: { + forkId, + forkSessionId: forkSessionIdValue + }, + idempotent: true + }) + } + + recordExternalForkMerge( + parentSessionId: string, + forkSessionIdValue: string, + forkId: string, + meta: Record = {} + ): DeepChatTapeEntryRow { + const table = this.table + const referencedEntryCount = table.countBySession(forkSessionIdValue) + return table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/merge', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: `fork:${parentSessionId}:${forkId}:external-merge:event`, + data: { + forkId, + forkSessionId: forkSessionIdValue, + referencedEntryCount, + ...meta + }, + idempotent: true + }) + } + + recordExternalForkDiscard( + parentSessionId: string, + forkSessionIdValue: string, + forkId: string, + meta: Record = {} + ): DeepChatTapeEntryRow { + const table = this.table + return table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/discard', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: `fork:${parentSessionId}:${forkId}:external-discard:event`, + data: { + forkId, + forkSessionId: forkSessionIdValue, + ...meta + }, + idempotent: true + }) + } +} diff --git a/src/main/tape/application/lineageService.ts b/src/main/tape/application/lineageService.ts new file mode 100644 index 0000000000..65893fc7ca --- /dev/null +++ b/src/main/tape/application/lineageService.ts @@ -0,0 +1,438 @@ +import { createHash } from 'crypto' +import type { + SubagentTapeLinkInput, + SubagentTapeLinkOutcome, + SubagentTapeLinkReceipt +} from '@shared/types/agent-interface' +import { + TAPE_INCARNATION_META_KEY, + type DeepChatTapeEntryRow, + type DeepChatTapeReadSource +} from '../domain/entry' +import type { TapeApplicationProviders } from '../ports/application' +import { parseJsonObject, parseJsonValue } from './common' + +type TapeLineageProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getLineageSessionReader' +> + +function compactText(value: string, maxLength = 1000): string { + const normalized = value.replace(/\s+/g, ' ').trim() + if (normalized.length <= maxLength) return normalized + return `${normalized.slice(0, Math.max(0, maxLength - 3)).trimEnd()}...` +} + +const SUBAGENT_TAPE_LINK_EVENT_NAME = 'subagent/tape_linked' + +const SUBAGENT_TAPE_LINK_VERSION = 2 + +const TAPE_IDENTITY_PATTERN = /^[a-f0-9]{64}$/ + +const SUBAGENT_TAPE_LINK_OUTCOMES = new Set([ + 'completed', + 'error', + 'cancelled' +]) + +type SubagentTapeLinkSnapshot = { + linkEntryId: number + childSessionId: string + childHeadEntryId: number + childEntryCount: number + outcome: SubagentTapeLinkOutcome + childTapeIdentity: string | null +} + +type ParsedSubagentTapeLink = { + snapshot: SubagentTapeLinkSnapshot + frozenInput: SubagentTapeLinkInput +} + +export type LinkedTapeSourceResolution = { + sources: DeepChatTapeReadSource[] + unavailableSourceIds: Set +} + +export type AgentTapeViewErrorCode = + | 'current_tape_unavailable' + | 'linked_tape_unavailable' + | 'linked_tape_unauthorized' + +export class AgentTapeViewError extends Error { + readonly name = 'AgentTapeViewError' + + constructor( + readonly code: AgentTapeViewErrorCode, + readonly parentSessionId: string, + readonly sourceSessionId: string, + message: string + ) { + super(message) + } +} + +export function normalizeSubagentTapeLinkInput( + input: SubagentTapeLinkInput +): SubagentTapeLinkInput { + const normalized = { + parentSessionId: input.parentSessionId.trim(), + childSessionId: input.childSessionId.trim(), + runId: input.runId.trim(), + taskId: input.taskId.trim(), + slotId: input.slotId.trim(), + taskTitle: compactText(input.taskTitle, 500), + outcome: input.outcome, + resultSummary: input.resultSummary?.trim() ? compactText(input.resultSummary, 2000) : null + } + for (const [name, value] of Object.entries(normalized)) { + if (name === 'resultSummary' || name === 'outcome') continue + if (typeof value !== 'string' || !value) { + throw new Error(`Subagent Tape link ${name} is required.`) + } + } + if (normalized.parentSessionId === normalized.childSessionId) { + throw new Error('Subagent Tape link child must differ from its parent.') + } + if (!SUBAGENT_TAPE_LINK_OUTCOMES.has(normalized.outcome)) { + throw new Error(`Invalid subagent Tape link outcome: ${String(normalized.outcome)}`) + } + return normalized +} + +function isUnmarkedLegacyTape(row: DeepChatTapeEntryRow): boolean { + const meta = parseJsonValue(row.meta_json) + return ( + meta !== null && + typeof meta === 'object' && + !Array.isArray(meta) && + !Object.prototype.hasOwnProperty.call(meta, TAPE_INCARNATION_META_KEY) + ) +} + +function computeTapeIdentity(row: DeepChatTapeEntryRow): string { + return createHash('sha256') + .update( + JSON.stringify([ + row.session_id, + row.entry_id, + row.kind, + row.name, + row.source_type, + row.source_id, + row.source_seq, + row.provenance_key, + row.payload_json, + row.meta_json, + row.created_at + ]) + ) + .digest('hex') +} + +function subagentTapeLinkProvenanceKey(input: SubagentTapeLinkInput): string { + // This version belongs to the stable task-identity key, independently of the evolving event + // payload's linkVersion. + const identityHash = createHash('sha256') + .update( + JSON.stringify([input.parentSessionId, input.childSessionId, input.runId, input.taskId]) + ) + .digest('hex') + return `subagent:tape-link:v1:${identityHash}` +} + +function parseSubagentTapeLink(row: DeepChatTapeEntryRow): ParsedSubagentTapeLink | null { + const payload = parseJsonObject(row.payload_json) + const data = + payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) + ? (payload.data as Record) + : {} + const childSessionId = data.childSessionId + const childHeadEntryId = data.childHeadEntryId + const childEntryCount = data.childEntryCount + const outcome = data.outcome + const runId = data.runId + const taskId = data.taskId + const slotId = data.slotId + const taskTitle = data.taskTitle + const resultSummary = data.resultSummary + const linkVersion = data.linkVersion + const childTapeIdentity = data.childTapeIdentity + const hasValidLinkVersion = + (linkVersion === 1 && childTapeIdentity === undefined) || + (linkVersion === SUBAGENT_TAPE_LINK_VERSION && + typeof childTapeIdentity === 'string' && + TAPE_IDENTITY_PATTERN.test(childTapeIdentity)) + if ( + row.kind !== 'event' || + row.name !== SUBAGENT_TAPE_LINK_EVENT_NAME || + typeof childSessionId !== 'string' || + !childSessionId || + typeof childHeadEntryId !== 'number' || + !Number.isSafeInteger(childHeadEntryId) || + childHeadEntryId < 0 || + typeof childEntryCount !== 'number' || + !Number.isSafeInteger(childEntryCount) || + childEntryCount < 0 || + typeof outcome !== 'string' || + !SUBAGENT_TAPE_LINK_OUTCOMES.has(outcome as SubagentTapeLinkOutcome) || + typeof runId !== 'string' || + typeof taskId !== 'string' || + typeof slotId !== 'string' || + typeof taskTitle !== 'string' || + (resultSummary !== null && typeof resultSummary !== 'string') || + !hasValidLinkVersion || + row.source_type !== 'subagent' || + row.source_id !== childSessionId || + row.source_seq !== childHeadEntryId || + childEntryCount > childHeadEntryId + ) { + return null + } + + let normalizedInput: SubagentTapeLinkInput + try { + normalizedInput = normalizeSubagentTapeLinkInput({ + parentSessionId: row.session_id, + childSessionId, + runId, + taskId, + slotId, + taskTitle, + outcome: outcome as SubagentTapeLinkOutcome, + resultSummary + }) + } catch { + return null + } + if ( + normalizedInput.parentSessionId !== row.session_id || + normalizedInput.childSessionId !== childSessionId || + normalizedInput.runId !== runId || + normalizedInput.taskId !== taskId || + normalizedInput.slotId !== slotId || + normalizedInput.taskTitle !== taskTitle || + normalizedInput.resultSummary !== resultSummary || + row.provenance_key !== subagentTapeLinkProvenanceKey(normalizedInput) + ) { + return null + } + + return { + snapshot: { + linkEntryId: row.entry_id, + childSessionId, + childHeadEntryId, + childEntryCount, + outcome: outcome as SubagentTapeLinkOutcome, + childTapeIdentity: + linkVersion === SUBAGENT_TAPE_LINK_VERSION ? (childTapeIdentity as string) : null + }, + frozenInput: normalizedInput + } +} + +function parseSubagentTapeLinkSnapshot(row: DeepChatTapeEntryRow): SubagentTapeLinkSnapshot | null { + return parseSubagentTapeLink(row)?.snapshot ?? null +} + +function parseLegacyExternalTapeLinkSnapshot( + row: DeepChatTapeEntryRow +): SubagentTapeLinkSnapshot | null { + if (row.kind !== 'event' || row.name !== 'fork/merge' || row.source_type !== 'fork') { + return null + } + const payload = parseJsonObject(row.payload_json) + const data = + payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) + ? (payload.data as Record) + : {} + const childSessionId = data.forkSessionId + const forkId = data.forkId + const referencedEntryCount = data.referencedEntryCount + if ( + typeof childSessionId !== 'string' || + !childSessionId || + forkId !== childSessionId || + row.source_id !== childSessionId || + row.source_seq !== 0 || + row.provenance_key !== `fork:${row.session_id}:${childSessionId}:external-merge:event` || + typeof referencedEntryCount !== 'number' || + !Number.isSafeInteger(referencedEntryCount) || + referencedEntryCount <= 0 + ) { + return null + } + return { + linkEntryId: row.entry_id, + childSessionId, + childHeadEntryId: referencedEntryCount, + childEntryCount: referencedEntryCount, + outcome: 'completed', + childTapeIdentity: null + } +} + +function toSubagentTapeLinkReceipt(row: DeepChatTapeEntryRow): SubagentTapeLinkReceipt { + const snapshot = parseSubagentTapeLinkSnapshot(row) + if (!snapshot) { + throw new Error(`Stored subagent Tape link receipt is malformed: ${row.entry_id}`) + } + return { + linkEntry: { + sessionId: row.session_id, + entryId: snapshot.linkEntryId + }, + childSessionId: snapshot.childSessionId, + childHeadEntryId: snapshot.childHeadEntryId, + childEntryCount: snapshot.childEntryCount, + outcome: snapshot.outcome + } +} + +function assertSubagentTapeLinkMatchesInput( + row: DeepChatTapeEntryRow, + input: SubagentTapeLinkInput +): void { + const parsed = parseSubagentTapeLink(row) + if (!parsed) { + throw new Error(`Stored subagent Tape link receipt is malformed: ${row.entry_id}`) + } + const storedInput = parsed.frozenInput + const storedKeys = Object.keys(storedInput) as Array + if ( + storedKeys.length !== Object.keys(input).length || + storedKeys.some((key) => storedInput[key] !== input[key]) + ) { + throw new Error( + `Subagent Tape link conflicts with finalized task ${input.runId}/${input.taskId}.` + ) + } +} + +export class TapeLineageService { + constructor(private readonly providers: TapeLineageProviders) {} + + private get table() { + return this.providers.getEntryStore() + } + + resolveLinkedTapeSources(parentSessionId: string): LinkedTapeSourceResolution { + const table = this.table + const sessionTable = this.providers.getLineageSessionReader() + if (!table || !sessionTable?.get(parentSessionId)) { + throw new AgentTapeViewError( + 'current_tape_unavailable', + parentSessionId, + parentSessionId, + `Current Tape ${parentSessionId} is unavailable.` + ) + } + + const parsedSnapshots = table + .getSubagentLineageEvents(parentSessionId) + .map((row) => parseSubagentTapeLinkSnapshot(row) ?? parseLegacyExternalTapeLinkSnapshot(row)) + .filter((snapshot): snapshot is SubagentTapeLinkSnapshot => snapshot !== null) + const latestSnapshotByChild = new Map() + for (const snapshot of parsedSnapshots) { + const current = latestSnapshotByChild.get(snapshot.childSessionId) + if (!current || snapshot.linkEntryId > current.linkEntryId) { + latestSnapshotByChild.set(snapshot.childSessionId, snapshot) + } + } + const snapshots = [...latestSnapshotByChild.values()] + const childSessionIds = [...new Set(snapshots.map((snapshot) => snapshot.childSessionId))] + const childById = new Map(sessionTable.getMany(childSessionIds).map((row) => [row.id, row])) + const unavailableSourceIds = new Set() + const snapshotBySource = new Map() + + for (const snapshot of snapshots) { + const child = childById.get(snapshot.childSessionId) + if (!child) { + unavailableSourceIds.add(snapshot.childSessionId) + continue + } + if (child.session_kind !== 'subagent' || child.parent_session_id !== parentSessionId) { + continue + } + snapshotBySource.set(snapshot.childSessionId, snapshot) + } + + const authorizedSourceIds = [...snapshotBySource.keys()] + const firstEntryBySource = new Map( + table.getFirstEntriesBySessions(authorizedSourceIds).map((row) => [row.session_id, row]) + ) + const liveHeads = table.getMaxEntryIdsBySessions(authorizedSourceIds) + const availableSources: DeepChatTapeReadSource[] = [] + for (const [sourceSessionId, snapshot] of snapshotBySource) { + const firstEntry = firstEntryBySource.get(sourceSessionId) + const identityMatches = snapshot.childTapeIdentity + ? firstEntry !== undefined && computeTapeIdentity(firstEntry) === snapshot.childTapeIdentity + : firstEntry !== undefined && isUnmarkedLegacyTape(firstEntry) + if (!identityMatches || (liveHeads.get(sourceSessionId) ?? 0) < snapshot.childHeadEntryId) { + unavailableSourceIds.add(sourceSessionId) + continue + } + availableSources.push({ + sessionId: sourceSessionId, + maxEntryId: snapshot.childHeadEntryId + }) + } + + return { + sources: availableSources.sort((left, right) => + left.sessionId < right.sessionId ? -1 : left.sessionId > right.sessionId ? 1 : 0 + ), + unavailableSourceIds + } + } + + linkSubagentTape(input: SubagentTapeLinkInput): SubagentTapeLinkReceipt { + const table = this.table + const normalized = normalizeSubagentTapeLinkInput(input) + const provenanceKey = subagentTapeLinkProvenanceKey(normalized) + return table.runInTransaction(() => { + const existing = table.getByProvenanceKey(normalized.parentSessionId, provenanceKey) + if (existing) { + assertSubagentTapeLinkMatchesInput(existing, normalized) + const receipt = toSubagentTapeLinkReceipt(existing) + return receipt + } + + const childFirstEntry = table.getFirstEntriesBySessions([normalized.childSessionId])[0] + if (!childFirstEntry || childFirstEntry.session_id !== normalized.childSessionId) { + throw new Error(`Subagent Tape ${normalized.childSessionId} is unavailable.`) + } + const childTapeIdentity = computeTapeIdentity(childFirstEntry) + const childHeadEntryId = table.getMaxEntryId(normalized.childSessionId) + const childEntryCount = table.countBySession(normalized.childSessionId) + const row = table.appendEvent({ + sessionId: normalized.parentSessionId, + name: SUBAGENT_TAPE_LINK_EVENT_NAME, + source: { + type: 'subagent', + id: normalized.childSessionId, + seq: childHeadEntryId + }, + provenanceKey, + data: { + linkVersion: SUBAGENT_TAPE_LINK_VERSION, + childSessionId: normalized.childSessionId, + childHeadEntryId, + childEntryCount, + childTapeIdentity, + runId: normalized.runId, + taskId: normalized.taskId, + slotId: normalized.slotId, + taskTitle: normalized.taskTitle, + outcome: normalized.outcome, + resultSummary: normalized.resultSummary + }, + idempotent: true + }) + assertSubagentTapeLinkMatchesInput(row, normalized) + const receipt = toSubagentTapeLinkReceipt(row) + return receipt + }) + } +} diff --git a/src/main/tape/application/recallProjection.ts b/src/main/tape/application/recallProjection.ts new file mode 100644 index 0000000000..e5336b5519 --- /dev/null +++ b/src/main/tape/application/recallProjection.ts @@ -0,0 +1,466 @@ +import type { AgentTapeSearchOptions, AgentTapeViewScope } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryRow, DeepChatTapeSearchInput } from '../domain/entry' +import { parseJsonObject, parseJsonValue } from './common' +import type { TapeSearchResult } from './contracts' + +function isRecordObject(value: unknown): value is Record { + return Boolean(value && typeof value === 'object' && !Array.isArray(value)) +} + +function compactText(value: string, maxLength = 1000): string { + const normalized = value.replace(/\s+/g, ' ').trim() + if (normalized.length <= maxLength) return normalized + return `${normalized.slice(0, Math.max(0, maxLength - 3)).trimEnd()}...` +} + +export function stringifyForSummary(value: unknown, maxLength = 1000): string { + if (typeof value === 'string') return compactText(value, maxLength) + if (value === null || value === undefined) return '' + try { + return compactText(JSON.stringify(value), maxLength) + } catch { + return compactText(String(value), maxLength) + } +} + +export function truncateToUtf8Bytes( + text: string, + maxBytes: number +): { text: string; truncated: boolean } { + const normalized = text.trim() + if (maxBytes <= 0) { + return { text: '', truncated: normalized.length > 0 } + } + if (maxBytes < 3) { + return { text: '', truncated: normalized.length > 0 } + } + if (Buffer.byteLength(normalized, 'utf8') <= maxBytes) { + return { text: normalized, truncated: false } + } + let bytes = 0 + let output = '' + for (const character of normalized) { + const nextBytes = Buffer.byteLength(character, 'utf8') + if (bytes + nextBytes > Math.max(0, maxBytes - 3)) break + output += character + bytes += nextBytes + } + return { text: `${output.trimEnd()}...`, truncated: true } +} + +export function normalizeContextByteLimit( + value: number | undefined, + fallback: number, + max: number +): number { + if (!Number.isFinite(value)) return fallback + return Math.min(Math.max(Math.floor(value as number), 0), max) +} + +function uniqueStrings(values: string[], limit = 10): string[] { + const seen = new Set() + const result: string[] = [] + for (const value of values) { + const normalized = value.trim() + if (!normalized || seen.has(normalized)) continue + seen.add(normalized) + result.push(normalized) + if (result.length >= limit) break + } + return result +} + +function extractFilePaths(text: string): string[] { + const matches = [ + ...text.matchAll( + /(?:^|[\s"'`([{<])((?:[A-Za-z]:\\|\/|\.{1,2}\/|[\w.-]+\/)[^\s"'`<>{}(),;!?]+(?:[/\\][^\s"'`<>{}(),;!?]+)*)/g + ) + ].map((match) => match[1].replace(/[.:]+$/g, '')) + return uniqueStrings(matches ?? []) +} + +function extractErrorCodes(text: string): string[] { + const matches = text.match(/\b(?:E[A-Z0-9_]{3,}|[A-Z][A-Z0-9_]*Error)\b/g) + return uniqueStrings(matches ?? []) +} + +function collectKeyedStrings( + value: unknown, + keys: Set, + output: string[] = [], + depth = 0 +): string[] { + if (depth > 4 || output.length >= 10 || !value || typeof value !== 'object') return output + if (Array.isArray(value)) { + for (const item of value) collectKeyedStrings(item, keys, output, depth + 1) + return output + } + for (const [key, nested] of Object.entries(value as Record)) { + if (keys.has(key) && typeof nested === 'string' && nested.trim()) { + output.push(compactText(nested, 500)) + if (output.length >= 10) return output + } + collectKeyedStrings(nested, keys, output, depth + 1) + if (output.length >= 10) return output + } + return output +} + +function collectUserMessageAttachmentRefs(files: unknown): { + searchText: string[] + filePaths: string[] + fileNames: string[] +} { + const searchText: string[] = [] + const filePaths: string[] = [] + const fileNames: string[] = [] + if (!Array.isArray(files)) { + return { searchText, filePaths, fileNames } + } + for (const file of files) { + if (!isRecordObject(file)) continue + const path = typeof file.path === 'string' ? file.path : '' + const name = typeof file.name === 'string' ? file.name : '' + const metadata = isRecordObject(file.metadata) ? file.metadata : null + const metadataFileName = + metadata && typeof metadata.fileName === 'string' ? metadata.fileName : '' + if (path) { + filePaths.push(compactText(path, 500)) + searchText.push(compactText(path, 500)) + } + for (const value of [name, metadataFileName]) { + if (!value) continue + fileNames.push(compactText(value, 500)) + searchText.push(compactText(value, 500)) + } + } + return { + searchText: uniqueStrings(searchText, 20), + filePaths: uniqueStrings(filePaths, 20), + fileNames: uniqueStrings(fileNames, 20) + } +} + +type UserMessageProjectionText = { + text: string + attachmentRefs: { + searchText: string[] + filePaths: string[] + fileNames: string[] + } +} + +function emptyUserMessageAttachmentRefs(): UserMessageProjectionText['attachmentRefs'] { + return { searchText: [], filePaths: [], fileNames: [] } +} + +function parseUserMessageProjectionText(content: string): UserMessageProjectionText { + const parsed = parseJsonValue(content) + if (isRecordObject(parsed) && typeof parsed.text === 'string') { + const attachmentRefs = collectUserMessageAttachmentRefs(parsed.files) + const parts = [parsed.text] + if (Array.isArray(parsed.files) && parsed.files.length > 0) { + parts.push(`files:${parsed.files.length}`) + parts.push(...attachmentRefs.searchText) + } + if (Array.isArray(parsed.links) && parsed.links.length > 0) { + parts.push(`links:${parsed.links.length}`) + } + return { text: parts.join(' '), attachmentRefs } + } + return { text: content, attachmentRefs: emptyUserMessageAttachmentRefs() } +} + +export function getUserMessageProjectionText( + row: DeepChatTapeEntryRow, + payload: Record +): UserMessageProjectionText | null { + if (row.kind !== 'message' || !isRecordObject(payload.record)) return null + const record = payload.record + const role = typeof record.role === 'string' ? record.role : 'message' + if (role === 'assistant') return null + const content = typeof record.content === 'string' ? record.content : '' + return parseUserMessageProjectionText(content) +} + +function collectUserMessageAttachmentRefsFromPayload(payload: Record): { + searchText: string[] + filePaths: string[] + fileNames: string[] +} { + if (!isRecordObject(payload.record)) { + return { searchText: [], filePaths: [], fileNames: [] } + } + const content = typeof payload.record.content === 'string' ? payload.record.content : '' + const parsed = parseJsonValue(content) + return isRecordObject(parsed) + ? collectUserMessageAttachmentRefs(parsed.files) + : { searchText: [], filePaths: [], fileNames: [] } +} + +function readUserMessageText(content: string, parsed?: UserMessageProjectionText): string { + return parsed?.text ?? parseUserMessageProjectionText(content).text +} + +function readAssistantMessageText(content: string): string { + const parsed = parseJsonValue(content) + if (!Array.isArray(parsed)) return content + const parts: string[] = [] + for (const block of parsed) { + if (!isRecordObject(block)) continue + if (typeof block.content === 'string' && block.content.trim()) { + parts.push(block.content) + continue + } + const toolCall = block.tool_call + if (isRecordObject(toolCall)) { + const name = typeof toolCall.name === 'string' ? toolCall.name : 'unknown' + const params = typeof toolCall.params === 'string' ? toolCall.params : '' + const response = typeof toolCall.response === 'string' ? toolCall.response : '' + parts.push(`tool ${name} ${params} ${response}`.trim()) + } + } + return parts.join(' ') +} + +export function summarizeTapeRow( + row: DeepChatTapeEntryRow, + payload: Record, + userMessage?: UserMessageProjectionText | null +): string { + if (row.kind === 'message') { + const record = payload.record + if (isRecordObject(record)) { + const role = typeof record.role === 'string' ? record.role : 'message' + const content = typeof record.content === 'string' ? record.content : '' + const text = + role === 'assistant' + ? readAssistantMessageText(content) + : readUserMessageText(content, userMessage ?? undefined) + return compactText(`${role}: ${text}`, 1200) + } + } + + if (row.kind === 'tool_call') { + const toolCall = payload.toolCall + if (isRecordObject(toolCall)) { + const name = typeof toolCall.name === 'string' ? toolCall.name : (row.name ?? 'unknown') + const params = typeof toolCall.params === 'string' ? toolCall.params : '' + return compactText(`tool_call ${name}: ${params}`, 1200) + } + } + + if (row.kind === 'tool_result') { + const response = typeof payload.response === 'string' ? payload.response : payload + return compactText( + `tool_result ${row.name ?? 'unknown'}: ${stringifyForSummary(response)}`, + 1200 + ) + } + + if (row.kind === 'anchor') { + const state = isRecordObject(payload.state) ? payload.state : payload + const summary = typeof state.summary === 'string' ? state.summary : stringifyForSummary(state) + return compactText(`anchor ${row.name ?? 'unknown'}: ${summary}`, 1200) + } + + if (row.kind === 'event') { + const data = isRecordObject(payload.data) ? payload.data : payload + return compactText(`event ${row.name ?? 'unknown'}: ${stringifyForSummary(data)}`, 1200) + } + + return compactText(`${row.kind} ${row.name ?? ''}`.trim(), 1200) +} + +export function buildTapeRowEvidenceText( + row: DeepChatTapeEntryRow, + payload: Record, + meta: Record, + userMessage?: UserMessageProjectionText | null +): string { + const parts: string[] = [] + if (row.kind === 'message' && isRecordObject(payload.record)) { + const record = payload.record + const content = typeof record.content === 'string' ? record.content : '' + const role = typeof record.role === 'string' ? record.role : 'message' + parts.push( + role === 'assistant' + ? readAssistantMessageText(content) + : readUserMessageText(content, userMessage ?? undefined) + ) + } else if (row.kind === 'tool_call' && isRecordObject(payload.toolCall)) { + const toolCall = payload.toolCall + parts.push(stringifyForSummary(toolCall.name, 200)) + parts.push(stringifyForSummary(toolCall.params, 3000)) + } else if (row.kind === 'tool_result') { + parts.push(stringifyForSummary(payload.response ?? payload, 4000)) + } else if (row.kind === 'anchor') { + parts.push(stringifyForSummary(isRecordObject(payload.state) ? payload.state : payload, 4000)) + } else if (row.kind === 'event') { + parts.push(stringifyForSummary(isRecordObject(payload.data) ? payload.data : payload, 4000)) + } else { + parts.push(stringifyForSummary(payload, 4000)) + } + if (typeof meta.status === 'string') parts.push(`status:${meta.status}`) + return compactText(parts.filter(Boolean).join('\n'), 5000) +} + +function setRef(target: Record, key: string, value: unknown): void { + if (value !== null && value !== undefined && value !== '') { + target[key] = value + } +} + +function enrichTapeRowRefs( + refs: Record, + payload: Record, + meta: Record, + evidenceText: string, + userMessage?: UserMessageProjectionText | null +): void { + const attachmentRefs = + userMessage?.attachmentRefs ?? collectUserMessageAttachmentRefsFromPayload(payload) + const filePaths = uniqueStrings( + [...extractFilePaths(evidenceText), ...attachmentRefs.filePaths], + 20 + ) + const errorCodes = extractErrorCodes(evidenceText) + const commands = uniqueStrings( + [ + ...collectKeyedStrings(payload, new Set(['command', 'cmd', 'script', 'shellCommand'])), + ...collectKeyedStrings(meta, new Set(['command', 'cmd', 'script', 'shellCommand'])) + ], + 10 + ) + setRef(refs, 'filePaths', filePaths.length ? filePaths : null) + setRef(refs, 'fileNames', attachmentRefs.fileNames.length ? attachmentRefs.fileNames : null) + setRef(refs, 'commands', commands.length ? commands : null) + setRef(refs, 'errorCodes', errorCodes.length ? errorCodes : null) + for (const key of ['exitCode', 'exitStatus', 'code']) { + const value = payload[key] ?? meta[key] + if (typeof value === 'number' || typeof value === 'string') { + setRef(refs, key, value) + } + } +} + +export function buildTapeRowRefs( + row: DeepChatTapeEntryRow, + payload: Record, + meta: Record, + userMessage?: UserMessageProjectionText | null, + evidenceText?: string +): Record { + const refs: Record = {} + setRef(refs, 'sourceType', row.source_type) + setRef(refs, 'sourceId', row.source_id) + setRef(refs, 'sourceSeq', row.source_seq) + setRef(refs, 'status', typeof meta.status === 'string' ? meta.status : null) + + if (row.kind === 'message' && isRecordObject(payload.record)) { + const record = payload.record + setRef(refs, 'messageId', record.id) + setRef(refs, 'orderSeq', record.orderSeq) + setRef(refs, 'role', record.role) + setRef(refs, 'messageStatus', record.status) + } else if (row.kind === 'tool_call' && isRecordObject(payload.toolCall)) { + const toolCall = payload.toolCall + setRef(refs, 'messageId', payload.messageId) + setRef(refs, 'orderSeq', payload.orderSeq) + setRef(refs, 'toolCallId', toolCall.id) + setRef(refs, 'toolName', toolCall.name ?? row.name) + setRef(refs, 'serverName', toolCall.serverName) + } else if (row.kind === 'tool_result') { + setRef(refs, 'messageId', payload.messageId) + setRef(refs, 'orderSeq', payload.orderSeq) + setRef(refs, 'toolCallId', payload.toolCallId) + setRef(refs, 'toolName', row.name) + } else if (row.kind === 'anchor') { + setRef(refs, 'anchorName', row.name) + } else if (row.kind === 'event') { + setRef(refs, 'eventName', row.name) + } + + enrichTapeRowRefs( + refs, + payload, + meta, + evidenceText ?? buildTapeRowEvidenceText(row, payload, meta, userMessage), + userMessage + ) + return refs +} + +export function parseProjectionRefs(raw: string): Record { + const parsed = parseJsonObject(raw) + return parsed +} + +export function normalizeContextWindowValue(value: number | undefined, fallback: number): number { + if (!Number.isFinite(value)) return fallback + return Math.min(Math.max(Math.floor(value as number), 0), 20) +} + +export function normalizeContextLimit(value: number | undefined): number { + if (!Number.isFinite(value)) return 50 + return Math.min(Math.max(Math.floor(value as number), 1), 100) +} + +function parseSearchBoundary(value: string | undefined, name: string): number | undefined { + const trimmed = value?.trim() + if (!trimmed) { + return undefined + } + + const numericValue = Number(trimmed) + if (Number.isFinite(numericValue)) { + return numericValue + } + + const parsedDate = Date.parse(trimmed) + if (Number.isFinite(parsedDate)) { + return parsedDate + } + + throw new Error(`${name} must be an ISO date/time or millisecond timestamp.`) +} + +export function toTapeSearchInput( + options: AgentTapeSearchOptions | undefined +): DeepChatTapeSearchInput { + return { + limit: options?.limit, + kinds: options?.kinds, + startCreatedAt: parseSearchBoundary(options?.start, 'start'), + endCreatedAt: parseSearchBoundary(options?.end, 'end') + } +} + +export function normalizeTapeViewScope(scope: AgentTapeViewScope | undefined): AgentTapeViewScope { + if (scope === undefined || scope === 'current') return 'current' + if (scope === 'linked_subagents' || scope === 'current_and_linked') return scope + throw new Error(`Invalid Tape view scope: ${String(scope)}`) +} + +export function normalizeTapeSearchLimit(value: number | undefined): number { + if (!Number.isFinite(value)) return 20 + return Math.min(Math.max(Math.floor(value as number), 1), 100) +} + +export function compareTapeSearchResults(left: TapeSearchResult, right: TapeSearchResult): number { + const leftHasScore = typeof left.score === 'number' && Number.isFinite(left.score) + const rightHasScore = typeof right.score === 'number' && Number.isFinite(right.score) + if (leftHasScore && rightHasScore && left.score !== right.score) { + return (left.score as number) - (right.score as number) + } + if (leftHasScore !== rightHasScore) { + return leftHasScore ? -1 : 1 + } + if (left.createdAt !== right.createdAt) { + return right.createdAt - left.createdAt + } + if (left.sessionId !== right.sessionId) { + return left.sessionId < right.sessionId ? -1 : 1 + } + return right.entryId - left.entryId +} diff --git a/src/main/tape/application/recallService.ts b/src/main/tape/application/recallService.ts new file mode 100644 index 0000000000..37190c0fbe --- /dev/null +++ b/src/main/tape/application/recallService.ts @@ -0,0 +1,561 @@ +import logger from 'electron-log' +import type { + AgentTapeAnchorsOptions, + AgentTapeContextEntry, + AgentTapeContextOptions, + AgentTapeContextResult, + AgentTapeSearchOptions +} from '@shared/types/agent-interface' +import { + buildEffectiveTapeView, + getLastEffectiveTokenUsage, + searchEffectiveTapeRows +} from '../domain/effectiveView' +import type { DeepChatTapeEntryRow, DeepChatTapeReadSource } from '../domain/entry' +import type { + TapeApplicationProviders, + TapeSearchProjectionInput as DeepChatTapeSearchProjectionInput, + TapeSearchProjectionResultRow as DeepChatTapeSearchProjectionResultRow, + TapeSearchProjectionRow as DeepChatTapeSearchProjectionRow, + TapeSearchProjectionStore +} from '../ports/application' +import { isEntryIdPrefix, migrationProvenanceKey, parseJsonObject } from './common' +import type { TapeAnchorResult, TapeInfo, TapeSearchResult } from './contracts' +import { AgentTapeViewError, type TapeLineageService } from './lineageService' +import { + buildTapeRowEvidenceText, + buildTapeRowRefs, + compareTapeSearchResults, + getUserMessageProjectionText, + normalizeContextByteLimit, + normalizeContextLimit, + normalizeContextWindowValue, + normalizeTapeSearchLimit, + normalizeTapeViewScope, + parseProjectionRefs, + stringifyForSummary, + summarizeTapeRow, + toTapeSearchInput, + truncateToUtf8Bytes +} from './recallProjection' + +type TapeRecallProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getSearchProjectionStore' +> + +const DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY = 2048 +const DEFAULT_CONTEXT_MAX_TOTAL_BYTES = 16384 +const MAX_CONTEXT_MAX_BYTES_PER_ENTRY = 8192 +const MAX_CONTEXT_MAX_TOTAL_BYTES = 65536 + +export class TapeRecallService { + constructor( + private readonly providers: TapeRecallProviders, + private readonly lineage: TapeLineageService + ) {} + + private get table() { + return this.providers.getEntryStore() + } + + private get searchProjectionTable() { + return this.providers.getSearchProjectionStore() + } + + info(sessionId: string): TapeInfo { + const table = this.table + const lastAnchor = table.getLatestAnchor(sessionId) + const rows = table.getBySession(sessionId) + return { + sessionId, + entries: table.countBySession(sessionId), + anchors: table.countAnchorsBySession(sessionId), + lastAnchor: lastAnchor?.name ?? null, + lastAnchorEntryId: lastAnchor?.entry_id ?? null, + entriesSinceLastAnchor: lastAnchor + ? table.countEntriesAfter(sessionId, lastAnchor.entry_id) + : 0, + lastTokenUsage: getLastEffectiveTokenUsage(rows), + migrationState: table.getByProvenanceKey(sessionId, migrationProvenanceKey(sessionId)) + ? 'ready' + : 'none' + } + } + + search(sessionId: string, query: string, options?: AgentTapeSearchOptions): TapeSearchResult[] { + const scope = normalizeTapeViewScope(options?.scope) + if (!query.trim()) { + return [] + } + if (scope === 'current') { + return this.searchCurrentTape(sessionId, query, options) + } + + const resolution = this.lineage.resolveLinkedTapeSources(sessionId) + if (resolution.unavailableSourceIds.size > 0) { + const sourceSessionId = [...resolution.unavailableSourceIds].sort()[0] + throw new AgentTapeViewError( + 'linked_tape_unavailable', + sessionId, + sourceSessionId, + `Linked Tape ${sourceSessionId} is unavailable.` + ) + } + const sources = [...resolution.sources] + if (scope === 'current_and_linked') { + sources.push({ sessionId, maxEntryId: this.table?.getMaxEntryId(sessionId) ?? 0 }) + } + return this.searchTapeSourcesReadOnly(sources, query, options) + } + + private searchCurrentTape( + sessionId: string, + query: string, + options?: AgentTapeSearchOptions + ): TapeSearchResult[] { + const table = this.table + const searchInput = toTapeSearchInput(options) + const projectionTable = this.searchProjectionTable + let skipProjectionSearch = false + + if (projectionTable) { + try { + const maxEntryId = table.getMaxEntryId(sessionId) + if (projectionTable.isCurrent(sessionId, maxEntryId)) { + return projectionTable + .search(sessionId, query, searchInput) + .map((row) => this.toProjectedSearchResult(row, undefined)) + } + } catch (error) { + skipProjectionSearch = true + logger.warn( + `[Tape] projection fast-path search failed; falling back to effective search: ${String(error)}` + ) + } + } + + const rows = table.getBySession(sessionId) + const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows + const preparedProjectionTable = skipProjectionSearch + ? null + : this.ensureSearchProjection(sessionId, rows, effectiveRows) + if (!preparedProjectionTable) { + return searchEffectiveTapeRows(rows, query, searchInput).map((row) => + this.toSearchResult(row) + ) + } + + const rowByEntryId = new Map(effectiveRows.map((row) => [row.entry_id, row])) + try { + return preparedProjectionTable + .search(sessionId, query, searchInput) + .map((row) => this.toProjectedSearchResult(row, rowByEntryId.get(row.entry_id))) + } catch (error) { + logger.warn( + `[Tape] projection search failed; falling back to effective search: ${String(error)}` + ) + return searchEffectiveTapeRows(rows, query, searchInput).map((row) => + this.toSearchResult(row) + ) + } + } + + getContext( + sessionId: string, + entryIds: number[], + options: AgentTapeContextOptions = {} + ): AgentTapeContextResult { + const sourceSessionId = options.sourceSessionId?.trim() || sessionId + if (sourceSessionId !== sessionId) { + return this.getLinkedTapeContext(sessionId, sourceSessionId, entryIds, options) + } + + const requestedEntryIds = [ + ...new Set(entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0)) + ].sort((left, right) => left - right) + const table = this.table + if (!table || requestedEntryIds.length === 0) { + return { + sessionId, + sourceSessionId, + requestedEntryIds, + matchedEntryIds: [], + entries: [] + } + } + + const rows = table.getBySession(sessionId) + const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows + const indexByEntryId = new Map(effectiveRows.map((row, index) => [row.entry_id, index])) + const before = normalizeContextWindowValue(options.before, 2) + const after = normalizeContextWindowValue(options.after, 2) + const limit = normalizeContextLimit(options.limit) + const maxBytesPerEntry = normalizeContextByteLimit( + options.maxBytesPerEntry, + DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY, + MAX_CONTEXT_MAX_BYTES_PER_ENTRY + ) + const maxTotalBytes = normalizeContextByteLimit( + options.maxTotalBytes, + DEFAULT_CONTEXT_MAX_TOTAL_BYTES, + MAX_CONTEXT_MAX_TOTAL_BYTES + ) + const selectedIndexes = new Set() + const requestedIndexes: number[] = [] + const windowIndexes: number[] = [] + + for (const entryId of requestedEntryIds) { + const index = indexByEntryId.get(entryId) + if (index === undefined) continue + requestedIndexes.push(index) + for ( + let cursor = Math.max(0, index - before); + cursor <= Math.min(effectiveRows.length - 1, index + after); + cursor += 1 + ) { + if (cursor === index) continue + windowIndexes.push(cursor) + } + } + + for (const index of requestedIndexes) { + if (selectedIndexes.size >= limit) break + selectedIndexes.add(index) + } + for (const index of windowIndexes) { + if (selectedIndexes.size >= limit) break + selectedIndexes.add(index) + } + + const selectedRows = [...selectedIndexes] + .sort((left, right) => left - right) + .map((index) => effectiveRows[index]) + let projectionRows = new Map() + try { + projectionRows = new Map( + this.searchProjectionTable + .getByEntryIds( + sessionId, + selectedRows.map((row) => row.entry_id) + ) + .map((row) => [row.entry_id, row]) + ) + } catch { + projectionRows = new Map() + } + let usedBytes = 0 + const entries: AgentTapeContextEntry[] = [] + const priorityIndexes = [...requestedIndexes, ...windowIndexes].filter( + (index, offset, indexes) => { + return selectedIndexes.has(index) && indexes.indexOf(index) === offset + } + ) + for (const index of priorityIndexes) { + const row = effectiveRows[index] + const remaining = Math.max(0, maxTotalBytes - usedBytes) + if (remaining <= 0) break + const maxEntryBytes = Math.min(maxBytesPerEntry, remaining) + if (maxEntryBytes <= 0) break + const entry = this.toContextEntry(row, projectionRows.get(row.entry_id), maxEntryBytes) + if (entry.evidence.bytes <= 0) continue + usedBytes += entry.evidence.bytes + entries.push(entry) + } + entries.sort((left, right) => left.entryId - right.entryId) + const returnedEntryIds = new Set(entries.map((entry) => entry.entryId)) + + return { + sessionId, + sourceSessionId, + requestedEntryIds, + matchedEntryIds: requestedEntryIds.filter((entryId) => returnedEntryIds.has(entryId)), + entries + } + } + + private searchTapeSourcesReadOnly( + sources: DeepChatTapeReadSource[], + query: string, + options: AgentTapeSearchOptions | undefined + ): TapeSearchResult[] { + const table = this.table + if (!table || !query.trim() || sources.length === 0) { + return [] + } + const searchInput = toTapeSearchInput(options) + const limit = normalizeTapeSearchLimit(options?.limit) + const results: TapeSearchResult[] = [] + let uncoveredSources = sources + + try { + const projected = this.searchProjectionTable?.searchSourcesReadOnly( + sources, + query, + searchInput + ) + if (projected) { + const coveredHeadBySource = new Map( + projected.coveredSources.map((source) => [source.sessionId, source.maxEntryId]) + ) + const hasCompleteProjection = + coveredHeadBySource.size === sources.length && + sources.every((source) => coveredHeadBySource.get(source.sessionId) === source.maxEntryId) + if (hasCompleteProjection) { + results.push(...projected.rows.map((row) => this.toProjectedSearchResult(row, undefined))) + uncoveredSources = [] + } + } + } catch (error) { + logger.warn( + `[Tape] linked projection search failed; using read-only Tape fallback: ${String(error)}` + ) + uncoveredSources = sources + } + + if (uncoveredSources.length > 0) { + results.push( + ...table + .searchEffectiveSourcesAtHeads(uncoveredSources, query, searchInput) + .map((row) => this.toSearchResult(row)) + ) + } + + const seen = new Set() + return results + .sort(compareTapeSearchResults) + .filter((result) => { + const key = `${result.sessionId}:${result.entryId}` + if (seen.has(key)) return false + seen.add(key) + return true + }) + .slice(0, limit) + } + + private getLinkedTapeContext( + parentSessionId: string, + sourceSessionId: string, + entryIds: number[], + options: AgentTapeContextOptions + ): AgentTapeContextResult { + const resolution = this.lineage.resolveLinkedTapeSources(parentSessionId) + if (resolution.unavailableSourceIds.has(sourceSessionId)) { + throw new AgentTapeViewError( + 'linked_tape_unavailable', + parentSessionId, + sourceSessionId, + `Linked Tape ${sourceSessionId} is unavailable.` + ) + } + const source = resolution.sources.find((candidate) => candidate.sessionId === sourceSessionId) + if (!source) { + throw new AgentTapeViewError( + 'linked_tape_unauthorized', + parentSessionId, + sourceSessionId, + `Tape ${sourceSessionId} is not an authorized direct child of ${parentSessionId}.` + ) + } + + const requestedEntryIds = [ + ...new Set(entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0)) + ].sort((left, right) => left - right) + const table = this.table + if (!table || requestedEntryIds.length === 0) { + return { + sessionId: parentSessionId, + sourceSessionId, + requestedEntryIds, + matchedEntryIds: [], + entries: [] + } + } + + const before = normalizeContextWindowValue(options.before, 2) + const after = normalizeContextWindowValue(options.after, 2) + const limit = normalizeContextLimit(options.limit) + const maxBytesPerEntry = normalizeContextByteLimit( + options.maxBytesPerEntry, + DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY, + MAX_CONTEXT_MAX_BYTES_PER_ENTRY + ) + const maxTotalBytes = normalizeContextByteLimit( + options.maxTotalBytes, + DEFAULT_CONTEXT_MAX_TOTAL_BYTES, + MAX_CONTEXT_MAX_TOTAL_BYTES + ) + const rows = table.getEffectiveContextRowsAtHead(source, requestedEntryIds, { + before, + after, + limit + }) + + let usedBytes = 0 + const entries: AgentTapeContextEntry[] = [] + for (const row of rows) { + const remaining = Math.max(0, maxTotalBytes - usedBytes) + if (remaining <= 0) break + const maxEntryBytes = Math.min(maxBytesPerEntry, remaining) + const entry = this.toContextEntry(row, undefined, maxEntryBytes) + if (entry.evidence.bytes <= 0) continue + usedBytes += entry.evidence.bytes + entries.push(entry) + } + entries.sort((left, right) => left.entryId - right.entryId) + const returnedEntryIds = new Set(entries.map((entry) => entry.entryId)) + + return { + sessionId: parentSessionId, + sourceSessionId, + requestedEntryIds, + matchedEntryIds: requestedEntryIds.filter((entryId) => returnedEntryIds.has(entryId)), + entries + } + } + + anchors(sessionId: string, options: AgentTapeAnchorsOptions = {}): TapeAnchorResult[] { + return this.table.getAnchors(sessionId, options.limit).map((row) => this.toAnchorResult(row)) + } + + private ensureSearchProjection( + sessionId: string, + rows: DeepChatTapeEntryRow[], + effectiveRows: DeepChatTapeEntryRow[] + ): TapeSearchProjectionStore | null { + const projectionTable = this.searchProjectionTable + const maxEntryId = rows.reduce((max, row) => Math.max(max, row.entry_id), 0) + try { + if (!projectionTable.isCurrent(sessionId, maxEntryId)) { + const meta = projectionTable.getSessionMeta(sessionId) + const projectedEntryIds = projectionTable.getProjectedEntryIds(sessionId) + const effectiveEntryIds = effectiveRows.map((row) => row.entry_id) + const canAppend = + !!meta && + projectionTable.isCurrent(sessionId, meta.maxEntryId) && + meta.maxEntryId <= maxEntryId && + isEntryIdPrefix(projectedEntryIds, effectiveEntryIds) + if (canAppend) { + projectionTable.appendSession( + sessionId, + effectiveRows.slice(projectedEntryIds.length).map((row) => this.toProjectionInput(row)), + maxEntryId + ) + } else { + projectionTable.replaceSession( + sessionId, + effectiveRows.map((row) => this.toProjectionInput(row)), + maxEntryId + ) + } + } + return projectionTable + } catch { + return null + } + } + + private toProjectionInput(row: DeepChatTapeEntryRow): DeepChatTapeSearchProjectionInput { + const payload = parseJsonObject(row.payload_json) + const meta = parseJsonObject(row.meta_json) + const userMessage = getUserMessageProjectionText(row, payload) + const summaryText = summarizeTapeRow(row, payload, userMessage) + const evidenceText = buildTapeRowEvidenceText(row, payload, meta, userMessage) + const refs = buildTapeRowRefs(row, payload, meta, userMessage, evidenceText) + const searchText = [ + row.kind, + row.name ?? '', + summaryText, + evidenceText, + Object.values(refs) + .map((value) => stringifyForSummary(value)) + .join(' ') + ] + .filter(Boolean) + .join('\n') + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + sourceType: row.source_type, + sourceId: row.source_id, + sourceSeq: row.source_seq, + searchText, + summaryText, + refs, + createdAt: row.created_at + } + } + + private toProjectedSearchResult( + row: DeepChatTapeSearchProjectionResultRow, + _sourceRow: DeepChatTapeEntryRow | undefined + ): TapeSearchResult { + const score = + typeof row.score === 'number' && Number.isFinite(row.score) ? row.score : undefined + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + createdAt: row.created_at, + summary: row.summary_text, + refs: parseProjectionRefs(row.refs_json), + ...(score === undefined ? {} : { score }) + } + } + + private toContextEntry( + row: DeepChatTapeEntryRow, + projectionRow: DeepChatTapeSearchProjectionRow | undefined, + maxBytes: number + ): AgentTapeContextEntry { + const fallbackProjection = projectionRow ? null : this.toProjectionInput(row) + const payload = parseJsonObject(row.payload_json) + const meta = parseJsonObject(row.meta_json) + const evidenceSource = buildTapeRowEvidenceText(row, payload, meta) + const clipped = truncateToUtf8Bytes(evidenceSource, maxBytes) + const bytes = Buffer.byteLength(clipped.text, 'utf8') + return { + entryId: row.entry_id, + kind: row.kind, + name: row.name, + summary: projectionRow?.summary_text ?? fallbackProjection?.summaryText ?? '', + refs: projectionRow + ? parseProjectionRefs(projectionRow.refs_json) + : (fallbackProjection?.refs ?? {}), + evidence: { + text: clipped.text, + truncated: clipped.truncated, + bytes + }, + createdAt: row.created_at + } + } + + private toSearchResult(row: DeepChatTapeEntryRow): TapeSearchResult { + const projection = this.toProjectionInput(row) + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + createdAt: row.created_at, + summary: projection.summaryText, + refs: projection.refs + } + } + + private toAnchorResult(row: DeepChatTapeEntryRow): TapeAnchorResult { + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + payload: parseJsonObject(row.payload_json), + meta: parseJsonObject(row.meta_json), + createdAt: row.created_at + } + } +} diff --git a/src/main/tape/application/reconcilerService.ts b/src/main/tape/application/reconcilerService.ts new file mode 100644 index 0000000000..ee1db5c6b7 --- /dev/null +++ b/src/main/tape/application/reconcilerService.ts @@ -0,0 +1,127 @@ +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { TapeApplicationProviders } from '../ports/application' +import { appendMessageRecordToTape } from './factPersistence' +import type { TapeBackfillResult } from './contracts' +import type { TapeFactService } from './factService' +import { migrationProvenanceKey } from './common' + +type TapeReconcilerProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getLegacySummaryReader' +> + +export interface TapeTranscriptReader { + getMessages(sessionId: string): ChatMessageRecord[] +} + +function legacySummaryProvenanceKey(sessionId: string): string { + return `summary:${sessionId}:legacy-summary:v1` +} + +export class TapeReconcilerService { + constructor( + private readonly providers: TapeReconcilerProviders, + private readonly facts: TapeFactService + ) {} + + private get table() { + return this.providers.getEntryStore() + } + + ensureSessionTapeReady( + sessionId: string, + messageStore: TapeTranscriptReader + ): TapeBackfillResult { + const table = this.table + const historyRecords = messageStore + .getMessages(sessionId) + .sort((left, right) => left.orderSeq - right.orderSeq) + const maxOrderSeq = historyRecords.reduce( + (currentMax, record) => Math.max(currentMax, record.orderSeq), + 0 + ) + + table.ensureBootstrapAnchor(sessionId) + + let appendedFactCount = 0 + for (const record of historyRecords) { + appendedFactCount += appendMessageRecordToTape(table, record, 'backfill') + } + + this.backfillLegacySummaryAnchor(sessionId, historyRecords) + + table.appendEvent({ + sessionId, + name: 'migration/backfill', + source: { + type: 'migration', + id: 'message-backfill', + seq: 1 + }, + provenanceKey: migrationProvenanceKey(sessionId), + data: { + source: 'deepchat_messages', + messageCount: historyRecords.length, + maxOrderSeq + }, + idempotent: true + }) + + return { + sessionId, + migrationState: 'ready', + messageCount: historyRecords.length, + maxOrderSeq, + appendedFactCount, + historyRecords: this.facts.getMessageRecords(sessionId) + } + } + + private backfillLegacySummaryAnchor( + sessionId: string, + historyRecords: ChatMessageRecord[] + ): void { + const table = this.table + if (table.getLatestSummaryAnchor(sessionId)) { + return + } + + const legacyState = this.providers.getLegacySummaryReader().getSummaryState(sessionId) + if (!legacyState) { + return + } + + const summary = legacyState.summary_text?.trim() + if (!summary) { + return + } + + const cursorOrderSeq = Math.max(1, legacyState.summary_cursor_order_seq ?? 1) + const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) + table.appendAnchor({ + sessionId, + name: 'compaction/migrated_summary', + source: { + type: 'summary', + id: 'legacy-summary', + seq: 1 + }, + provenanceKey: legacySummaryProvenanceKey(sessionId), + state: { + summary, + cursorOrderSeq, + range: + sourceRecords.length > 0 + ? { + fromOrderSeq: sourceRecords[0].orderSeq, + toOrderSeq: sourceRecords[sourceRecords.length - 1].orderSeq + } + : null, + sourceMessageIds: sourceRecords.map((record) => record.id), + migratedFrom: 'deepchat_sessions.summary_text' + }, + idempotent: true, + createdAt: legacyState.summary_updated_at ?? undefined + }) + } +} diff --git a/src/main/tape/application/sessionTape.ts b/src/main/tape/application/sessionTape.ts new file mode 100644 index 0000000000..0688801430 --- /dev/null +++ b/src/main/tape/application/sessionTape.ts @@ -0,0 +1,201 @@ +import type { + AgentTapeAnchorsOptions, + AgentTapeContextOptions, + AgentTapeContextResult, + AgentTapeHandoffState, + AgentTapeSearchOptions, + ChatMessageRecord, + SubagentTapeLinkInput, + SubagentTapeLinkReceipt +} from '@shared/types/agent-interface' +import type { + DeepChatTapeViewManifest, + DeepChatTapeViewManifestRecord +} from '@shared/types/tape-view-manifest' +import type { + DeepChatCausalObservationReadOptions, + DeepChatCausalObservationSlice, + DeepChatTapeReplayExportOptions, + DeepChatTapeReplaySlice +} from '@shared/types/tape-replay' +import type { DeepChatTapeEntryRow } from '../domain/entry' +import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' +import type { TapeToolFactWriter } from '../ports/capabilities' +import { createTapeApplicationProviders, type TapeApplicationDatabase } from '../ports/application' +import type { + TapeAnchorResult, + TapeBackfillResult, + TapeForkHandle, + TapeInfo, + TapeMigrationState, + TapeSearchResult, + TapeViewManifestSourceMaps +} from './contracts' +import { TapeFactService } from './factService' +import { normalizeTapeHandoffState, TapeForkService } from './forkService' +import { + AgentTapeViewError, + normalizeSubagentTapeLinkInput, + TapeLineageService, + type AgentTapeViewErrorCode +} from './lineageService' +import { TapeRecallService } from './recallService' +import { TapeReconcilerService, type TapeTranscriptReader } from './reconcilerService' +import { TapeViewReplayService } from './viewReplayService' + +export type { + AgentTapeViewErrorCode, + TapeAnchorResult, + TapeBackfillResult, + TapeForkHandle, + TapeInfo, + TapeMigrationState, + TapeSearchResult, + TapeViewManifestSourceMaps +} +export { AgentTapeViewError, normalizeSubagentTapeLinkInput, normalizeTapeHandoffState } + +export class SessionTape implements TapeToolFactWriter { + private readonly facts: TapeFactService + private readonly reconciler: TapeReconcilerService + private readonly recall: TapeRecallService + private readonly lineage: TapeLineageService + private readonly viewReplay: TapeViewReplayService + private readonly forks: TapeForkService + + constructor(database: TapeApplicationDatabase) { + const providers = createTapeApplicationProviders(database) + this.facts = new TapeFactService(providers) + this.lineage = new TapeLineageService(providers) + this.reconciler = new TapeReconcilerService(providers, this.facts) + this.recall = new TapeRecallService(providers, this.lineage) + this.viewReplay = new TapeViewReplayService(providers) + this.forks = new TapeForkService(providers, this.facts) + } + + ensureSessionTapeReady( + sessionId: string, + messageStore: TapeTranscriptReader + ): TapeBackfillResult { + return this.reconciler.ensureSessionTapeReady(sessionId, messageStore) + } + + appendMessageRecord(record: ChatMessageRecord): number { + return this.facts.appendMessageRecord(record) + } + + appendToolFact(input: TapeToolFactInput): Promise { + return this.facts.appendToolFact(input) + } + + getMessageRecords(sessionId: string): ChatMessageRecord[] { + return this.facts.getMessageRecords(sessionId) + } + + info(sessionId: string): TapeInfo { + return this.recall.info(sessionId) + } + + search(sessionId: string, query: string, options?: AgentTapeSearchOptions): TapeSearchResult[] { + return this.recall.search(sessionId, query, options) + } + + getContext( + sessionId: string, + entryIds: number[], + options: AgentTapeContextOptions = {} + ): AgentTapeContextResult { + return this.recall.getContext(sessionId, entryIds, options) + } + + anchors(sessionId: string, options: AgentTapeAnchorsOptions = {}): TapeAnchorResult[] { + return this.recall.anchors(sessionId, options) + } + + getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestSourceMaps { + return this.viewReplay.getViewManifestSourceMaps(sessionId, messageId) + } + + appendViewManifest(manifest: DeepChatTapeViewManifest): DeepChatTapeEntryRow { + return this.viewReplay.appendViewManifest(manifest) + } + + listViewManifestsByMessage( + sessionId: string, + messageId: string + ): DeepChatTapeViewManifestRecord[] { + return this.viewReplay.listViewManifestsByMessage(sessionId, messageId) + } + + exportReplaySlice( + sessionId: string, + messageId: string, + options: DeepChatTapeReplayExportOptions = {} + ): DeepChatTapeReplaySlice | null { + return this.viewReplay.exportReplaySlice(sessionId, messageId, options) + } + + readCausalObservationSlice( + sessionId: string, + messageId: string, + options: DeepChatCausalObservationReadOptions = {} + ): DeepChatCausalObservationSlice { + return this.viewReplay.readCausalObservationSlice(sessionId, messageId, options) + } + + handoff( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): DeepChatTapeEntryRow { + return this.forks.handoff(sessionId, name, state, meta) + } + + handoffResult( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): TapeAnchorResult { + return this.forks.handoffResult(sessionId, name, state, meta) + } + + createFork(parentSessionId: string, forkId?: string): TapeForkHandle { + return this.forks.createFork(parentSessionId, forkId) + } + + appendForkMessageRecord(handle: TapeForkHandle, record: ChatMessageRecord): number { + return this.forks.appendForkMessageRecord(handle, record) + } + + mergeFork(parentSessionId: string, forkId: string): number { + return this.forks.mergeFork(parentSessionId, forkId) + } + + discardFork(parentSessionId: string, forkId: string): void { + this.forks.discardFork(parentSessionId, forkId) + } + + recordExternalForkMerge( + parentSessionId: string, + forkSessionId: string, + forkId: string, + meta: Record = {} + ): DeepChatTapeEntryRow { + return this.forks.recordExternalForkMerge(parentSessionId, forkSessionId, forkId, meta) + } + + recordExternalForkDiscard( + parentSessionId: string, + forkSessionId: string, + forkId: string, + meta: Record = {} + ): DeepChatTapeEntryRow { + return this.forks.recordExternalForkDiscard(parentSessionId, forkSessionId, forkId, meta) + } + + linkSubagentTape(input: SubagentTapeLinkInput): SubagentTapeLinkReceipt { + return this.lineage.linkSubagentTape(input) + } +} diff --git a/src/main/tape/application/viewReplayService.ts b/src/main/tape/application/viewReplayService.ts new file mode 100644 index 0000000000..08660720d0 --- /dev/null +++ b/src/main/tape/application/viewReplayService.ts @@ -0,0 +1,621 @@ +import type { + DeepChatTapeViewExcludedRange, + DeepChatTapeViewManifest, + DeepChatTapeViewManifestRecord +} from '@shared/types/tape-view-manifest' +import type { + DeepChatCausalObservationReadOptions, + DeepChatCausalObservationRequest, + DeepChatCausalObservationSlice, + DeepChatTapeReplayEntrySnapshot, + DeepChatTapeReplayExportOptions, + DeepChatTapeReplaySlice, + DeepChatTapeReplayTraceSnapshot +} from '@shared/types/tape-replay' +import { SUMMARY_ANCHOR_NAMES, type DeepChatTapeEntryRow } from '../domain/entry' +import { buildEffectiveTapeView } from '../domain/effectiveView' +import { + hashJson, + TAPE_VIEW_MANIFEST_EVENT_NAME, + verifyTapeViewManifestHash +} from '../domain/viewManifest' +import type { + TapeApplicationProviders, + TapeMessageTraceRow as DeepChatMessageTraceRow +} from '../ports/application' +import { + collectEntryIds, + hashString, + isPositiveInteger, + parseJsonObject, + withReplaySliceHash +} from './common' +import type { TapeViewManifestSourceMaps } from './contracts' + +type TapeViewReplayProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getMessageTraceReader' | 'getTerminalMessageReader' +> + +const BOOTSTRAP_ANCHOR_NAME = 'session/start' + +function isReconstructionAnchorName(name: string | null): boolean { + if (name === null) { + return false + } + return ( + (SUMMARY_ANCHOR_NAMES as readonly string[]).includes(name) || + name.startsWith('handoff/') || + name.startsWith('auto_handoff/') + ) +} + +function readToolFactStatus(row: DeepChatTapeEntryRow): string | null { + const status = parseJsonObject(row.meta_json).status + return typeof status === 'string' ? status : null +} + +function readToolFactToolCallId(row: DeepChatTapeEntryRow): string | null { + const payload = parseJsonObject(row.payload_json) + if (row.kind === 'tool_call') { + const toolCall = payload.toolCall + if (toolCall && typeof toolCall === 'object' && !Array.isArray(toolCall)) { + const id = (toolCall as Record).id + return typeof id === 'string' && id.length > 0 ? id : null + } + return null + } + const toolCallId = payload.toolCallId + return typeof toolCallId === 'string' && toolCallId.length > 0 ? toolCallId : null +} + +function readToolFactMessageId(row: DeepChatTapeEntryRow): string | null { + const messageId = parseJsonObject(row.payload_json).messageId + return typeof messageId === 'string' && messageId.length > 0 ? messageId : null +} + +const VIEW_POLICIES = new Set([ + 'legacy_context_v1', + 'legacy_context_shadow', + 'resume_shadow', + 'tool_loop_shadow', + 'context_pressure_recovery_shadow' +]) + +const VIEW_ENTRY_REASONS = new Set([ + 'system_prompt', + 'selected_history', + 'new_user_input', + 'resume_target', + 'tool_loop_message' +]) + +const VIEW_EXCLUDED_REASONS = new Set([ + 'before_summary_cursor', + 'compaction_indicator', + 'pending_not_context_history', + 'out_of_budget', + 'empty_after_formatting', + 'superseded', + 'retracted' +]) + +function isRecordObject(value: unknown): value is Record { + return Boolean(value && typeof value === 'object' && !Array.isArray(value)) +} + +function isNullableString(value: unknown): value is string | null { + return value === null || typeof value === 'string' +} + +function isNullableNumber(value: unknown): value is number | null { + return value === null || typeof value === 'number' +} + +function isViewEntryRef(value: unknown): value is DeepChatTapeViewManifest['included'][number] { + if (!isRecordObject(value)) { + return false + } + + return ( + isNullableNumber(value.entryId) && + isNullableString(value.messageId) && + isNullableNumber(value.orderSeq) && + (value.role === 'system' || + value.role === 'user' || + value.role === 'assistant' || + value.role === 'tool' || + value.role === null) && + (value.source === 'tape' || value.source === 'synthetic') && + typeof value.reason === 'string' && + VIEW_ENTRY_REASONS.has(value.reason) + ) +} + +function isViewExcludedRef(value: unknown): value is DeepChatTapeViewManifest['excluded'][number] { + if (!isRecordObject(value)) { + return false + } + + return ( + isNullableNumber(value.entryId) && + isNullableString(value.messageId) && + isNullableNumber(value.orderSeq) && + typeof value.reason === 'string' && + VIEW_EXCLUDED_REASONS.has(value.reason) + ) +} + +function isViewExcludedRange(value: unknown): value is DeepChatTapeViewExcludedRange { + if (!isRecordObject(value)) { + return false + } + + return ( + typeof value.fromOrderSeq === 'number' && + typeof value.toOrderSeq === 'number' && + typeof value.count === 'number' && + typeof value.reason === 'string' && + VIEW_EXCLUDED_REASONS.has(value.reason) + ) +} + +function hasNumberFields(value: unknown, fields: string[]): value is Record { + if (!isRecordObject(value)) { + return false + } + + return fields.every((field) => typeof value[field] === 'number') +} + +function hasStringFields(value: unknown, fields: string[]): value is Record { + if (!isRecordObject(value)) { + return false + } + + return fields.every((field) => typeof value[field] === 'string') +} + +function isViewManifestMeta(value: unknown): value is DeepChatTapeViewManifest['meta'] { + if (!isRecordObject(value)) { + return false + } + + return ( + typeof value.providerId === 'string' && + typeof value.modelId === 'string' && + typeof value.summaryCursorOrderSeq === 'number' && + typeof value.supportsVision === 'boolean' && + typeof value.supportsAudioInput === 'boolean' && + typeof value.traceDebugEnabled === 'boolean' + ) +} + +function isViewManifest(value: unknown, sessionId: string): value is DeepChatTapeViewManifest { + if (!isRecordObject(value)) { + return false + } + + return ( + (value.schemaVersion === 1 || value.schemaVersion === 2) && + typeof value.hashVersion === 'number' && + value.sessionId === sessionId && + typeof value.viewId === 'string' && + typeof value.messageId === 'string' && + typeof value.requestSeq === 'number' && + (value.taskType === 'chat' || value.taskType === 'resume' || value.taskType === 'tool_loop') && + typeof value.policy === 'string' && + VIEW_POLICIES.has(value.policy) && + (typeof value.policyVersion === 'number' || value.policyVersion === null) && + value.contextBuilderVersion === 'legacy-v1' && + typeof value.latestEntryId === 'number' && + Array.isArray(value.anchorEntryIds) && + value.anchorEntryIds.every((entryId) => typeof entryId === 'number') && + (value.reconstructionAnchorEntryId === undefined || + isNullableNumber(value.reconstructionAnchorEntryId)) && + (value.excludedRanges === undefined || + (Array.isArray(value.excludedRanges) && value.excludedRanges.every(isViewExcludedRange))) && + Array.isArray(value.included) && + value.included.every(isViewEntryRef) && + Array.isArray(value.excluded) && + value.excluded.every(isViewExcludedRef) && + hasNumberFields(value.tokenBudget, [ + 'contextLength', + 'requestedMaxTokens', + 'effectiveMaxTokens', + 'reserveTokens', + 'toolReserveTokens', + 'estimatedPromptTokens' + ]) && + hasStringFields(value.hashes, ['promptHash', 'toolDefinitionsHash', 'manifestHash']) && + isViewManifestMeta(value.meta) && + typeof value.assembledAt === 'number' + ) +} + +export class TapeViewReplayService { + constructor(private readonly providers: TapeViewReplayProviders) {} + + private get table() { + return this.providers.getEntryStore() + } + + getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestSourceMaps { + const table = this.table + const rows = table.getBySession(sessionId) + const entryIdByMessageId = new Map() + const toolCallEntryIdByToolId = new Map() + const toolResultEntryIdByToolId = new Map() + let latestEntryId = 0 + const anchorEntryIds: number[] = [] + let reconstructionAnchorEntryId: number | null = null + let bootstrapAnchorEntryId: number | null = null + + for (const row of rows) { + latestEntryId = Math.max(latestEntryId, row.entry_id) + if (row.kind === 'anchor') { + anchorEntryIds.push(row.entry_id) + if (isReconstructionAnchorName(row.name)) { + if (reconstructionAnchorEntryId === null || row.entry_id > reconstructionAnchorEntryId) { + reconstructionAnchorEntryId = row.entry_id + } + } else if (row.name === BOOTSTRAP_ANCHOR_NAME) { + bootstrapAnchorEntryId = row.entry_id + } + continue + } + if (row.kind === 'message' && row.source_type === 'message' && row.source_id) { + entryIdByMessageId.set(row.source_id, row.entry_id) + continue + } + if (row.kind === 'tool_call' || row.kind === 'tool_result') { + if (messageId && readToolFactMessageId(row) !== messageId) { + continue + } + const toolCallId = readToolFactToolCallId(row) + if (!toolCallId || readToolFactStatus(row) === 'pending') { + continue + } + const target = + row.kind === 'tool_call' ? toolCallEntryIdByToolId : toolResultEntryIdByToolId + target.set(toolCallId, row.entry_id) + } + } + + const reconstructionAnchorEntryIds = + reconstructionAnchorEntryId !== null + ? [reconstructionAnchorEntryId] + : bootstrapAnchorEntryId !== null + ? [bootstrapAnchorEntryId] + : [] + + return { + latestEntryId, + anchorEntryIds, + reconstructionAnchorEntryIds, + reconstructionAnchorEntryId, + entryIdByMessageId, + toolCallEntryIdByToolId, + toolResultEntryIdByToolId + } + } + + appendViewManifest(manifest: DeepChatTapeViewManifest): DeepChatTapeEntryRow { + const table = this.table + table.ensureBootstrapAnchor(manifest.sessionId) + return table.appendEvent({ + sessionId: manifest.sessionId, + name: TAPE_VIEW_MANIFEST_EVENT_NAME, + source: { + type: 'runtime_event', + id: manifest.messageId, + seq: manifest.requestSeq + }, + provenanceKey: `view:${manifest.sessionId}:${manifest.messageId}:${manifest.requestSeq}:${manifest.hashes.manifestHash}`, + data: { + manifest + }, + meta: { + viewId: manifest.viewId, + requestSeq: manifest.requestSeq, + taskType: manifest.taskType, + policy: manifest.policy, + policyVersion: manifest.policyVersion + }, + createdAt: manifest.assembledAt, + idempotent: true + }) + } + + listViewManifestsByMessage( + sessionId: string, + messageId: string + ): DeepChatTapeViewManifestRecord[] { + const table = this.table + return table + .getBySession(sessionId) + .filter( + (row) => + row.kind === 'event' && + row.name === TAPE_VIEW_MANIFEST_EVENT_NAME && + row.source_type === 'runtime_event' && + row.source_id === messageId + ) + .map((row) => this.toViewManifestRecord(row)) + .filter((record): record is DeepChatTapeViewManifestRecord => Boolean(record)) + .sort((left, right) => right.requestSeq - left.requestSeq || right.entryId - left.entryId) + } + + exportReplaySlice( + sessionId: string, + messageId: string, + options: DeepChatTapeReplayExportOptions = {} + ): DeepChatTapeReplaySlice | null { + if (options.requestSeq !== undefined && !isPositiveInteger(options.requestSeq)) { + throw new Error('requestSeq must be a positive integer.') + } + + const manifests = this.listViewManifestsByMessage(sessionId, messageId) + const manifestRecord = + options.requestSeq === undefined + ? manifests[0] + : manifests.find((record) => record.requestSeq === options.requestSeq) + if (!manifestRecord) { + return null + } + + return this.buildReplaySlice(sessionId, messageId, manifestRecord, options) + } + + readCausalObservationSlice( + sessionId: string, + messageId: string, + options: DeepChatCausalObservationReadOptions = {} + ): DeepChatCausalObservationSlice { + if (options.requestSeq !== undefined && !isPositiveInteger(options.requestSeq)) { + throw new Error('requestSeq must be a positive integer.') + } + + const rows = this.table.getBySession(sessionId) + const manifestRows = rows.filter( + (row) => + row.kind === 'event' && + row.name === TAPE_VIEW_MANIFEST_EVENT_NAME && + row.source_type === 'runtime_event' && + row.source_id === messageId + ) + const traces = this.providers + .getMessageTraceReader() + .listByMessageId(messageId) + .filter( + (row) => + row.session_id === sessionId && + row.message_id === messageId && + isPositiveInteger(row.request_seq) + ) + + const requestSeq = + options.requestSeq ?? + [...manifestRows.map((row) => row.source_seq), ...traces.map((row) => row.request_seq)] + .filter((value): value is number => typeof value === 'number' && isPositiveInteger(value)) + .reduce((latest, value) => Math.max(latest ?? value, value), null) + + let request: DeepChatCausalObservationRequest + if (requestSeq === null) { + request = { state: 'request_unavailable', requestSeq: null, trace: null } + } else { + const selectedManifestRows = manifestRows.filter((row) => row.source_seq === requestSeq) + const manifestRecord = selectedManifestRows + .map((row) => this.toViewManifestRecord(row)) + .find((record) => record?.messageId === messageId && record.requestSeq === requestSeq) + const trace = traces.find((row) => row.request_seq === requestSeq) ?? null + + if (manifestRecord) { + request = { + state: 'manifest_bound', + requestSeq, + replay: this.buildReplaySlice(sessionId, messageId, manifestRecord, options) + } + } else { + request = { + state: selectedManifestRows.length > 0 ? 'manifest_malformed' : 'manifest_missing', + requestSeq, + trace: trace + ? this.toReplayTraceSnapshot(trace, options.includeTracePayload === true) + : null + } + } + } + + const outputEntries = buildEffectiveTapeView(rows, { includePending: false }) + .rows.filter( + (row) => + (row.kind === 'message' && + row.source_type === 'message' && + row.source_id === messageId) || + ((row.kind === 'tool_call' || row.kind === 'tool_result') && + readToolFactMessageId(row) === messageId) + ) + .map((row) => this.toReplayEntrySnapshot(row, options.includeTapePayloads === true)) + const message = this.providers.getTerminalMessageReader().get(messageId) + const terminalMessage = + message?.session_id === sessionId && + message.role === 'assistant' && + (message.status === 'sent' || message.status === 'error') + ? { + status: message.status, + orderSeq: message.order_seq, + createdAt: message.created_at, + updatedAt: message.updated_at, + contentHash: hashString(message.content), + metadataHash: hashString(message.metadata) + } + : null + + return { + schemaVersion: 1, + sessionId, + messageId, + request, + output: { + correlation: 'message_only', + entries: outputEntries, + terminalMessage + }, + runtime: + options.currentRuntimeStatus === undefined + ? { scope: 'unavailable', status: null, eventHistory: 'not_persisted' } + : { + scope: 'current_only', + status: options.currentRuntimeStatus, + eventHistory: 'not_persisted' + } + } + } + + private buildReplaySlice( + sessionId: string, + messageId: string, + manifestRecord: DeepChatTapeViewManifestRecord, + options: DeepChatTapeReplayExportOptions + ): DeepChatTapeReplaySlice { + const table = this.table + const manifest = manifestRecord.manifest + const includedEntryIds = collectEntryIds(manifest.included.map((ref) => ref.entryId)) + const excludedEntryIds = collectEntryIds(manifest.excluded.map((ref) => ref.entryId)) + const anchorEntryIds = collectEntryIds(manifest.anchorEntryIds) + const selectedEntryIds = new Set([ + manifestRecord.entryId, + ...includedEntryIds, + ...excludedEntryIds, + ...anchorEntryIds + ]) + const entries = table + .getBySession(sessionId) + .filter((row) => selectedEntryIds.has(row.entry_id)) + .map((row) => this.toReplayEntrySnapshot(row, options.includeTapePayloads === true)) + + const trace = this.findReplayTrace(sessionId, messageId, manifestRecord.requestSeq) + const createdAt = Date.now() + const sliceBase: Omit & { + hashes: Omit & { sliceHash: '' } + } = { + schemaVersion: 1 as const, + sliceId: `replay_${hashJson({ + sessionId, + messageId, + requestSeq: manifestRecord.requestSeq, + manifestHash: manifest.hashes.manifestHash + }).slice(0, 16)}`, + sessionId, + messageId, + requestSeq: manifestRecord.requestSeq, + mode: trace ? 'trace_bound' : 'manifest_only', + manifestRecord, + trace: trace ? this.toReplayTraceSnapshot(trace, options.includeTracePayload === true) : null, + entries, + refs: { + manifestEntryId: manifestRecord.entryId, + includedEntryIds, + excludedEntryIds, + anchorEntryIds + }, + hashes: { + manifestHash: manifest.hashes.manifestHash, + sliceHash: '' + }, + integrity: manifestRecord.integrity, + createdAt + } + + return withReplaySliceHash(sliceBase, hashJson) + } + + private toViewManifestRecord(row: DeepChatTapeEntryRow): DeepChatTapeViewManifestRecord | null { + const payload = parseJsonObject(row.payload_json) + const data = payload.data + const rawManifest = + data && typeof data === 'object' && !Array.isArray(data) + ? (data as Record).manifest + : undefined + const manifest = + isRecordObject(rawManifest) && rawManifest.hashVersion === undefined + ? { ...rawManifest, hashVersion: 1 } + : rawManifest + if (!isViewManifest(manifest, row.session_id)) { + return null + } + + return { + sessionId: row.session_id, + messageId: manifest.messageId, + requestSeq: manifest.requestSeq, + entryId: row.entry_id, + createdAt: row.created_at, + integrity: verifyTapeViewManifestHash(manifest), + manifest + } + } + + private findReplayTrace( + sessionId: string, + messageId: string, + requestSeq: number + ): DeepChatMessageTraceRow | null { + const traceTable = this.providers.getMessageTraceReader() + return ( + traceTable + .listByMessageId(messageId) + .find((row) => row.session_id === sessionId && row.request_seq === requestSeq) ?? null + ) + } + + private toReplayEntrySnapshot( + row: DeepChatTapeEntryRow, + includePayloads: boolean + ): DeepChatTapeReplayEntrySnapshot { + const snapshot: DeepChatTapeReplayEntrySnapshot = { + entryId: row.entry_id, + kind: row.kind, + name: row.name, + sourceType: row.source_type, + sourceId: row.source_id, + sourceSeq: row.source_seq, + provenanceKey: row.provenance_key, + payloadHash: hashString(row.payload_json), + metaHash: hashString(row.meta_json), + createdAt: row.created_at + } + + if (includePayloads) { + snapshot.payload = parseJsonObject(row.payload_json) + snapshot.meta = parseJsonObject(row.meta_json) + } + + return snapshot + } + + private toReplayTraceSnapshot( + row: DeepChatMessageTraceRow, + includePayload: boolean + ): DeepChatTapeReplayTraceSnapshot { + const snapshot: DeepChatTapeReplayTraceSnapshot = { + id: row.id, + requestSeq: row.request_seq, + providerId: row.provider_id, + modelId: row.model_id, + endpoint: row.endpoint, + headersHash: hashString(row.headers_json), + bodyHash: hashString(row.body_json), + truncated: row.truncated === 1, + createdAt: row.created_at + } + + if (includePayload) { + snapshot.headersJson = row.headers_json + snapshot.bodyJson = row.body_json + } + + return snapshot + } +} diff --git a/src/main/tape/ports/application.ts b/src/main/tape/ports/application.ts new file mode 100644 index 0000000000..ccda7e4fd1 --- /dev/null +++ b/src/main/tape/ports/application.ts @@ -0,0 +1,173 @@ +import type { + DeepChatTapeEntryKind, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + DeepChatTapeSourceType +} from '../domain/entry' +import type { + TapeBootstrapStore, + TapeEntryLifecycleStore, + TapeEntryStore, + TapeTransactionRunner +} from './storage' + +export interface TapeSearchProjectionInput { + sessionId: string + entryId: number + kind: DeepChatTapeEntryKind + name: string | null + sourceType: DeepChatTapeSourceType | null + sourceId: string | null + sourceSeq: number | null + searchText: string + summaryText: string + refs: Record + createdAt: number +} + +export interface TapeSearchProjectionRow { + session_id: string + entry_id: number + kind: DeepChatTapeEntryKind + name: string | null + source_type: DeepChatTapeSourceType | null + source_id: string | null + source_seq: number | null + search_text: string + summary_text: string + refs_json: string + created_at: number +} + +export interface TapeSearchProjectionResultRow extends TapeSearchProjectionRow { + score: number | null +} + +export interface TapeSearchProjectionMeta { + projectionVersion: number + maxEntryId: number +} + +export interface TapeSearchProjectionReadResult { + rows: TapeSearchProjectionResultRow[] + coveredSources: DeepChatTapeReadSource[] +} + +export interface TapeSearchProjectionStore { + getSessionMeta(sessionId: string): TapeSearchProjectionMeta | null + isCurrent(sessionId: string, maxEntryId: number, projectionVersion?: number): boolean + getProjectedEntryIds(sessionId: string): number[] + appendSession( + sessionId: string, + rows: TapeSearchProjectionInput[], + maxEntryId: number, + projectionVersion?: number + ): void + replaceSession( + sessionId: string, + rows: TapeSearchProjectionInput[], + maxEntryId: number, + projectionVersion?: number + ): void + getByEntryIds(sessionId: string, entryIds: number[]): TapeSearchProjectionRow[] + searchSourcesReadOnly( + sources: readonly DeepChatTapeReadSource[], + query: string, + options?: DeepChatTapeSearchInput + ): TapeSearchProjectionReadResult + search( + sessionId: string, + query: string, + options?: DeepChatTapeSearchInput + ): TapeSearchProjectionResultRow[] + deleteBySession(sessionId: string): void +} + +export interface TapeLineageSessionRow { + id: string + session_kind: 'regular' | 'subagent' + parent_session_id: string | null +} + +export interface TapeLineageSessionReader { + get(sessionId: string): TapeLineageSessionRow | undefined + getMany(sessionIds: string[]): TapeLineageSessionRow[] +} + +export interface TapeLegacySummaryState { + summary_text: string | null + summary_cursor_order_seq: number | null + summary_updated_at: number | null +} + +export interface TapeLegacySummaryReader { + getSummaryState(sessionId: string): TapeLegacySummaryState | null +} + +export interface TapeMessageTraceRow { + id: string + message_id: string + session_id: string + provider_id: string + model_id: string + request_seq: number + endpoint: string + headers_json: string + body_json: string + truncated: number + created_at: number +} + +export interface TapeMessageTraceReader { + listByMessageId(messageId: string): TapeMessageTraceRow[] +} + +export interface TapeTerminalMessageRow { + session_id: string + order_seq: number + role: 'user' | 'assistant' + content: string + status: 'pending' | 'sent' | 'error' + metadata: string + created_at: number + updated_at: number +} + +export interface TapeTerminalMessageReader { + get(messageId: string): TapeTerminalMessageRow | undefined +} + +export type TapeApplicationEntryStore = TapeEntryStore & TapeTransactionRunner & TapeBootstrapStore + +export interface TapeApplicationDatabase { + readonly deepchatTapeEntriesTable: TapeApplicationEntryStore & TapeEntryLifecycleStore + readonly deepchatTapeSearchProjectionTable: TapeSearchProjectionStore + readonly newSessionsTable: TapeLineageSessionReader + readonly deepchatSessionsTable: TapeLegacySummaryReader + readonly deepchatMessageTracesTable: TapeMessageTraceReader + readonly deepchatMessagesTable: TapeTerminalMessageReader +} + +export interface TapeApplicationProviders { + getEntryStore(): TapeApplicationEntryStore + getForkStore(): TapeApplicationEntryStore & TapeEntryLifecycleStore + getSearchProjectionStore(): TapeSearchProjectionStore + getLineageSessionReader(): TapeLineageSessionReader + getLegacySummaryReader(): TapeLegacySummaryReader + getMessageTraceReader(): TapeMessageTraceReader + getTerminalMessageReader(): TapeTerminalMessageReader +} + +export function createTapeApplicationProviders( + database: TapeApplicationDatabase +): TapeApplicationProviders { + return { + getEntryStore: () => database.deepchatTapeEntriesTable, + getForkStore: () => database.deepchatTapeEntriesTable, + getSearchProjectionStore: () => database.deepchatTapeSearchProjectionTable, + getLineageSessionReader: () => database.newSessionsTable, + getLegacySummaryReader: () => database.deepchatSessionsTable, + getMessageTraceReader: () => database.deepchatMessageTracesTable, + getTerminalMessageReader: () => database.deepchatMessagesTable + } +} From efab7846a0477d28d5bbb8a8076c981937d51706 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 16:18:23 +0800 Subject: [PATCH 05/29] refactor(tape): close storage bypasses --- docs/architecture/tape-layering/tasks.md | 14 +- scripts/generate-architecture-baseline.mjs | 9 +- .../memory/memoryRuntimeCoordinator.ts | 18 +- .../runtime/deepChatRuntimeCoordinator.ts | 7 +- src/main/app/composition.ts | 2 +- .../legacyChatImportService.ts | 2 + src/main/data/schemaCatalog.ts | 4 +- .../deepchatMemoryIngestionProjection.ts | 2 + src/main/session/data/database.ts | 9 +- src/main/session/data/index.ts | 4 +- src/main/session/data/settings.ts | 36 +- .../data/tables/deepchatTapeEntries.ts | 1080 +---------------- .../tables/deepchatTapeSearchProjection.ts | 899 +------------- src/main/session/data/transcript.ts | 40 +- src/main/tape/application/forkService.ts | 13 +- src/main/tape/application/sessionTape.ts | 105 +- .../infrastructure/sqlite/tapeEntryStore.ts | 1070 ++++++++++++++++ .../sqlite/tapeLifecycleAdapter.ts | 17 + .../sqlite/tapeSearchProjectionStore.ts | 895 ++++++++++++++ src/main/tape/ports/application.ts | 7 +- src/main/tape/ports/capabilities.ts | 1 + .../memory/memoryRuntimeCoordinator.test.ts | 14 +- .../deepChatRuntimeCoordinator.test.ts | 1 + .../tables/deepchatTapeEntriesTable.test.ts | 51 +- test/main/session/data/tapeFork.test.ts | 8 + test/main/session/data/tapeLifecycle.test.ts | 56 + test/main/session/data/tapeTestHarness.ts | 4 + test/main/session/runtimeIntegration.test.ts | 1 + 28 files changed, 2294 insertions(+), 2075 deletions(-) create mode 100644 src/main/tape/infrastructure/sqlite/tapeEntryStore.ts create mode 100644 src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts create mode 100644 src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts create mode 100644 test/main/session/data/tapeLifecycle.test.ts diff --git a/docs/architecture/tape-layering/tasks.md b/docs/architecture/tape-layering/tasks.md index 9c9f2acc56..48bf64fcc6 100644 --- a/docs/architecture/tape-layering/tasks.md +++ b/docs/architecture/tape-layering/tasks.md @@ -33,13 +33,13 @@ ## Infrastructure and Bypass Closure -- [ ] Separate normal SQLite entry operations from destructive lifecycle operations. -- [ ] Inject message fact capabilities into transcript without changing transaction boundaries. -- [ ] Inject anchor and lifecycle capabilities into Session settings. -- [ ] Replace Memory runtime and route table access with explicit capabilities. -- [ ] Document and allowlist startup migration and Memory projection infrastructure exceptions. -- [ ] Add lifecycle, transaction, projection, and authorization tests. -- [ ] Review and commit the storage-boundary slice. +- [x] Separate normal SQLite entry operations from destructive lifecycle operations. +- [x] Inject message fact capabilities into transcript without changing transaction boundaries. +- [x] Inject anchor and lifecycle capabilities into Session settings. +- [x] Replace Memory runtime and route table access with explicit capabilities. +- [x] Document and allowlist startup migration and Memory projection infrastructure exceptions. +- [x] Add lifecycle, transaction, projection, and authorization tests. +- [x] Review and commit the storage-boundary slice. ## Architecture Enforcement diff --git a/scripts/generate-architecture-baseline.mjs b/scripts/generate-architecture-baseline.mjs index 2172da73c4..ec66e5d901 100644 --- a/scripts/generate-architecture-baseline.mjs +++ b/scripts/generate-architecture-baseline.mjs @@ -24,7 +24,8 @@ const AGENT_SYSTEM_RUNTIME_BOUNDARY_FILES = [ 'src/main/agent/deepchat/runtime/process.ts', 'src/main/agent/deepchat/runtime/dispatch.ts', 'src/main/session/data/transcript.ts', - 'src/main/session/data/tape.ts', + 'src/main/tape/application/sessionTape.ts', + 'src/main/tape/ports/capabilities.ts', 'src/main/provider/providers/acpProvider.ts' ] const AGENT_SYSTEM_EXPECTED_FILES = [ @@ -73,7 +74,11 @@ const AGENT_SYSTEM_OWNER_EVIDENCE = [ 'src/main/agent/deepchat/loop/deepChatLoopEngine.ts', /\bclass DeepChatLoopEngine\b/g ], - ['tapeRecorder', 'src/main/agent/deepchat/loop/ports.ts', /\binterface TapeRecorder\b/g], + [ + 'tapeToolFactWriter', + 'src/main/tape/ports/capabilities.ts', + /\binterface TapeToolFactWriter\b/g + ], [ 'memoryRuntimeCoordinator', 'src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts', diff --git a/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts b/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts index c3c90d5b7f..59719edcc0 100644 --- a/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts +++ b/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts @@ -11,6 +11,7 @@ import type { DeepChatMemoryIngestionProjectionRow } from '@/memory/data/tables/deepchatMemoryIngestionProjection' import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeAnchorWriter, TapeRawEntryReader } from '@/tape/ports/capabilities' import { MEMORY_EXTRACTION_CHUNKS_PER_QUEUE_TASK, buildMemoryExtractionChunks, @@ -68,13 +69,8 @@ export interface MemoryRuntimeCoordinatorDependencies { getMemoryCursorOrderSeq(sessionId: string): number | null updateMemoryCursorOrderSeq(sessionId: string, orderSeq: number): void rewindMemoryCursorOrderSeq(sessionId: string, orderSeq: number): void - getTapeRows(sessionId: string): DeepChatTapeEntryRow[] - appendTapeAnchor(input: { - sessionId: string - name: string - state: Record - meta?: Record - }): void + tapeReader: TapeRawEntryReader + tapeAnchorWriter: TapeAnchorWriter getIngestionProjection(): MemoryIngestionProjection | undefined } @@ -193,7 +189,7 @@ export class MemoryRuntimeCoordinator implements MemoryPromptContributor, Memory } if (this.canContinueExecution(sessionId, executionToken)) { try { - this.deps.appendTapeAnchor({ + this.deps.tapeAnchorWriter.appendAnchor({ sessionId, name: 'memory/view_assembled', state: assembled.manifest as unknown as Record, @@ -406,7 +402,7 @@ export class MemoryRuntimeCoordinator implements MemoryPromptContributor, Memory this.deps.updateMemoryCursorOrderSeq(sessionId, chunk.cursorCommitOrderSeq) } if (result.createdIds.length > 0) { - this.deps.appendTapeAnchor({ + this.deps.tapeAnchorWriter.appendAnchor({ sessionId, name: 'memory/extract', state: { @@ -678,7 +674,7 @@ export class MemoryRuntimeCoordinator implements MemoryPromptContributor, Memory maxEntryId: number, projection: MemoryIngestionProjection ): { rows: DeepChatMemoryIngestionProjectionRow[]; cursorCommitAllowed: boolean } { - const view = buildEffectiveTapeView(this.deps.getTapeRows(sessionId)) + const view = buildEffectiveTapeView(this.deps.tapeReader.getBySession(sessionId)) const projectionRows = this.projectionRowsFromEffectiveView(sessionId, view) try { projection.replaceSession(sessionId, projectionRows, maxEntryId) @@ -732,7 +728,7 @@ export class MemoryRuntimeCoordinator implements MemoryPromptContributor, Memory cursorCommitAllowed: boolean ): { rows: DeepChatMemoryIngestionProjectionRow[]; cursorCommitAllowed: boolean } | null { try { - const view = buildEffectiveTapeView(this.deps.getTapeRows(sessionId)) + const view = buildEffectiveTapeView(this.deps.tapeReader.getBySession(sessionId)) const rows = this.projectionRowsFromEffectiveView(sessionId, view) return { rows: this.filterIngestionRange(rows, fromOrderSeqExclusive, toOrderSeqInclusive), diff --git a/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts b/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts index d4eee9c9b7..3091c269e7 100644 --- a/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts +++ b/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts @@ -329,11 +329,8 @@ export class DeepChatRuntimeCoordinator { this.sqlitePresenter.deepchatSessionsTable.updateMemoryCursorOrderSeq(sessionId, orderSeq), rewindMemoryCursorOrderSeq: (sessionId, orderSeq) => this.sqlitePresenter.deepchatSessionsTable.rewindMemoryCursorOrderSeq(sessionId, orderSeq), - getTapeRows: (sessionId) => - this.sqlitePresenter.deepchatTapeEntriesTable.getBySession(sessionId), - appendTapeAnchor: (input) => { - this.sqlitePresenter.deepchatTapeEntriesTable.appendAnchor(input) - }, + tapeReader: this.tapeService, + tapeAnchorWriter: this.tapeService, getIngestionProjection: runtimePorts.getMemoryIngestionProjection }) this.memoryPromptContributor = this.memoryCoordinator diff --git a/src/main/app/composition.ts b/src/main/app/composition.ts index fc74d0499d..95ea0adbc2 100644 --- a/src/main/app/composition.ts +++ b/src/main/app/composition.ts @@ -1608,7 +1608,7 @@ export async function createMainProcessControl(dependencies: { const memoryRoutes = createMemoryRoutes({ memoryService, getAgentType: (agentId) => agentSettings.getAgentType(agentId), - getTapeEntries: () => sessionData.database.deepchatTapeEntriesTable, + getTapeEntries: () => sessionData.tapeStore, getAuditEntries: () => memoryDatabase.agentMemoryAuditTable }) const desktopRoutes = createDesktopRoutes({ diff --git a/src/main/app/startupMigrations/legacyChatImportService.ts b/src/main/app/startupMigrations/legacyChatImportService.ts index 89904f61f7..03544175a8 100644 --- a/src/main/app/startupMigrations/legacyChatImportService.ts +++ b/src/main/app/startupMigrations/legacyChatImportService.ts @@ -157,6 +157,8 @@ export class LegacyChatImportService { private async clearImportedSessionData(): Promise { const db = this.sessionDatabase.getDatabase() db.transaction(() => { + // Migration-only exception: overwrite import intentionally clears all legacy-owned tables + // in one transaction, including Tape and its projections. db.exec(` DELETE FROM deepchat_message_search_results; DELETE FROM deepchat_search_documents; diff --git a/src/main/data/schemaCatalog.ts b/src/main/data/schemaCatalog.ts index 5922b76c10..55f4a11899 100644 --- a/src/main/data/schemaCatalog.ts +++ b/src/main/data/schemaCatalog.ts @@ -19,9 +19,9 @@ import { DeepChatMessageSearchResultsTable } from '@/session/data/tables/deepcha import { DeepChatSearchDocumentsTable } from '@/session/data/tables/deepchatSearchDocuments' import { DeepChatPendingInputsTable } from '@/session/data/tables/deepchatPendingInputs' import { DeepChatUsageStatsTable } from '@/session/data/tables/deepchatUsageStats' -import { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' +import { DeepChatTapeEntriesTable } from '@/tape/infrastructure/sqlite/tapeEntryStore' import { DeepChatMemoryIngestionProjectionTable } from '@/memory/data/tables/deepchatMemoryIngestionProjection' -import { DeepChatTapeSearchProjectionTable } from '@/session/data/tables/deepchatTapeSearchProjection' +import { DeepChatTapeSearchProjectionTable } from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' import { DeepChatSessionMetadataTable } from '@/session/data/tables/deepchatSessionMetadata' import { LegacyImportStatusTable } from '@/app/data/tables/legacyImportStatus' import { AgentsTable } from '@/agent/data/tables/agents' diff --git a/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts b/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts index 40dd788de5..c59dfc857d 100644 --- a/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts +++ b/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts @@ -271,6 +271,8 @@ export class DeepChatMemoryIngestionProjectionTable toOrderSeqInclusive: number ): DeepChatMemoryIngestionCurrentRange { this.perfObserver?.increment('repositoryCalls') + // Read-only infrastructure exception: the Tape head and projection head must be observed by + // one SQL statement so concurrent appends cannot create a false-current projection window. const records = this.db .prepare( `WITH state AS ( diff --git a/src/main/session/data/database.ts b/src/main/session/data/database.ts index 14efffc10f..a28c7c4c59 100644 --- a/src/main/session/data/database.ts +++ b/src/main/session/data/database.ts @@ -15,9 +15,10 @@ import { DeepChatMessageSearchResultsTable } from './tables/deepchatMessageSearc import { DeepChatSearchDocumentsTable } from './tables/deepchatSearchDocuments' import { DeepChatPendingInputsTable } from './tables/deepchatPendingInputs' import { DeepChatUsageStatsTable } from './tables/deepchatUsageStats' -import { DeepChatTapeEntriesTable } from './tables/deepchatTapeEntries' +import { DeepChatTapeEntriesTable } from '@/tape/infrastructure/sqlite/tapeEntryStore' import type { TapeMutationProjection } from '@/tape/ports/storage' -import { DeepChatTapeSearchProjectionTable } from './tables/deepchatTapeSearchProjection' +import { SqliteTapeLifecycleAdapter } from '@/tape/infrastructure/sqlite/tapeLifecycleAdapter' +import { DeepChatTapeSearchProjectionTable } from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' import { DeepChatSessionMetadataTable } from './tables/deepchatSessionMetadata' import { NewSessionActiveSkillsTable } from './tables/newSessionActiveSkills' import { NewSessionDisabledAgentToolsTable } from './tables/newSessionDisabledAgentTools' @@ -96,6 +97,10 @@ export class SessionDatabase { return new DeepChatTapeEntriesTable(this.getDatabase(), this.getTapeMutationProjection?.()) } + get tapeLifecycle() { + return new SqliteTapeLifecycleAdapter(this.getDatabase(), this.getTapeMutationProjection?.()) + } + get deepchatTapeSearchProjectionTable() { return new DeepChatTapeSearchProjectionTable(this.getDatabase()) } diff --git a/src/main/session/data/index.ts b/src/main/session/data/index.ts index fc2d05aca4..e1d48032a8 100644 --- a/src/main/session/data/index.ts +++ b/src/main/session/data/index.ts @@ -26,8 +26,8 @@ export function createSessionDataFromDatabase( database: SessionDatabase, events: SessionDataEvents ) { - const transcript = new SessionTranscript(database) const tapeStore = new SessionTape(database) + const transcript = new SessionTranscript(database, tapeStore) const pendingInputStore = new SessionPendingInputStore(database) const ensureTape = (sessionId: string) => tapeStore.ensureSessionTapeReady(sessionId, transcript) const toTapeAnchor = (row: DeepChatTapeEntryRow) => ({ @@ -80,7 +80,7 @@ export function createSessionDataFromDatabase( return { database, - settings: new SessionSettingsStore(database), + settings: new SessionSettingsStore(database, tapeStore), transcript, tape, tapeStore, diff --git a/src/main/session/data/settings.ts b/src/main/session/data/settings.ts index 7b308d3b52..27a2faeba2 100644 --- a/src/main/session/data/settings.ts +++ b/src/main/session/data/settings.ts @@ -2,6 +2,12 @@ import { SessionDatabase } from './database' import type { PermissionMode, SessionGenerationSettings } from '@shared/types/agent-interface' import type { DeepChatSessionSummaryRow } from '@/session/data/tables/deepchatSessions' import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { + TapeAnchorReader, + TapeAnchorWriter, + TapeLifecycleAdmin +} from '@/tape/ports/capabilities' +import { SessionTape } from './tape' export type SessionSummaryState = { summaryText: string | null @@ -131,9 +137,14 @@ function summaryStatesEqual(left: SessionSummaryState, right: SessionSummaryStat export class SessionSettingsStore { private database: SessionDatabase + private readonly tape: TapeAnchorReader & TapeAnchorWriter & TapeLifecycleAdmin - constructor(database: SessionDatabase) { + constructor( + database: SessionDatabase, + tape: TapeAnchorReader & TapeAnchorWriter & TapeLifecycleAdmin = new SessionTape(database) + ) { this.database = database + this.tape = tape } create( @@ -150,7 +161,7 @@ export class SessionSettingsStore { permissionMode, generationSettings ) - this.database.deepchatTapeEntriesTable.ensureBootstrapAnchor(id) + this.tape.initializeSessionTape(id) } get(id: string) { @@ -158,8 +169,7 @@ export class SessionSettingsStore { } delete(id: string): void { - this.database.deepchatTapeEntriesTable.deleteBySession(id) - this.database.deepchatTapeSearchProjectionTable.deleteBySession(id) + this.tape.deleteSessionTape(id) this.database.deepchatSessionsTable.delete(id) } @@ -198,8 +208,7 @@ export class SessionSettingsStore { } getSummaryState(id: string): SessionSummaryState { - const tapeTable = this.database.deepchatTapeEntriesTable - const tapeState = summaryStateFromTapeAnchor(tapeTable.getLatestReconstructionAnchor(id)) + const tapeState = summaryStateFromTapeAnchor(this.tape.getLatestReconstructionAnchor(id)) if (tapeState) { return tapeState } @@ -208,9 +217,7 @@ export class SessionSettingsStore { } getReconstructionAnchorPromptState(id: string): ReconstructionAnchorPromptState | null { - return reconstructionAnchorPromptStateFromRow( - this.database.deepchatTapeEntriesTable.getLatestReconstructionAnchor(id) - ) + return reconstructionAnchorPromptStateFromRow(this.tape.getLatestReconstructionAnchor(id)) } updateSummaryState(id: string, state: SessionSummaryState): void { @@ -224,8 +231,7 @@ export class SessionSettingsStore { tapeAnchor?: SummaryTapeAnchorInput ): SummaryStateCompareAndSetResult { const applyUpdate = (): boolean => { - const tapeTable = this.database.deepchatTapeEntriesTable - const latestTapeAnchor = tapeTable.getLatestReconstructionAnchor(id) + const latestTapeAnchor = this.tape.getLatestReconstructionAnchor(id) const currentState = this.getSummaryState(id) if (!summaryStatesEqual(currentState, expectedState)) { return false @@ -236,7 +242,7 @@ export class SessionSettingsStore { this.database.deepchatSessionsTable.updateSummaryState(id, nextState) if (tapeAnchor) { - tapeTable.appendAnchor({ + this.tape.appendAnchor({ sessionId: id, name: tapeAnchor.name, state: tapeAnchor.state, @@ -265,7 +271,7 @@ export class SessionSettingsStore { resetSummaryState(id: string): void { const reset = (): void => { this.database.deepchatSessionsTable.resetSummaryState(id) - this.database.deepchatTapeEntriesTable.appendAnchor({ + this.tape.appendAnchor({ sessionId: id, name: 'summary/reset', state: { @@ -278,8 +284,6 @@ export class SessionSettingsStore { } resetTape(id: string): void { - this.database.deepchatTapeEntriesTable.deleteBySession(id) - this.database.deepchatTapeSearchProjectionTable.deleteBySession(id) - this.database.deepchatTapeEntriesTable.ensureBootstrapAnchor(id) + this.tape.resetSessionTape(id) } } diff --git a/src/main/session/data/tables/deepchatTapeEntries.ts b/src/main/session/data/tables/deepchatTapeEntries.ts index 018a5ce2ef..6d84885f1d 100644 --- a/src/main/session/data/tables/deepchatTapeEntries.ts +++ b/src/main/session/data/tables/deepchatTapeEntries.ts @@ -1,1079 +1 @@ -import Database from 'better-sqlite3-multiple-ciphers' -import logger from '@shared/logger' -import { BaseTable } from '@/data/baseTable' -import { randomUUID } from 'crypto' -import { - normalizeDeepChatTapeReadSources, - serializeDeepChatTapeReadSources, - SUMMARY_ANCHOR_NAMES, - TAPE_INCARNATION_META_KEY, - type DeepChatTapeAppendInput, - type DeepChatTapeEntryRow, - type DeepChatTapeReadSource, - type DeepChatTapeSearchInput, - type TapeAnchorAppendInput, - type TapeEventAppendInput -} from '@/tape/domain/entry' -import type { - TapeBootstrapStore, - TapeEntryLifecycleStore, - TapeEntryStore, - TapeMutationProjection, - TapeTransactionRunner -} from '@/tape/ports/storage' - -export { - normalizeDeepChatTapeReadSources, - serializeDeepChatTapeReadSources, - SUMMARY_ANCHOR_NAMES, - TAPE_INCARNATION_META_KEY -} from '@/tape/domain/entry' -export type { - DeepChatTapeAppendInput, - DeepChatTapeEntryKind, - DeepChatTapeEntryRow, - DeepChatTapeReadSource, - DeepChatTapeSearchInput, - DeepChatTapeSourceInput, - DeepChatTapeSourceType, - TapeAnchorAppendInput, - TapeEventAppendInput -} from '@/tape/domain/entry' - -export type DeepChatTapeMutationProjection = TapeMutationProjection - -const RECONSTRUCTION_ANCHOR_NAMES = SUMMARY_ANCHOR_NAMES - -const TAPE_ENTRY_INDEX_SQL = ` - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_kind - ON deepchat_tape_entries(session_id, kind, entry_id); - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_name - ON deepchat_tape_entries(session_id, name, entry_id); - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_source - ON deepchat_tape_entries(session_id, source_type, source_id, source_seq); - CREATE UNIQUE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_provenance - ON deepchat_tape_entries(session_id, provenance_key) - WHERE provenance_key IS NOT NULL; -` - -function safeJsonStringify(value: Record | undefined): string { - return JSON.stringify(value ?? {}) -} - -function buildProvenanceKey(input: DeepChatTapeAppendInput): string | null { - if (input.provenanceKey !== undefined) { - return input.provenanceKey - } - if (!input.source?.type || !input.source.id) { - return null - } - return [ - input.source.type, - input.source.id, - input.source.seq ?? 0, - input.kind, - input.name ?? '' - ].join(':') -} - -function escapeLikePattern(value: string): string { - return value.replace(/[\\%_]/g, (character) => `\\${character}`) -} - -// Three LIKE parameters per field group stay below SQLite's portable 999-variable floor after -// source, filter, and limit bindings. -const MAX_TAPE_SEARCH_TOKEN_CLAUSES = 256 - -function tokenizeDeepChatTapeSearchQuery(value: string): string[] { - return value - .split(/\s+/) - .map((token) => token.trim()) - .filter(Boolean) -} - -export function buildDeepChatTapeFtsMatch(value: string): string { - const tokens = tokenizeDeepChatTapeSearchQuery(value) - const values = - tokens.length > 1 && tokens.length <= MAX_TAPE_SEARCH_TOKEN_CLAUSES ? tokens : [value] - return values.map((token) => `"${token.replace(/"/g, '""')}"`).join(' AND ') -} - -export function buildDeepChatTapeLikeSearchPredicate( - fieldExpressions: readonly [string, ...string[]], - normalizedQuery: string -): { sql: string; params: string[] } { - const fieldClause = `(${fieldExpressions - .map((field) => `${field} LIKE ? ESCAPE '\\'`) - .join(' OR ')})` - const queryClauses = [fieldClause] - const queryPattern = `%${escapeLikePattern(normalizedQuery)}%` - const params = fieldExpressions.map(() => queryPattern) - const tokens = tokenizeDeepChatTapeSearchQuery(normalizedQuery) - - if (tokens.length > 1 && tokens.length <= MAX_TAPE_SEARCH_TOKEN_CLAUSES) { - queryClauses.push(`(${tokens.map(() => fieldClause).join(' AND ')})`) - for (const token of tokens) { - const tokenPattern = `%${escapeLikePattern(token)}%` - params.push(...fieldExpressions.map(() => tokenPattern)) - } - } - - return { - sql: `(${queryClauses.join(' OR ')})`, - params - } -} - -const AUTHORIZED_TAPE_SOURCES_CTE_SQL = ` - authorized_sources(session_id, max_entry_id) AS ( - SELECT - json_extract(value, '$.sessionId'), - CAST(json_extract(value, '$.maxEntryId') AS INTEGER) - FROM json_each(?) - ) -` - -function effectiveTapeMessagePredicateSql(alias: string): string { - return ` - json_type(${alias}.payload_json, '$.record') = 'object' - AND typeof(json_extract(${alias}.payload_json, '$.record.id')) = 'text' - AND typeof(json_extract(${alias}.payload_json, '$.record.sessionId')) = 'text' - AND typeof(json_extract(${alias}.payload_json, '$.record.orderSeq')) IN ('integer', 'real') - AND json_extract(${alias}.payload_json, '$.record.role') IN ('user', 'assistant') - AND typeof(json_extract(${alias}.payload_json, '$.record.content')) = 'text' - AND ( - json_extract(${alias}.payload_json, '$.record.status') IS NULL - OR json_extract(${alias}.payload_json, '$.record.status') != 'pending' - ) - ` -} - -function tapeRetractionMessageIdSql(alias: string): string { - return ` - CASE - WHEN json_type(${alias}.payload_json, '$.data') = 'object' - THEN json_extract(${alias}.payload_json, '$.data.messageId') - WHEN json_type(${alias}.payload_json, '$.data') = 'text' - AND json_valid(json_extract(${alias}.payload_json, '$.data')) - THEN json_extract(json_extract(${alias}.payload_json, '$.data'), '$.messageId') - ELSE NULL - END - ` -} - -function tapeToolCallIdSql(alias: string): string { - return ` - CASE - WHEN ${alias}.kind = 'tool_result' - THEN json_extract(${alias}.payload_json, '$.toolCallId') - WHEN json_type(${alias}.payload_json, '$.toolCall') = 'object' - THEN json_extract(${alias}.payload_json, '$.toolCall.id') - WHEN json_type(${alias}.payload_json, '$.toolCall') = 'text' - AND json_valid(json_extract(${alias}.payload_json, '$.toolCall')) - THEN json_extract(json_extract(${alias}.payload_json, '$.toolCall'), '$.id') - ELSE NULL - END - ` -} - -// These read-only SQL forms mirror deepchatTapeEffectiveSemantics. Search uses correlated -// candidate validation to avoid materializing a whole linked Tape; context uses the complete -// effective CTE because it needs stable neighboring-row positions. Native tests cover parity for -// replacement, retraction, head cutoff, and window ordering. -const EFFECTIVE_TAPE_ROWS_CTE_SQL = ` - bounded_rows AS ( - SELECT tape.* - FROM deepchat_tape_entries AS tape - INNER JOIN authorized_sources AS source - ON source.session_id = tape.session_id - AND tape.entry_id <= source.max_entry_id - ), - raw_message_candidates AS ( - SELECT - bounded_rows.*, - json_extract(payload_json, '$.record.id') AS message_id - FROM bounded_rows - WHERE kind = 'message' - ), - message_candidates AS ( - SELECT * - FROM raw_message_candidates - WHERE ${effectiveTapeMessagePredicateSql('raw_message_candidates')} - ), - ranked_messages AS ( - SELECT - message_candidates.*, - ROW_NUMBER() OVER ( - PARTITION BY session_id, message_id - ORDER BY entry_id DESC - ) AS candidate_rank - FROM message_candidates - ), - effective_message_rows AS ( - SELECT ranked_messages.* - FROM ranked_messages - WHERE candidate_rank = 1 - AND NOT EXISTS ( - SELECT 1 - FROM bounded_rows AS retraction - WHERE retraction.session_id = ranked_messages.session_id - AND retraction.kind = 'event' - AND retraction.name = 'message/retracted' - AND retraction.entry_id > ranked_messages.entry_id - AND (${tapeRetractionMessageIdSql('retraction')}) = ranked_messages.message_id - ) - ), - raw_tool_candidates AS ( - SELECT - bounded_rows.*, - json_extract(payload_json, '$.messageId') AS message_id, - (${tapeToolCallIdSql('bounded_rows')}) AS tool_call_id, - json_extract(meta_json, '$.status') AS tool_status - FROM bounded_rows - WHERE kind IN ('tool_call', 'tool_result') - ), - tool_candidates AS ( - SELECT * - FROM raw_tool_candidates - WHERE typeof(message_id) = 'text' - AND length(message_id) > 0 - AND typeof(tool_call_id) = 'text' - AND length(tool_call_id) > 0 - AND tool_status IN ('success', 'error') - ), - ranked_tools AS ( - SELECT - tool_candidates.*, - ROW_NUMBER() OVER ( - PARTITION BY session_id, kind, message_id, tool_call_id - ORDER BY entry_id DESC - ) AS candidate_rank - FROM tool_candidates - ), - effective_rows AS ( - SELECT - session_id, entry_id, kind, name, source_type, source_id, source_seq, - provenance_key, payload_json, meta_json, created_at - FROM bounded_rows - WHERE kind = 'anchor' - UNION ALL - SELECT - session_id, entry_id, kind, name, source_type, source_id, source_seq, - provenance_key, payload_json, meta_json, created_at - FROM bounded_rows - WHERE kind = 'event' - AND (name IS NULL OR name NOT IN ( - 'message/retracted', - 'message/compaction_indicator', - 'migration/backfill' - )) - UNION ALL - SELECT - session_id, entry_id, kind, name, source_type, source_id, source_seq, - provenance_key, payload_json, meta_json, created_at - FROM effective_message_rows - UNION ALL - SELECT - ranked_tools.session_id, - ranked_tools.entry_id, - ranked_tools.kind, - ranked_tools.name, - ranked_tools.source_type, - ranked_tools.source_id, - ranked_tools.source_seq, - ranked_tools.provenance_key, - ranked_tools.payload_json, - ranked_tools.meta_json, - ranked_tools.created_at - FROM ranked_tools - INNER JOIN effective_message_rows AS message - ON message.session_id = ranked_tools.session_id - AND message.message_id = ranked_tools.message_id - WHERE ranked_tools.candidate_rank = 1 - ) -` - -const EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL = ` - candidate.kind = 'anchor' - OR ( - candidate.kind = 'event' - AND (candidate.name IS NULL OR candidate.name NOT IN ( - 'message/retracted', - 'message/compaction_indicator', - 'migration/backfill' - )) - ) - OR ( - candidate.kind = 'message' - AND ${effectiveTapeMessagePredicateSql('candidate')} - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS later_message - WHERE later_message.session_id = candidate.session_id - AND later_message.entry_id > candidate.entry_id - AND later_message.entry_id <= source.max_entry_id - AND later_message.kind = 'message' - AND ${effectiveTapeMessagePredicateSql('later_message')} - AND json_extract(later_message.payload_json, '$.record.id') = - json_extract(candidate.payload_json, '$.record.id') - ) - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS retraction - WHERE retraction.session_id = candidate.session_id - AND retraction.entry_id > candidate.entry_id - AND retraction.entry_id <= source.max_entry_id - AND retraction.kind = 'event' - AND retraction.name = 'message/retracted' - AND (${tapeRetractionMessageIdSql('retraction')}) = - json_extract(candidate.payload_json, '$.record.id') - ) - ) - OR ( - candidate.kind IN ('tool_call', 'tool_result') - AND json_extract(candidate.meta_json, '$.status') IN ('success', 'error') - AND typeof(json_extract(candidate.payload_json, '$.messageId')) = 'text' - AND length(json_extract(candidate.payload_json, '$.messageId')) > 0 - AND typeof((${tapeToolCallIdSql('candidate')})) = 'text' - AND length((${tapeToolCallIdSql('candidate')})) > 0 - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS later_tool - WHERE later_tool.session_id = candidate.session_id - AND later_tool.entry_id > candidate.entry_id - AND later_tool.entry_id <= source.max_entry_id - AND later_tool.kind = candidate.kind - AND json_extract(later_tool.meta_json, '$.status') IN ('success', 'error') - AND json_extract(later_tool.payload_json, '$.messageId') = - json_extract(candidate.payload_json, '$.messageId') - AND (${tapeToolCallIdSql('later_tool')}) = (${tapeToolCallIdSql('candidate')}) - ) - AND EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS message - WHERE message.session_id = candidate.session_id - AND message.entry_id <= source.max_entry_id - AND message.kind = 'message' - AND ${effectiveTapeMessagePredicateSql('message')} - AND json_extract(message.payload_json, '$.record.id') = - json_extract(candidate.payload_json, '$.messageId') - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS later_message - WHERE later_message.session_id = message.session_id - AND later_message.entry_id > message.entry_id - AND later_message.entry_id <= source.max_entry_id - AND later_message.kind = 'message' - AND ${effectiveTapeMessagePredicateSql('later_message')} - AND json_extract(later_message.payload_json, '$.record.id') = - json_extract(message.payload_json, '$.record.id') - ) - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS retraction - WHERE retraction.session_id = message.session_id - AND retraction.entry_id > message.entry_id - AND retraction.entry_id <= source.max_entry_id - AND retraction.kind = 'event' - AND retraction.name = 'message/retracted' - AND (${tapeRetractionMessageIdSql('retraction')}) = - json_extract(message.payload_json, '$.record.id') - ) - ) - ) -` - -export class DeepChatTapeEntriesTable - extends BaseTable - implements TapeEntryStore, TapeTransactionRunner, TapeBootstrapStore, TapeEntryLifecycleStore -{ - constructor( - db: Database.Database, - private readonly mutationProjection?: DeepChatTapeMutationProjection - ) { - super(db, 'deepchat_tape_entries') - } - - getCreateTableSQL(): string { - return ` - CREATE TABLE IF NOT EXISTS deepchat_tape_entries ( - session_id TEXT NOT NULL, - entry_id INTEGER NOT NULL, - kind TEXT NOT NULL, - name TEXT, - source_type TEXT, - source_id TEXT, - source_seq INTEGER, - provenance_key TEXT, - payload_json TEXT NOT NULL DEFAULT '{}', - meta_json TEXT NOT NULL DEFAULT '{}', - created_at INTEGER NOT NULL, - PRIMARY KEY (session_id, entry_id) - ); - ${TAPE_ENTRY_INDEX_SQL} - ` - } - - public createTable(): void { - if (!this.tableExists()) { - this.db.exec(this.getCreateTableSQL()) - return - } - this.ensureProvenanceColumns() - this.db.exec(TAPE_ENTRY_INDEX_SQL) - } - - getMigrationSQL(_version: number): string | null { - return null - } - - getLatestVersion(): number { - return 0 - } - - runInTransaction(operation: () => T): T { - return this.db.transaction(operation)() - } - - append(input: DeepChatTapeAppendInput): DeepChatTapeEntryRow { - const append = this.db.transaction(() => { - const provenanceKey = buildProvenanceKey(input) - if (input.idempotent && provenanceKey) { - const existing = this.getByProvenanceKey(input.sessionId, provenanceKey) - if (existing) { - return existing - } - } - - const createdAt = input.createdAt ?? Date.now() - const previousSessionMaxEntryId = this.getMaxEntryId(input.sessionId) - const row = { - session_id: input.sessionId, - entry_id: previousSessionMaxEntryId + 1, - kind: input.kind, - name: input.name ?? null, - source_type: input.source?.type ?? null, - source_id: input.source?.id ?? null, - source_seq: input.source?.seq ?? null, - provenance_key: provenanceKey, - payload_json: safeJsonStringify(input.payload), - meta_json: safeJsonStringify(input.meta), - created_at: createdAt - } satisfies DeepChatTapeEntryRow - - try { - this.db - .prepare( - `INSERT INTO deepchat_tape_entries ( - session_id, - entry_id, - kind, - name, - source_type, - source_id, - source_seq, - provenance_key, - payload_json, - meta_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - .run( - row.session_id, - row.entry_id, - row.kind, - row.name, - row.source_type, - row.source_id, - row.source_seq, - row.provenance_key, - row.payload_json, - row.meta_json, - row.created_at - ) - } catch (error) { - if (input.idempotent && provenanceKey) { - const existing = this.getByProvenanceKey(input.sessionId, provenanceKey) - if (existing) { - return existing - } - } - throw error - } - - if (this.mutationProjection) { - try { - const applyProjection = this.db.transaction(() => - this.mutationProjection?.applyAppendedEntry(row, previousSessionMaxEntryId) - ) - applyProjection() - } catch (error) { - this.mutationProjection.invalidateSession(row.session_id) - logger.warn( - `[Tape] memory ingestion projection append failed; session marked stale: ${String(error)}` - ) - } - } - - return row - }) - - return append() - } - - appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow { - return this.append({ - sessionId: input.sessionId, - kind: 'anchor', - name: input.name, - source: input.source, - provenanceKey: input.provenanceKey, - payload: { - name: input.name, - state: input.state - }, - meta: input.meta, - createdAt: input.createdAt, - idempotent: input.idempotent - }) - } - - appendEvent(input: TapeEventAppendInput): DeepChatTapeEntryRow { - return this.append({ - sessionId: input.sessionId, - kind: 'event', - name: input.name, - source: input.source, - provenanceKey: input.provenanceKey, - payload: { - name: input.name, - data: input.data - }, - meta: input.meta, - createdAt: input.createdAt, - idempotent: input.idempotent - }) - } - - ensureBootstrapAnchor(sessionId: string): void { - const existing = this.db - .prepare( - `SELECT entry_id - FROM deepchat_tape_entries - WHERE session_id = ? AND kind = 'anchor' - ORDER BY entry_id ASC - LIMIT 1` - ) - .get(sessionId) as { entry_id: number } | undefined - - if (existing) { - return - } - - this.appendAnchor({ - sessionId, - name: 'session/start', - source: { - type: 'session', - id: sessionId, - seq: 0 - }, - state: { - owner: 'human' - }, - meta: { - [TAPE_INCARNATION_META_KEY]: randomUUID() - }, - idempotent: true - }) - } - - getBySession(sessionId: string): DeepChatTapeEntryRow[] { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - ORDER BY entry_id ASC` - ) - .all(sessionId) as DeepChatTapeEntryRow[] - } - - getSubagentLineageEvents(sessionId: string): DeepChatTapeEntryRow[] { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - AND kind = 'event' - AND name IN ('subagent/tape_linked', 'fork/merge') - ORDER BY entry_id ASC` - ) - .all(sessionId) as DeepChatTapeEntryRow[] - } - - getFirstEntriesBySessions(sessionIds: string[]): DeepChatTapeEntryRow[] { - const ids = [...new Set(sessionIds.map((id) => id.trim()).filter(Boolean))] - if (ids.length === 0) { - return [] - } - return this.db - .prepare( - `WITH requested_sessions(session_id) AS ( - SELECT value FROM json_each(?) - ), - first_entries(session_id, entry_id) AS ( - SELECT tape.session_id, MIN(tape.entry_id) - FROM deepchat_tape_entries AS tape - INNER JOIN requested_sessions AS requested - ON requested.session_id = tape.session_id - GROUP BY tape.session_id - ) - SELECT tape.* - FROM deepchat_tape_entries AS tape - INNER JOIN first_entries AS first - ON first.session_id = tape.session_id - AND first.entry_id = tape.entry_id - ORDER BY tape.session_id ASC` - ) - .all(JSON.stringify(ids)) as DeepChatTapeEntryRow[] - } - - getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND entry_id <= ? - ORDER BY entry_id ASC` - ) - .all(sessionId, maxEntryId) as DeepChatTapeEntryRow[] - } - - listMemoryViewManifestAnchorsBySessions( - sessionIds: string[], - optionsOrLimit: number | { limit?: number; messageId?: string } = 100 - ): DeepChatTapeEntryRow[] { - const uniqueSessionIds = [...new Set(sessionIds.filter((id) => id.trim().length > 0))] - if (uniqueSessionIds.length === 0) { - return [] - } - const options = typeof optionsOrLimit === 'number' ? { limit: optionsOrLimit } : optionsOrLimit - const cappedLimit = Math.min(Math.max(Math.floor(options.limit ?? 100), 1), 500) - const placeholders = uniqueSessionIds.map(() => '?').join(', ') - const whereClauses = [ - `session_id IN (${placeholders})`, - "kind = 'anchor'", - "name = 'memory/view_assembled'" - ] - const params: Array = [...uniqueSessionIds] - if (options.messageId) { - whereClauses.push("json_extract(meta_json, '$.messageId') = ?") - params.push(options.messageId) - } - params.push(cappedLimit) - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE ${whereClauses.join(' AND ')} - ORDER BY created_at DESC, entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeEntryRow[] - } - - listMemoryViewManifestAnchorsByAgent( - agentId: string, - options: { sessionId?: string; limit?: number; messageId?: string } = {} - ): DeepChatTapeEntryRow[] { - const cappedLimit = Math.min(Math.max(Math.floor(options.limit ?? 100), 1), 500) - const whereClauses = [ - 'sessions.agent_id = ?', - "tape.kind = 'anchor'", - "tape.name = 'memory/view_assembled'" - ] - const params: Array = [agentId] - if (options.sessionId) { - whereClauses.push('tape.session_id = ?') - params.push(options.sessionId) - } - if (options.messageId) { - whereClauses.push("json_extract(tape.meta_json, '$.messageId') = ?") - params.push(options.messageId) - } - params.push(cappedLimit) - return this.db - .prepare( - `SELECT tape.* - FROM deepchat_tape_entries AS tape - INNER JOIN new_sessions AS sessions - ON sessions.id = tape.session_id - WHERE ${whereClauses.join(' AND ')} - ORDER BY tape.created_at DESC, tape.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeEntryRow[] - } - - getEntriesAfter(sessionId: string, entryId: number): DeepChatTapeEntryRow[] { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND entry_id > ? - ORDER BY entry_id ASC` - ) - .all(sessionId, entryId) as DeepChatTapeEntryRow[] - } - - getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND kind = 'anchor' - ORDER BY entry_id DESC - LIMIT 1` - ) - .get(sessionId) as DeepChatTapeEntryRow | undefined - } - - getAnchors(sessionId: string, limit: number = 20): DeepChatTapeEntryRow[] { - const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) - const rows = this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND kind = 'anchor' - ORDER BY entry_id DESC - LIMIT ?` - ) - .all(sessionId, cappedLimit) as DeepChatTapeEntryRow[] - - return rows.reverse() - } - - getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { - const placeholders = SUMMARY_ANCHOR_NAMES.map(() => '?').join(', ') - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - AND kind = 'anchor' - AND name IN (${placeholders}) - ORDER BY entry_id DESC - LIMIT 1` - ) - .get(sessionId, ...SUMMARY_ANCHOR_NAMES) as DeepChatTapeEntryRow | undefined - } - - getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { - const placeholders = RECONSTRUCTION_ANCHOR_NAMES.map(() => '?').join(', ') - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - AND kind = 'anchor' - AND ( - name IN (${placeholders}) - OR name LIKE 'handoff/%' - OR name LIKE 'auto_handoff/%' - ) - ORDER BY entry_id DESC - LIMIT 1` - ) - .get(sessionId, ...RECONSTRUCTION_ANCHOR_NAMES) as DeepChatTapeEntryRow | undefined - } - - getByProvenanceKey(sessionId: string, provenanceKey: string): DeepChatTapeEntryRow | undefined { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND provenance_key = ? - LIMIT 1` - ) - .get(sessionId, provenanceKey) as DeepChatTapeEntryRow | undefined - } - - getMaxEntryId(sessionId: string): number { - const row = this.db - .prepare( - `SELECT MAX(entry_id) AS max_entry_id - FROM deepchat_tape_entries - WHERE session_id = ?` - ) - .get(sessionId) as { max_entry_id: number | null } | undefined - return row?.max_entry_id ?? 0 - } - - getMaxEntryIdsBySessions(sessionIds: string[]): Map { - const ids = [...new Set(sessionIds.map((id) => id.trim()).filter(Boolean))] - const maxEntryIdBySession = new Map(ids.map((id) => [id, 0])) - if (ids.length === 0) { - return maxEntryIdBySession - } - const rows = this.db - .prepare( - `WITH requested_sessions(session_id) AS ( - SELECT value FROM json_each(?) - ) - SELECT tape.session_id, MAX(tape.entry_id) AS max_entry_id - FROM deepchat_tape_entries AS tape - INNER JOIN requested_sessions AS requested - ON requested.session_id = tape.session_id - GROUP BY tape.session_id` - ) - .all(JSON.stringify(ids)) as Array<{ session_id: string; max_entry_id: number }> - for (const row of rows) { - maxEntryIdBySession.set(row.session_id, row.max_entry_id) - } - return maxEntryIdBySession - } - - countAnchorsBySession(sessionId: string): number { - const row = this.db - .prepare( - `SELECT COUNT(*) AS count - FROM deepchat_tape_entries - WHERE session_id = ? AND kind = 'anchor'` - ) - .get(sessionId) as { count: number } | undefined - return row?.count ?? 0 - } - - countEntriesAfter(sessionId: string, entryId: number): number { - const row = this.db - .prepare( - `SELECT COUNT(*) AS count - FROM deepchat_tape_entries - WHERE session_id = ? AND entry_id > ?` - ) - .get(sessionId, entryId) as { count: number } | undefined - return row?.count ?? 0 - } - - countBySession(sessionId: string): number { - const row = this.db - .prepare( - `SELECT COUNT(*) AS count - FROM deepchat_tape_entries - WHERE session_id = ?` - ) - .get(sessionId) as { count: number } | undefined - return row?.count ?? 0 - } - - search( - sessionId: string, - query: string, - options: DeepChatTapeSearchInput = {} - ): DeepChatTapeEntryRow[] { - const normalizedQuery = query.trim() - if (!normalizedQuery) { - return [] - } - const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 - const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) - const queryPredicate = buildDeepChatTapeLikeSearchPredicate( - ['payload_json', 'meta_json', 'name'], - normalizedQuery - ) - const whereClauses = ['session_id = ?', queryPredicate.sql] - const params: Array = [sessionId, ...queryPredicate.params] - - if (options.kinds?.length) { - whereClauses.push(`kind IN (${options.kinds.map(() => '?').join(', ')})`) - params.push(...options.kinds) - } - - if (Number.isFinite(options.startCreatedAt)) { - whereClauses.push('created_at >= ?') - params.push(options.startCreatedAt as number) - } - - if (Number.isFinite(options.endCreatedAt)) { - whereClauses.push('created_at <= ?') - params.push(options.endCreatedAt as number) - } - - params.push(cappedLimit) - - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE ${whereClauses.join(' AND ')} - ORDER BY entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeEntryRow[] - } - - searchEffectiveSourcesAtHeads( - sources: readonly DeepChatTapeReadSource[], - query: string, - options: DeepChatTapeSearchInput = {} - ): DeepChatTapeEntryRow[] { - const normalizedSources = normalizeDeepChatTapeReadSources(sources) - const normalizedQuery = query.trim() - if (normalizedSources.length === 0 || !normalizedQuery) { - return [] - } - - const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 - const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) - const queryPredicate = buildDeepChatTapeLikeSearchPredicate( - ['candidate.payload_json', 'candidate.meta_json', 'candidate.name'], - normalizedQuery - ) - const whereClauses = [queryPredicate.sql] - const params: Array = [ - serializeDeepChatTapeReadSources(normalizedSources), - ...queryPredicate.params - ] - - if (options.kinds?.length) { - whereClauses.push(`candidate.kind IN (${options.kinds.map(() => '?').join(', ')})`) - params.push(...options.kinds) - } - if (Number.isFinite(options.startCreatedAt)) { - whereClauses.push('candidate.created_at >= ?') - params.push(options.startCreatedAt as number) - } - if (Number.isFinite(options.endCreatedAt)) { - whereClauses.push('candidate.created_at <= ?') - params.push(options.endCreatedAt as number) - } - params.push(cappedLimit) - - return this.db - .prepare( - `WITH - ${AUTHORIZED_TAPE_SOURCES_CTE_SQL} - SELECT candidate.* - FROM deepchat_tape_entries AS candidate - INNER JOIN authorized_sources AS source - ON source.session_id = candidate.session_id - AND candidate.entry_id <= source.max_entry_id - WHERE ${whereClauses.join(' AND ')} - AND (${EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL}) - ORDER BY candidate.created_at DESC, candidate.session_id ASC, candidate.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeEntryRow[] - } - - getEffectiveContextRowsAtHead( - source: DeepChatTapeReadSource, - entryIds: number[], - options: { before: number; after: number; limit: number } - ): DeepChatTapeEntryRow[] { - const normalizedSource = normalizeDeepChatTapeReadSources([source])[0] - const requestedEntryIds = [ - ...new Set(entryIds.filter((entryId) => Number.isSafeInteger(entryId) && entryId > 0)) - ].sort((left, right) => left - right) - if (!normalizedSource || requestedEntryIds.length === 0) { - return [] - } - const before = Math.min(Math.max(Math.floor(options.before), 0), 20) - const after = Math.min(Math.max(Math.floor(options.after), 0), 20) - const limit = Math.min(Math.max(Math.floor(options.limit), 1), 100) - - return this.db - .prepare( - `WITH - ${AUTHORIZED_TAPE_SOURCES_CTE_SQL}, - ${EFFECTIVE_TAPE_ROWS_CTE_SQL}, - ordered_rows AS ( - SELECT - effective_rows.*, - ROW_NUMBER() OVER (ORDER BY entry_id ASC) AS row_position - FROM effective_rows - ), - requested_ids(entry_id, request_ordinal) AS ( - SELECT CAST(value AS INTEGER), CAST(key AS INTEGER) - FROM json_each(?) - ), - requested_positions AS ( - SELECT - requested_ids.request_ordinal, - ordered_rows.entry_id, - ordered_rows.row_position - FROM requested_ids - INNER JOIN ordered_rows - ON ordered_rows.entry_id = requested_ids.entry_id - ), - context_candidates AS ( - SELECT - ordered_rows.*, - 0 AS priority_group, - requested_positions.request_ordinal, - 0 AS neighbor_position - FROM requested_positions - INNER JOIN ordered_rows - ON ordered_rows.entry_id = requested_positions.entry_id - UNION ALL - SELECT - ordered_rows.*, - 1 AS priority_group, - requested_positions.request_ordinal, - ordered_rows.row_position AS neighbor_position - FROM requested_positions - INNER JOIN ordered_rows - ON ordered_rows.row_position BETWEEN requested_positions.row_position - ? - AND requested_positions.row_position + ? - AND ordered_rows.entry_id != requested_positions.entry_id - ), - ranked_context_candidates AS ( - SELECT - context_candidates.*, - ROW_NUMBER() OVER ( - PARTITION BY session_id, entry_id - ORDER BY priority_group, request_ordinal, neighbor_position - ) AS duplicate_rank - FROM context_candidates - ) - SELECT - session_id, entry_id, kind, name, source_type, source_id, source_seq, - provenance_key, payload_json, meta_json, created_at - FROM ranked_context_candidates - WHERE duplicate_rank = 1 - ORDER BY priority_group, request_ordinal, neighbor_position - LIMIT ?` - ) - .all( - serializeDeepChatTapeReadSources([normalizedSource]), - JSON.stringify(requestedEntryIds), - before, - after, - limit - ) as DeepChatTapeEntryRow[] - } - - deleteBySession(sessionId: string): void { - const remove = this.db.transaction(() => { - this.db.prepare('DELETE FROM deepchat_tape_entries WHERE session_id = ?').run(sessionId) - this.mutationProjection?.deleteBySession(sessionId) - }) - remove() - } - - private ensureProvenanceColumns(): void { - const columns: Array<[string, string]> = [ - ['source_type', 'TEXT'], - ['source_id', 'TEXT'], - ['source_seq', 'INTEGER'], - ['provenance_key', 'TEXT'] - ] - for (const [columnName, columnType] of columns) { - if (!this.hasColumn(columnName)) { - this.db.exec(`ALTER TABLE deepchat_tape_entries ADD COLUMN ${columnName} ${columnType}`) - } - } - } -} +export * from '@/tape/infrastructure/sqlite/tapeEntryStore' diff --git a/src/main/session/data/tables/deepchatTapeSearchProjection.ts b/src/main/session/data/tables/deepchatTapeSearchProjection.ts index 75ca28ce79..16f6784dff 100644 --- a/src/main/session/data/tables/deepchatTapeSearchProjection.ts +++ b/src/main/session/data/tables/deepchatTapeSearchProjection.ts @@ -1,898 +1 @@ -import Database from 'better-sqlite3-multiple-ciphers' -import { BaseTable } from '@/data/baseTable' -import { - buildDeepChatTapeFtsMatch, - buildDeepChatTapeLikeSearchPredicate -} from './deepchatTapeEntries' -import { - normalizeDeepChatTapeReadSources, - serializeDeepChatTapeReadSources -} from '@/tape/domain/entry' -import type { DeepChatTapeReadSource, DeepChatTapeSearchInput } from '@/tape/domain/entry' -import type { - TapeSearchProjectionInput, - TapeSearchProjectionMeta, - TapeSearchProjectionReadResult, - TapeSearchProjectionResultRow, - TapeSearchProjectionRow, - TapeSearchProjectionStore -} from '@/tape/ports/application' - -export const DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION = 2 - -export type DeepChatTapeSearchProjectionInput = TapeSearchProjectionInput -export type DeepChatTapeSearchProjectionRow = TapeSearchProjectionRow -export type DeepChatTapeSearchProjectionResultRow = TapeSearchProjectionResultRow -export type DeepChatTapeSearchProjectionMeta = TapeSearchProjectionMeta -export type DeepChatTapeSearchProjectionReadResult = TapeSearchProjectionReadResult - -type FtsCapability = { available: boolean; tokenizer: 'trigram' | 'unicode61' } - -const TAPE_SEARCH_PROJECTION_INDEX_SQL = ` - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_kind - ON deepchat_tape_search_projection(session_id, kind, entry_id); - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_created - ON deepchat_tape_search_projection(session_id, created_at, entry_id); -` - -function normalizeLimit(limit: number | undefined): number { - return Math.min(Math.max(Math.floor(Number.isFinite(limit) ? (limit as number) : 20), 1), 100) -} - -function safeJsonStringify(value: Record): string { - return JSON.stringify(value ?? {}) -} - -function parseRefs(value: string): Record { - try { - const parsed = JSON.parse(value) - return parsed && typeof parsed === 'object' && !Array.isArray(parsed) - ? (parsed as Record) - : {} - } catch { - return {} - } -} - -export class DeepChatTapeSearchProjectionTable - extends BaseTable - implements TapeSearchProjectionStore -{ - private ftsCapability: FtsCapability | undefined - private ftsReady = false - - constructor(db: Database.Database) { - super(db, 'deepchat_tape_search_projection') - } - - getCreateTableSQL(): string { - return ` - CREATE TABLE IF NOT EXISTS deepchat_tape_search_projection ( - session_id TEXT NOT NULL, - entry_id INTEGER NOT NULL, - kind TEXT NOT NULL, - name TEXT, - source_type TEXT, - source_id TEXT, - source_seq INTEGER, - search_text TEXT NOT NULL, - summary_text TEXT NOT NULL, - refs_json TEXT NOT NULL DEFAULT '{}', - created_at INTEGER NOT NULL, - PRIMARY KEY (session_id, entry_id) - ); - CREATE TABLE IF NOT EXISTS deepchat_tape_search_projection_meta ( - session_id TEXT PRIMARY KEY, - projection_version INTEGER NOT NULL, - max_entry_id INTEGER NOT NULL, - updated_at INTEGER NOT NULL - ); - CREATE TABLE IF NOT EXISTS deepchat_tape_search_fts_meta ( - session_id TEXT PRIMARY KEY, - projection_version INTEGER NOT NULL, - max_entry_id INTEGER NOT NULL, - updated_at INTEGER NOT NULL - ); - ${TAPE_SEARCH_PROJECTION_INDEX_SQL} - ` - } - - override createTable(): void { - this.db.exec(this.getCreateTableSQL()) - if (!this.ftsReady) { - this.ensureFtsIndex() - } - } - - getMigrationSQL(_version: number): string | null { - return null - } - - getLatestVersion(): number { - return 0 - } - - getSessionMeta(sessionId: string): DeepChatTapeSearchProjectionMeta | null { - const row = this.db - .prepare( - `SELECT projection_version, max_entry_id - FROM deepchat_tape_search_projection_meta - WHERE session_id = ?` - ) - .get(sessionId) as - | { - projection_version: number - max_entry_id: number - } - | undefined - if (!row) return null - return { - projectionVersion: row.projection_version, - maxEntryId: row.max_entry_id - } - } - - isCurrent( - sessionId: string, - maxEntryId: number, - projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - ): boolean { - const row = this.getSessionMeta(sessionId) - return row?.projectionVersion === projectionVersion && row.maxEntryId === maxEntryId - } - - getProjectedEntryIds(sessionId: string): number[] { - return ( - this.db - .prepare( - `SELECT entry_id - FROM deepchat_tape_search_projection - WHERE session_id = ? - ORDER BY entry_id ASC` - ) - .all(sessionId) as Array<{ entry_id: number }> - ).map((row) => row.entry_id) - } - - appendSession( - sessionId: string, - rows: DeepChatTapeSearchProjectionInput[], - maxEntryId: number, - projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - ): void { - const previousMeta = this.getSessionMeta(sessionId) - try { - this.db.transaction(() => { - if (!this.ftsReady) { - this.clearSessionFtsForBaseWrite(sessionId) - } - this.insertProjectionRows(rows) - if (this.ftsReady) { - if (previousMeta && this.isFtsCurrent(sessionId, previousMeta)) { - this.insertFtsRows(rows) - } else { - this.replaceSessionFtsRows(sessionId, this.getSessionProjectionInputs(sessionId)) - } - this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) - } - this.upsertMeta(sessionId, projectionVersion, maxEntryId) - })() - } catch (error) { - if (this.ftsReady) { - this.ftsReady = false - } - throw error - } - } - - replaceSession( - sessionId: string, - rows: DeepChatTapeSearchProjectionInput[], - maxEntryId: number, - projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - ): void { - try { - this.db.transaction(() => { - if (this.ftsReady) { - this.db - .prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?') - .run(sessionId) - this.db - .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') - .run(sessionId) - } else { - this.clearSessionFtsForBaseWrite(sessionId) - } - this.db - .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') - .run(sessionId) - this.db - .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') - .run(sessionId) - this.insertProjectionRows(rows) - if (this.ftsReady) { - this.insertFtsRows(rows) - this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) - } - this.upsertMeta(sessionId, projectionVersion, maxEntryId) - })() - } catch (error) { - if (this.ftsReady) { - this.ftsReady = false - } - throw error - } - } - - private insertProjectionRows(rows: DeepChatTapeSearchProjectionInput[]): void { - if (!rows.length) return - const insertProjection = this.db.prepare( - `INSERT OR REPLACE INTO deepchat_tape_search_projection ( - session_id, - entry_id, - kind, - name, - source_type, - source_id, - source_seq, - search_text, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - for (const row of rows) { - insertProjection.run( - row.sessionId, - row.entryId, - row.kind, - row.name, - row.sourceType, - row.sourceId, - row.sourceSeq, - row.searchText, - row.summaryText, - safeJsonStringify(row.refs), - row.createdAt - ) - } - } - - private upsertMeta(sessionId: string, projectionVersion: number, maxEntryId: number): void { - this.db - .prepare( - `INSERT INTO deepchat_tape_search_projection_meta ( - session_id, - projection_version, - max_entry_id, - updated_at - ) - VALUES (?, ?, ?, ?) - ON CONFLICT(session_id) DO UPDATE SET - projection_version = excluded.projection_version, - max_entry_id = excluded.max_entry_id, - updated_at = excluded.updated_at` - ) - .run(sessionId, projectionVersion, maxEntryId, Date.now()) - } - - private getFtsMeta(sessionId: string): DeepChatTapeSearchProjectionMeta | null { - const row = this.db - .prepare( - `SELECT projection_version, max_entry_id - FROM deepchat_tape_search_fts_meta - WHERE session_id = ?` - ) - .get(sessionId) as - | { - projection_version: number - max_entry_id: number - } - | undefined - if (!row) return null - return { - projectionVersion: row.projection_version, - maxEntryId: row.max_entry_id - } - } - - private isFtsCurrent(sessionId: string, meta: DeepChatTapeSearchProjectionMeta): boolean { - const row = this.getFtsMeta(sessionId) - return row?.projectionVersion === meta.projectionVersion && row.maxEntryId === meta.maxEntryId - } - - private upsertFtsMeta(sessionId: string, projectionVersion: number, maxEntryId: number): void { - this.db - .prepare( - `INSERT INTO deepchat_tape_search_fts_meta ( - session_id, - projection_version, - max_entry_id, - updated_at - ) - VALUES (?, ?, ?, ?) - ON CONFLICT(session_id) DO UPDATE SET - projection_version = excluded.projection_version, - max_entry_id = excluded.max_entry_id, - updated_at = excluded.updated_at` - ) - .run(sessionId, projectionVersion, maxEntryId, Date.now()) - } - - getByEntryIds(sessionId: string, entryIds: number[]): DeepChatTapeSearchProjectionRow[] { - const ids = [...new Set(entryIds.filter((id) => Number.isInteger(id) && id > 0))] - if (!ids.length) return [] - const placeholders = ids.map(() => '?').join(', ') - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_search_projection - WHERE session_id = ? AND entry_id IN (${placeholders}) - ORDER BY entry_id ASC` - ) - .all(sessionId, ...ids) as DeepChatTapeSearchProjectionRow[] - } - - searchSourcesReadOnly( - sources: readonly DeepChatTapeReadSource[], - query: string, - options: DeepChatTapeSearchInput = {} - ): DeepChatTapeSearchProjectionReadResult { - const normalizedSources = normalizeDeepChatTapeReadSources(sources) - const coveredSources = this.getCurrentReadSources( - normalizedSources, - 'deepchat_tape_search_projection_meta' - ) - const normalizedQuery = query.trim() - if (coveredSources.length === 0 || !normalizedQuery) { - return { rows: [], coveredSources } - } - if (coveredSources.length !== normalizedSources.length) { - return { rows: [], coveredSources } - } - - const limit = normalizeLimit(options.limit) - const ordered: DeepChatTapeSearchProjectionResultRow[] = [] - const seen = new Set() - const collect = (rows: DeepChatTapeSearchProjectionResultRow[]): void => { - for (const row of rows) { - const key = `${row.session_id}:${row.entry_id}` - if (seen.has(key)) continue - seen.add(key) - ordered.push(row) - } - } - - if (this.ftsReady) { - const ftsSources = this.getCurrentReadSources(coveredSources, 'deepchat_tape_search_fts_meta') - if (ftsSources.length === coveredSources.length) { - try { - collect(this.searchFtsSourcesReadOnly(ftsSources, normalizedQuery, options, limit)) - } catch { - // The base projection remains a complete read-only fallback. - } - } - } - if (ordered.length < limit) { - collect(this.searchLikeSourcesReadOnly(coveredSources, normalizedQuery, options, limit)) - } - - return { rows: ordered.slice(0, limit), coveredSources } - } - - search( - sessionId: string, - query: string, - options: DeepChatTapeSearchInput = {} - ): DeepChatTapeSearchProjectionResultRow[] { - const normalized = query.trim() - if (!normalized) return [] - const limit = normalizeLimit(options.limit) - const ordered: DeepChatTapeSearchProjectionResultRow[] = [] - const seen = new Set() - const collect = (rows: DeepChatTapeSearchProjectionResultRow[]): void => { - for (const row of rows) { - if (seen.has(row.entry_id)) continue - seen.add(row.entry_id) - ordered.push(row) - } - } - this.recoverSessionFts(sessionId) - if (this.ftsReady) { - collect(this.searchFts(sessionId, normalized, options, limit)) - } - if (!this.ftsReady || ordered.length < limit) { - collect(this.searchLike(sessionId, normalized, options, limit)) - } - return ordered.slice(0, limit) - } - - deleteBySession(sessionId: string): void { - this.db - .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') - .run(sessionId) - this.db - .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') - .run(sessionId) - this.deleteSessionFts(sessionId) - } - - clearAll(): void { - this.db.prepare('DELETE FROM deepchat_tape_search_projection').run() - this.db.prepare('DELETE FROM deepchat_tape_search_projection_meta').run() - if (this.ftsMetaTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() - } - this.clearFts() - } - - private detectFtsCapability(): FtsCapability { - if (this.ftsCapability) return this.ftsCapability - const probe = (tokenizer: string): boolean => { - const name = `temp.tape_search_fts_probe_${tokenizer}` - try { - this.db.exec( - `CREATE VIRTUAL TABLE IF NOT EXISTS ${name} USING fts5(c, tokenize='${tokenizer}');` - ) - this.db.exec(`DROP TABLE IF EXISTS ${name};`) - return true - } catch { - return false - } - } - if (probe('trigram')) this.ftsCapability = { available: true, tokenizer: 'trigram' } - else if (probe('unicode61')) this.ftsCapability = { available: true, tokenizer: 'unicode61' } - else this.ftsCapability = { available: false, tokenizer: 'unicode61' } - return this.ftsCapability - } - - private ensureFtsIndex(): void { - const capability = this.detectFtsCapability() - if (!capability.available) { - this.ftsReady = false - return - } - try { - this.db.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS deepchat_tape_search_fts USING fts5( - search_text, - name, - session_id UNINDEXED, - entry_id UNINDEXED, - kind UNINDEXED, - source_type UNINDEXED, - source_id UNINDEXED, - source_seq UNINDEXED, - summary_text UNINDEXED, - refs_json UNINDEXED, - created_at UNINDEXED, - tokenize='${capability.tokenizer}' - ); - `) - this.ftsReady = true - } catch { - this.ftsReady = false - } - } - - private toProjectionInput( - row: DeepChatTapeSearchProjectionRow - ): DeepChatTapeSearchProjectionInput { - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - sourceType: row.source_type, - sourceId: row.source_id, - sourceSeq: row.source_seq, - searchText: row.search_text, - summaryText: row.summary_text, - refs: parseRefs(row.refs_json), - createdAt: row.created_at - } - } - - private getSessionProjectionInputs(sessionId: string): DeepChatTapeSearchProjectionInput[] { - return ( - this.db - .prepare( - `SELECT * - FROM deepchat_tape_search_projection - WHERE session_id = ? - ORDER BY entry_id ASC` - ) - .all(sessionId) as DeepChatTapeSearchProjectionRow[] - ).map((row) => this.toProjectionInput(row)) - } - - private ftsTableExists(): boolean { - const row = this.db - .prepare( - `SELECT name - FROM sqlite_master - WHERE type = 'table' AND name = 'deepchat_tape_search_fts' - LIMIT 1` - ) - .get() as { name: string } | undefined - return Boolean(row) - } - - private ftsMetaTableExists(): boolean { - const row = this.db - .prepare( - `SELECT name - FROM sqlite_master - WHERE type = 'table' AND name = 'deepchat_tape_search_fts_meta' - LIMIT 1` - ) - .get() as { name: string } | undefined - return Boolean(row) - } - - private clearSessionFtsForBaseWrite(sessionId: string): void { - if (this.ftsMetaTableExists()) { - this.db - .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') - .run(sessionId) - } - if (this.ftsTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) - } - } - - private getProjectionRowId(sessionId: string, entryId: number): number { - const row = this.db - .prepare( - `SELECT rowid - FROM deepchat_tape_search_projection - WHERE session_id = ? AND entry_id = ?` - ) - .get(sessionId, entryId) as { rowid: number } | undefined - if (!row) { - throw new Error(`Missing tape search projection row for ${sessionId}:${entryId}`) - } - return row.rowid - } - - private insertFtsRows(rows: DeepChatTapeSearchProjectionInput[]): void { - if (!rows.length) return - const insertFts = this.db.prepare( - `INSERT INTO deepchat_tape_search_fts ( - rowid, - search_text, - name, - session_id, - entry_id, - kind, - source_type, - source_id, - source_seq, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - for (const row of rows) { - this.db - .prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ? AND entry_id = ?') - .run(row.sessionId, row.entryId) - insertFts.run( - this.getProjectionRowId(row.sessionId, row.entryId), - row.searchText, - row.name ?? '', - row.sessionId, - row.entryId, - row.kind, - row.sourceType, - row.sourceId, - row.sourceSeq, - row.summaryText, - safeJsonStringify(row.refs), - row.createdAt - ) - } - } - - private replaceSessionFtsRows( - sessionId: string, - rows: DeepChatTapeSearchProjectionInput[] - ): void { - this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) - this.insertFtsRows(rows) - } - - private replaceSessionFts( - sessionId: string, - rows: DeepChatTapeSearchProjectionInput[], - projectionVersion: number, - maxEntryId: number - ): boolean { - if (!this.ftsReady) return false - try { - this.db.transaction(() => { - this.replaceSessionFtsRows(sessionId, rows) - this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) - })() - return true - } catch { - this.ftsReady = false - return false - } - } - - private recoverSessionFts(sessionId: string): void { - const meta = this.getSessionMeta(sessionId) - if (!meta) { - this.deleteSessionFts(sessionId) - return - } - if (this.ftsReady && this.isFtsCurrent(sessionId, meta)) return - if (!this.ftsReady) { - this.ensureFtsIndex() - } - if (!this.ftsReady) return - if (this.isFtsCurrent(sessionId, meta)) return - this.replaceSessionFts( - sessionId, - this.getSessionProjectionInputs(sessionId), - meta.projectionVersion, - meta.maxEntryId - ) - } - - hasFtsReadyForTesting(): boolean { - return this.ftsReady - } - - disableFtsForTesting(): void { - this.ftsReady = false - } - - dropFtsForTesting(): void { - this.db.exec('DROP TABLE IF EXISTS deepchat_tape_search_fts') - if (this.ftsMetaTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() - } - this.ftsReady = false - } - - private deleteSessionFts(sessionId: string): void { - if (this.ftsMetaTableExists()) { - this.db - .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') - .run(sessionId) - } - if (!this.ftsTableExists()) return - try { - this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) - } catch { - this.ftsReady = false - } - } - - private clearFts(): void { - if (this.ftsMetaTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() - } - if (!this.ftsTableExists()) return - try { - this.db.prepare('DELETE FROM deepchat_tape_search_fts').run() - } catch { - this.ftsReady = false - } - } - - private searchFts( - sessionId: string, - normalized: string, - options: DeepChatTapeSearchInput, - limit: number - ): DeepChatTapeSearchProjectionResultRow[] { - const match = buildDeepChatTapeFtsMatch(normalized) - const whereClauses = [ - 'deepchat_tape_search_fts MATCH ?', - 'deepchat_tape_search_fts.session_id = ?', - 'projection.session_id = ?' - ] - const params: Array = [match, sessionId, sessionId] - this.addFilters(whereClauses, params, options, true, 'projection') - params.push(limit) - try { - return this.db - .prepare( - `SELECT - projection.session_id, - projection.entry_id, - projection.kind, - projection.name, - projection.source_type, - projection.source_id, - projection.source_seq, - projection.search_text, - projection.summary_text, - projection.refs_json, - projection.created_at, - bm25(deepchat_tape_search_fts) AS score - FROM deepchat_tape_search_fts - INNER JOIN deepchat_tape_search_projection AS projection - ON projection.session_id = deepchat_tape_search_fts.session_id - AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) - AND projection.search_text = deepchat_tape_search_fts.search_text - WHERE ${whereClauses.join(' AND ')} - ORDER BY score ASC, projection.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeSearchProjectionResultRow[] - } catch { - return [] - } - } - - private getCurrentReadSources( - sources: readonly DeepChatTapeReadSource[], - metaTable: 'deepchat_tape_search_projection_meta' | 'deepchat_tape_search_fts_meta' - ): DeepChatTapeReadSource[] { - const normalizedSources = normalizeDeepChatTapeReadSources(sources) - if (normalizedSources.length === 0) { - return [] - } - const rows = this.db - .prepare( - `WITH requested_sources(session_id, max_entry_id) AS ( - SELECT - json_extract(value, '$.sessionId'), - CAST(json_extract(value, '$.maxEntryId') AS INTEGER) - FROM json_each(?) - ) - SELECT requested.session_id, requested.max_entry_id - FROM requested_sources AS requested - INNER JOIN ${metaTable} AS meta - ON meta.session_id = requested.session_id - AND meta.max_entry_id = requested.max_entry_id - AND meta.projection_version = ? - ORDER BY requested.session_id ASC` - ) - .all( - serializeDeepChatTapeReadSources(normalizedSources), - DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - ) as Array<{ session_id: string; max_entry_id: number }> - return rows.map((row) => ({ sessionId: row.session_id, maxEntryId: row.max_entry_id })) - } - - private searchFtsSourcesReadOnly( - sources: readonly DeepChatTapeReadSource[], - normalized: string, - options: DeepChatTapeSearchInput, - limit: number - ): DeepChatTapeSearchProjectionResultRow[] { - const match = buildDeepChatTapeFtsMatch(normalized) - const whereClauses = ['deepchat_tape_search_fts MATCH ?'] - const params: Array = [serializeDeepChatTapeReadSources(sources), match] - this.addFilters(whereClauses, params, options, true, 'projection') - params.push(limit) - return this.db - .prepare( - `WITH authorized_sources(session_id, max_entry_id) AS ( - SELECT - json_extract(value, '$.sessionId'), - CAST(json_extract(value, '$.maxEntryId') AS INTEGER) - FROM json_each(?) - ) - SELECT - projection.session_id, - projection.entry_id, - projection.kind, - projection.name, - projection.source_type, - projection.source_id, - projection.source_seq, - projection.search_text, - projection.summary_text, - projection.refs_json, - projection.created_at, - bm25(deepchat_tape_search_fts) AS score - FROM deepchat_tape_search_fts - INNER JOIN deepchat_tape_search_projection AS projection - ON projection.session_id = deepchat_tape_search_fts.session_id - AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) - AND projection.search_text = deepchat_tape_search_fts.search_text - INNER JOIN authorized_sources AS source - ON source.session_id = projection.session_id - AND projection.entry_id <= source.max_entry_id - WHERE ${whereClauses.join(' AND ')} - ORDER BY score ASC, projection.created_at DESC, projection.session_id ASC, - projection.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeSearchProjectionResultRow[] - } - - private searchLikeSourcesReadOnly( - sources: readonly DeepChatTapeReadSource[], - normalized: string, - options: DeepChatTapeSearchInput, - limit: number - ): DeepChatTapeSearchProjectionResultRow[] { - const queryPredicate = buildDeepChatTapeLikeSearchPredicate( - ['projection.search_text', 'projection.summary_text', 'projection.name'], - normalized - ) - const whereClauses = [queryPredicate.sql] - const params: Array = [serializeDeepChatTapeReadSources(sources)] - params.push(...queryPredicate.params) - this.addFilters(whereClauses, params, options, false, 'projection') - params.push(limit) - return this.db - .prepare( - `WITH authorized_sources(session_id, max_entry_id) AS ( - SELECT - json_extract(value, '$.sessionId'), - CAST(json_extract(value, '$.maxEntryId') AS INTEGER) - FROM json_each(?) - ) - SELECT projection.*, NULL AS score - FROM deepchat_tape_search_projection AS projection - INNER JOIN authorized_sources AS source - ON source.session_id = projection.session_id - AND projection.entry_id <= source.max_entry_id - WHERE ${whereClauses.join(' AND ')} - ORDER BY projection.created_at DESC, projection.session_id ASC, projection.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeSearchProjectionResultRow[] - } - - private searchLike( - sessionId: string, - normalized: string, - options: DeepChatTapeSearchInput, - limit: number - ): DeepChatTapeSearchProjectionResultRow[] { - const queryPredicate = buildDeepChatTapeLikeSearchPredicate( - ['search_text', 'summary_text', 'name'], - normalized - ) - const whereClauses = ['session_id = ?', queryPredicate.sql] - const params: Array = [sessionId] - params.push(...queryPredicate.params) - this.addFilters(whereClauses, params, options) - params.push(limit) - return this.db - .prepare( - `SELECT *, NULL AS score - FROM deepchat_tape_search_projection - WHERE ${whereClauses.join(' AND ')} - ORDER BY entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeSearchProjectionResultRow[] - } - - private addFilters( - whereClauses: string[], - params: Array, - options: DeepChatTapeSearchInput, - castCreatedAt = false, - tableAlias?: string - ): void { - const column = (name: string) => (tableAlias ? `${tableAlias}.${name}` : name) - if (options.kinds?.length) { - whereClauses.push(`${column('kind')} IN (${options.kinds.map(() => '?').join(', ')})`) - params.push(...options.kinds) - } - const createdAtColumn = castCreatedAt - ? `CAST(${column('created_at')} AS INTEGER)` - : column('created_at') - if (Number.isFinite(options.startCreatedAt)) { - whereClauses.push(`${createdAtColumn} >= ?`) - params.push(options.startCreatedAt as number) - } - if (Number.isFinite(options.endCreatedAt)) { - whereClauses.push(`${createdAtColumn} <= ?`) - params.push(options.endCreatedAt as number) - } - } -} +export * from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' diff --git a/src/main/session/data/transcript.ts b/src/main/session/data/transcript.ts index 4e986d9e3d..6d0f05b46d 100644 --- a/src/main/session/data/transcript.ts +++ b/src/main/session/data/transcript.ts @@ -23,11 +23,8 @@ import { resolveUsageModelId, resolveUsageProviderId } from '@/session/usageStats' -import { - appendMessageRecordToTape, - appendMessageReplacementToTape, - appendMessageRetractionToTape -} from '@/session/data/tapeFacts' +import type { TapeMessageFactWriter } from '@/tape/ports/capabilities' +import { SessionTape } from './tape' function shouldConvertPendingBlockToError( status: AssistantMessageBlock['status'] @@ -128,9 +125,14 @@ function extractSearchableMessageContent(rawContent: string): string { export class SessionTranscript { private database: SessionDatabase + private readonly tapeFacts: TapeMessageFactWriter - constructor(database: SessionDatabase) { + constructor( + database: SessionDatabase, + tapeFacts: TapeMessageFactWriter = new SessionTape(database) + ) { this.database = database + this.tapeFacts = tapeFacts } private runInDatabaseTransaction(operation: () => T): T { @@ -372,11 +374,7 @@ export class SessionTranscript { } const updated = this.getMessage(messageId) if (updated) { - appendMessageReplacementToTape( - this.database.deepchatTapeEntriesTable, - updated, - 'message_content_updated' - ) + this.tapeFacts.appendMessageReplacement(updated, 'message_content_updated') } return } @@ -394,11 +392,7 @@ export class SessionTranscript { } const updated = this.getMessage(messageId) if (updated) { - appendMessageReplacementToTape( - this.database.deepchatTapeEntriesTable, - updated, - 'message_content_updated' - ) + this.tapeFacts.appendMessageReplacement(updated, 'message_content_updated') } } @@ -421,11 +415,7 @@ export class SessionTranscript { this.runInDatabaseTransaction(() => { const record = this.getMessage(messageId) if (record) { - appendMessageRetractionToTape( - this.database.deepchatTapeEntriesTable, - record, - 'message_deleted' - ) + this.tapeFacts.appendMessageRetraction(record, 'message_deleted') } this.database.deepchatSearchDocumentsTable.delete(`message:${messageId}`) this.database.deepchatAssistantBlocksTable.delete(messageId) @@ -444,11 +434,7 @@ export class SessionTranscript { (record) => record.orderSeq >= fromOrderSeq ) for (const record of records) { - appendMessageRetractionToTape( - this.database.deepchatTapeEntriesTable, - record, - 'messages_deleted_from_order_seq' - ) + this.tapeFacts.appendMessageRetraction(record, 'messages_deleted_from_order_seq') } const messageIds = records.map((record) => record.id) if (messageIds.length > 0) { @@ -685,7 +671,7 @@ export class SessionTranscript { if (!record) { return } - appendMessageRecordToTape(this.database.deepchatTapeEntriesTable, record, 'live') + this.tapeFacts.appendMessageRecord(record) } private toRecord(row: DeepChatMessageRow): ChatMessageRecord { diff --git a/src/main/tape/application/forkService.ts b/src/main/tape/application/forkService.ts index 1b71f29f2e..6d9e4f81d8 100644 --- a/src/main/tape/application/forkService.ts +++ b/src/main/tape/application/forkService.ts @@ -8,7 +8,10 @@ import { parseJsonObject } from './common' import type { TapeAnchorResult, TapeForkHandle } from './contracts' import type { TapeFactService } from './factService' -type TapeForkProviders = Pick +type TapeForkProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getEntryLifecycleStore' | 'getSearchProjectionStore' +> function readForkMergeReceiptCount( row: DeepChatTapeEntryRow, @@ -172,7 +175,11 @@ export class TapeForkService { ) {} private get table() { - return this.providers.getForkStore() + return this.providers.getEntryStore() + } + + private get lifecycle() { + return this.providers.getEntryLifecycleStore() } private get searchProjectionTable() { @@ -367,7 +374,7 @@ export class TapeForkService { discardFork(parentSessionId: string, forkId: string): void { const table = this.table const forkSessionIdValue = forkSessionId(parentSessionId, forkId) - table.deleteBySession(forkSessionIdValue) + this.lifecycle.deleteBySession(forkSessionIdValue) try { this.searchProjectionTable.deleteBySession(forkSessionIdValue) } catch (error) { diff --git a/src/main/tape/application/sessionTape.ts b/src/main/tape/application/sessionTape.ts index 0688801430..4ed5c669b2 100644 --- a/src/main/tape/application/sessionTape.ts +++ b/src/main/tape/application/sessionTape.ts @@ -18,10 +18,22 @@ import type { DeepChatTapeReplayExportOptions, DeepChatTapeReplaySlice } from '@shared/types/tape-replay' -import type { DeepChatTapeEntryRow } from '../domain/entry' +import type { DeepChatTapeEntryRow, TapeAnchorAppendInput } from '../domain/entry' import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' -import type { TapeToolFactWriter } from '../ports/capabilities' -import { createTapeApplicationProviders, type TapeApplicationDatabase } from '../ports/application' +import type { + TapeAnchorReader, + TapeAnchorWriter, + TapeInspectionReader, + TapeLifecycleAdmin, + TapeMessageFactWriter, + TapeRawEntryReader, + TapeToolFactWriter +} from '../ports/capabilities' +import { + createTapeApplicationProviders, + type TapeApplicationDatabase, + type TapeApplicationProviders +} from '../ports/application' import type { TapeAnchorResult, TapeBackfillResult, @@ -55,7 +67,17 @@ export type { } export { AgentTapeViewError, normalizeSubagentTapeLinkInput, normalizeTapeHandoffState } -export class SessionTape implements TapeToolFactWriter { +export class SessionTape + implements + TapeToolFactWriter, + TapeMessageFactWriter, + TapeRawEntryReader, + TapeAnchorReader, + TapeAnchorWriter, + TapeInspectionReader, + TapeLifecycleAdmin +{ + private readonly providers: TapeApplicationProviders private readonly facts: TapeFactService private readonly reconciler: TapeReconcilerService private readonly recall: TapeRecallService @@ -64,13 +86,13 @@ export class SessionTape implements TapeToolFactWriter { private readonly forks: TapeForkService constructor(database: TapeApplicationDatabase) { - const providers = createTapeApplicationProviders(database) - this.facts = new TapeFactService(providers) - this.lineage = new TapeLineageService(providers) - this.reconciler = new TapeReconcilerService(providers, this.facts) - this.recall = new TapeRecallService(providers, this.lineage) - this.viewReplay = new TapeViewReplayService(providers) - this.forks = new TapeForkService(providers, this.facts) + this.providers = createTapeApplicationProviders(database) + this.facts = new TapeFactService(this.providers) + this.lineage = new TapeLineageService(this.providers) + this.reconciler = new TapeReconcilerService(this.providers, this.facts) + this.recall = new TapeRecallService(this.providers, this.lineage) + this.viewReplay = new TapeViewReplayService(this.providers) + this.forks = new TapeForkService(this.providers, this.facts) } ensureSessionTapeReady( @@ -84,6 +106,14 @@ export class SessionTape implements TapeToolFactWriter { return this.facts.appendMessageRecord(record) } + appendMessageReplacement(record: ChatMessageRecord, reason: string): number { + return this.facts.appendMessageReplacement(record, reason) + } + + appendMessageRetraction(record: ChatMessageRecord, reason: string): number { + return this.facts.appendMessageRetraction(record, reason) + } + appendToolFact(input: TapeToolFactInput): Promise { return this.facts.appendToolFact(input) } @@ -198,4 +228,57 @@ export class SessionTape implements TapeToolFactWriter { linkSubagentTape(input: SubagentTapeLinkInput): SubagentTapeLinkReceipt { return this.lineage.linkSubagentTape(input) } + + getBySession(sessionId: string): DeepChatTapeEntryRow[] { + return this.providers.getEntryStore().getBySession(sessionId) + } + + getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] { + return this.providers.getEntryStore().getBySessionUpToEntryId(sessionId, maxEntryId) + } + + getMaxEntryId(sessionId: string): number { + return this.providers.getEntryStore().getMaxEntryId(sessionId) + } + + getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + return this.providers.getEntryStore().getLatestAnchor(sessionId) + } + + getAnchors(sessionId: string, limit?: number): DeepChatTapeEntryRow[] { + return this.providers.getEntryStore().getAnchors(sessionId, limit) + } + + getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + return this.providers.getEntryStore().getLatestSummaryAnchor(sessionId) + } + + getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + return this.providers.getEntryStore().getLatestReconstructionAnchor(sessionId) + } + + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow { + return this.providers.getEntryStore().appendAnchor(input) + } + + listMemoryViewManifestAnchorsByAgent( + agentId: string, + options?: { sessionId?: string; limit?: number; messageId?: string } + ): DeepChatTapeEntryRow[] { + return this.providers.getEntryStore().listMemoryViewManifestAnchorsByAgent(agentId, options) + } + + initializeSessionTape(sessionId: string): void { + this.providers.getEntryStore().ensureBootstrapAnchor(sessionId) + } + + deleteSessionTape(sessionId: string): void { + this.providers.getEntryLifecycleStore().deleteBySession(sessionId) + this.providers.getSearchProjectionStore().deleteBySession(sessionId) + } + + resetSessionTape(sessionId: string): void { + this.deleteSessionTape(sessionId) + this.initializeSessionTape(sessionId) + } } diff --git a/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts b/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts new file mode 100644 index 0000000000..1965e58aaf --- /dev/null +++ b/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts @@ -0,0 +1,1070 @@ +import Database from 'better-sqlite3-multiple-ciphers' +import logger from '@shared/logger' +import { BaseTable } from '@/data/baseTable' +import { randomUUID } from 'crypto' +import { + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources, + SUMMARY_ANCHOR_NAMES, + TAPE_INCARNATION_META_KEY, + type DeepChatTapeAppendInput, + type DeepChatTapeEntryRow, + type DeepChatTapeReadSource, + type DeepChatTapeSearchInput, + type TapeAnchorAppendInput, + type TapeEventAppendInput +} from '@/tape/domain/entry' +import type { + TapeBootstrapStore, + TapeEntryStore, + TapeMutationProjection, + TapeTransactionRunner +} from '@/tape/ports/storage' + +export { + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources, + SUMMARY_ANCHOR_NAMES, + TAPE_INCARNATION_META_KEY +} from '@/tape/domain/entry' +export type { + DeepChatTapeAppendInput, + DeepChatTapeEntryKind, + DeepChatTapeEntryRow, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + DeepChatTapeSourceInput, + DeepChatTapeSourceType, + TapeAnchorAppendInput, + TapeEventAppendInput +} from '@/tape/domain/entry' + +export type DeepChatTapeMutationProjection = TapeMutationProjection + +const RECONSTRUCTION_ANCHOR_NAMES = SUMMARY_ANCHOR_NAMES + +const TAPE_ENTRY_INDEX_SQL = ` + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_kind + ON deepchat_tape_entries(session_id, kind, entry_id); + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_name + ON deepchat_tape_entries(session_id, name, entry_id); + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_source + ON deepchat_tape_entries(session_id, source_type, source_id, source_seq); + CREATE UNIQUE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_provenance + ON deepchat_tape_entries(session_id, provenance_key) + WHERE provenance_key IS NOT NULL; +` + +function safeJsonStringify(value: Record | undefined): string { + return JSON.stringify(value ?? {}) +} + +function buildProvenanceKey(input: DeepChatTapeAppendInput): string | null { + if (input.provenanceKey !== undefined) { + return input.provenanceKey + } + if (!input.source?.type || !input.source.id) { + return null + } + return [ + input.source.type, + input.source.id, + input.source.seq ?? 0, + input.kind, + input.name ?? '' + ].join(':') +} + +function escapeLikePattern(value: string): string { + return value.replace(/[\\%_]/g, (character) => `\\${character}`) +} + +// Three LIKE parameters per field group stay below SQLite's portable 999-variable floor after +// source, filter, and limit bindings. +const MAX_TAPE_SEARCH_TOKEN_CLAUSES = 256 + +function tokenizeDeepChatTapeSearchQuery(value: string): string[] { + return value + .split(/\s+/) + .map((token) => token.trim()) + .filter(Boolean) +} + +export function buildDeepChatTapeFtsMatch(value: string): string { + const tokens = tokenizeDeepChatTapeSearchQuery(value) + const values = + tokens.length > 1 && tokens.length <= MAX_TAPE_SEARCH_TOKEN_CLAUSES ? tokens : [value] + return values.map((token) => `"${token.replace(/"/g, '""')}"`).join(' AND ') +} + +export function buildDeepChatTapeLikeSearchPredicate( + fieldExpressions: readonly [string, ...string[]], + normalizedQuery: string +): { sql: string; params: string[] } { + const fieldClause = `(${fieldExpressions + .map((field) => `${field} LIKE ? ESCAPE '\\'`) + .join(' OR ')})` + const queryClauses = [fieldClause] + const queryPattern = `%${escapeLikePattern(normalizedQuery)}%` + const params = fieldExpressions.map(() => queryPattern) + const tokens = tokenizeDeepChatTapeSearchQuery(normalizedQuery) + + if (tokens.length > 1 && tokens.length <= MAX_TAPE_SEARCH_TOKEN_CLAUSES) { + queryClauses.push(`(${tokens.map(() => fieldClause).join(' AND ')})`) + for (const token of tokens) { + const tokenPattern = `%${escapeLikePattern(token)}%` + params.push(...fieldExpressions.map(() => tokenPattern)) + } + } + + return { + sql: `(${queryClauses.join(' OR ')})`, + params + } +} + +const AUTHORIZED_TAPE_SOURCES_CTE_SQL = ` + authorized_sources(session_id, max_entry_id) AS ( + SELECT + json_extract(value, '$.sessionId'), + CAST(json_extract(value, '$.maxEntryId') AS INTEGER) + FROM json_each(?) + ) +` + +function effectiveTapeMessagePredicateSql(alias: string): string { + return ` + json_type(${alias}.payload_json, '$.record') = 'object' + AND typeof(json_extract(${alias}.payload_json, '$.record.id')) = 'text' + AND typeof(json_extract(${alias}.payload_json, '$.record.sessionId')) = 'text' + AND typeof(json_extract(${alias}.payload_json, '$.record.orderSeq')) IN ('integer', 'real') + AND json_extract(${alias}.payload_json, '$.record.role') IN ('user', 'assistant') + AND typeof(json_extract(${alias}.payload_json, '$.record.content')) = 'text' + AND ( + json_extract(${alias}.payload_json, '$.record.status') IS NULL + OR json_extract(${alias}.payload_json, '$.record.status') != 'pending' + ) + ` +} + +function tapeRetractionMessageIdSql(alias: string): string { + return ` + CASE + WHEN json_type(${alias}.payload_json, '$.data') = 'object' + THEN json_extract(${alias}.payload_json, '$.data.messageId') + WHEN json_type(${alias}.payload_json, '$.data') = 'text' + AND json_valid(json_extract(${alias}.payload_json, '$.data')) + THEN json_extract(json_extract(${alias}.payload_json, '$.data'), '$.messageId') + ELSE NULL + END + ` +} + +function tapeToolCallIdSql(alias: string): string { + return ` + CASE + WHEN ${alias}.kind = 'tool_result' + THEN json_extract(${alias}.payload_json, '$.toolCallId') + WHEN json_type(${alias}.payload_json, '$.toolCall') = 'object' + THEN json_extract(${alias}.payload_json, '$.toolCall.id') + WHEN json_type(${alias}.payload_json, '$.toolCall') = 'text' + AND json_valid(json_extract(${alias}.payload_json, '$.toolCall')) + THEN json_extract(json_extract(${alias}.payload_json, '$.toolCall'), '$.id') + ELSE NULL + END + ` +} + +// These read-only SQL forms mirror deepchatTapeEffectiveSemantics. Search uses correlated +// candidate validation to avoid materializing a whole linked Tape; context uses the complete +// effective CTE because it needs stable neighboring-row positions. Native tests cover parity for +// replacement, retraction, head cutoff, and window ordering. +const EFFECTIVE_TAPE_ROWS_CTE_SQL = ` + bounded_rows AS ( + SELECT tape.* + FROM deepchat_tape_entries AS tape + INNER JOIN authorized_sources AS source + ON source.session_id = tape.session_id + AND tape.entry_id <= source.max_entry_id + ), + raw_message_candidates AS ( + SELECT + bounded_rows.*, + json_extract(payload_json, '$.record.id') AS message_id + FROM bounded_rows + WHERE kind = 'message' + ), + message_candidates AS ( + SELECT * + FROM raw_message_candidates + WHERE ${effectiveTapeMessagePredicateSql('raw_message_candidates')} + ), + ranked_messages AS ( + SELECT + message_candidates.*, + ROW_NUMBER() OVER ( + PARTITION BY session_id, message_id + ORDER BY entry_id DESC + ) AS candidate_rank + FROM message_candidates + ), + effective_message_rows AS ( + SELECT ranked_messages.* + FROM ranked_messages + WHERE candidate_rank = 1 + AND NOT EXISTS ( + SELECT 1 + FROM bounded_rows AS retraction + WHERE retraction.session_id = ranked_messages.session_id + AND retraction.kind = 'event' + AND retraction.name = 'message/retracted' + AND retraction.entry_id > ranked_messages.entry_id + AND (${tapeRetractionMessageIdSql('retraction')}) = ranked_messages.message_id + ) + ), + raw_tool_candidates AS ( + SELECT + bounded_rows.*, + json_extract(payload_json, '$.messageId') AS message_id, + (${tapeToolCallIdSql('bounded_rows')}) AS tool_call_id, + json_extract(meta_json, '$.status') AS tool_status + FROM bounded_rows + WHERE kind IN ('tool_call', 'tool_result') + ), + tool_candidates AS ( + SELECT * + FROM raw_tool_candidates + WHERE typeof(message_id) = 'text' + AND length(message_id) > 0 + AND typeof(tool_call_id) = 'text' + AND length(tool_call_id) > 0 + AND tool_status IN ('success', 'error') + ), + ranked_tools AS ( + SELECT + tool_candidates.*, + ROW_NUMBER() OVER ( + PARTITION BY session_id, kind, message_id, tool_call_id + ORDER BY entry_id DESC + ) AS candidate_rank + FROM tool_candidates + ), + effective_rows AS ( + SELECT + session_id, entry_id, kind, name, source_type, source_id, source_seq, + provenance_key, payload_json, meta_json, created_at + FROM bounded_rows + WHERE kind = 'anchor' + UNION ALL + SELECT + session_id, entry_id, kind, name, source_type, source_id, source_seq, + provenance_key, payload_json, meta_json, created_at + FROM bounded_rows + WHERE kind = 'event' + AND (name IS NULL OR name NOT IN ( + 'message/retracted', + 'message/compaction_indicator', + 'migration/backfill' + )) + UNION ALL + SELECT + session_id, entry_id, kind, name, source_type, source_id, source_seq, + provenance_key, payload_json, meta_json, created_at + FROM effective_message_rows + UNION ALL + SELECT + ranked_tools.session_id, + ranked_tools.entry_id, + ranked_tools.kind, + ranked_tools.name, + ranked_tools.source_type, + ranked_tools.source_id, + ranked_tools.source_seq, + ranked_tools.provenance_key, + ranked_tools.payload_json, + ranked_tools.meta_json, + ranked_tools.created_at + FROM ranked_tools + INNER JOIN effective_message_rows AS message + ON message.session_id = ranked_tools.session_id + AND message.message_id = ranked_tools.message_id + WHERE ranked_tools.candidate_rank = 1 + ) +` + +const EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL = ` + candidate.kind = 'anchor' + OR ( + candidate.kind = 'event' + AND (candidate.name IS NULL OR candidate.name NOT IN ( + 'message/retracted', + 'message/compaction_indicator', + 'migration/backfill' + )) + ) + OR ( + candidate.kind = 'message' + AND ${effectiveTapeMessagePredicateSql('candidate')} + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS later_message + WHERE later_message.session_id = candidate.session_id + AND later_message.entry_id > candidate.entry_id + AND later_message.entry_id <= source.max_entry_id + AND later_message.kind = 'message' + AND ${effectiveTapeMessagePredicateSql('later_message')} + AND json_extract(later_message.payload_json, '$.record.id') = + json_extract(candidate.payload_json, '$.record.id') + ) + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS retraction + WHERE retraction.session_id = candidate.session_id + AND retraction.entry_id > candidate.entry_id + AND retraction.entry_id <= source.max_entry_id + AND retraction.kind = 'event' + AND retraction.name = 'message/retracted' + AND (${tapeRetractionMessageIdSql('retraction')}) = + json_extract(candidate.payload_json, '$.record.id') + ) + ) + OR ( + candidate.kind IN ('tool_call', 'tool_result') + AND json_extract(candidate.meta_json, '$.status') IN ('success', 'error') + AND typeof(json_extract(candidate.payload_json, '$.messageId')) = 'text' + AND length(json_extract(candidate.payload_json, '$.messageId')) > 0 + AND typeof((${tapeToolCallIdSql('candidate')})) = 'text' + AND length((${tapeToolCallIdSql('candidate')})) > 0 + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS later_tool + WHERE later_tool.session_id = candidate.session_id + AND later_tool.entry_id > candidate.entry_id + AND later_tool.entry_id <= source.max_entry_id + AND later_tool.kind = candidate.kind + AND json_extract(later_tool.meta_json, '$.status') IN ('success', 'error') + AND json_extract(later_tool.payload_json, '$.messageId') = + json_extract(candidate.payload_json, '$.messageId') + AND (${tapeToolCallIdSql('later_tool')}) = (${tapeToolCallIdSql('candidate')}) + ) + AND EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS message + WHERE message.session_id = candidate.session_id + AND message.entry_id <= source.max_entry_id + AND message.kind = 'message' + AND ${effectiveTapeMessagePredicateSql('message')} + AND json_extract(message.payload_json, '$.record.id') = + json_extract(candidate.payload_json, '$.messageId') + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS later_message + WHERE later_message.session_id = message.session_id + AND later_message.entry_id > message.entry_id + AND later_message.entry_id <= source.max_entry_id + AND later_message.kind = 'message' + AND ${effectiveTapeMessagePredicateSql('later_message')} + AND json_extract(later_message.payload_json, '$.record.id') = + json_extract(message.payload_json, '$.record.id') + ) + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS retraction + WHERE retraction.session_id = message.session_id + AND retraction.entry_id > message.entry_id + AND retraction.entry_id <= source.max_entry_id + AND retraction.kind = 'event' + AND retraction.name = 'message/retracted' + AND (${tapeRetractionMessageIdSql('retraction')}) = + json_extract(message.payload_json, '$.record.id') + ) + ) + ) +` + +export class DeepChatTapeEntriesTable + extends BaseTable + implements TapeEntryStore, TapeTransactionRunner, TapeBootstrapStore +{ + constructor( + db: Database.Database, + private readonly mutationProjection?: DeepChatTapeMutationProjection + ) { + super(db, 'deepchat_tape_entries') + } + + getCreateTableSQL(): string { + return ` + CREATE TABLE IF NOT EXISTS deepchat_tape_entries ( + session_id TEXT NOT NULL, + entry_id INTEGER NOT NULL, + kind TEXT NOT NULL, + name TEXT, + source_type TEXT, + source_id TEXT, + source_seq INTEGER, + provenance_key TEXT, + payload_json TEXT NOT NULL DEFAULT '{}', + meta_json TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, entry_id) + ); + ${TAPE_ENTRY_INDEX_SQL} + ` + } + + public createTable(): void { + if (!this.tableExists()) { + this.db.exec(this.getCreateTableSQL()) + return + } + this.ensureProvenanceColumns() + this.db.exec(TAPE_ENTRY_INDEX_SQL) + } + + getMigrationSQL(_version: number): string | null { + return null + } + + getLatestVersion(): number { + return 0 + } + + runInTransaction(operation: () => T): T { + return this.db.transaction(operation)() + } + + append(input: DeepChatTapeAppendInput): DeepChatTapeEntryRow { + const append = this.db.transaction(() => { + const provenanceKey = buildProvenanceKey(input) + if (input.idempotent && provenanceKey) { + const existing = this.getByProvenanceKey(input.sessionId, provenanceKey) + if (existing) { + return existing + } + } + + const createdAt = input.createdAt ?? Date.now() + const previousSessionMaxEntryId = this.getMaxEntryId(input.sessionId) + const row = { + session_id: input.sessionId, + entry_id: previousSessionMaxEntryId + 1, + kind: input.kind, + name: input.name ?? null, + source_type: input.source?.type ?? null, + source_id: input.source?.id ?? null, + source_seq: input.source?.seq ?? null, + provenance_key: provenanceKey, + payload_json: safeJsonStringify(input.payload), + meta_json: safeJsonStringify(input.meta), + created_at: createdAt + } satisfies DeepChatTapeEntryRow + + try { + this.db + .prepare( + `INSERT INTO deepchat_tape_entries ( + session_id, + entry_id, + kind, + name, + source_type, + source_id, + source_seq, + provenance_key, + payload_json, + meta_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run( + row.session_id, + row.entry_id, + row.kind, + row.name, + row.source_type, + row.source_id, + row.source_seq, + row.provenance_key, + row.payload_json, + row.meta_json, + row.created_at + ) + } catch (error) { + if (input.idempotent && provenanceKey) { + const existing = this.getByProvenanceKey(input.sessionId, provenanceKey) + if (existing) { + return existing + } + } + throw error + } + + if (this.mutationProjection) { + try { + const applyProjection = this.db.transaction(() => + this.mutationProjection?.applyAppendedEntry(row, previousSessionMaxEntryId) + ) + applyProjection() + } catch (error) { + this.mutationProjection.invalidateSession(row.session_id) + logger.warn( + `[Tape] memory ingestion projection append failed; session marked stale: ${String(error)}` + ) + } + } + + return row + }) + + return append() + } + + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow { + return this.append({ + sessionId: input.sessionId, + kind: 'anchor', + name: input.name, + source: input.source, + provenanceKey: input.provenanceKey, + payload: { + name: input.name, + state: input.state + }, + meta: input.meta, + createdAt: input.createdAt, + idempotent: input.idempotent + }) + } + + appendEvent(input: TapeEventAppendInput): DeepChatTapeEntryRow { + return this.append({ + sessionId: input.sessionId, + kind: 'event', + name: input.name, + source: input.source, + provenanceKey: input.provenanceKey, + payload: { + name: input.name, + data: input.data + }, + meta: input.meta, + createdAt: input.createdAt, + idempotent: input.idempotent + }) + } + + ensureBootstrapAnchor(sessionId: string): void { + const existing = this.db + .prepare( + `SELECT entry_id + FROM deepchat_tape_entries + WHERE session_id = ? AND kind = 'anchor' + ORDER BY entry_id ASC + LIMIT 1` + ) + .get(sessionId) as { entry_id: number } | undefined + + if (existing) { + return + } + + this.appendAnchor({ + sessionId, + name: 'session/start', + source: { + type: 'session', + id: sessionId, + seq: 0 + }, + state: { + owner: 'human' + }, + meta: { + [TAPE_INCARNATION_META_KEY]: randomUUID() + }, + idempotent: true + }) + } + + getBySession(sessionId: string): DeepChatTapeEntryRow[] { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? + ORDER BY entry_id ASC` + ) + .all(sessionId) as DeepChatTapeEntryRow[] + } + + getSubagentLineageEvents(sessionId: string): DeepChatTapeEntryRow[] { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? + AND kind = 'event' + AND name IN ('subagent/tape_linked', 'fork/merge') + ORDER BY entry_id ASC` + ) + .all(sessionId) as DeepChatTapeEntryRow[] + } + + getFirstEntriesBySessions(sessionIds: string[]): DeepChatTapeEntryRow[] { + const ids = [...new Set(sessionIds.map((id) => id.trim()).filter(Boolean))] + if (ids.length === 0) { + return [] + } + return this.db + .prepare( + `WITH requested_sessions(session_id) AS ( + SELECT value FROM json_each(?) + ), + first_entries(session_id, entry_id) AS ( + SELECT tape.session_id, MIN(tape.entry_id) + FROM deepchat_tape_entries AS tape + INNER JOIN requested_sessions AS requested + ON requested.session_id = tape.session_id + GROUP BY tape.session_id + ) + SELECT tape.* + FROM deepchat_tape_entries AS tape + INNER JOIN first_entries AS first + ON first.session_id = tape.session_id + AND first.entry_id = tape.entry_id + ORDER BY tape.session_id ASC` + ) + .all(JSON.stringify(ids)) as DeepChatTapeEntryRow[] + } + + getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND entry_id <= ? + ORDER BY entry_id ASC` + ) + .all(sessionId, maxEntryId) as DeepChatTapeEntryRow[] + } + + listMemoryViewManifestAnchorsBySessions( + sessionIds: string[], + optionsOrLimit: number | { limit?: number; messageId?: string } = 100 + ): DeepChatTapeEntryRow[] { + const uniqueSessionIds = [...new Set(sessionIds.filter((id) => id.trim().length > 0))] + if (uniqueSessionIds.length === 0) { + return [] + } + const options = typeof optionsOrLimit === 'number' ? { limit: optionsOrLimit } : optionsOrLimit + const cappedLimit = Math.min(Math.max(Math.floor(options.limit ?? 100), 1), 500) + const placeholders = uniqueSessionIds.map(() => '?').join(', ') + const whereClauses = [ + `session_id IN (${placeholders})`, + "kind = 'anchor'", + "name = 'memory/view_assembled'" + ] + const params: Array = [...uniqueSessionIds] + if (options.messageId) { + whereClauses.push("json_extract(meta_json, '$.messageId') = ?") + params.push(options.messageId) + } + params.push(cappedLimit) + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE ${whereClauses.join(' AND ')} + ORDER BY created_at DESC, entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeEntryRow[] + } + + listMemoryViewManifestAnchorsByAgent( + agentId: string, + options: { sessionId?: string; limit?: number; messageId?: string } = {} + ): DeepChatTapeEntryRow[] { + const cappedLimit = Math.min(Math.max(Math.floor(options.limit ?? 100), 1), 500) + const whereClauses = [ + 'sessions.agent_id = ?', + "tape.kind = 'anchor'", + "tape.name = 'memory/view_assembled'" + ] + const params: Array = [agentId] + if (options.sessionId) { + whereClauses.push('tape.session_id = ?') + params.push(options.sessionId) + } + if (options.messageId) { + whereClauses.push("json_extract(tape.meta_json, '$.messageId') = ?") + params.push(options.messageId) + } + params.push(cappedLimit) + return this.db + .prepare( + `SELECT tape.* + FROM deepchat_tape_entries AS tape + INNER JOIN new_sessions AS sessions + ON sessions.id = tape.session_id + WHERE ${whereClauses.join(' AND ')} + ORDER BY tape.created_at DESC, tape.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeEntryRow[] + } + + getEntriesAfter(sessionId: string, entryId: number): DeepChatTapeEntryRow[] { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND entry_id > ? + ORDER BY entry_id ASC` + ) + .all(sessionId, entryId) as DeepChatTapeEntryRow[] + } + + getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND kind = 'anchor' + ORDER BY entry_id DESC + LIMIT 1` + ) + .get(sessionId) as DeepChatTapeEntryRow | undefined + } + + getAnchors(sessionId: string, limit: number = 20): DeepChatTapeEntryRow[] { + const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) + const rows = this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND kind = 'anchor' + ORDER BY entry_id DESC + LIMIT ?` + ) + .all(sessionId, cappedLimit) as DeepChatTapeEntryRow[] + + return rows.reverse() + } + + getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + const placeholders = SUMMARY_ANCHOR_NAMES.map(() => '?').join(', ') + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? + AND kind = 'anchor' + AND name IN (${placeholders}) + ORDER BY entry_id DESC + LIMIT 1` + ) + .get(sessionId, ...SUMMARY_ANCHOR_NAMES) as DeepChatTapeEntryRow | undefined + } + + getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + const placeholders = RECONSTRUCTION_ANCHOR_NAMES.map(() => '?').join(', ') + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? + AND kind = 'anchor' + AND ( + name IN (${placeholders}) + OR name LIKE 'handoff/%' + OR name LIKE 'auto_handoff/%' + ) + ORDER BY entry_id DESC + LIMIT 1` + ) + .get(sessionId, ...RECONSTRUCTION_ANCHOR_NAMES) as DeepChatTapeEntryRow | undefined + } + + getByProvenanceKey(sessionId: string, provenanceKey: string): DeepChatTapeEntryRow | undefined { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND provenance_key = ? + LIMIT 1` + ) + .get(sessionId, provenanceKey) as DeepChatTapeEntryRow | undefined + } + + getMaxEntryId(sessionId: string): number { + const row = this.db + .prepare( + `SELECT MAX(entry_id) AS max_entry_id + FROM deepchat_tape_entries + WHERE session_id = ?` + ) + .get(sessionId) as { max_entry_id: number | null } | undefined + return row?.max_entry_id ?? 0 + } + + getMaxEntryIdsBySessions(sessionIds: string[]): Map { + const ids = [...new Set(sessionIds.map((id) => id.trim()).filter(Boolean))] + const maxEntryIdBySession = new Map(ids.map((id) => [id, 0])) + if (ids.length === 0) { + return maxEntryIdBySession + } + const rows = this.db + .prepare( + `WITH requested_sessions(session_id) AS ( + SELECT value FROM json_each(?) + ) + SELECT tape.session_id, MAX(tape.entry_id) AS max_entry_id + FROM deepchat_tape_entries AS tape + INNER JOIN requested_sessions AS requested + ON requested.session_id = tape.session_id + GROUP BY tape.session_id` + ) + .all(JSON.stringify(ids)) as Array<{ session_id: string; max_entry_id: number }> + for (const row of rows) { + maxEntryIdBySession.set(row.session_id, row.max_entry_id) + } + return maxEntryIdBySession + } + + countAnchorsBySession(sessionId: string): number { + const row = this.db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_entries + WHERE session_id = ? AND kind = 'anchor'` + ) + .get(sessionId) as { count: number } | undefined + return row?.count ?? 0 + } + + countEntriesAfter(sessionId: string, entryId: number): number { + const row = this.db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_entries + WHERE session_id = ? AND entry_id > ?` + ) + .get(sessionId, entryId) as { count: number } | undefined + return row?.count ?? 0 + } + + countBySession(sessionId: string): number { + const row = this.db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_entries + WHERE session_id = ?` + ) + .get(sessionId) as { count: number } | undefined + return row?.count ?? 0 + } + + search( + sessionId: string, + query: string, + options: DeepChatTapeSearchInput = {} + ): DeepChatTapeEntryRow[] { + const normalizedQuery = query.trim() + if (!normalizedQuery) { + return [] + } + const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 + const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) + const queryPredicate = buildDeepChatTapeLikeSearchPredicate( + ['payload_json', 'meta_json', 'name'], + normalizedQuery + ) + const whereClauses = ['session_id = ?', queryPredicate.sql] + const params: Array = [sessionId, ...queryPredicate.params] + + if (options.kinds?.length) { + whereClauses.push(`kind IN (${options.kinds.map(() => '?').join(', ')})`) + params.push(...options.kinds) + } + + if (Number.isFinite(options.startCreatedAt)) { + whereClauses.push('created_at >= ?') + params.push(options.startCreatedAt as number) + } + + if (Number.isFinite(options.endCreatedAt)) { + whereClauses.push('created_at <= ?') + params.push(options.endCreatedAt as number) + } + + params.push(cappedLimit) + + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE ${whereClauses.join(' AND ')} + ORDER BY entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeEntryRow[] + } + + searchEffectiveSourcesAtHeads( + sources: readonly DeepChatTapeReadSource[], + query: string, + options: DeepChatTapeSearchInput = {} + ): DeepChatTapeEntryRow[] { + const normalizedSources = normalizeDeepChatTapeReadSources(sources) + const normalizedQuery = query.trim() + if (normalizedSources.length === 0 || !normalizedQuery) { + return [] + } + + const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 + const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) + const queryPredicate = buildDeepChatTapeLikeSearchPredicate( + ['candidate.payload_json', 'candidate.meta_json', 'candidate.name'], + normalizedQuery + ) + const whereClauses = [queryPredicate.sql] + const params: Array = [ + serializeDeepChatTapeReadSources(normalizedSources), + ...queryPredicate.params + ] + + if (options.kinds?.length) { + whereClauses.push(`candidate.kind IN (${options.kinds.map(() => '?').join(', ')})`) + params.push(...options.kinds) + } + if (Number.isFinite(options.startCreatedAt)) { + whereClauses.push('candidate.created_at >= ?') + params.push(options.startCreatedAt as number) + } + if (Number.isFinite(options.endCreatedAt)) { + whereClauses.push('candidate.created_at <= ?') + params.push(options.endCreatedAt as number) + } + params.push(cappedLimit) + + return this.db + .prepare( + `WITH + ${AUTHORIZED_TAPE_SOURCES_CTE_SQL} + SELECT candidate.* + FROM deepchat_tape_entries AS candidate + INNER JOIN authorized_sources AS source + ON source.session_id = candidate.session_id + AND candidate.entry_id <= source.max_entry_id + WHERE ${whereClauses.join(' AND ')} + AND (${EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL}) + ORDER BY candidate.created_at DESC, candidate.session_id ASC, candidate.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeEntryRow[] + } + + getEffectiveContextRowsAtHead( + source: DeepChatTapeReadSource, + entryIds: number[], + options: { before: number; after: number; limit: number } + ): DeepChatTapeEntryRow[] { + const normalizedSource = normalizeDeepChatTapeReadSources([source])[0] + const requestedEntryIds = [ + ...new Set(entryIds.filter((entryId) => Number.isSafeInteger(entryId) && entryId > 0)) + ].sort((left, right) => left - right) + if (!normalizedSource || requestedEntryIds.length === 0) { + return [] + } + const before = Math.min(Math.max(Math.floor(options.before), 0), 20) + const after = Math.min(Math.max(Math.floor(options.after), 0), 20) + const limit = Math.min(Math.max(Math.floor(options.limit), 1), 100) + + return this.db + .prepare( + `WITH + ${AUTHORIZED_TAPE_SOURCES_CTE_SQL}, + ${EFFECTIVE_TAPE_ROWS_CTE_SQL}, + ordered_rows AS ( + SELECT + effective_rows.*, + ROW_NUMBER() OVER (ORDER BY entry_id ASC) AS row_position + FROM effective_rows + ), + requested_ids(entry_id, request_ordinal) AS ( + SELECT CAST(value AS INTEGER), CAST(key AS INTEGER) + FROM json_each(?) + ), + requested_positions AS ( + SELECT + requested_ids.request_ordinal, + ordered_rows.entry_id, + ordered_rows.row_position + FROM requested_ids + INNER JOIN ordered_rows + ON ordered_rows.entry_id = requested_ids.entry_id + ), + context_candidates AS ( + SELECT + ordered_rows.*, + 0 AS priority_group, + requested_positions.request_ordinal, + 0 AS neighbor_position + FROM requested_positions + INNER JOIN ordered_rows + ON ordered_rows.entry_id = requested_positions.entry_id + UNION ALL + SELECT + ordered_rows.*, + 1 AS priority_group, + requested_positions.request_ordinal, + ordered_rows.row_position AS neighbor_position + FROM requested_positions + INNER JOIN ordered_rows + ON ordered_rows.row_position BETWEEN requested_positions.row_position - ? + AND requested_positions.row_position + ? + AND ordered_rows.entry_id != requested_positions.entry_id + ), + ranked_context_candidates AS ( + SELECT + context_candidates.*, + ROW_NUMBER() OVER ( + PARTITION BY session_id, entry_id + ORDER BY priority_group, request_ordinal, neighbor_position + ) AS duplicate_rank + FROM context_candidates + ) + SELECT + session_id, entry_id, kind, name, source_type, source_id, source_seq, + provenance_key, payload_json, meta_json, created_at + FROM ranked_context_candidates + WHERE duplicate_rank = 1 + ORDER BY priority_group, request_ordinal, neighbor_position + LIMIT ?` + ) + .all( + serializeDeepChatTapeReadSources([normalizedSource]), + JSON.stringify(requestedEntryIds), + before, + after, + limit + ) as DeepChatTapeEntryRow[] + } + + private ensureProvenanceColumns(): void { + const columns: Array<[string, string]> = [ + ['source_type', 'TEXT'], + ['source_id', 'TEXT'], + ['source_seq', 'INTEGER'], + ['provenance_key', 'TEXT'] + ] + for (const [columnName, columnType] of columns) { + if (!this.hasColumn(columnName)) { + this.db.exec(`ALTER TABLE deepchat_tape_entries ADD COLUMN ${columnName} ${columnType}`) + } + } + } +} diff --git a/src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts b/src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts new file mode 100644 index 0000000000..f245c8d9aa --- /dev/null +++ b/src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts @@ -0,0 +1,17 @@ +import type Database from 'better-sqlite3-multiple-ciphers' +import type { TapeEntryLifecycleStore, TapeMutationProjection } from '../../ports/storage' + +export class SqliteTapeLifecycleAdapter implements TapeEntryLifecycleStore { + constructor( + private readonly db: Database.Database, + private readonly mutationProjection?: TapeMutationProjection + ) {} + + deleteBySession(sessionId: string): void { + const remove = this.db.transaction(() => { + this.db.prepare('DELETE FROM deepchat_tape_entries WHERE session_id = ?').run(sessionId) + this.mutationProjection?.deleteBySession(sessionId) + }) + remove() + } +} diff --git a/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts new file mode 100644 index 0000000000..9ff20d26ef --- /dev/null +++ b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts @@ -0,0 +1,895 @@ +import Database from 'better-sqlite3-multiple-ciphers' +import { BaseTable } from '@/data/baseTable' +import { buildDeepChatTapeFtsMatch, buildDeepChatTapeLikeSearchPredicate } from './tapeEntryStore' +import { + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources +} from '@/tape/domain/entry' +import type { DeepChatTapeReadSource, DeepChatTapeSearchInput } from '@/tape/domain/entry' +import type { + TapeSearchProjectionInput, + TapeSearchProjectionMeta, + TapeSearchProjectionReadResult, + TapeSearchProjectionResultRow, + TapeSearchProjectionRow, + TapeSearchProjectionStore +} from '@/tape/ports/application' + +export const DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION = 2 + +export type DeepChatTapeSearchProjectionInput = TapeSearchProjectionInput +export type DeepChatTapeSearchProjectionRow = TapeSearchProjectionRow +export type DeepChatTapeSearchProjectionResultRow = TapeSearchProjectionResultRow +export type DeepChatTapeSearchProjectionMeta = TapeSearchProjectionMeta +export type DeepChatTapeSearchProjectionReadResult = TapeSearchProjectionReadResult + +type FtsCapability = { available: boolean; tokenizer: 'trigram' | 'unicode61' } + +const TAPE_SEARCH_PROJECTION_INDEX_SQL = ` + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_kind + ON deepchat_tape_search_projection(session_id, kind, entry_id); + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_created + ON deepchat_tape_search_projection(session_id, created_at, entry_id); +` + +function normalizeLimit(limit: number | undefined): number { + return Math.min(Math.max(Math.floor(Number.isFinite(limit) ? (limit as number) : 20), 1), 100) +} + +function safeJsonStringify(value: Record): string { + return JSON.stringify(value ?? {}) +} + +function parseRefs(value: string): Record { + try { + const parsed = JSON.parse(value) + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : {} + } catch { + return {} + } +} + +export class DeepChatTapeSearchProjectionTable + extends BaseTable + implements TapeSearchProjectionStore +{ + private ftsCapability: FtsCapability | undefined + private ftsReady = false + + constructor(db: Database.Database) { + super(db, 'deepchat_tape_search_projection') + } + + getCreateTableSQL(): string { + return ` + CREATE TABLE IF NOT EXISTS deepchat_tape_search_projection ( + session_id TEXT NOT NULL, + entry_id INTEGER NOT NULL, + kind TEXT NOT NULL, + name TEXT, + source_type TEXT, + source_id TEXT, + source_seq INTEGER, + search_text TEXT NOT NULL, + summary_text TEXT NOT NULL, + refs_json TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, entry_id) + ); + CREATE TABLE IF NOT EXISTS deepchat_tape_search_projection_meta ( + session_id TEXT PRIMARY KEY, + projection_version INTEGER NOT NULL, + max_entry_id INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS deepchat_tape_search_fts_meta ( + session_id TEXT PRIMARY KEY, + projection_version INTEGER NOT NULL, + max_entry_id INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + ${TAPE_SEARCH_PROJECTION_INDEX_SQL} + ` + } + + override createTable(): void { + this.db.exec(this.getCreateTableSQL()) + if (!this.ftsReady) { + this.ensureFtsIndex() + } + } + + getMigrationSQL(_version: number): string | null { + return null + } + + getLatestVersion(): number { + return 0 + } + + getSessionMeta(sessionId: string): DeepChatTapeSearchProjectionMeta | null { + const row = this.db + .prepare( + `SELECT projection_version, max_entry_id + FROM deepchat_tape_search_projection_meta + WHERE session_id = ?` + ) + .get(sessionId) as + | { + projection_version: number + max_entry_id: number + } + | undefined + if (!row) return null + return { + projectionVersion: row.projection_version, + maxEntryId: row.max_entry_id + } + } + + isCurrent( + sessionId: string, + maxEntryId: number, + projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ): boolean { + const row = this.getSessionMeta(sessionId) + return row?.projectionVersion === projectionVersion && row.maxEntryId === maxEntryId + } + + getProjectedEntryIds(sessionId: string): number[] { + return ( + this.db + .prepare( + `SELECT entry_id + FROM deepchat_tape_search_projection + WHERE session_id = ? + ORDER BY entry_id ASC` + ) + .all(sessionId) as Array<{ entry_id: number }> + ).map((row) => row.entry_id) + } + + appendSession( + sessionId: string, + rows: DeepChatTapeSearchProjectionInput[], + maxEntryId: number, + projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ): void { + const previousMeta = this.getSessionMeta(sessionId) + try { + this.db.transaction(() => { + if (!this.ftsReady) { + this.clearSessionFtsForBaseWrite(sessionId) + } + this.insertProjectionRows(rows) + if (this.ftsReady) { + if (previousMeta && this.isFtsCurrent(sessionId, previousMeta)) { + this.insertFtsRows(rows) + } else { + this.replaceSessionFtsRows(sessionId, this.getSessionProjectionInputs(sessionId)) + } + this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) + } + this.upsertMeta(sessionId, projectionVersion, maxEntryId) + })() + } catch (error) { + if (this.ftsReady) { + this.ftsReady = false + } + throw error + } + } + + replaceSession( + sessionId: string, + rows: DeepChatTapeSearchProjectionInput[], + maxEntryId: number, + projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ): void { + try { + this.db.transaction(() => { + if (this.ftsReady) { + this.db + .prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?') + .run(sessionId) + this.db + .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') + .run(sessionId) + } else { + this.clearSessionFtsForBaseWrite(sessionId) + } + this.db + .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') + .run(sessionId) + this.db + .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') + .run(sessionId) + this.insertProjectionRows(rows) + if (this.ftsReady) { + this.insertFtsRows(rows) + this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) + } + this.upsertMeta(sessionId, projectionVersion, maxEntryId) + })() + } catch (error) { + if (this.ftsReady) { + this.ftsReady = false + } + throw error + } + } + + private insertProjectionRows(rows: DeepChatTapeSearchProjectionInput[]): void { + if (!rows.length) return + const insertProjection = this.db.prepare( + `INSERT OR REPLACE INTO deepchat_tape_search_projection ( + session_id, + entry_id, + kind, + name, + source_type, + source_id, + source_seq, + search_text, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + for (const row of rows) { + insertProjection.run( + row.sessionId, + row.entryId, + row.kind, + row.name, + row.sourceType, + row.sourceId, + row.sourceSeq, + row.searchText, + row.summaryText, + safeJsonStringify(row.refs), + row.createdAt + ) + } + } + + private upsertMeta(sessionId: string, projectionVersion: number, maxEntryId: number): void { + this.db + .prepare( + `INSERT INTO deepchat_tape_search_projection_meta ( + session_id, + projection_version, + max_entry_id, + updated_at + ) + VALUES (?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + projection_version = excluded.projection_version, + max_entry_id = excluded.max_entry_id, + updated_at = excluded.updated_at` + ) + .run(sessionId, projectionVersion, maxEntryId, Date.now()) + } + + private getFtsMeta(sessionId: string): DeepChatTapeSearchProjectionMeta | null { + const row = this.db + .prepare( + `SELECT projection_version, max_entry_id + FROM deepchat_tape_search_fts_meta + WHERE session_id = ?` + ) + .get(sessionId) as + | { + projection_version: number + max_entry_id: number + } + | undefined + if (!row) return null + return { + projectionVersion: row.projection_version, + maxEntryId: row.max_entry_id + } + } + + private isFtsCurrent(sessionId: string, meta: DeepChatTapeSearchProjectionMeta): boolean { + const row = this.getFtsMeta(sessionId) + return row?.projectionVersion === meta.projectionVersion && row.maxEntryId === meta.maxEntryId + } + + private upsertFtsMeta(sessionId: string, projectionVersion: number, maxEntryId: number): void { + this.db + .prepare( + `INSERT INTO deepchat_tape_search_fts_meta ( + session_id, + projection_version, + max_entry_id, + updated_at + ) + VALUES (?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + projection_version = excluded.projection_version, + max_entry_id = excluded.max_entry_id, + updated_at = excluded.updated_at` + ) + .run(sessionId, projectionVersion, maxEntryId, Date.now()) + } + + getByEntryIds(sessionId: string, entryIds: number[]): DeepChatTapeSearchProjectionRow[] { + const ids = [...new Set(entryIds.filter((id) => Number.isInteger(id) && id > 0))] + if (!ids.length) return [] + const placeholders = ids.map(() => '?').join(', ') + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_search_projection + WHERE session_id = ? AND entry_id IN (${placeholders}) + ORDER BY entry_id ASC` + ) + .all(sessionId, ...ids) as DeepChatTapeSearchProjectionRow[] + } + + searchSourcesReadOnly( + sources: readonly DeepChatTapeReadSource[], + query: string, + options: DeepChatTapeSearchInput = {} + ): DeepChatTapeSearchProjectionReadResult { + const normalizedSources = normalizeDeepChatTapeReadSources(sources) + const coveredSources = this.getCurrentReadSources( + normalizedSources, + 'deepchat_tape_search_projection_meta' + ) + const normalizedQuery = query.trim() + if (coveredSources.length === 0 || !normalizedQuery) { + return { rows: [], coveredSources } + } + if (coveredSources.length !== normalizedSources.length) { + return { rows: [], coveredSources } + } + + const limit = normalizeLimit(options.limit) + const ordered: DeepChatTapeSearchProjectionResultRow[] = [] + const seen = new Set() + const collect = (rows: DeepChatTapeSearchProjectionResultRow[]): void => { + for (const row of rows) { + const key = `${row.session_id}:${row.entry_id}` + if (seen.has(key)) continue + seen.add(key) + ordered.push(row) + } + } + + if (this.ftsReady) { + const ftsSources = this.getCurrentReadSources(coveredSources, 'deepchat_tape_search_fts_meta') + if (ftsSources.length === coveredSources.length) { + try { + collect(this.searchFtsSourcesReadOnly(ftsSources, normalizedQuery, options, limit)) + } catch { + // The base projection remains a complete read-only fallback. + } + } + } + if (ordered.length < limit) { + collect(this.searchLikeSourcesReadOnly(coveredSources, normalizedQuery, options, limit)) + } + + return { rows: ordered.slice(0, limit), coveredSources } + } + + search( + sessionId: string, + query: string, + options: DeepChatTapeSearchInput = {} + ): DeepChatTapeSearchProjectionResultRow[] { + const normalized = query.trim() + if (!normalized) return [] + const limit = normalizeLimit(options.limit) + const ordered: DeepChatTapeSearchProjectionResultRow[] = [] + const seen = new Set() + const collect = (rows: DeepChatTapeSearchProjectionResultRow[]): void => { + for (const row of rows) { + if (seen.has(row.entry_id)) continue + seen.add(row.entry_id) + ordered.push(row) + } + } + this.recoverSessionFts(sessionId) + if (this.ftsReady) { + collect(this.searchFts(sessionId, normalized, options, limit)) + } + if (!this.ftsReady || ordered.length < limit) { + collect(this.searchLike(sessionId, normalized, options, limit)) + } + return ordered.slice(0, limit) + } + + deleteBySession(sessionId: string): void { + this.db + .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') + .run(sessionId) + this.db + .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') + .run(sessionId) + this.deleteSessionFts(sessionId) + } + + clearAll(): void { + this.db.prepare('DELETE FROM deepchat_tape_search_projection').run() + this.db.prepare('DELETE FROM deepchat_tape_search_projection_meta').run() + if (this.ftsMetaTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() + } + this.clearFts() + } + + private detectFtsCapability(): FtsCapability { + if (this.ftsCapability) return this.ftsCapability + const probe = (tokenizer: string): boolean => { + const name = `temp.tape_search_fts_probe_${tokenizer}` + try { + this.db.exec( + `CREATE VIRTUAL TABLE IF NOT EXISTS ${name} USING fts5(c, tokenize='${tokenizer}');` + ) + this.db.exec(`DROP TABLE IF EXISTS ${name};`) + return true + } catch { + return false + } + } + if (probe('trigram')) this.ftsCapability = { available: true, tokenizer: 'trigram' } + else if (probe('unicode61')) this.ftsCapability = { available: true, tokenizer: 'unicode61' } + else this.ftsCapability = { available: false, tokenizer: 'unicode61' } + return this.ftsCapability + } + + private ensureFtsIndex(): void { + const capability = this.detectFtsCapability() + if (!capability.available) { + this.ftsReady = false + return + } + try { + this.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS deepchat_tape_search_fts USING fts5( + search_text, + name, + session_id UNINDEXED, + entry_id UNINDEXED, + kind UNINDEXED, + source_type UNINDEXED, + source_id UNINDEXED, + source_seq UNINDEXED, + summary_text UNINDEXED, + refs_json UNINDEXED, + created_at UNINDEXED, + tokenize='${capability.tokenizer}' + ); + `) + this.ftsReady = true + } catch { + this.ftsReady = false + } + } + + private toProjectionInput( + row: DeepChatTapeSearchProjectionRow + ): DeepChatTapeSearchProjectionInput { + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + sourceType: row.source_type, + sourceId: row.source_id, + sourceSeq: row.source_seq, + searchText: row.search_text, + summaryText: row.summary_text, + refs: parseRefs(row.refs_json), + createdAt: row.created_at + } + } + + private getSessionProjectionInputs(sessionId: string): DeepChatTapeSearchProjectionInput[] { + return ( + this.db + .prepare( + `SELECT * + FROM deepchat_tape_search_projection + WHERE session_id = ? + ORDER BY entry_id ASC` + ) + .all(sessionId) as DeepChatTapeSearchProjectionRow[] + ).map((row) => this.toProjectionInput(row)) + } + + private ftsTableExists(): boolean { + const row = this.db + .prepare( + `SELECT name + FROM sqlite_master + WHERE type = 'table' AND name = 'deepchat_tape_search_fts' + LIMIT 1` + ) + .get() as { name: string } | undefined + return Boolean(row) + } + + private ftsMetaTableExists(): boolean { + const row = this.db + .prepare( + `SELECT name + FROM sqlite_master + WHERE type = 'table' AND name = 'deepchat_tape_search_fts_meta' + LIMIT 1` + ) + .get() as { name: string } | undefined + return Boolean(row) + } + + private clearSessionFtsForBaseWrite(sessionId: string): void { + if (this.ftsMetaTableExists()) { + this.db + .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') + .run(sessionId) + } + if (this.ftsTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) + } + } + + private getProjectionRowId(sessionId: string, entryId: number): number { + const row = this.db + .prepare( + `SELECT rowid + FROM deepchat_tape_search_projection + WHERE session_id = ? AND entry_id = ?` + ) + .get(sessionId, entryId) as { rowid: number } | undefined + if (!row) { + throw new Error(`Missing tape search projection row for ${sessionId}:${entryId}`) + } + return row.rowid + } + + private insertFtsRows(rows: DeepChatTapeSearchProjectionInput[]): void { + if (!rows.length) return + const insertFts = this.db.prepare( + `INSERT INTO deepchat_tape_search_fts ( + rowid, + search_text, + name, + session_id, + entry_id, + kind, + source_type, + source_id, + source_seq, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + for (const row of rows) { + this.db + .prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ? AND entry_id = ?') + .run(row.sessionId, row.entryId) + insertFts.run( + this.getProjectionRowId(row.sessionId, row.entryId), + row.searchText, + row.name ?? '', + row.sessionId, + row.entryId, + row.kind, + row.sourceType, + row.sourceId, + row.sourceSeq, + row.summaryText, + safeJsonStringify(row.refs), + row.createdAt + ) + } + } + + private replaceSessionFtsRows( + sessionId: string, + rows: DeepChatTapeSearchProjectionInput[] + ): void { + this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) + this.insertFtsRows(rows) + } + + private replaceSessionFts( + sessionId: string, + rows: DeepChatTapeSearchProjectionInput[], + projectionVersion: number, + maxEntryId: number + ): boolean { + if (!this.ftsReady) return false + try { + this.db.transaction(() => { + this.replaceSessionFtsRows(sessionId, rows) + this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) + })() + return true + } catch { + this.ftsReady = false + return false + } + } + + private recoverSessionFts(sessionId: string): void { + const meta = this.getSessionMeta(sessionId) + if (!meta) { + this.deleteSessionFts(sessionId) + return + } + if (this.ftsReady && this.isFtsCurrent(sessionId, meta)) return + if (!this.ftsReady) { + this.ensureFtsIndex() + } + if (!this.ftsReady) return + if (this.isFtsCurrent(sessionId, meta)) return + this.replaceSessionFts( + sessionId, + this.getSessionProjectionInputs(sessionId), + meta.projectionVersion, + meta.maxEntryId + ) + } + + hasFtsReadyForTesting(): boolean { + return this.ftsReady + } + + disableFtsForTesting(): void { + this.ftsReady = false + } + + dropFtsForTesting(): void { + this.db.exec('DROP TABLE IF EXISTS deepchat_tape_search_fts') + if (this.ftsMetaTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() + } + this.ftsReady = false + } + + private deleteSessionFts(sessionId: string): void { + if (this.ftsMetaTableExists()) { + this.db + .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') + .run(sessionId) + } + if (!this.ftsTableExists()) return + try { + this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) + } catch { + this.ftsReady = false + } + } + + private clearFts(): void { + if (this.ftsMetaTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() + } + if (!this.ftsTableExists()) return + try { + this.db.prepare('DELETE FROM deepchat_tape_search_fts').run() + } catch { + this.ftsReady = false + } + } + + private searchFts( + sessionId: string, + normalized: string, + options: DeepChatTapeSearchInput, + limit: number + ): DeepChatTapeSearchProjectionResultRow[] { + const match = buildDeepChatTapeFtsMatch(normalized) + const whereClauses = [ + 'deepchat_tape_search_fts MATCH ?', + 'deepchat_tape_search_fts.session_id = ?', + 'projection.session_id = ?' + ] + const params: Array = [match, sessionId, sessionId] + this.addFilters(whereClauses, params, options, true, 'projection') + params.push(limit) + try { + return this.db + .prepare( + `SELECT + projection.session_id, + projection.entry_id, + projection.kind, + projection.name, + projection.source_type, + projection.source_id, + projection.source_seq, + projection.search_text, + projection.summary_text, + projection.refs_json, + projection.created_at, + bm25(deepchat_tape_search_fts) AS score + FROM deepchat_tape_search_fts + INNER JOIN deepchat_tape_search_projection AS projection + ON projection.session_id = deepchat_tape_search_fts.session_id + AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) + AND projection.search_text = deepchat_tape_search_fts.search_text + WHERE ${whereClauses.join(' AND ')} + ORDER BY score ASC, projection.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeSearchProjectionResultRow[] + } catch { + return [] + } + } + + private getCurrentReadSources( + sources: readonly DeepChatTapeReadSource[], + metaTable: 'deepchat_tape_search_projection_meta' | 'deepchat_tape_search_fts_meta' + ): DeepChatTapeReadSource[] { + const normalizedSources = normalizeDeepChatTapeReadSources(sources) + if (normalizedSources.length === 0) { + return [] + } + const rows = this.db + .prepare( + `WITH requested_sources(session_id, max_entry_id) AS ( + SELECT + json_extract(value, '$.sessionId'), + CAST(json_extract(value, '$.maxEntryId') AS INTEGER) + FROM json_each(?) + ) + SELECT requested.session_id, requested.max_entry_id + FROM requested_sources AS requested + INNER JOIN ${metaTable} AS meta + ON meta.session_id = requested.session_id + AND meta.max_entry_id = requested.max_entry_id + AND meta.projection_version = ? + ORDER BY requested.session_id ASC` + ) + .all( + serializeDeepChatTapeReadSources(normalizedSources), + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ) as Array<{ session_id: string; max_entry_id: number }> + return rows.map((row) => ({ sessionId: row.session_id, maxEntryId: row.max_entry_id })) + } + + private searchFtsSourcesReadOnly( + sources: readonly DeepChatTapeReadSource[], + normalized: string, + options: DeepChatTapeSearchInput, + limit: number + ): DeepChatTapeSearchProjectionResultRow[] { + const match = buildDeepChatTapeFtsMatch(normalized) + const whereClauses = ['deepchat_tape_search_fts MATCH ?'] + const params: Array = [serializeDeepChatTapeReadSources(sources), match] + this.addFilters(whereClauses, params, options, true, 'projection') + params.push(limit) + return this.db + .prepare( + `WITH authorized_sources(session_id, max_entry_id) AS ( + SELECT + json_extract(value, '$.sessionId'), + CAST(json_extract(value, '$.maxEntryId') AS INTEGER) + FROM json_each(?) + ) + SELECT + projection.session_id, + projection.entry_id, + projection.kind, + projection.name, + projection.source_type, + projection.source_id, + projection.source_seq, + projection.search_text, + projection.summary_text, + projection.refs_json, + projection.created_at, + bm25(deepchat_tape_search_fts) AS score + FROM deepchat_tape_search_fts + INNER JOIN deepchat_tape_search_projection AS projection + ON projection.session_id = deepchat_tape_search_fts.session_id + AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) + AND projection.search_text = deepchat_tape_search_fts.search_text + INNER JOIN authorized_sources AS source + ON source.session_id = projection.session_id + AND projection.entry_id <= source.max_entry_id + WHERE ${whereClauses.join(' AND ')} + ORDER BY score ASC, projection.created_at DESC, projection.session_id ASC, + projection.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeSearchProjectionResultRow[] + } + + private searchLikeSourcesReadOnly( + sources: readonly DeepChatTapeReadSource[], + normalized: string, + options: DeepChatTapeSearchInput, + limit: number + ): DeepChatTapeSearchProjectionResultRow[] { + const queryPredicate = buildDeepChatTapeLikeSearchPredicate( + ['projection.search_text', 'projection.summary_text', 'projection.name'], + normalized + ) + const whereClauses = [queryPredicate.sql] + const params: Array = [serializeDeepChatTapeReadSources(sources)] + params.push(...queryPredicate.params) + this.addFilters(whereClauses, params, options, false, 'projection') + params.push(limit) + return this.db + .prepare( + `WITH authorized_sources(session_id, max_entry_id) AS ( + SELECT + json_extract(value, '$.sessionId'), + CAST(json_extract(value, '$.maxEntryId') AS INTEGER) + FROM json_each(?) + ) + SELECT projection.*, NULL AS score + FROM deepchat_tape_search_projection AS projection + INNER JOIN authorized_sources AS source + ON source.session_id = projection.session_id + AND projection.entry_id <= source.max_entry_id + WHERE ${whereClauses.join(' AND ')} + ORDER BY projection.created_at DESC, projection.session_id ASC, projection.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeSearchProjectionResultRow[] + } + + private searchLike( + sessionId: string, + normalized: string, + options: DeepChatTapeSearchInput, + limit: number + ): DeepChatTapeSearchProjectionResultRow[] { + const queryPredicate = buildDeepChatTapeLikeSearchPredicate( + ['search_text', 'summary_text', 'name'], + normalized + ) + const whereClauses = ['session_id = ?', queryPredicate.sql] + const params: Array = [sessionId] + params.push(...queryPredicate.params) + this.addFilters(whereClauses, params, options) + params.push(limit) + return this.db + .prepare( + `SELECT *, NULL AS score + FROM deepchat_tape_search_projection + WHERE ${whereClauses.join(' AND ')} + ORDER BY entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeSearchProjectionResultRow[] + } + + private addFilters( + whereClauses: string[], + params: Array, + options: DeepChatTapeSearchInput, + castCreatedAt = false, + tableAlias?: string + ): void { + const column = (name: string) => (tableAlias ? `${tableAlias}.${name}` : name) + if (options.kinds?.length) { + whereClauses.push(`${column('kind')} IN (${options.kinds.map(() => '?').join(', ')})`) + params.push(...options.kinds) + } + const createdAtColumn = castCreatedAt + ? `CAST(${column('created_at')} AS INTEGER)` + : column('created_at') + if (Number.isFinite(options.startCreatedAt)) { + whereClauses.push(`${createdAtColumn} >= ?`) + params.push(options.startCreatedAt as number) + } + if (Number.isFinite(options.endCreatedAt)) { + whereClauses.push(`${createdAtColumn} <= ?`) + params.push(options.endCreatedAt as number) + } + } +} diff --git a/src/main/tape/ports/application.ts b/src/main/tape/ports/application.ts index ccda7e4fd1..cdf7b2d598 100644 --- a/src/main/tape/ports/application.ts +++ b/src/main/tape/ports/application.ts @@ -140,7 +140,8 @@ export interface TapeTerminalMessageReader { export type TapeApplicationEntryStore = TapeEntryStore & TapeTransactionRunner & TapeBootstrapStore export interface TapeApplicationDatabase { - readonly deepchatTapeEntriesTable: TapeApplicationEntryStore & TapeEntryLifecycleStore + readonly deepchatTapeEntriesTable: TapeApplicationEntryStore + readonly tapeLifecycle: TapeEntryLifecycleStore readonly deepchatTapeSearchProjectionTable: TapeSearchProjectionStore readonly newSessionsTable: TapeLineageSessionReader readonly deepchatSessionsTable: TapeLegacySummaryReader @@ -150,7 +151,7 @@ export interface TapeApplicationDatabase { export interface TapeApplicationProviders { getEntryStore(): TapeApplicationEntryStore - getForkStore(): TapeApplicationEntryStore & TapeEntryLifecycleStore + getEntryLifecycleStore(): TapeEntryLifecycleStore getSearchProjectionStore(): TapeSearchProjectionStore getLineageSessionReader(): TapeLineageSessionReader getLegacySummaryReader(): TapeLegacySummaryReader @@ -163,7 +164,7 @@ export function createTapeApplicationProviders( ): TapeApplicationProviders { return { getEntryStore: () => database.deepchatTapeEntriesTable, - getForkStore: () => database.deepchatTapeEntriesTable, + getEntryLifecycleStore: () => database.tapeLifecycle, getSearchProjectionStore: () => database.deepchatTapeSearchProjectionTable, getLineageSessionReader: () => database.newSessionsTable, getLegacySummaryReader: () => database.deepchatSessionsTable, diff --git a/src/main/tape/ports/capabilities.ts b/src/main/tape/ports/capabilities.ts index 8851136905..ca7b5fc6ce 100644 --- a/src/main/tape/ports/capabilities.ts +++ b/src/main/tape/ports/capabilities.ts @@ -38,6 +38,7 @@ export interface TapeInspectionReader { } export interface TapeLifecycleAdmin { + initializeSessionTape(sessionId: string): void deleteSessionTape(sessionId: string): void resetSessionTape(sessionId: string): void } diff --git a/test/main/agent/deepchat/memory/memoryRuntimeCoordinator.test.ts b/test/main/agent/deepchat/memory/memoryRuntimeCoordinator.test.ts index ee02cefd3a..2b724c9616 100644 --- a/test/main/agent/deepchat/memory/memoryRuntimeCoordinator.test.ts +++ b/test/main/agent/deepchat/memory/memoryRuntimeCoordinator.test.ts @@ -95,6 +95,8 @@ function createHarness() { replaceSession: vi.fn(), invalidateSession: vi.fn() } + const getTapeRows = vi.fn(() => tapeRows) + const appendTapeAnchor = vi.fn() const deps = { memoryPort: port as any, getSessionAgentId: vi.fn(() => 'agent-a'), @@ -116,8 +118,16 @@ function createHarness() { rewindMemoryCursorOrderSeq: vi.fn((_sessionId: string, orderSeq: number) => { cursor = orderSeq }), - getTapeRows: vi.fn(() => tapeRows), - appendTapeAnchor: vi.fn(), + tapeReader: { + getBySession: getTapeRows, + getBySessionUpToEntryId: vi.fn((_sessionId: string, maxEntryId: number) => + tapeRows.filter((row) => row.entry_id <= maxEntryId) + ), + getMaxEntryId: vi.fn(() => tapeRows.at(-1)?.entry_id ?? 0) + }, + tapeAnchorWriter: { appendAnchor: appendTapeAnchor }, + getTapeRows, + appendTapeAnchor, getIngestionProjection: vi.fn(() => projection) } const coordinator = new MemoryRuntimeCoordinator(deps) diff --git a/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts b/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts index 16498a5563..2520e905f2 100644 --- a/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts +++ b/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts @@ -443,6 +443,7 @@ function createMockSqlitePresenter() { memoryIngestionProjectionCurrent = false }) }), + tapeLifecycle: deepchatTapeEntriesTable, deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn(), isCurrent: vi.fn().mockReturnValue(false), diff --git a/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts b/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts index 13e7c9e9d3..aa8efc4a2c 100644 --- a/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts +++ b/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts @@ -1,12 +1,17 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' const sqliteModule = await import('better-sqlite3-multiple-ciphers').catch(() => null) const tableModule = sqliteModule ? await import('@/session/data/tables/deepchatTapeEntries') : null +const lifecycleModule = sqliteModule + ? await import('@/tape/infrastructure/sqlite/tapeLifecycleAdapter') + : null const Database = sqliteModule?.default const DeepChatTapeEntriesTable = tableModule?.DeepChatTapeEntriesTable +const SqliteTapeLifecycleAdapter = lifecycleModule?.SqliteTapeLifecycleAdapter const DatabaseCtor = Database! const DeepChatTapeEntriesTableCtor = DeepChatTapeEntriesTable! +const SqliteTapeLifecycleAdapterCtor = SqliteTapeLifecycleAdapter! let sqliteAvailable = false if (Database) { @@ -26,7 +31,8 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { const db = new DatabaseCtor(':memory:') const table = new DeepChatTapeEntriesTableCtor(db) table.createTable() - return { db, table } + const lifecycle = new SqliteTapeLifecycleAdapterCtor(db) + return { db, table, lifecycle } } it('keeps memory/persona anchors out of context reconstruction (C7, AC-7.3)', () => { @@ -86,7 +92,7 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { }) it('assigns a new Tape incarnation when a session Tape is rebuilt', () => { - const { db, table } = createTable() + const { db, table, lifecycle } = createTable() table.ensureBootstrapAnchor('s1') table.ensureBootstrapAnchor('s2') @@ -101,7 +107,7 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { expect(secondIncarnation).toEqual(expect.any(String)) expect(secondIncarnation).not.toBe(firstIncarnation) - table.deleteBySession('s1') + lifecycle.deleteBySession('s1') table.ensureBootstrapAnchor('s1') const rebuiltIncarnation = JSON.parse( table.getFirstEntriesBySessions(['s1'])[0].meta_json @@ -112,6 +118,43 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { db.close() }) + it('deletes Tape and its mutation projection through the lifecycle adapter', () => { + const db = new DatabaseCtor(':memory:') + const projection = { + applyAppendedEntry: vi.fn(() => true), + invalidateSession: vi.fn(), + deleteBySession: vi.fn() + } + const table = new DeepChatTapeEntriesTableCtor(db, projection) + table.createTable() + const lifecycle = new SqliteTapeLifecycleAdapterCtor(db, projection) + table.ensureBootstrapAnchor('s1') + + lifecycle.deleteBySession('s1') + + expect(table.getBySession('s1')).toEqual([]) + expect(projection.deleteBySession).toHaveBeenCalledWith('s1') + db.close() + }) + + it('rolls back Tape deletion when mutation projection cleanup fails', () => { + const db = new DatabaseCtor(':memory:') + const table = new DeepChatTapeEntriesTableCtor(db) + table.createTable() + table.ensureBootstrapAnchor('s1') + const lifecycle = new SqliteTapeLifecycleAdapterCtor(db, { + applyAppendedEntry: vi.fn(() => true), + invalidateSession: vi.fn(), + deleteBySession: vi.fn(() => { + throw new Error('projection cleanup failed') + }) + }) + + expect(() => lifecycle.deleteBySession('s1')).toThrow('projection cleanup failed') + expect(table.getBySession('s1')).toHaveLength(1) + db.close() + }) + it('tracks the latest summary-related anchor only within the requested session', () => { const { db, table } = createTable() diff --git a/test/main/session/data/tapeFork.test.ts b/test/main/session/data/tapeFork.test.ts index 27dfb7d3b0..6d30b000ad 100644 --- a/test/main/session/data/tapeFork.test.ts +++ b/test/main/session/data/tapeFork.test.ts @@ -5,6 +5,7 @@ import { vi, SessionTape, DeepChatTapeEntriesTable, + SqliteTapeLifecycleAdapter, DatabaseCtor, itIfSqlite, createTapeTableMock, @@ -16,6 +17,8 @@ describe('SessionTape forks', () => { const { table, entries } = createTapeTableMock() const service = new SessionTape({ deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn() }, deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } } as any) @@ -183,6 +186,8 @@ describe('SessionTape forks', () => { const { table, entries } = createTapeTableMock() const service = new SessionTape({ deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn() }, deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } } as any) @@ -332,6 +337,8 @@ describe('SessionTape forks', () => { table.createTable() const service = new SessionTape({ deepchatTapeEntriesTable: table, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn() }, deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } } as any) @@ -358,6 +365,7 @@ describe('SessionTape forks', () => { } const service = new SessionTape({ deepchatTapeEntriesTable: table, + tapeLifecycle: table, deepchatTapeSearchProjectionTable: projectionTable, deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } } as any) diff --git a/test/main/session/data/tapeLifecycle.test.ts b/test/main/session/data/tapeLifecycle.test.ts new file mode 100644 index 0000000000..421d2ea7e1 --- /dev/null +++ b/test/main/session/data/tapeLifecycle.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it, vi } from 'vitest' +import { SessionTape } from '@/session/data/tape' + +function createHarness() { + const calls: string[] = [] + const entryStore = { + ensureBootstrapAnchor: vi.fn((sessionId: string) => { + calls.push(`bootstrap:${sessionId}`) + }) + } + const lifecycle = { + deleteBySession: vi.fn((sessionId: string) => { + calls.push(`entries:${sessionId}`) + }) + } + const searchProjection = { + deleteBySession: vi.fn((sessionId: string) => { + calls.push(`search:${sessionId}`) + }) + } + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: lifecycle, + deepchatTapeSearchProjectionTable: searchProjection + } as any) + + return { calls, entryStore, lifecycle, searchProjection, tape } +} + +describe('SessionTape lifecycle administration', () => { + it('deletes entries before the search projection', () => { + const { calls, tape } = createHarness() + + tape.deleteSessionTape('s1') + + expect(calls).toEqual(['entries:s1', 'search:s1']) + }) + + it('rebuilds the bootstrap only after both destructive stores are cleared', () => { + const { calls, tape } = createHarness() + + tape.resetSessionTape('s1') + + expect(calls).toEqual(['entries:s1', 'search:s1', 'bootstrap:s1']) + }) + + it('does not create a mixed-generation Tape when projection cleanup fails', () => { + const { entryStore, searchProjection, tape } = createHarness() + searchProjection.deleteBySession.mockImplementationOnce(() => { + throw new Error('search cleanup failed') + }) + + expect(() => tape.resetSessionTape('s1')).toThrow('search cleanup failed') + expect(entryStore.ensureBootstrapAnchor).not.toHaveBeenCalled() + }) +}) diff --git a/test/main/session/data/tapeTestHarness.ts b/test/main/session/data/tapeTestHarness.ts index 0aed0aad2d..5128b968b6 100644 --- a/test/main/session/data/tapeTestHarness.ts +++ b/test/main/session/data/tapeTestHarness.ts @@ -16,6 +16,7 @@ import { } from '@/session/data/tapeFacts' import { buildRequestRefs } from '@/session/data/tapeViewManifest' import { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' +import { SqliteTapeLifecycleAdapter } from '@/tape/infrastructure/sqlite/tapeLifecycleAdapter' import { DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, DeepChatTapeSearchProjectionTable @@ -411,6 +412,7 @@ function createTapeService( ) { return new SessionTape({ deepchatTapeEntriesTable: table, + tapeLifecycle: table, deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn(), getByEntryIds: vi.fn().mockReturnValue([]) @@ -440,6 +442,7 @@ function createLinkedTapeService( return { service: new SessionTape({ deepchatTapeEntriesTable: table, + tapeLifecycle: table, deepchatTapeSearchProjectionTable: projectionTable, newSessionsTable: { get: vi.fn((sessionId: string) => sessionById.get(sessionId)), @@ -611,6 +614,7 @@ export { appendToolFactsToTape, buildRequestRefs, DeepChatTapeEntriesTable, + SqliteTapeLifecycleAdapter, DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, DeepChatTapeSearchProjectionTable, DeepChatMemoryIngestionProjectionTable, diff --git a/test/main/session/runtimeIntegration.test.ts b/test/main/session/runtimeIntegration.test.ts index fcfc6f412f..a9df4ec780 100644 --- a/test/main/session/runtimeIntegration.test.ts +++ b/test/main/session/runtimeIntegration.test.ts @@ -638,6 +638,7 @@ function createMockSqlitePresenter() { }) }, deepchatTapeEntriesTable: tapeTable, + tapeLifecycle: tapeTable, deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn(), isCurrent: vi.fn().mockReturnValue(false), From afc61a5040e66465ed87c431158277385c1c34e9 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 16:26:09 +0800 Subject: [PATCH 06/29] test(tape): enforce layer boundaries --- docs/architecture/tape-layering/tasks.md | 8 +- test/main/tape/layerBoundaries.test.ts | 154 +++++++++++++++++++++++ 2 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 test/main/tape/layerBoundaries.test.ts diff --git a/docs/architecture/tape-layering/tasks.md b/docs/architecture/tape-layering/tasks.md index 48bf64fcc6..7dba2eae47 100644 --- a/docs/architecture/tape-layering/tasks.md +++ b/docs/architecture/tape-layering/tasks.md @@ -43,10 +43,10 @@ ## Architecture Enforcement -- [ ] Add domain dependency-direction tests. -- [ ] Add a physical Tape table access guard with a narrow explicit allowlist. -- [ ] Run the Tape contract and scale suites. -- [ ] Review and commit the architecture-guard slice. +- [x] Add domain dependency-direction tests. +- [x] Add a physical Tape table access guard with a narrow explicit allowlist. +- [x] Run the Tape contract and scale suites. +- [x] Review and commit the architecture-guard slice. ## Documentation and Final Validation diff --git a/test/main/tape/layerBoundaries.test.ts b/test/main/tape/layerBoundaries.test.ts new file mode 100644 index 0000000000..7fc313f107 --- /dev/null +++ b/test/main/tape/layerBoundaries.test.ts @@ -0,0 +1,154 @@ +import path from 'node:path' +import ts from 'typescript' +import { describe, expect, it, vi } from 'vitest' + +const MAIN_SOURCE_ROOT = path.resolve(process.cwd(), 'src/main') +const TAPE_DOMAIN_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape/domain') +const TAPE_SQLITE_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape/infrastructure/sqlite') +const TAPE_SQLITE_RELATIVE_ROOT = 'tape/infrastructure/sqlite/' +const TYPESCRIPT_SOURCE_EXTENSION = /\.[cm]?tsx?$/ + +const PHYSICAL_TAPE_STORAGE_PATTERN = + /\b(?:deepchat_tape_(?:entries|search_(?:projection(?:_meta)?|fts(?:_meta)?))|DeepChatTape(?:Entries|SearchProjection)Table|deepchatTape(?:Entries|SearchProjection)(?:Table)?)\b/ + +interface StorageBoundaryException { + physicalName?: string + sqliteImport?: string +} + +const ALLOWED_STORAGE_EXCEPTIONS = new Map([ + ['app/databaseSecurity.ts', { physicalName: 'database table-name security allowlist' }], + [ + 'app/startupMigrations/legacyChatImportService.ts', + { physicalName: 'migration-only full-table replacement and projection cleanup' } + ], + [ + 'data/schemaCatalog.ts', + { + physicalName: 'schema creation and migration registry', + sqliteImport: 'schema adapter construction' + } + ], + [ + 'data/sqliteCopyExclusions.ts', + { physicalName: 'SQLite virtual-table copy exclusion metadata' } + ], + [ + 'memory/data/tables/deepchatMemoryIngestionProjection.ts', + { physicalName: 'read-only single-statement Tape-head consistency check' } + ], + [ + 'session/data/database.ts', + { + physicalName: 'SQLite adapter compatibility getters', + sqliteImport: 'SQLite adapter composition' + } + ], + [ + 'session/data/tables/deepchatTapeEntries.ts', + { sqliteImport: 'legacy import-path compatibility re-export' } + ], + [ + 'session/data/tables/deepchatTapeSearchProjection.ts', + { sqliteImport: 'legacy import-path compatibility re-export' } + ], + ['tape/ports/application.ts', { physicalName: 'legacy database-shape compatibility adapter' }] +]) + +function listTypeScriptSources(root: string, fs: typeof import('node:fs')): string[] { + return fs + .readdirSync(root, { withFileTypes: true }) + .flatMap((entry) => { + const entryPath = path.join(root, entry.name) + if (entry.isDirectory()) return listTypeScriptSources(entryPath, fs) + if (entry.isFile() && TYPESCRIPT_SOURCE_EXTENSION.test(entry.name)) return [entryPath] + return [] + }) + .sort() +} + +function relativeToMain(file: string): string { + return path.relative(MAIN_SOURCE_ROOT, file).split(path.sep).join('/') +} + +function isInside(root: string, target: string): boolean { + const relativePath = path.relative(root, target) + return relativePath === '' || (!relativePath.startsWith('..') && !path.isAbsolute(relativePath)) +} + +function resolveMainImport(importingFile: string, specifier: string): string | null { + if (specifier.startsWith('@/')) { + return path.resolve(MAIN_SOURCE_ROOT, specifier.slice(2)) + } + if (specifier.startsWith('.')) { + return path.resolve(path.dirname(importingFile), specifier) + } + return null +} + +describe('Tape layer boundaries', () => { + it('keeps the Tape domain independent from other main-process layers', async () => { + const fs = await vi.importActual('node:fs') + const violations = listTypeScriptSources(TAPE_DOMAIN_ROOT, fs).flatMap((file) => { + const source = fs.readFileSync(file, 'utf8') + const imports = ts.preProcessFile(source, true, true).importedFiles + + return imports.flatMap(({ fileName: specifier }) => { + const target = resolveMainImport(file, specifier) + if (!target || isInside(TAPE_DOMAIN_ROOT, target)) return [] + return [`${relativeToMain(file)} -> ${specifier}`] + }) + }) + + expect(violations).toEqual([]) + }) + + it('allows physical Tape storage access only at explicit infrastructure boundaries', async () => { + const fs = await vi.importActual('node:fs') + const matchedExceptionCapabilities = new Set() + const violations = listTypeScriptSources(MAIN_SOURCE_ROOT, fs).flatMap((file) => { + const relativeFile = relativeToMain(file) + if (relativeFile.startsWith(TAPE_SQLITE_RELATIVE_ROOT)) return [] + + const source = fs.readFileSync(file, 'utf8') + const physicalName = source.match(PHYSICAL_TAPE_STORAGE_PATTERN)?.[0] + const sqliteImport = ts + .preProcessFile(source, true, true) + .importedFiles.map(({ fileName }) => ({ + fileName, + target: resolveMainImport(file, fileName) + })) + .find(({ target }) => target && isInside(TAPE_SQLITE_ROOT, target))?.fileName + const exception = ALLOWED_STORAGE_EXCEPTIONS.get(relativeFile) + const fileViolations: string[] = [] + + if (physicalName) { + if (exception?.physicalName) { + matchedExceptionCapabilities.add(`${relativeFile}:physicalName`) + } else { + fileViolations.push(`${relativeFile}: physical name ${physicalName}`) + } + } + if (sqliteImport) { + if (exception?.sqliteImport) { + matchedExceptionCapabilities.add(`${relativeFile}:sqliteImport`) + } else { + fileViolations.push(`${relativeFile}: SQLite import ${sqliteImport}`) + } + } + return fileViolations + }) + + const staleExceptions = [...ALLOWED_STORAGE_EXCEPTIONS.entries()].flatMap(([file, exception]) => + (Object.entries(exception) as Array<[keyof StorageBoundaryException, string]>).flatMap( + ([capability, reason]) => + matchedExceptionCapabilities.has(`${file}:${capability}`) + ? [] + : [`${file} (${capability}): ${reason}`] + ) + ) + + expect(violations).toEqual([]) + expect(staleExceptions).toEqual([]) + }) +}) From 42f80e9ee0ebd7d5697042b2f3381bbf5f0c96b5 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 16:36:39 +0800 Subject: [PATCH 07/29] test(tape): align writer mock naming --- .../agent/deepchat/runtime/process.test.ts | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/test/main/agent/deepchat/runtime/process.test.ts b/test/main/agent/deepchat/runtime/process.test.ts index db31730680..75de6fe02b 100644 --- a/test/main/agent/deepchat/runtime/process.test.ts +++ b/test/main/agent/deepchat/runtime/process.test.ts @@ -160,7 +160,7 @@ function makeStreamEvents(...events: LLMCoreStreamEvent[]): LLMCoreStreamEvent[] describe('processStream', () => { let messageStore: ReturnType - let tapeRecorder: { appendToolFact: ReturnType } + let tapeToolFactWriter: { appendToolFact: ReturnType } let tempHome: string | null = null let homedirSpy: ReturnType | null = null @@ -168,7 +168,7 @@ describe('processStream', () => { vi.useFakeTimers() vi.clearAllMocks() messageStore = createMockMessageStore() - tapeRecorder = { + tapeToolFactWriter = { appendToolFact: vi.fn(async (input) => ({ sessionId: input.sessionId, entryId: 1 @@ -241,7 +241,7 @@ describe('processStream', () => { permissionMode: 'full_access', io: { messageStore, - tapeToolFactWriter: tapeRecorder, + tapeToolFactWriter, publishEvent: publishDeepchatEventMock, publishSessionUpdate: vi.fn() }, @@ -273,7 +273,7 @@ describe('processStream', () => { messageStore.setMessageError.mockImplementation(() => { order.push('message:error') }) - tapeRecorder.appendToolFact.mockImplementation(async (input) => { + tapeToolFactWriter.appendToolFact.mockImplementation(async (input) => { order.push(`tape:${input.provenance.source}`) return { sessionId: input.sessionId, entryId: 1 } }) @@ -374,7 +374,7 @@ describe('processStream', () => { 'renderer:update', 'renderer:complete' ]) - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() expect(JSON.parse(messageStore.finalizeAssistantMessage.mock.calls[0][2])).toMatchObject({ provider: 'openai', model: 'gpt-4' @@ -410,7 +410,7 @@ describe('processStream', () => { ]) expect(messageStore.finalizeAssistantMessage).toHaveBeenCalledTimes(1) expect(messageStore.setMessageError).toHaveBeenCalledTimes(1) - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() }) it('persists each tool round before entering the next provider round', async () => { @@ -470,9 +470,9 @@ describe('processStream', () => { 'renderer:update', 'renderer:complete' ]) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(6) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(6) expect( - tapeRecorder.appendToolFact.mock.calls.map(([input]) => [ + tapeToolFactWriter.appendToolFact.mock.calls.map(([input]) => [ input.provenance.source, input.provenance.sourceId, input.provenance.sequence @@ -488,7 +488,7 @@ describe('processStream', () => { }) it('keeps the tool loop fail-open when TapeToolFactWriter rejects a fact', async () => { - tapeRecorder.appendToolFact.mockRejectedValue(new Error('tape unavailable')) + tapeToolFactWriter.appendToolFact.mockRejectedValue(new Error('tape unavailable')) const coreStream = createToolThenCompleteStream('action') const result = await processStream( @@ -501,7 +501,7 @@ describe('processStream', () => { expect(result.status).toBe('completed') expect(coreStream).toHaveBeenCalledTimes(2) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(1) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(1) }) it('persists a paused tool round before its terminal projection', async () => { @@ -553,7 +553,7 @@ describe('processStream', () => { 'renderer:update', 'renderer:complete' ]) - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() }) it('accounts for executed tools before pausing a mixed tool batch', async () => { @@ -705,7 +705,7 @@ describe('processStream', () => { ]) expect(messageStore.setMessageError).toHaveBeenCalled() expect(messageStore.finalizeAssistantMessage).not.toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() expectDeepchatEvent('chat.stream.failed', { sessionId: 's1', messageId: 'm1', @@ -738,7 +738,7 @@ describe('processStream', () => { const abortMetadata = JSON.parse(messageStore.setMessageError.mock.calls[0][2]) expect(abortMetadata).toMatchObject({ provider: 'openai', model: 'gpt-4' }) expect(messageStore.finalizeAssistantMessage).not.toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() expectDeepchatEvent('chat.stream.failed', { sessionId: 's1', messageId: 'm1', @@ -767,7 +767,7 @@ describe('processStream', () => { }) expect(order).toEqual([...TOOL_ROUND_COMMIT_ORDER, ...ERROR_TERMINAL_COMMIT_ORDER]) expect(coreStream).toHaveBeenCalledTimes(1) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) it('does not snapshot an oversized tool batch that never executes', async () => { @@ -807,7 +807,7 @@ describe('processStream', () => { ...COMPLETED_TERMINAL_COMMIT_ORDER ]) expect(toolService.callTool).not.toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() }) it('persists a terminal tool-output error before the failed projection', async () => { @@ -831,7 +831,7 @@ describe('processStream', () => { expect(order).toEqual([...TOOL_ROUND_COMMIT_ORDER, ...ERROR_TERMINAL_COMMIT_ORDER]) expect(coreStream).toHaveBeenCalledTimes(1) expect(messageStore.setMessageError).toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) it('settles a post-stream abort without a tool Tape snapshot', async () => { @@ -848,7 +848,7 @@ describe('processStream', () => { expect(result.status).toBe('aborted') expect(order).toEqual(['renderer:update', 'message:update', ...ERROR_TERMINAL_COMMIT_ORDER]) - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() }) it('persists the completed tool batch before a post-tool abort', async () => { @@ -878,7 +878,7 @@ describe('processStream', () => { expect(toolService.callTool).toHaveBeenCalledTimes(1) expect(messageStore.setMessageError).toHaveBeenCalled() expect(messageStore.finalizeAssistantMessage).not.toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) it('keeps a tool-local AbortError as a tool failure while the run remains active', async () => { @@ -929,7 +929,7 @@ describe('processStream', () => { }) ]) ) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) it('persists the completed batch before settling for pending input', async () => { @@ -953,7 +953,7 @@ describe('processStream', () => { }) expect(order).toEqual([...TOOL_ROUND_COMMIT_ORDER, ...COMPLETED_TERMINAL_COMMIT_ORDER]) expect(coreStream).toHaveBeenCalledTimes(1) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) }) From 4e69cef1db64fe5ce2aff87491945b45edc2ca3e Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 16:37:44 +0800 Subject: [PATCH 08/29] docs(tape): refresh architecture map --- docs/FLOWS.md | 4 +- docs/architecture/memory-system.md | 18 ++++++- docs/architecture/session-management.md | 18 ++++++- docs/architecture/tape-layering/tasks.md | 12 ++--- docs/architecture/tape-system.md | 69 ++++++++++++++++++++---- 5 files changed, 101 insertions(+), 20 deletions(-) diff --git a/docs/FLOWS.md b/docs/FLOWS.md index 7552be6279..7c5de88cfb 100644 --- a/docs/FLOWS.md +++ b/docs/FLOWS.md @@ -81,8 +81,8 @@ Provider、Tool、Skill、Memory 和 Session data 都通过创建时传入的必 不会伪造新的 outer round。 - Memory prompt contribution 必须等待结果、清理内容、限制大小并允许失败；terminal extraction 在后台 执行，并保持 epoch、cursor 和 fence 约束。 -- `TapeRecorder.appendToolFact` 在 message projection 完成后写 terminal tool call/result；写入失败不影响 - 当前回复完成。 +- `TapeToolFactWriter.appendToolFact` 在 message projection 完成后写 terminal tool call/result； + 写入失败不影响当前回复完成。 ## 4. ACP 执行 diff --git a/docs/architecture/memory-system.md b/docs/architecture/memory-system.md index 65e3ef16cc..a31fccb23e 100644 --- a/docs/architecture/memory-system.md +++ b/docs/architecture/memory-system.md @@ -23,6 +23,9 @@ flowchart LR epoch、cursor 和 fence。 - Session 保存 Memory cursor/settings，不拥有 Memory row 或 vector store。 - App 负责 shutdown/database maintenance 时的全局 fence 和停止顺序，不解释 Memory 业务状态。 +- Memory runtime 通过 `TapeRawEntryReader` 和 `TapeAnchorWriter` 读取执行事实、记录 + `memory/view_assembled` 与 `memory/extract` anchor；Memory routes 只通过 `TapeInspectionReader` + 检查 manifest，不接收 Tape table。 ## 数据与状态 @@ -61,6 +64,8 @@ Vector store v2 使用 `.v2.duckdb`、plain `FLOAT[]` table 和 exact s ```text terminal turn projection + -> read bounded ingestion projection range + -> rebuild from effective Tape or fall back when projection is stale/unavailable -> collect bounded text chunks -> extraction / reflection -> normalize candidates @@ -76,6 +81,16 @@ terminal turn projection - stale result、partial batch 和 provider cancellation 有明确 terminal outcome； - vector store 异常进入 typed error/quarantine，不得把消息发送永久挂起。 +## Tape 与 ingestion projection 边界 + +`DeepChatMemoryIngestionProjectionTable.readCurrentRange` 在一条只读 SQL 中同时观察 Tape head 和 +projection head。这是明确的基础设施例外：拆成两次查询会让并发 append 产生 false-current 窗口。 +除此之外 Memory 不得直接读取物理 Tape 表。 + +projection current 时只 materialize cursor 区间；head 不一致时，runtime 通过 `TapeRawEntryReader` +构建 effective Tape view 并重建 projection。projection 查询或重建失败时保留既有 Tape fallback 和 +cursor commit 保护，不能因为拆层新增全历史 hot-path 查询，也不能在不完整 projection 上推进 cursor。 + ## Privacy 与隔离 - 所有查询显式携带 Agent identity；不得依赖进程全局“当前 Agent”。 @@ -101,7 +116,8 @@ metric 名称、retrieval evaluation 和 artifact upload 的未完成工作保 5. `src/main/memory/infra/vectorStoreManager.ts` 6. `src/main/memory/infra/memoryVectorStore.ts` 7. `src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts` -8. `test/main/memory/` +8. `src/main/tape/ports/capabilities.ts` +9. `test/main/memory/` Memory tests 必须防止旧 `src/main/presenter/memoryPresenter`、HNSW hot path、 无 Agent namespace 查询和无 deadline provider call 回流。 diff --git a/docs/architecture/session-management.md b/docs/architecture/session-management.md index fff32610bb..4ec3a411ec 100644 --- a/docs/architecture/session-management.md +++ b/docs/architecture/session-management.md @@ -13,7 +13,8 @@ Session 是可长期保存的产品对象；window、renderer、Remote endpoint | list、restore、status、projection query | `src/main/session/query.ts` | | full delete transaction | `src/main/session/deletion.ts` | | transcript | `src/main/session/data/transcript.ts` | -| Tape / ViewManifest | `src/main/session/data/tape*.ts` | +| Tape public port / composition | `src/main/session/data/index.ts` | +| Tape domain / application / SQLite adapters | `src/main/tape/` | | generation settings / Memory cursor | `src/main/session/data/settings.ts` | | pending input | `src/main/session/data/pendingInputs.ts` | | renderer binding | `src/main/desktop/sessionBinding.ts` | @@ -54,6 +55,19 @@ send input；Remote、Scheduler 和 renderer 不得各自维护不同的默认 - history search 优先使用 search document / FTS path，失败时回退受控 SQL search；坐标和 scroll 归 renderer viewport owner，不写回 Session。 +## Tape boundary + +Session data composition 创建一个 `SessionTape`，对外继续暴露现有 `SessionTapePort`，并按每个 IPC +操作原有的条件和时序调用 `ensureSessionTapeReady`。`src/main/session/data/tape*.ts` 和旧 table path +只是 compatibility re-export，不再拥有 Tape policy 或 persistence。 + +- transcript 只接收 `TapeMessageFactWriter`；message replace/retract 与对应 Tape fact 继续共享调用方 + SQLite transaction； +- settings/compaction 只接收 anchor reader/writer 与 lifecycle admin；summary 更新和 anchor append + 继续使用同一个 connection； +- Agent、Memory 和 routes 分别接收自己的最小 Tape capability，不接收物理 table； +- Tape 的运行中修订通过 append 表达；物理 delete/reset 只由 Session lifecycle 触发。 + ## Binding `DesktopSessionBinding` 维护 `webContentsId -> sessionId`： @@ -70,7 +84,7 @@ send input；Remote、Scheduler 和 renderer 不得各自维护不同的默认 ```text cleanup both backend caches without hydration -> remove ACP durable binding - -> remove transcript / Tape / pending / settings + -> remove transcript / Tape lifecycle data / pending / settings -> clear permission and active Skill state -> delete app-session row ``` diff --git a/docs/architecture/tape-layering/tasks.md b/docs/architecture/tape-layering/tasks.md index 7dba2eae47..c4675eaac0 100644 --- a/docs/architecture/tape-layering/tasks.md +++ b/docs/architecture/tape-layering/tasks.md @@ -50,9 +50,9 @@ ## Documentation and Final Validation -- [ ] Update Tape, Session, and Memory architecture references. -- [ ] Run the full main-process test suite and Memory performance suite. -- [ ] Run full type checks, formatting, i18n validation, and lint. -- [ ] Review the complete `dev...HEAD` diff and fix every finding. -- [ ] Complete the task checklist and commit the final documentation slice. -- [ ] Confirm the working tree is clean and the branch has not been pushed. +- [x] Update Tape, Session, and Memory architecture references. +- [x] Run the full main-process test suite and Memory performance suite. +- [x] Run full type checks, formatting, i18n validation, and lint. +- [x] Review the complete `dev...HEAD` diff and fix every finding. +- [x] Complete the task checklist and commit the final documentation slice. +- [x] Confirm the working tree is clean and the branch has not been pushed. diff --git a/docs/architecture/tape-system.md b/docs/architecture/tape-system.md index 745d9d30f6..f9b7da106e 100644 --- a/docs/architecture/tape-system.md +++ b/docs/architecture/tape-system.md @@ -3,19 +3,63 @@ Tape 是 Session 同寿命的 append-only execution fact log。它保存可回放事实、anchor、ViewManifest 和 Subagent lineage；message transcript 是面向 UI 的 projection，不是 Tape 的替代品。 -## 所有权和数据 +## 所有权和分层 | 能力 | 当前 owner | | --- | --- | -| append/query/replay | `src/main/session/data/tape.ts` | -| effective view | `src/main/session/data/tapeEffectiveView.ts` | -| facts | `src/main/session/data/tapeFacts.ts` | -| ViewManifest storage | `src/main/session/data/tapeViewManifest.ts` | +| entry/fact/ref、effective semantics、ViewManifest 纯逻辑 | `src/main/tape/domain/` | +| 消费方能力和 storage ports | `src/main/tape/ports/` | +| Fact、Reconciler、Recall、Lineage、View/Replay、Fork services | `src/main/tape/application/` | +| `SessionTape` 兼容 facade | `src/main/tape/application/sessionTape.ts` | +| append/read/query store | `src/main/tape/infrastructure/sqlite/tapeEntryStore.ts` | +| search projection | `src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts` | +| 物理 lifecycle delete | `src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts` | | runtime assembly | `src/main/agent/deepchat/runtime/tapeViewAssembler.ts` | | policy selection | `src/main/agent/deepchat/runtime/tapeViewPolicy.ts` | | model-facing tools | `src/main/tool/agentTools/agentTapeTools.ts` | Tape entry 只能 append。更正、压缩和 handoff 通过新 fact/anchor 表达，不原地改写旧 entry。 +anchor 改变后续读取起点或重建状态，但不删除被覆盖的历史。 + +```mermaid +flowchart TD + Consumers["Agent / Transcript / Memory / Settings / IPC"] --> Ports["Tape capability ports"] + Ports --> Facade["SessionTape compatibility facade"] + Facade --> Services["Six application services"] + Services --> Stores["Entry store / Search projection / Lifecycle adapter"] + Stores --> SQLite["Shared Session SQLite connection"] +``` + +`src/main/session/data/tape*.ts` 和旧 table modules 只保留 compatibility re-export。新代码必须从 +`src/main/tape/` 或能力 port 导入，不能把兼容路径重新当作 owner。 + +## 能力端口和组合 + +| 消费方 | 允许依赖的 Tape 能力 | +| --- | --- | +| Agent loop | `TapeToolFactWriter` | +| Transcript | `TapeMessageFactWriter` | +| Memory runtime | `TapeRawEntryReader`、`TapeAnchorWriter` | +| Settings / compaction | `TapeAnchorReader`、`TapeAnchorWriter`、`TapeLifecycleAdmin` | +| Memory routes | `TapeInspectionReader` | +| IPC / Session data | 现有 `SessionTapePort` | + +`createSessionDataFromDatabase` 组合一个 `SessionTape`，把窄能力传给 transcript 和 settings，并在既有 +IPC boundary 按原时序执行 `ensureSessionTapeReady`。facade 只做 service 组合和兼容转发，不承载新的 +domain policy；外部方法的签名、同步/异步行为、异常和 fallback 语义保持稳定。 + +## 存储与事务边界 + +- `TapeEntryStore` 只负责 append/read/query；物理删除由独立 lifecycle adapter 执行，只服务于 + Session lifecycle（包含 fork Session cleanup），不属于运行中 Tape 语义。 +- transcript message mutation 与 replacement/retraction fact、summary compare-and-set 与 anchor append + 使用同一个 SQLite connection 和调用方 transaction，拆层不能拆开其原子边界。 +- search projection 可以重建；projection 不可用或 coverage 不完整时回退 effective Tape search，fork + cleanup 的 projection 删除失败仍不阻断主流程。 +- legacy chat import 的全表删除是 migration-only 例外；Memory ingestion projection 为避免并发窗口， + 可以在一条只读 SQL 中同时比较 Tape head 和 projection head。除此之外消费方不得访问物理 Tape 表。 +- reset 物理删除当前 Session Tape 后重新 bootstrap；本阶段没有 archive-on-reset，不能把 reset 解释成 + append-only 运行语义的一部分。 ## View 和 provenance @@ -54,20 +98,27 @@ read boundary 兼容，新写入必须使用 canonical policy/provenance。 `tape_info`、`tape_anchors` 是 diagnostic；`tape_handoff` 是 runtime-only。五个名称全部 reserved， MCP 不能 shadow，持久化 disabled-tool 配置也不能关闭 system capability。 -## Subagent lineage +## Fork 和 Subagent lineage Subagent 使用独立 Session 和独立 Tape。完成后父 Session append 一个 link，固定 child Tape head： - 查询时只读该 frozen head，不自动读取 child 后续 entry； - child entries 不复制进父 Tape； -- discard/merge 只改变父 Session 的 lineage fact，不篡改 child 历史； +- 只有显式授权的直接 child 可以跨 Tape 读取；missing、recreated 或 incarnation 不匹配必须 fail + closed； - 非直接 child、未授权 Session 或递归 Subagent 不能通过 Tape tool 越权读取。 +普通 fork merge 只把 fork head 相对基线的 delta 作为新 entry append 到父 Tape，并追加 merge receipt； +不得改写父 Tape 旧 entry，也不得把整份 fork 历史重复复制。discard 和重复 merge 保持既有审计、幂等及 +best-effort projection cleanup 语义。 + ## 回放和兼容 Replay 必须保持 entry order、role、tool call/result pairing、anchor cursor 和 policy version。未知旧 fact 可以按兼容规则跳过或映射，但不能静默改变已知 fact 的含义。测试至少覆盖正常 chat、resume、tool interaction、compaction、context pressure、Subagent frozen head 和旧 manifest 读取。 -关键测试位于 `test/main/session/`、`test/main/agent/deepchat/` 和 `test/main/tool/`。历史的 Tape -increment SDD 已合并到本文，详细实施顺序从 Git 历史查询。 +关键行为测试位于 `test/main/session/data/tape*.test.ts`，分层守护位于 +`test/main/tape/layerBoundaries.test.ts`；runtime 和 tool 契约继续位于 +`test/main/agent/deepchat/` 与 `test/main/tool/`。历史的 Tape increment SDD 已合并到本文，详细实施 +顺序从 Git 历史查询。 From 1d71e9cc60b8d08fd931fd96d9bc66bb1b957cb0 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 17:03:13 +0800 Subject: [PATCH 09/29] fix(tape): make generation resets atomic --- src/main/tape/application/forkService.ts | 72 +++++---- .../tape/application/generationLifecycle.ts | 29 ++++ src/main/tape/application/recallService.ts | 4 +- src/main/tape/application/sessionTape.ts | 4 +- .../sqlite/tapeSearchProjectionStore.ts | 44 ++++-- src/main/tape/ports/application.ts | 6 + .../deepChatRuntimeCoordinator.test.ts | 4 +- test/main/session/data/tapeFork.test.ts | 38 ++++- test/main/session/data/tapeLifecycle.test.ts | 147 +++++++++++++++++- test/main/session/data/tapeLineage.test.ts | 7 +- test/main/session/data/tapeRecall.test.ts | 127 +++++++++++++++ test/main/session/data/tapeTestHarness.ts | 3 +- test/main/session/runtimeIntegration.test.ts | 4 +- 13 files changed, 436 insertions(+), 53 deletions(-) create mode 100644 src/main/tape/application/generationLifecycle.ts diff --git a/src/main/tape/application/forkService.ts b/src/main/tape/application/forkService.ts index 6d9e4f81d8..9831eb5410 100644 --- a/src/main/tape/application/forkService.ts +++ b/src/main/tape/application/forkService.ts @@ -4,6 +4,7 @@ import type { AgentTapeHandoffState, ChatMessageRecord } from '@shared/types/age import type { DeepChatTapeEntryRow } from '../domain/entry' import type { TapeApplicationProviders } from '../ports/application' import { appendMessageRecordToTape } from './factPersistence' +import { deleteTapeGeneration } from './generationLifecycle' import { parseJsonObject } from './common' import type { TapeAnchorResult, TapeForkHandle } from './contracts' import type { TapeFactService } from './factService' @@ -168,6 +169,10 @@ function forkSessionId(parentSessionId: string, forkId: string): string { return `${parentSessionId}::fork::${forkId}` } +function forkDiscardProvenanceKey(parentSessionId: string, forkId: string): string { + return `fork:${parentSessionId}:${forkId}:discard:event` +} + export class TapeForkService { constructor( private readonly providers: TapeForkProviders, @@ -178,14 +183,6 @@ export class TapeForkService { return this.providers.getEntryStore() } - private get lifecycle() { - return this.providers.getEntryLifecycleStore() - } - - private get searchProjectionTable() { - return this.providers.getSearchProjectionStore() - } - private toAnchorResult(row: DeepChatTapeEntryRow): TapeAnchorResult { return { sessionId: row.session_id, @@ -239,6 +236,14 @@ export class TapeForkService { createFork(parentSessionId: string, forkId: string = nanoid()): TapeForkHandle { const table = this.table const forkIdValue = forkId.trim() || nanoid() + if ( + table.getByProvenanceKey( + parentSessionId, + forkDiscardProvenanceKey(parentSessionId, forkIdValue) + ) + ) { + throw new Error(`Fork ${forkIdValue} has been discarded and cannot be reused.`) + } const forkSessionIdValue = forkSessionId(parentSessionId, forkIdValue) const parentHeadEntryId = table.getMaxEntryId(parentSessionId) table.ensureBootstrapAnchor(forkSessionIdValue) @@ -308,6 +313,12 @@ export class TapeForkService { ) } + if ( + table.getByProvenanceKey(parentSessionId, forkDiscardProvenanceKey(parentSessionId, forkId)) + ) { + throw new Error(`Fork ${forkId} does not exist or has been discarded.`) + } + assertValidForkStart( table.getByProvenanceKey(forkSessionIdValue, `fork:${parentSessionId}:${forkId}:start`), parentSessionId, @@ -374,27 +385,32 @@ export class TapeForkService { discardFork(parentSessionId: string, forkId: string): void { const table = this.table const forkSessionIdValue = forkSessionId(parentSessionId, forkId) - this.lifecycle.deleteBySession(forkSessionIdValue) - try { - this.searchProjectionTable.deleteBySession(forkSessionIdValue) - } catch (error) { - logger.warn(`[Tape] failed to delete fork search projection: ${String(error)}`) - } - table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/discard', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:discard:event`, - data: { - forkId, - forkSessionId: forkSessionIdValue - }, - idempotent: true + let cleanupError: unknown + table.runInTransaction(() => { + try { + deleteTapeGeneration(this.providers, forkSessionIdValue) + } catch (error) { + cleanupError = error + } + table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/discard', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: forkDiscardProvenanceKey(parentSessionId, forkId), + data: { + forkId, + forkSessionId: forkSessionIdValue + }, + idempotent: true + }) }) + if (cleanupError) { + logger.warn(`[Tape] failed to delete fork Tape generation: ${String(cleanupError)}`) + } } recordExternalForkMerge( diff --git a/src/main/tape/application/generationLifecycle.ts b/src/main/tape/application/generationLifecycle.ts new file mode 100644 index 0000000000..54f4dac3f0 --- /dev/null +++ b/src/main/tape/application/generationLifecycle.ts @@ -0,0 +1,29 @@ +import type { TapeApplicationProviders } from '../ports/application' + +type TapeGenerationLifecycleProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getEntryLifecycleStore' | 'getSearchProjectionStore' +> + +export function deleteTapeGeneration( + providers: TapeGenerationLifecycleProviders, + sessionId: string +): void { + const table = providers.getEntryStore() + table.runInTransaction(() => { + providers.getEntryLifecycleStore().deleteBySession(sessionId) + providers.getSearchProjectionStore().deleteBySession(sessionId) + }) +} + +export function resetTapeGeneration( + providers: TapeGenerationLifecycleProviders, + sessionId: string +): void { + const table = providers.getEntryStore() + table.runInTransaction(() => { + providers.getEntryLifecycleStore().deleteBySession(sessionId) + providers.getSearchProjectionStore().deleteBySession(sessionId) + table.ensureBootstrapAnchor(sessionId) + }) +} diff --git a/src/main/tape/application/recallService.ts b/src/main/tape/application/recallService.ts index 37190c0fbe..a7aa11512d 100644 --- a/src/main/tape/application/recallService.ts +++ b/src/main/tape/application/recallService.ts @@ -233,10 +233,12 @@ export class TapeRecallService { .map((index) => effectiveRows[index]) let projectionRows = new Map() try { + const maxEntryId = rows.reduce((max, row) => Math.max(max, row.entry_id), 0) projectionRows = new Map( this.searchProjectionTable - .getByEntryIds( + .getByEntryIdsIfCurrent( sessionId, + maxEntryId, selectedRows.map((row) => row.entry_id) ) .map((row) => [row.entry_id, row]) diff --git a/src/main/tape/application/sessionTape.ts b/src/main/tape/application/sessionTape.ts index 4ed5c669b2..7688557a2c 100644 --- a/src/main/tape/application/sessionTape.ts +++ b/src/main/tape/application/sessionTape.ts @@ -45,6 +45,7 @@ import type { } from './contracts' import { TapeFactService } from './factService' import { normalizeTapeHandoffState, TapeForkService } from './forkService' +import { resetTapeGeneration } from './generationLifecycle' import { AgentTapeViewError, normalizeSubagentTapeLinkInput, @@ -278,7 +279,6 @@ export class SessionTape } resetSessionTape(sessionId: string): void { - this.deleteSessionTape(sessionId) - this.initializeSessionTape(sessionId) + resetTapeGeneration(this.providers, sessionId) } } diff --git a/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts index 9ff20d26ef..0c84a325ab 100644 --- a/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts +++ b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts @@ -331,6 +331,29 @@ export class DeepChatTapeSearchProjectionTable .all(sessionId, ...ids) as DeepChatTapeSearchProjectionRow[] } + getByEntryIdsIfCurrent( + sessionId: string, + maxEntryId: number, + entryIds: number[], + projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ): DeepChatTapeSearchProjectionRow[] { + const ids = [...new Set(entryIds.filter((id) => Number.isInteger(id) && id > 0))] + if (!ids.length) return [] + const placeholders = ids.map(() => '?').join(', ') + return this.db + .prepare( + `SELECT projection.* + FROM deepchat_tape_search_projection AS projection + INNER JOIN deepchat_tape_search_projection_meta AS meta + ON meta.session_id = projection.session_id + AND meta.projection_version = ? + AND meta.max_entry_id = ? + WHERE projection.session_id = ? AND projection.entry_id IN (${placeholders}) + ORDER BY projection.entry_id ASC` + ) + .all(projectionVersion, maxEntryId, sessionId, ...ids) as DeepChatTapeSearchProjectionRow[] + } + searchSourcesReadOnly( sources: readonly DeepChatTapeReadSource[], query: string, @@ -406,13 +429,15 @@ export class DeepChatTapeSearchProjectionTable } deleteBySession(sessionId: string): void { - this.db - .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') - .run(sessionId) - this.db - .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') - .run(sessionId) - this.deleteSessionFts(sessionId) + this.db.transaction(() => { + this.db + .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') + .run(sessionId) + this.db + .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') + .run(sessionId) + this.deleteSessionFts(sessionId, true) + })() } clearAll(): void { @@ -656,7 +681,7 @@ export class DeepChatTapeSearchProjectionTable this.ftsReady = false } - private deleteSessionFts(sessionId: string): void { + private deleteSessionFts(sessionId: string, strict = false): void { if (this.ftsMetaTableExists()) { this.db .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') @@ -665,8 +690,9 @@ export class DeepChatTapeSearchProjectionTable if (!this.ftsTableExists()) return try { this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) - } catch { + } catch (error) { this.ftsReady = false + if (strict) throw error } } diff --git a/src/main/tape/ports/application.ts b/src/main/tape/ports/application.ts index cdf7b2d598..9c2db87387 100644 --- a/src/main/tape/ports/application.ts +++ b/src/main/tape/ports/application.ts @@ -70,6 +70,12 @@ export interface TapeSearchProjectionStore { projectionVersion?: number ): void getByEntryIds(sessionId: string, entryIds: number[]): TapeSearchProjectionRow[] + getByEntryIdsIfCurrent( + sessionId: string, + maxEntryId: number, + entryIds: number[], + projectionVersion?: number + ): TapeSearchProjectionRow[] searchSourcesReadOnly( sources: readonly DeepChatTapeReadSource[], query: string, diff --git a/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts b/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts index 2520e905f2..f72295de49 100644 --- a/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts +++ b/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts @@ -322,6 +322,7 @@ function createMockSqlitePresenter() { delete: vi.fn() }, deepchatTapeEntriesTable: (deepchatTapeEntriesTable = { + runInTransaction: vi.fn((operation: () => unknown) => operation()), ensureBootstrapAnchor: vi.fn(), append: vi.fn((input: any) => { const provenanceKey = @@ -447,7 +448,8 @@ function createMockSqlitePresenter() { deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn(), isCurrent: vi.fn().mockReturnValue(false), - getByEntryIds: vi.fn().mockReturnValue([]) + getByEntryIds: vi.fn().mockReturnValue([]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) }, deepchatMemoryIngestionProjectionTable: { readCurrentRange: vi.fn( diff --git a/test/main/session/data/tapeFork.test.ts b/test/main/session/data/tapeFork.test.ts index 6d30b000ad..eee8f43ee9 100644 --- a/test/main/session/data/tapeFork.test.ts +++ b/test/main/session/data/tapeFork.test.ts @@ -356,7 +356,7 @@ describe('SessionTape forks', () => { } }) - it('cleans fork search projection on discard without blocking the discard event', () => { + it('keeps a failed fork cleanup isolated and makes its discard receipt fail closed', () => { const { table, entries } = createTapeTableMock() const projectionTable = { deleteBySession: vi.fn(() => { @@ -376,9 +376,43 @@ describe('SessionTape forks', () => { expect(table.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) expect(projectionTable.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) - expect(entries.some((entry) => entry.session_id === fork.forkSessionId)).toBe(false) + expect(entries.some((entry) => entry.session_id === fork.forkSessionId)).toBe(true) expect( entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') ).toBe(true) + expect(() => service.mergeFork('s1', 'fork-cleanup')).toThrow( + 'Fork fork-cleanup does not exist or has been discarded.' + ) + expect(() => service.createFork('s1', 'fork-cleanup')).toThrow( + 'Fork fork-cleanup has been discarded and cannot be reused.' + ) + }) + + it('restores fork entries when the discard receipt cannot be appended', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn() }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('s1', 'receipt-cleanup') + service.appendForkMessageRecord( + fork, + createRecord({ id: 'receipt-message', sessionId: 'ignored' }) + ) + const appendEvent = table.appendEvent.getMockImplementation()! + table.appendEvent.mockImplementation((input: any) => { + if (input.sessionId === 's1' && input.name === 'fork/discard') { + throw new Error('discard receipt failed') + } + return appendEvent(input) + }) + + expect(() => service.discardFork('s1', 'receipt-cleanup')).toThrow('discard receipt failed') + expect(entries.some((entry) => entry.session_id === fork.forkSessionId)).toBe(true) + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') + ).toBe(false) }) }) diff --git a/test/main/session/data/tapeLifecycle.test.ts b/test/main/session/data/tapeLifecycle.test.ts index 421d2ea7e1..a2c768a4b6 100644 --- a/test/main/session/data/tapeLifecycle.test.ts +++ b/test/main/session/data/tapeLifecycle.test.ts @@ -1,21 +1,45 @@ -import { describe, expect, it, vi } from 'vitest' -import { SessionTape } from '@/session/data/tape' +import { + DatabaseCtor, + DeepChatTapeEntriesTable, + DeepChatTapeSearchProjectionTable, + describe, + expect, + it, + itIfSqlite, + SessionTape, + SqliteTapeLifecycleAdapter, + vi +} from './tapeTestHarness' function createHarness() { const calls: string[] = [] + const state = { entriesPresent: true, searchPresent: true, bootstrapCount: 0 } const entryStore = { + runInTransaction: vi.fn((operation: () => unknown) => { + const snapshot = { ...state } + try { + return operation() + } catch (error) { + Object.assign(state, snapshot) + throw error + } + }), ensureBootstrapAnchor: vi.fn((sessionId: string) => { calls.push(`bootstrap:${sessionId}`) + state.entriesPresent = true + state.bootstrapCount += 1 }) } const lifecycle = { deleteBySession: vi.fn((sessionId: string) => { calls.push(`entries:${sessionId}`) + state.entriesPresent = false }) } const searchProjection = { deleteBySession: vi.fn((sessionId: string) => { calls.push(`search:${sessionId}`) + state.searchPresent = false }) } const tape = new SessionTape({ @@ -24,7 +48,7 @@ function createHarness() { deepchatTapeSearchProjectionTable: searchProjection } as any) - return { calls, entryStore, lifecycle, searchProjection, tape } + return { calls, entryStore, lifecycle, searchProjection, state, tape } } describe('SessionTape lifecycle administration', () => { @@ -37,20 +61,133 @@ describe('SessionTape lifecycle administration', () => { }) it('rebuilds the bootstrap only after both destructive stores are cleared', () => { - const { calls, tape } = createHarness() + const { calls, entryStore, tape } = createHarness() tape.resetSessionTape('s1') expect(calls).toEqual(['entries:s1', 'search:s1', 'bootstrap:s1']) + expect(entryStore.runInTransaction).toHaveBeenCalledOnce() }) it('does not create a mixed-generation Tape when projection cleanup fails', () => { - const { entryStore, searchProjection, tape } = createHarness() + const { entryStore, searchProjection, state, tape } = createHarness() searchProjection.deleteBySession.mockImplementationOnce(() => { throw new Error('search cleanup failed') }) expect(() => tape.resetSessionTape('s1')).toThrow('search cleanup failed') expect(entryStore.ensureBootstrapAnchor).not.toHaveBeenCalled() + expect(state).toEqual({ entriesPresent: true, searchPresent: true, bootstrapCount: 0 }) + }) + + itIfSqlite('rolls back a failed reset and creates a fresh incarnation only on retry', () => { + const db = new DatabaseCtor(':memory:') + try { + const entryStore = new DeepChatTapeEntriesTable(db) + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + entryStore.createTable() + searchProjection.createTable() + entryStore.ensureBootstrapAnchor('s1') + const oldBootstrap = entryStore.getBySession('s1')[0] + entryStore.appendEvent({ + sessionId: 's1', + name: 'old/generation', + data: { marker: 'old generation' } + }) + searchProjection.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'event', + name: 'old/generation', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'old generation', + summaryText: 'old generation', + refs: { generation: 'old' }, + createdAt: 100 + } + ], + 2 + ) + const deleteProjection = searchProjection.deleteBySession.bind(searchProjection) + const deleteProjectionSpy = vi + .spyOn(searchProjection, 'deleteBySession') + .mockImplementationOnce(() => { + throw new Error('search cleanup failed') + }) + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: searchProjection + } as any) + + expect(() => tape.resetSessionTape('s1')).toThrow('search cleanup failed') + expect(entryStore.getBySession('s1').map((entry) => entry.name)).toEqual([ + 'session/start', + 'old/generation' + ]) + expect(entryStore.getBySession('s1')[0].meta_json).toBe(oldBootstrap.meta_json) + expect(searchProjection.isCurrent('s1', 2)).toBe(true) + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([2]) + + deleteProjectionSpy.mockImplementation(deleteProjection) + tape.resetSessionTape('s1') + + const newEntries = entryStore.getBySession('s1') + expect(newEntries).toHaveLength(1) + expect(newEntries[0]).toMatchObject({ entry_id: 1, name: 'session/start' }) + expect(newEntries[0].meta_json).not.toBe(oldBootstrap.meta_json) + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([]) + expect(searchProjection.getSessionMeta('s1')).toBeNull() + } finally { + db.close() + } + }) + + itIfSqlite('rolls back partial search projection deletion', () => { + const db = new DatabaseCtor(':memory:') + try { + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + searchProjection.createTable() + searchProjection.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'event', + name: 'old/generation', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'old generation', + summaryText: 'old generation', + refs: { generation: 'old' }, + createdAt: 100 + } + ], + 1 + ) + db.exec(` + CREATE TRIGGER fail_projection_meta_delete + BEFORE DELETE ON deepchat_tape_search_projection_meta + WHEN old.session_id = 's1' + BEGIN + SELECT RAISE(ABORT, 'injected projection cleanup failure'); + END; + `) + + expect(() => searchProjection.deleteBySession('s1')).toThrow( + 'injected projection cleanup failure' + ) + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([1]) + expect(searchProjection.isCurrent('s1', 1)).toBe(true) + } finally { + db.close() + } }) }) diff --git a/test/main/session/data/tapeLineage.test.ts b/test/main/session/data/tapeLineage.test.ts index 19dbfe2d3e..7b35c9b66a 100644 --- a/test/main/session/data/tapeLineage.test.ts +++ b/test/main/session/data/tapeLineage.test.ts @@ -359,7 +359,8 @@ describe('SessionTape lineage', () => { })), replaceSession: vi.fn(), appendSession: vi.fn(), - getByEntryIds: vi.fn().mockReturnValue([]) + getByEntryIds: vi.fn().mockReturnValue([]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) } const { service } = createLinkedTapeService( table, @@ -439,7 +440,7 @@ describe('SessionTape lineage', () => { it('expands linked context within one source and never crosses the frozen head', () => { const { table } = createTapeTableMock() const projectionTable = { - getByEntryIds: vi.fn(() => [ + getByEntryIdsIfCurrent: vi.fn(() => [ { session_id: 'child', entry_id: 2, @@ -506,7 +507,7 @@ describe('SessionTape lineage', () => { [2], expect.objectContaining({ before: 0, after: 5 }) ) - expect(projectionTable.getByEntryIds).not.toHaveBeenCalled() + expect(projectionTable.getByEntryIdsIfCurrent).not.toHaveBeenCalled() expect(table.append).not.toHaveBeenCalled() }) diff --git a/test/main/session/data/tapeRecall.test.ts b/test/main/session/data/tapeRecall.test.ts index 247b8cfdcd..2c428b8280 100644 --- a/test/main/session/data/tapeRecall.test.ts +++ b/test/main/session/data/tapeRecall.test.ts @@ -458,6 +458,113 @@ describe('SessionTape recall', () => { expect(context.entries[0].evidence.text).toContain('target-ok') }) + it('falls back to the current Tape when the context projection is not current', () => { + const { table } = createTapeTableMock() + const entry = table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'current-message', seq: 0 }, + payload: { + record: createRecord({ + id: 'current-message', + content: JSON.stringify({ text: 'current generation context', files: [], links: [] }) + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' } + }) + const projectionTable = { + getByEntryIds: vi.fn(() => [ + { + session_id: 's1', + entry_id: entry.entry_id, + summary_text: 'old private context', + refs_json: '{"messageId":"old-message"}' + } + ]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const context = service.getContext('s1', [entry.entry_id], { before: 0, after: 0 }) + + expect(projectionTable.getByEntryIdsIfCurrent).toHaveBeenCalledWith('s1', entry.entry_id, [ + entry.entry_id + ]) + expect(projectionTable.getByEntryIds).not.toHaveBeenCalled() + expect(context.entries[0]).toMatchObject({ + summary: expect.stringContaining('current generation context'), + refs: { messageId: 'current-message' } + }) + expect(context.entries[0].summary).not.toContain('old private context') + }) + + itIfSqlite('ignores stale same-entry projection context when the Tape head has advanced', () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + table.createTable() + projectionTable.createTable() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'current-message', seq: 0 }, + payload: { + record: createRecord({ + id: 'current-message', + content: JSON.stringify({ + text: 'current generation context', + files: [], + links: [] + }) + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' } + }) + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'old-message', + sourceSeq: 0, + searchText: 'old private context', + summaryText: 'old private context', + refs: { messageId: 'old-message' }, + createdAt: 100 + } + ], + 0 + ) + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const context = service.getContext('s1', [1], { before: 0, after: 0 }) + + expect(context.entries[0]).toMatchObject({ + entryId: 1, + summary: expect.stringContaining('current generation context'), + refs: { messageId: 'current-message' } + }) + expect(context.entries[0].summary).not.toContain('old private context') + } finally { + db.close() + } + }) + it('binds tape projection LIKE fallback params for single and multi-term queries', () => { const all = vi.fn().mockReturnValue([]) const db = { @@ -502,6 +609,26 @@ describe('SessionTape recall', () => { ) }) + it('checks projection version and head in the same context row query', () => { + const reads: Array<{ sql: string; params: unknown[] }> = [] + const projectionTable = new DeepChatTapeSearchProjectionTable({ + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => { + reads.push({ sql, params }) + return [] + } + })) + } as any) + + projectionTable.getByEntryIdsIfCurrent('s1', 7, [3, 3, 0], 42) + + expect(reads).toHaveLength(1) + expect(reads[0].sql).toContain('INNER JOIN deepchat_tape_search_projection_meta AS meta') + expect(reads[0].sql).toContain('meta.projection_version = ?') + expect(reads[0].sql).toContain('meta.max_entry_id = ?') + expect(reads[0].params).toEqual([42, 7, 's1', 3]) + }) + it('uses current tape projection without loading full session rows', () => { const { table } = createTapeTableMock() table.append({ diff --git a/test/main/session/data/tapeTestHarness.ts b/test/main/session/data/tapeTestHarness.ts index 5128b968b6..155b013a70 100644 --- a/test/main/session/data/tapeTestHarness.ts +++ b/test/main/session/data/tapeTestHarness.ts @@ -415,7 +415,8 @@ function createTapeService( tapeLifecycle: table, deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn(), - getByEntryIds: vi.fn().mockReturnValue([]) + getByEntryIds: vi.fn().mockReturnValue([]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) }, deepchatMessageTracesTable: { listByMessageId: vi.fn((messageId: string) => diff --git a/test/main/session/runtimeIntegration.test.ts b/test/main/session/runtimeIntegration.test.ts index a9df4ec780..9d67a3b37f 100644 --- a/test/main/session/runtimeIntegration.test.ts +++ b/test/main/session/runtimeIntegration.test.ts @@ -74,6 +74,7 @@ function createMockSqlitePresenter() { let messagesList: any[] = [] const tapeTable = { + runInTransaction: vi.fn((operation: () => unknown) => operation()), ensureBootstrapAnchor: vi.fn(), append: vi.fn((input: any) => { const row = { @@ -642,7 +643,8 @@ function createMockSqlitePresenter() { deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn(), isCurrent: vi.fn().mockReturnValue(false), - getByEntryIds: vi.fn().mockReturnValue([]) + getByEntryIds: vi.fn().mockReturnValue([]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) }, // Expose internal stores for assertion _sessionsStore: sessionsStore, From 292c2beb79c57f625ac1edf833cc1ca69a65819f Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 17:14:11 +0800 Subject: [PATCH 10/29] refactor(tape): narrow consumer ports --- src/main/agent/acp/compatibility/adapters.ts | 10 +- .../agent/acp/compatibility/dependencies.ts | 6 +- .../deepchat/runtime/deepChatLoopRunner.ts | 26 ++- .../runtime/deepChatRuntimeCoordinator.ts | 9 +- src/main/agent/deepchat/runtime/process.ts | 2 +- .../agent/deepchat/runtime/turnCoordinator.ts | 19 +- src/main/app/composition.ts | 2 +- src/main/memory/routes.ts | 64 +----- src/main/session/data/index.ts | 2 +- src/main/session/data/settings.ts | 2 +- src/main/session/data/transcript.ts | 2 +- src/main/tape/application/contracts.ts | 28 +-- src/main/tape/application/recallService.ts | 21 ++ .../tape/application/reconcilerService.ts | 6 +- src/main/tape/application/sessionTape.ts | 34 +-- .../tape/application/viewReplayService.ts | 57 +++++ src/main/tape/ports/capabilities.ts | 72 ++++++- .../agent/acp/compatibility/adapters.test.ts | 2 +- test/main/routes/dispatcher.test.ts | 201 +++++++++--------- test/main/session/data/tapeRecall.test.ts | 40 ++++ test/main/session/data/tapeViewReplay.test.ts | 64 ++++++ 21 files changed, 435 insertions(+), 234 deletions(-) diff --git a/src/main/agent/acp/compatibility/adapters.ts b/src/main/agent/acp/compatibility/adapters.ts index 3bfc85a76d..fb1b5c1956 100644 --- a/src/main/agent/acp/compatibility/adapters.ts +++ b/src/main/agent/acp/compatibility/adapters.ts @@ -24,8 +24,8 @@ import { type IoParams, type StreamState } from '@/agent/deepchat/runtime/types' -import { SessionTranscript } from '@/session/data/transcript' -import { SessionTape } from '@/session/data/tape' +import type { SessionTranscript } from '@/session/data/transcript' +import type { TapeReconciliationPort } from '@/tape/ports/capabilities' import { buildPersistableMessageTracePayload } from '@/agent/deepchat/runtime/messageTracePayload' interface ProjectionState { @@ -36,7 +36,7 @@ interface ProjectionState { export interface AcpCompatibilityProjectionAdapterOptions { messageStore: SessionTranscript - tapeService: SessionTape + tapeReconciliation: TapeReconciliationPort writeViewManifest: (input: AcpViewManifestInput) => void | Promise setStatus: (status: 'generating' | 'idle' | 'error') => void publishEvent: DeepChatEventPublisher @@ -56,8 +56,8 @@ export class AcpCompatibilityProjectionAdapter implements AcpCompatibilityProjec sessionId: AcpViewManifestInput['sessionId'] userContent: Parameters[2] }): AcpProjectionHandle { - const { messageStore, tapeService } = this.options - tapeService.ensureSessionTapeReady(input.sessionId, messageStore) + const { messageStore, tapeReconciliation } = this.options + tapeReconciliation.ensureSessionTapeReady(input.sessionId, messageStore) const userMessageId = messageStore.createUserMessage( input.sessionId, messageStore.getNextOrderSeq(input.sessionId), diff --git a/src/main/agent/acp/compatibility/dependencies.ts b/src/main/agent/acp/compatibility/dependencies.ts index 32b6b1908d..42755c8e51 100644 --- a/src/main/agent/acp/compatibility/dependencies.ts +++ b/src/main/agent/acp/compatibility/dependencies.ts @@ -17,7 +17,7 @@ import { createUserChatMessage } from '@/agent/deepchat/runtime/contextBuilder' import { resolveEffectiveActiveSkillNames } from '@/agent/deepchat/resources/systemPromptBuilder' import type { SessionSettingsStore } from '@/session/data/settings' import type { SessionTranscript } from '@/session/data/transcript' -import type { SessionTape } from '@/session/data/tape' +import type { TapeReconciliationPort } from '@/tape/ports/capabilities' import type { DeepChatToolResolver } from '@/agent/deepchat/runtime/toolResolver' import type { DeepChatEventPublisher, @@ -34,7 +34,7 @@ export interface AcpCompatibilityDependencyBuilderDependencies { providerRuntime: Pick sessionStore: SessionSettingsStore messageStore: SessionTranscript - tapeService: SessionTape + tapeReconciliation: TapeReconciliationPort toolResolver: DeepChatToolResolver appendViewManifest(input: AcpViewManifestInput): void setStatus(sessionId: string, status: DeepChatSessionState['status']): void @@ -84,7 +84,7 @@ export function createAcpCompatibilityDependencies( publishEvent: dependencies.publishEvent, publishSessionUpdate: dependencies.publishSessionUpdate, messageStore: dependencies.messageStore, - tapeService: dependencies.tapeService, + tapeReconciliation: dependencies.tapeReconciliation, writeViewManifest: async (manifest) => dependencies.appendViewManifest(manifest), setStatus: (status) => dependencies.setStatus(sessionId, status) }) diff --git a/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts b/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts index 9be85d5f23..7f1773be95 100644 --- a/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts +++ b/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts @@ -67,7 +67,12 @@ import { resolveTapeViewManifestPolicy, type TapeViewContextSelection } from '@/tape/domain/viewManifest' -import type { SessionTape } from '@/session/data/tape' +import type { + TapeReconciliationPort, + TapeToolFactWriter, + TapeViewManifestReader, + TapeViewManifestWriter +} from '@/tape/ports/capabilities' import type { AgentTraceSettingsPort } from '@/agent/traceSettings' import type { DeepChatToolResolver } from '@/agent/deepchat/runtime/toolResolver' import type { @@ -179,7 +184,10 @@ export interface DeepChatLoopRunnerPorts { traceSettings: AgentTraceSettingsPort sessionStore: SessionSettingsStore messageStore: SessionTranscript - tapeService: SessionTape + tapeReconciliation: TapeReconciliationPort + tapeViewManifestReader: TapeViewManifestReader + tapeViewManifestWriter: TapeViewManifestWriter + tapeToolFactWriter: TapeToolFactWriter pendingInputCoordinator: SessionPendingInputs toolResolver: DeepChatToolResolver providerPermissionCoordinator: ProviderPermissionCoordinator @@ -391,7 +399,8 @@ export class DeepChatLoopRunner { const traceEnabled = this.ports.traceSettings.isEnabled() const initialRequestSeq = Math.max( - this.ports.tapeService.listViewManifestsByMessage(sessionId, messageId)[0]?.requestSeq ?? 0, + this.ports.tapeViewManifestReader.listViewManifestsByMessage(sessionId, messageId)[0] + ?.requestSeq ?? 0, this.ports.messageStore.getMaxMessageTraceRequestSeq(messageId) ) const temperature = generationSettings.temperature @@ -754,7 +763,7 @@ export class DeepChatLoopRunner { }, io: { messageStore: this.ports.messageStore, - tapeToolFactWriter: this.ports.tapeService, + tapeToolFactWriter: this.ports.tapeToolFactWriter, publishEvent: this.ports.publishEvent, publishSessionUpdate: this.ports.publishSessionUpdate } @@ -770,7 +779,7 @@ export class DeepChatLoopRunner { } appendTapeViewManifest(params: AppendTapeViewManifestInput): void { - const sourceMaps = this.ports.tapeService.getViewManifestSourceMaps( + const sourceMaps = this.ports.tapeViewManifestReader.getViewManifestSourceMaps( params.sessionId, params.messageId ) @@ -799,7 +808,7 @@ export class DeepChatLoopRunner { supportsAudioInput: params.supportsAudioInput, traceDebugEnabled: params.traceDebugEnabled }) - this.ports.tapeService.appendViewManifest(manifest) + this.ports.tapeViewManifestWriter.appendViewManifest(manifest) } emitRateLimitWaitingMessage( @@ -893,7 +902,10 @@ export class DeepChatLoopRunner { prepareCompaction: async (systemPrompt) => { const prepared = await this.ports.inputPreparationCoordinator.prepareExisting({ ensureHistory: () => - this.ports.tapeService.ensureSessionTapeReady(params.sessionId, this.ports.messageStore) + this.ports.tapeReconciliation.ensureSessionTapeReady( + params.sessionId, + this.ports.messageStore + ) .historyRecords, prepareIntent: async (historyRecords) => await this.ports.compactionService.prepareForContextPressureRecovery({ diff --git a/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts b/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts index 3091c269e7..8c1b365a6f 100644 --- a/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts +++ b/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts @@ -410,7 +410,10 @@ export class DeepChatRuntimeCoordinator { traceSettings: this.traceSettings, sessionStore: this.sessionStore, messageStore: this.messageStore, - tapeService: this.tapeService, + tapeReconciliation: this.tapeService, + tapeViewManifestReader: this.tapeService, + tapeViewManifestWriter: this.tapeService, + tapeToolFactWriter: this.tapeService, pendingInputCoordinator: this.pendingInputCoordinator, toolResolver: this.toolResolver, providerPermissionCoordinator: this.providerPermissionCoordinator, @@ -457,7 +460,7 @@ export class DeepChatRuntimeCoordinator { toolService: this.toolService, sessionStore: this.sessionStore, messageStore: this.messageStore, - tapeService: this.tapeService, + tapeReconciliation: this.tapeService, pendingInputCoordinator: this.pendingInputCoordinator, toolResolver: this.toolResolver, compactionService: this.compactionService, @@ -575,7 +578,7 @@ export class DeepChatRuntimeCoordinator { providerRuntime: this.providerRuntime, sessionStore: this.sessionStore, messageStore: this.messageStore, - tapeService: this.tapeService, + tapeReconciliation: this.tapeService, toolResolver: this.toolResolver, appendViewManifest: (manifest) => { this.loopRunner.appendTapeViewManifest({ diff --git a/src/main/agent/deepchat/runtime/process.ts b/src/main/agent/deepchat/runtime/process.ts index e9ee0e67ef..b75acefbc0 100644 --- a/src/main/agent/deepchat/runtime/process.ts +++ b/src/main/agent/deepchat/runtime/process.ts @@ -31,7 +31,7 @@ import { } from '@/agent/deepchat/loop/deepChatLoopEngine' import { emitDeepChatLoopNotification } from '@/agent/deepchat/loop/notificationObserver' import type { OutputSink } from '@/agent/deepchat/loop/ports' -import { buildTapeToolFactInputs } from '@/session/data/tapeFacts' +import { buildTapeToolFactInputs } from '@/tape/application/factPersistence' const UNKNOWN_CONTEXT_LIMIT = Number.MAX_SAFE_INTEGER const USER_CANCELED_GENERATION_ERROR = 'common.error.userCanceledGeneration' diff --git a/src/main/agent/deepchat/runtime/turnCoordinator.ts b/src/main/agent/deepchat/runtime/turnCoordinator.ts index f16490e4e5..48eb29e9b9 100644 --- a/src/main/agent/deepchat/runtime/turnCoordinator.ts +++ b/src/main/agent/deepchat/runtime/turnCoordinator.ts @@ -41,7 +41,7 @@ import { buildTerminalErrorBlocks, type SessionTranscript } from '@/session/data import type { DeepChatEventPublisher, ProcessResult } from './types' import { buildUsageFromMetadata, stampTerminalMetadata } from './runtimeMetadata' import type { SessionSettingsStore } from '@/session/data/settings' -import type { SessionTape } from '@/session/data/tape' +import type { TapeReconciliationPort } from '@/tape/ports/capabilities' import { getTapeContextHistoryRecords, buildTapeChatView, @@ -96,7 +96,7 @@ export interface TurnCoordinatorPorts { toolService: Pick sessionStore: SessionSettingsStore messageStore: SessionTranscript - tapeService: SessionTape + tapeReconciliation: TapeReconciliationPort pendingInputCoordinator: SessionPendingInputs toolResolver: DeepChatToolResolver compactionService: CompactionService @@ -352,7 +352,10 @@ export class TurnCoordinator { ensureHistory: () => this.ports.runSynchronousPreStreamStep(sessionId, 'tape-ready', () => getTapeContextHistoryRecords( - this.ports.tapeService.ensureSessionTapeReady(sessionId, this.ports.messageStore) + this.ports.tapeReconciliation.ensureSessionTapeReady( + sessionId, + this.ports.messageStore + ) .historyRecords ) ), @@ -840,7 +843,10 @@ export class TurnCoordinator { sessionId, 'tape-ready', () => - this.ports.tapeService.ensureSessionTapeReady(sessionId, this.ports.messageStore) + this.ports.tapeReconciliation.ensureSessionTapeReady( + sessionId, + this.ports.messageStore + ) .historyRecords ), refreshHistory: () => @@ -848,7 +854,10 @@ export class TurnCoordinator { sessionId, 'tape-ready', () => - this.ports.tapeService.ensureSessionTapeReady(sessionId, this.ports.messageStore) + this.ports.tapeReconciliation.ensureSessionTapeReady( + sessionId, + this.ports.messageStore + ) .historyRecords ), prepareIntent: async (historyRecords) => { diff --git a/src/main/app/composition.ts b/src/main/app/composition.ts index 95ea0adbc2..f20d5362eb 100644 --- a/src/main/app/composition.ts +++ b/src/main/app/composition.ts @@ -1608,7 +1608,7 @@ export async function createMainProcessControl(dependencies: { const memoryRoutes = createMemoryRoutes({ memoryService, getAgentType: (agentId) => agentSettings.getAgentType(agentId), - getTapeEntries: () => sessionData.tapeStore, + getTapeInspection: () => sessionData.tapeStore, getAuditEntries: () => memoryDatabase.agentMemoryAuditTable }) const desktopRoutes = createDesktopRoutes({ diff --git a/src/main/memory/routes.ts b/src/main/memory/routes.ts index 7c7fd8b381..c9f8f5b4d9 100644 --- a/src/main/memory/routes.ts +++ b/src/main/memory/routes.ts @@ -40,8 +40,6 @@ import { type MemoryUpdateResult } from '@shared/contracts/routes/memory.routes' import { createRouteMap, type DeepchatRouteMap } from '@/routes/routeRegistry' -import { buildEffectiveTapeView } from '@/tape/domain/effectiveView' -import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' import type { TapeInspectionReader } from '@/tape/ports/capabilities' import type { MemoryConflictPair, @@ -59,7 +57,9 @@ const MEMORY_PERSONA_STATES = ['draft', 'active', 'superseded', 'rejected'] as c type MemoryPersonaState = (typeof MEMORY_PERSONA_STATES)[number] const MEMORY_PERSONA_STATE_SET: ReadonlySet = new Set(MEMORY_PERSONA_STATES) -export function formatMemorySourceRecordContent(record: ChatMessageRecord): string { +export function formatMemorySourceRecordContent( + record: Pick +): string { try { const parsed = JSON.parse(record.content) as unknown if (record.role === 'user') { @@ -182,49 +182,6 @@ function toMemoryAuditEventDto(row: AgentMemoryAuditRow) { } } -function deriveSelectedMemoryIds(value: unknown): string[] | null { - if (!Array.isArray(value)) return null - const ids = new Set() - for (const item of value) { - const id = - typeof item === 'string' - ? item - : item && typeof item === 'object' && !Array.isArray(item) - ? (item as Record).id - : null - if (typeof id === 'string' && id.length > 0) ids.add(id) - } - return [...ids] -} - -function toMemoryViewManifestDto(row: DeepChatTapeEntryRow) { - const payload = parseJsonRecord(row.payload_json) - const meta = parseJsonRecord(row.meta_json) - const manifest = - payload.state && typeof payload.state === 'object' && !Array.isArray(payload.state) - ? (payload.state as Record) - : null - if (!manifest) return null - const readNumber = (value: unknown): number => - typeof value === 'number' && Number.isFinite(value) ? value : 0 - return { - sessionId: row.session_id, - messageId: typeof meta.messageId === 'string' ? meta.messageId : null, - entryId: row.entry_id, - policyVersion: - typeof manifest.policyVersion === 'number' && Number.isFinite(manifest.policyVersion) - ? manifest.policyVersion - : null, - tokenBudget: readNumber(manifest.tokenBudget), - estimatedTokens: readNumber(manifest.estimatedTokens), - selectedCount: Array.isArray(manifest.selected) ? manifest.selected.length : 0, - selectedIds: deriveSelectedMemoryIds(manifest.selected), - droppedCount: Array.isArray(manifest.dropped) ? manifest.dropped.length : 0, - queryHash: typeof manifest.queryHash === 'string' ? manifest.queryHash : null, - createdAt: row.created_at - } -} - type MemoryAuditEntries = { listByAgent(agentId: string, options: MemoryAuditListOptions): AgentMemoryAuditRow[] } @@ -286,7 +243,7 @@ interface MemoryRouteService { export function createMemoryRoutes(deps: { memoryService: MemoryRouteService getAgentType(agentId: string): Promise - getTapeEntries(): TapeInspectionReader + getTapeInspection(): TapeInspectionReader getAuditEntries(): MemoryAuditEntries }): DeepchatRouteMap { const { memoryService } = deps @@ -295,9 +252,9 @@ export function createMemoryRoutes(deps: { if (!row || row.agent_id !== agentId || !row.source_session) return null const sourceEntryIds = parseAgentMemorySourceEntryIds(row.source_entry_ids) if (!sourceEntryIds?.length) return null - const sourceSet = new Set(sourceEntryIds) - const entries = buildEffectiveTapeView(deps.getTapeEntries().getBySession(row.source_session)) - .messageEntries.filter((entry) => sourceSet.has(entry.entryId)) + const entries = deps + .getTapeInspection() + .getEffectiveMessageSourceSpan(row.source_session, sourceEntryIds) .map((entry) => ({ entryId: entry.entryId, role: entry.record.role, @@ -489,15 +446,12 @@ export function createMemoryRoutes(deps: { } const limit = input.limit ?? 100 const manifests = deps - .getTapeEntries() - .listMemoryViewManifestAnchorsByAgent(input.agentId, { + .getTapeInspection() + .listMemoryViewManifestsByAgent(input.agentId, { sessionId: input.sessionId, limit, messageId: input.messageId }) - .map(toMemoryViewManifestDto) - .filter((manifest): manifest is NonNullable => Boolean(manifest)) - .filter((manifest) => !input.messageId || manifest.messageId === input.messageId) .slice(0, limit) return memoryListViewManifestsRoute.output.parse({ manifests }) } diff --git a/src/main/session/data/index.ts b/src/main/session/data/index.ts index e1d48032a8..a64f168ae5 100644 --- a/src/main/session/data/index.ts +++ b/src/main/session/data/index.ts @@ -5,7 +5,7 @@ import type { SessionTapePort } from './contracts' import { SessionPendingInputStore } from './pendingInputStore' import { SessionPendingInputs } from './pendingInputs' import { SessionSettingsStore } from './settings' -import { normalizeTapeHandoffState, SessionTape } from './tape' +import { normalizeTapeHandoffState, SessionTape } from '@/tape/application/sessionTape' import { SessionTranscript } from './transcript' import { SessionDatabase } from './database' diff --git a/src/main/session/data/settings.ts b/src/main/session/data/settings.ts index 27a2faeba2..289be81b0f 100644 --- a/src/main/session/data/settings.ts +++ b/src/main/session/data/settings.ts @@ -7,7 +7,7 @@ import type { TapeAnchorWriter, TapeLifecycleAdmin } from '@/tape/ports/capabilities' -import { SessionTape } from './tape' +import { SessionTape } from '@/tape/application/sessionTape' export type SessionSummaryState = { summaryText: string | null diff --git a/src/main/session/data/transcript.ts b/src/main/session/data/transcript.ts index 6d0f05b46d..4e24893495 100644 --- a/src/main/session/data/transcript.ts +++ b/src/main/session/data/transcript.ts @@ -24,7 +24,7 @@ import { resolveUsageProviderId } from '@/session/usageStats' import type { TapeMessageFactWriter } from '@/tape/ports/capabilities' -import { SessionTape } from './tape' +import { SessionTape } from '@/tape/application/sessionTape' function shouldConvertPendingBlockToError( status: AssistantMessageBlock['status'] diff --git a/src/main/tape/application/contracts.ts b/src/main/tape/application/contracts.ts index 504fba28ba..465d96f909 100644 --- a/src/main/tape/application/contracts.ts +++ b/src/main/tape/application/contracts.ts @@ -1,15 +1,11 @@ -import type { AgentTapeAnchorResult, ChatMessageRecord } from '@shared/types/agent-interface' +import type { AgentTapeAnchorResult } from '@shared/types/agent-interface' +import type { TapeMigrationState } from '../ports/capabilities' -export type TapeMigrationState = 'none' | 'ready' - -export type TapeBackfillResult = { - sessionId: string - migrationState: TapeMigrationState - messageCount: number - maxOrderSeq: number - appendedFactCount: number - historyRecords: ChatMessageRecord[] -} +export type { + TapeBackfillResult, + TapeMigrationState, + TapeViewManifestSourceMaps +} from '../ports/capabilities' export type TapeInfo = { sessionId: string @@ -41,13 +37,3 @@ export type TapeForkHandle = { forkSessionId: string parentHeadEntryId: number } - -export type TapeViewManifestSourceMaps = { - latestEntryId: number - anchorEntryIds: number[] - reconstructionAnchorEntryIds: number[] - reconstructionAnchorEntryId: number | null - entryIdByMessageId: Map - toolCallEntryIdByToolId: Map - toolResultEntryIdByToolId: Map -} diff --git a/src/main/tape/application/recallService.ts b/src/main/tape/application/recallService.ts index a7aa11512d..93a4e74f0f 100644 --- a/src/main/tape/application/recallService.ts +++ b/src/main/tape/application/recallService.ts @@ -19,6 +19,7 @@ import type { TapeSearchProjectionRow as DeepChatTapeSearchProjectionRow, TapeSearchProjectionStore } from '../ports/application' +import type { TapeEffectiveMessageSourceEntry } from '../ports/capabilities' import { isEntryIdPrefix, migrationProvenanceKey, parseJsonObject } from './common' import type { TapeAnchorResult, TapeInfo, TapeSearchResult } from './contracts' import { AgentTapeViewError, type TapeLineageService } from './lineageService' @@ -83,6 +84,26 @@ export class TapeRecallService { } } + getEffectiveMessageSourceSpan( + sessionId: string, + entryIds: number[] + ): TapeEffectiveMessageSourceEntry[] { + const requestedEntryIds = new Set( + entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0) + ) + if (requestedEntryIds.size === 0) return [] + return buildEffectiveTapeView(this.table.getBySession(sessionId), { includePending: false }) + .messageEntries.filter((entry) => requestedEntryIds.has(entry.entryId)) + .map((entry) => ({ + entryId: entry.entryId, + record: { + role: entry.record.role, + content: entry.record.content, + orderSeq: entry.record.orderSeq + } + })) + } + search(sessionId: string, query: string, options?: AgentTapeSearchOptions): TapeSearchResult[] { const scope = normalizeTapeViewScope(options?.scope) if (!query.trim()) { diff --git a/src/main/tape/application/reconcilerService.ts b/src/main/tape/application/reconcilerService.ts index ee1db5c6b7..575d4bac4b 100644 --- a/src/main/tape/application/reconcilerService.ts +++ b/src/main/tape/application/reconcilerService.ts @@ -1,7 +1,7 @@ import type { ChatMessageRecord } from '@shared/types/agent-interface' import type { TapeApplicationProviders } from '../ports/application' +import type { TapeBackfillResult, TapeTranscriptReader } from '../ports/capabilities' import { appendMessageRecordToTape } from './factPersistence' -import type { TapeBackfillResult } from './contracts' import type { TapeFactService } from './factService' import { migrationProvenanceKey } from './common' @@ -10,10 +10,6 @@ type TapeReconcilerProviders = Pick< 'getEntryStore' | 'getLegacySummaryReader' > -export interface TapeTranscriptReader { - getMessages(sessionId: string): ChatMessageRecord[] -} - function legacySummaryProvenanceKey(sessionId: string): string { return `summary:${sessionId}:legacy-summary:v1` } diff --git a/src/main/tape/application/sessionTape.ts b/src/main/tape/application/sessionTape.ts index 7688557a2c..b1589bc71b 100644 --- a/src/main/tape/application/sessionTape.ts +++ b/src/main/tape/application/sessionTape.ts @@ -23,11 +23,17 @@ import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' import type { TapeAnchorReader, TapeAnchorWriter, + TapeEffectiveMessageSourceEntry, TapeInspectionReader, TapeLifecycleAdmin, TapeMessageFactWriter, TapeRawEntryReader, - TapeToolFactWriter + TapeReconciliationPort, + TapeToolFactWriter, + TapeTranscriptReader, + TapeMemoryViewManifestInspection, + TapeViewManifestReader, + TapeViewManifestWriter } from '../ports/capabilities' import { createTapeApplicationProviders, @@ -53,7 +59,7 @@ import { type AgentTapeViewErrorCode } from './lineageService' import { TapeRecallService } from './recallService' -import { TapeReconcilerService, type TapeTranscriptReader } from './reconcilerService' +import { TapeReconcilerService } from './reconcilerService' import { TapeViewReplayService } from './viewReplayService' export type { @@ -73,6 +79,9 @@ export class SessionTape TapeToolFactWriter, TapeMessageFactWriter, TapeRawEntryReader, + TapeReconciliationPort, + TapeViewManifestReader, + TapeViewManifestWriter, TapeAnchorReader, TapeAnchorWriter, TapeInspectionReader, @@ -234,14 +243,6 @@ export class SessionTape return this.providers.getEntryStore().getBySession(sessionId) } - getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] { - return this.providers.getEntryStore().getBySessionUpToEntryId(sessionId, maxEntryId) - } - - getMaxEntryId(sessionId: string): number { - return this.providers.getEntryStore().getMaxEntryId(sessionId) - } - getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { return this.providers.getEntryStore().getLatestAnchor(sessionId) } @@ -262,11 +263,18 @@ export class SessionTape return this.providers.getEntryStore().appendAnchor(input) } - listMemoryViewManifestAnchorsByAgent( + getEffectiveMessageSourceSpan( + sessionId: string, + entryIds: number[] + ): TapeEffectiveMessageSourceEntry[] { + return this.recall.getEffectiveMessageSourceSpan(sessionId, entryIds) + } + + listMemoryViewManifestsByAgent( agentId: string, options?: { sessionId?: string; limit?: number; messageId?: string } - ): DeepChatTapeEntryRow[] { - return this.providers.getEntryStore().listMemoryViewManifestAnchorsByAgent(agentId, options) + ): TapeMemoryViewManifestInspection[] { + return this.viewReplay.listMemoryViewManifestsByAgent(agentId, options) } initializeSessionTape(sessionId: string): void { diff --git a/src/main/tape/application/viewReplayService.ts b/src/main/tape/application/viewReplayService.ts index 08660720d0..b8b637968a 100644 --- a/src/main/tape/application/viewReplayService.ts +++ b/src/main/tape/application/viewReplayService.ts @@ -23,6 +23,7 @@ import type { TapeApplicationProviders, TapeMessageTraceRow as DeepChatMessageTraceRow } from '../ports/application' +import type { TapeMemoryViewManifestInspection } from '../ports/capabilities' import { collectEntryIds, hashString, @@ -74,6 +75,51 @@ function readToolFactMessageId(row: DeepChatTapeEntryRow): string | null { return typeof messageId === 'string' && messageId.length > 0 ? messageId : null } +function deriveSelectedMemoryIds(value: unknown): string[] | null { + if (!Array.isArray(value)) return null + const ids = new Set() + for (const item of value) { + const id = + typeof item === 'string' + ? item + : item && typeof item === 'object' && !Array.isArray(item) + ? (item as Record).id + : null + if (typeof id === 'string' && id.length > 0) ids.add(id) + } + return [...ids] +} + +function toMemoryViewManifestInspection( + row: DeepChatTapeEntryRow +): TapeMemoryViewManifestInspection | null { + const payload = parseJsonObject(row.payload_json) + const meta = parseJsonObject(row.meta_json) + const manifest = + payload.state && typeof payload.state === 'object' && !Array.isArray(payload.state) + ? (payload.state as Record) + : null + if (!manifest) return null + const readNumber = (value: unknown): number => + typeof value === 'number' && Number.isFinite(value) ? value : 0 + return { + sessionId: row.session_id, + messageId: typeof meta.messageId === 'string' ? meta.messageId : null, + entryId: row.entry_id, + policyVersion: + typeof manifest.policyVersion === 'number' && Number.isFinite(manifest.policyVersion) + ? manifest.policyVersion + : null, + tokenBudget: readNumber(manifest.tokenBudget), + estimatedTokens: readNumber(manifest.estimatedTokens), + selectedCount: Array.isArray(manifest.selected) ? manifest.selected.length : 0, + selectedIds: deriveSelectedMemoryIds(manifest.selected), + droppedCount: Array.isArray(manifest.dropped) ? manifest.dropped.length : 0, + queryHash: typeof manifest.queryHash === 'string' ? manifest.queryHash : null, + createdAt: row.created_at + } +} + const VIEW_POLICIES = new Set([ 'legacy_context_v1', 'legacy_context_shadow', @@ -300,6 +346,17 @@ export class TapeViewReplayService { } } + listMemoryViewManifestsByAgent( + agentId: string, + options?: { sessionId?: string; limit?: number; messageId?: string } + ): TapeMemoryViewManifestInspection[] { + return this.table + .listMemoryViewManifestAnchorsByAgent(agentId, options) + .map(toMemoryViewManifestInspection) + .filter((manifest): manifest is TapeMemoryViewManifestInspection => manifest !== null) + .filter((manifest) => !options?.messageId || manifest.messageId === options.messageId) + } + appendViewManifest(manifest: DeepChatTapeViewManifest): DeepChatTapeEntryRow { const table = this.table table.ensureBootstrapAnchor(manifest.sessionId) diff --git a/src/main/tape/ports/capabilities.ts b/src/main/tape/ports/capabilities.ts index ca7b5fc6ce..2e8f511063 100644 --- a/src/main/tape/ports/capabilities.ts +++ b/src/main/tape/ports/capabilities.ts @@ -1,7 +1,49 @@ import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { + DeepChatTapeViewManifest, + DeepChatTapeViewManifestRecord +} from '@shared/types/tape-view-manifest' import type { DeepChatTapeEntryRow, TapeAnchorAppendInput } from '../domain/entry' import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' +export type TapeMigrationState = 'none' | 'ready' + +export type TapeBackfillResult = { + sessionId: string + migrationState: TapeMigrationState + messageCount: number + maxOrderSeq: number + appendedFactCount: number + historyRecords: ChatMessageRecord[] +} + +export interface TapeTranscriptReader { + getMessages(sessionId: string): ChatMessageRecord[] +} + +export interface TapeReconciliationPort { + ensureSessionTapeReady(sessionId: string, messageStore: TapeTranscriptReader): TapeBackfillResult +} + +export type TapeViewManifestSourceMaps = { + latestEntryId: number + anchorEntryIds: number[] + reconstructionAnchorEntryIds: number[] + reconstructionAnchorEntryId: number | null + entryIdByMessageId: Map + toolCallEntryIdByToolId: Map + toolResultEntryIdByToolId: Map +} + +export interface TapeViewManifestReader { + getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestSourceMaps + listViewManifestsByMessage(sessionId: string, messageId: string): DeepChatTapeViewManifestRecord[] +} + +export interface TapeViewManifestWriter { + appendViewManifest(manifest: DeepChatTapeViewManifest): void +} + export interface TapeToolFactWriter { appendToolFact(input: TapeToolFactInput): Promise } @@ -14,8 +56,6 @@ export interface TapeMessageFactWriter { export interface TapeRawEntryReader { getBySession(sessionId: string): DeepChatTapeEntryRow[] - getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] - getMaxEntryId(sessionId: string): number } export interface TapeAnchorReader { @@ -30,11 +70,33 @@ export interface TapeAnchorWriter { } export interface TapeInspectionReader { - getBySession(sessionId: string): DeepChatTapeEntryRow[] - listMemoryViewManifestAnchorsByAgent( + getEffectiveMessageSourceSpan( + sessionId: string, + entryIds: number[] + ): TapeEffectiveMessageSourceEntry[] + listMemoryViewManifestsByAgent( agentId: string, options?: { sessionId?: string; limit?: number; messageId?: string } - ): DeepChatTapeEntryRow[] + ): TapeMemoryViewManifestInspection[] +} + +export interface TapeEffectiveMessageSourceEntry { + entryId: number + record: Pick +} + +export interface TapeMemoryViewManifestInspection { + sessionId: string + messageId: string | null + entryId: number + policyVersion: number | null + tokenBudget: number + estimatedTokens: number + selectedCount: number + selectedIds: string[] | null + droppedCount: number + queryHash: string | null + createdAt: number } export interface TapeLifecycleAdmin { diff --git a/test/main/agent/acp/compatibility/adapters.test.ts b/test/main/agent/acp/compatibility/adapters.test.ts index 9388d1a4f5..1d8c164496 100644 --- a/test/main/agent/acp/compatibility/adapters.test.ts +++ b/test/main/agent/acp/compatibility/adapters.test.ts @@ -188,7 +188,7 @@ function createProjectionHarness() { publishEvent: publishDeepchatEvent, publishSessionUpdate: vi.fn(), messageStore, - tapeService, + tapeReconciliation: tapeService, writeViewManifest: vi.fn(), setStatus: vi.fn() }) diff --git a/test/main/routes/dispatcher.test.ts b/test/main/routes/dispatcher.test.ts index a788bcc968..7595b91e85 100644 --- a/test/main/routes/dispatcher.test.ts +++ b/test/main/routes/dispatcher.test.ts @@ -1273,8 +1273,8 @@ function createRuntime() { listByAgent: vi.fn(() => []) }, deepchatTapeEntriesTable: { - getBySession: vi.fn(() => []), - listMemoryViewManifestAnchorsByAgent: vi.fn(() => []) + getEffectiveMessageSourceSpan: vi.fn(() => []), + listMemoryViewManifestsByAgent: vi.fn(() => []) } } as unknown as MainDatabase const cronJob = { @@ -1515,7 +1515,7 @@ function createRuntime() { const memoryRoutes = createMemoryRoutes({ memoryService, getAgentType: (agentId) => providerSettings.getAgentType(agentId), - getTapeEntries: () => (sqlitePresenter as any).deepchatTapeEntriesTable, + getTapeInspection: () => (sqlitePresenter as any).deepchatTapeEntriesTable, getAuditEntries: () => (sqlitePresenter as any).agentMemoryAuditTable }) const desktopRoutes = createDesktopRoutes({ @@ -2383,50 +2383,19 @@ describe('dispatchDeepchatRoute', () => { const { runtime, providerSettings } = createRuntime() vi.mocked(providerSettings.getAgentType).mockResolvedValueOnce('deepchat') const listSessions = vi.fn() - const listMemoryViewManifestAnchorsByAgent = vi.fn().mockReturnValue([ - { - session_id: 's1', - entry_id: 20, - kind: 'anchor', - name: 'memory/view_assembled', - source_type: 'memory', - source_id: 'msg-new', - source_seq: 0, - provenance_key: null, - payload_json: JSON.stringify({ - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 10, - selected: ['new'], - dropped: [], - queryHash: 'newhash' - } - }), - meta_json: JSON.stringify({ messageId: 'msg-new' }), - created_at: 200 - }, - { - session_id: 's1', - entry_id: 10, - kind: 'anchor', - name: 'memory/view_assembled', - source_type: 'memory', - source_id: 'msg-old', - source_seq: 0, - provenance_key: null, - payload_json: JSON.stringify({ - state: { - policyVersion: 1, - tokenBudget: 900, - estimatedTokens: 9, - selected: ['old'], - dropped: ['drop'], - queryHash: 'oldhash' - } - }), - meta_json: JSON.stringify({ messageId: 'msg-old' }), - created_at: 100 + const listMemoryViewManifestsByAgent = vi.fn().mockReturnValue([ + { + sessionId: 's1', + messageId: 'msg-old', + entryId: 10, + policyVersion: 1, + tokenBudget: 900, + estimatedTokens: 9, + selectedCount: 1, + selectedIds: ['old'], + droppedCount: 1, + queryHash: 'oldhash', + createdAt: 100 } ]) ;(runtime as any).sqlitePresenter = { @@ -2434,7 +2403,7 @@ describe('dispatchDeepchatRoute', () => { list: listSessions }, deepchatTapeEntriesTable: { - listMemoryViewManifestAnchorsByAgent + listMemoryViewManifestsByAgent } } @@ -2446,7 +2415,7 @@ describe('dispatchDeepchatRoute', () => { ) expect(listSessions).not.toHaveBeenCalled() - expect(listMemoryViewManifestAnchorsByAgent).toHaveBeenCalledWith('a', { + expect(listMemoryViewManifestsByAgent).toHaveBeenCalledWith('a', { sessionId: 's1', limit: 1, messageId: 'msg-old' @@ -2465,43 +2434,27 @@ describe('dispatchDeepchatRoute', () => { }) }) - it('derives selected memory ids from string and object manifest selections', async () => { + it('returns Tape inspection manifest DTOs without exposing raw rows', async () => { const { runtime, providerSettings } = createRuntime() vi.mocked(providerSettings.getAgentType).mockResolvedValueOnce('deepchat') - const listMemoryViewManifestAnchorsByAgent = vi.fn().mockReturnValue([ - { - session_id: 's1', - entry_id: 30, - kind: 'anchor', - name: 'memory/view_assembled', - source_type: 'memory', - source_id: 'msg-1', - source_seq: 0, - provenance_key: null, - payload_json: JSON.stringify({ - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 10, - selected: [ - 'm-string', - { id: 'm-object' }, - 'm-string', - { id: 'm-object' }, - { nope: 'ignored' }, - 3 - ], - dropped: [], - queryHash: 'hash' - } - }), - meta_json: JSON.stringify({ messageId: 'msg-1' }), - created_at: 300 + const listMemoryViewManifestsByAgent = vi.fn().mockReturnValue([ + { + sessionId: 's1', + messageId: 'msg-1', + entryId: 30, + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: 10, + selectedCount: 6, + selectedIds: ['m-string', 'm-object'], + droppedCount: 0, + queryHash: 'hash', + createdAt: 300 } ]) ;(runtime as any).sqlitePresenter = { deepchatTapeEntriesTable: { - listMemoryViewManifestAnchorsByAgent + listMemoryViewManifestsByAgent } } @@ -2520,6 +2473,51 @@ describe('dispatchDeepchatRoute', () => { }) ] }) + expect(result.manifests[0]).not.toHaveProperty('payload_json') + expect(result.manifests[0]).not.toHaveProperty('meta_json') + }) + + it('reads memory source spans through the DTO-only Tape inspection port', async () => { + const { runtime } = createRuntime() + const getManagementVisibleByIds = vi.fn().mockReturnValue([ + { + id: 'memory-1', + agent_id: 'deepchat', + source_session: 's1', + source_entry_ids: '[2,3]' + } + ]) + const getEffectiveMessageSourceSpan = vi.fn().mockReturnValue([ + { + entryId: 2, + record: { + role: 'user', + orderSeq: 1, + content: JSON.stringify({ text: 'source context' }) + } + } + ]) + ;(runtime as any).memoryService = { getManagementVisibleByIds } + ;(runtime as any).sqlitePresenter = { + deepchatTapeEntriesTable: { + getEffectiveMessageSourceSpan + } + } + + const result = await dispatchDeepchatRoute( + runtime, + 'memory.getSourceSpan', + { agentId: 'deepchat', memoryId: 'memory-1' }, + { webContentsId: 42, windowId: 7 } + ) + + expect(getEffectiveMessageSourceSpan).toHaveBeenCalledWith('s1', [2, 3]) + expect(result).toEqual({ + span: { + sessionId: 's1', + entries: [{ entryId: 2, role: 'user', content: 'source context', orderSeq: 1 }] + } + }) }) it('dispatches memory.getByIds with deepchat guard and input order projection', async () => { @@ -2703,10 +2701,10 @@ describe('dispatchDeepchatRoute', () => { it('returns no memory view manifests for missing or non-DeepChat agents', async () => { const { runtime, providerSettings } = createRuntime() - const listMemoryViewManifestAnchorsByAgent = vi.fn() + const listMemoryViewManifestsByAgent = vi.fn() ;(runtime as any).sqlitePresenter = { deepchatTapeEntriesTable: { - listMemoryViewManifestAnchorsByAgent + listMemoryViewManifestsByAgent } } vi.mocked(providerSettings.getAgentType) @@ -2729,7 +2727,7 @@ describe('dispatchDeepchatRoute', () => { { webContentsId: 42, windowId: 7 } ) ).resolves.toEqual({ manifests: [] }) - expect(listMemoryViewManifestAnchorsByAgent).not.toHaveBeenCalled() + expect(listMemoryViewManifestsByAgent).not.toHaveBeenCalled() }) it('dispatches bounded memory pages and returns an opaque keyset cursor', async () => { @@ -2808,28 +2806,19 @@ describe('dispatchDeepchatRoute', () => { const listSessions = vi.fn(() => Array.from({ length: 1200 }, (_, index) => ({ id: `s-${index}` })) ) - const listMemoryViewManifestAnchorsByAgent = vi.fn().mockReturnValue([ + const listMemoryViewManifestsByAgent = vi.fn().mockReturnValue([ { - session_id: 's-1199', - entry_id: 1, - kind: 'anchor', - name: 'memory/view_assembled', - source_type: 'memory', - source_id: 'msg-1', - source_seq: 0, - provenance_key: null, - payload_json: JSON.stringify({ - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 10, - selected: ['m1'], - dropped: [], - queryHash: 'hash' - } - }), - meta_json: JSON.stringify({ messageId: 'msg-1' }), - created_at: 100 + sessionId: 's-1199', + messageId: 'msg-1', + entryId: 1, + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: 10, + selectedCount: 1, + selectedIds: ['m1'], + droppedCount: 0, + queryHash: 'hash', + createdAt: 100 } ]) ;(runtime as any).sqlitePresenter = { @@ -2837,7 +2826,7 @@ describe('dispatchDeepchatRoute', () => { list: listSessions }, deepchatTapeEntriesTable: { - listMemoryViewManifestAnchorsByAgent + listMemoryViewManifestsByAgent } } @@ -2849,7 +2838,7 @@ describe('dispatchDeepchatRoute', () => { ) expect(listSessions).not.toHaveBeenCalled() - expect(listMemoryViewManifestAnchorsByAgent).toHaveBeenCalledWith('a', { + expect(listMemoryViewManifestsByAgent).toHaveBeenCalledWith('a', { sessionId: undefined, limit: 100, messageId: undefined diff --git a/test/main/session/data/tapeRecall.test.ts b/test/main/session/data/tapeRecall.test.ts index 2c428b8280..25309b3075 100644 --- a/test/main/session/data/tapeRecall.test.ts +++ b/test/main/session/data/tapeRecall.test.ts @@ -84,6 +84,46 @@ describe('SessionTape recall', () => { expect(service.search('s2', 'hello')).toHaveLength(0) }) + it('returns only effective message DTOs for a requested source span', () => { + const { table } = createTapeTableMock() + const first = table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { record: createRecord({ id: 'm1', orderSeq: 1 }) }, + meta: { source: 'live', orderSeq: 1, role: 'user' } + }) + const second = table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm2', seq: 0 }, + payload: { record: createRecord({ id: 'm2', orderSeq: 2 }) }, + meta: { source: 'live', orderSeq: 2, role: 'user' } + }) + table.appendEvent({ + sessionId: 's1', + name: 'message/retracted', + source: { type: 'message', id: 'm1', seq: 1 }, + data: { messageId: 'm1', reason: 'deleted' } + }) + const service = new SessionTape({ deepchatTapeEntriesTable: table } as any) + + const span = service.getEffectiveMessageSourceSpan('s1', [first.entry_id, second.entry_id]) + + expect(span).toHaveLength(1) + expect(span[0]).toEqual({ + entryId: second.entry_id, + record: { + role: 'user', + content: createRecord({ id: 'm2', orderSeq: 2 }).content, + orderSeq: 2 + } + }) + expect(span[0]).not.toHaveProperty('payload_json') + }) + it('projects fallback tape search into compact results and bounded context', () => { const { table } = createTapeTableMock() const service = createTapeService(table) diff --git a/test/main/session/data/tapeViewReplay.test.ts b/test/main/session/data/tapeViewReplay.test.ts index 126e8e7804..2f83c9f8a8 100644 --- a/test/main/session/data/tapeViewReplay.test.ts +++ b/test/main/session/data/tapeViewReplay.test.ts @@ -113,6 +113,70 @@ describe('SessionTape view and replay', () => { ]) }) + it('projects memory view anchors into inspection DTOs', () => { + const { table, entries } = createTapeTableMock() + table.listMemoryViewManifestAnchorsByAgent = vi.fn( + (_agentId: string, options?: { messageId?: string }) => + entries.filter( + (entry) => + entry.kind === 'anchor' && + entry.name === 'memory/view_assembled' && + (!options?.messageId || JSON.parse(entry.meta_json).messageId === options.messageId) + ) + ) + const service = new SessionTape({ deepchatTapeEntriesTable: table } as any) + table.appendAnchor({ + sessionId: 's1', + name: 'memory/view_assembled', + state: { + policyVersion: 2, + tokenBudget: 1000, + estimatedTokens: 15, + selected: [ + 'm-string', + { id: 'm-object' }, + 'm-string', + { id: 'm-object' }, + { ignored: true }, + 3 + ], + dropped: ['m-dropped'], + queryHash: 'query-hash' + }, + meta: { messageId: 'msg-1' }, + createdAt: 300 + }) + + const manifests = service.listMemoryViewManifestsByAgent('agent-1', { + sessionId: 's1', + messageId: 'msg-1', + limit: 1 + }) + + expect(table.listMemoryViewManifestAnchorsByAgent).toHaveBeenCalledWith('agent-1', { + sessionId: 's1', + messageId: 'msg-1', + limit: 1 + }) + expect(manifests).toEqual([ + { + sessionId: 's1', + messageId: 'msg-1', + entryId: 1, + policyVersion: 2, + tokenBudget: 1000, + estimatedTokens: 15, + selectedCount: 6, + selectedIds: ['m-string', 'm-object'], + droppedCount: 1, + queryHash: 'query-hash', + createdAt: 300 + } + ]) + expect(manifests[0]).not.toHaveProperty('payload_json') + expect(manifests[0]).not.toHaveProperty('meta_json') + }) + it('indexes effective tool facts so tool-loop manifests reference real entries', () => { const { table } = createTapeTableMock() const assistantRecord = createRecord({ From 8a8bfe0f42c87423588e8d860bb2e9e7b81f052b Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 17:19:56 +0800 Subject: [PATCH 11/29] refactor(tape): align service ownership --- src/main/tape/application/common.ts | 35 ---- src/main/tape/application/contracts.ts | 1 + src/main/tape/application/factService.ts | 128 +++++++++++- src/main/tape/application/forkService.ts | 148 +------------- src/main/tape/application/sessionTape.ts | 21 +- .../tape/application/viewReplayService.ts | 193 ++---------------- src/main/tape/domain/replay.ts | 193 ++++++++++++++++++ src/main/tape/ports/capabilities.ts | 7 +- 8 files changed, 355 insertions(+), 371 deletions(-) create mode 100644 src/main/tape/domain/replay.ts diff --git a/src/main/tape/application/common.ts b/src/main/tape/application/common.ts index 13c6bbcd5c..5eb330afa3 100644 --- a/src/main/tape/application/common.ts +++ b/src/main/tape/application/common.ts @@ -1,6 +1,3 @@ -import { createHash } from 'crypto' -import type { DeepChatTapeReplaySlice } from '@shared/types/tape-replay' - export function parseJsonObject(raw: string): Record { try { const parsed = JSON.parse(raw) as unknown @@ -19,20 +16,6 @@ export function parseJsonValue(raw: string): unknown { } } -export function hashString(value: string): string { - return createHash('sha256').update(value).digest('hex') -} - -export function isPositiveInteger(value: number): boolean { - return Number.isInteger(value) && value > 0 -} - -export function collectEntryIds(values: Array): number[] { - return [...new Set(values.filter((value): value is number => typeof value === 'number'))].sort( - (left, right) => left - right - ) -} - export function isEntryIdPrefix(prefix: number[], values: number[]): boolean { if (prefix.length > values.length) return false for (let index = 0; index < prefix.length; index += 1) { @@ -44,21 +27,3 @@ export function isEntryIdPrefix(prefix: number[], values: number[]): boolean { export function migrationProvenanceKey(sessionId: string): string { return `migration:${sessionId}:message-backfill:v1` } - -export function withReplaySliceHash( - slice: Omit & { - hashes: Omit & { sliceHash: '' } - }, - hashJson: (value: unknown) => string -): DeepChatTapeReplaySlice { - const sliceForHash = { ...slice } as Partial - delete sliceForHash.createdAt - delete sliceForHash.integrity - return { - ...slice, - hashes: { - ...slice.hashes, - sliceHash: hashJson(sliceForHash) - } - } -} diff --git a/src/main/tape/application/contracts.ts b/src/main/tape/application/contracts.ts index 465d96f909..ed947e3298 100644 --- a/src/main/tape/application/contracts.ts +++ b/src/main/tape/application/contracts.ts @@ -4,6 +4,7 @@ import type { TapeMigrationState } from '../ports/capabilities' export type { TapeBackfillResult, TapeMigrationState, + TapeViewManifestAssemblySources, TapeViewManifestSourceMaps } from '../ports/capabilities' diff --git a/src/main/tape/application/factService.ts b/src/main/tape/application/factService.ts index 05e782ff2a..36684e5a76 100644 --- a/src/main/tape/application/factService.ts +++ b/src/main/tape/application/factService.ts @@ -1,7 +1,12 @@ -import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { AgentTapeHandoffState, ChatMessageRecord } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryRow, TapeAnchorAppendInput } from '../domain/entry' import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' import { buildEffectiveTapeView } from '../domain/effectiveView' -import type { TapeToolFactWriter, TapeMessageFactWriter } from '../ports/capabilities' +import type { + TapeAnchorWriter, + TapeMessageFactWriter, + TapeToolFactWriter +} from '../ports/capabilities' import type { TapeApplicationProviders } from '../ports/application' import { appendMessageRecordToTape, @@ -9,10 +14,75 @@ import { appendMessageRetractionToTape, appendTapeToolFact } from './factPersistence' +import { parseJsonObject } from './common' +import type { TapeAnchorResult } from './contracts' type TapeFactProviders = Pick -export class TapeFactService implements TapeToolFactWriter, TapeMessageFactWriter { +function normalizeHandoffName(name: string): string { + const trimmed = name.trim() + if (!trimmed) return 'handoff/manual' + if (trimmed.startsWith('handoff/') || trimmed.startsWith('auto_handoff/')) return trimmed + return `handoff/${trimmed}` +} + +function normalizePositiveInteger(value: unknown): number | null { + if (typeof value !== 'number' || !Number.isFinite(value)) return null + return Math.max(1, Math.floor(value)) +} + +function buildOrderSeqRange(records: ChatMessageRecord[]): Record | null { + if (records.length === 0) return null + return { + fromOrderSeq: records[0].orderSeq, + toOrderSeq: records[records.length - 1].orderSeq + } +} + +function enrichHandoffState( + state: Record, + historyRecords: ChatMessageRecord[] +): Record { + const maxOrderSeq = historyRecords.reduce( + (currentMax, record) => Math.max(currentMax, record.orderSeq), + 0 + ) + const cursorOrderSeq = + normalizePositiveInteger(state.cursorOrderSeq ?? state.summaryCursorOrderSeq) ?? maxOrderSeq + 1 + const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) + const enrichedState: Record = { ...state, cursorOrderSeq } + + if (!Object.prototype.hasOwnProperty.call(enrichedState, 'range')) { + enrichedState.range = buildOrderSeqRange(sourceRecords) + } + + const sourceMessageIds = enrichedState.sourceMessageIds + if (!Array.isArray(sourceMessageIds) || sourceMessageIds.some((id) => typeof id !== 'string')) { + enrichedState.sourceMessageIds = sourceRecords.map((record) => record.id) + } + + return enrichedState +} + +export function normalizeTapeHandoffState(state: unknown): AgentTapeHandoffState { + if (!state || typeof state !== 'object' || Array.isArray(state)) { + throw new Error('Tape handoff requires a non-empty summary.') + } + + const summary = (state as Record).summary + if (typeof summary !== 'string' || !summary.trim()) { + throw new Error('Tape handoff requires a non-empty summary.') + } + + return { + ...(state as Record), + summary: summary.trim() + } +} + +export class TapeFactService + implements TapeToolFactWriter, TapeMessageFactWriter, TapeAnchorWriter +{ constructor(private readonly providers: TapeFactProviders) {} private get table() { @@ -23,6 +93,10 @@ export class TapeFactService implements TapeToolFactWriter, TapeMessageFactWrite return appendMessageRecordToTape(this.table, record, 'live') } + appendMessageRecordForSession(sessionId: string, record: ChatMessageRecord): number { + return appendMessageRecordToTape(this.table, { ...record, sessionId }, 'live') + } + appendMessageReplacement(record: ChatMessageRecord, reason: string): number { return appendMessageReplacementToTape(this.table, record, reason) } @@ -41,4 +115,52 @@ export class TapeFactService implements TapeToolFactWriter, TapeMessageFactWrite return buildEffectiveTapeView(this.table.getBySession(sessionId), { includePending: true }) .messageRecords } + + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow { + return this.table.appendAnchor(input) + } + + handoff( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): DeepChatTapeEntryRow { + const normalizedState = normalizeTapeHandoffState(state) + const table = this.table + table.ensureBootstrapAnchor(sessionId) + const handoffState = enrichHandoffState(normalizedState, this.getMessageRecords(sessionId)) + return table.appendAnchor({ + sessionId, + name: normalizeHandoffName(name), + source: { + type: 'runtime_event', + id: `handoff:${Date.now()}`, + seq: 0 + }, + state: handoffState, + meta: { + ...meta, + handoff: true + } + }) + } + + handoffResult( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): TapeAnchorResult { + const row = this.handoff(sessionId, name, state, meta) + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + payload: parseJsonObject(row.payload_json), + meta: parseJsonObject(row.meta_json), + createdAt: row.created_at + } + } } diff --git a/src/main/tape/application/forkService.ts b/src/main/tape/application/forkService.ts index 9831eb5410..1de7e21c56 100644 --- a/src/main/tape/application/forkService.ts +++ b/src/main/tape/application/forkService.ts @@ -1,13 +1,10 @@ import { nanoid } from 'nanoid' import logger from 'electron-log' -import type { AgentTapeHandoffState, ChatMessageRecord } from '@shared/types/agent-interface' import type { DeepChatTapeEntryRow } from '../domain/entry' import type { TapeApplicationProviders } from '../ports/application' -import { appendMessageRecordToTape } from './factPersistence' import { deleteTapeGeneration } from './generationLifecycle' import { parseJsonObject } from './common' -import type { TapeAnchorResult, TapeForkHandle } from './contracts' -import type { TapeFactService } from './factService' +import type { TapeForkHandle } from './contracts' type TapeForkProviders = Pick< TapeApplicationProviders, @@ -88,83 +85,6 @@ function assertValidForkStart( } } -function normalizeHandoffName(name: string): string { - const trimmed = name.trim() - if (!trimmed) { - return 'handoff/manual' - } - if (trimmed.startsWith('handoff/') || trimmed.startsWith('auto_handoff/')) { - return trimmed - } - return `handoff/${trimmed}` -} - -function normalizePositiveInteger(value: unknown): number | null { - if (typeof value === 'number' && Number.isFinite(value)) { - return Math.max(1, Math.floor(value)) - } - return null -} - -function hasOwnKey(value: Record, key: string): boolean { - return Object.prototype.hasOwnProperty.call(value, key) -} - -function buildOrderSeqRange(records: ChatMessageRecord[]): Record | null { - if (records.length === 0) { - return null - } - - return { - fromOrderSeq: records[0].orderSeq, - toOrderSeq: records[records.length - 1].orderSeq - } -} - -function enrichHandoffState( - state: Record, - historyRecords: ChatMessageRecord[] -): Record { - const maxOrderSeq = historyRecords.reduce( - (currentMax, record) => Math.max(currentMax, record.orderSeq), - 0 - ) - const cursorOrderSeq = - normalizePositiveInteger(state.cursorOrderSeq ?? state.summaryCursorOrderSeq) ?? maxOrderSeq + 1 - const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) - const enrichedState: Record = { - ...state, - cursorOrderSeq - } - - if (!hasOwnKey(enrichedState, 'range')) { - enrichedState.range = buildOrderSeqRange(sourceRecords) - } - - const sourceMessageIds = enrichedState.sourceMessageIds - if (!Array.isArray(sourceMessageIds) || sourceMessageIds.some((id) => typeof id !== 'string')) { - enrichedState.sourceMessageIds = sourceRecords.map((record) => record.id) - } - - return enrichedState -} - -export function normalizeTapeHandoffState(state: unknown): AgentTapeHandoffState { - if (!state || typeof state !== 'object' || Array.isArray(state)) { - throw new Error('Tape handoff requires a non-empty summary.') - } - - const summary = (state as Record).summary - if (typeof summary !== 'string' || !summary.trim()) { - throw new Error('Tape handoff requires a non-empty summary.') - } - - return { - ...(state as Record), - summary: summary.trim() - } -} - function forkSessionId(parentSessionId: string, forkId: string): string { return `${parentSessionId}::fork::${forkId}` } @@ -174,65 +94,12 @@ function forkDiscardProvenanceKey(parentSessionId: string, forkId: string): stri } export class TapeForkService { - constructor( - private readonly providers: TapeForkProviders, - private readonly facts: TapeFactService - ) {} + constructor(private readonly providers: TapeForkProviders) {} private get table() { return this.providers.getEntryStore() } - private toAnchorResult(row: DeepChatTapeEntryRow): TapeAnchorResult { - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - payload: parseJsonObject(row.payload_json), - meta: parseJsonObject(row.meta_json), - createdAt: row.created_at - } - } - - handoff( - sessionId: string, - name: string, - state: AgentTapeHandoffState, - meta: Record = {} - ): DeepChatTapeEntryRow { - const normalizedState = normalizeTapeHandoffState(state) - const table = this.table - table.ensureBootstrapAnchor(sessionId) - const handoffState = enrichHandoffState( - normalizedState, - this.facts.getMessageRecords(sessionId) - ) - return table.appendAnchor({ - sessionId, - name: normalizeHandoffName(name), - source: { - type: 'runtime_event', - id: `handoff:${Date.now()}`, - seq: 0 - }, - state: handoffState, - meta: { - ...meta, - handoff: true - } - }) - } - - handoffResult( - sessionId: string, - name: string, - state: AgentTapeHandoffState, - meta: Record = {} - ): TapeAnchorResult { - return this.toAnchorResult(this.handoff(sessionId, name, state, meta)) - } - createFork(parentSessionId: string, forkId: string = nanoid()): TapeForkHandle { const table = this.table const forkIdValue = forkId.trim() || nanoid() @@ -286,17 +153,6 @@ export class TapeForkService { } } - appendForkMessageRecord(handle: TapeForkHandle, record: ChatMessageRecord): number { - return appendMessageRecordToTape( - this.table, - { - ...record, - sessionId: handle.forkSessionId - }, - 'live' - ) - } - mergeFork(parentSessionId: string, forkId: string): number { const table = this.table const forkSessionIdValue = forkSessionId(parentSessionId, forkId) diff --git a/src/main/tape/application/sessionTape.ts b/src/main/tape/application/sessionTape.ts index b1589bc71b..63521b0527 100644 --- a/src/main/tape/application/sessionTape.ts +++ b/src/main/tape/application/sessionTape.ts @@ -47,10 +47,11 @@ import type { TapeInfo, TapeMigrationState, TapeSearchResult, + TapeViewManifestAssemblySources, TapeViewManifestSourceMaps } from './contracts' -import { TapeFactService } from './factService' -import { normalizeTapeHandoffState, TapeForkService } from './forkService' +import { normalizeTapeHandoffState, TapeFactService } from './factService' +import { TapeForkService } from './forkService' import { resetTapeGeneration } from './generationLifecycle' import { AgentTapeViewError, @@ -70,6 +71,7 @@ export type { TapeInfo, TapeMigrationState, TapeSearchResult, + TapeViewManifestAssemblySources, TapeViewManifestSourceMaps } export { AgentTapeViewError, normalizeSubagentTapeLinkInput, normalizeTapeHandoffState } @@ -102,7 +104,7 @@ export class SessionTape this.reconciler = new TapeReconcilerService(this.providers, this.facts) this.recall = new TapeRecallService(this.providers, this.lineage) this.viewReplay = new TapeViewReplayService(this.providers) - this.forks = new TapeForkService(this.providers, this.facts) + this.forks = new TapeForkService(this.providers) } ensureSessionTapeReady( @@ -152,7 +154,10 @@ export class SessionTape return this.recall.anchors(sessionId, options) } - getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestSourceMaps { + getViewManifestSourceMaps( + sessionId: string, + messageId?: string + ): TapeViewManifestAssemblySources { return this.viewReplay.getViewManifestSourceMaps(sessionId, messageId) } @@ -189,7 +194,7 @@ export class SessionTape state: AgentTapeHandoffState, meta: Record = {} ): DeepChatTapeEntryRow { - return this.forks.handoff(sessionId, name, state, meta) + return this.facts.handoff(sessionId, name, state, meta) } handoffResult( @@ -198,7 +203,7 @@ export class SessionTape state: AgentTapeHandoffState, meta: Record = {} ): TapeAnchorResult { - return this.forks.handoffResult(sessionId, name, state, meta) + return this.facts.handoffResult(sessionId, name, state, meta) } createFork(parentSessionId: string, forkId?: string): TapeForkHandle { @@ -206,7 +211,7 @@ export class SessionTape } appendForkMessageRecord(handle: TapeForkHandle, record: ChatMessageRecord): number { - return this.forks.appendForkMessageRecord(handle, record) + return this.facts.appendMessageRecordForSession(handle.forkSessionId, record) } mergeFork(parentSessionId: string, forkId: string): number { @@ -260,7 +265,7 @@ export class SessionTape } appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow { - return this.providers.getEntryStore().appendAnchor(input) + return this.facts.appendAnchor(input) } getEffectiveMessageSourceSpan( diff --git a/src/main/tape/application/viewReplayService.ts b/src/main/tape/application/viewReplayService.ts index b8b637968a..271b42f9bc 100644 --- a/src/main/tape/application/viewReplayService.ts +++ b/src/main/tape/application/viewReplayService.ts @@ -1,5 +1,4 @@ import type { - DeepChatTapeViewExcludedRange, DeepChatTapeViewManifest, DeepChatTapeViewManifestRecord } from '@shared/types/tape-view-manifest' @@ -14,6 +13,13 @@ import type { } from '@shared/types/tape-replay' import { SUMMARY_ANCHOR_NAMES, type DeepChatTapeEntryRow } from '../domain/entry' import { buildEffectiveTapeView } from '../domain/effectiveView' +import { + collectEntryIds, + hashString, + isPositiveInteger, + normalizeStoredTapeViewManifest, + withReplaySliceHash +} from '../domain/replay' import { hashJson, TAPE_VIEW_MANIFEST_EVENT_NAME, @@ -24,14 +30,8 @@ import type { TapeMessageTraceRow as DeepChatMessageTraceRow } from '../ports/application' import type { TapeMemoryViewManifestInspection } from '../ports/capabilities' -import { - collectEntryIds, - hashString, - isPositiveInteger, - parseJsonObject, - withReplaySliceHash -} from './common' -import type { TapeViewManifestSourceMaps } from './contracts' +import { parseJsonObject } from './common' +import type { TapeViewManifestAssemblySources } from './contracts' type TapeViewReplayProviders = Pick< TapeApplicationProviders, @@ -120,165 +120,6 @@ function toMemoryViewManifestInspection( } } -const VIEW_POLICIES = new Set([ - 'legacy_context_v1', - 'legacy_context_shadow', - 'resume_shadow', - 'tool_loop_shadow', - 'context_pressure_recovery_shadow' -]) - -const VIEW_ENTRY_REASONS = new Set([ - 'system_prompt', - 'selected_history', - 'new_user_input', - 'resume_target', - 'tool_loop_message' -]) - -const VIEW_EXCLUDED_REASONS = new Set([ - 'before_summary_cursor', - 'compaction_indicator', - 'pending_not_context_history', - 'out_of_budget', - 'empty_after_formatting', - 'superseded', - 'retracted' -]) - -function isRecordObject(value: unknown): value is Record { - return Boolean(value && typeof value === 'object' && !Array.isArray(value)) -} - -function isNullableString(value: unknown): value is string | null { - return value === null || typeof value === 'string' -} - -function isNullableNumber(value: unknown): value is number | null { - return value === null || typeof value === 'number' -} - -function isViewEntryRef(value: unknown): value is DeepChatTapeViewManifest['included'][number] { - if (!isRecordObject(value)) { - return false - } - - return ( - isNullableNumber(value.entryId) && - isNullableString(value.messageId) && - isNullableNumber(value.orderSeq) && - (value.role === 'system' || - value.role === 'user' || - value.role === 'assistant' || - value.role === 'tool' || - value.role === null) && - (value.source === 'tape' || value.source === 'synthetic') && - typeof value.reason === 'string' && - VIEW_ENTRY_REASONS.has(value.reason) - ) -} - -function isViewExcludedRef(value: unknown): value is DeepChatTapeViewManifest['excluded'][number] { - if (!isRecordObject(value)) { - return false - } - - return ( - isNullableNumber(value.entryId) && - isNullableString(value.messageId) && - isNullableNumber(value.orderSeq) && - typeof value.reason === 'string' && - VIEW_EXCLUDED_REASONS.has(value.reason) - ) -} - -function isViewExcludedRange(value: unknown): value is DeepChatTapeViewExcludedRange { - if (!isRecordObject(value)) { - return false - } - - return ( - typeof value.fromOrderSeq === 'number' && - typeof value.toOrderSeq === 'number' && - typeof value.count === 'number' && - typeof value.reason === 'string' && - VIEW_EXCLUDED_REASONS.has(value.reason) - ) -} - -function hasNumberFields(value: unknown, fields: string[]): value is Record { - if (!isRecordObject(value)) { - return false - } - - return fields.every((field) => typeof value[field] === 'number') -} - -function hasStringFields(value: unknown, fields: string[]): value is Record { - if (!isRecordObject(value)) { - return false - } - - return fields.every((field) => typeof value[field] === 'string') -} - -function isViewManifestMeta(value: unknown): value is DeepChatTapeViewManifest['meta'] { - if (!isRecordObject(value)) { - return false - } - - return ( - typeof value.providerId === 'string' && - typeof value.modelId === 'string' && - typeof value.summaryCursorOrderSeq === 'number' && - typeof value.supportsVision === 'boolean' && - typeof value.supportsAudioInput === 'boolean' && - typeof value.traceDebugEnabled === 'boolean' - ) -} - -function isViewManifest(value: unknown, sessionId: string): value is DeepChatTapeViewManifest { - if (!isRecordObject(value)) { - return false - } - - return ( - (value.schemaVersion === 1 || value.schemaVersion === 2) && - typeof value.hashVersion === 'number' && - value.sessionId === sessionId && - typeof value.viewId === 'string' && - typeof value.messageId === 'string' && - typeof value.requestSeq === 'number' && - (value.taskType === 'chat' || value.taskType === 'resume' || value.taskType === 'tool_loop') && - typeof value.policy === 'string' && - VIEW_POLICIES.has(value.policy) && - (typeof value.policyVersion === 'number' || value.policyVersion === null) && - value.contextBuilderVersion === 'legacy-v1' && - typeof value.latestEntryId === 'number' && - Array.isArray(value.anchorEntryIds) && - value.anchorEntryIds.every((entryId) => typeof entryId === 'number') && - (value.reconstructionAnchorEntryId === undefined || - isNullableNumber(value.reconstructionAnchorEntryId)) && - (value.excludedRanges === undefined || - (Array.isArray(value.excludedRanges) && value.excludedRanges.every(isViewExcludedRange))) && - Array.isArray(value.included) && - value.included.every(isViewEntryRef) && - Array.isArray(value.excluded) && - value.excluded.every(isViewExcludedRef) && - hasNumberFields(value.tokenBudget, [ - 'contextLength', - 'requestedMaxTokens', - 'effectiveMaxTokens', - 'reserveTokens', - 'toolReserveTokens', - 'estimatedPromptTokens' - ]) && - hasStringFields(value.hashes, ['promptHash', 'toolDefinitionsHash', 'manifestHash']) && - isViewManifestMeta(value.meta) && - typeof value.assembledAt === 'number' - ) -} - export class TapeViewReplayService { constructor(private readonly providers: TapeViewReplayProviders) {} @@ -286,7 +127,10 @@ export class TapeViewReplayService { return this.providers.getEntryStore() } - getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestSourceMaps { + getViewManifestSourceMaps( + sessionId: string, + messageId?: string + ): TapeViewManifestAssemblySources { const table = this.table const rows = table.getBySession(sessionId) const entryIdByMessageId = new Map() @@ -585,7 +429,7 @@ export class TapeViewReplayService { createdAt } - return withReplaySliceHash(sliceBase, hashJson) + return withReplaySliceHash(sliceBase) } private toViewManifestRecord(row: DeepChatTapeEntryRow): DeepChatTapeViewManifestRecord | null { @@ -595,13 +439,8 @@ export class TapeViewReplayService { data && typeof data === 'object' && !Array.isArray(data) ? (data as Record).manifest : undefined - const manifest = - isRecordObject(rawManifest) && rawManifest.hashVersion === undefined - ? { ...rawManifest, hashVersion: 1 } - : rawManifest - if (!isViewManifest(manifest, row.session_id)) { - return null - } + const manifest = normalizeStoredTapeViewManifest(rawManifest, row.session_id) + if (!manifest) return null return { sessionId: row.session_id, diff --git a/src/main/tape/domain/replay.ts b/src/main/tape/domain/replay.ts new file mode 100644 index 0000000000..c9b5e75b37 --- /dev/null +++ b/src/main/tape/domain/replay.ts @@ -0,0 +1,193 @@ +import { createHash } from 'crypto' +import type { + DeepChatTapeViewExcludedRange, + DeepChatTapeViewManifest +} from '@shared/types/tape-view-manifest' +import type { DeepChatTapeReplaySlice } from '@shared/types/tape-replay' +import { hashJson } from './viewManifest' + +const VIEW_POLICIES = new Set([ + 'legacy_context_v1', + 'legacy_context_shadow', + 'resume_shadow', + 'tool_loop_shadow', + 'context_pressure_recovery_shadow' +]) + +const VIEW_ENTRY_REASONS = new Set([ + 'system_prompt', + 'selected_history', + 'new_user_input', + 'resume_target', + 'tool_loop_message' +]) + +const VIEW_EXCLUDED_REASONS = new Set([ + 'before_summary_cursor', + 'compaction_indicator', + 'pending_not_context_history', + 'out_of_budget', + 'empty_after_formatting', + 'superseded', + 'retracted' +]) + +function isRecordObject(value: unknown): value is Record { + return Boolean(value && typeof value === 'object' && !Array.isArray(value)) +} + +function isNullableString(value: unknown): value is string | null { + return value === null || typeof value === 'string' +} + +function isNullableNumber(value: unknown): value is number | null { + return value === null || typeof value === 'number' +} + +function isViewEntryRef(value: unknown): value is DeepChatTapeViewManifest['included'][number] { + if (!isRecordObject(value)) return false + + return ( + isNullableNumber(value.entryId) && + isNullableString(value.messageId) && + isNullableNumber(value.orderSeq) && + (value.role === 'system' || + value.role === 'user' || + value.role === 'assistant' || + value.role === 'tool' || + value.role === null) && + (value.source === 'tape' || value.source === 'synthetic') && + typeof value.reason === 'string' && + VIEW_ENTRY_REASONS.has(value.reason) + ) +} + +function isViewExcludedRef(value: unknown): value is DeepChatTapeViewManifest['excluded'][number] { + if (!isRecordObject(value)) return false + + return ( + isNullableNumber(value.entryId) && + isNullableString(value.messageId) && + isNullableNumber(value.orderSeq) && + typeof value.reason === 'string' && + VIEW_EXCLUDED_REASONS.has(value.reason) + ) +} + +function isViewExcludedRange(value: unknown): value is DeepChatTapeViewExcludedRange { + if (!isRecordObject(value)) return false + + return ( + typeof value.fromOrderSeq === 'number' && + typeof value.toOrderSeq === 'number' && + typeof value.count === 'number' && + typeof value.reason === 'string' && + VIEW_EXCLUDED_REASONS.has(value.reason) + ) +} + +function hasNumberFields(value: unknown, fields: string[]): value is Record { + if (!isRecordObject(value)) return false + return fields.every((field) => typeof value[field] === 'number') +} + +function hasStringFields(value: unknown, fields: string[]): value is Record { + if (!isRecordObject(value)) return false + return fields.every((field) => typeof value[field] === 'string') +} + +function isViewManifestMeta(value: unknown): value is DeepChatTapeViewManifest['meta'] { + if (!isRecordObject(value)) return false + + return ( + typeof value.providerId === 'string' && + typeof value.modelId === 'string' && + typeof value.summaryCursorOrderSeq === 'number' && + typeof value.supportsVision === 'boolean' && + typeof value.supportsAudioInput === 'boolean' && + typeof value.traceDebugEnabled === 'boolean' + ) +} + +export function isTapeViewManifest( + value: unknown, + sessionId: string +): value is DeepChatTapeViewManifest { + if (!isRecordObject(value)) return false + + return ( + (value.schemaVersion === 1 || value.schemaVersion === 2) && + typeof value.hashVersion === 'number' && + value.sessionId === sessionId && + typeof value.viewId === 'string' && + typeof value.messageId === 'string' && + typeof value.requestSeq === 'number' && + (value.taskType === 'chat' || value.taskType === 'resume' || value.taskType === 'tool_loop') && + typeof value.policy === 'string' && + VIEW_POLICIES.has(value.policy) && + (typeof value.policyVersion === 'number' || value.policyVersion === null) && + value.contextBuilderVersion === 'legacy-v1' && + typeof value.latestEntryId === 'number' && + Array.isArray(value.anchorEntryIds) && + value.anchorEntryIds.every((entryId) => typeof entryId === 'number') && + (value.reconstructionAnchorEntryId === undefined || + isNullableNumber(value.reconstructionAnchorEntryId)) && + (value.excludedRanges === undefined || + (Array.isArray(value.excludedRanges) && value.excludedRanges.every(isViewExcludedRange))) && + Array.isArray(value.included) && + value.included.every(isViewEntryRef) && + Array.isArray(value.excluded) && + value.excluded.every(isViewExcludedRef) && + hasNumberFields(value.tokenBudget, [ + 'contextLength', + 'requestedMaxTokens', + 'effectiveMaxTokens', + 'reserveTokens', + 'toolReserveTokens', + 'estimatedPromptTokens' + ]) && + hasStringFields(value.hashes, ['promptHash', 'toolDefinitionsHash', 'manifestHash']) && + isViewManifestMeta(value.meta) && + typeof value.assembledAt === 'number' + ) +} + +export function normalizeStoredTapeViewManifest( + value: unknown, + sessionId: string +): DeepChatTapeViewManifest | null { + const candidate = + isRecordObject(value) && value.hashVersion === undefined ? { ...value, hashVersion: 1 } : value + return isTapeViewManifest(candidate, sessionId) ? candidate : null +} + +export function hashString(value: string): string { + return createHash('sha256').update(value).digest('hex') +} + +export function isPositiveInteger(value: number): boolean { + return Number.isInteger(value) && value > 0 +} + +export function collectEntryIds(values: Array): number[] { + return [...new Set(values.filter((value): value is number => typeof value === 'number'))].sort( + (left, right) => left - right + ) +} + +export function withReplaySliceHash( + slice: Omit & { + hashes: Omit & { sliceHash: '' } + } +): DeepChatTapeReplaySlice { + const sliceForHash = { ...slice } as Partial + delete sliceForHash.createdAt + delete sliceForHash.integrity + return { + ...slice, + hashes: { + ...slice.hashes, + sliceHash: hashJson(sliceForHash) + } + } +} diff --git a/src/main/tape/ports/capabilities.ts b/src/main/tape/ports/capabilities.ts index 2e8f511063..539e6ebd78 100644 --- a/src/main/tape/ports/capabilities.ts +++ b/src/main/tape/ports/capabilities.ts @@ -25,7 +25,7 @@ export interface TapeReconciliationPort { ensureSessionTapeReady(sessionId: string, messageStore: TapeTranscriptReader): TapeBackfillResult } -export type TapeViewManifestSourceMaps = { +export type TapeViewManifestAssemblySources = { latestEntryId: number anchorEntryIds: number[] reconstructionAnchorEntryIds: number[] @@ -35,8 +35,11 @@ export type TapeViewManifestSourceMaps = { toolResultEntryIdByToolId: Map } +/** @deprecated Use TapeViewManifestAssemblySources for the complete assembly source set. */ +export type TapeViewManifestSourceMaps = TapeViewManifestAssemblySources + export interface TapeViewManifestReader { - getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestSourceMaps + getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestAssemblySources listViewManifestsByMessage(sessionId: string, messageId: string): DeepChatTapeViewManifestRecord[] } From 0a6d11ab03f2139aac390036b2c731b4a039a800 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 17:25:48 +0800 Subject: [PATCH 12/29] test(tape): harden layer boundaries --- test/main/tape/layerBoundaries.test.ts | 237 ++++++++++++++++++++++++- 1 file changed, 234 insertions(+), 3 deletions(-) diff --git a/test/main/tape/layerBoundaries.test.ts b/test/main/tape/layerBoundaries.test.ts index 7fc313f107..c979451c24 100644 --- a/test/main/tape/layerBoundaries.test.ts +++ b/test/main/tape/layerBoundaries.test.ts @@ -3,11 +3,41 @@ import ts from 'typescript' import { describe, expect, it, vi } from 'vitest' const MAIN_SOURCE_ROOT = path.resolve(process.cwd(), 'src/main') +const TAPE_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape') const TAPE_DOMAIN_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape/domain') const TAPE_SQLITE_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape/infrastructure/sqlite') +const TAPE_CAPABILITIES_MODULE = path.join(MAIN_SOURCE_ROOT, 'tape/ports/capabilities') +const MEMORY_ROUTES_FILE = path.join(MAIN_SOURCE_ROOT, 'memory/routes.ts') const TAPE_SQLITE_RELATIVE_ROOT = 'tape/infrastructure/sqlite/' const TYPESCRIPT_SOURCE_EXTENSION = /\.[cm]?tsx?$/ +const LEGACY_TAPE_COMPATIBILITY_MODULES = new Map([ + ['session/data/tape', '@/tape/application/sessionTape'], + ['session/data/tapeEffectiveView', '@/tape/domain/effectiveView'], + ['session/data/tapeFacts', '@/tape/application/factPersistence'], + ['session/data/tapeViewManifest', '@/tape/domain/viewManifest'], + ['session/data/tables/deepchatTapeEntries', '@/tape/infrastructure/sqlite/tapeEntryStore'], + [ + 'session/data/tables/deepchatTapeSearchProjection', + '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' + ] +]) + +const FORBIDDEN_DOMAIN_SQLITE_IMPORTS = new Set([ + 'better-sqlite3', + 'bun:sqlite', + 'node:sqlite', + 'sql.js', + 'sqlite3' +]) +const FORBIDDEN_DOMAIN_LOGGING_IMPORTS = new Set([ + '@shared/logger', + 'electron-log', + 'loglevel', + 'pino', + 'winston' +]) + const PHYSICAL_TAPE_STORAGE_PATTERN = /\b(?:deepchat_tape_(?:entries|search_(?:projection(?:_meta)?|fts(?:_meta)?))|DeepChatTape(?:Entries|SearchProjection)Table|deepchatTape(?:Entries|SearchProjection)(?:Table)?)\b/ @@ -86,6 +116,111 @@ function resolveMainImport(importingFile: string, specifier: string): string | n return null } +function withoutTypeScriptExtension(file: string): string { + return file.replace(TYPESCRIPT_SOURCE_EXTENSION, '') +} + +function matchesPackageOrSubpath(specifier: string, packages: ReadonlySet): boolean { + return [...packages].some( + (packageName) => specifier === packageName || specifier.startsWith(`${packageName}/`) + ) +} + +function getForbiddenDomainPackageCategory(specifier: string): string | null { + if ( + specifier === 'electron' || + specifier.startsWith('electron/') || + specifier.startsWith('@electron/') + ) { + return 'Electron runtime' + } + if ( + matchesPackageOrSubpath(specifier, FORBIDDEN_DOMAIN_SQLITE_IMPORTS) || + specifier.startsWith('@libsql/') + ) { + return 'SQLite runtime' + } + if (matchesPackageOrSubpath(specifier, FORBIDDEN_DOMAIN_LOGGING_IMPORTS)) { + return 'logging runtime' + } + return null +} + +function getDomainImportViolation(importingFile: string, specifier: string): string | null { + const forbiddenPackageCategory = getForbiddenDomainPackageCategory(specifier) + if (forbiddenPackageCategory) { + return `${forbiddenPackageCategory} import ${specifier}` + } + + const target = resolveMainImport(importingFile, specifier) + if (target && !isInside(TAPE_DOMAIN_ROOT, target)) { + return `main-process dependency ${specifier}` + } + return null +} + +function isLegacyTapeCompatibilityImport(importingFile: string, specifier: string): boolean { + const target = resolveMainImport(importingFile, specifier) + if (!target) return false + const relativeTarget = relativeToMain(withoutTypeScriptExtension(target)) + return LEGACY_TAPE_COMPATIBILITY_MODULES.has(relativeTarget) +} + +function isTapeModuleImport(importingFile: string, specifier: string): boolean { + const target = resolveMainImport(importingFile, specifier) + return Boolean(target && isInside(TAPE_ROOT, target)) +} + +function findMemoryRouteTapeImportViolations(source: string, file: string): string[] { + const sourceFile = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true) + return sourceFile.statements.flatMap((statement) => { + if (!ts.isImportDeclaration(statement) || !ts.isStringLiteral(statement.moduleSpecifier)) { + return [] + } + const specifier = statement.moduleSpecifier.text + if (!isTapeModuleImport(file, specifier)) return [] + + const target = resolveMainImport(file, specifier) + if (!target || withoutTypeScriptExtension(target) !== TAPE_CAPABILITIES_MODULE) { + return [`Tape import must use the inspection port: ${specifier}`] + } + + const importClause = statement.importClause + const namedBindings = importClause?.namedBindings + if ( + !importClause || + importClause.name || + !namedBindings || + !ts.isNamedImports(namedBindings) || + namedBindings.elements.length !== 1 + ) { + return [`Tape capabilities import must name only TapeInspectionReader: ${specifier}`] + } + + const [element] = namedBindings.elements + const importedName = element.propertyName?.text ?? element.name.text + const isTypeOnly = importClause.isTypeOnly || element.isTypeOnly + return importedName === 'TapeInspectionReader' && isTypeOnly + ? [] + : [`Memory routes may import only the TapeInspectionReader type: ${importedName}`] + }) +} + +function isCanonicalCompatibilityReexport(source: string, expectedTarget: string): boolean { + const sourceFile = ts.createSourceFile('compatibility.ts', source, ts.ScriptTarget.Latest, true) + if (sourceFile.statements.length !== 1) return false + const [statement] = sourceFile.statements + return ( + ts.isExportDeclaration(statement) && + statement.exportClause === undefined && + Boolean( + statement.moduleSpecifier && + ts.isStringLiteral(statement.moduleSpecifier) && + statement.moduleSpecifier.text === expectedTarget + ) + ) +} + describe('Tape layer boundaries', () => { it('keeps the Tape domain independent from other main-process layers', async () => { const fs = await vi.importActual('node:fs') @@ -94,15 +229,111 @@ describe('Tape layer boundaries', () => { const imports = ts.preProcessFile(source, true, true).importedFiles return imports.flatMap(({ fileName: specifier }) => { - const target = resolveMainImport(file, specifier) - if (!target || isInside(TAPE_DOMAIN_ROOT, target)) return [] - return [`${relativeToMain(file)} -> ${specifier}`] + const violation = getDomainImportViolation(file, specifier) + return violation ? [`${relativeToMain(file)}: ${violation}`] : [] }) }) expect(violations).toEqual([]) }) + it('keeps production code off legacy Tape compatibility imports', async () => { + const fs = await vi.importActual('node:fs') + const violations = listTypeScriptSources(MAIN_SOURCE_ROOT, fs).flatMap((file) => { + const source = fs.readFileSync(file, 'utf8') + return ts + .preProcessFile(source, true, true) + .importedFiles.flatMap(({ fileName: specifier }) => + isLegacyTapeCompatibilityImport(file, specifier) + ? [`${relativeToMain(file)} -> ${specifier}`] + : [] + ) + }) + + expect(violations).toEqual([]) + }) + + it('keeps legacy Tape compatibility modules as canonical re-exports', async () => { + const fs = await vi.importActual('node:fs') + const violations = [...LEGACY_TAPE_COMPATIBILITY_MODULES.entries()].flatMap( + ([relativeModule, expectedTarget]) => { + const file = path.join(MAIN_SOURCE_ROOT, `${relativeModule}.ts`) + const source = fs.readFileSync(file, 'utf8') + return isCanonicalCompatibilityReexport(source, expectedTarget) + ? [] + : [`${relativeModule}.ts must re-export only ${expectedTarget}`] + } + ) + + expect(violations).toEqual([]) + }) + + it('keeps Memory routes on the Tape inspection DTO port', async () => { + const fs = await vi.importActual('node:fs') + const source = fs.readFileSync(MEMORY_ROUTES_FILE, 'utf8') + expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).toEqual([]) + }) + + it.each([ + ['Session', '@/session/data/transcript'], + ['Agent', '@/agent/deepchat/runtime/process'], + ['Memory', '@/memory/routes'], + ['App', '@/app/composition'], + ['Tape ports', '@/tape/ports/capabilities'], + ['Tape SQLite infrastructure', '@/tape/infrastructure/sqlite/tapeEntryStore'], + ['bare SQLite', 'better-sqlite3'], + ['Node SQLite', 'node:sqlite'], + ['Electron', 'electron'], + ['Electron subpath', 'electron/main'], + ['shared logging', '@shared/logger'], + ['Electron logging', 'electron-log'] + ])('detects forbidden %s imports in the Tape domain', (_category, specifier) => { + const importingFile = path.join(TAPE_DOMAIN_ROOT, 'negative-case.ts') + expect(getDomainImportViolation(importingFile, specifier)).not.toBeNull() + }) + + it.each([ + ['domain sibling', './entry'], + ['domain alias', '@/tape/domain/effectiveView'], + ['shared type', '@shared/types/tape-replay'], + ['Node crypto', 'node:crypto'] + ])('allows pure %s imports in the Tape domain', (_category, specifier) => { + const importingFile = path.join(TAPE_DOMAIN_ROOT, 'allowed-case.ts') + expect(getDomainImportViolation(importingFile, specifier)).toBeNull() + }) + + it.each([ + [path.join(MAIN_SOURCE_ROOT, 'agent/example.ts'), '@/session/data/tape'], + [path.join(MAIN_SOURCE_ROOT, 'session/data/index.ts'), './tapeFacts'], + [path.join(MAIN_SOURCE_ROOT, 'memory/example.ts'), '@/session/data/tables/deepchatTapeEntries'], + [ + path.join(MAIN_SOURCE_ROOT, 'app/example.ts'), + '@/session/data/tables/deepchatTapeSearchProjection' + ] + ])('detects legacy Tape compatibility import %s -> %s', (importingFile, specifier) => { + expect(isLegacyTapeCompatibilityImport(importingFile, specifier)).toBe(true) + }) + + it.each([ + [ + 'raw reader capability', + "import type { TapeRawEntryReader } from '@/tape/ports/capabilities'" + ], + [ + 'effective-view helper', + "import { buildEffectiveTapeView } from '@/tape/domain/effectiveView'" + ], + ['application facade', "import { SessionTape } from '@/tape/application/sessionTape'"], + ['inspection value import', "import { TapeInspectionReader } from '@/tape/ports/capabilities'"] + ])('detects Memory route Tape bypass through %s', (_category, source) => { + expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).not.toEqual([]) + }) + + it('allows Memory routes to import only the inspection reader type', () => { + const source = "import type { TapeInspectionReader } from '@/tape/ports/capabilities'" + expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).toEqual([]) + }) + it('allows physical Tape storage access only at explicit infrastructure boundaries', async () => { const fs = await vi.importActual('node:fs') const matchedExceptionCapabilities = new Set() From 9d85ebbb61dfe197982ad05d9a42f44f62cdf312 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 17:35:15 +0800 Subject: [PATCH 13/29] test(tape): guard semantics compatibility path --- test/main/tape/layerBoundaries.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/main/tape/layerBoundaries.test.ts b/test/main/tape/layerBoundaries.test.ts index c979451c24..71ecd013a5 100644 --- a/test/main/tape/layerBoundaries.test.ts +++ b/test/main/tape/layerBoundaries.test.ts @@ -16,6 +16,7 @@ const LEGACY_TAPE_COMPATIBILITY_MODULES = new Map([ ['session/data/tapeEffectiveView', '@/tape/domain/effectiveView'], ['session/data/tapeFacts', '@/tape/application/factPersistence'], ['session/data/tapeViewManifest', '@/tape/domain/viewManifest'], + ['session/data/tables/deepchatTapeEffectiveSemantics', '@/tape/domain/effectiveSemantics'], ['session/data/tables/deepchatTapeEntries', '@/tape/infrastructure/sqlite/tapeEntryStore'], [ 'session/data/tables/deepchatTapeSearchProjection', @@ -305,6 +306,10 @@ describe('Tape layer boundaries', () => { it.each([ [path.join(MAIN_SOURCE_ROOT, 'agent/example.ts'), '@/session/data/tape'], [path.join(MAIN_SOURCE_ROOT, 'session/data/index.ts'), './tapeFacts'], + [ + path.join(MAIN_SOURCE_ROOT, 'memory/example.ts'), + '@/session/data/tables/deepchatTapeEffectiveSemantics' + ], [path.join(MAIN_SOURCE_ROOT, 'memory/example.ts'), '@/session/data/tables/deepchatTapeEntries'], [ path.join(MAIN_SOURCE_ROOT, 'app/example.ts'), From 610261072da61f8c6f6d436b0e4d15834c443a29 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 17:36:41 +0800 Subject: [PATCH 14/29] test(tape): guard project sqlite driver --- test/main/tape/layerBoundaries.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/main/tape/layerBoundaries.test.ts b/test/main/tape/layerBoundaries.test.ts index 71ecd013a5..e75efdc392 100644 --- a/test/main/tape/layerBoundaries.test.ts +++ b/test/main/tape/layerBoundaries.test.ts @@ -26,6 +26,7 @@ const LEGACY_TAPE_COMPATIBILITY_MODULES = new Map([ const FORBIDDEN_DOMAIN_SQLITE_IMPORTS = new Set([ 'better-sqlite3', + 'better-sqlite3-multiple-ciphers', 'bun:sqlite', 'node:sqlite', 'sql.js', @@ -283,6 +284,7 @@ describe('Tape layer boundaries', () => { ['Tape ports', '@/tape/ports/capabilities'], ['Tape SQLite infrastructure', '@/tape/infrastructure/sqlite/tapeEntryStore'], ['bare SQLite', 'better-sqlite3'], + ['project SQLite driver', 'better-sqlite3-multiple-ciphers'], ['Node SQLite', 'node:sqlite'], ['Electron', 'electron'], ['Electron subpath', 'electron/main'], From 12edd0b8ff8bc18d12465f150f9d69d0ad007484 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 17:39:11 +0800 Subject: [PATCH 15/29] fix(tape): invalidate legacy projections --- .../sqlite/tapeSearchProjectionStore.ts | 5 +- test/main/session/data/tapeRecall.test.ts | 63 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts index 0c84a325ab..fdcf5c867d 100644 --- a/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts +++ b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts @@ -15,7 +15,10 @@ import type { TapeSearchProjectionStore } from '@/tape/ports/application' -export const DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION = 2 +// Version 3 invalidates projections that may predate atomic Tape generation transitions. A +// matching entry-id head alone cannot prove that a version 2 row belongs to the current +// incarnation after a previously interrupted reset. +export const DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION = 3 export type DeepChatTapeSearchProjectionInput = TapeSearchProjectionInput export type DeepChatTapeSearchProjectionRow = TapeSearchProjectionRow diff --git a/test/main/session/data/tapeRecall.test.ts b/test/main/session/data/tapeRecall.test.ts index 25309b3075..ec764b3af8 100644 --- a/test/main/session/data/tapeRecall.test.ts +++ b/test/main/session/data/tapeRecall.test.ts @@ -22,6 +22,10 @@ import { } from './tapeTestHarness' describe('SessionTape recall', () => { + it('invalidates projections written before atomic generation transitions', () => { + expect(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION).toBeGreaterThan(2) + }) + it('reports info, search, and handoff within one session scope', () => { const { table, entries } = createTapeTableMock() const service = new SessionTape({ @@ -543,6 +547,65 @@ describe('SessionTape recall', () => { expect(context.entries[0].summary).not.toContain('old private context') }) + itIfSqlite('ignores a pre-transition projection version at the same Tape head', () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + table.createTable() + projectionTable.createTable() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'current-message', seq: 0 }, + payload: { + record: createRecord({ + id: 'current-message', + content: JSON.stringify({ text: 'current generation context', files: [], links: [] }) + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' } + }) + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'old-message', + sourceSeq: 0, + searchText: 'old private context', + summaryText: 'old private context', + refs: { messageId: 'old-message' }, + createdAt: 100 + } + ], + 1, + 2 + ) + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const context = service.getContext('s1', [1], { before: 0, after: 0 }) + + expect(context.entries[0]).toMatchObject({ + entryId: 1, + summary: expect.stringContaining('current generation context'), + refs: { messageId: 'current-message' } + }) + expect(context.entries[0].summary).not.toContain('old private context') + } finally { + db.close() + } + }) + itIfSqlite('ignores stale same-entry projection context when the Tape head has advanced', () => { const db = new DatabaseCtor(':memory:') try { From 922bbed7204bb9db64f091e72b1526224bb7065d Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 17:41:35 +0800 Subject: [PATCH 16/29] refactor(tape): require capability injection --- .../legacyChatImportService.ts | 6 ++- src/main/session/data/settings.ts | 3 +- src/main/session/data/transcript.ts | 6 +-- .../agent/acp/compatibility/adapters.test.ts | 2 +- test/main/session/data/transcript.test.ts | 3 +- test/main/session/usageStatsService.test.ts | 3 +- test/main/tape/layerBoundaries.test.ts | 41 +++++++++++++++++++ 7 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/main/app/startupMigrations/legacyChatImportService.ts b/src/main/app/startupMigrations/legacyChatImportService.ts index 03544175a8..d93bfdb23a 100644 --- a/src/main/app/startupMigrations/legacyChatImportService.ts +++ b/src/main/app/startupMigrations/legacyChatImportService.ts @@ -12,6 +12,7 @@ import { isReasoningEffort } from '@shared/types/model-db' import { resolveAcpAgentAlias } from '@shared/utils/acpAgentAlias' import { SessionTranscript } from '@/session/data/transcript' import { SessionDatabase } from '@/session/data/database' +import { SessionTape } from '@/tape/application/sessionTape' import type { ProjectDatabase } from '@/project/data/database' import type { AppDatabase } from '@/app/data/database' import type { MemoryDatabase } from '@/memory/data/database' @@ -50,7 +51,10 @@ export class LegacyChatImportService { this.sessionDatabase = sessionDatabase this.projectDatabase = projectDatabase this.memoryDatabase = memoryDatabase - this.messageStore = new SessionTranscript(this.sessionDatabase) + this.messageStore = new SessionTranscript( + this.sessionDatabase, + new SessionTape(this.sessionDatabase) + ) this.sourceDbPath = sourceDbPath ?? path.join(app.getPath('userData'), 'app_db', 'chat.db') } diff --git a/src/main/session/data/settings.ts b/src/main/session/data/settings.ts index 289be81b0f..c6831d1821 100644 --- a/src/main/session/data/settings.ts +++ b/src/main/session/data/settings.ts @@ -7,7 +7,6 @@ import type { TapeAnchorWriter, TapeLifecycleAdmin } from '@/tape/ports/capabilities' -import { SessionTape } from '@/tape/application/sessionTape' export type SessionSummaryState = { summaryText: string | null @@ -141,7 +140,7 @@ export class SessionSettingsStore { constructor( database: SessionDatabase, - tape: TapeAnchorReader & TapeAnchorWriter & TapeLifecycleAdmin = new SessionTape(database) + tape: TapeAnchorReader & TapeAnchorWriter & TapeLifecycleAdmin ) { this.database = database this.tape = tape diff --git a/src/main/session/data/transcript.ts b/src/main/session/data/transcript.ts index 4e24893495..f1d9764af3 100644 --- a/src/main/session/data/transcript.ts +++ b/src/main/session/data/transcript.ts @@ -24,7 +24,6 @@ import { resolveUsageProviderId } from '@/session/usageStats' import type { TapeMessageFactWriter } from '@/tape/ports/capabilities' -import { SessionTape } from '@/tape/application/sessionTape' function shouldConvertPendingBlockToError( status: AssistantMessageBlock['status'] @@ -127,10 +126,7 @@ export class SessionTranscript { private database: SessionDatabase private readonly tapeFacts: TapeMessageFactWriter - constructor( - database: SessionDatabase, - tapeFacts: TapeMessageFactWriter = new SessionTape(database) - ) { + constructor(database: SessionDatabase, tapeFacts: TapeMessageFactWriter) { this.database = database this.tapeFacts = tapeFacts } diff --git a/test/main/agent/acp/compatibility/adapters.test.ts b/test/main/agent/acp/compatibility/adapters.test.ts index 1d8c164496..62d7a732b9 100644 --- a/test/main/agent/acp/compatibility/adapters.test.ts +++ b/test/main/agent/acp/compatibility/adapters.test.ts @@ -182,8 +182,8 @@ function createProjectionHarness() { ) } } as unknown as MainDatabase - const messageStore = new SessionTranscript(sqlitePresenter) const tapeService = new SessionTape(sqlitePresenter) + const messageStore = new SessionTranscript(sqlitePresenter, tapeService) const adapter = new AcpCompatibilityProjectionAdapter({ publishEvent: publishDeepchatEvent, publishSessionUpdate: vi.fn(), diff --git a/test/main/session/data/transcript.test.ts b/test/main/session/data/transcript.test.ts index 42e5e55072..bf6252cfc9 100644 --- a/test/main/session/data/transcript.test.ts +++ b/test/main/session/data/transcript.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { SessionTranscript } from '@/session/data/transcript' +import { SessionTape } from '@/tape/application/sessionTape' import { cloneBlocksForRenderer } from '@/agent/deepchat/runtime/echo' import logger from '@shared/logger' @@ -142,7 +143,7 @@ describe('SessionTranscript', () => { beforeEach(() => { sqlitePresenter = createMockSqlitePresenter() - store = new SessionTranscript(sqlitePresenter) + store = new SessionTranscript(sqlitePresenter, new SessionTape(sqlitePresenter)) }) describe('createUserMessage', () => { diff --git a/test/main/session/usageStatsService.test.ts b/test/main/session/usageStatsService.test.ts index ff5a1c7698..a1ba721c41 100644 --- a/test/main/session/usageStatsService.test.ts +++ b/test/main/session/usageStatsService.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { SessionTranscript } from '@/session/data/transcript' +import { SessionTape } from '@/tape/application/sessionTape' import { DASHBOARD_STATS_BACKFILL_KEY, type UsageStatsRecordInput } from '@/session/usageStats' import { UsageStatsService } from '@/session/usageStatsService' import type { PermissionMode } from '@shared/types/agent-interface' @@ -556,7 +557,7 @@ describe('UsageStatsService', () => { it('keeps a single stats row when live finalize updates a previously backfilled message', async () => { const { service, sqlitePresenter } = createService() - const messageStore = new SessionTranscript(sqlitePresenter) + const messageStore = new SessionTranscript(sqlitePresenter, new SessionTape(sqlitePresenter)) sqlitePresenter.deepchatSessionsTable.create('session-1', 'openai', 'gpt-4o', 'full_access') sqlitePresenter.deepchatMessagesTable.insert({ diff --git a/test/main/tape/layerBoundaries.test.ts b/test/main/tape/layerBoundaries.test.ts index e75efdc392..73052322bf 100644 --- a/test/main/tape/layerBoundaries.test.ts +++ b/test/main/tape/layerBoundaries.test.ts @@ -7,6 +7,7 @@ const TAPE_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape') const TAPE_DOMAIN_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape/domain') const TAPE_SQLITE_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape/infrastructure/sqlite') const TAPE_CAPABILITIES_MODULE = path.join(MAIN_SOURCE_ROOT, 'tape/ports/capabilities') +const TAPE_SESSION_FACADE_MODULE = path.join(MAIN_SOURCE_ROOT, 'tape/application/sessionTape') const MEMORY_ROUTES_FILE = path.join(MAIN_SOURCE_ROOT, 'memory/routes.ts') const TAPE_SQLITE_RELATIVE_ROOT = 'tape/infrastructure/sqlite/' const TYPESCRIPT_SOURCE_EXTENSION = /\.[cm]?tsx?$/ @@ -24,6 +25,17 @@ const LEGACY_TAPE_COMPATIBILITY_MODULES = new Map([ ] ]) +const CAPABILITY_SCOPED_CONSUMER_FILES = [ + 'agent/acp/compatibility/adapters.ts', + 'agent/acp/compatibility/dependencies.ts', + 'agent/deepchat/memory/memoryRuntimeCoordinator.ts', + 'agent/deepchat/runtime/deepChatLoopRunner.ts', + 'agent/deepchat/runtime/turnCoordinator.ts', + 'memory/routes.ts', + 'session/data/settings.ts', + 'session/data/transcript.ts' +].map((file) => path.join(MAIN_SOURCE_ROOT, file)) + const FORBIDDEN_DOMAIN_SQLITE_IMPORTS = new Set([ 'better-sqlite3', 'better-sqlite3-multiple-ciphers', @@ -173,6 +185,15 @@ function isTapeModuleImport(importingFile: string, specifier: string): boolean { return Boolean(target && isInside(TAPE_ROOT, target)) } +function findConcreteTapeFacadeImportViolations(source: string, file: string): string[] { + return ts.preProcessFile(source, true, true).importedFiles.flatMap(({ fileName: specifier }) => { + const target = resolveMainImport(file, specifier) + return target && withoutTypeScriptExtension(target) === TAPE_SESSION_FACADE_MODULE + ? [`Concrete Tape facade import: ${specifier}`] + : [] + }) +} + function findMemoryRouteTapeImportViolations(source: string, file: string): string[] { const sourceFile = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true) return sourceFile.statements.flatMap((statement) => { @@ -276,6 +297,17 @@ describe('Tape layer boundaries', () => { expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).toEqual([]) }) + it('keeps capability-scoped consumers off the concrete Tape facade', async () => { + const fs = await vi.importActual('node:fs') + const violations = CAPABILITY_SCOPED_CONSUMER_FILES.flatMap((file) => + findConcreteTapeFacadeImportViolations(fs.readFileSync(file, 'utf8'), file).map( + (violation) => `${relativeToMain(file)}: ${violation}` + ) + ) + + expect(violations).toEqual([]) + }) + it.each([ ['Session', '@/session/data/transcript'], ['Agent', '@/agent/deepchat/runtime/process'], @@ -341,6 +373,15 @@ describe('Tape layer boundaries', () => { expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).toEqual([]) }) + it.each(['@/tape/application/sessionTape', '../../tape/application/sessionTape'])( + 'detects concrete Tape facade import %s in a capability-scoped consumer', + (specifier) => { + const file = path.join(MAIN_SOURCE_ROOT, 'session/data/consumer.ts') + const source = `import { SessionTape } from '${specifier}'` + expect(findConcreteTapeFacadeImportViolations(source, file)).not.toEqual([]) + } + ) + it('allows physical Tape storage access only at explicit infrastructure boundaries', async () => { const fs = await vi.importActual('node:fs') const matchedExceptionCapabilities = new Set() From 468e36a81db7d7612ede50478e53bdf57eda9e72 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 17:43:10 +0800 Subject: [PATCH 17/29] refactor(tape): narrow anchor reader port --- src/main/tape/ports/capabilities.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/tape/ports/capabilities.ts b/src/main/tape/ports/capabilities.ts index 539e6ebd78..568c47fa17 100644 --- a/src/main/tape/ports/capabilities.ts +++ b/src/main/tape/ports/capabilities.ts @@ -62,9 +62,6 @@ export interface TapeRawEntryReader { } export interface TapeAnchorReader { - getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined - getAnchors(sessionId: string, limit?: number): DeepChatTapeEntryRow[] - getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined } From ae52a0624de3ebdce4d7d43f66bafeea392b9c9c Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 17:44:12 +0800 Subject: [PATCH 18/29] refactor(tape): narrow storage protocols --- src/main/tape/ports/application.ts | 1 - src/main/tape/ports/storage.ts | 5 ----- 2 files changed, 6 deletions(-) diff --git a/src/main/tape/ports/application.ts b/src/main/tape/ports/application.ts index 9c2db87387..c1b220f029 100644 --- a/src/main/tape/ports/application.ts +++ b/src/main/tape/ports/application.ts @@ -69,7 +69,6 @@ export interface TapeSearchProjectionStore { maxEntryId: number, projectionVersion?: number ): void - getByEntryIds(sessionId: string, entryIds: number[]): TapeSearchProjectionRow[] getByEntryIdsIfCurrent( sessionId: string, maxEntryId: number, diff --git a/src/main/tape/ports/storage.ts b/src/main/tape/ports/storage.ts index ee1b274fbb..1a33d390a0 100644 --- a/src/main/tape/ports/storage.ts +++ b/src/main/tape/ports/storage.ts @@ -22,15 +22,10 @@ export interface TapeEntryStore { getSubagentLineageEvents(sessionId: string): DeepChatTapeEntryRow[] getFirstEntriesBySessions(sessionIds: string[]): DeepChatTapeEntryRow[] getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] - listMemoryViewManifestAnchorsBySessions( - sessionIds: string[], - optionsOrLimit?: number | { limit?: number; messageId?: string } - ): DeepChatTapeEntryRow[] listMemoryViewManifestAnchorsByAgent( agentId: string, options?: { sessionId?: string; limit?: number; messageId?: string } ): DeepChatTapeEntryRow[] - getEntriesAfter(sessionId: string, entryId: number): DeepChatTapeEntryRow[] getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined getAnchors(sessionId: string, limit?: number): DeepChatTapeEntryRow[] getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined From 4fbf219325c8e4f05176df63698bb05f8f57ea32 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 17:52:25 +0800 Subject: [PATCH 19/29] docs(tape): clarify layer contracts --- ...agent-system-layered-runtime-baseline.json | 27 +++--- docs/architecture/memory-system.md | 7 +- docs/architecture/session-management.md | 14 ++- docs/architecture/tape-layering/plan.md | 95 ++++++++++++++----- docs/architecture/tape-layering/spec.md | 92 +++++++++++++----- docs/architecture/tape-layering/tasks.md | 41 +++++++- docs/architecture/tape-system.md | 30 +++++- 7 files changed, 237 insertions(+), 69 deletions(-) diff --git a/docs/architecture/baselines/agent-system-layered-runtime-baseline.json b/docs/architecture/baselines/agent-system-layered-runtime-baseline.json index 1baa3fce5b..2bbf2caa1c 100644 --- a/docs/architecture/baselines/agent-system-layered-runtime-baseline.json +++ b/docs/architecture/baselines/agent-system-layered-runtime-baseline.json @@ -1,7 +1,7 @@ { "schemaVersion": 2, "goal": "agent-system-layered-runtime", - "headCommit": "d87cb4f69dca7af499e153d9a1d71015e1bfad60", + "headCommit": "ae52a0624de3ebdce4d7d43f66bafeea392b9c9c", "relevantWorkingTree": { "dirty": false, "files": [] @@ -127,11 +127,12 @@ "src/main/agent/shared/storage/sessionPaths.ts", "src/main/provider/providers/acpProvider.ts", "src/main/session/assignment.ts", - "src/main/session/data/tape.ts", "src/main/session/data/transcript.ts", "src/main/session/lifecycle.ts", "src/main/session/query.ts", - "src/main/session/turn.ts" + "src/main/session/turn.ts", + "src/main/tape/application/sessionTape.ts", + "src/main/tape/ports/capabilities.ts" ], "expectedFiles": { "src/main/agent/acp/instance/acpAgentInstance.ts": true, @@ -155,11 +156,12 @@ "src/main/agent/shared/appSessionService.ts": true, "src/main/provider/providers/acpProvider.ts": true, "src/main/session/assignment.ts": true, - "src/main/session/data/tape.ts": true, "src/main/session/data/transcript.ts": true, "src/main/session/lifecycle.ts": true, "src/main/session/query.ts": true, - "src/main/session/turn.ts": true + "src/main/session/turn.ts": true, + "src/main/tape/application/sessionTape.ts": true, + "src/main/tape/ports/capabilities.ts": true }, "ownerEvidence": { "agentManager": { @@ -192,8 +194,8 @@ "exists": true, "declarationCount": 1 }, - "tapeRecorder": { - "file": "src/main/agent/deepchat/loop/ports.ts", + "tapeToolFactWriter": { + "file": "src/main/tape/ports/capabilities.ts", "exists": true, "declarationCount": 1 }, @@ -297,11 +299,12 @@ "src/main/agent/deepchat/runtime/process.ts", "src/main/provider/providers/acpProvider.ts", "src/main/session/assignment.ts", - "src/main/session/data/tape.ts", "src/main/session/data/transcript.ts", "src/main/session/lifecycle.ts", "src/main/session/query.ts", - "src/main/session/turn.ts" + "src/main/session/turn.ts", + "src/main/tape/application/sessionTape.ts", + "src/main/tape/ports/capabilities.ts" ] }, "contracts": { @@ -366,7 +369,7 @@ "src/shared/contracts/routes/window.routes.ts", "src/shared/contracts/routes/workspace.routes.ts" ], - "sha256": "406ef501353454e8677f773c556b8938141d931229eebabae6fcb9b0c37abe3c" + "sha256": "e624008521f11befaa0497a325cb65b4bf38f36e575dd4285622ad3705955b9e" }, "storage": { "sqlite": { @@ -376,7 +379,7 @@ "src/main/data/schemaTypes.ts" ], "tableIdentifiers": [], - "sha256": "411d0c6078cf4264f2241e1198315e52dcf085342975d344e5d8967333b8129b" + "sha256": "3465d92dff615489b7037de640077ce95bda7a7abbb0144ce84bb144dcd85a9e" }, "memoryDuckDbSidecar": { "files": [ @@ -396,7 +399,7 @@ "src/main/app/mainProcess.ts", "src/main/appMain.ts" ], - "sha256": "dc13adbe8b76f0c4bb463605c780fc83e4a78696b8875d6765e767a4c2dcf70f" + "sha256": "8968bb535eeabb3d9a154ca777cc83c7c8ec5183ccae7c165fe11bd51e6f7299" }, "dependencyMetrics": { "loopFiles": [ diff --git a/docs/architecture/memory-system.md b/docs/architecture/memory-system.md index a31fccb23e..5128e74282 100644 --- a/docs/architecture/memory-system.md +++ b/docs/architecture/memory-system.md @@ -25,7 +25,7 @@ flowchart LR - App 负责 shutdown/database maintenance 时的全局 fence 和停止顺序，不解释 Memory 业务状态。 - Memory runtime 通过 `TapeRawEntryReader` 和 `TapeAnchorWriter` 读取执行事实、记录 `memory/view_assembled` 与 `memory/extract` anchor；Memory routes 只通过 `TapeInspectionReader` - 检查 manifest，不接收 Tape table。 + 获取 effective source span 和 manifest DTO，不接收 Tape table 或 raw Tape row。 ## 数据与状态 @@ -91,6 +91,11 @@ projection current 时只 materialize cursor 区间；head 不一致时，runtim 构建 effective Tape view 并重建 projection。projection 查询或重建失败时保留既有 Tape fallback 和 cursor commit 保护，不能因为拆层新增全历史 hot-path 查询，也不能在不完整 projection 上推进 cursor。 +`TapeRawEntryReader` 只提供 `getBySession`。Memory management route 先验证 memory row 属于请求 Agent， +再用 `getEffectiveMessageSourceSpan` 读取 retraction/replacement 生效后的最小 message DTO；manifest +列表通过 `listMemoryViewManifestsByAgent` 在 storage query 中执行 Agent、Session、message 和 limit +过滤，route 不自行解析 `payload_json` 或 `meta_json`。 + ## Privacy 与隔离 - 所有查询显式携带 Agent identity；不得依赖进程全局“当前 Agent”。 diff --git a/docs/architecture/session-management.md b/docs/architecture/session-management.md index 4ec3a411ec..81b8f76ccc 100644 --- a/docs/architecture/session-management.md +++ b/docs/architecture/session-management.md @@ -65,9 +65,21 @@ Session data composition 创建一个 `SessionTape`，对外继续暴露现有 ` SQLite transaction； - settings/compaction 只接收 anchor reader/writer 与 lifecycle admin；summary 更新和 anchor append 继续使用同一个 connection； -- Agent、Memory 和 routes 分别接收自己的最小 Tape capability，不接收物理 table； +- loop runner 分别接收 reconciliation、ViewManifest reader/writer 和 tool fact writer；Turn coordinator + 与 ACP compatibility 只接收 reconciliation；Memory 和 routes 分别接收自己的最小 Tape capability， + 不接收物理 table； - Tape 的运行中修订通过 append 表达；物理 delete/reset 只由 Session lifecycle 触发。 +`SessionTranscript` 和 `SessionSettingsStore` 不提供隐式 `new SessionTape(...)` fallback，必须由正常 +composition 注入共享 connection 上的最小 capability。legacy import 作为 migration composition 显式 +构造同 connection facade 后再注入，避免隐藏的独立 writer 或事务上下文。 + +`clearSessionMessages` 会创建新的 Tape incarnation：entry、mutation projection、search/FTS projection +删除和新 bootstrap 必须在同一个 SQLite transaction 内完成；lifecycle、cleanup 或 bootstrap hard +failure 会完整保留旧 incarnation。mutation projection 沿用 fail-open：新 bootstrap 的 projection +apply 失败时旧 projection row 已删除且 meta 标 stale。最终 Session delete 不创建新 incarnation，继续 +遵循下面的 staged cleanup 顺序。 + ## Binding `DesktopSessionBinding` 维护 `webContentsId -> sessionId`： diff --git a/docs/architecture/tape-layering/plan.md b/docs/architecture/tape-layering/plan.md index da70828d3c..d630c47dc2 100644 --- a/docs/architecture/tape-layering/plan.md +++ b/docs/architecture/tape-layering/plan.md @@ -20,18 +20,26 @@ will target `@/tape/*`. Tape entry rows, append inputs, source identities, entry references, fact provenance, and tool fact inputs move out of Agent and table modules into Tape-owned types. Effective-view selection, -ViewManifest hashing, lineage validation, and replay value conversion remain pure. +ViewManifest hashing, lineage validation, stored-manifest validation, and replay hashing remain +pure. Database row parsing and trace-evidence reads remain in the application layer. The primary ports are: -- `TapeEntryStore`: append, anchor/event append helpers, bootstrap, and read/query operations. It - has no destructive method. +- `TapeEntryStore`: append, anchor/event append helpers, and read/query operations. It has no + destructive method. `TapeBootstrapStore` and `TapeTransactionRunner` are separate application + composition capabilities. - `TapeSearchProjectionStore`: rebuildable search projection behavior. - `TapeToolFactWriter`: the single `appendToolFact` capability used by the Agent loop. - `TapeMessageFactWriter`: message, replacement, and retraction fact operations used by transcript. -- `TapeAnchorReader` and `TapeAnchorWriter`: narrow anchor capabilities for settings and Memory. -- `TapeRawEntryReader`: intentional raw-row access for effective-view rebuilding in Memory. -- `TapeInspectionReader`: effective source spans and Memory ViewManifest inspection queries. +- `TapeReconciliationPort`: bootstrap and transcript reconciliation used by the loop runner, Turn + coordinator, and ACP compatibility projection. +- `TapeViewManifestReader` and `TapeViewManifestWriter`: the manifest capabilities used by the + loop runner without exposing the full facade. +- `TapeAnchorReader`: only the latest reconstruction-anchor read required by settings. +- `TapeAnchorWriter`: narrow anchor append capability for settings and Memory. +- `TapeRawEntryReader`: only `getBySession`, retained for effective-view rebuilding in Memory. +- `TapeInspectionReader`: effective source spans and Memory ViewManifest inspection DTOs; no raw + entry rows cross this boundary. - `TapeLifecycleAdmin`: Session-owned delete and reset operations across entries and projections. - Explicit transcript and trace evidence read ports used by reconciliation and replay. @@ -42,19 +50,20 @@ at each call site. The current `SessionTape` behavior is divided without changing method semantics: -1. **TapeFactService** owns message, tool, event, anchor, handoff, and ViewManifest fact appends. -2. **TapeReconciler** owns bootstrap, legacy transcript backfill, and legacy summary-anchor repair. +1. **TapeFactService** owns message, tool, generic anchor, handoff, and fork-message fact appends. +2. **TapeReconcilerService** owns bootstrap, legacy transcript backfill, and legacy summary-anchor + repair. 3. **TapeRecallService** owns info, search, context windows, anchor listing, and authorized source resolution needed by recall. 4. **TapeLineageService** owns link validation, frozen child heads, authorization, and lineage receipts. -5. **TapeViewReplayService** owns ViewManifest source maps, manifest listing, replay exports, and - explicit trace-evidence reads. -6. **TapeForkService** owns fork creation, isolated writes, delta merge, discard, and external fork +5. **TapeViewReplayService** owns ViewManifest append and source assembly, manifest listing, replay + exports, Memory inspection projection, and explicit trace-evidence reads. +6. **TapeForkService** owns only fork creation, delta merge, discard, and external lifecycle receipts. -`SessionTape` becomes a compatibility facade that constructs or receives these services and -forwards the existing methods. `SessionTapePort` in Session contracts remains unchanged. +`SessionTape` becomes a compatibility facade that constructs these services and forwards the +existing methods. `SessionTapePort` in Session contracts remains unchanged. ## SQLite Infrastructure @@ -63,9 +72,10 @@ query SQL, a normal SQLite entry store, and a lifecycle adapter. Table names, in predicates, provenance uniqueness, and payload serialization remain byte-compatible. Physical entry deletion is removed from the normal entry-store interface. `TapeLifecycleAdmin` -coordinates entry deletion, search projection deletion, and reset bootstrap. Startup legacy import -may continue to execute whole-database cleanup SQL because it rebuilds persisted state before -normal runtime composition. +coordinates entry deletion, search projection deletion, and reset bootstrap. A reset executes +entry and mutation-projection deletion, search and FTS deletion, and new bootstrap creation in one +transaction. Startup legacy import may continue to execute whole-database cleanup SQL because it +rebuilds persisted state before normal runtime composition. The Memory ingestion projection retains its current single SQL statement that compares `MAX(deepchat_tape_entries.entry_id)` with the projection metadata head. Moving this comparison to @@ -85,9 +95,12 @@ SQLite connection -> SessionTape facade and existing SessionTapePort adapter ``` -Runtime composition passes `TapeToolFactWriter` to the loop, raw-row and anchor capabilities to -the Memory coordinator, and `TapeInspectionReader` to Memory routes. No application consumer gets -the concrete entry table. +Runtime composition passes reconciliation, ViewManifest read/write, and tool-fact capabilities to +the loop runner; only reconciliation to the Turn coordinator and ACP adapter; raw-row and anchor +capabilities to the Memory coordinator; and `TapeInspectionReader` to Memory routes. No +application consumer gets the concrete entry table. Transcript and settings have no concrete +facade default: normal composition injects their capabilities from the shared `SessionTape`, and +legacy import composition constructs and injects an explicitly shared-connection facade. `ensureSessionTapeReady` remains at the current Session port boundary. Search and context requests with linked-source scopes keep their existing conditional reconciliation behavior. @@ -98,8 +111,15 @@ with linked-source scopes keep their existing conditional reconciliation behavio that deletes projection rows. - Summary compare-and-set appends its reconstruction anchor inside the same transaction that updates summary state. -- Reset and Session deletion coordinate entry and search-projection cleanup through lifecycle - operations while preserving the current external ordering. +- Reset deletes entries, mutation projection, search projection, FTS metadata, and FTS rows and + appends the new bootstrap within one shared-connection transaction. A propagated transition + failure restores the old incarnation. The pre-existing fail-open mutation-projection append + policy may instead commit the new Tape with that derivative marked stale after old projection + rows have been removed. +- Fork discard performs the same atomic cleanup attempt. Cleanup failure rolls that attempt back + but still appends a fail-closed discard receipt, preserving the non-blocking contract. +- Final Session deletion keeps its staged lifecycle ordering and does not create a replacement + incarnation. - Port implementations use the same connection provider and remain synchronous, so extracting a service does not cross a transaction boundary. @@ -110,6 +130,12 @@ behavior where the current implementation is atomic. - Keep all shared DTOs and `SessionTapePort` signatures unchanged. - Preserve old internal exported symbol names through compatibility re-exports. +- Keep `TapeViewManifestSourceMaps` as a deprecated alias of + `TapeViewManifestAssemblySources`; use the latter for the complete application source set. +- Advance the rebuildable search projection to version 3 so same-head version 2 rows from a + possible interrupted legacy reset are never trusted. The first current-Tape search performs a + one-time rebuild; linked read-only search uses the existing effective-Tape fallback until a + current rebuild exists. - Preserve schema SQL, existing rows, canonical policy identifiers, hashes, source identities, provenance keys, error messages where tested, and bounded query limits. - Preserve projection failure fallback and best-effort fork projection cleanup. @@ -124,7 +150,10 @@ behavior where the current implementation is atomic. fallback, and lifecycle reset. 4. Add contract coverage for append-only correction, frozen-head authorization, fork delta merge, ViewManifest hashes, replay evidence, and projection rebuild equivalence. -5. Add a source-boundary test that rejects domain reverse imports and non-allowlisted table access. +5. Add source-boundary tests that reject domain reverse/runtime imports, production legacy Tape + imports, concrete facade imports from capability-scoped consumers, Memory route capability + expansion, the project SQLite driver, and non-allowlisted table access. Exercise each guard with + table-driven negative fixtures. 6. Run the full main-process suite, Tape scale suite, type checks, formatting, i18n validation, and lint before handoff. @@ -135,7 +164,7 @@ commit, review the complete unstaged and staged diff for hidden side effects, co boundary cases, performance, security, misleading names, missing tests, and long-term maintenance cost. Fix all findings and repeat validation before committing. -The planned commits are: +The initial implementation commits were: 1. `docs(tape): specify layering refactor` 2. `test(tape): split behavior contracts` @@ -143,7 +172,25 @@ The planned commits are: 4. `refactor(tape): split application services` 5. `refactor(tape): close storage bypasses` 6. `test(tape): enforce layer boundaries` -7. `docs(tape): refresh architecture map` +7. `test(tape): align writer mock naming` +8. `docs(tape): refresh architecture map` + +The review remediation commits are: + +1. `fix(tape): make generation resets atomic` +2. `refactor(tape): narrow consumer ports` +3. `refactor(tape): align service ownership` +4. `test(tape): harden layer boundaries` +5. `docs(tape): clarify layer contracts` + +The cumulative review added these focused fixes before the documentation commit: + +1. `test(tape): guard semantics compatibility path` +2. `test(tape): guard project sqlite driver` +3. `fix(tape): invalidate legacy projections` +4. `refactor(tape): require capability injection` +5. `refactor(tape): narrow anchor reader port` +6. `refactor(tape): narrow storage protocols` No commit is pushed. The final review compares the complete branch with `dev`. diff --git a/docs/architecture/tape-layering/spec.md b/docs/architecture/tape-layering/spec.md index eda8c5164f..e4b3a1fdcc 100644 --- a/docs/architecture/tape-layering/spec.md +++ b/docs/architecture/tape-layering/spec.md @@ -2,17 +2,17 @@ ## Background -DeepChat's Tape implementation has strong runtime semantics but weak module boundaries. The main -`SessionTape` implementation combines fact writing, migration and reconciliation, search and -context recall, ViewManifest and replay assembly, subagent lineage, and fork management in one -large module. The SQLite entry table also exposes destructive lifecycle operations beside normal -append and read operations. - -Several consumers bypass `SessionTape` and depend directly on the SQLite table. Transcript writes, -Memory ingestion, Memory management routes, Session settings, and startup migration each use a -different subset of Tape behavior, but the current table-shaped dependency gives them more -authority than they need. Tape types also flow in the wrong direction because the Tape layer -imports Agent loop port types. +Before this refactor, DeepChat's Tape implementation had strong runtime semantics but weak module +boundaries. The main `SessionTape` implementation combined fact writing, migration and +reconciliation, search and context recall, ViewManifest and replay assembly, subagent lineage, +and fork management in one large module. The SQLite entry table also exposed destructive +lifecycle operations beside normal append and read operations. + +Several consumers bypassed `SessionTape` and depended directly on the SQLite table. Transcript +writes, Memory ingestion, Memory management routes, Session settings, and startup migration each +used a different subset of Tape behavior, but the table-shaped dependency gave them more +authority than they needed. Tape types also flowed in the wrong direction because the Tape layer +imported Agent loop port types. This refactor adopts Bub's useful dependency pattern—domain primitives, narrow store protocols, application services, and independent view selection—without copying Bub's simpler schema or @@ -58,34 +58,49 @@ to treat trace evidence as transcript data or to move it into the Tape entry sch - Search projections are rebuildable derivatives. Projection failures retain the existing bounded effective-view fallback. - Destructive Session cleanup is a lifecycle operation, not a normal Tape store operation. +- A reset that creates a new Tape incarnation deletes entries, mutation projection state, and + search projection state and appends the new bootstrap anchor in one SQLite transaction. +- A discarded fork is fail-closed for merge and identifier reuse even when best-effort physical + cleanup fails. ## Capability Boundaries | Consumer | Allowed capability | | --- | --- | -| Agent loop | `TapeToolFactWriter` | +| DeepChat loop runner | `TapeReconciliationPort`, `TapeViewManifestReader`, `TapeViewManifestWriter`, and `TapeToolFactWriter` | +| Turn coordinator and ACP compatibility adapter | `TapeReconciliationPort` | | Session transcript | `TapeMessageFactWriter` | | Memory runtime | `TapeRawEntryReader` and `TapeAnchorWriter` | | Session settings and compaction | `TapeAnchorReader`, `TapeAnchorWriter`, and `TapeLifecycleAdmin` | | Memory management routes | `TapeInspectionReader` | | Session IPC | Existing `SessionTapePort` facade | -A single implementation may satisfy several ports, but each consumer receives only the structural -type it needs. +A single implementation may satisfy several ports, but each consumer receives only the +structural type it needs. `TapeRawEntryReader` exposes only `getBySession`. The inspection port +returns purpose-built effective-message and Memory ViewManifest DTOs; it never returns a physical +Tape row. `TapeAnchorReader` exposes only the latest reconstruction anchor required by settings. +Transcript and settings require these capabilities to be injected; only normal or migration +composition may construct the concrete facade. + +`TapeViewManifestAssemblySources` names the complete source set assembled by the application +service. The old `TapeViewManifestSourceMaps` name remains a deprecated type alias for source +compatibility and must not be confused with the smaller domain lookup map used by pure +ViewManifest builders. ## Direct Storage Access Inventory The implementation must account for every current physical-table access: -- `session/data/tape.ts`: current core implementation; replaced by the new facade and services. +- `session/data/tape.ts`: compatibility re-export of the new facade; production imports use + `@/tape/*` directly. - `session/data/transcript.ts`: legitimate message fact producer; migrated to `TapeMessageFactWriter` while preserving same-connection transactions. - `session/data/settings.ts`: bootstrap, reconstruction-anchor reads, summary/reset anchors, and destructive cleanup; migrated to anchor and lifecycle capabilities. -- `agent/deepchat/runtime/deepChatRuntimeCoordinator.ts`: Memory raw-row reads and anchor writes; - migrated to explicit read and write ports. -- `memory/routes.ts` and app composition: raw table object escape; replaced with domain-level - inspection queries. +- `agent/deepchat/runtime/deepChatRuntimeCoordinator.ts`: composition root that distributes + reconciliation, fact, manifest, raw-read, and anchor capabilities to narrower consumers. +- `memory/routes.ts` and app composition: use `TapeInspectionReader`; effective source spans and + Memory ViewManifest records cross the boundary only as domain DTOs. - `memory/data/tables/deepchatMemoryIngestionProjection.ts`: one-statement freshness comparison between Tape head and projection head; retained as an explicit read-only infrastructure exception to preserve atomicity and query count. @@ -93,10 +108,37 @@ The implementation must account for every current physical-table access: as an explicit startup-migration exception. - Schema catalog and database security table-name lists: metadata, not runtime Tape access. +## Generation and Failure Semantics + +`resetSessionTape` performs entry deletion, mutation-projection deletion, search-projection and FTS +deletion, and new bootstrap creation inside one transaction on the shared Session SQLite +connection. Any propagated lifecycle, cleanup, or bootstrap failure restores the complete prior +incarnation. The existing fail-open mutation-projection append policy remains: if applying the new +bootstrap to that derived projection fails, its metadata is invalidated after old rows have been +removed, so the new Tape can commit without trusting partial projection state. Search-projection +session cleanup is also internally transactional so it cannot leave base, metadata, and FTS rows +at different generations. + +Fork discard makes one atomic cleanup attempt. If cleanup succeeds, cleanup and the discard +receipt commit together. If cleanup fails, the cleanup attempt rolls back, the parent still +appends the discard receipt, and the failure remains non-blocking. Merge checks an existing merge +receipt first for idempotency and then rejects a discard receipt before reading the fork. Creating +a fork with an explicitly discarded identifier also fails closed. + +Context projection reads use `getByEntryIdsIfCurrent`, which verifies projection version and the +projection metadata head against the current Tape head supplied by the synchronous caller in the +same SQL statement that reads the requested rows. A non-current projection is ignored and summary +or reference context is rebuilt from the current effective Tape. Projection version 3 invalidates +version 2 data that may have survived an interrupted pre-atomic reset with the same entry-id head; +current search rebuilds that derivative on demand, while read-only linked search retains its +effective-Tape fallback. + ## Acceptance Criteria -1. `src/main/tape/domain/` does not import Agent, Session, Memory, App, or SQLite infrastructure. -2. Agent loop compilation depends on `TapeToolFactWriter`, not the broad `TapeRecorder` interface. +1. `src/main/tape/domain/` does not import Agent, Session, Memory, App, SQLite, Electron, or logging + runtime modules. +2. Agent execution consumers compile against reconciliation, manifest, and fact ports rather than + concrete `SessionTape` or the deleted broad `TapeRecorder` interface. 3. Runtime, transcript, settings, routes, and normal application composition do not receive a `DeepChatTapeEntriesTable` instance. 4. `TapeEntryStore` exposes no reset or delete method. @@ -105,12 +147,16 @@ The implementation must account for every current physical-table access: 6. Transcript mutation plus Tape correction, and summary mutation plus anchor append, retain their current transaction semantics. 7. `ensureSessionTapeReady` remains idempotent and runs at the same Session port boundaries. -8. Projection search fallback and non-blocking fork projection cleanup remain unchanged. +8. Projection search fallback remains unchanged. Fork cleanup failure remains non-blocking while + its discard receipt prevents later merge or identifier reuse. 9. Baseline Tape tests remain green; the pre-refactor baseline is 120 passed and 26 environment-gated skipped tests across seven files. 10. Tape scale coverage confirms bounded tail materialization and no added full-history query on the Memory projection fast path. -11. Architecture tests reject forbidden imports and new physical-table bypasses. +11. Architecture tests reject forbidden imports, production use of legacy compatibility paths, + concrete facade imports from capability-scoped consumers, Memory route capability expansion, + the actual SQLite driver, and new physical-table bypasses. Negative fixtures prove that each + guard recognizes the prohibited dependency. 12. No remote Git operations are performed as part of this work. ## Constraints diff --git a/docs/architecture/tape-layering/tasks.md b/docs/architecture/tape-layering/tasks.md index c4675eaac0..512b93bb1f 100644 --- a/docs/architecture/tape-layering/tasks.md +++ b/docs/architecture/tape-layering/tasks.md @@ -45,14 +45,47 @@ - [x] Add domain dependency-direction tests. - [x] Add a physical Tape table access guard with a narrow explicit allowlist. -- [x] Run the Tape contract and scale suites. - [x] Review and commit the architecture-guard slice. +## Review Remediation + +- [x] Make reset generation replacement atomic across entries, mutation projection, search + projection, FTS state, and the new bootstrap. +- [x] Make fork cleanup atomic while preserving a non-blocking, fail-closed discard receipt. +- [x] Read context projection rows only when projection version and metadata head match the + synchronous caller's current Tape head in the same SQL statement. +- [x] Replace concrete `SessionTape` dependencies in the loop runner, Turn coordinator, and ACP + compatibility adapter with narrow ports. +- [x] Reduce `TapeRawEntryReader` to the Memory runtime's `getBySession` requirement. +- [x] Replace Memory route raw rows with effective-source and ViewManifest inspection DTOs. +- [x] Remove production imports of legacy Tape compatibility paths. +- [x] Move handoff, generic anchor, and fork-message fact ownership into `TapeFactService`. +- [x] Restrict `TapeForkService` to fork lifecycle behavior. +- [x] Move stored-manifest validation and replay hash helpers into the domain layer. +- [x] Rename the complete source-set DTO to `TapeViewManifestAssemblySources` and retain the old + name as a deprecated alias. +- [x] Harden architecture guards for main-layer, SQLite, Electron, logging, legacy-path, and Memory + route violations, including table-driven negative fixtures. +- [x] Cover every legacy compatibility wrapper and the project's actual SQLite driver in the + boundary guards. +- [x] Invalidate pre-atomic version 2 search projections and verify same-head context fallback. +- [x] Require capability injection for transcript and settings, with concrete facade construction + limited to composition boundaries. +- [x] Keep capability-scoped consumers off the concrete facade through a negative-tested guard. +- [x] Remove unused anchor and storage operations from application-facing protocols while keeping + concrete compatibility methods intact. +- [x] Refresh the canonical agent-system architecture baseline with the new Tape owner evidence. + ## Documentation and Final Validation -- [x] Update Tape, Session, and Memory architecture references. +- [x] Synchronize the English SDD with the implemented capabilities, transactions, ownership, and + failure semantics. +- [x] Update Tape, Session, and Memory architecture references in their existing languages. +- [x] Run the Tape contract and scale suites. - [x] Run the full main-process test suite and Memory performance suite. - [x] Run full type checks, formatting, i18n validation, and lint. +- [x] Scan all three SDD artifacts for non-English prose and unresolved clarification markers. - [x] Review the complete `dev...HEAD` diff and fix every finding. -- [x] Complete the task checklist and commit the final documentation slice. -- [x] Confirm the working tree is clean and the branch has not been pushed. +- [x] Prepare the completed task checklist and final documentation slice for a local commit. +- [x] Confirm no unexpected working-tree files exist and no remote push has been performed before + the final commit. diff --git a/docs/architecture/tape-system.md b/docs/architecture/tape-system.md index f9b7da106e..0b07cc5c41 100644 --- a/docs/architecture/tape-system.md +++ b/docs/architecture/tape-system.md @@ -7,7 +7,7 @@ Subagent lineage；message transcript 是面向 UI 的 projection，不是 Tape | 能力 | 当前 owner | | --- | --- | -| entry/fact/ref、effective semantics、ViewManifest 纯逻辑 | `src/main/tape/domain/` | +| entry/fact/ref、effective semantics、ViewManifest/replay 纯逻辑 | `src/main/tape/domain/` | | 消费方能力和 storage ports | `src/main/tape/ports/` | | Fact、Reconciler、Recall、Lineage、View/Replay、Fork services | `src/main/tape/application/` | | `SessionTape` 兼容 facade | `src/main/tape/application/sessionTape.ts` | @@ -37,7 +37,8 @@ flowchart TD | 消费方 | 允许依赖的 Tape 能力 | | --- | --- | -| Agent loop | `TapeToolFactWriter` | +| DeepChat loop runner | `TapeReconciliationPort`、`TapeViewManifestReader`、`TapeViewManifestWriter`、`TapeToolFactWriter` | +| Turn coordinator / ACP compatibility | `TapeReconciliationPort` | | Transcript | `TapeMessageFactWriter` | | Memory runtime | `TapeRawEntryReader`、`TapeAnchorWriter` | | Settings / compaction | `TapeAnchorReader`、`TapeAnchorWriter`、`TapeLifecycleAdmin` | @@ -48,14 +49,30 @@ flowchart TD IPC boundary 按原时序执行 `ensureSessionTapeReady`。facade 只做 service 组合和兼容转发，不承载新的 domain policy；外部方法的签名、同步/异步行为、异常和 fallback 语义保持稳定。 +`TapeRawEntryReader` 只暴露 Memory runtime 实际需要的 `getBySession`。Memory routes 使用的 +`TapeInspectionReader` 只返回 effective message source span 与 Memory ViewManifest DTO，不返回 +`DeepChatTapeEntryRow`。完整的 manifest assembly source set 命名为 +`TapeViewManifestAssemblySources`；旧 `TapeViewManifestSourceMaps` 仅作为兼容 type alias 保留。 +`TapeAnchorReader` 只暴露 settings 实际使用的 latest reconstruction anchor；transcript/settings 必须 +由 composition 注入 port，不允许在 consumer 内隐式构造 concrete facade。 + ## 存储与事务边界 - `TapeEntryStore` 只负责 append/read/query；物理删除由独立 lifecycle adapter 执行，只服务于 Session lifecycle（包含 fork Session cleanup），不属于运行中 Tape 语义。 - transcript message mutation 与 replacement/retraction fact、summary compare-and-set 与 anchor append 使用同一个 SQLite connection 和调用方 transaction，拆层不能拆开其原子边界。 +- `resetSessionTape` 在同一 transaction 内删除 entry、mutation projection、search/FTS projection 并 + 创建新 bootstrap；lifecycle/cleanup/bootstrap 的 hard failure 会恢复旧 incarnation。既有 mutation + projection append fail-open 策略仍可提交新 Tape，但旧 projection row 已删除且 meta 会标 stale。 +- context projection 通过单条 `getByEntryIdsIfCurrent` SQL 校验 projection version、projection meta head + 与同步调用方提供的 current Tape head，并读取请求行；不 current 时从 effective Tape 重建 + summary/ref context。 +- search projection 升为 version 3，一次性拒绝可能来自 pre-atomic reset、恰好复用相同 head 的 version + 2 row；current search 按需重建，linked read-only search 在重建前沿用 effective-Tape fallback。 - search projection 可以重建；projection 不可用或 coverage 不完整时回退 effective Tape search，fork - cleanup 的 projection 删除失败仍不阻断主流程。 + cleanup 的 projection 删除失败仍不阻断主流程，但 discard receipt 会使后续 merge 和相同 fork ID + 的显式复用 fail closed。 - legacy chat import 的全表删除是 migration-only 例外；Memory ingestion projection 为避免并发窗口， 可以在一条只读 SQL 中同时比较 Tape head 和 projection head。除此之外消费方不得访问物理 Tape 表。 - reset 物理删除当前 Session Tape 后重新 bootstrap；本阶段没有 archive-on-reset，不能把 reset 解释成 @@ -110,7 +127,8 @@ Subagent 使用独立 Session 和独立 Tape。完成后父 Session append 一 普通 fork merge 只把 fork head 相对基线的 delta 作为新 entry append 到父 Tape，并追加 merge receipt； 不得改写父 Tape 旧 entry，也不得把整份 fork 历史重复复制。discard 和重复 merge 保持既有审计、幂等及 -best-effort projection cleanup 语义。 +best-effort projection cleanup 语义。discard cleanup 成功时与 receipt 一起提交；cleanup 失败时回滚本次 +cleanup、仍 append receipt 并记录 warning，残留 fork 也不能再次 merge。 ## 回放和兼容 @@ -118,6 +136,10 @@ Replay 必须保持 entry order、role、tool call/result pairing、anchor curso 可以按兼容规则跳过或映射，但不能静默改变已知 fact 的含义。测试至少覆盖正常 chat、resume、tool interaction、compaction、context pressure、Subagent frozen head 和旧 manifest 读取。 +stored manifest validation、legacy `hashVersion` normalization、entry-id collection 和 replay slice hash +属于 `src/main/tape/domain/replay.ts` 的纯逻辑；SQLite row parsing、message trace 和 terminal evidence +读取仍属于 `TapeViewReplayService`，不能反向放进 domain。 + 关键行为测试位于 `test/main/session/data/tape*.test.ts`，分层守护位于 `test/main/tape/layerBoundaries.test.ts`；runtime 和 tool 契约继续位于 `test/main/agent/deepchat/` 与 `test/main/tool/`。历史的 Tape increment SDD 已合并到本文，详细实施 From 38902efecb410365995f98f5684ed8d161e691c4 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 22:15:55 +0800 Subject: [PATCH 20/29] docs(tape): specify follow-up hardening --- docs/architecture/tape-layering/plan.md | 36 ++++++++++++++++++++---- docs/architecture/tape-layering/spec.md | 24 ++++++++++++++-- docs/architecture/tape-layering/tasks.md | 20 +++++++++++++ 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/docs/architecture/tape-layering/plan.md b/docs/architecture/tape-layering/plan.md index d630c47dc2..61b2ef0fdd 100644 --- a/docs/architecture/tape-layering/plan.md +++ b/docs/architecture/tape-layering/plan.md @@ -119,7 +119,12 @@ with linked-source scopes keep their existing conditional reconciliation behavio - Fork discard performs the same atomic cleanup attempt. Cleanup failure rolls that attempt back but still appends a fail-closed discard receipt, preserving the non-blocking contract. - Final Session deletion keeps its staged lifecycle ordering and does not create a replacement - incarnation. + incarnation. Its Tape entry and search-projection cleanup still runs as one generation + transaction before the Session row is removed. +- FTS is a rebuildable derivative. If session-row deletion from the virtual table fails, the + adapter drops the FTS table and clears freshness metadata inside the same lifecycle transaction; + the next search recreates and repopulates it. Failure to drop the damaged derivative remains a + hard transaction failure rather than committing a mixed generation. - Port implementations use the same connection provider and remain synchronous, so extracting a service does not cross a transaction boundary. @@ -129,9 +134,11 @@ behavior where the current implementation is atomic. ## Compatibility Strategy - Keep all shared DTOs and `SessionTapePort` signatures unchanged. -- Preserve old internal exported symbol names through compatibility re-exports. -- Keep `TapeViewManifestSourceMaps` as a deprecated alias of - `TapeViewManifestAssemblySources`; use the latter for the complete application source set. +- Preserve old internal exported symbol names through explicit, frozen compatibility re-exports + marked as deprecated. +- Use `TapeViewManifestAssemblySources` for the complete application source set and + `TapeViewManifestLookupMaps` for pure domain lookups. Preserve each historical + `TapeViewManifestSourceMaps` shape only at its original legacy import path. - Advance the rebuildable search projection to version 3 so same-head version 2 rows from a possible interrupted legacy reset are never trusted. The first current-Tape search performs a one-time rebuild; linked read-only search uses the existing effective-Tape fallback until a @@ -141,6 +148,23 @@ behavior where the current implementation is atomic. - Preserve projection failure fallback and best-effort fork projection cleanup. - Keep trace evidence distinct from transcript projection in replay dependencies. +## Follow-up Hardening + +1. Replace the deleted monolithic Tape test in the Memory native scope with every split suite that + contains `itIfSqlite` or `describeIfSqlite`, and make the scope validator discover this required + coverage independently. +2. Route final Session Tape deletion through `deleteTapeGeneration` and recover a failed FTS row + delete by invalidating and dropping the derivative within the generation transaction. +3. Prune pre-version-3 and metadata-orphaned projection rows during schema initialization without + rebuilding every current projection eagerly. +4. Freeze legacy shim export surfaces; remove unused concrete-store helpers while retaining the + required compatibility facade methods as deprecated exports. +5. Rename the canonical domain ViewManifest lookup-map type and make Memory route boundary scans + reject static, dynamic, CommonJS, type-import, and re-export bypasses. +6. Reuse the composition-owned Tape fact writer in legacy import, document the same-connection + anchor requirement, cache SQLite FTS capability per connection, and replace exception-by-missing + mock behavior with explicit failure fixtures. + ## Test Strategy 1. Record the current seven-file baseline: 120 passed and 26 native-SQLite-gated skipped tests. @@ -154,7 +178,9 @@ behavior where the current implementation is atomic. imports, concrete facade imports from capability-scoped consumers, Memory route capability expansion, the project SQLite driver, and non-allowlisted table access. Exercise each guard with table-driven negative fixtures. -6. Run the full main-process suite, Tape scale suite, type checks, formatting, i18n validation, and +6. Add native-scope discovery, corrupt-FTS recovery, stale-projection cleanup, bootstrap rollback, + frozen-shim, and non-static Memory route import coverage. +7. Run the full main-process suite, Tape scale suite, type checks, formatting, i18n validation, and lint before handoff. ## Commit and Review Strategy diff --git a/docs/architecture/tape-layering/spec.md b/docs/architecture/tape-layering/spec.md index e4b3a1fdcc..0ebdc2bbc9 100644 --- a/docs/architecture/tape-layering/spec.md +++ b/docs/architecture/tape-layering/spec.md @@ -83,9 +83,10 @@ Transcript and settings require these capabilities to be injected; only normal o composition may construct the concrete facade. `TapeViewManifestAssemblySources` names the complete source set assembled by the application -service. The old `TapeViewManifestSourceMaps` name remains a deprecated type alias for source -compatibility and must not be confused with the smaller domain lookup map used by pure -ViewManifest builders. +service. `TapeViewManifestLookupMaps` names the smaller domain lookup map used by pure +ViewManifest builders. The two historical `TapeViewManifestSourceMaps` exports remain available +only from their respective legacy compatibility modules, where each is an explicit deprecated +alias of the original shape. ## Direct Storage Access Inventory @@ -133,6 +134,18 @@ version 2 data that may have survived an interrupted pre-atomic reset with the s current search rebuilds that derivative on demand, while read-only linked search retains its effective-Tape fallback. +Corrupt or unavailable FTS storage must not block deletion of authoritative Tape or transcript +state. A failed FTS row deletion invalidates FTS metadata and drops the rebuildable virtual table +before the lifecycle transaction continues; if that recovery operation itself cannot complete, +the enclosing Tape generation transaction fails atomically. Startup removes base and FTS +projection rows owned only by pre-version-3 metadata so inert legacy text does not remain on disk +or force linked read-only searches onto permanent fallback paths. + +The Memory native-test manifest must include every split Tape suite that contains an active native +SQLite gate. Scope validation discovers these gated suites independently from the manifest so +removing or renaming one cannot silently eliminate the only real-SQLite lifecycle and FTS CI +coverage. + ## Acceptance Criteria 1. `src/main/tape/domain/` does not import Agent, Session, Memory, App, SQLite, Electron, or logging @@ -158,6 +171,11 @@ effective-Tape fallback. the actual SQLite driver, and new physical-table bypasses. Negative fixtures prove that each guard recognizes the prohibited dependency. 12. No remote Git operations are performed as part of this work. +13. Native Memory CI discovers and executes every SQLite-gated Tape suite after test splitting. +14. Corrupt FTS cleanup cannot leave transcript, Tape entries, and base search projection in + different generations; pre-version-3 projection data is removed during schema initialization. +15. Legacy Tape modules expose frozen deprecated export lists, while canonical modules use + unambiguous ViewManifest source-map names. ## Constraints diff --git a/docs/architecture/tape-layering/tasks.md b/docs/architecture/tape-layering/tasks.md index 512b93bb1f..1c73add7a7 100644 --- a/docs/architecture/tape-layering/tasks.md +++ b/docs/architecture/tape-layering/tasks.md @@ -89,3 +89,23 @@ - [x] Prepare the completed task checklist and final documentation slice for a local commit. - [x] Confirm no unexpected working-tree files exist and no remote push has been performed before the final commit. + +## Post-Review Hardening + +- [ ] Replace the stale monolithic Tape path in Memory native CI with all SQLite-gated split suites. +- [ ] Make scope validation discover required native Tape coverage independently from the manifest. +- [ ] Make final Session Tape deletion reuse the atomic generation lifecycle helper. +- [ ] Recover failed FTS row deletion by dropping and invalidating the rebuildable derivative. +- [ ] Remove pre-version-3 and metadata-orphaned search projection rows at schema initialization. +- [ ] Add direct reset-bootstrap rollback and corrupt-FTS lifecycle regression tests. +- [ ] Freeze and deprecate legacy Tape compatibility export lists without removing old symbols. +- [ ] Use distinct canonical names for application assembly sources and domain lookup maps. +- [ ] Remove unused concrete storage helpers that are not compatibility contracts. +- [ ] Detect dynamic import, CommonJS require, type import, and re-export Memory route bypasses. +- [ ] Reuse the composition-owned Tape message writer in legacy import. +- [ ] Document the same-connection transaction requirement for settings anchor writers. +- [ ] Cache FTS capability detection per SQLite connection. +- [ ] Replace fallback tests that depend on missing mock methods with explicit failures. +- [ ] Document permanent fork residue when best-effort discard cleanup fails. +- [ ] Run focused native scope, lifecycle, recall, boundary, migration, and compatibility tests. +- [ ] Run full validation and review every new local commit without pushing. From 10604a3c802984eab7e2e84474764bf8a9635cb3 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 22:19:21 +0800 Subject: [PATCH 21/29] test(memory): restore native tape scope --- scripts/check-memory-test-scope.mjs | 25 +++++++- .../deepchatMemoryIngestionProjection.test.ts | 13 +++-- test/main/scripts/memoryTestScope.test.ts | 58 ++++++++++++++++++- test/main/session/data/tapeLineage.test.ts | 6 +- test/memory-test-scope.json | 7 ++- 5 files changed, 98 insertions(+), 11 deletions(-) diff --git a/scripts/check-memory-test-scope.mjs b/scripts/check-memory-test-scope.mjs index 993f806530..ff50fe4913 100644 --- a/scripts/check-memory-test-scope.mjs +++ b/scripts/check-memory-test-scope.mjs @@ -6,6 +6,9 @@ const CATEGORY_NAMES = ['behavior', 'native', 'eval', 'perf'] const MEMORY_IMPORT = /from ['"][^'"]*(?:\/memory(?:\/|['"])|agent-memory|memory\.routes|agentMemory|recallKeyword)[^'"]*['"]/ const TEST_FILE = /\.(?:test|spec)\.[cm]?[jt]sx?$|\.perf\.[cm]?[jt]sx?$/ +const NATIVE_TAPE_TEST = + /^test\/main\/session\/data\/(?:tape[^/]*\.test\.ts|tables\/deepchatTapeEntriesTable\.test\.ts)$/ +const NATIVE_SQLITE_GATE = /\b(?:itIfSqlite|describeIfSqlite)\b/ function walkTestFiles(rootDir) { const testRoot = join(rootDir, 'test', 'main') @@ -30,11 +33,24 @@ function isMemoryOwned(path, readContent) { return MEMORY_IMPORT.test(readContent(path)) } +export function findRequiredNativeTapeTests(paths, readContent) { + return paths.filter( + (path) => NATIVE_TAPE_TEST.test(path) && NATIVE_SQLITE_GATE.test(readContent(path)) + ) +} + +function discoverRequiredNativeTapeTests(rootDir) { + return findRequiredNativeTapeTests(walkTestFiles(rootDir), (path) => + readFileSync(join(rootDir, path), 'utf8') + ) +} + export function validateMemoryTestScope({ rootDir, manifest, existingPaths, discoveredPaths, + requiredNativePaths = [], fileContents, readFile }) { @@ -101,6 +117,12 @@ export function validateMemoryTestScope({ } } + for (const path of requiredNativePaths) { + if (ownerByPath.get(path) !== 'native') { + errors.push(`Required native Tape test is not classified as native: ${path}`) + } + } + return errors.sort() } @@ -108,7 +130,8 @@ function runCli() { const scriptDirectory = dirname(fileURLToPath(import.meta.url)) const rootDir = resolve(scriptDirectory, '..') const manifest = JSON.parse(readFileSync(join(rootDir, 'test/memory-test-scope.json'), 'utf8')) - const errors = validateMemoryTestScope({ rootDir, manifest }) + const requiredNativePaths = discoverRequiredNativeTapeTests(rootDir) + const errors = validateMemoryTestScope({ rootDir, manifest, requiredNativePaths }) if (errors.length > 0) { console.error(['Memory test scope validation failed:', ...errors.map((error) => `- ${error}`)].join('\n')) process.exitCode = 1 diff --git a/test/main/memory/deepchatMemoryIngestionProjection.test.ts b/test/main/memory/deepchatMemoryIngestionProjection.test.ts index 5549ae0550..564875c303 100644 --- a/test/main/memory/deepchatMemoryIngestionProjection.test.ts +++ b/test/main/memory/deepchatMemoryIngestionProjection.test.ts @@ -1,5 +1,6 @@ import { describe, expect, vi } from 'vitest' import { buildEffectiveTapeView } from '@/session/data/tapeEffectiveView' +import { SqliteTapeLifecycleAdapter } from '@/tape/infrastructure/sqlite/tapeLifecycleAdapter' import { Database, nativeSqliteItIf } from '../nativeSqliteHarness' const entriesModule = Database ? await import('@/session/data/tables/deepchatTapeEntries') : null @@ -25,7 +26,8 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { projection.createTable() const tape = new TapeTableCtor(db, projection) tape.createTable() - return { db, projection, tape } + const lifecycle = new SqliteTapeLifecycleAdapter(db, projection) + return { db, lifecycle, projection, tape } } function appendMessage( @@ -243,7 +245,7 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { itIfSqlite( 'marks retractions stale and keeps a final tool-before-message sequence current', () => { - const { db, projection, tape } = createTables() + const { db, lifecycle, projection, tape } = createTables() try { appendMessage(tape, { id: 'm1', @@ -259,8 +261,7 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { }) expect(projection.isCurrent('s1', tape.getMaxEntryId('s1'))).toBe(false) - projection.deleteBySession('s1') - tape.deleteBySession('s1') + lifecycle.deleteBySession('s1') appendToolCall(tape, { messageId: 'future-message', toolCallId: 'tool-before-message', @@ -434,7 +435,7 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { ) itIfSqlite('cleans projection rows and meta with the authoritative session delete', () => { - const { db, projection, tape } = createTables() + const { db, lifecycle, projection, tape } = createTables() try { appendMessage(tape, { id: 'm1', @@ -442,7 +443,7 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { status: 'sent', content: 'first' }) - tape.deleteBySession('s1') + lifecycle.deleteBySession('s1') expect(tape.getBySession('s1')).toEqual([]) expect(projection.listRange('s1', 0, 10)).toEqual([]) diff --git a/test/main/scripts/memoryTestScope.test.ts b/test/main/scripts/memoryTestScope.test.ts index d7c6d1604b..4ecbdad881 100644 --- a/test/main/scripts/memoryTestScope.test.ts +++ b/test/main/scripts/memoryTestScope.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it, vi } from 'vitest' -import { validateMemoryTestScope } from '../../../scripts/check-memory-test-scope.mjs' +import { + findRequiredNativeTapeTests, + validateMemoryTestScope +} from '../../../scripts/check-memory-test-scope.mjs' const rootDir = process.cwd() const baseManifest = { @@ -120,4 +123,57 @@ describe('memory test scope guard', () => { ).toEqual([]) expect(readFile).not.toHaveBeenCalled() }) + + it('requires discovered native Tape tests to stay in the native scope', () => { + const requiredPath = 'tape-native.test.ts' + const paths = new Set([...existingPaths, requiredPath]) + const contents = new Map([...fileContents, [requiredPath, 'export {}']]) + + expect( + validateMemoryTestScope({ + rootDir, + manifest: baseManifest, + existingPaths: paths, + discoveredPaths: [], + requiredNativePaths: [requiredPath], + fileContents: contents + }) + ).toContainEqual(expect.stringContaining('not classified as native')) + + const manifest = structuredClone(baseManifest) + manifest.native.push(requiredPath) + expect( + validateMemoryTestScope({ + rootDir, + manifest, + existingPaths: paths, + discoveredPaths: [], + requiredNativePaths: [requiredPath], + fileContents: contents + }) + ).toEqual([]) + }) + + it('discovers only split Tape suites with native SQLite gates', () => { + const paths = [ + 'test/main/session/data/tapeRecall.test.ts', + 'test/main/session/data/tapeReconciler.test.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts', + 'test/main/memory/agentMemoryTable.test.ts' + ] + const contents = new Map([ + ['test/main/session/data/tapeRecall.test.ts', 'itIfSqlite(\'uses SQLite\', () => {})'], + ['test/main/session/data/tapeReconciler.test.ts', 'it(\'stays portable\', () => {})'], + [ + 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts', + 'describeIfSqlite(\'uses SQLite\', () => {})' + ], + ['test/main/memory/agentMemoryTable.test.ts', 'itIfSqlite(\'outside Tape\', () => {})'] + ]) + + expect(findRequiredNativeTapeTests(paths, (path) => contents.get(path) ?? '')).toEqual([ + 'test/main/session/data/tapeRecall.test.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts' + ]) + }) }) diff --git a/test/main/session/data/tapeLineage.test.ts b/test/main/session/data/tapeLineage.test.ts index 7b35c9b66a..6c13bd63b0 100644 --- a/test/main/session/data/tapeLineage.test.ts +++ b/test/main/session/data/tapeLineage.test.ts @@ -9,7 +9,8 @@ import { itIfSqlite, createTapeTableMock, createLinkedTapeService, - createSubagentLinkInput + createSubagentLinkInput, + SqliteTapeLifecycleAdapter } from './tapeTestHarness' describe('SessionTape lineage', () => { @@ -602,6 +603,7 @@ describe('SessionTape lineage', () => { const db = new DatabaseCtor(':memory:') try { const table = new DeepChatTapeEntriesTable(db) + const lifecycle = new SqliteTapeLifecycleAdapter(db) table.createTable() const { service } = createLinkedTapeService(table, [ { id: 'parent', session_kind: 'regular', parent_session_id: null }, @@ -616,7 +618,7 @@ describe('SessionTape lineage', () => { }) service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - table.deleteBySession('child') + lifecycle.deleteBySession('child') table.ensureBootstrapAnchor('child') table.appendEvent({ sessionId: 'child', diff --git a/test/memory-test-scope.json b/test/memory-test-scope.json index f0a41a447f..25630eaf77 100644 --- a/test/memory-test-scope.json +++ b/test/memory-test-scope.json @@ -51,7 +51,12 @@ "test/main/memory/agentMemoryAuditRetention.test.ts", "test/main/memory/agentMemoryPaginationNative.test.ts", "test/main/memory/agentMemoryTable.test.ts", - "test/main/session/data/tape.test.ts", + "test/main/session/data/tables/deepchatTapeEntriesTable.test.ts", + "test/main/session/data/tapeFork.test.ts", + "test/main/session/data/tapeLifecycle.test.ts", + "test/main/session/data/tapeLineage.test.ts", + "test/main/session/data/tapeRecall.test.ts", + "test/main/session/data/tapeViewReplay.test.ts", "test/main/memory/memoryNativeMigration.test.ts", "test/main/memory/memoryVectorStoreV2Native.test.ts", "test/main/memory/memoryUpdateNative.test.ts", From 52e0c4824cc1bcfe198df2330609ceed654997b3 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 22:29:14 +0800 Subject: [PATCH 22/29] test(tape): restore layered settings fixture --- test/main/session/data/settings.test.ts | 100 +++++++++++++----------- 1 file changed, 55 insertions(+), 45 deletions(-) diff --git a/test/main/session/data/settings.test.ts b/test/main/session/data/settings.test.ts index 2773200a3e..76f9394c57 100644 --- a/test/main/session/data/settings.test.ts +++ b/test/main/session/data/settings.test.ts @@ -7,12 +7,22 @@ const sqlitePresenterModule = sqliteModule const sessionStoreModule = sqliteModule ? await import('../../../../src/main/session/data/settings') : null +const sessionDatabaseModule = sqliteModule + ? await import('../../../../src/main/session/data/database') + : null +const sessionTapeModule = sqliteModule + ? await import('../../../../src/main/tape/application/sessionTape') + : null const Database = sqliteModule?.default const MainDatabase = sqlitePresenterModule?.MainDatabase const SessionSettingsStore = sessionStoreModule?.SessionSettingsStore +const SessionDatabase = sessionDatabaseModule?.SessionDatabase +const SessionTape = sessionTapeModule?.SessionTape const MainDatabaseCtor = MainDatabase! const SessionSettingsStoreCtor = SessionSettingsStore! +const SessionDatabaseCtor = SessionDatabase! +const SessionTapeCtor = SessionTape! let sqliteAvailable = false if (Database) { @@ -29,18 +39,20 @@ const describeIfSqlite = sqliteAvailable ? describe : describe.skip describeIfSqlite('SessionSettingsStore tape summary state', () => { function createStore() { - const sqlitePresenter = new MainDatabaseCtor(':memory:') - const store = new SessionSettingsStoreCtor(sqlitePresenter) - return { sqlitePresenter, store } + const connection = new MainDatabaseCtor(':memory:') + const database = new SessionDatabaseCtor(connection) + const tape = new SessionTapeCtor(database) + const store = new SessionSettingsStoreCtor(database, tape) + return { connection, database, store } } it('creates a bootstrap anchor for each session', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') - store.create('s2', 'openai', 'gpt-4o-mini') + store.create('s1', 'openai', 'gpt-4o', 'full_access') + store.create('s2', 'openai', 'gpt-4o-mini', 'full_access') - expect(sqlitePresenter.deepchatTapeEntriesTable.getBySession('s1')).toMatchObject([ + expect(database.deepchatTapeEntriesTable.getBySession('s1')).toMatchObject([ { session_id: 's1', entry_id: 1, @@ -48,7 +60,7 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { name: 'session/start' } ]) - expect(sqlitePresenter.deepchatTapeEntriesTable.getBySession('s2')).toMatchObject([ + expect(database.deepchatTapeEntriesTable.getBySession('s2')).toMatchObject([ { session_id: 's2', entry_id: 1, @@ -57,13 +69,13 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { } ]) - sqlitePresenter.close() + connection.close() }) it('prefers compaction summary anchors over legacy summary columns', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', { summaryText: 'legacy summary', summaryCursorOrderSeq: 2, @@ -101,24 +113,24 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { } }) expect(store.getSummaryState('s1')).toEqual(result.currentState) - expect(sqlitePresenter.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toMatchObject({ + expect(database.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toMatchObject({ name: 'compaction/manual', created_at: 100 }) - sqlitePresenter.close() + connection.close() }) it('uses handoff anchors as context reconstruction state', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', { summaryText: 'legacy summary', summaryCursorOrderSeq: 2, summaryUpdatedAt: 50 }) - sqlitePresenter.deepchatTapeEntriesTable.appendAnchor({ + database.deepchatTapeEntriesTable.appendAnchor({ sessionId: 's1', name: 'handoff/manual', state: { @@ -134,14 +146,14 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryUpdatedAt: 120 }) - sqlitePresenter.close() + connection.close() }) it('uses handoff cursor even when handoff state has no summary', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') - sqlitePresenter.deepchatTapeEntriesTable.appendAnchor({ + store.create('s1', 'openai', 'gpt-4o', 'full_access') + database.deepchatTapeEntriesTable.appendAnchor({ sessionId: 's1', name: 'handoff/manual', state: { @@ -157,19 +169,19 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryUpdatedAt: null }) - sqlitePresenter.close() + connection.close() }) it('compares summary state against tape reconstruction anchors before writing compaction anchors', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', { summaryText: 'legacy summary', summaryCursorOrderSeq: 2, summaryUpdatedAt: 50 }) - sqlitePresenter.deepchatTapeEntriesTable.appendAnchor({ + database.deepchatTapeEntriesTable.appendAnchor({ sessionId: 's1', name: 'handoff/manual', state: { @@ -208,21 +220,19 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryUpdatedAt: 200 } }) - expect( - sqlitePresenter.deepchatTapeEntriesTable.getLatestReconstructionAnchor('s1') - ).toMatchObject({ + expect(database.deepchatTapeEntriesTable.getLatestReconstructionAnchor('s1')).toMatchObject({ name: 'compaction/auto', created_at: 200 }) - sqlitePresenter.close() + connection.close() }) it('does not apply no-anchor summary updates over tape-backed state', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') - sqlitePresenter.deepchatTapeEntriesTable.appendAnchor({ + store.create('s1', 'openai', 'gpt-4o', 'full_access') + database.deepchatTapeEntriesTable.appendAnchor({ sessionId: 's1', name: 'handoff/manual', state: { @@ -256,13 +266,13 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { }) expect(store.getSummaryState('s1')).toEqual(result.currentState) - sqlitePresenter.close() + connection.close() }) it('does not write a stale anchor when summary compare-and-set fails', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', { summaryText: 'newer summary', summaryCursorOrderSeq: 5, @@ -298,22 +308,22 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryUpdatedAt: 200 } }) - expect(sqlitePresenter.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toBeUndefined() + expect(database.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toBeUndefined() - sqlitePresenter.close() + connection.close() }) it('rolls back summary state when the matching tape anchor cannot be appended', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() const originalState = { summaryText: 'stable summary', summaryCursorOrderSeq: 3, summaryUpdatedAt: 50 } - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', originalState) - sqlitePresenter.getDatabase().exec(` + database.getDatabase().exec(` CREATE TRIGGER fail_compaction_anchor BEFORE INSERT ON deepchat_tape_entries WHEN NEW.name = 'compaction/failing' @@ -342,15 +352,15 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { ).toThrow('forced tape anchor failure') expect(store.getSummaryState('s1')).toEqual(originalState) - expect(sqlitePresenter.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toBeUndefined() + expect(database.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toBeUndefined() - sqlitePresenter.close() + connection.close() }) it('uses reset anchors to invalidate older compaction anchors', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.compareAndSetSummaryState( 's1', { @@ -379,10 +389,10 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryCursorOrderSeq: 1, summaryUpdatedAt: null }) - expect(sqlitePresenter.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toMatchObject({ + expect(database.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toMatchObject({ name: 'summary/reset' }) - sqlitePresenter.close() + connection.close() }) }) From c69690720aeac46047eada8eebc30c0458d9ff6f Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 22:30:34 +0800 Subject: [PATCH 23/29] fix(tape): harden generation cleanup --- src/main/app/composition.ts | 3 +- src/main/session/transcriptMutations.ts | 9 +- src/main/tape/application/sessionTape.ts | 5 +- .../sqlite/tapeSearchProjectionStore.ts | 110 ++++++-- .../deepChatRuntimeCoordinator.test.ts | 4 +- test/main/session/data/tapeLifecycle.test.ts | 238 +++++++++++++++++- test/main/session/data/tapeRecall.test.ts | 98 ++++++++ test/main/session/data/tapeTestHarness.ts | 4 +- test/main/session/runtimeIntegration.test.ts | 3 +- test/main/session/transcriptMutations.test.ts | 79 ++++++ 10 files changed, 527 insertions(+), 26 deletions(-) create mode 100644 test/main/session/transcriptMutations.test.ts diff --git a/src/main/app/composition.ts b/src/main/app/composition.ts index f20d5362eb..554ad0af65 100644 --- a/src/main/app/composition.ts +++ b/src/main/app/composition.ts @@ -986,7 +986,8 @@ export async function createMainProcessControl(dependencies: { transcript: sessionData.transcript, settings: sessionData.settings, pendingInputs: sessionData.pendingInputs, - runtime: deepChatRuntimeCoordinator + runtime: deepChatRuntimeCoordinator, + runInTransaction: (operation) => sessionData.database.getDatabase().transaction(operation)() }) memoryIngestionObserver = deepChatRuntimeCoordinator.memoryIngestionObserver acpAgentRuntime = new AcpAgentRuntime( diff --git a/src/main/session/transcriptMutations.ts b/src/main/session/transcriptMutations.ts index eb15255dee..c3bbf21d27 100644 --- a/src/main/session/transcriptMutations.ts +++ b/src/main/session/transcriptMutations.ts @@ -20,6 +20,7 @@ export interface SessionTranscriptMutationDependencies { settings: SessionSettingsStore pendingInputs: SessionPendingInputs runtime: SessionTranscriptRuntimePort + runInTransaction(operation: () => T): T } export class SessionTranscriptMutations { @@ -27,9 +28,11 @@ export class SessionTranscriptMutations { async clearMessages(sessionId: string): Promise { await this.dependencies.runtime.prepareClearMessages(sessionId) - this.dependencies.pendingInputs.deleteBySession(sessionId) - this.dependencies.transcript.deleteBySession(sessionId) - this.dependencies.settings.resetTape(sessionId) + this.dependencies.runInTransaction(() => { + this.dependencies.pendingInputs.deleteBySession(sessionId) + this.dependencies.transcript.deleteBySession(sessionId) + this.dependencies.settings.resetTape(sessionId) + }) this.dependencies.runtime.finishClearMessages(sessionId) } diff --git a/src/main/tape/application/sessionTape.ts b/src/main/tape/application/sessionTape.ts index 63521b0527..120d11bd9a 100644 --- a/src/main/tape/application/sessionTape.ts +++ b/src/main/tape/application/sessionTape.ts @@ -52,7 +52,7 @@ import type { } from './contracts' import { normalizeTapeHandoffState, TapeFactService } from './factService' import { TapeForkService } from './forkService' -import { resetTapeGeneration } from './generationLifecycle' +import { deleteTapeGeneration, resetTapeGeneration } from './generationLifecycle' import { AgentTapeViewError, normalizeSubagentTapeLinkInput, @@ -287,8 +287,7 @@ export class SessionTape } deleteSessionTape(sessionId: string): void { - this.providers.getEntryLifecycleStore().deleteBySession(sessionId) - this.providers.getSearchProjectionStore().deleteBySession(sessionId) + deleteTapeGeneration(this.providers, sessionId) } resetSessionTape(sessionId: string): void { diff --git a/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts index fdcf5c867d..f476bc7817 100644 --- a/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts +++ b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts @@ -28,6 +28,8 @@ export type DeepChatTapeSearchProjectionReadResult = TapeSearchProjectionReadRes type FtsCapability = { available: boolean; tokenizer: 'trigram' | 'unicode61' } +const FTS_CAPABILITY_BY_DATABASE = new WeakMap() + const TAPE_SEARCH_PROJECTION_INDEX_SQL = ` CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_kind ON deepchat_tape_search_projection(session_id, kind, entry_id); @@ -99,6 +101,7 @@ export class DeepChatTapeSearchProjectionTable override createTable(): void { this.db.exec(this.getCreateTableSQL()) + this.pruneInvalidProjectionRows() if (!this.ftsReady) { this.ensureFtsIndex() } @@ -444,16 +447,23 @@ export class DeepChatTapeSearchProjectionTable } clearAll(): void { - this.db.prepare('DELETE FROM deepchat_tape_search_projection').run() - this.db.prepare('DELETE FROM deepchat_tape_search_projection_meta').run() - if (this.ftsMetaTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() - } - this.clearFts() + this.db.transaction(() => { + this.db.prepare('DELETE FROM deepchat_tape_search_projection').run() + this.db.prepare('DELETE FROM deepchat_tape_search_projection_meta').run() + if (this.ftsMetaTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() + } + this.clearFts(true) + })() } private detectFtsCapability(): FtsCapability { if (this.ftsCapability) return this.ftsCapability + const cached = FTS_CAPABILITY_BY_DATABASE.get(this.db) + if (cached) { + this.ftsCapability = cached + return cached + } const probe = (tokenizer: string): boolean => { const name = `temp.tape_search_fts_probe_${tokenizer}` try { @@ -469,6 +479,7 @@ export class DeepChatTapeSearchProjectionTable if (probe('trigram')) this.ftsCapability = { available: true, tokenizer: 'trigram' } else if (probe('unicode61')) this.ftsCapability = { available: true, tokenizer: 'unicode61' } else this.ftsCapability = { available: false, tokenizer: 'unicode61' } + FTS_CAPABILITY_BY_DATABASE.set(this.db, this.ftsCapability) return this.ftsCapability } @@ -677,14 +688,10 @@ export class DeepChatTapeSearchProjectionTable } dropFtsForTesting(): void { - this.db.exec('DROP TABLE IF EXISTS deepchat_tape_search_fts') - if (this.ftsMetaTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() - } - this.ftsReady = false + this.dropFtsIndex() } - private deleteSessionFts(sessionId: string, strict = false): void { + private deleteSessionFts(sessionId: string, dropOnFailure = false): void { if (this.ftsMetaTableExists()) { this.db .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') @@ -693,13 +700,15 @@ export class DeepChatTapeSearchProjectionTable if (!this.ftsTableExists()) return try { this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) - } catch (error) { + } catch { this.ftsReady = false - if (strict) throw error + if (dropOnFailure) { + this.dropFtsIndex() + } } } - private clearFts(): void { + private clearFts(dropOnFailure = false): void { if (this.ftsMetaTableExists()) { this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() } @@ -708,9 +717,80 @@ export class DeepChatTapeSearchProjectionTable this.db.prepare('DELETE FROM deepchat_tape_search_fts').run() } catch { this.ftsReady = false + if (dropOnFailure) { + this.dropFtsIndex() + } + } + } + + private dropFtsIndex(): void { + this.ftsReady = false + this.db.exec('DROP TABLE IF EXISTS deepchat_tape_search_fts') + if (this.ftsMetaTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() } } + private pruneInvalidProjectionRows(): void { + this.db.transaction(() => { + this.db + .prepare( + `DELETE FROM deepchat_tape_search_projection + WHERE NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_search_projection_meta AS meta + WHERE meta.session_id = deepchat_tape_search_projection.session_id + AND meta.projection_version >= ? + )` + ) + .run(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION) + this.db + .prepare( + `DELETE FROM deepchat_tape_search_projection_meta + WHERE projection_version < ?` + ) + .run(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION) + + if (this.ftsTableExists()) { + try { + this.db + .prepare( + `DELETE FROM deepchat_tape_search_fts + WHERE NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_search_fts_meta AS meta + WHERE meta.session_id = deepchat_tape_search_fts.session_id + AND meta.projection_version >= ? + ) + OR NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_search_projection_meta AS meta + WHERE meta.session_id = deepchat_tape_search_fts.session_id + AND meta.projection_version >= ? + )` + ) + .run(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION) + } catch { + this.dropFtsIndex() + } + } + if (this.ftsMetaTableExists()) { + this.db + .prepare( + `DELETE FROM deepchat_tape_search_fts_meta + WHERE projection_version < ? + OR NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_search_projection_meta AS projection_meta + WHERE projection_meta.session_id = deepchat_tape_search_fts_meta.session_id + AND projection_meta.projection_version >= ? + )` + ) + .run(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION) + } + })() + } + private searchFts( sessionId: string, normalized: string, diff --git a/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts b/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts index f72295de49..c85b15a1ca 100644 --- a/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts +++ b/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts @@ -877,7 +877,9 @@ describe('DeepChatRuntimeCoordinator', () => { transcript: sessionData.transcript, settings: sessionData.settings, pendingInputs: sessionData.pendingInputs, - runtime: agent + runtime: agent, + runInTransaction: (operation) => + sessionData.database.getDatabase().transaction(operation)() }) }) diff --git a/test/main/session/data/tapeLifecycle.test.ts b/test/main/session/data/tapeLifecycle.test.ts index a2c768a4b6..87a3493837 100644 --- a/test/main/session/data/tapeLifecycle.test.ts +++ b/test/main/session/data/tapeLifecycle.test.ts @@ -10,6 +10,7 @@ import { SqliteTapeLifecycleAdapter, vi } from './tapeTestHarness' +import { SessionTranscriptMutations } from '@/session/transcriptMutations' function createHarness() { const calls: string[] = [] @@ -53,11 +54,22 @@ function createHarness() { describe('SessionTape lifecycle administration', () => { it('deletes entries before the search projection', () => { - const { calls, tape } = createHarness() + const { calls, entryStore, tape } = createHarness() tape.deleteSessionTape('s1') expect(calls).toEqual(['entries:s1', 'search:s1']) + expect(entryStore.runInTransaction).toHaveBeenCalledOnce() + }) + + it('rolls back final Tape deletion when search cleanup fails', () => { + const { searchProjection, state, tape } = createHarness() + searchProjection.deleteBySession.mockImplementationOnce(() => { + throw new Error('search cleanup failed') + }) + + expect(() => tape.deleteSessionTape('s1')).toThrow('search cleanup failed') + expect(state).toEqual({ entriesPresent: true, searchPresent: true, bootstrapCount: 0 }) }) it('rebuilds the bootstrap only after both destructive stores are cleared', () => { @@ -80,6 +92,16 @@ describe('SessionTape lifecycle administration', () => { expect(state).toEqual({ entriesPresent: true, searchPresent: true, bootstrapCount: 0 }) }) + it('rolls a reset back when the replacement bootstrap fails', () => { + const { entryStore, state, tape } = createHarness() + entryStore.ensureBootstrapAnchor.mockImplementationOnce(() => { + throw new Error('bootstrap failed') + }) + + expect(() => tape.resetSessionTape('s1')).toThrow('bootstrap failed') + expect(state).toEqual({ entriesPresent: true, searchPresent: true, bootstrapCount: 0 }) + }) + itIfSqlite('rolls back a failed reset and creates a fresh incarnation only on retry', () => { const db = new DatabaseCtor(':memory:') try { @@ -190,4 +212,218 @@ describe('SessionTape lifecycle administration', () => { db.close() } }) + + itIfSqlite('rolls back SQLite reset deletion when the replacement bootstrap fails', () => { + const db = new DatabaseCtor(':memory:') + try { + const entryStore = new DeepChatTapeEntriesTable(db) + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + entryStore.createTable() + searchProjection.createTable() + entryStore.ensureBootstrapAnchor('s1') + entryStore.appendEvent({ + sessionId: 's1', + name: 'old/generation', + data: { marker: 'old generation' } + }) + searchProjection.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'event', + name: 'old/generation', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'old generation', + summaryText: 'old generation', + refs: {}, + createdAt: 100 + } + ], + 2 + ) + const bootstrapSpy = vi + .spyOn(entryStore, 'ensureBootstrapAnchor') + .mockImplementationOnce(() => { + throw new Error('bootstrap failed') + }) + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: searchProjection + } as any) + + expect(() => tape.resetSessionTape('s1')).toThrow('bootstrap failed') + expect(entryStore.getBySession('s1').map((entry) => entry.name)).toEqual([ + 'session/start', + 'old/generation' + ]) + expect(searchProjection.isCurrent('s1', 2)).toBe(true) + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([2]) + bootstrapSpy.mockRestore() + } finally { + db.close() + } + }) + + itIfSqlite( + 'rolls transcript cleanup back with a failed Tape reset on one connection', + async () => { + const db = new DatabaseCtor(':memory:') + try { + db.exec(` + CREATE TABLE clear_pending (session_id TEXT NOT NULL); + CREATE TABLE clear_transcript (session_id TEXT NOT NULL); + INSERT INTO clear_pending (session_id) VALUES ('s1'); + INSERT INTO clear_transcript (session_id) VALUES ('s1'); + `) + const entryStore = new DeepChatTapeEntriesTable(db) + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + entryStore.createTable() + searchProjection.createTable() + entryStore.ensureBootstrapAnchor('s1') + entryStore.appendEvent({ + sessionId: 's1', + name: 'old/generation', + data: { marker: 'old generation' } + }) + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: searchProjection + } as any) + vi.spyOn(searchProjection, 'deleteBySession').mockImplementationOnce(() => { + throw new Error('search cleanup failed') + }) + const runtime = { + prepareClearMessages: vi.fn().mockResolvedValue(undefined), + finishClearMessages: vi.fn() + } + const mutations = new SessionTranscriptMutations({ + pendingInputs: { + deleteBySession: () => + db.prepare('DELETE FROM clear_pending WHERE session_id = ?').run('s1') + }, + transcript: { + deleteBySession: () => + db.prepare('DELETE FROM clear_transcript WHERE session_id = ?').run('s1') + }, + settings: { resetTape: () => tape.resetSessionTape('s1') }, + runtime, + runInTransaction: (operation) => db.transaction(operation)() + } as any) + + await expect(mutations.clearMessages('s1')).rejects.toThrow('search cleanup failed') + + expect(db.prepare('SELECT COUNT(*) AS count FROM clear_pending').get()).toEqual({ + count: 1 + }) + expect(db.prepare('SELECT COUNT(*) AS count FROM clear_transcript').get()).toEqual({ + count: 1 + }) + expect(entryStore.getBySession('s1').map((entry) => entry.name)).toEqual([ + 'session/start', + 'old/generation' + ]) + expect(runtime.finishClearMessages).not.toHaveBeenCalled() + } finally { + vi.restoreAllMocks() + db.close() + } + } + ) + + itIfSqlite('drops corrupt FTS state instead of blocking a Tape generation reset', () => { + const db = new DatabaseCtor(':memory:') + try { + const entryStore = new DeepChatTapeEntriesTable(db) + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + entryStore.createTable() + searchProjection.createTable() + if (!searchProjection.hasFtsReadyForTesting()) return + entryStore.ensureBootstrapAnchor('s1') + entryStore.appendEvent({ + sessionId: 's1', + name: 'old/generation', + data: { marker: 'private old generation' } + }) + searchProjection.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'event', + name: 'old/generation', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'private old generation', + summaryText: 'private old generation', + refs: {}, + createdAt: 100 + } + ], + 2 + ) + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: searchProjection + } as any) + const prepare = db.prepare.bind(db) + const prepareSpy = vi.spyOn(db, 'prepare').mockImplementation(((sql: string) => { + if ( + sql.trim().replace(/\s+/g, ' ') === + 'DELETE FROM deepchat_tape_search_fts WHERE session_id = ?' + ) { + throw new Error('injected corrupt FTS delete') + } + return prepare(sql) + }) as typeof db.prepare) + const exec = db.exec.bind(db) + const execSpy = vi.spyOn(db, 'exec').mockImplementation(((sql: string) => { + if (sql.trim() === 'DROP TABLE IF EXISTS deepchat_tape_search_fts') { + throw new Error('injected corrupt FTS drop') + } + return exec(sql) + }) as typeof db.exec) + + expect(() => tape.resetSessionTape('s1')).toThrow('injected corrupt FTS drop') + expect(entryStore.getBySession('s1').map((entry) => entry.name)).toEqual([ + 'session/start', + 'old/generation' + ]) + expect(searchProjection.isCurrent('s1', 2)).toBe(true) + execSpy.mockRestore() + + expect(() => tape.resetSessionTape('s1')).not.toThrow() + prepareSpy.mockRestore() + + expect(entryStore.getBySession('s1')).toMatchObject([{ entry_id: 1, name: 'session/start' }]) + expect(searchProjection.getSessionMeta('s1')).toBeNull() + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([]) + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'deepchat_tape_search_fts'" + ) + .get() + ).toBeUndefined() + expect(tape.search('s1', 'private old generation')).toEqual([]) + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'deepchat_tape_search_fts'" + ) + .get() + ).toEqual({ name: 'deepchat_tape_search_fts' }) + } finally { + vi.restoreAllMocks() + db.close() + } + }) }) diff --git a/test/main/session/data/tapeRecall.test.ts b/test/main/session/data/tapeRecall.test.ts index ec764b3af8..54e9519027 100644 --- a/test/main/session/data/tapeRecall.test.ts +++ b/test/main/session/data/tapeRecall.test.ts @@ -26,6 +26,104 @@ describe('SessionTape recall', () => { expect(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION).toBeGreaterThan(2) }) + itIfSqlite('prunes legacy and metadata-orphaned search projections during initialization', () => { + const db = new DatabaseCtor(':memory:') + try { + const projection = new DeepChatTapeSearchProjectionTable(db) + projection.createTable() + const row = (sessionId: string, searchText: string) => ({ + sessionId, + entryId: 1, + kind: 'event' as const, + name: 'projection/test', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText, + summaryText: searchText, + refs: {}, + createdAt: 100 + }) + projection.replaceSession('legacy', [row('legacy', 'private legacy text')], 1, 2) + projection.replaceSession('current', [row('current', 'current text')], 1) + db.prepare( + `INSERT INTO deepchat_tape_search_projection ( + session_id, + entry_id, + kind, + name, + source_type, + source_id, + source_seq, + search_text, + summary_text, + refs_json, + created_at + ) + VALUES ('orphan', 1, 'event', 'projection/test', NULL, NULL, NULL, ?, ?, '{}', 100)` + ).run('private orphan text', 'private orphan text') + + const restarted = new DeepChatTapeSearchProjectionTable(db) + restarted.createTable() + + expect(restarted.getProjectedEntryIds('legacy')).toEqual([]) + expect(restarted.getSessionMeta('legacy')).toBeNull() + expect(restarted.getProjectedEntryIds('orphan')).toEqual([]) + expect(restarted.getProjectedEntryIds('current')).toEqual([1]) + expect(restarted.isCurrent('current', 1)).toBe(true) + const ftsExists = db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'deepchat_tape_search_fts'" + ) + .get() + if (ftsExists) { + expect( + db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_search_fts + WHERE session_id IN ('legacy', 'orphan')` + ) + .get() + ).toEqual({ count: 0 }) + expect( + db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_search_fts + WHERE session_id = 'current'` + ) + .get() + ).toEqual({ count: 1 }) + } + } finally { + db.close() + } + }) + + itIfSqlite('caches FTS capability detection per SQLite connection', () => { + const db = new DatabaseCtor(':memory:') + try { + const exec = db.exec.bind(db) + let probeStatementCount = 0 + const execSpy = vi.spyOn(db, 'exec').mockImplementation(((sql: string) => { + if (sql.includes('tape_search_fts_probe_')) probeStatementCount += 1 + return exec(sql) + }) as typeof db.exec) + + new DeepChatTapeSearchProjectionTable(db).createTable() + const firstDetectionCount = probeStatementCount + new DeepChatTapeSearchProjectionTable(db).createTable() + + expect(firstDetectionCount).toBeGreaterThan(0) + expect(probeStatementCount).toBe(firstDetectionCount) + execSpy.mockRestore() + } finally { + vi.restoreAllMocks() + db.close() + } + }) + it('reports info, search, and handoff within one session scope', () => { const { table, entries } = createTapeTableMock() const service = new SessionTape({ diff --git a/test/main/session/data/tapeTestHarness.ts b/test/main/session/data/tapeTestHarness.ts index 155b013a70..3b1249c6fd 100644 --- a/test/main/session/data/tapeTestHarness.ts +++ b/test/main/session/data/tapeTestHarness.ts @@ -415,7 +415,9 @@ function createTapeService( tapeLifecycle: table, deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn(), - getByEntryIds: vi.fn().mockReturnValue([]), + isCurrent: vi.fn(() => { + throw new Error('projection unavailable') + }), getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) }, deepchatMessageTracesTable: { diff --git a/test/main/session/runtimeIntegration.test.ts b/test/main/session/runtimeIntegration.test.ts index 9d67a3b37f..6e723ed0c2 100644 --- a/test/main/session/runtimeIntegration.test.ts +++ b/test/main/session/runtimeIntegration.test.ts @@ -792,7 +792,8 @@ function createTranscriptMutations( transcript: sessionData.transcript, settings: sessionData.settings, pendingInputs: sessionData.pendingInputs, - runtime + runtime, + runInTransaction: (operation) => sessionData.database.getDatabase().transaction(operation)() }) } diff --git a/test/main/session/transcriptMutations.test.ts b/test/main/session/transcriptMutations.test.ts new file mode 100644 index 0000000000..b0e076f20d --- /dev/null +++ b/test/main/session/transcriptMutations.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it, vi } from 'vitest' +import { SessionTranscriptMutations } from '@/session/transcriptMutations' + +describe('SessionTranscriptMutations', () => { + it('rolls pending inputs and transcript deletion back when Tape reset fails', async () => { + const state = { pendingInputs: 1, transcriptMessages: 2, tapeEntries: 3 } + const runtime = { + prepareClearMessages: vi.fn().mockResolvedValue(undefined), + finishClearMessages: vi.fn() + } + const mutations = new SessionTranscriptMutations({ + pendingInputs: { + deleteBySession: vi.fn(() => { + state.pendingInputs = 0 + }) + }, + transcript: { + deleteBySession: vi.fn(() => { + state.transcriptMessages = 0 + }) + }, + settings: { + resetTape: vi.fn(() => { + state.tapeEntries = 0 + throw new Error('Tape reset failed') + }) + }, + runtime, + runInTransaction: (operation) => { + const snapshot = { ...state } + try { + return operation() + } catch (error) { + Object.assign(state, snapshot) + throw error + } + } + } as any) + + await expect(mutations.clearMessages('s1')).rejects.toThrow('Tape reset failed') + + expect(state).toEqual({ pendingInputs: 1, transcriptMessages: 2, tapeEntries: 3 }) + expect(runtime.prepareClearMessages).toHaveBeenCalledWith('s1') + expect(runtime.finishClearMessages).not.toHaveBeenCalled() + }) + + it('finishes runtime cleanup only after the shared transaction commits', async () => { + const calls: string[] = [] + const mutations = new SessionTranscriptMutations({ + pendingInputs: { deleteBySession: vi.fn(() => calls.push('pending')) }, + transcript: { deleteBySession: vi.fn(() => calls.push('transcript')) }, + settings: { resetTape: vi.fn(() => calls.push('tape')) }, + runtime: { + prepareClearMessages: vi.fn(async () => { + calls.push('prepare') + }), + finishClearMessages: vi.fn(() => calls.push('finish')) + }, + runInTransaction: (operation) => { + calls.push('transaction:start') + const result = operation() + calls.push('transaction:commit') + return result + } + } as any) + + await mutations.clearMessages('s1') + + expect(calls).toEqual([ + 'prepare', + 'transaction:start', + 'pending', + 'transcript', + 'tape', + 'transaction:commit', + 'finish' + ]) + }) +}) From 16274f7bd497ae8264895d6130cb10e98cf3d75c Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 22:38:12 +0800 Subject: [PATCH 24/29] refactor(tape): harden compatibility boundaries --- src/main/app/composition.ts | 3 +- .../legacyChatImportService.ts | 8 +- .../tables/deepchatTapeEffectiveSemantics.ts | 17 +- .../data/tables/deepchatTapeEntries.ts | 23 +- .../tables/deepchatTapeSearchProjection.ts | 15 +- src/main/session/data/tape.ts | 20 +- src/main/session/data/tapeEffectiveView.ts | 10 +- src/main/session/data/tapeFacts.ts | 15 +- src/main/session/data/tapeViewManifest.ts | 26 +- src/main/tape/application/contracts.ts | 3 +- src/main/tape/application/sessionTape.ts | 18 +- src/main/tape/domain/viewManifest.ts | 8 +- src/main/tape/ports/capabilities.ts | 8 +- test/main/app/compositionBoundaries.test.ts | 1 + .../legacyChatImportService.test.ts | 5 + test/main/tape/layerBoundaries.test.ts | 316 +++++++++++++++--- 16 files changed, 405 insertions(+), 91 deletions(-) diff --git a/src/main/app/composition.ts b/src/main/app/composition.ts index 554ad0af65..d4d93e143f 100644 --- a/src/main/app/composition.ts +++ b/src/main/app/composition.ts @@ -386,7 +386,8 @@ export async function createMainProcessControl(dependencies: { appDatabase, sessionData.database, projectDatabase, - memoryDatabase + memoryDatabase, + sessionData.tapeStore ) usageStatsService = new UsageStatsService( sessionData.database, diff --git a/src/main/app/startupMigrations/legacyChatImportService.ts b/src/main/app/startupMigrations/legacyChatImportService.ts index d93bfdb23a..acd72273f5 100644 --- a/src/main/app/startupMigrations/legacyChatImportService.ts +++ b/src/main/app/startupMigrations/legacyChatImportService.ts @@ -12,7 +12,7 @@ import { isReasoningEffort } from '@shared/types/model-db' import { resolveAcpAgentAlias } from '@shared/utils/acpAgentAlias' import { SessionTranscript } from '@/session/data/transcript' import { SessionDatabase } from '@/session/data/database' -import { SessionTape } from '@/tape/application/sessionTape' +import type { TapeMessageFactWriter } from '@/tape/ports/capabilities' import type { ProjectDatabase } from '@/project/data/database' import type { AppDatabase } from '@/app/data/database' import type { MemoryDatabase } from '@/memory/data/database' @@ -45,16 +45,14 @@ export class LegacyChatImportService { sessionDatabase: SessionDatabase, projectDatabase: ProjectDatabase, memoryDatabase: MemoryDatabase, + tapeFacts: TapeMessageFactWriter, sourceDbPath?: string ) { this.appDatabase = appDatabase this.sessionDatabase = sessionDatabase this.projectDatabase = projectDatabase this.memoryDatabase = memoryDatabase - this.messageStore = new SessionTranscript( - this.sessionDatabase, - new SessionTape(this.sessionDatabase) - ) + this.messageStore = new SessionTranscript(this.sessionDatabase, tapeFacts) this.sourceDbPath = sourceDbPath ?? path.join(app.getPath('userData'), 'app_db', 'chat.db') } diff --git a/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts b/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts index 7a707b39b9..3223a637b4 100644 --- a/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts +++ b/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts @@ -1 +1,16 @@ -export * from '@/tape/domain/effectiveSemantics' +/** @deprecated Import Tape semantics from `@/tape/domain/effectiveSemantics`. */ +export { + messageRecordHasFinalToolUse, + parseAssistantBlocks, + parseNestedTapeJsonObject, + parseTapeJsonObject, + readTapeMessageRetractionId, + readTapeToolIdentity, + readTapeToolStatus, + tapeEntryToMessageRecord, + tapeMessageRank, + tapeToolRank +} from '@/tape/domain/effectiveSemantics' + +/** @deprecated Import Tape semantic types from `@/tape/domain/effectiveSemantics`. */ +export type { DeepChatTapeToolIdentity } from '@/tape/domain/effectiveSemantics' diff --git a/src/main/session/data/tables/deepchatTapeEntries.ts b/src/main/session/data/tables/deepchatTapeEntries.ts index 6d84885f1d..c34220272c 100644 --- a/src/main/session/data/tables/deepchatTapeEntries.ts +++ b/src/main/session/data/tables/deepchatTapeEntries.ts @@ -1 +1,22 @@ -export * from '@/tape/infrastructure/sqlite/tapeEntryStore' +/** @deprecated Import the SQLite Tape store from `@/tape/infrastructure/sqlite/tapeEntryStore`. */ +export { + buildDeepChatTapeFtsMatch, + buildDeepChatTapeLikeSearchPredicate, + DeepChatTapeEntriesTable, + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources, + SUMMARY_ANCHOR_NAMES, + TAPE_INCARNATION_META_KEY +} from '@/tape/infrastructure/sqlite/tapeEntryStore' + +/** @deprecated Import SQLite Tape types from `@/tape/infrastructure/sqlite/tapeEntryStore`. */ +export type { + DeepChatTapeAppendInput, + DeepChatTapeEntryKind, + DeepChatTapeEntryRow, + DeepChatTapeMutationProjection, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + DeepChatTapeSourceInput, + DeepChatTapeSourceType +} from '@/tape/infrastructure/sqlite/tapeEntryStore' diff --git a/src/main/session/data/tables/deepchatTapeSearchProjection.ts b/src/main/session/data/tables/deepchatTapeSearchProjection.ts index 16f6784dff..6869f22cea 100644 --- a/src/main/session/data/tables/deepchatTapeSearchProjection.ts +++ b/src/main/session/data/tables/deepchatTapeSearchProjection.ts @@ -1 +1,14 @@ -export * from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' +/** @deprecated Import the SQLite projection from `@/tape/infrastructure/sqlite/tapeSearchProjectionStore`. */ +export { + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, + DeepChatTapeSearchProjectionTable +} from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' + +/** @deprecated Import projection types from `@/tape/infrastructure/sqlite/tapeSearchProjectionStore`. */ +export type { + DeepChatTapeSearchProjectionInput, + DeepChatTapeSearchProjectionMeta, + DeepChatTapeSearchProjectionReadResult, + DeepChatTapeSearchProjectionResultRow, + DeepChatTapeSearchProjectionRow +} from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' diff --git a/src/main/session/data/tape.ts b/src/main/session/data/tape.ts index 38e1db43db..6915099d38 100644 --- a/src/main/session/data/tape.ts +++ b/src/main/session/data/tape.ts @@ -1 +1,19 @@ -export * from '@/tape/application/sessionTape' +/** @deprecated Import Tape application APIs from `@/tape/application/sessionTape`. */ +export { + AgentTapeViewError, + normalizeSubagentTapeLinkInput, + normalizeTapeHandoffState, + SessionTape +} from '@/tape/application/sessionTape' + +/** @deprecated Import Tape application types from `@/tape/application/sessionTape`. */ +export type { + AgentTapeViewErrorCode, + TapeAnchorResult, + TapeBackfillResult, + TapeForkHandle, + TapeInfo, + TapeMigrationState, + TapeSearchResult, + TapeViewManifestAssemblySources as TapeViewManifestSourceMaps +} from '@/tape/application/sessionTape' diff --git a/src/main/session/data/tapeEffectiveView.ts b/src/main/session/data/tapeEffectiveView.ts index 0cb25c20b6..a8ee4faa81 100644 --- a/src/main/session/data/tapeEffectiveView.ts +++ b/src/main/session/data/tapeEffectiveView.ts @@ -1 +1,9 @@ -export * from '@/tape/domain/effectiveView' +/** @deprecated Import effective Tape helpers from `@/tape/domain/effectiveView`. */ +export { + buildEffectiveTapeView, + getLastEffectiveTokenUsage, + searchEffectiveTapeRows +} from '@/tape/domain/effectiveView' + +/** @deprecated Import effective Tape types from `@/tape/domain/effectiveView`. */ +export type { EffectiveMessageEntry, EffectiveTapeView } from '@/tape/domain/effectiveView' diff --git a/src/main/session/data/tapeFacts.ts b/src/main/session/data/tapeFacts.ts index 1b31870d9a..9d06afbce1 100644 --- a/src/main/session/data/tapeFacts.ts +++ b/src/main/session/data/tapeFacts.ts @@ -1 +1,14 @@ -export * from '@/tape/application/factPersistence' +/** @deprecated Import Tape fact helpers from `@/tape/application/factPersistence`. */ +export { + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + appendTapeToolFact, + appendToolFactsToTape, + buildTapeToolFactInputs, + tapeEntriesToEffectiveMessageRecords, + tapeEntryToMessageRecord +} from '@/tape/application/factPersistence' + +/** @deprecated Import Tape fact types from `@/tape/application/factPersistence`. */ +export type { TapeFactSource } from '@/tape/application/factPersistence' diff --git a/src/main/session/data/tapeViewManifest.ts b/src/main/session/data/tapeViewManifest.ts index 4b7cec35ef..529d6f7707 100644 --- a/src/main/session/data/tapeViewManifest.ts +++ b/src/main/session/data/tapeViewManifest.ts @@ -1 +1,25 @@ -export * from '@/tape/domain/viewManifest' +/** @deprecated Import ViewManifest helpers from `@/tape/domain/viewManifest`. */ +export { + buildExcludedRefs, + buildIncludedRefs, + buildRequestRefs, + createTapeViewManifest, + hashJson, + isCompactionRecord, + resolveTapeViewManifestPolicy, + stableJsonStringify, + TAPE_VIEW_CONTEXT_BUILDER_VERSION, + TAPE_VIEW_MANIFEST_EVENT_NAME, + TAPE_VIEW_MANIFEST_HASH_VERSION, + verifyTapeViewManifestHash +} from '@/tape/domain/viewManifest' + +/** @deprecated Import ViewManifest types from `@/tape/domain/viewManifest`. */ +export type { + ContextSummaryCursorMetadata, + TapeViewContextSelection, + TapeViewManifestBuildInput, + TapeViewManifestLookupMaps as TapeViewManifestSourceMaps, + TapeViewManifestPolicyInput, + TapeViewManifestPolicyResult +} from '@/tape/domain/viewManifest' diff --git a/src/main/tape/application/contracts.ts b/src/main/tape/application/contracts.ts index ed947e3298..4c537623cb 100644 --- a/src/main/tape/application/contracts.ts +++ b/src/main/tape/application/contracts.ts @@ -4,8 +4,7 @@ import type { TapeMigrationState } from '../ports/capabilities' export type { TapeBackfillResult, TapeMigrationState, - TapeViewManifestAssemblySources, - TapeViewManifestSourceMaps + TapeViewManifestAssemblySources } from '../ports/capabilities' export type TapeInfo = { diff --git a/src/main/tape/application/sessionTape.ts b/src/main/tape/application/sessionTape.ts index 120d11bd9a..8a1fccd651 100644 --- a/src/main/tape/application/sessionTape.ts +++ b/src/main/tape/application/sessionTape.ts @@ -47,8 +47,7 @@ import type { TapeInfo, TapeMigrationState, TapeSearchResult, - TapeViewManifestAssemblySources, - TapeViewManifestSourceMaps + TapeViewManifestAssemblySources } from './contracts' import { normalizeTapeHandoffState, TapeFactService } from './factService' import { TapeForkService } from './forkService' @@ -71,8 +70,7 @@ export type { TapeInfo, TapeMigrationState, TapeSearchResult, - TapeViewManifestAssemblySources, - TapeViewManifestSourceMaps + TapeViewManifestAssemblySources } export { AgentTapeViewError, normalizeSubagentTapeLinkInput, normalizeTapeHandoffState } @@ -248,18 +246,6 @@ export class SessionTape return this.providers.getEntryStore().getBySession(sessionId) } - getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { - return this.providers.getEntryStore().getLatestAnchor(sessionId) - } - - getAnchors(sessionId: string, limit?: number): DeepChatTapeEntryRow[] { - return this.providers.getEntryStore().getAnchors(sessionId, limit) - } - - getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { - return this.providers.getEntryStore().getLatestSummaryAnchor(sessionId) - } - getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { return this.providers.getEntryStore().getLatestReconstructionAnchor(sessionId) } diff --git a/src/main/tape/domain/viewManifest.ts b/src/main/tape/domain/viewManifest.ts index 8aacdac00e..9f4ac268f0 100644 --- a/src/main/tape/domain/viewManifest.ts +++ b/src/main/tape/domain/viewManifest.ts @@ -34,7 +34,7 @@ export function isCompactionRecord(record: ChatMessageRecord): boolean { export const TAPE_VIEW_MANIFEST_EVENT_NAME = 'view/assembled' export const TAPE_VIEW_CONTEXT_BUILDER_VERSION = 'legacy-v1' as const -export type TapeViewManifestSourceMaps = { +export type TapeViewManifestLookupMaps = { entryIdByMessageId?: Map toolCallEntryIdByToolId?: Map toolResultEntryIdByToolId?: Map @@ -240,7 +240,7 @@ export function createTapeViewManifest( export function buildIncludedRefs( selection: TapeViewContextSelection, - sourceMaps: TapeViewManifestSourceMaps = {} + sourceMaps: TapeViewManifestLookupMaps = {} ): DeepChatTapeViewEntryRef[] { const refs: DeepChatTapeViewEntryRef[] = [] @@ -282,7 +282,7 @@ export function buildIncludedRefs( export function buildExcludedRefs( selection: TapeViewContextSelection, - sourceMaps: TapeViewManifestSourceMaps = {} + sourceMaps: TapeViewManifestLookupMaps = {} ): DeepChatTapeViewExcludedRef[] { return selection.excludedRecords.map((item) => ({ entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, @@ -294,7 +294,7 @@ export function buildExcludedRefs( export function buildRequestRefs( messages: ChatMessage[], - sourceMaps: TapeViewManifestSourceMaps = {} + sourceMaps: TapeViewManifestLookupMaps = {} ): DeepChatTapeViewEntryRef[] { const lastToolCallIndex = new Map() const lastToolResultIndex = new Map() diff --git a/src/main/tape/ports/capabilities.ts b/src/main/tape/ports/capabilities.ts index 568c47fa17..68e5898aa7 100644 --- a/src/main/tape/ports/capabilities.ts +++ b/src/main/tape/ports/capabilities.ts @@ -35,9 +35,6 @@ export type TapeViewManifestAssemblySources = { toolResultEntryIdByToolId: Map } -/** @deprecated Use TapeViewManifestAssemblySources for the complete assembly source set. */ -export type TapeViewManifestSourceMaps = TapeViewManifestAssemblySources - export interface TapeViewManifestReader { getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestAssemblySources listViewManifestsByMessage(sessionId: string, messageId: string): DeepChatTapeViewManifestRecord[] @@ -66,6 +63,11 @@ export interface TapeAnchorReader { } export interface TapeAnchorWriter { + /** + * Compound Session settings updates require this writer to share the caller's synchronous + * SQLite connection and transaction context. Cross-connection or asynchronous adapters cannot + * preserve summary-and-anchor atomicity. + */ appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow } diff --git a/test/main/app/compositionBoundaries.test.ts b/test/main/app/compositionBoundaries.test.ts index 95af980b7e..571253b695 100644 --- a/test/main/app/compositionBoundaries.test.ts +++ b/test/main/app/compositionBoundaries.test.ts @@ -10,6 +10,7 @@ describe('session boundary composition', () => { ) expect(compositionSource.match(/new LegacyChatImportService\(/g)).toHaveLength(1) + expect(compositionSource).toContain('memoryDatabase,\n sessionData.tapeStore') expect(compositionSource).toContain( 'legacyChatImportService.repairImportedLegacySessionSkills(conversationId)' ) diff --git a/test/main/app/startupMigrations/legacyChatImportService.test.ts b/test/main/app/startupMigrations/legacyChatImportService.test.ts index de24962edc..f2d083b8a1 100644 --- a/test/main/app/startupMigrations/legacyChatImportService.test.ts +++ b/test/main/app/startupMigrations/legacyChatImportService.test.ts @@ -123,6 +123,11 @@ describe('LegacyChatImportService', () => { sqlitePresenter as any, sqlitePresenter as any, sqlitePresenter as any, + { + appendMessageRecord: vi.fn(() => 0), + appendMessageReplacement: vi.fn(() => 0), + appendMessageRetraction: vi.fn(() => 0) + }, '/mock/legacy.db' ) }) diff --git a/test/main/tape/layerBoundaries.test.ts b/test/main/tape/layerBoundaries.test.ts index 73052322bf..5a9487188d 100644 --- a/test/main/tape/layerBoundaries.test.ts +++ b/test/main/tape/layerBoundaries.test.ts @@ -12,16 +12,153 @@ const MEMORY_ROUTES_FILE = path.join(MAIN_SOURCE_ROOT, 'memory/routes.ts') const TAPE_SQLITE_RELATIVE_ROOT = 'tape/infrastructure/sqlite/' const TYPESCRIPT_SOURCE_EXTENSION = /\.[cm]?tsx?$/ -const LEGACY_TAPE_COMPATIBILITY_MODULES = new Map([ - ['session/data/tape', '@/tape/application/sessionTape'], - ['session/data/tapeEffectiveView', '@/tape/domain/effectiveView'], - ['session/data/tapeFacts', '@/tape/application/factPersistence'], - ['session/data/tapeViewManifest', '@/tape/domain/viewManifest'], - ['session/data/tables/deepchatTapeEffectiveSemantics', '@/tape/domain/effectiveSemantics'], - ['session/data/tables/deepchatTapeEntries', '@/tape/infrastructure/sqlite/tapeEntryStore'], +interface LegacyTapeCompatibilityContract { + target: string + valueExports: readonly string[] + typeExports: readonly string[] + typeAliases?: Readonly> +} + +const LEGACY_TAPE_COMPATIBILITY_MODULES = new Map([ + [ + 'session/data/tape', + { + target: '@/tape/application/sessionTape', + valueExports: [ + 'AgentTapeViewError', + 'normalizeSubagentTapeLinkInput', + 'normalizeTapeHandoffState', + 'SessionTape' + ], + typeExports: [ + 'AgentTapeViewErrorCode', + 'TapeAnchorResult', + 'TapeBackfillResult', + 'TapeForkHandle', + 'TapeInfo', + 'TapeMigrationState', + 'TapeSearchResult' + ], + typeAliases: { TapeViewManifestSourceMaps: 'TapeViewManifestAssemblySources' } + } + ], + [ + 'session/data/tapeEffectiveView', + { + target: '@/tape/domain/effectiveView', + valueExports: [ + 'buildEffectiveTapeView', + 'getLastEffectiveTokenUsage', + 'searchEffectiveTapeRows' + ], + typeExports: ['EffectiveMessageEntry', 'EffectiveTapeView'] + } + ], + [ + 'session/data/tapeFacts', + { + target: '@/tape/application/factPersistence', + valueExports: [ + 'appendMessageRecordToTape', + 'appendMessageReplacementToTape', + 'appendMessageRetractionToTape', + 'appendTapeToolFact', + 'appendToolFactsToTape', + 'buildTapeToolFactInputs', + 'tapeEntriesToEffectiveMessageRecords', + 'tapeEntryToMessageRecord' + ], + typeExports: ['TapeFactSource'] + } + ], + [ + 'session/data/tapeViewManifest', + { + target: '@/tape/domain/viewManifest', + valueExports: [ + 'buildExcludedRefs', + 'buildIncludedRefs', + 'buildRequestRefs', + 'createTapeViewManifest', + 'hashJson', + 'isCompactionRecord', + 'resolveTapeViewManifestPolicy', + 'stableJsonStringify', + 'TAPE_VIEW_CONTEXT_BUILDER_VERSION', + 'TAPE_VIEW_MANIFEST_EVENT_NAME', + 'TAPE_VIEW_MANIFEST_HASH_VERSION', + 'verifyTapeViewManifestHash' + ], + typeExports: [ + 'ContextSummaryCursorMetadata', + 'TapeViewContextSelection', + 'TapeViewManifestBuildInput', + 'TapeViewManifestPolicyInput', + 'TapeViewManifestPolicyResult' + ], + typeAliases: { TapeViewManifestSourceMaps: 'TapeViewManifestLookupMaps' } + } + ], + [ + 'session/data/tables/deepchatTapeEffectiveSemantics', + { + target: '@/tape/domain/effectiveSemantics', + valueExports: [ + 'messageRecordHasFinalToolUse', + 'parseAssistantBlocks', + 'parseNestedTapeJsonObject', + 'parseTapeJsonObject', + 'readTapeMessageRetractionId', + 'readTapeToolIdentity', + 'readTapeToolStatus', + 'tapeEntryToMessageRecord', + 'tapeMessageRank', + 'tapeToolRank' + ], + typeExports: ['DeepChatTapeToolIdentity'] + } + ], + [ + 'session/data/tables/deepchatTapeEntries', + { + target: '@/tape/infrastructure/sqlite/tapeEntryStore', + valueExports: [ + 'buildDeepChatTapeFtsMatch', + 'buildDeepChatTapeLikeSearchPredicate', + 'DeepChatTapeEntriesTable', + 'normalizeDeepChatTapeReadSources', + 'serializeDeepChatTapeReadSources', + 'SUMMARY_ANCHOR_NAMES', + 'TAPE_INCARNATION_META_KEY' + ], + typeExports: [ + 'DeepChatTapeAppendInput', + 'DeepChatTapeEntryKind', + 'DeepChatTapeEntryRow', + 'DeepChatTapeMutationProjection', + 'DeepChatTapeReadSource', + 'DeepChatTapeSearchInput', + 'DeepChatTapeSourceInput', + 'DeepChatTapeSourceType' + ] + } + ], [ 'session/data/tables/deepchatTapeSearchProjection', - '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' + { + target: '@/tape/infrastructure/sqlite/tapeSearchProjectionStore', + valueExports: [ + 'DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION', + 'DeepChatTapeSearchProjectionTable' + ], + typeExports: [ + 'DeepChatTapeSearchProjectionInput', + 'DeepChatTapeSearchProjectionMeta', + 'DeepChatTapeSearchProjectionReadResult', + 'DeepChatTapeSearchProjectionResultRow', + 'DeepChatTapeSearchProjectionRow' + ] + } ] ]) @@ -31,6 +168,7 @@ const CAPABILITY_SCOPED_CONSUMER_FILES = [ 'agent/deepchat/memory/memoryRuntimeCoordinator.ts', 'agent/deepchat/runtime/deepChatLoopRunner.ts', 'agent/deepchat/runtime/turnCoordinator.ts', + 'app/startupMigrations/legacyChatImportService.ts', 'memory/routes.ts', 'session/data/settings.ts', 'session/data/transcript.ts' @@ -90,11 +228,17 @@ const ALLOWED_STORAGE_EXCEPTIONS = new Map([ ], [ 'session/data/tables/deepchatTapeEntries.ts', - { sqliteImport: 'legacy import-path compatibility re-export' } + { + physicalName: 'frozen legacy compatibility export', + sqliteImport: 'legacy import-path compatibility re-export' + } ], [ 'session/data/tables/deepchatTapeSearchProjection.ts', - { sqliteImport: 'legacy import-path compatibility re-export' } + { + physicalName: 'frozen legacy compatibility export', + sqliteImport: 'legacy import-path compatibility re-export' + } ], ['tape/ports/application.ts', { physicalName: 'legacy database-shape compatibility adapter' }] ]) @@ -196,51 +340,90 @@ function findConcreteTapeFacadeImportViolations(source: string, file: string): s function findMemoryRouteTapeImportViolations(source: string, file: string): string[] { const sourceFile = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true) - return sourceFile.statements.flatMap((statement) => { - if (!ts.isImportDeclaration(statement) || !ts.isStringLiteral(statement.moduleSpecifier)) { - return [] - } - const specifier = statement.moduleSpecifier.text - if (!isTapeModuleImport(file, specifier)) return [] - + const tapeReferences = ts + .preProcessFile(source, true, true) + .importedFiles.filter(({ fileName }) => isTapeModuleImport(file, fileName)) + const staticTapeImports = sourceFile.statements.filter( + (statement): statement is ts.ImportDeclaration => + ts.isImportDeclaration(statement) && + ts.isStringLiteral(statement.moduleSpecifier) && + isTapeModuleImport(file, statement.moduleSpecifier.text) + ) + const violations = tapeReferences.flatMap(({ fileName: specifier }) => { const target = resolveMainImport(file, specifier) - if (!target || withoutTypeScriptExtension(target) !== TAPE_CAPABILITIES_MODULE) { - return [`Tape import must use the inspection port: ${specifier}`] - } + return !target || withoutTypeScriptExtension(target) !== TAPE_CAPABILITIES_MODULE + ? [`Tape import must use the inspection port: ${specifier}`] + : [] + }) - const importClause = statement.importClause - const namedBindings = importClause?.namedBindings - if ( - !importClause || - importClause.name || - !namedBindings || - !ts.isNamedImports(namedBindings) || - namedBindings.elements.length !== 1 - ) { - return [`Tape capabilities import must name only TapeInspectionReader: ${specifier}`] - } + if (tapeReferences.length !== staticTapeImports.length) { + violations.push('Tape references must use a static type-only import declaration') + } - const [element] = namedBindings.elements - const importedName = element.propertyName?.text ?? element.name.text - const isTypeOnly = importClause.isTypeOnly || element.isTypeOnly - return importedName === 'TapeInspectionReader' && isTypeOnly - ? [] - : [`Memory routes may import only the TapeInspectionReader type: ${importedName}`] - }) + violations.push( + ...staticTapeImports.flatMap((statement) => { + const specifier = statement.moduleSpecifier.text + const importClause = statement.importClause + const namedBindings = importClause?.namedBindings + if ( + !importClause || + importClause.name || + !namedBindings || + !ts.isNamedImports(namedBindings) || + namedBindings.elements.length !== 1 + ) { + return [`Tape capabilities import must name only TapeInspectionReader: ${specifier}`] + } + + const [element] = namedBindings.elements + const importedName = element.propertyName?.text ?? element.name.text + const isTypeOnly = importClause.isTypeOnly || element.isTypeOnly + return importedName === 'TapeInspectionReader' && isTypeOnly + ? [] + : [`Memory routes may import only the TapeInspectionReader type: ${importedName}`] + }) + ) + + return [...new Set(violations)] } -function isCanonicalCompatibilityReexport(source: string, expectedTarget: string): boolean { +function compatibilityExportDescriptors(contract: LegacyTapeCompatibilityContract): string[] { + return [ + ...contract.valueExports.map((name) => `value:${name}:${name}`), + ...contract.typeExports.map((name) => `type:${name}:${name}`), + ...Object.entries(contract.typeAliases ?? {}).map( + ([exportedName, importedName]) => `type:${importedName}:${exportedName}` + ) + ].sort() +} + +function isFrozenCompatibilityReexport( + source: string, + contract: LegacyTapeCompatibilityContract +): boolean { + if (!source.includes('@deprecated')) return false const sourceFile = ts.createSourceFile('compatibility.ts', source, ts.ScriptTarget.Latest, true) - if (sourceFile.statements.length !== 1) return false - const [statement] = sourceFile.statements + const descriptors = sourceFile.statements.flatMap((statement) => { + if ( + !ts.isExportDeclaration(statement) || + !statement.exportClause || + !ts.isNamedExports(statement.exportClause) || + !statement.moduleSpecifier || + !ts.isStringLiteral(statement.moduleSpecifier) || + statement.moduleSpecifier.text !== contract.target + ) { + return ['invalid'] + } + return statement.exportClause.elements.map((element) => { + const importedName = element.propertyName?.text ?? element.name.text + const exportedName = element.name.text + const kind = statement.isTypeOnly || element.isTypeOnly ? 'type' : 'value' + return `${kind}:${importedName}:${exportedName}` + }) + }) return ( - ts.isExportDeclaration(statement) && - statement.exportClause === undefined && - Boolean( - statement.moduleSpecifier && - ts.isStringLiteral(statement.moduleSpecifier) && - statement.moduleSpecifier.text === expectedTarget - ) + sourceFile.statements.length > 0 && + JSON.stringify(descriptors.sort()) === JSON.stringify(compatibilityExportDescriptors(contract)) ) } @@ -276,15 +459,15 @@ describe('Tape layer boundaries', () => { expect(violations).toEqual([]) }) - it('keeps legacy Tape compatibility modules as canonical re-exports', async () => { + it('keeps legacy Tape compatibility modules on frozen deprecated export contracts', async () => { const fs = await vi.importActual('node:fs') const violations = [...LEGACY_TAPE_COMPATIBILITY_MODULES.entries()].flatMap( - ([relativeModule, expectedTarget]) => { + ([relativeModule, contract]) => { const file = path.join(MAIN_SOURCE_ROOT, `${relativeModule}.ts`) const source = fs.readFileSync(file, 'utf8') - return isCanonicalCompatibilityReexport(source, expectedTarget) + return isFrozenCompatibilityReexport(source, contract) ? [] - : [`${relativeModule}.ts must re-export only ${expectedTarget}`] + : [`${relativeModule}.ts must match its frozen ${contract.target} export contract`] } ) @@ -363,7 +546,10 @@ describe('Tape layer boundaries', () => { "import { buildEffectiveTapeView } from '@/tape/domain/effectiveView'" ], ['application facade', "import { SessionTape } from '@/tape/application/sessionTape'"], - ['inspection value import', "import { TapeInspectionReader } from '@/tape/ports/capabilities'"] + ['inspection value import', "import { TapeInspectionReader } from '@/tape/ports/capabilities'"], + ['dynamic import', "void import('@/tape/application/sessionTape')"], + ['CommonJS require', "const tape = require('@/tape/domain/effectiveView')"], + ['type re-export', "export type { TapeInspectionReader } from '@/tape/ports/capabilities'"] ])('detects Memory route Tape bypass through %s', (_category, source) => { expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).not.toEqual([]) }) @@ -373,6 +559,30 @@ describe('Tape layer boundaries', () => { expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).toEqual([]) }) + it('allows inline type syntax for the Memory inspection reader', () => { + const source = "import { type TapeInspectionReader } from '@/tape/ports/capabilities'" + expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).toEqual([]) + }) + + it.each([ + ['star export', "/** @deprecated */\nexport * from '@/tape/domain/effectiveView'"], + [ + 'extra export', + "/** @deprecated */\nexport { buildEffectiveTapeView, getLastEffectiveTokenUsage, searchEffectiveTapeRows, unexpected } from '@/tape/domain/effectiveView'\nexport type { EffectiveMessageEntry, EffectiveTapeView } from '@/tape/domain/effectiveView'" + ], + [ + 'missing export', + "/** @deprecated */\nexport { buildEffectiveTapeView } from '@/tape/domain/effectiveView'\nexport type { EffectiveMessageEntry, EffectiveTapeView } from '@/tape/domain/effectiveView'" + ], + [ + 'missing deprecation marker', + "export { buildEffectiveTapeView, getLastEffectiveTokenUsage, searchEffectiveTapeRows } from '@/tape/domain/effectiveView'\nexport type { EffectiveMessageEntry, EffectiveTapeView } from '@/tape/domain/effectiveView'" + ] + ])('rejects a legacy compatibility contract with a %s', (_case, source) => { + const contract = LEGACY_TAPE_COMPATIBILITY_MODULES.get('session/data/tapeEffectiveView')! + expect(isFrozenCompatibilityReexport(source, contract)).toBe(false) + }) + it.each(['@/tape/application/sessionTape', '../../tape/application/sessionTape'])( 'detects concrete Tape facade import %s in a capability-scoped consumer', (specifier) => { From df1cd3ae5b19d78e6c3cab79cdc9e4c5938fb2d4 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 22:45:06 +0800 Subject: [PATCH 25/29] test(tape): repair memory type contracts --- test/main/session/data/tapeTestHarness.ts | 8 ++++++++ test/main/session/data/tapeViewReplay.test.ts | 2 ++ 2 files changed, 10 insertions(+) diff --git a/test/main/session/data/tapeTestHarness.ts b/test/main/session/data/tapeTestHarness.ts index 3b1249c6fd..29819a3625 100644 --- a/test/main/session/data/tapeTestHarness.ts +++ b/test/main/session/data/tapeTestHarness.ts @@ -147,6 +147,14 @@ function createTapeTableMock() { getBySession: vi.fn((sessionId: string) => entries.filter((entry) => entry.session_id === sessionId) ), + listMemoryViewManifestAnchorsByAgent: vi.fn( + ( + _agentId: string, + _options?: { sessionId?: string; limit?: number; messageId?: string } + ): any[] => { + throw new Error('configure listMemoryViewManifestAnchorsByAgent for this test') + } + ), getSubagentLineageEvents: vi.fn((sessionId: string) => entries.filter( (entry) => diff --git a/test/main/session/data/tapeViewReplay.test.ts b/test/main/session/data/tapeViewReplay.test.ts index 2f83c9f8a8..ebb7e6fa62 100644 --- a/test/main/session/data/tapeViewReplay.test.ts +++ b/test/main/session/data/tapeViewReplay.test.ts @@ -1,3 +1,5 @@ +import type { TapeViewManifestBuildInput } from '@/tape/domain/viewManifest' +import type { ChatMessageRecord } from '@shared/types/agent-interface' import { describe, expect, From 13a86789c334ff29fba41dd5911c61711498dd05 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 22:53:02 +0800 Subject: [PATCH 26/29] docs(tape): finalize hardening record --- ...agent-system-layered-runtime-baseline.json | 4 +- docs/architecture/memory-system.md | 4 +- docs/architecture/session-management.md | 20 ++++---- docs/architecture/tape-layering/plan.md | 47 +++++++++++++++++-- docs/architecture/tape-layering/spec.md | 22 +++++++-- docs/architecture/tape-layering/tasks.md | 41 ++++++++-------- docs/architecture/tape-system.md | 18 ++++--- 7 files changed, 111 insertions(+), 45 deletions(-) diff --git a/docs/architecture/baselines/agent-system-layered-runtime-baseline.json b/docs/architecture/baselines/agent-system-layered-runtime-baseline.json index 2bbf2caa1c..be21f707cc 100644 --- a/docs/architecture/baselines/agent-system-layered-runtime-baseline.json +++ b/docs/architecture/baselines/agent-system-layered-runtime-baseline.json @@ -1,7 +1,7 @@ { "schemaVersion": 2, "goal": "agent-system-layered-runtime", - "headCommit": "ae52a0624de3ebdce4d7d43f66bafeea392b9c9c", + "headCommit": "df1cd3ae5b19d78e6c3cab79cdc9e4c5938fb2d4", "relevantWorkingTree": { "dirty": false, "files": [] @@ -399,7 +399,7 @@ "src/main/app/mainProcess.ts", "src/main/appMain.ts" ], - "sha256": "8968bb535eeabb3d9a154ca777cc83c7c8ec5183ccae7c165fe11bd51e6f7299" + "sha256": "394cd861f9fe06a29159d06f87f83b7ba1866287f129fae2a42a382e7e2e6c95" }, "dependencyMetrics": { "loopFiles": [ diff --git a/docs/architecture/memory-system.md b/docs/architecture/memory-system.md index 5128e74282..d3c17a586a 100644 --- a/docs/architecture/memory-system.md +++ b/docs/architecture/memory-system.md @@ -94,7 +94,9 @@ cursor commit 保护，不能因为拆层新增全历史 hot-path 查询，也 `TapeRawEntryReader` 只提供 `getBySession`。Memory management route 先验证 memory row 属于请求 Agent， 再用 `getEffectiveMessageSourceSpan` 读取 retraction/replacement 生效后的最小 message DTO；manifest 列表通过 `listMemoryViewManifestsByAgent` 在 storage query 中执行 Agent、Session、message 和 limit -过滤，route 不自行解析 `payload_json` 或 `meta_json`。 +过滤，route 不自行解析 `payload_json` 或 `meta_json`。架构守卫同时扫描 static import、dynamic +import、CommonJS require、type import 和 re-export，Memory route 不能绕过 inspection port 重新取得 +raw reader、facade 或 domain helper。 ## Privacy 与隔离 diff --git a/docs/architecture/session-management.md b/docs/architecture/session-management.md index 81b8f76ccc..e80d0d837f 100644 --- a/docs/architecture/session-management.md +++ b/docs/architecture/session-management.md @@ -59,7 +59,7 @@ send input；Remote、Scheduler 和 renderer 不得各自维护不同的默认 Session data composition 创建一个 `SessionTape`，对外继续暴露现有 `SessionTapePort`，并按每个 IPC 操作原有的条件和时序调用 `ensureSessionTapeReady`。`src/main/session/data/tape*.ts` 和旧 table path -只是 compatibility re-export，不再拥有 Tape policy 或 persistence。 +只是显式冻结并标记 deprecated 的 compatibility re-export，不再拥有 Tape policy 或 persistence。 - transcript 只接收 `TapeMessageFactWriter`；message replace/retract 与对应 Tape fact 继续共享调用方 SQLite transaction； @@ -71,14 +71,16 @@ Session data composition 创建一个 `SessionTape`，对外继续暴露现有 ` - Tape 的运行中修订通过 append 表达；物理 delete/reset 只由 Session lifecycle 触发。 `SessionTranscript` 和 `SessionSettingsStore` 不提供隐式 `new SessionTape(...)` fallback，必须由正常 -composition 注入共享 connection 上的最小 capability。legacy import 作为 migration composition 显式 -构造同 connection facade 后再注入，避免隐藏的独立 writer 或事务上下文。 - -`clearSessionMessages` 会创建新的 Tape incarnation：entry、mutation projection、search/FTS projection -删除和新 bootstrap 必须在同一个 SQLite transaction 内完成；lifecycle、cleanup 或 bootstrap hard -failure 会完整保留旧 incarnation。mutation projection 沿用 fail-open：新 bootstrap 的 projection -apply 失败时旧 projection row 已删除且 meta 标 stale。最终 Session delete 不创建新 incarnation，继续 -遵循下面的 staged cleanup 顺序。 +composition 注入共享 connection 上的最小 capability。legacy import 作为 migration consumer 复用 +`sessionData.tapeStore` 的 message fact writer，不再构造第二个 facade，避免隐藏的独立 writer 或事务 +上下文。 + +`clearSessionMessages` 会创建新的 Tape incarnation：pending input 删除、transcript 删除和 Tape reset +位于同一个外层 SQLite transaction；Tape 内部的 entry、mutation projection、search/FTS projection +删除和新 bootstrap 作为 savepoint 嵌套。lifecycle、cleanup 或 bootstrap hard failure 会同时恢复上述 +数据并完整保留旧 incarnation。mutation projection 沿用 fail-open：新 bootstrap 的 projection apply +失败时旧 projection row 已删除且 meta 标 stale。最终 Session delete 不创建新 incarnation，继续遵循 +下面的 staged cleanup 顺序。 ## Binding diff --git a/docs/architecture/tape-layering/plan.md b/docs/architecture/tape-layering/plan.md index 61b2ef0fdd..f54ae5d399 100644 --- a/docs/architecture/tape-layering/plan.md +++ b/docs/architecture/tape-layering/plan.md @@ -100,7 +100,8 @@ the loop runner; only reconciliation to the Turn coordinator and ACP adapter; ra capabilities to the Memory coordinator; and `TapeInspectionReader` to Memory routes. No application consumer gets the concrete entry table. Transcript and settings have no concrete facade default: normal composition injects their capabilities from the shared `SessionTape`, and -legacy import composition constructs and injects an explicitly shared-connection facade. +legacy import reuses that composition-owned `TapeMessageFactWriter` instead of constructing a +second facade. `ensureSessionTapeReady` remains at the current Session port boundary. Search and context requests with linked-source scopes keep their existing conditional reconciliation behavior. @@ -111,6 +112,9 @@ with linked-source scopes keep their existing conditional reconciliation behavio that deletes projection rows. - Summary compare-and-set appends its reconstruction anchor inside the same transaction that updates summary state. +- Clear-time pending-input deletion, transcript deletion, and Tape reset run in one outer + shared-connection transaction. The Tape generation transaction nests as a savepoint, so a reset + or bootstrap failure restores every clear-time data family. - Reset deletes entries, mutation projection, search projection, FTS metadata, and FTS rows and appends the new bootstrap within one shared-connection transaction. A propagated transition failure restores the old incarnation. The pre-existing fail-open mutation-projection append @@ -136,6 +140,9 @@ behavior where the current implementation is atomic. - Keep all shared DTOs and `SessionTapePort` signatures unchanged. - Preserve old internal exported symbol names through explicit, frozen compatibility re-exports marked as deprecated. +- Keep methods that existed on the historically exported concrete SQLite classes, while excluding + them from application-facing protocols. Remove non-historical raw-row forwarding helpers from + the facade. - Use `TapeViewManifestAssemblySources` for the complete application source set and `TapeViewManifestLookupMaps` for pure domain lookups. Preserve each historical `TapeViewManifestSourceMaps` shape only at its original legacy import path. @@ -157,13 +164,16 @@ behavior where the current implementation is atomic. delete by invalidating and dropping the derivative within the generation transaction. 3. Prune pre-version-3 and metadata-orphaned projection rows during schema initialization without rebuilding every current projection eagerly. -4. Freeze legacy shim export surfaces; remove unused concrete-store helpers while retaining the - required compatibility facade methods as deprecated exports. +4. Freeze legacy shim export surfaces; remove non-historical facade raw-row helpers, and retain + historical concrete-store methods because the exported classes are compatibility contracts. 5. Rename the canonical domain ViewManifest lookup-map type and make Memory route boundary scans reject static, dynamic, CommonJS, type-import, and re-export bypasses. 6. Reuse the composition-owned Tape fact writer in legacy import, document the same-connection anchor requirement, cache SQLite FTS capability per connection, and replace exception-by-missing mock behavior with explicit failure fixtures. +7. Put the complete `clearMessages` mutation set inside one shared-connection transaction and + document that a failed best-effort fork cleanup leaves permanent, non-retried residue that is + nevertheless fail-closed for merge and identifier reuse. ## Test Strategy @@ -218,8 +228,39 @@ The cumulative review added these focused fixes before the documentation commit: 5. `refactor(tape): narrow anchor reader port` 6. `refactor(tape): narrow storage protocols` +The post-review hardening added these local commits: + +1. `docs(tape): specify follow-up hardening` +2. `test(memory): restore native tape scope` +3. `test(tape): restore layered settings fixture` +4. `fix(tape): harden generation cleanup` +5. `refactor(tape): harden compatibility boundaries` +6. `test(tape): repair memory type contracts` + No commit is pushed. The final review compares the complete branch with `dev`. +## Final Validation Record + +The final focused gates passed: + +- Memory scope discovery classified 65 files with 3 explicit exemptions, and the independent + Memory type gate passed all 65 scoped files. +- Memory behavior passed 749 tests across 46 files. +- Native SQLite and Tape coverage passed 242 tests across 13 files; the 2 skipped tests are + Windows-only handle-locking cases on the current macOS host. +- Memory performance passed all 8 tests, including the 10k/100k Tape range-bound comparison. +- Full node and renderer type checks, formatting, i18n validation, and lint passed. +- The architecture baseline generator passed and refreshed the canonical snapshot against the + final verified code commit. + +The full main-process command completed with 390 passing files and 3 failing files: 4,471 tests +passed, 2 were skipped, and 9 failed. Each affected file was then run in an isolated detached +worktree at the exact `dev` baseline (`e84428b66`), reproducing the same 6 failures in +`mainDatabase.test.ts`, 1 failure in `schedulerService.test.ts`, and 2 failures in +`sessionDataMigrations.sqlite.test.ts`. These are pre-existing baseline failures, not branch +regressions; the project-wide main-process gate therefore remains red for reasons outside this Tape +refactor. + ## Rollback The work is organized into locally reviewable commits. Reverting must proceed in reverse order diff --git a/docs/architecture/tape-layering/spec.md b/docs/architecture/tape-layering/spec.md index 0ebdc2bbc9..7d57b0c63d 100644 --- a/docs/architecture/tape-layering/spec.md +++ b/docs/architecture/tape-layering/spec.md @@ -61,7 +61,7 @@ to treat trace evidence as transcript data or to move it into the Tape entry sch - A reset that creates a new Tape incarnation deletes entries, mutation projection state, and search projection state and appends the new bootstrap anchor in one SQLite transaction. - A discarded fork is fail-closed for merge and identifier reuse even when best-effort physical - cleanup fails. + cleanup fails. Failed cleanup leaves permanent inert residue; no automatic retry is scheduled. ## Capability Boundaries @@ -80,7 +80,10 @@ structural type it needs. `TapeRawEntryReader` exposes only `getBySession`. The returns purpose-built effective-message and Memory ViewManifest DTOs; it never returns a physical Tape row. `TapeAnchorReader` exposes only the latest reconstruction anchor required by settings. Transcript and settings require these capabilities to be injected; only normal or migration -composition may construct the concrete facade. +composition may provide them, and only normal Session composition constructs the concrete facade. +Legacy import reuses the composition-owned `TapeMessageFactWriter`. `TapeViewManifestWriter` +intentionally exposes a `void` append contract because its consumer does not observe the stored +row; the concrete facade's richer return value remains an internal compatibility detail. `TapeViewManifestAssemblySources` names the complete source set assembled by the application service. `TapeViewManifestLookupMaps` names the smaller domain lookup map used by pure @@ -106,7 +109,8 @@ The implementation must account for every current physical-table access: between Tape head and projection head; retained as an explicit read-only infrastructure exception to preserve atomicity and query count. - `app/startupMigrations/legacyChatImportService.ts`: destructive whole-database rebuild; retained - as an explicit startup-migration exception. + as an explicit startup-migration exception while reusing the composition-owned message fact + writer. - Schema catalog and database security table-name lists: metadata, not runtime Tape access. ## Generation and Failure Semantics @@ -141,6 +145,11 @@ the enclosing Tape generation transaction fails atomically. Startup removes base projection rows owned only by pre-version-3 metadata so inert legacy text does not remain on disk or force linked read-only searches onto permanent fallback paths. +`clearMessages` places pending-input deletion, transcript deletion, and Tape reset inside one +outer transaction on that same connection. The Tape generation transaction becomes a nested +savepoint, so any reset, projection cleanup, or bootstrap failure restores all three data families +instead of leaving transcript and Tape at different generations. + The Memory native-test manifest must include every split Tape suite that contains an active native SQLite gate. Scope validation discovers these gated suites independently from the manifest so removing or renaming one cannot silently eliminate the only real-SQLite lifecycle and FTS CI @@ -157,8 +166,8 @@ coverage. 4. `TapeEntryStore` exposes no reset or delete method. 5. Existing `SessionTapePort`, persisted schema, table names, entry payloads, View policy IDs, and renderer contracts remain unchanged. -6. Transcript mutation plus Tape correction, and summary mutation plus anchor append, retain their - current transaction semantics. +6. Transcript mutation plus Tape correction, summary mutation plus anchor append, and clear-time + pending/transcript deletion plus Tape reset share their required transaction context. 7. `ensureSessionTapeReady` remains idempotent and runs at the same Session port boundaries. 8. Projection search fallback remains unchanged. Fork cleanup failure remains non-blocking while its discard receipt prevents later merge or identifier reuse. @@ -176,6 +185,9 @@ coverage. different generations; pre-version-3 projection data is removed during schema initialization. 15. Legacy Tape modules expose frozen deprecated export lists, while canonical modules use unambiguous ViewManifest source-map names. +16. Historical concrete SQLite-class methods remain available through the frozen legacy class + exports, but non-historical raw-row helpers are absent from the `SessionTape` facade and every + application-facing port. ## Constraints diff --git a/docs/architecture/tape-layering/tasks.md b/docs/architecture/tape-layering/tasks.md index 1c73add7a7..54314337c2 100644 --- a/docs/architecture/tape-layering/tasks.md +++ b/docs/architecture/tape-layering/tasks.md @@ -62,8 +62,8 @@ - [x] Move handoff, generic anchor, and fork-message fact ownership into `TapeFactService`. - [x] Restrict `TapeForkService` to fork lifecycle behavior. - [x] Move stored-manifest validation and replay hash helpers into the domain layer. -- [x] Rename the complete source-set DTO to `TapeViewManifestAssemblySources` and retain the old - name as a deprecated alias. +- [x] Rename the complete source-set DTO to `TapeViewManifestAssemblySources` and retain its old + name only as a deprecated legacy-path alias. - [x] Harden architecture guards for main-layer, SQLite, Electron, logging, legacy-path, and Memory route violations, including table-driven negative fixtures. - [x] Cover every legacy compatibility wrapper and the project's actual SQLite driver in the @@ -92,20 +92,23 @@ ## Post-Review Hardening -- [ ] Replace the stale monolithic Tape path in Memory native CI with all SQLite-gated split suites. -- [ ] Make scope validation discover required native Tape coverage independently from the manifest. -- [ ] Make final Session Tape deletion reuse the atomic generation lifecycle helper. -- [ ] Recover failed FTS row deletion by dropping and invalidating the rebuildable derivative. -- [ ] Remove pre-version-3 and metadata-orphaned search projection rows at schema initialization. -- [ ] Add direct reset-bootstrap rollback and corrupt-FTS lifecycle regression tests. -- [ ] Freeze and deprecate legacy Tape compatibility export lists without removing old symbols. -- [ ] Use distinct canonical names for application assembly sources and domain lookup maps. -- [ ] Remove unused concrete storage helpers that are not compatibility contracts. -- [ ] Detect dynamic import, CommonJS require, type import, and re-export Memory route bypasses. -- [ ] Reuse the composition-owned Tape message writer in legacy import. -- [ ] Document the same-connection transaction requirement for settings anchor writers. -- [ ] Cache FTS capability detection per SQLite connection. -- [ ] Replace fallback tests that depend on missing mock methods with explicit failures. -- [ ] Document permanent fork residue when best-effort discard cleanup fails. -- [ ] Run focused native scope, lifecycle, recall, boundary, migration, and compatibility tests. -- [ ] Run full validation and review every new local commit without pushing. +- [x] Replace the stale monolithic Tape path in Memory native CI with all SQLite-gated split suites. +- [x] Make scope validation discover required native Tape coverage independently from the manifest. +- [x] Make final Session Tape deletion reuse the atomic generation lifecycle helper. +- [x] Recover failed FTS row deletion by dropping and invalidating the rebuildable derivative. +- [x] Remove pre-version-3 and metadata-orphaned search projection rows at schema initialization. +- [x] Add direct reset-bootstrap rollback and corrupt-FTS lifecycle regression tests. +- [x] Put pending-input deletion, transcript deletion, and Tape reset in one clear-time transaction. +- [x] Freeze and deprecate legacy Tape compatibility export lists without removing old symbols. +- [x] Use distinct canonical names for application assembly sources and domain lookup maps. +- [x] Audit unused concrete storage helpers and retain those present on historically exported + SQLite classes while removing non-historical facade raw-row forwarding methods. +- [x] Detect dynamic import, CommonJS require, type import, and re-export Memory route bypasses. +- [x] Reuse the composition-owned Tape message writer in legacy import. +- [x] Document the same-connection transaction requirement for settings anchor writers. +- [x] Cache FTS capability detection per SQLite connection. +- [x] Replace fallback tests that depend on missing mock methods with explicit failures. +- [x] Document permanent fork residue when best-effort discard cleanup fails. +- [x] Run focused native scope, lifecycle, recall, boundary, migration, and compatibility tests. +- [x] Run full validation, reproduce the nine unrelated main-suite failures on the exact `dev` + baseline, and review every new local commit without pushing. diff --git a/docs/architecture/tape-system.md b/docs/architecture/tape-system.md index 0b07cc5c41..b5eeb6174b 100644 --- a/docs/architecture/tape-system.md +++ b/docs/architecture/tape-system.md @@ -30,8 +30,9 @@ flowchart TD Stores --> SQLite["Shared Session SQLite connection"] ``` -`src/main/session/data/tape*.ts` 和旧 table modules 只保留 compatibility re-export。新代码必须从 -`src/main/tape/` 或能力 port 导入，不能把兼容路径重新当作 owner。 +`src/main/session/data/tape*.ts` 和旧 table modules 只保留显式、冻结且标记 deprecated 的 +compatibility re-export。新代码必须从 `src/main/tape/` 或能力 port 导入，不能把兼容路径重新当作 +owner，也不能通过 canonical module 的新增导出隐式扩大旧路径合同。 ## 能力端口和组合 @@ -52,7 +53,8 @@ domain policy；外部方法的签名、同步/异步行为、异常和 fallback `TapeRawEntryReader` 只暴露 Memory runtime 实际需要的 `getBySession`。Memory routes 使用的 `TapeInspectionReader` 只返回 effective message source span 与 Memory ViewManifest DTO，不返回 `DeepChatTapeEntryRow`。完整的 manifest assembly source set 命名为 -`TapeViewManifestAssemblySources`；旧 `TapeViewManifestSourceMaps` 仅作为兼容 type alias 保留。 +`TapeViewManifestAssemblySources`，domain lookup map 命名为 `TapeViewManifestLookupMaps`；两种历史 +`TapeViewManifestSourceMaps` 形状只在各自原有 compatibility path 作为 type alias 保留。 `TapeAnchorReader` 只暴露 settings 实际使用的 latest reconstruction anchor；transcript/settings 必须 由 composition 注入 port，不允许在 consumer 内隐式构造 concrete facade。 @@ -62,6 +64,8 @@ domain policy；外部方法的签名、同步/异步行为、异常和 fallback Session lifecycle（包含 fork Session cleanup），不属于运行中 Tape 语义。 - transcript message mutation 与 replacement/retraction fact、summary compare-and-set 与 anchor append 使用同一个 SQLite connection 和调用方 transaction，拆层不能拆开其原子边界。 +- `clearMessages` 在同一外层 transaction 中删除 pending input、transcript projection 并 reset Tape； + Tape generation transaction 作为 savepoint 嵌套，任一 hard failure 会同时恢复三类数据。 - `resetSessionTape` 在同一 transaction 内删除 entry、mutation projection、search/FTS projection 并 创建新 bootstrap；lifecycle/cleanup/bootstrap 的 hard failure 会恢复旧 incarnation。既有 mutation projection append fail-open 策略仍可提交新 Tape，但旧 projection row 已删除且 meta 会标 stale。 @@ -73,8 +77,9 @@ domain policy；外部方法的签名、同步/异步行为、异常和 fallback - search projection 可以重建；projection 不可用或 coverage 不完整时回退 effective Tape search，fork cleanup 的 projection 删除失败仍不阻断主流程，但 discard receipt 会使后续 merge 和相同 fork ID 的显式复用 fail closed。 -- legacy chat import 的全表删除是 migration-only 例外；Memory ingestion projection 为避免并发窗口， - 可以在一条只读 SQL 中同时比较 Tape head 和 projection head。除此之外消费方不得访问物理 Tape 表。 +- legacy chat import 的全表删除是 migration-only 例外，但消息 fact writer 复用 composition 已创建的 + `SessionTape` capability，不再另建 facade；Memory ingestion projection 为避免并发窗口，可以在一条 + 只读 SQL 中同时比较 Tape head 和 projection head。除此之外消费方不得访问物理 Tape 表。 - reset 物理删除当前 Session Tape 后重新 bootstrap；本阶段没有 archive-on-reset，不能把 reset 解释成 append-only 运行语义的一部分。 @@ -128,7 +133,8 @@ Subagent 使用独立 Session 和独立 Tape。完成后父 Session append 一 普通 fork merge 只把 fork head 相对基线的 delta 作为新 entry append 到父 Tape，并追加 merge receipt； 不得改写父 Tape 旧 entry，也不得把整份 fork 历史重复复制。discard 和重复 merge 保持既有审计、幂等及 best-effort projection cleanup 语义。discard cleanup 成功时与 receipt 一起提交；cleanup 失败时回滚本次 -cleanup、仍 append receipt 并记录 warning，残留 fork 也不能再次 merge。 +cleanup、仍 append receipt 并记录 warning。此时残留 fork 是永久惰性数据，本阶段没有自动或后台重试 +路径；discard receipt 仍保证它不能再次 merge，也不能用相同 fork ID 显式复用。 ## 回放和兼容 From a25ff46716470b200cdef3dd7962ffe963d34048 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 23:32:14 +0800 Subject: [PATCH 27/29] fix(tape): validate durable link identities --- src/main/tape/application/forkService.ts | 22 +++++-- src/main/tape/application/lineageService.ts | 20 +++++++ .../tape/application/viewReplayService.ts | 8 ++- test/main/session/data/tapeFork.test.ts | 41 +++++++++++++ test/main/session/data/tapeLineage.test.ts | 58 +++++++++++++++++-- test/main/session/data/tapeViewReplay.test.ts | 23 +++++++- 6 files changed, 158 insertions(+), 14 deletions(-) diff --git a/src/main/tape/application/forkService.ts b/src/main/tape/application/forkService.ts index 1de7e21c56..3dd69371a5 100644 --- a/src/main/tape/application/forkService.ts +++ b/src/main/tape/application/forkService.ts @@ -93,6 +93,10 @@ function forkDiscardProvenanceKey(parentSessionId: string, forkId: string): stri return `fork:${parentSessionId}:${forkId}:discard:event` } +function forkMergeProvenanceKey(parentSessionId: string, forkId: string): string { + return `fork:${parentSessionId}:${forkId}:merge:event` +} + export class TapeForkService { constructor(private readonly providers: TapeForkProviders) {} @@ -103,6 +107,14 @@ export class TapeForkService { createFork(parentSessionId: string, forkId: string = nanoid()): TapeForkHandle { const table = this.table const forkIdValue = forkId.trim() || nanoid() + if ( + table.getByProvenanceKey( + parentSessionId, + forkMergeProvenanceKey(parentSessionId, forkIdValue) + ) + ) { + throw new Error(`Fork ${forkIdValue} has already been merged and cannot be reused.`) + } if ( table.getByProvenanceKey( parentSessionId, @@ -156,7 +168,7 @@ export class TapeForkService { mergeFork(parentSessionId: string, forkId: string): number { const table = this.table const forkSessionIdValue = forkSessionId(parentSessionId, forkId) - const mergeProvenanceKey = `fork:${parentSessionId}:${forkId}:merge:event` + const mergeProvenanceKey = forkMergeProvenanceKey(parentSessionId, forkId) return table.runInTransaction(() => { const existingReceipt = table.getByProvenanceKey(parentSessionId, mergeProvenanceKey) @@ -287,10 +299,10 @@ export class TapeForkService { }, provenanceKey: `fork:${parentSessionId}:${forkId}:external-merge:event`, data: { + ...meta, forkId, forkSessionId: forkSessionIdValue, - referencedEntryCount, - ...meta + referencedEntryCount }, idempotent: true }) @@ -313,9 +325,9 @@ export class TapeForkService { }, provenanceKey: `fork:${parentSessionId}:${forkId}:external-discard:event`, data: { + ...meta, forkId, - forkSessionId: forkSessionIdValue, - ...meta + forkSessionId: forkSessionIdValue }, idempotent: true }) diff --git a/src/main/tape/application/lineageService.ts b/src/main/tape/application/lineageService.ts index 65893fc7ca..42aba422c7 100644 --- a/src/main/tape/application/lineageService.ts +++ b/src/main/tape/application/lineageService.ts @@ -389,6 +389,7 @@ export class TapeLineageService { linkSubagentTape(input: SubagentTapeLinkInput): SubagentTapeLinkReceipt { const table = this.table + const sessionTable = this.providers.getLineageSessionReader() const normalized = normalizeSubagentTapeLinkInput(input) const provenanceKey = subagentTapeLinkProvenanceKey(normalized) return table.runInTransaction(() => { @@ -399,6 +400,25 @@ export class TapeLineageService { return receipt } + const sessionById = new Map( + sessionTable + .getMany([normalized.parentSessionId, normalized.childSessionId]) + .map((session) => [session.id, session]) + ) + const parent = sessionById.get(normalized.parentSessionId) + const child = sessionById.get(normalized.childSessionId) + if ( + !parent || + !child || + child.session_kind !== 'subagent' || + child.parent_session_id !== normalized.parentSessionId + ) { + throw new Error( + `Session ${normalized.childSessionId} is not a direct subagent child of ` + + `${normalized.parentSessionId}.` + ) + } + const childFirstEntry = table.getFirstEntriesBySessions([normalized.childSessionId])[0] if (!childFirstEntry || childFirstEntry.session_id !== normalized.childSessionId) { throw new Error(`Subagent Tape ${normalized.childSessionId} is unavailable.`) diff --git a/src/main/tape/application/viewReplayService.ts b/src/main/tape/application/viewReplayService.ts index 271b42f9bc..1e1f8dbff0 100644 --- a/src/main/tape/application/viewReplayService.ts +++ b/src/main/tape/application/viewReplayService.ts @@ -440,7 +440,13 @@ export class TapeViewReplayService { ? (data as Record).manifest : undefined const manifest = normalizeStoredTapeViewManifest(rawManifest, row.session_id) - if (!manifest) return null + if ( + !manifest || + manifest.messageId !== row.source_id || + manifest.requestSeq !== row.source_seq + ) { + return null + } return { sessionId: row.session_id, diff --git a/test/main/session/data/tapeFork.test.ts b/test/main/session/data/tapeFork.test.ts index eee8f43ee9..be7da20c8a 100644 --- a/test/main/session/data/tapeFork.test.ts +++ b/test/main/session/data/tapeFork.test.ts @@ -87,6 +87,9 @@ describe('SessionTape forks', () => { expect(service.mergeFork('parent', 'bounded')).toBe(1) expect(service.mergeFork('parent', 'bounded')).toBe(1) + expect(() => service.createFork('parent', 'bounded')).toThrow( + 'Fork bounded has already been merged and cannot be reused.' + ) const parentEntries = entries.filter((entry) => entry.session_id === 'parent') expect(parentEntries.filter((entry) => entry.source_type === 'fork')).toHaveLength(2) @@ -281,6 +284,44 @@ describe('SessionTape forks', () => { ) }) + it('keeps external fork receipt identity fields authoritative over metadata', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.ensureBootstrapAnchor('external-child') + + const merge = service.recordExternalForkMerge('parent', 'external-child', 'external-child', { + forkId: 'spoofed', + forkSessionId: 'spoofed-session', + referencedEntryCount: 999, + taskId: 'task-1' + }) + const discard = service.recordExternalForkDiscard( + 'parent', + 'external-child', + 'external-child', + { + forkId: 'spoofed', + forkSessionId: 'spoofed-session', + taskId: 'task-1' + } + ) + + expect(JSON.parse(merge.payload_json).data).toEqual({ + forkId: 'external-child', + forkSessionId: 'external-child', + referencedEntryCount: 1, + taskId: 'task-1' + }) + expect(JSON.parse(discard.payload_json).data).toEqual({ + forkId: 'external-child', + forkSessionId: 'external-child', + taskId: 'task-1' + }) + }) + itIfSqlite('rolls back copied fork entries and the receipt when merge fails', () => { const db = new DatabaseCtor(':memory:') const table = new DeepChatTapeEntriesTable(db) diff --git a/test/main/session/data/tapeLineage.test.ts b/test/main/session/data/tapeLineage.test.ts index 6c13bd63b0..da26bf7726 100644 --- a/test/main/session/data/tapeLineage.test.ts +++ b/test/main/session/data/tapeLineage.test.ts @@ -16,10 +16,11 @@ import { describe('SessionTape lineage', () => { it('links a frozen subagent Tape without copying child entries and retries idempotently', () => { const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'child-2', session_kind: 'subagent', parent_session_id: 'parent' } + ]) table.ensureBootstrapAnchor('parent') table.ensureBootstrapAnchor('child') @@ -512,16 +513,36 @@ describe('SessionTape lineage', () => { expect(table.append).not.toHaveBeenCalled() }) - it('rejects sibling and unlinked source ids even when a forged link event exists', () => { + it('rejects non-direct children at write time and after reparenting', () => { const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ + const { service, sessionById } = createLinkedTapeService(table, [ { id: 'parent', session_kind: 'regular', parent_session_id: null }, { id: 'other-parent', session_kind: 'regular', parent_session_id: null }, { id: 'sibling', session_kind: 'subagent', parent_session_id: 'other-parent' } ]) table.ensureBootstrapAnchor('parent') table.ensureBootstrapAnchor('sibling') + + expect(() => service.linkSubagentTape(createSubagentLinkInput('parent', 'sibling'))).toThrow( + 'Session sibling is not a direct subagent child of parent.' + ) + expect( + table + .getBySession('parent') + .some((entry: { name: string | null }) => entry.name === 'subagent/tape_linked') + ).toBe(false) + + sessionById.set('sibling', { + id: 'sibling', + session_kind: 'subagent', + parent_session_id: 'parent' + }) service.linkSubagentTape(createSubagentLinkInput('parent', 'sibling')) + sessionById.set('sibling', { + id: 'sibling', + session_kind: 'subagent', + parent_session_id: 'other-parent' + }) let error: unknown try { @@ -536,6 +557,31 @@ describe('SessionTape lineage', () => { }) }) + it('rejects missing parents and non-subagent children before persisting a link', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'regular-child', session_kind: 'regular', parent_session_id: 'parent' }, + { id: 'orphan', session_kind: 'subagent', parent_session_id: 'missing-parent' } + ]) + table.ensureBootstrapAnchor('regular-child') + table.ensureBootstrapAnchor('orphan') + + expect(() => + service.linkSubagentTape(createSubagentLinkInput('parent', 'regular-child')) + ).toThrow('Session regular-child is not a direct subagent child of parent.') + expect(() => + service.linkSubagentTape(createSubagentLinkInput('missing-parent', 'orphan')) + ).toThrow('Session orphan is not a direct subagent child of missing-parent.') + expect( + ['parent', 'missing-parent'].some((sessionId) => + table + .getBySession(sessionId) + .some((entry: { name: string | null }) => entry.name === 'subagent/tape_linked') + ) + ).toBe(false) + }) + it('reports a finalized linked Tape as unavailable after its durable session is deleted', () => { const { table } = createTapeTableMock() const { service, sessionById } = createLinkedTapeService(table, [ diff --git a/test/main/session/data/tapeViewReplay.test.ts b/test/main/session/data/tapeViewReplay.test.ts index ebb7e6fa62..1af5777a55 100644 --- a/test/main/session/data/tapeViewReplay.test.ts +++ b/test/main/session/data/tapeViewReplay.test.ts @@ -363,6 +363,25 @@ describe('SessionTape view and replay', () => { expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) }) + it('rejects view manifests that disagree with their persisted source envelope', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 1 }, + data: { manifest: createObservationManifest({ messageId: 'other', requestSeq: 1 }) } + }) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 2 }, + data: { manifest: createObservationManifest({ messageId: 'a1', requestSeq: 1 }) } + }) + + expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) + }) + it('normalizes legacy manifests without hashVersion to hashVersion 1', () => { const { table } = createTapeTableMock() const service = new SessionTape({ @@ -408,7 +427,7 @@ describe('SessionTape view and replay', () => { table.appendEvent({ sessionId: 's1', name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 99 }, + source: { type: 'runtime_event', id: 'a1', seq: 1 }, data: { manifest: legacyManifest } }) @@ -514,7 +533,7 @@ describe('SessionTape view and replay', () => { table.appendEvent({ sessionId: 's1', name: 'view/assembled', - source: { type: 'runtime_event', id: 'a2', seq: 99 }, + source: { type: 'runtime_event', id: 'a2', seq: 1 }, data: { manifest: { ...tamperedManifest, latestEntryId: tamperedManifest.latestEntryId + 1 } } }) From 756ec816cb6f9d491518ecd2bd181b9e3bd23835 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 23:36:19 +0800 Subject: [PATCH 28/29] fix(tape): harden derived state recovery --- src/main/tape/application/factPersistence.ts | 4 +- .../tape/application/reconcilerService.ts | 6 +- src/main/tape/domain/effectiveSemantics.ts | 4 +- .../sqlite/tapeSearchProjectionStore.ts | 5 ++ test/main/session/data/tapeFacts.test.ts | 25 +++++- test/main/session/data/tapeRecall.test.ts | 86 +++++++++++++++++++ test/main/session/data/tapeReconciler.test.ts | 5 +- 7 files changed, 125 insertions(+), 10 deletions(-) diff --git a/src/main/tape/application/factPersistence.ts b/src/main/tape/application/factPersistence.ts index 186ccde9d2..a320777677 100644 --- a/src/main/tape/application/factPersistence.ts +++ b/src/main/tape/application/factPersistence.ts @@ -53,7 +53,7 @@ function collectPendingInteractionToolIds(blocks: AssistantMessageBlock[]): Set< const ids = new Set() for (const block of blocks) { if ( - block.type === 'action' && + block?.type === 'action' && (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && block.status === 'pending' && typeof block.tool_call?.id === 'string' && @@ -72,7 +72,7 @@ export function buildTapeToolFactInputs(record: ChatMessageRecord): TapeToolFact const blocks = parseAssistantBlocks(record.content) const pendingInteractionToolIds = collectPendingInteractionToolIds(blocks) blocks.forEach((block, blockIndex) => { - if (block.type !== 'tool_call' || !block.tool_call) return + if (block?.type !== 'tool_call' || !block.tool_call) return if (block.status !== 'success' && block.status !== 'error') return const toolCallId = block.tool_call.id diff --git a/src/main/tape/application/reconcilerService.ts b/src/main/tape/application/reconcilerService.ts index 575d4bac4b..b8379cbda2 100644 --- a/src/main/tape/application/reconcilerService.ts +++ b/src/main/tape/application/reconcilerService.ts @@ -29,9 +29,9 @@ export class TapeReconcilerService { messageStore: TapeTranscriptReader ): TapeBackfillResult { const table = this.table - const historyRecords = messageStore - .getMessages(sessionId) - .sort((left, right) => left.orderSeq - right.orderSeq) + const historyRecords = [...messageStore.getMessages(sessionId)].sort( + (left, right) => left.orderSeq - right.orderSeq + ) const maxOrderSeq = historyRecords.reduce( (currentMax, record) => Math.max(currentMax, record.orderSeq), 0 diff --git a/src/main/tape/domain/effectiveSemantics.ts b/src/main/tape/domain/effectiveSemantics.ts index 058d96de25..a50091b1a9 100644 --- a/src/main/tape/domain/effectiveSemantics.ts +++ b/src/main/tape/domain/effectiveSemantics.ts @@ -44,7 +44,7 @@ export function messageRecordHasFinalToolUse(record: ChatMessageRecord): boolean const blocks = parseAssistantBlocks(record.content) const pendingInteractionToolIds = new Set( blocks.flatMap((block) => - block.type === 'action' && + block?.type === 'action' && (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && block.status === 'pending' && typeof block.tool_call?.id === 'string' @@ -54,7 +54,7 @@ export function messageRecordHasFinalToolUse(record: ChatMessageRecord): boolean ) return blocks.some( (block) => - block.type === 'tool_call' && + block?.type === 'tool_call' && (block.status === 'success' || block.status === 'error') && typeof block.tool_call?.id === 'string' && !pendingInteractionToolIds.has(block.tool_call.id) diff --git a/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts index f476bc7817..3c4d8db625 100644 --- a/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts +++ b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts @@ -833,6 +833,11 @@ export class DeepChatTapeSearchProjectionTable ) .all(...params) as DeepChatTapeSearchProjectionResultRow[] } catch { + try { + this.dropFtsIndex() + } catch { + this.ftsReady = false + } return [] } } diff --git a/test/main/session/data/tapeFacts.test.ts b/test/main/session/data/tapeFacts.test.ts index 6dbfcf63c5..98e10024c6 100644 --- a/test/main/session/data/tapeFacts.test.ts +++ b/test/main/session/data/tapeFacts.test.ts @@ -2,7 +2,10 @@ import { describe, expect, it } from 'vitest' import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' import { appendMessageRecordToTape, appendToolFactsToTape } from '@/session/data/tapeFacts' import { buildEffectiveTapeView } from '@/session/data/tapeEffectiveView' -import { tapeToolRank } from '@/session/data/tables/deepchatTapeEffectiveSemantics' +import { + messageRecordHasFinalToolUse, + tapeToolRank +} from '@/session/data/tables/deepchatTapeEffectiveSemantics' import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' function createTable() { @@ -162,6 +165,26 @@ describe('appendToolFactsToTape', () => { expect(toolCallIds).toEqual(['done1']) }) + it('ignores malformed block elements without shifting valid tool provenance', () => { + const table = createTable() + const record = assistantRecord([ + null, + 'malformed', + toolCallBlock('success', 'valid', 'done') + ] as unknown as AssistantMessageBlock[]) + + expect(messageRecordHasFinalToolUse(record)).toBe(true) + expect(appendToolFactsToTape(table as any, record, 'live', 'tool_loop')).toBe(2) + expect(table.rows.filter((row) => row.kind === 'tool_call')).toMatchObject([ + { source_id: 'a1:valid', source_seq: 2 } + ]) + expect( + messageRecordHasFinalToolUse( + assistantRecord([null, 42] as unknown as AssistantMessageBlock[]) + ) + ).toBe(false) + }) + it('dedupes a mid-loop snapshot and the finalize write to one effective tool fact', () => { const table = createTable() const pending = assistantRecord([toolCallBlock('success', 'tc1', 'result')], { diff --git a/test/main/session/data/tapeRecall.test.ts b/test/main/session/data/tapeRecall.test.ts index 54e9519027..52840d8c2f 100644 --- a/test/main/session/data/tapeRecall.test.ts +++ b/test/main/session/data/tapeRecall.test.ts @@ -1090,6 +1090,92 @@ describe('SessionTape recall', () => { ).toBe(false) }) + it('invalidates a failed FTS query and rebuilds the derivative before the next search', () => { + const projectionRow = { + session_id: 's1', + entry_id: 1, + kind: 'message', + name: 'message/user', + source_type: 'message', + source_id: 'm1', + source_seq: 0, + search_text: 'Redis TTL', + summary_text: 'Redis TTL', + refs_json: '{}', + created_at: 100 + } + let ftsMetaPresent = true + let shouldFailFtsQuery = true + let likeQueryCount = 0 + let ftsQueryCount = 0 + const exec = vi.fn() + const prepare = vi.fn((sql: string) => ({ + all: (..._params: unknown[]) => { + if (sql.includes('bm25(deepchat_tape_search_fts)')) { + ftsQueryCount += 1 + if (shouldFailFtsQuery) { + shouldFailFtsQuery = false + throw new Error('injected corrupt FTS query') + } + return [{ ...projectionRow, score: -1 }] + } + if (sql.includes('NULL AS score')) { + likeQueryCount += 1 + return [{ ...projectionRow, score: null }] + } + if (sql.includes('SELECT *') && sql.includes('deepchat_tape_search_projection')) { + return [projectionRow] + } + return [] + }, + get: (..._params: unknown[]) => { + if (sql.includes('sqlite_master') && sql.includes('deepchat_tape_search_fts_meta')) { + return { name: 'deepchat_tape_search_fts_meta' } + } + if (sql.includes('FROM deepchat_tape_search_projection_meta')) { + return { projection_version: 1, max_entry_id: 1 } + } + if (sql.includes('FROM deepchat_tape_search_fts_meta')) { + return ftsMetaPresent ? { projection_version: 1, max_entry_id: 1 } : undefined + } + if (sql.includes('SELECT rowid')) { + return { rowid: 1 } + } + return undefined + }, + run: (..._params: unknown[]) => { + if (sql.includes('DELETE FROM deepchat_tape_search_fts_meta')) { + ftsMetaPresent = false + } + if (sql.includes('INSERT INTO deepchat_tape_search_fts_meta')) { + ftsMetaPresent = true + } + } + })) + const projectionTable = new DeepChatTapeSearchProjectionTable({ + exec, + prepare, + transaction: (callback: () => void) => callback + } as any) + ;(projectionTable as any).ftsReady = true + + expect(projectionTable.search('s1', 'Redis', { limit: 1 })).toMatchObject([ + { entry_id: 1, score: null } + ]) + expect(projectionTable.hasFtsReadyForTesting()).toBe(false) + expect(exec).toHaveBeenCalledWith('DROP TABLE IF EXISTS deepchat_tape_search_fts') + + expect(projectionTable.search('s1', 'Redis', { limit: 1 })).toMatchObject([ + { entry_id: 1, score: -1 } + ]) + expect(projectionTable.hasFtsReadyForTesting()).toBe(true) + expect(ftsQueryCount).toBe(2) + expect(likeQueryCount).toBe(1) + expect( + exec.mock.calls.some(([sql]) => String(sql).includes('CREATE VIRTUAL TABLE IF NOT EXISTS')) + ).toBe(true) + }) + it('queries memory view manifests by agent without expanding session ids', () => { const all = vi.fn().mockReturnValue([]) const db = { diff --git a/test/main/session/data/tapeReconciler.test.ts b/test/main/session/data/tapeReconciler.test.ts index 8dfa450bf1..7178ce313c 100644 --- a/test/main/session/data/tapeReconciler.test.ts +++ b/test/main/session/data/tapeReconciler.test.ts @@ -25,7 +25,6 @@ describe('SessionTape reconciliation and facts', () => { } ] const records = [ - createRecord({ id: 'u1', orderSeq: 1 }), createRecord({ id: 'a1', orderSeq: 2, @@ -33,7 +32,8 @@ describe('SessionTape reconciliation and facts', () => { content: JSON.stringify(assistantBlocks), createdAt: 120, updatedAt: 120 - }) + }), + createRecord({ id: 'u1', orderSeq: 1 }) ] const messageStore = { getMessages: vi.fn().mockReturnValue(records) @@ -48,6 +48,7 @@ describe('SessionTape reconciliation and facts', () => { expect(first.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) expect(second.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) + expect(records.map((record) => record.id)).toEqual(['a1', 'u1']) expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) From a0470d5b380f20f845781b17132d7084791d59a7 Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Fri, 17 Jul 2026 23:40:41 +0800 Subject: [PATCH 29/29] test(tape): align boundary fixtures --- scripts/check-memory-test-scope.mjs | 2 +- test/main/app/compositionBoundaries.test.ts | 4 +- test/main/routes/dispatcher.test.ts | 132 ++++++++---------- test/main/scripts/memoryTestScope.test.ts | 11 +- test/main/session/data/tapeReconciler.test.ts | 17 +++ test/main/session/data/tapeTestHarness.ts | 13 +- 6 files changed, 97 insertions(+), 82 deletions(-) diff --git a/scripts/check-memory-test-scope.mjs b/scripts/check-memory-test-scope.mjs index ff50fe4913..1ce061efc7 100644 --- a/scripts/check-memory-test-scope.mjs +++ b/scripts/check-memory-test-scope.mjs @@ -7,7 +7,7 @@ const MEMORY_IMPORT = /from ['"][^'"]*(?:\/memory(?:\/|['"])|agent-memory|memory\.routes|agentMemory|recallKeyword)[^'"]*['"]/ const TEST_FILE = /\.(?:test|spec)\.[cm]?[jt]sx?$|\.perf\.[cm]?[jt]sx?$/ const NATIVE_TAPE_TEST = - /^test\/main\/session\/data\/(?:tape[^/]*\.test\.ts|tables\/deepchatTapeEntriesTable\.test\.ts)$/ + /^test\/main\/session\/data\/(?:tape[^/]*|tables\/deepchatTapeEntriesTable)\.(?:test|spec)\.ts$/ const NATIVE_SQLITE_GATE = /\b(?:itIfSqlite|describeIfSqlite)\b/ function walkTestFiles(rootDir) { diff --git a/test/main/app/compositionBoundaries.test.ts b/test/main/app/compositionBoundaries.test.ts index 571253b695..6222d0e172 100644 --- a/test/main/app/compositionBoundaries.test.ts +++ b/test/main/app/compositionBoundaries.test.ts @@ -10,7 +10,9 @@ describe('session boundary composition', () => { ) expect(compositionSource.match(/new LegacyChatImportService\(/g)).toHaveLength(1) - expect(compositionSource).toContain('memoryDatabase,\n sessionData.tapeStore') + expect(compositionSource).toMatch( + /new LegacyChatImportService\([^)]*memoryDatabase,\s*sessionData\.tapeStore/ + ) expect(compositionSource).toContain( 'legacyChatImportService.repairImportedLegacySessionSkills(conversationId)' ) diff --git a/test/main/routes/dispatcher.test.ts b/test/main/routes/dispatcher.test.ts index 7595b91e85..5c79a6e5e2 100644 --- a/test/main/routes/dispatcher.test.ts +++ b/test/main/routes/dispatcher.test.ts @@ -14,6 +14,7 @@ import type { IYoBrowserPresenter } from '@shared/types/desktop' import type { MainDatabase } from '@/data/mainDatabase' +import type { TapeInspectionReader } from '@/tape/ports/capabilities' import type { OAuthServicePort } from '@shared/types/oauth' import type { DialogServicePort } from '@shared/types/dialog' import type { DeviceServicePort } from '@shared/types/device' @@ -1271,12 +1272,12 @@ function createRuntime() { repairSchema: vi.fn().mockResolvedValue(databaseRepairReport), agentMemoryAuditTable: { listByAgent: vi.fn(() => []) - }, - deepchatTapeEntriesTable: { - getEffectiveMessageSourceSpan: vi.fn(() => []), - listMemoryViewManifestsByAgent: vi.fn(() => []) } } as unknown as MainDatabase + const tapeInspection: TapeInspectionReader = { + getEffectiveMessageSourceSpan: vi.fn(() => []), + listMemoryViewManifestsByAgent: vi.fn(() => []) + } const cronJob = { id: 'cron-1', name: 'Cron smoke', @@ -1515,7 +1516,7 @@ function createRuntime() { const memoryRoutes = createMemoryRoutes({ memoryService, getAgentType: (agentId) => providerSettings.getAgentType(agentId), - getTapeInspection: () => (sqlitePresenter as any).deepchatTapeEntriesTable, + getTapeInspection: () => tapeInspection, getAuditEntries: () => (sqlitePresenter as any).agentMemoryAuditTable }) const desktopRoutes = createDesktopRoutes({ @@ -1754,6 +1755,7 @@ function createRuntime() { remoteService, shortcutPresenter, sqlitePresenter, + tapeInspection, windowPresenter, deviceService, appDataReset, @@ -2380,30 +2382,29 @@ describe('dispatchDeepchatRoute', () => { }) it('filters memory view manifests by message before applying the requested limit', async () => { - const { runtime, providerSettings } = createRuntime() + const { runtime, providerSettings, tapeInspection } = createRuntime() vi.mocked(providerSettings.getAgentType).mockResolvedValueOnce('deepchat') const listSessions = vi.fn() - const listMemoryViewManifestsByAgent = vi.fn().mockReturnValue([ - { - sessionId: 's1', - messageId: 'msg-old', - entryId: 10, - policyVersion: 1, - tokenBudget: 900, - estimatedTokens: 9, - selectedCount: 1, - selectedIds: ['old'], - droppedCount: 1, - queryHash: 'oldhash', - createdAt: 100 - } - ]) + const listMemoryViewManifestsByAgent = vi + .mocked(tapeInspection.listMemoryViewManifestsByAgent) + .mockReturnValue([ + { + sessionId: 's1', + messageId: 'msg-old', + entryId: 10, + policyVersion: 1, + tokenBudget: 900, + estimatedTokens: 9, + selectedCount: 1, + selectedIds: ['old'], + droppedCount: 1, + queryHash: 'oldhash', + createdAt: 100 + } + ]) ;(runtime as any).sqlitePresenter = { newSessionsTable: { list: listSessions - }, - deepchatTapeEntriesTable: { - listMemoryViewManifestsByAgent } } @@ -2435,9 +2436,9 @@ describe('dispatchDeepchatRoute', () => { }) it('returns Tape inspection manifest DTOs without exposing raw rows', async () => { - const { runtime, providerSettings } = createRuntime() + const { runtime, providerSettings, tapeInspection } = createRuntime() vi.mocked(providerSettings.getAgentType).mockResolvedValueOnce('deepchat') - const listMemoryViewManifestsByAgent = vi.fn().mockReturnValue([ + vi.mocked(tapeInspection.listMemoryViewManifestsByAgent).mockReturnValue([ { sessionId: 's1', messageId: 'msg-1', @@ -2452,12 +2453,6 @@ describe('dispatchDeepchatRoute', () => { createdAt: 300 } ]) - ;(runtime as any).sqlitePresenter = { - deepchatTapeEntriesTable: { - listMemoryViewManifestsByAgent - } - } - const result = await dispatchDeepchatRoute( runtime, 'memory.listViewManifests', @@ -2478,7 +2473,7 @@ describe('dispatchDeepchatRoute', () => { }) it('reads memory source spans through the DTO-only Tape inspection port', async () => { - const { runtime } = createRuntime() + const { runtime, tapeInspection } = createRuntime() const getManagementVisibleByIds = vi.fn().mockReturnValue([ { id: 'memory-1', @@ -2487,22 +2482,19 @@ describe('dispatchDeepchatRoute', () => { source_entry_ids: '[2,3]' } ]) - const getEffectiveMessageSourceSpan = vi.fn().mockReturnValue([ - { - entryId: 2, - record: { - role: 'user', - orderSeq: 1, - content: JSON.stringify({ text: 'source context' }) + const getEffectiveMessageSourceSpan = vi + .mocked(tapeInspection.getEffectiveMessageSourceSpan) + .mockReturnValue([ + { + entryId: 2, + record: { + role: 'user', + orderSeq: 1, + content: JSON.stringify({ text: 'source context' }) + } } - } - ]) + ]) ;(runtime as any).memoryService = { getManagementVisibleByIds } - ;(runtime as any).sqlitePresenter = { - deepchatTapeEntriesTable: { - getEffectiveMessageSourceSpan - } - } const result = await dispatchDeepchatRoute( runtime, @@ -2700,13 +2692,8 @@ describe('dispatchDeepchatRoute', () => { }) it('returns no memory view manifests for missing or non-DeepChat agents', async () => { - const { runtime, providerSettings } = createRuntime() - const listMemoryViewManifestsByAgent = vi.fn() - ;(runtime as any).sqlitePresenter = { - deepchatTapeEntriesTable: { - listMemoryViewManifestsByAgent - } - } + const { runtime, providerSettings, tapeInspection } = createRuntime() + const listMemoryViewManifestsByAgent = vi.mocked(tapeInspection.listMemoryViewManifestsByAgent) vi.mocked(providerSettings.getAgentType) .mockResolvedValueOnce(null) .mockResolvedValueOnce('acp') @@ -2801,32 +2788,31 @@ describe('dispatchDeepchatRoute', () => { }) it('does not expand all sessions when listing memory view manifests', async () => { - const { runtime, providerSettings } = createRuntime() + const { runtime, providerSettings, tapeInspection } = createRuntime() vi.mocked(providerSettings.getAgentType).mockResolvedValueOnce('deepchat') const listSessions = vi.fn(() => Array.from({ length: 1200 }, (_, index) => ({ id: `s-${index}` })) ) - const listMemoryViewManifestsByAgent = vi.fn().mockReturnValue([ - { - sessionId: 's-1199', - messageId: 'msg-1', - entryId: 1, - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 10, - selectedCount: 1, - selectedIds: ['m1'], - droppedCount: 0, - queryHash: 'hash', - createdAt: 100 - } - ]) + const listMemoryViewManifestsByAgent = vi + .mocked(tapeInspection.listMemoryViewManifestsByAgent) + .mockReturnValue([ + { + sessionId: 's-1199', + messageId: 'msg-1', + entryId: 1, + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: 10, + selectedCount: 1, + selectedIds: ['m1'], + droppedCount: 0, + queryHash: 'hash', + createdAt: 100 + } + ]) ;(runtime as any).sqlitePresenter = { newSessionsTable: { list: listSessions - }, - deepchatTapeEntriesTable: { - listMemoryViewManifestsByAgent } } diff --git a/test/main/scripts/memoryTestScope.test.ts b/test/main/scripts/memoryTestScope.test.ts index 4ecbdad881..4c5117c4e0 100644 --- a/test/main/scripts/memoryTestScope.test.ts +++ b/test/main/scripts/memoryTestScope.test.ts @@ -157,23 +157,32 @@ describe('memory test scope guard', () => { it('discovers only split Tape suites with native SQLite gates', () => { const paths = [ 'test/main/session/data/tapeRecall.test.ts', + 'test/main/session/data/tapeReplay.spec.ts', 'test/main/session/data/tapeReconciler.test.ts', 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.spec.ts', 'test/main/memory/agentMemoryTable.test.ts' ] const contents = new Map([ ['test/main/session/data/tapeRecall.test.ts', 'itIfSqlite(\'uses SQLite\', () => {})'], + ['test/main/session/data/tapeReplay.spec.ts', 'describeIfSqlite(\'uses SQLite\', () => {})'], ['test/main/session/data/tapeReconciler.test.ts', 'it(\'stays portable\', () => {})'], [ 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts', 'describeIfSqlite(\'uses SQLite\', () => {})' ], + [ + 'test/main/session/data/tables/deepchatTapeEntriesTable.spec.ts', + 'itIfSqlite(\'uses SQLite\', () => {})' + ], ['test/main/memory/agentMemoryTable.test.ts', 'itIfSqlite(\'outside Tape\', () => {})'] ]) expect(findRequiredNativeTapeTests(paths, (path) => contents.get(path) ?? '')).toEqual([ 'test/main/session/data/tapeRecall.test.ts', - 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts' + 'test/main/session/data/tapeReplay.spec.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.spec.ts' ]) }) }) diff --git a/test/main/session/data/tapeReconciler.test.ts b/test/main/session/data/tapeReconciler.test.ts index 7178ce313c..28bf1077fd 100644 --- a/test/main/session/data/tapeReconciler.test.ts +++ b/test/main/session/data/tapeReconciler.test.ts @@ -14,6 +14,23 @@ import { } from './tapeTestHarness' describe('SessionTape reconciliation and facts', () => { + it('keeps unkeyed idempotent harness appends distinct like the SQLite store', () => { + const { table, entries } = createTapeTableMock() + const input = { + sessionId: 's1', + kind: 'event', + name: 'unkeyed', + payload: { value: 1 }, + idempotent: true + } + + table.append(input) + table.append(input) + + expect(entries).toHaveLength(2) + expect(entries.map((entry) => entry.entry_id)).toEqual([1, 2]) + }) + it('backfills message and tool facts idempotently before returning tape records', () => { const { table, entries } = createTapeTableMock() const assistantBlocks = [ diff --git a/test/main/session/data/tapeTestHarness.ts b/test/main/session/data/tapeTestHarness.ts index 29819a3625..5853059a3b 100644 --- a/test/main/session/data/tapeTestHarness.ts +++ b/test/main/session/data/tapeTestHarness.ts @@ -90,12 +90,13 @@ function createTapeTableMock() { input.name ?? '' ].join(':') : null - const existing = input.idempotent - ? entries.find( - (entry) => - entry.session_id === input.sessionId && entry.provenance_key === provenanceKey - ) - : null + const existing = + input.idempotent && provenanceKey + ? entries.find( + (entry) => + entry.session_id === input.sessionId && entry.provenance_key === provenanceKey + ) + : null if (existing) { return existing }