From 7223f4584b90ce0ceb2eaf9c32f72dc16875dec7 Mon Sep 17 00:00:00 2001 From: xiaomo Date: Sat, 18 Jul 2026 09:10:53 +0800 Subject: [PATCH 1/6] fix(remote): alert on Feishu delivery errors (#1992) --- src/main/app/composition.ts | 3 + src/main/remote/channels/feishu/adapter.ts | 6 ++ .../remote/channels/feishu/feishuRuntime.ts | 18 ++++ src/main/remote/index.ts | 46 +++++++++ src/main/remote/ports.ts | 10 ++ test/main/remote/feishuAdapter.test.ts | 29 ++++++ test/main/remote/feishuRuntime.test.ts | 94 ++++++++++++++++++- test/main/remote/remoteService.test.ts | 67 +++++++++++++ 8 files changed, 272 insertions(+), 1 deletion(-) diff --git a/src/main/app/composition.ts b/src/main/app/composition.ts index fc74d0499d..c2eef8f0ab 100644 --- a/src/main/app/composition.ts +++ b/src/main/app/composition.ts @@ -1263,6 +1263,9 @@ export async function createMainProcessControl(dependencies: { projection: sessionQuery, desktop: { openSession: (sessionId) => openRemoteSession(sessionId) + }, + notifications: { + showNotification: (options) => notificationService.showNotification(options) } }) cronJobs = new SchedulerService({ diff --git a/src/main/remote/channels/feishu/adapter.ts b/src/main/remote/channels/feishu/adapter.ts index b2e64c088f..a99aa5c703 100644 --- a/src/main/remote/channels/feishu/adapter.ts +++ b/src/main/remote/channels/feishu/adapter.ts @@ -21,6 +21,7 @@ type FeishuAdapterDeps = { bindingStore: RemoteBindingStore createConversationRunner: () => RemoteConversationRunner onFatalError?: (message: string) => Promise | void + onDeliveryError?: (message: string) => void configSignature?: string } @@ -28,6 +29,7 @@ export class FeishuAdapter extends ChannelAdapter { private readonly bindingStore: RemoteBindingStore private readonly createConversationRunner: () => RemoteConversationRunner private readonly onFatalError?: (message: string) => Promise | void + private readonly onDeliveryError?: (message: string) => void private readonly credentials: { brand: FeishuBrand appId: string @@ -47,6 +49,7 @@ export class FeishuAdapter extends ChannelAdapter { this.bindingStore = deps.bindingStore this.createConversationRunner = deps.createConversationRunner this.onFatalError = deps.onFatalError + this.onDeliveryError = deps.onDeliveryError this.credentials = { brand: config.channelConfig.brand === 'lark' ? 'lark' : 'feishu', appId: String(config.channelConfig.appId ?? '').trim(), @@ -80,6 +83,9 @@ export class FeishuAdapter extends ChannelAdapter { }, onFatalError: (message) => { void this.onFatalError?.(message) + }, + onDeliveryError: (message) => { + this.onDeliveryError?.(message) } }) diff --git a/src/main/remote/channels/feishu/feishuRuntime.ts b/src/main/remote/channels/feishu/feishuRuntime.ts index f80ba91cd5..3b89414396 100644 --- a/src/main/remote/channels/feishu/feishuRuntime.ts +++ b/src/main/remote/channels/feishu/feishuRuntime.ts @@ -31,6 +31,8 @@ const safeErrorMessage = (error: unknown): string => error instanceof Error ? error.message : String(error) const FEISHU_INTERNAL_ERROR_REPLY = 'An internal error occurred while processing your request.' +const FEISHU_INBOUND_MESSAGE_ERROR = 'Failed to handle inbound message.' +const FEISHU_REPLY_DELIVERY_ERROR = 'Failed to deliver reply to Feishu.' const FEISHU_STREAMING_CARD_FALLBACK_NOTICE = 'Feishu CardKit streaming failed. Falling back to normal message updates. Check that the app has im:message and cardkit:card:write permissions.' @@ -45,6 +47,7 @@ type FeishuRuntimeDeps = { } onStatusChange?: (snapshot: FeishuRuntimeStatusSnapshot) => void onFatalError?: (message: string) => void + onDeliveryError?: (message: string) => void } type FeishuProcessedInboundEntry = { @@ -321,6 +324,10 @@ export class FeishuRuntime { } } } + + if (this.isCurrentRun(runId) && this.statusSnapshot.lastError) { + this.setStatus({ lastError: null }) + } } catch (error) { const diagnostics = { runId, @@ -345,6 +352,10 @@ export class FeishuRuntime { return } + // Keep the runtime running but surface the failure in the status snapshot + // (settings page and /status), since the user otherwise sees nothing. + this.setStatus({ lastError: FEISHU_INBOUND_MESSAGE_ERROR }) + try { if (!this.isCurrentRun(runId)) { return @@ -358,6 +369,13 @@ export class FeishuRuntime { eventId: message.eventId, error: sendError }) + if (this.isCurrentRun(runId)) { + const deliveryError = FEISHU_REPLY_DELIVERY_ERROR + this.setStatus({ + lastError: deliveryError + }) + this.deps.onDeliveryError?.(deliveryError) + } } } } diff --git a/src/main/remote/index.ts b/src/main/remote/index.ts index 0f25b31e44..0a732b7540 100644 --- a/src/main/remote/index.ts +++ b/src/main/remote/index.ts @@ -87,6 +87,7 @@ import { const DEFAULT_CHANNEL_ID = 'default' const WEIXIN_TRACE_LOG_ENABLED = process.env.DEEPCHAT_WEIXIN_TRACE === '1' +const REMOTE_DELIVERY_ERROR_NOTIFY_INTERVAL_MS = 60 * 1000 const FEISHU_AUTH_SESSION_TTL_MS = 5 * 60 * 1000 const FEISHU_AUTH_DEFAULT_WAIT_TIMEOUT_MS = 5 * 60 * 1000 const FEISHU_INSTALL_DEFAULT_WAIT_TIMEOUT_MS = 5 * 60 * 1000 @@ -222,6 +223,8 @@ export class RemoteService { private weixinIlinkLoginWindow: BrowserWindow | null = null private weixinIlinkLoginWindowUrl: string | null = null private readonly weixinIlinkLoginWaits = new Map>() + private readonly deliveryErrorNotifiedAt = new Map() + private readonly deliveryErrorNotificationInFlight = new Set() constructor(private readonly deps: RemoteServiceDeps) { this.bindingStore = new RemoteBindingStore(this.deps.settings) @@ -1255,6 +1258,9 @@ export class RemoteService { await this.disableFeishuRuntimeForFatalError(config.configSignature ?? '', message) }) }, + onDeliveryError: (message) => { + this.notifyChannelDeliveryError('feishu', 'Feishu', message) + }, configSignature: config.configSignature }) }) @@ -2560,6 +2566,46 @@ export class RemoteService { this.weixinIlinkLoginWindowUrl = null } + private notifyChannelDeliveryError( + channel: RemoteChannel, + channelLabel: string, + message: string + ): void { + const notifications = this.deps.notifications + if (!notifications || this.deliveryErrorNotificationInFlight.has(channel)) { + return + } + + // Throttled: a dead proxy fails every message; one notification per minute is enough. + const now = Date.now() + const lastNotifiedAt = this.deliveryErrorNotifiedAt.get(channel) ?? 0 + if (now - lastNotifiedAt < REMOTE_DELIVERY_ERROR_NOTIFY_INTERVAL_MS) { + return + } + + this.deliveryErrorNotificationInFlight.add(channel) + void notifications + .showNotification({ + id: `remote-delivery-error:${channel}`, + title: `DeepChat ${channelLabel} Remote`, + body: message + }) + .then((notificationId) => { + if (notificationId) { + this.deliveryErrorNotifiedAt.set(channel, now) + } + }) + .catch((error) => { + logger.warn('[RemoteService] Failed to show delivery error notification.', { + channel, + error + }) + }) + .finally(() => { + this.deliveryErrorNotificationInFlight.delete(channel) + }) + } + private createConversationRunner(channel: RemoteChannel): RemoteConversationRunner { return new RemoteConversationRunner( { diff --git a/src/main/remote/ports.ts b/src/main/remote/ports.ts index 55a05f4fb4..1b551fedd5 100644 --- a/src/main/remote/ports.ts +++ b/src/main/remote/ports.ts @@ -64,6 +64,15 @@ export interface RemoteDesktopPort { openSession(sessionId: string): Promise } +export interface RemoteNotificationPort { + showNotification(options: { + id: string + title: string + body: string + silent?: boolean + }): Promise +} + export interface RemoteServiceDeps { settings: Pick catalog: RemoteCatalogPort @@ -73,6 +82,7 @@ export interface RemoteServiceDeps { assignment: RemoteSessionAssignmentPort projection: RemoteSessionProjectionPort desktop: RemoteDesktopPort + notifications?: RemoteNotificationPort } export interface RemoteRuntimeLifecycle { diff --git a/test/main/remote/feishuAdapter.test.ts b/test/main/remote/feishuAdapter.test.ts index 60419382a1..f80f6208e9 100644 --- a/test/main/remote/feishuAdapter.test.ts +++ b/test/main/remote/feishuAdapter.test.ts @@ -4,6 +4,7 @@ import type { FeishuRuntimeStatusSnapshot } from '@/remote/types' type MockRuntimeDeps = { onStatusChange?: (snapshot: FeishuRuntimeStatusSnapshot) => void onFatalError?: (message: string) => void + onDeliveryError?: (message: string) => void } const runtimeInstances: Array<{ @@ -121,6 +122,34 @@ describe('FeishuAdapter', () => { expect(onFatalError).toHaveBeenCalledWith('fatal feishu error') }) + it('forwards delivery errors from the wrapped runtime', async () => { + const onDeliveryError = vi.fn() + const adapter = new FeishuAdapter( + { + channelId: 'default', + channelType: 'feishu', + agentId: 'deepchat', + channelConfig: { + appId: 'cli_a', + appSecret: 'secret', + verificationToken: 'verify', + encryptKey: 'encrypt' + }, + configSignature: 'feishu:test' + }, + { + bindingStore: {} as any, + createConversationRunner: () => ({}) as any, + onDeliveryError + } + ) + + await adapter.connect() + runtimeInstances[0].deps.onDeliveryError?.('Failed to deliver reply to Feishu.') + + expect(onDeliveryError).toHaveBeenCalledWith('Failed to deliver reply to Feishu.') + }) + it('returns Feishu image message ids from sendImage', async () => { const adapter = new FeishuAdapter( { diff --git a/test/main/remote/feishuRuntime.test.ts b/test/main/remote/feishuRuntime.test.ts index 1e8004a6de..cfd421272a 100644 --- a/test/main/remote/feishuRuntime.test.ts +++ b/test/main/remote/feishuRuntime.test.ts @@ -42,6 +42,7 @@ const createDeferred = () => { const createHarness = async (options?: { logger?: { error: (...params: unknown[]) => void } enableStreamingCards?: boolean + onDeliveryError?: (message: string) => void }) => { let onMessage: ((event: unknown) => Promise) | null = null const streamHandlers: Array<(event: unknown) => Promise> = [] @@ -109,7 +110,8 @@ const createHarness = async (options?: { router: router as any, bindingStore: bindingStore as any, enableStreamingCards: options?.enableStreamingCards, - logger: options?.logger + logger: options?.logger, + onDeliveryError: options?.onDeliveryError }) await runtime.start() @@ -1285,6 +1287,96 @@ describe('FeishuRuntime', () => { messageId: 'om-error', eventId: 'evt-error' }) + expect(harness.runtime.getStatusSnapshot().lastError).toBe('Failed to handle inbound message.') + + await harness.runtime.stop() + }) + + it('records handling failures in the status snapshot without stopping the runtime', async () => { + const harness = await createHarness() + harness.router.handleMessage.mockRejectedValueOnce(new Error('Insufficient Balance')) + + await harness.emitMessage({ + parsed: createParsedMessage({ + eventId: 'evt-error', + messageId: 'om-error' + }) + }) + + await vi.waitFor(() => { + expect(harness.runtime.getStatusSnapshot()).toMatchObject({ + state: 'running', + lastError: 'Failed to handle inbound message.' + }) + }) + + harness.router.handleMessage.mockResolvedValueOnce({ + replies: ['ok'] + }) + await harness.emitMessage({ + parsed: createParsedMessage({ + eventId: 'evt-recovered', + messageId: 'om-recovered' + }) + }) + + await vi.waitFor(() => { + expect(harness.runtime.getStatusSnapshot()).toMatchObject({ + state: 'running', + lastError: null + }) + }) + + await harness.runtime.stop() + }) + + it('records reply delivery failures when even the error reply cannot be sent', async () => { + const onDeliveryError = vi.fn() + const harness = await createHarness({ onDeliveryError }) + harness.router.handleMessage.mockRejectedValueOnce(new Error('Insufficient Balance')) + harness.client.sendText.mockRejectedValueOnce( + new Error('Connect Timeout Error (attempted address: 127.0.0.1:7890)') + ) + + await harness.emitMessage({ + parsed: createParsedMessage({ + eventId: 'evt-error', + messageId: 'om-error' + }) + }) + + const expectedDeliveryError = 'Failed to deliver reply to Feishu.' + await vi.waitFor(() => { + expect(harness.runtime.getStatusSnapshot()).toMatchObject({ + state: 'running', + lastError: expectedDeliveryError + }) + }) + expect(onDeliveryError).toHaveBeenCalledWith(expectedDeliveryError) + + await harness.runtime.stop() + }) + + it('does not raise the delivery fallback when the error reply is sent successfully', async () => { + const onDeliveryError = vi.fn() + const harness = await createHarness({ onDeliveryError }) + harness.router.handleMessage.mockRejectedValueOnce(new Error('Insufficient Balance')) + + await harness.emitMessage({ + parsed: createParsedMessage({ + eventId: 'evt-error', + messageId: 'om-error' + }) + }) + + await vi.waitFor(() => { + expect(harness.client.sendText).toHaveBeenCalled() + expect(harness.runtime.getStatusSnapshot()).toMatchObject({ + state: 'running', + lastError: 'Failed to handle inbound message.' + }) + }) + expect(onDeliveryError).not.toHaveBeenCalled() await harness.runtime.stop() }) diff --git a/test/main/remote/remoteService.test.ts b/test/main/remote/remoteService.test.ts index 0ac7b2f465..b9ddf2b22a 100644 --- a/test/main/remote/remoteService.test.ts +++ b/test/main/remote/remoteService.test.ts @@ -66,6 +66,7 @@ import type { RemoteSessionLifecyclePort, RemoteSessionProjectionPort, RemoteSessionTurnPort, + RemoteNotificationPort, RemoteServiceDeps, RemoteWorkspacePort } from '@/remote/ports' @@ -188,6 +189,12 @@ const createWorkspace = (): RemoteWorkspacePort => ({ }) }) +const createNotifications = (): RemoteNotificationPort & { + showNotification: ReturnType +} => ({ + showNotification: vi.fn().mockResolvedValue('remote-delivery-error:feishu') +}) + const createRemoteService = (settings: TestSettings, overrides: Partial = {}) => new RemoteService({ settings, @@ -212,6 +219,66 @@ describe('RemoteService', () => { vi.restoreAllMocks() }) + it('notifies once for a Feishu delivery failure and throttles later failures', async () => { + const notifications = createNotifications() + const presenter = createRemoteService(createProviderSettings(), { notifications }) + const notifyDeliveryError = (presenter as any).notifyChannelDeliveryError.bind(presenter) + + notifyDeliveryError('feishu', 'Feishu', 'Failed to deliver reply to Feishu.') + await vi.waitFor(() => { + expect(notifications.showNotification).toHaveBeenCalledWith({ + id: 'remote-delivery-error:feishu', + title: 'DeepChat Feishu Remote', + body: 'Failed to deliver reply to Feishu.' + }) + }) + + notifyDeliveryError('feishu', 'Feishu', 'Failed to deliver reply to Feishu.') + await Promise.resolve() + + expect(notifications.showNotification).toHaveBeenCalledTimes(1) + }) + + it('retries a Feishu delivery notification after the previous attempt fails', async () => { + const notifications = createNotifications() + notifications.showNotification.mockRejectedValueOnce(new Error('notification unavailable')) + const presenter = createRemoteService(createProviderSettings(), { notifications }) + const notifyDeliveryError = (presenter as any).notifyChannelDeliveryError.bind(presenter) + + notifyDeliveryError('feishu', 'Feishu', 'Failed to deliver reply to Feishu.') + await vi.waitFor(() => { + expect(notifications.showNotification).toHaveBeenCalledTimes(1) + }) + await vi.waitFor(() => { + expect((presenter as any).deliveryErrorNotificationInFlight.has('feishu')).toBe(false) + }) + + notifyDeliveryError('feishu', 'Feishu', 'Failed to deliver reply to Feishu.') + await vi.waitFor(() => { + expect(notifications.showNotification).toHaveBeenCalledTimes(2) + }) + }) + + it('does not throttle a Feishu delivery notification when notifications are disabled', async () => { + const notifications = createNotifications() + notifications.showNotification.mockResolvedValue(undefined) + const presenter = createRemoteService(createProviderSettings(), { notifications }) + const notifyDeliveryError = (presenter as any).notifyChannelDeliveryError.bind(presenter) + + notifyDeliveryError('feishu', 'Feishu', 'Failed to deliver reply to Feishu.') + await vi.waitFor(() => { + expect(notifications.showNotification).toHaveBeenCalledTimes(1) + }) + await vi.waitFor(() => { + expect((presenter as any).deliveryErrorNotificationInFlight.has('feishu')).toBe(false) + }) + + notifyDeliveryError('feishu', 'Feishu', 'Failed to deliver reply to Feishu.') + await vi.waitFor(() => { + expect(notifications.showNotification).toHaveBeenCalledTimes(2) + }) + }) + it('serializes runtime rebuilds so only one poller starts per token', async () => { const providerSettings = createProviderSettings() From 500fb3c958796b7fac5f3dbe3a4129767b8e2a44 Mon Sep 17 00:00:00 2001 From: Dorence Deng Date: Sat, 18 Jul 2026 09:11:59 +0800 Subject: [PATCH 2/6] fix(app): honor DEEPCHAT_E2E_USER_DATA_DIR before startup (#1995) --- .../user-data-profile-path/spec.md | 107 ++++++++++++++++++ src/main/provider/providerDbLoader.ts | 2 +- src/shared/logger.ts | 2 +- test/e2e/fixtures/electronApp.ts | 5 +- test/main/provider/providerDbLoader.test.ts | 15 +++ 5 files changed, 127 insertions(+), 4 deletions(-) create mode 100644 docs/architecture/user-data-profile-path/spec.md diff --git a/docs/architecture/user-data-profile-path/spec.md b/docs/architecture/user-data-profile-path/spec.md new file mode 100644 index 0000000000..d79a054aa1 --- /dev/null +++ b/docs/architecture/user-data-profile-path/spec.md @@ -0,0 +1,107 @@ +# User Data Profile Path + +Status: implemented as a runtime profile path contract and maintained as an architecture spec. + +## Summary + +DeepChat stores profile-scoped application files under one Electron `userData` profile path, used for settings, +SQLite databases, provider catalog cache, plugin data, OAuth credentials, generated assets, temporary files, and logs. + +By default DeepChat uses Electron's standard `userData` path and does not override it. Development, preview, E2E, and +build validation may launch DeepChat with an explicit profile path through `process.env.DEEPCHAT_E2E_USER_DATA_DIR`. + +## Path Contract + +### Definitions + +- **Profile path**: the effective directory returned by `app.getPath('userData')` after startup path selection has run. +- **Default profile path**: Electron's standard `userData` path, `path.join(app.getPath('appData'), 'DeepChat')`. The + `appData` base directory follows Electron's default and is platform-dependent. App name must stay `DeepChat`, as + configured in `package.json` and `electron-builder.yml`. +- **Explicit profile path**: a profile path set through `process.env.DEEPCHAT_E2E_USER_DATA_DIR`. When it holds a + trimmed non-whitespace value, it overrides the default profile path for the current process. + +### Explicit Profile Source + +The explicit profile path is process-local and does not migrate, copy, or delete data from any other profile. +`DEEPCHAT_E2E_USER_DATA_DIR` reaches the application from two sources: + +1. **User-defined**: a user sets it manually for local development, preview, or packaged-build validation, + e.g. `export DEEPCHAT_E2E_USER_DATA_DIR=/tmp/deepchat-profile`. The Playwright fixture reuses this profile and + preserves directory after the run. +2. **E2E-defined**: when the variable is unset, the Playwright fixture creates a temporary profile under + `os.tmpdir()/deepchat-e2e-user-data-*` and deletes it after the run. + +## Required Behavior + +- Modules covered by this contract must resolve profile-scoped files under the effective profile path. +- Startup must apply the explicit profile path before presenters, windows, databases, caches, or profile-backed + services are created. +- Modules that resolve profile paths before normal startup must follow the same rule. They may either delay path + resolution until after startup applies `app.setPath('userData', ...)`, or read `DEEPCHAT_E2E_USER_DATA_DIR` directly + and fall back to Electron's default path. +- Main-process logs must be written under `/logs/main.log`. +- Provider database cache files must be written under `/provider-db/`. +- While an explicit profile is in effect, the process must confine all profile-scoped reads and writes to that + profile and must not touch the default profile path, so a default-profile build and an explicit-profile build can + run at the same time. +- Selecting a profile path must not bypass privacy mode, credential handling, database encryption, or normal app + settings semantics. +- During E2E test runs, the fixture must pass its selected profile to the launched Electron process through + `DEEPCHAT_E2E_USER_DATA_DIR`. + +## Current Implementation + +`src/main/appMain.ts` applies the explicit profile path at the start of `startApp()`: + +```typescript +const e2eUserDataDir = process.env.DEEPCHAT_E2E_USER_DATA_DIR?.trim() +if (e2eUserDataDir) { + app.setPath('userData', e2eUserDataDir) +} +``` + +Once the Electron app has started and is ready, normal profile data resolves through `app.getPath('userData')`: +application settings (`app-settings.json`), SQLite databases (`app_db/`), OAuth credentials (`*-auth/`), plugins +(`plugins/`), workspaces (`workspaces/`), and generated images (`images/`). + +**Early path resolvers** run before startup applies `app.setPath('userData', ...)`, so they should read +`DEEPCHAT_E2E_USER_DATA_DIR` first and fall back to the default profile path. + +| Resolver | File | Provides | Rationale (Why not `app.getPath`) | +| --- | --- | --- | --- | +| `log.transports.file.resolvePathFn` | `src/shared/logger.ts` | log `logs/main.log` | `appMain.ts` → `logger.ts` | +| `ProviderDbLoader.userDataDir` | `src/main/provider/providerDbLoader.ts` | model provider `provider-db/` | `appMain.ts` → `app/mainProcess.ts` → `app/composition.ts` → `providerDbLoader.ts` | +| `getDefaultUserDataDir()` | `test/e2e/fixtures/electronApp.ts` | profile root | No Electron `app` | + +## Acceptance Criteria + +- With `DEEPCHAT_E2E_USER_DATA_DIR` unset or whitespace-only, startup does not call `app.setPath('userData', ...)` and + DeepChat runs on Electron's default profile path. +- With `DEEPCHAT_E2E_USER_DATA_DIR` set to a non-empty absolute path, startup applies that path before profile-backed + runtime initialization. +- In application startup and early path resolvers, whitespace-only `DEEPCHAT_E2E_USER_DATA_DIR` is treated as unset. +- Main-process logs are created under `/logs/main.log`. +- Provider database cache files are created under `/provider-db/`. +- With `DEEPCHAT_E2E_USER_DATA_DIR` already set, the E2E fixture uses that directory and must not delete it. +- With `DEEPCHAT_E2E_USER_DATA_DIR` unset, the E2E fixture creates a temporary profile and deletes only that + fixture-owned directory after the run. +- No code path uses `DEEPCHAT_E2E_USER_DATA_DIR` as a signal for smoke tests, CI, mocks, disabled network access, or + alternate provider behavior. + +## Non-Goals + +- Profile-path override mechanism other than `DEEPCHAT_E2E_USER_DATA_DIR`. + +## Validation + +- Launch with `DEEPCHAT_E2E_USER_DATA_DIR` set to a profile directory and verify profile-backed files appear under + that directory. +- Verify `logs/main.log` is created under the selected profile path. +- Verify provider DB cache files are created under the selected profile path. +- Run a Playwright spec with `DEEPCHAT_E2E_USER_DATA_DIR` unset and verify the fixture-owned temporary directory is + deleted. + +## Open Questions + +None. diff --git a/src/main/provider/providerDbLoader.ts b/src/main/provider/providerDbLoader.ts index c835cad047..197e2c8afb 100644 --- a/src/main/provider/providerDbLoader.ts +++ b/src/main/provider/providerDbLoader.ts @@ -51,7 +51,7 @@ export class ProviderDbLoader { private readonly catalogListeners = new Set() constructor() { - this.userDataDir = app.getPath('userData') + this.userDataDir = process.env.DEEPCHAT_E2E_USER_DATA_DIR?.trim() || app.getPath('userData') this.cacheDir = path.join(this.userDataDir, 'provider-db') this.cacheFilePath = path.join(this.cacheDir, 'providers.json') this.metaFilePath = path.join(this.cacheDir, 'meta.json') diff --git a/src/shared/logger.ts b/src/shared/logger.ts index 0540f2707c..648f8ba089 100644 --- a/src/shared/logger.ts +++ b/src/shared/logger.ts @@ -5,7 +5,7 @@ import { is } from '@electron-toolkit/utils' // Configure log file path // Use logger for recording instead of console -const userData = app?.getPath('userData') || '' +const userData = process.env.DEEPCHAT_E2E_USER_DATA_DIR?.trim() || app?.getPath('userData') || '' if (userData) { log.transports.file.resolvePathFn = () => path.join(userData, 'logs/main.log') } diff --git a/test/e2e/fixtures/electronApp.ts b/test/e2e/fixtures/electronApp.ts index 2079d57649..d75205757c 100644 --- a/test/e2e/fixtures/electronApp.ts +++ b/test/e2e/fixtures/electronApp.ts @@ -129,8 +129,9 @@ const attachDiagnostics = async ( } const getDefaultUserDataDir = (): string => { - if (process.env.DEEPCHAT_E2E_USER_DATA_DIR) { - return resolve(process.env.DEEPCHAT_E2E_USER_DATA_DIR) + const e2eUserDataDir = process.env.DEEPCHAT_E2E_USER_DATA_DIR?.trim() + if (e2eUserDataDir) { + return resolve(e2eUserDataDir) } if (process.platform === 'win32') { diff --git a/test/main/provider/providerDbLoader.test.ts b/test/main/provider/providerDbLoader.test.ts index f9a3a7d829..dc9de71bc8 100644 --- a/test/main/provider/providerDbLoader.test.ts +++ b/test/main/provider/providerDbLoader.test.ts @@ -106,6 +106,21 @@ describe('ProviderDbLoader', () => { fs.rmSync(tempRoot, { recursive: true, force: true }) }) + it('resolves DEEPCHAT_E2E_USER_DATA_DIR as provider-db base directory', async () => { + const oldDir = process.env.DEEPCHAT_E2E_USER_DATA_DIR + const e2eUserDataRoot = path.join(tempRoot, 'e2e-user-data') + process.env.DEEPCHAT_E2E_USER_DATA_DIR = e2eUserDataRoot + const ProviderDbLoader = await importLoader() + new ProviderDbLoader() + expect(fs.existsSync(path.join(e2eUserDataRoot, 'provider-db'))).toBe(true) + expect(fs.existsSync(getCacheDir())).toBe(false) + if (oldDir === undefined) { + delete process.env.DEEPCHAT_E2E_USER_DATA_DIR + } else { + process.env.DEEPCHAT_E2E_USER_DATA_DIR = oldDir + } + }) + it('initializes from cache and still triggers a startup refresh when cache is fresh', async () => { writeBuiltInDb(createAggregate(['builtin'])) writeCachedDb(createAggregate(['openai'])) From 3faf784e7b54f5c57a0e1a0778967651b3e8a77c Mon Sep 17 00:00:00 2001 From: yyhhyyyyyy Date: Sat, 18 Jul 2026 09:12:12 +0800 Subject: [PATCH 3/6] refactor(tape): establish layered architecture (#1996) * docs(tape): specify layering refactor * test(tape): split behavior contracts * refactor(tape): establish domain ports * refactor(tape): split application services * refactor(tape): close storage bypasses * test(tape): enforce layer boundaries * test(tape): align writer mock naming * docs(tape): refresh architecture map * fix(tape): make generation resets atomic * refactor(tape): narrow consumer ports * refactor(tape): align service ownership * test(tape): harden layer boundaries * test(tape): guard semantics compatibility path * test(tape): guard project sqlite driver * fix(tape): invalidate legacy projections * refactor(tape): require capability injection * refactor(tape): narrow anchor reader port * refactor(tape): narrow storage protocols * docs(tape): clarify layer contracts * docs(tape): specify follow-up hardening * test(memory): restore native tape scope * test(tape): restore layered settings fixture * fix(tape): harden generation cleanup * refactor(tape): harden compatibility boundaries * test(tape): repair memory type contracts * docs(tape): finalize hardening record * fix(tape): validate durable link identities * fix(tape): harden derived state recovery * test(tape): align boundary fixtures --- docs/FLOWS.md | 4 +- ...agent-system-layered-runtime-baseline.json | 27 +- docs/architecture/memory-system.md | 25 +- docs/architecture/session-management.md | 32 +- docs/architecture/tape-layering/plan.md | 269 + docs/architecture/tape-layering/spec.md | 213 + docs/architecture/tape-layering/tasks.md | 114 + docs/architecture/tape-system.md | 97 +- scripts/check-memory-test-scope.mjs | 25 +- scripts/generate-architecture-baseline.mjs | 9 +- src/main/agent/acp/compatibility/adapters.ts | 10 +- .../agent/acp/compatibility/dependencies.ts | 6 +- src/main/agent/deepchat/loop/ports.ts | 51 +- .../memory/memoryRuntimeCoordinator.ts | 22 +- .../agent/deepchat/runtime/contextBuilder.ts | 2 +- .../deepchat/runtime/deepChatLoopRunner.ts | 28 +- .../runtime/deepChatRuntimeCoordinator.ts | 16 +- src/main/agent/deepchat/runtime/process.ts | 4 +- .../agent/deepchat/runtime/turnCoordinator.ts | 19 +- src/main/agent/deepchat/runtime/types.ts | 4 +- src/main/app/composition.ts | 8 +- .../legacyChatImportService.ts | 6 +- src/main/data/schemaCatalog.ts | 4 +- .../deepchatMemoryIngestionProjection.ts | 12 +- src/main/memory/routes.ts | 96 +- src/main/session/data/database.ts | 15 +- src/main/session/data/index.ts | 12 +- src/main/session/data/settings.ts | 37 +- .../tables/deepchatTapeEffectiveSemantics.ts | 177 +- .../data/tables/deepchatTapeEntries.ts | 1180 +--- .../tables/deepchatTapeSearchProjection.ts | 940 +-- src/main/session/data/tape.ts | 2650 +------- src/main/session/data/tapeEffectiveView.ts | 264 +- src/main/session/data/tapeFacts.ts | 366 +- src/main/session/data/tapeViewManifest.ts | 387 +- src/main/session/data/transcript.ts | 36 +- src/main/session/transcriptMutations.ts | 9 +- src/main/tape/application/common.ts | 29 + src/main/tape/application/contracts.ts | 39 + src/main/tape/application/factPersistence.ts | 352 ++ src/main/tape/application/factService.ts | 166 + src/main/tape/application/forkService.ts | 335 + .../tape/application/generationLifecycle.ts | 29 + src/main/tape/application/lineageService.ts | 458 ++ src/main/tape/application/recallProjection.ts | 466 ++ src/main/tape/application/recallService.ts | 584 ++ .../tape/application/reconcilerService.ts | 123 + src/main/tape/application/sessionTape.ts | 282 + .../tape/application/viewReplayService.ts | 523 ++ src/main/tape/domain/effectiveSemantics.ts | 161 + src/main/tape/domain/effectiveView.ts | 251 + src/main/tape/domain/entry.ts | 118 + src/main/tape/domain/facts.ts | 29 + src/main/tape/domain/replay.ts | 193 + src/main/tape/domain/viewManifest.ts | 363 ++ .../infrastructure/sqlite/tapeEntryStore.ts | 1070 ++++ .../sqlite/tapeLifecycleAdapter.ts | 17 + .../sqlite/tapeSearchProjectionStore.ts | 1009 +++ src/main/tape/ports/application.ts | 179 + src/main/tape/ports/capabilities.ts | 108 + src/main/tape/ports/storage.ts | 67 + .../agent/acp/compatibility/adapters.test.ts | 4 +- .../memory/memoryRuntimeCoordinator.test.ts | 14 +- .../deepChatRuntimeCoordinator.test.ts | 9 +- .../agent/deepchat/runtime/process.test.ts | 42 +- test/main/app/compositionBoundaries.test.ts | 3 + .../legacyChatImportService.test.ts | 5 + test/main/evals/nativeAgent/harness.ts | 2 +- .../deepchatMemoryIngestionProjection.test.ts | 13 +- test/main/routes/dispatcher.test.ts | 235 +- test/main/scripts/memoryTestScope.test.ts | 67 +- test/main/session/data/settings.test.ts | 134 +- .../tables/deepchatTapeEntriesTable.test.ts | 51 +- test/main/session/data/tape.test.ts | 5531 ----------------- test/main/session/data/tapeFacts.test.ts | 25 +- test/main/session/data/tapeFork.test.ts | 459 ++ test/main/session/data/tapeLifecycle.test.ts | 429 ++ test/main/session/data/tapeLineage.test.ts | 759 +++ test/main/session/data/tapeRecall.test.ts | 2491 ++++++++ test/main/session/data/tapeReconciler.test.ts | 399 ++ test/main/session/data/tapeTestHarness.ts | 654 ++ test/main/session/data/tapeViewReplay.test.ts | 1578 +++++ test/main/session/data/transcript.test.ts | 31 +- test/main/session/runtimeIntegration.test.ts | 19 +- test/main/session/transcriptMutations.test.ts | 79 + test/main/session/usageStatsService.test.ts | 3 +- test/main/tape/layerBoundaries.test.ts | 643 ++ test/memory-test-scope.json | 7 +- 88 files changed, 15942 insertions(+), 11871 deletions(-) create mode 100644 docs/architecture/tape-layering/plan.md create mode 100644 docs/architecture/tape-layering/spec.md create mode 100644 docs/architecture/tape-layering/tasks.md create mode 100644 src/main/tape/application/common.ts create mode 100644 src/main/tape/application/contracts.ts create mode 100644 src/main/tape/application/factPersistence.ts create mode 100644 src/main/tape/application/factService.ts create mode 100644 src/main/tape/application/forkService.ts create mode 100644 src/main/tape/application/generationLifecycle.ts create mode 100644 src/main/tape/application/lineageService.ts create mode 100644 src/main/tape/application/recallProjection.ts create mode 100644 src/main/tape/application/recallService.ts create mode 100644 src/main/tape/application/reconcilerService.ts create mode 100644 src/main/tape/application/sessionTape.ts create mode 100644 src/main/tape/application/viewReplayService.ts create mode 100644 src/main/tape/domain/effectiveSemantics.ts create mode 100644 src/main/tape/domain/effectiveView.ts create mode 100644 src/main/tape/domain/entry.ts create mode 100644 src/main/tape/domain/facts.ts create mode 100644 src/main/tape/domain/replay.ts create mode 100644 src/main/tape/domain/viewManifest.ts create mode 100644 src/main/tape/infrastructure/sqlite/tapeEntryStore.ts create mode 100644 src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts create mode 100644 src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts create mode 100644 src/main/tape/ports/application.ts create mode 100644 src/main/tape/ports/capabilities.ts create mode 100644 src/main/tape/ports/storage.ts delete mode 100644 test/main/session/data/tape.test.ts create mode 100644 test/main/session/data/tapeFork.test.ts create mode 100644 test/main/session/data/tapeLifecycle.test.ts create mode 100644 test/main/session/data/tapeLineage.test.ts create mode 100644 test/main/session/data/tapeRecall.test.ts create mode 100644 test/main/session/data/tapeReconciler.test.ts create mode 100644 test/main/session/data/tapeTestHarness.ts create mode 100644 test/main/session/data/tapeViewReplay.test.ts create mode 100644 test/main/session/transcriptMutations.test.ts create mode 100644 test/main/tape/layerBoundaries.test.ts diff --git a/docs/FLOWS.md b/docs/FLOWS.md index 7552be6279..7c5de88cfb 100644 --- a/docs/FLOWS.md +++ b/docs/FLOWS.md @@ -81,8 +81,8 @@ Provider、Tool、Skill、Memory 和 Session data 都通过创建时传入的必 不会伪造新的 outer round。 - Memory prompt contribution 必须等待结果、清理内容、限制大小并允许失败；terminal extraction 在后台 执行，并保持 epoch、cursor 和 fence 约束。 -- `TapeRecorder.appendToolFact` 在 message projection 完成后写 terminal tool call/result；写入失败不影响 - 当前回复完成。 +- `TapeToolFactWriter.appendToolFact` 在 message projection 完成后写 terminal tool call/result； + 写入失败不影响当前回复完成。 ## 4. ACP 执行 diff --git a/docs/architecture/baselines/agent-system-layered-runtime-baseline.json b/docs/architecture/baselines/agent-system-layered-runtime-baseline.json index 1baa3fce5b..be21f707cc 100644 --- a/docs/architecture/baselines/agent-system-layered-runtime-baseline.json +++ b/docs/architecture/baselines/agent-system-layered-runtime-baseline.json @@ -1,7 +1,7 @@ { "schemaVersion": 2, "goal": "agent-system-layered-runtime", - "headCommit": "d87cb4f69dca7af499e153d9a1d71015e1bfad60", + "headCommit": "df1cd3ae5b19d78e6c3cab79cdc9e4c5938fb2d4", "relevantWorkingTree": { "dirty": false, "files": [] @@ -127,11 +127,12 @@ "src/main/agent/shared/storage/sessionPaths.ts", "src/main/provider/providers/acpProvider.ts", "src/main/session/assignment.ts", - "src/main/session/data/tape.ts", "src/main/session/data/transcript.ts", "src/main/session/lifecycle.ts", "src/main/session/query.ts", - "src/main/session/turn.ts" + "src/main/session/turn.ts", + "src/main/tape/application/sessionTape.ts", + "src/main/tape/ports/capabilities.ts" ], "expectedFiles": { "src/main/agent/acp/instance/acpAgentInstance.ts": true, @@ -155,11 +156,12 @@ "src/main/agent/shared/appSessionService.ts": true, "src/main/provider/providers/acpProvider.ts": true, "src/main/session/assignment.ts": true, - "src/main/session/data/tape.ts": true, "src/main/session/data/transcript.ts": true, "src/main/session/lifecycle.ts": true, "src/main/session/query.ts": true, - "src/main/session/turn.ts": true + "src/main/session/turn.ts": true, + "src/main/tape/application/sessionTape.ts": true, + "src/main/tape/ports/capabilities.ts": true }, "ownerEvidence": { "agentManager": { @@ -192,8 +194,8 @@ "exists": true, "declarationCount": 1 }, - "tapeRecorder": { - "file": "src/main/agent/deepchat/loop/ports.ts", + "tapeToolFactWriter": { + "file": "src/main/tape/ports/capabilities.ts", "exists": true, "declarationCount": 1 }, @@ -297,11 +299,12 @@ "src/main/agent/deepchat/runtime/process.ts", "src/main/provider/providers/acpProvider.ts", "src/main/session/assignment.ts", - "src/main/session/data/tape.ts", "src/main/session/data/transcript.ts", "src/main/session/lifecycle.ts", "src/main/session/query.ts", - "src/main/session/turn.ts" + "src/main/session/turn.ts", + "src/main/tape/application/sessionTape.ts", + "src/main/tape/ports/capabilities.ts" ] }, "contracts": { @@ -366,7 +369,7 @@ "src/shared/contracts/routes/window.routes.ts", "src/shared/contracts/routes/workspace.routes.ts" ], - "sha256": "406ef501353454e8677f773c556b8938141d931229eebabae6fcb9b0c37abe3c" + "sha256": "e624008521f11befaa0497a325cb65b4bf38f36e575dd4285622ad3705955b9e" }, "storage": { "sqlite": { @@ -376,7 +379,7 @@ "src/main/data/schemaTypes.ts" ], "tableIdentifiers": [], - "sha256": "411d0c6078cf4264f2241e1198315e52dcf085342975d344e5d8967333b8129b" + "sha256": "3465d92dff615489b7037de640077ce95bda7a7abbb0144ce84bb144dcd85a9e" }, "memoryDuckDbSidecar": { "files": [ @@ -396,7 +399,7 @@ "src/main/app/mainProcess.ts", "src/main/appMain.ts" ], - "sha256": "dc13adbe8b76f0c4bb463605c780fc83e4a78696b8875d6765e767a4c2dcf70f" + "sha256": "394cd861f9fe06a29159d06f87f83b7ba1866287f129fae2a42a382e7e2e6c95" }, "dependencyMetrics": { "loopFiles": [ diff --git a/docs/architecture/memory-system.md b/docs/architecture/memory-system.md index 65e3ef16cc..d3c17a586a 100644 --- a/docs/architecture/memory-system.md +++ b/docs/architecture/memory-system.md @@ -23,6 +23,9 @@ flowchart LR epoch、cursor 和 fence。 - Session 保存 Memory cursor/settings，不拥有 Memory row 或 vector store。 - App 负责 shutdown/database maintenance 时的全局 fence 和停止顺序，不解释 Memory 业务状态。 +- Memory runtime 通过 `TapeRawEntryReader` 和 `TapeAnchorWriter` 读取执行事实、记录 + `memory/view_assembled` 与 `memory/extract` anchor；Memory routes 只通过 `TapeInspectionReader` + 获取 effective source span 和 manifest DTO，不接收 Tape table 或 raw Tape row。 ## 数据与状态 @@ -61,6 +64,8 @@ Vector store v2 使用 `.v2.duckdb`、plain `FLOAT[]` table 和 exact s ```text terminal turn projection + -> read bounded ingestion projection range + -> rebuild from effective Tape or fall back when projection is stale/unavailable -> collect bounded text chunks -> extraction / reflection -> normalize candidates @@ -76,6 +81,23 @@ terminal turn projection - stale result、partial batch 和 provider cancellation 有明确 terminal outcome； - vector store 异常进入 typed error/quarantine，不得把消息发送永久挂起。 +## Tape 与 ingestion projection 边界 + +`DeepChatMemoryIngestionProjectionTable.readCurrentRange` 在一条只读 SQL 中同时观察 Tape head 和 +projection head。这是明确的基础设施例外：拆成两次查询会让并发 append 产生 false-current 窗口。 +除此之外 Memory 不得直接读取物理 Tape 表。 + +projection current 时只 materialize cursor 区间；head 不一致时，runtime 通过 `TapeRawEntryReader` +构建 effective Tape view 并重建 projection。projection 查询或重建失败时保留既有 Tape fallback 和 +cursor commit 保护，不能因为拆层新增全历史 hot-path 查询，也不能在不完整 projection 上推进 cursor。 + +`TapeRawEntryReader` 只提供 `getBySession`。Memory management route 先验证 memory row 属于请求 Agent， +再用 `getEffectiveMessageSourceSpan` 读取 retraction/replacement 生效后的最小 message DTO；manifest +列表通过 `listMemoryViewManifestsByAgent` 在 storage query 中执行 Agent、Session、message 和 limit +过滤，route 不自行解析 `payload_json` 或 `meta_json`。架构守卫同时扫描 static import、dynamic +import、CommonJS require、type import 和 re-export，Memory route 不能绕过 inspection port 重新取得 +raw reader、facade 或 domain helper。 + ## Privacy 与隔离 - 所有查询显式携带 Agent identity；不得依赖进程全局“当前 Agent”。 @@ -101,7 +123,8 @@ metric 名称、retrieval evaluation 和 artifact upload 的未完成工作保 5. `src/main/memory/infra/vectorStoreManager.ts` 6. `src/main/memory/infra/memoryVectorStore.ts` 7. `src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts` -8. `test/main/memory/` +8. `src/main/tape/ports/capabilities.ts` +9. `test/main/memory/` Memory tests 必须防止旧 `src/main/presenter/memoryPresenter`、HNSW hot path、 无 Agent namespace 查询和无 deadline provider call 回流。 diff --git a/docs/architecture/session-management.md b/docs/architecture/session-management.md index fff32610bb..e80d0d837f 100644 --- a/docs/architecture/session-management.md +++ b/docs/architecture/session-management.md @@ -13,7 +13,8 @@ Session 是可长期保存的产品对象；window、renderer、Remote endpoint | list、restore、status、projection query | `src/main/session/query.ts` | | full delete transaction | `src/main/session/deletion.ts` | | transcript | `src/main/session/data/transcript.ts` | -| Tape / ViewManifest | `src/main/session/data/tape*.ts` | +| Tape public port / composition | `src/main/session/data/index.ts` | +| Tape domain / application / SQLite adapters | `src/main/tape/` | | generation settings / Memory cursor | `src/main/session/data/settings.ts` | | pending input | `src/main/session/data/pendingInputs.ts` | | renderer binding | `src/main/desktop/sessionBinding.ts` | @@ -54,6 +55,33 @@ send input；Remote、Scheduler 和 renderer 不得各自维护不同的默认 - history search 优先使用 search document / FTS path，失败时回退受控 SQL search；坐标和 scroll 归 renderer viewport owner，不写回 Session。 +## Tape boundary + +Session data composition 创建一个 `SessionTape`，对外继续暴露现有 `SessionTapePort`，并按每个 IPC +操作原有的条件和时序调用 `ensureSessionTapeReady`。`src/main/session/data/tape*.ts` 和旧 table path +只是显式冻结并标记 deprecated 的 compatibility re-export，不再拥有 Tape policy 或 persistence。 + +- transcript 只接收 `TapeMessageFactWriter`；message replace/retract 与对应 Tape fact 继续共享调用方 + SQLite transaction； +- settings/compaction 只接收 anchor reader/writer 与 lifecycle admin；summary 更新和 anchor append + 继续使用同一个 connection； +- loop runner 分别接收 reconciliation、ViewManifest reader/writer 和 tool fact writer；Turn coordinator + 与 ACP compatibility 只接收 reconciliation；Memory 和 routes 分别接收自己的最小 Tape capability， + 不接收物理 table； +- Tape 的运行中修订通过 append 表达；物理 delete/reset 只由 Session lifecycle 触发。 + +`SessionTranscript` 和 `SessionSettingsStore` 不提供隐式 `new SessionTape(...)` fallback，必须由正常 +composition 注入共享 connection 上的最小 capability。legacy import 作为 migration consumer 复用 +`sessionData.tapeStore` 的 message fact writer，不再构造第二个 facade，避免隐藏的独立 writer 或事务 +上下文。 + +`clearSessionMessages` 会创建新的 Tape incarnation：pending input 删除、transcript 删除和 Tape reset +位于同一个外层 SQLite transaction；Tape 内部的 entry、mutation projection、search/FTS projection +删除和新 bootstrap 作为 savepoint 嵌套。lifecycle、cleanup 或 bootstrap hard failure 会同时恢复上述 +数据并完整保留旧 incarnation。mutation projection 沿用 fail-open：新 bootstrap 的 projection apply +失败时旧 projection row 已删除且 meta 标 stale。最终 Session delete 不创建新 incarnation，继续遵循 +下面的 staged cleanup 顺序。 + ## Binding `DesktopSessionBinding` 维护 `webContentsId -> sessionId`： @@ -70,7 +98,7 @@ send input；Remote、Scheduler 和 renderer 不得各自维护不同的默认 ```text cleanup both backend caches without hydration -> remove ACP durable binding - -> remove transcript / Tape / pending / settings + -> remove transcript / Tape lifecycle data / pending / settings -> clear permission and active Skill state -> delete app-session row ``` diff --git a/docs/architecture/tape-layering/plan.md b/docs/architecture/tape-layering/plan.md new file mode 100644 index 0000000000..f54ae5d399 --- /dev/null +++ b/docs/architecture/tape-layering/plan.md @@ -0,0 +1,269 @@ +# Tape Layering Refactor Implementation Plan + +## Target Structure + +The implementation will create a top-level Tape subsystem: + +```text +src/main/tape/ + domain/ pure entry, fact, view, manifest, lineage, and replay logic + ports/ storage and consumer capability interfaces + application/ fact, reconciliation, recall, lineage, view/replay, and fork services + infrastructure/sqlite/ SQLite entry store, query SQL, search projection, and lifecycle adapter +``` + +Existing `src/main/session/data/tape*.ts` and table modules will become compatibility re-exports +where an old import path is still part of the current internal contract. New production imports +will target `@/tape/*`. + +## Domain and Port Design + +Tape entry rows, append inputs, source identities, entry references, fact provenance, and tool fact +inputs move out of Agent and table modules into Tape-owned types. Effective-view selection, +ViewManifest hashing, lineage validation, stored-manifest validation, and replay hashing remain +pure. Database row parsing and trace-evidence reads remain in the application layer. + +The primary ports are: + +- `TapeEntryStore`: append, anchor/event append helpers, and read/query operations. It has no + destructive method. `TapeBootstrapStore` and `TapeTransactionRunner` are separate application + composition capabilities. +- `TapeSearchProjectionStore`: rebuildable search projection behavior. +- `TapeToolFactWriter`: the single `appendToolFact` capability used by the Agent loop. +- `TapeMessageFactWriter`: message, replacement, and retraction fact operations used by transcript. +- `TapeReconciliationPort`: bootstrap and transcript reconciliation used by the loop runner, Turn + coordinator, and ACP compatibility projection. +- `TapeViewManifestReader` and `TapeViewManifestWriter`: the manifest capabilities used by the + loop runner without exposing the full facade. +- `TapeAnchorReader`: only the latest reconstruction-anchor read required by settings. +- `TapeAnchorWriter`: narrow anchor append capability for settings and Memory. +- `TapeRawEntryReader`: only `getBySession`, retained for effective-view rebuilding in Memory. +- `TapeInspectionReader`: effective source spans and Memory ViewManifest inspection DTOs; no raw + entry rows cross this boundary. +- `TapeLifecycleAdmin`: Session-owned delete and reset operations across entries and projections. +- Explicit transcript and trace evidence read ports used by reconciliation and replay. + +One application object may implement multiple interfaces. Composition injects the narrow interface +at each call site. + +## Application Services + +The current `SessionTape` behavior is divided without changing method semantics: + +1. **TapeFactService** owns message, tool, generic anchor, handoff, and fork-message fact appends. +2. **TapeReconcilerService** owns bootstrap, legacy transcript backfill, and legacy summary-anchor + repair. +3. **TapeRecallService** owns info, search, context windows, anchor listing, and authorized source + resolution needed by recall. +4. **TapeLineageService** owns link validation, frozen child heads, authorization, and lineage + receipts. +5. **TapeViewReplayService** owns ViewManifest append and source assembly, manifest listing, replay + exports, Memory inspection projection, and explicit trace-evidence reads. +6. **TapeForkService** owns only fork creation, delta merge, discard, and external lifecycle + receipts. + +`SessionTape` becomes a compatibility facade that constructs these services and forwards the +existing methods. `SessionTapePort` in Session contracts remains unchanged. + +## SQLite Infrastructure + +The large entry table module is separated into Tape-owned row and append types, reusable effective +query SQL, a normal SQLite entry store, and a lifecycle adapter. Table names, indexes, SQL +predicates, provenance uniqueness, and payload serialization remain byte-compatible. + +Physical entry deletion is removed from the normal entry-store interface. `TapeLifecycleAdmin` +coordinates entry deletion, search projection deletion, and reset bootstrap. A reset executes +entry and mutation-projection deletion, search and FTS deletion, and new bootstrap creation in one +transaction. Startup legacy import may continue to execute whole-database cleanup SQL because it +rebuilds persisted state before normal runtime composition. + +The Memory ingestion projection retains its current single SQL statement that compares +`MAX(deepchat_tape_entries.entry_id)` with the projection metadata head. Moving this comparison to +two independent port calls would introduce a freshness race and an extra query, so it is an +allowlisted read-only infrastructure dependency. + +## Composition and Data Flow + +Session data composition will create the entry store and Tape services before constructing +transcript and settings: + +```text +SQLite connection + -> Tape stores and services + -> Transcript with TapeMessageFactWriter + -> Settings with anchor and lifecycle capabilities + -> SessionTape facade and existing SessionTapePort adapter +``` + +Runtime composition passes reconciliation, ViewManifest read/write, and tool-fact capabilities to +the loop runner; only reconciliation to the Turn coordinator and ACP adapter; raw-row and anchor +capabilities to the Memory coordinator; and `TapeInspectionReader` to Memory routes. No +application consumer gets the concrete entry table. Transcript and settings have no concrete +facade default: normal composition injects their capabilities from the shared `SessionTape`, and +legacy import reuses that composition-owned `TapeMessageFactWriter` instead of constructing a +second facade. + +`ensureSessionTapeReady` remains at the current Session port boundary. Search and context requests +with linked-source scopes keep their existing conditional reconciliation behavior. + +## Transaction Boundaries + +- Transcript deletion and retry truncation append retractions inside the same SQLite transaction + that deletes projection rows. +- Summary compare-and-set appends its reconstruction anchor inside the same transaction that + updates summary state. +- Clear-time pending-input deletion, transcript deletion, and Tape reset run in one outer + shared-connection transaction. The Tape generation transaction nests as a savepoint, so a reset + or bootstrap failure restores every clear-time data family. +- Reset deletes entries, mutation projection, search projection, FTS metadata, and FTS rows and + appends the new bootstrap within one shared-connection transaction. A propagated transition + failure restores the old incarnation. The pre-existing fail-open mutation-projection append + policy may instead commit the new Tape with that derivative marked stale after old projection + rows have been removed. +- Fork discard performs the same atomic cleanup attempt. Cleanup failure rolls that attempt back + but still appends a fail-closed discard receipt, preserving the non-blocking contract. +- Final Session deletion keeps its staged lifecycle ordering and does not create a replacement + incarnation. Its Tape entry and search-projection cleanup still runs as one generation + transaction before the Session row is removed. +- FTS is a rebuildable derivative. If session-row deletion from the virtual table fails, the + adapter drops the FTS table and clears freshness metadata inside the same lifecycle transaction; + the next search recreates and repopulates it. Failure to drop the damaged derivative remains a + hard transaction failure rather than committing a mixed generation. +- Port implementations use the same connection provider and remain synchronous, so extracting a + service does not cross a transaction boundary. + +Contract tests will force failures between paired operations and verify rollback or unchanged +behavior where the current implementation is atomic. + +## Compatibility Strategy + +- Keep all shared DTOs and `SessionTapePort` signatures unchanged. +- Preserve old internal exported symbol names through explicit, frozen compatibility re-exports + marked as deprecated. +- Keep methods that existed on the historically exported concrete SQLite classes, while excluding + them from application-facing protocols. Remove non-historical raw-row forwarding helpers from + the facade. +- Use `TapeViewManifestAssemblySources` for the complete application source set and + `TapeViewManifestLookupMaps` for pure domain lookups. Preserve each historical + `TapeViewManifestSourceMaps` shape only at its original legacy import path. +- Advance the rebuildable search projection to version 3 so same-head version 2 rows from a + possible interrupted legacy reset are never trusted. The first current-Tape search performs a + one-time rebuild; linked read-only search uses the existing effective-Tape fallback until a + current rebuild exists. +- Preserve schema SQL, existing rows, canonical policy identifiers, hashes, source identities, + provenance keys, error messages where tested, and bounded query limits. +- Preserve projection failure fallback and best-effort fork projection cleanup. +- Keep trace evidence distinct from transcript projection in replay dependencies. + +## Follow-up Hardening + +1. Replace the deleted monolithic Tape test in the Memory native scope with every split suite that + contains `itIfSqlite` or `describeIfSqlite`, and make the scope validator discover this required + coverage independently. +2. Route final Session Tape deletion through `deleteTapeGeneration` and recover a failed FTS row + delete by invalidating and dropping the derivative within the generation transaction. +3. Prune pre-version-3 and metadata-orphaned projection rows during schema initialization without + rebuilding every current projection eagerly. +4. Freeze legacy shim export surfaces; remove non-historical facade raw-row helpers, and retain + historical concrete-store methods because the exported classes are compatibility contracts. +5. Rename the canonical domain ViewManifest lookup-map type and make Memory route boundary scans + reject static, dynamic, CommonJS, type-import, and re-export bypasses. +6. Reuse the composition-owned Tape fact writer in legacy import, document the same-connection + anchor requirement, cache SQLite FTS capability per connection, and replace exception-by-missing + mock behavior with explicit failure fixtures. +7. Put the complete `clearMessages` mutation set inside one shared-connection transaction and + document that a failed best-effort fork cleanup leaves permanent, non-retried residue that is + nevertheless fail-closed for merge and identifier reuse. + +## Test Strategy + +1. Record the current seven-file baseline: 120 passed and 26 native-SQLite-gated skipped tests. +2. Mechanically split the monolithic test suite by application-service boundary without changing + assertions or skip gates. +3. Add characterization coverage for reconciliation ordering, transaction atomicity, projection + fallback, and lifecycle reset. +4. Add contract coverage for append-only correction, frozen-head authorization, fork delta merge, + ViewManifest hashes, replay evidence, and projection rebuild equivalence. +5. Add source-boundary tests that reject domain reverse/runtime imports, production legacy Tape + imports, concrete facade imports from capability-scoped consumers, Memory route capability + expansion, the project SQLite driver, and non-allowlisted table access. Exercise each guard with + table-driven negative fixtures. +6. Add native-scope discovery, corrupt-FTS recovery, stale-projection cleanup, bootstrap rollback, + frozen-shim, and non-static Memory route import coverage. +7. Run the full main-process suite, Tape scale suite, type checks, formatting, i18n validation, and + lint before handoff. + +## Commit and Review Strategy + +Each implementation slice remains green and receives a local conventional commit. Before every +commit, review the complete unstaged and staged diff for hidden side effects, compatibility, +boundary cases, performance, security, misleading names, missing tests, and long-term maintenance +cost. Fix all findings and repeat validation before committing. + +The initial implementation commits were: + +1. `docs(tape): specify layering refactor` +2. `test(tape): split behavior contracts` +3. `refactor(tape): establish domain ports` +4. `refactor(tape): split application services` +5. `refactor(tape): close storage bypasses` +6. `test(tape): enforce layer boundaries` +7. `test(tape): align writer mock naming` +8. `docs(tape): refresh architecture map` + +The review remediation commits are: + +1. `fix(tape): make generation resets atomic` +2. `refactor(tape): narrow consumer ports` +3. `refactor(tape): align service ownership` +4. `test(tape): harden layer boundaries` +5. `docs(tape): clarify layer contracts` + +The cumulative review added these focused fixes before the documentation commit: + +1. `test(tape): guard semantics compatibility path` +2. `test(tape): guard project sqlite driver` +3. `fix(tape): invalidate legacy projections` +4. `refactor(tape): require capability injection` +5. `refactor(tape): narrow anchor reader port` +6. `refactor(tape): narrow storage protocols` + +The post-review hardening added these local commits: + +1. `docs(tape): specify follow-up hardening` +2. `test(memory): restore native tape scope` +3. `test(tape): restore layered settings fixture` +4. `fix(tape): harden generation cleanup` +5. `refactor(tape): harden compatibility boundaries` +6. `test(tape): repair memory type contracts` + +No commit is pushed. The final review compares the complete branch with `dev`. + +## Final Validation Record + +The final focused gates passed: + +- Memory scope discovery classified 65 files with 3 explicit exemptions, and the independent + Memory type gate passed all 65 scoped files. +- Memory behavior passed 749 tests across 46 files. +- Native SQLite and Tape coverage passed 242 tests across 13 files; the 2 skipped tests are + Windows-only handle-locking cases on the current macOS host. +- Memory performance passed all 8 tests, including the 10k/100k Tape range-bound comparison. +- Full node and renderer type checks, formatting, i18n validation, and lint passed. +- The architecture baseline generator passed and refreshed the canonical snapshot against the + final verified code commit. + +The full main-process command completed with 390 passing files and 3 failing files: 4,471 tests +passed, 2 were skipped, and 9 failed. Each affected file was then run in an isolated detached +worktree at the exact `dev` baseline (`e84428b66`), reproducing the same 6 failures in +`mainDatabase.test.ts`, 1 failure in `schedulerService.test.ts`, and 2 failures in +`sessionDataMigrations.sqlite.test.ts`. These are pre-existing baseline failures, not branch +regressions; the project-wide main-process gate therefore remains red for reasons outside this Tape +refactor. + +## Rollback + +The work is organized into locally reviewable commits. Reverting must proceed in reverse order +because later composition and boundary changes depend on the earlier domain and port extraction. +The unchanged schema and compatibility re-exports allow a complete branch rollback without a data +migration. diff --git a/docs/architecture/tape-layering/spec.md b/docs/architecture/tape-layering/spec.md new file mode 100644 index 0000000000..7d57b0c63d --- /dev/null +++ b/docs/architecture/tape-layering/spec.md @@ -0,0 +1,213 @@ +# Tape Layering Refactor Specification + +## Background + +Before this refactor, DeepChat's Tape implementation had strong runtime semantics but weak module +boundaries. The main `SessionTape` implementation combined fact writing, migration and +reconciliation, search and context recall, ViewManifest and replay assembly, subagent lineage, +and fork management in one large module. The SQLite entry table also exposed destructive +lifecycle operations beside normal append and read operations. + +Several consumers bypassed `SessionTape` and depended directly on the SQLite table. Transcript +writes, Memory ingestion, Memory management routes, Session settings, and startup migration each +used a different subset of Tape behavior, but the table-shaped dependency gave them more +authority than they needed. Tape types also flowed in the wrong direction because the Tape layer +imported Agent loop port types. + +This refactor adopts Bub's useful dependency pattern—domain primitives, narrow store protocols, +application services, and independent view selection—without copying Bub's simpler schema or +reset semantics. DeepChat retains its stronger revision, retraction, ViewManifest, frozen-head, +and fork contracts. + +## Goals + +1. Establish a top-level `src/main/tape/` subsystem with explicit domain, port, application, and + SQLite infrastructure boundaries. +2. Split `SessionTape` along its existing cohesive behavior groups while retaining a compatibility + facade. +3. Replace raw table dependencies with the smallest capability required by each consumer. +4. Keep destructive reset and delete operations outside the append-only entry store contract. +5. Remove the Tape-to-Agent reverse dependency and delete unused `TapeRecorder` capabilities. +6. Preserve all persisted data, public IPC behavior, runtime ordering, transaction boundaries, + failure fallbacks, and performance characteristics. +7. Add enforceable dependency and behavioral contracts so the layering does not regress. + +## Data Families + +The refactor keeps three distinct data families: + +| Data family | Role | Authority | +| --- | --- | --- | +| Tape facts | Append-only execution facts, anchors, manifests, lineage, and fork receipts | Tape | +| Transcript projection | UI-oriented structured messages and a legacy backfill source | Session data | +| Trace evidence | Provider request and terminal execution evidence used by replay | Session trace storage | + +Replay may combine Tape facts with trace evidence through explicit read ports. This is not a reason +to treat trace evidence as transcript data or to move it into the Tape entry schema. + +## Required Invariants + +- Entries in an active Tape are append-only. Known facts are never updated in place. +- Corrections and deletions of projected messages are represented by appended replacement or + retraction facts. +- Anchors are reconstruction points and never imply deletion of earlier entries. +- Compaction changes the selected view, not the retained history. +- Fork merge appends only the fork delta and a merge receipt to the parent. +- Cross-Tape reads require an explicit direct-child lineage fact and remain bounded by the stored + child head. +- Search projections are rebuildable derivatives. Projection failures retain the existing bounded + effective-view fallback. +- Destructive Session cleanup is a lifecycle operation, not a normal Tape store operation. +- A reset that creates a new Tape incarnation deletes entries, mutation projection state, and + search projection state and appends the new bootstrap anchor in one SQLite transaction. +- A discarded fork is fail-closed for merge and identifier reuse even when best-effort physical + cleanup fails. Failed cleanup leaves permanent inert residue; no automatic retry is scheduled. + +## Capability Boundaries + +| Consumer | Allowed capability | +| --- | --- | +| DeepChat loop runner | `TapeReconciliationPort`, `TapeViewManifestReader`, `TapeViewManifestWriter`, and `TapeToolFactWriter` | +| Turn coordinator and ACP compatibility adapter | `TapeReconciliationPort` | +| Session transcript | `TapeMessageFactWriter` | +| Memory runtime | `TapeRawEntryReader` and `TapeAnchorWriter` | +| Session settings and compaction | `TapeAnchorReader`, `TapeAnchorWriter`, and `TapeLifecycleAdmin` | +| Memory management routes | `TapeInspectionReader` | +| Session IPC | Existing `SessionTapePort` facade | + +A single implementation may satisfy several ports, but each consumer receives only the +structural type it needs. `TapeRawEntryReader` exposes only `getBySession`. The inspection port +returns purpose-built effective-message and Memory ViewManifest DTOs; it never returns a physical +Tape row. `TapeAnchorReader` exposes only the latest reconstruction anchor required by settings. +Transcript and settings require these capabilities to be injected; only normal or migration +composition may provide them, and only normal Session composition constructs the concrete facade. +Legacy import reuses the composition-owned `TapeMessageFactWriter`. `TapeViewManifestWriter` +intentionally exposes a `void` append contract because its consumer does not observe the stored +row; the concrete facade's richer return value remains an internal compatibility detail. + +`TapeViewManifestAssemblySources` names the complete source set assembled by the application +service. `TapeViewManifestLookupMaps` names the smaller domain lookup map used by pure +ViewManifest builders. The two historical `TapeViewManifestSourceMaps` exports remain available +only from their respective legacy compatibility modules, where each is an explicit deprecated +alias of the original shape. + +## Direct Storage Access Inventory + +The implementation must account for every current physical-table access: + +- `session/data/tape.ts`: compatibility re-export of the new facade; production imports use + `@/tape/*` directly. +- `session/data/transcript.ts`: legitimate message fact producer; migrated to + `TapeMessageFactWriter` while preserving same-connection transactions. +- `session/data/settings.ts`: bootstrap, reconstruction-anchor reads, summary/reset anchors, and + destructive cleanup; migrated to anchor and lifecycle capabilities. +- `agent/deepchat/runtime/deepChatRuntimeCoordinator.ts`: composition root that distributes + reconciliation, fact, manifest, raw-read, and anchor capabilities to narrower consumers. +- `memory/routes.ts` and app composition: use `TapeInspectionReader`; effective source spans and + Memory ViewManifest records cross the boundary only as domain DTOs. +- `memory/data/tables/deepchatMemoryIngestionProjection.ts`: one-statement freshness comparison + between Tape head and projection head; retained as an explicit read-only infrastructure + exception to preserve atomicity and query count. +- `app/startupMigrations/legacyChatImportService.ts`: destructive whole-database rebuild; retained + as an explicit startup-migration exception while reusing the composition-owned message fact + writer. +- Schema catalog and database security table-name lists: metadata, not runtime Tape access. + +## Generation and Failure Semantics + +`resetSessionTape` performs entry deletion, mutation-projection deletion, search-projection and FTS +deletion, and new bootstrap creation inside one transaction on the shared Session SQLite +connection. Any propagated lifecycle, cleanup, or bootstrap failure restores the complete prior +incarnation. The existing fail-open mutation-projection append policy remains: if applying the new +bootstrap to that derived projection fails, its metadata is invalidated after old rows have been +removed, so the new Tape can commit without trusting partial projection state. Search-projection +session cleanup is also internally transactional so it cannot leave base, metadata, and FTS rows +at different generations. + +Fork discard makes one atomic cleanup attempt. If cleanup succeeds, cleanup and the discard +receipt commit together. If cleanup fails, the cleanup attempt rolls back, the parent still +appends the discard receipt, and the failure remains non-blocking. Merge checks an existing merge +receipt first for idempotency and then rejects a discard receipt before reading the fork. Creating +a fork with an explicitly discarded identifier also fails closed. + +Context projection reads use `getByEntryIdsIfCurrent`, which verifies projection version and the +projection metadata head against the current Tape head supplied by the synchronous caller in the +same SQL statement that reads the requested rows. A non-current projection is ignored and summary +or reference context is rebuilt from the current effective Tape. Projection version 3 invalidates +version 2 data that may have survived an interrupted pre-atomic reset with the same entry-id head; +current search rebuilds that derivative on demand, while read-only linked search retains its +effective-Tape fallback. + +Corrupt or unavailable FTS storage must not block deletion of authoritative Tape or transcript +state. A failed FTS row deletion invalidates FTS metadata and drops the rebuildable virtual table +before the lifecycle transaction continues; if that recovery operation itself cannot complete, +the enclosing Tape generation transaction fails atomically. Startup removes base and FTS +projection rows owned only by pre-version-3 metadata so inert legacy text does not remain on disk +or force linked read-only searches onto permanent fallback paths. + +`clearMessages` places pending-input deletion, transcript deletion, and Tape reset inside one +outer transaction on that same connection. The Tape generation transaction becomes a nested +savepoint, so any reset, projection cleanup, or bootstrap failure restores all three data families +instead of leaving transcript and Tape at different generations. + +The Memory native-test manifest must include every split Tape suite that contains an active native +SQLite gate. Scope validation discovers these gated suites independently from the manifest so +removing or renaming one cannot silently eliminate the only real-SQLite lifecycle and FTS CI +coverage. + +## Acceptance Criteria + +1. `src/main/tape/domain/` does not import Agent, Session, Memory, App, SQLite, Electron, or logging + runtime modules. +2. Agent execution consumers compile against reconciliation, manifest, and fact ports rather than + concrete `SessionTape` or the deleted broad `TapeRecorder` interface. +3. Runtime, transcript, settings, routes, and normal application composition do not receive a + `DeepChatTapeEntriesTable` instance. +4. `TapeEntryStore` exposes no reset or delete method. +5. Existing `SessionTapePort`, persisted schema, table names, entry payloads, View policy IDs, and + renderer contracts remain unchanged. +6. Transcript mutation plus Tape correction, summary mutation plus anchor append, and clear-time + pending/transcript deletion plus Tape reset share their required transaction context. +7. `ensureSessionTapeReady` remains idempotent and runs at the same Session port boundaries. +8. Projection search fallback remains unchanged. Fork cleanup failure remains non-blocking while + its discard receipt prevents later merge or identifier reuse. +9. Baseline Tape tests remain green; the pre-refactor baseline is 120 passed and 26 + environment-gated skipped tests across seven files. +10. Tape scale coverage confirms bounded tail materialization and no added full-history query on + the Memory projection fast path. +11. Architecture tests reject forbidden imports, production use of legacy compatibility paths, + concrete facade imports from capability-scoped consumers, Memory route capability expansion, + the actual SQLite driver, and new physical-table bypasses. Negative fixtures prove that each + guard recognizes the prohibited dependency. +12. No remote Git operations are performed as part of this work. +13. Native Memory CI discovers and executes every SQLite-gated Tape suite after test splitting. +14. Corrupt FTS cleanup cannot leave transcript, Tape entries, and base search projection in + different generations; pre-version-3 projection data is removed during schema initialization. +15. Legacy Tape modules expose frozen deprecated export lists, while canonical modules use + unambiguous ViewManifest source-map names. +16. Historical concrete SQLite-class methods remain available through the frozen legacy class + exports, but non-historical raw-row helpers are absent from the `SessionTape` facade and every + application-facing port. + +## Constraints + +- Use synchronous ports where the current SQLite operation is synchronous; do not introduce + artificial asynchronous transaction boundaries. +- Preserve ordering, idempotency keys, hashes, error classes, and fallback logging semantics. +- Compatibility re-exports may remain at old module paths to control import churn. +- New SDD artifacts in this directory must use English prose. +- Every local commit requires a complete unstaged and staged diff review plus relevant validation. + +## Non-Goals + +- No database schema or data migration. +- No archive-on-reset behavior. +- No change to compaction, context selection, or ViewManifest policy. +- No redesign of transcript or trace storage. +- No renderer or IPC feature change. +- No GitHub issue, pull request, branch push, or other remote mutation. + +## Open Questions + +None. The implementation decisions required for this refactor are recorded in this specification +and the accompanying plan. diff --git a/docs/architecture/tape-layering/tasks.md b/docs/architecture/tape-layering/tasks.md new file mode 100644 index 0000000000..54314337c2 --- /dev/null +++ b/docs/architecture/tape-layering/tasks.md @@ -0,0 +1,114 @@ +# Tape Layering Refactor Tasks + +## Specification and Baseline + +- [x] Record the pre-refactor Tape baseline: 120 passed and 26 environment-gated skipped tests. +- [x] Write the English `spec.md`, `plan.md`, and `tasks.md` artifacts. +- [x] Confirm the SDD artifacts contain no unresolved clarification markers or non-English prose. +- [x] Review and commit the SDD slice. + +## Behavior Characterization + +- [x] Split the monolithic Tape suite by reconciliation, recall, lineage, view/replay, and fork + behavior. +- [x] Preserve every existing assertion and environment skip gate during the mechanical split. +- [x] Add transaction, reconciliation-order, and projection-fallback characterization coverage. +- [x] Review and commit the characterization slice. + +## Domain and Ports + +- [x] Move Tape-owned entry, source, provenance, and fact types out of Agent and SQLite modules. +- [x] Move effective-view and ViewManifest pure logic into `src/main/tape/domain/`. +- [x] Introduce normal storage and narrow consumer capability ports. +- [x] Replace the broad `TapeRecorder` dependency with `TapeToolFactWriter`. +- [x] Preserve old module paths through compatibility re-exports. +- [x] Review and commit the domain and port slice. + +## Application Services + +- [x] Extract fact, reconciliation, recall, lineage, view/replay, and fork services. +- [x] Convert `SessionTape` into a compatibility facade. +- [x] Preserve `SessionTapePort` and current reconciliation timing. +- [x] Review and commit the application-service slice. + +## Infrastructure and Bypass Closure + +- [x] Separate normal SQLite entry operations from destructive lifecycle operations. +- [x] Inject message fact capabilities into transcript without changing transaction boundaries. +- [x] Inject anchor and lifecycle capabilities into Session settings. +- [x] Replace Memory runtime and route table access with explicit capabilities. +- [x] Document and allowlist startup migration and Memory projection infrastructure exceptions. +- [x] Add lifecycle, transaction, projection, and authorization tests. +- [x] Review and commit the storage-boundary slice. + +## Architecture Enforcement + +- [x] Add domain dependency-direction tests. +- [x] Add a physical Tape table access guard with a narrow explicit allowlist. +- [x] Review and commit the architecture-guard slice. + +## Review Remediation + +- [x] Make reset generation replacement atomic across entries, mutation projection, search + projection, FTS state, and the new bootstrap. +- [x] Make fork cleanup atomic while preserving a non-blocking, fail-closed discard receipt. +- [x] Read context projection rows only when projection version and metadata head match the + synchronous caller's current Tape head in the same SQL statement. +- [x] Replace concrete `SessionTape` dependencies in the loop runner, Turn coordinator, and ACP + compatibility adapter with narrow ports. +- [x] Reduce `TapeRawEntryReader` to the Memory runtime's `getBySession` requirement. +- [x] Replace Memory route raw rows with effective-source and ViewManifest inspection DTOs. +- [x] Remove production imports of legacy Tape compatibility paths. +- [x] Move handoff, generic anchor, and fork-message fact ownership into `TapeFactService`. +- [x] Restrict `TapeForkService` to fork lifecycle behavior. +- [x] Move stored-manifest validation and replay hash helpers into the domain layer. +- [x] Rename the complete source-set DTO to `TapeViewManifestAssemblySources` and retain its old + name only as a deprecated legacy-path alias. +- [x] Harden architecture guards for main-layer, SQLite, Electron, logging, legacy-path, and Memory + route violations, including table-driven negative fixtures. +- [x] Cover every legacy compatibility wrapper and the project's actual SQLite driver in the + boundary guards. +- [x] Invalidate pre-atomic version 2 search projections and verify same-head context fallback. +- [x] Require capability injection for transcript and settings, with concrete facade construction + limited to composition boundaries. +- [x] Keep capability-scoped consumers off the concrete facade through a negative-tested guard. +- [x] Remove unused anchor and storage operations from application-facing protocols while keeping + concrete compatibility methods intact. +- [x] Refresh the canonical agent-system architecture baseline with the new Tape owner evidence. + +## Documentation and Final Validation + +- [x] Synchronize the English SDD with the implemented capabilities, transactions, ownership, and + failure semantics. +- [x] Update Tape, Session, and Memory architecture references in their existing languages. +- [x] Run the Tape contract and scale suites. +- [x] Run the full main-process test suite and Memory performance suite. +- [x] Run full type checks, formatting, i18n validation, and lint. +- [x] Scan all three SDD artifacts for non-English prose and unresolved clarification markers. +- [x] Review the complete `dev...HEAD` diff and fix every finding. +- [x] Prepare the completed task checklist and final documentation slice for a local commit. +- [x] Confirm no unexpected working-tree files exist and no remote push has been performed before + the final commit. + +## Post-Review Hardening + +- [x] Replace the stale monolithic Tape path in Memory native CI with all SQLite-gated split suites. +- [x] Make scope validation discover required native Tape coverage independently from the manifest. +- [x] Make final Session Tape deletion reuse the atomic generation lifecycle helper. +- [x] Recover failed FTS row deletion by dropping and invalidating the rebuildable derivative. +- [x] Remove pre-version-3 and metadata-orphaned search projection rows at schema initialization. +- [x] Add direct reset-bootstrap rollback and corrupt-FTS lifecycle regression tests. +- [x] Put pending-input deletion, transcript deletion, and Tape reset in one clear-time transaction. +- [x] Freeze and deprecate legacy Tape compatibility export lists without removing old symbols. +- [x] Use distinct canonical names for application assembly sources and domain lookup maps. +- [x] Audit unused concrete storage helpers and retain those present on historically exported + SQLite classes while removing non-historical facade raw-row forwarding methods. +- [x] Detect dynamic import, CommonJS require, type import, and re-export Memory route bypasses. +- [x] Reuse the composition-owned Tape message writer in legacy import. +- [x] Document the same-connection transaction requirement for settings anchor writers. +- [x] Cache FTS capability detection per SQLite connection. +- [x] Replace fallback tests that depend on missing mock methods with explicit failures. +- [x] Document permanent fork residue when best-effort discard cleanup fails. +- [x] Run focused native scope, lifecycle, recall, boundary, migration, and compatibility tests. +- [x] Run full validation, reproduce the nine unrelated main-suite failures on the exact `dev` + baseline, and review every new local commit without pushing. diff --git a/docs/architecture/tape-system.md b/docs/architecture/tape-system.md index 745d9d30f6..b5eeb6174b 100644 --- a/docs/architecture/tape-system.md +++ b/docs/architecture/tape-system.md @@ -3,19 +3,85 @@ Tape 是 Session 同寿命的 append-only execution fact log。它保存可回放事实、anchor、ViewManifest 和 Subagent lineage；message transcript 是面向 UI 的 projection，不是 Tape 的替代品。 -## 所有权和数据 +## 所有权和分层 | 能力 | 当前 owner | | --- | --- | -| append/query/replay | `src/main/session/data/tape.ts` | -| effective view | `src/main/session/data/tapeEffectiveView.ts` | -| facts | `src/main/session/data/tapeFacts.ts` | -| ViewManifest storage | `src/main/session/data/tapeViewManifest.ts` | +| entry/fact/ref、effective semantics、ViewManifest/replay 纯逻辑 | `src/main/tape/domain/` | +| 消费方能力和 storage ports | `src/main/tape/ports/` | +| Fact、Reconciler、Recall、Lineage、View/Replay、Fork services | `src/main/tape/application/` | +| `SessionTape` 兼容 facade | `src/main/tape/application/sessionTape.ts` | +| append/read/query store | `src/main/tape/infrastructure/sqlite/tapeEntryStore.ts` | +| search projection | `src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts` | +| 物理 lifecycle delete | `src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts` | | runtime assembly | `src/main/agent/deepchat/runtime/tapeViewAssembler.ts` | | policy selection | `src/main/agent/deepchat/runtime/tapeViewPolicy.ts` | | model-facing tools | `src/main/tool/agentTools/agentTapeTools.ts` | Tape entry 只能 append。更正、压缩和 handoff 通过新 fact/anchor 表达，不原地改写旧 entry。 +anchor 改变后续读取起点或重建状态，但不删除被覆盖的历史。 + +```mermaid +flowchart TD + Consumers["Agent / Transcript / Memory / Settings / IPC"] --> Ports["Tape capability ports"] + Ports --> Facade["SessionTape compatibility facade"] + Facade --> Services["Six application services"] + Services --> Stores["Entry store / Search projection / Lifecycle adapter"] + Stores --> SQLite["Shared Session SQLite connection"] +``` + +`src/main/session/data/tape*.ts` 和旧 table modules 只保留显式、冻结且标记 deprecated 的 +compatibility re-export。新代码必须从 `src/main/tape/` 或能力 port 导入，不能把兼容路径重新当作 +owner，也不能通过 canonical module 的新增导出隐式扩大旧路径合同。 + +## 能力端口和组合 + +| 消费方 | 允许依赖的 Tape 能力 | +| --- | --- | +| DeepChat loop runner | `TapeReconciliationPort`、`TapeViewManifestReader`、`TapeViewManifestWriter`、`TapeToolFactWriter` | +| Turn coordinator / ACP compatibility | `TapeReconciliationPort` | +| Transcript | `TapeMessageFactWriter` | +| Memory runtime | `TapeRawEntryReader`、`TapeAnchorWriter` | +| Settings / compaction | `TapeAnchorReader`、`TapeAnchorWriter`、`TapeLifecycleAdmin` | +| Memory routes | `TapeInspectionReader` | +| IPC / Session data | 现有 `SessionTapePort` | + +`createSessionDataFromDatabase` 组合一个 `SessionTape`，把窄能力传给 transcript 和 settings，并在既有 +IPC boundary 按原时序执行 `ensureSessionTapeReady`。facade 只做 service 组合和兼容转发，不承载新的 +domain policy；外部方法的签名、同步/异步行为、异常和 fallback 语义保持稳定。 + +`TapeRawEntryReader` 只暴露 Memory runtime 实际需要的 `getBySession`。Memory routes 使用的 +`TapeInspectionReader` 只返回 effective message source span 与 Memory ViewManifest DTO，不返回 +`DeepChatTapeEntryRow`。完整的 manifest assembly source set 命名为 +`TapeViewManifestAssemblySources`，domain lookup map 命名为 `TapeViewManifestLookupMaps`；两种历史 +`TapeViewManifestSourceMaps` 形状只在各自原有 compatibility path 作为 type alias 保留。 +`TapeAnchorReader` 只暴露 settings 实际使用的 latest reconstruction anchor；transcript/settings 必须 +由 composition 注入 port，不允许在 consumer 内隐式构造 concrete facade。 + +## 存储与事务边界 + +- `TapeEntryStore` 只负责 append/read/query；物理删除由独立 lifecycle adapter 执行，只服务于 + Session lifecycle（包含 fork Session cleanup），不属于运行中 Tape 语义。 +- transcript message mutation 与 replacement/retraction fact、summary compare-and-set 与 anchor append + 使用同一个 SQLite connection 和调用方 transaction，拆层不能拆开其原子边界。 +- `clearMessages` 在同一外层 transaction 中删除 pending input、transcript projection 并 reset Tape； + Tape generation transaction 作为 savepoint 嵌套，任一 hard failure 会同时恢复三类数据。 +- `resetSessionTape` 在同一 transaction 内删除 entry、mutation projection、search/FTS projection 并 + 创建新 bootstrap；lifecycle/cleanup/bootstrap 的 hard failure 会恢复旧 incarnation。既有 mutation + projection append fail-open 策略仍可提交新 Tape，但旧 projection row 已删除且 meta 会标 stale。 +- context projection 通过单条 `getByEntryIdsIfCurrent` SQL 校验 projection version、projection meta head + 与同步调用方提供的 current Tape head，并读取请求行；不 current 时从 effective Tape 重建 + summary/ref context。 +- search projection 升为 version 3，一次性拒绝可能来自 pre-atomic reset、恰好复用相同 head 的 version + 2 row；current search 按需重建，linked read-only search 在重建前沿用 effective-Tape fallback。 +- search projection 可以重建；projection 不可用或 coverage 不完整时回退 effective Tape search，fork + cleanup 的 projection 删除失败仍不阻断主流程，但 discard receipt 会使后续 merge 和相同 fork ID + 的显式复用 fail closed。 +- legacy chat import 的全表删除是 migration-only 例外，但消息 fact writer 复用 composition 已创建的 + `SessionTape` capability，不再另建 facade；Memory ingestion projection 为避免并发窗口，可以在一条 + 只读 SQL 中同时比较 Tape head 和 projection head。除此之外消费方不得访问物理 Tape 表。 +- reset 物理删除当前 Session Tape 后重新 bootstrap；本阶段没有 archive-on-reset，不能把 reset 解释成 + append-only 运行语义的一部分。 ## View 和 provenance @@ -54,20 +120,33 @@ read boundary 兼容，新写入必须使用 canonical policy/provenance。 `tape_info`、`tape_anchors` 是 diagnostic；`tape_handoff` 是 runtime-only。五个名称全部 reserved， MCP 不能 shadow，持久化 disabled-tool 配置也不能关闭 system capability。 -## Subagent lineage +## Fork 和 Subagent lineage Subagent 使用独立 Session 和独立 Tape。完成后父 Session append 一个 link，固定 child Tape head： - 查询时只读该 frozen head，不自动读取 child 后续 entry； - child entries 不复制进父 Tape； -- discard/merge 只改变父 Session 的 lineage fact，不篡改 child 历史； +- 只有显式授权的直接 child 可以跨 Tape 读取；missing、recreated 或 incarnation 不匹配必须 fail + closed； - 非直接 child、未授权 Session 或递归 Subagent 不能通过 Tape tool 越权读取。 +普通 fork merge 只把 fork head 相对基线的 delta 作为新 entry append 到父 Tape，并追加 merge receipt； +不得改写父 Tape 旧 entry，也不得把整份 fork 历史重复复制。discard 和重复 merge 保持既有审计、幂等及 +best-effort projection cleanup 语义。discard cleanup 成功时与 receipt 一起提交；cleanup 失败时回滚本次 +cleanup、仍 append receipt 并记录 warning。此时残留 fork 是永久惰性数据，本阶段没有自动或后台重试 +路径；discard receipt 仍保证它不能再次 merge，也不能用相同 fork ID 显式复用。 + ## 回放和兼容 Replay 必须保持 entry order、role、tool call/result pairing、anchor cursor 和 policy version。未知旧 fact 可以按兼容规则跳过或映射，但不能静默改变已知 fact 的含义。测试至少覆盖正常 chat、resume、tool interaction、compaction、context pressure、Subagent frozen head 和旧 manifest 读取。 -关键测试位于 `test/main/session/`、`test/main/agent/deepchat/` 和 `test/main/tool/`。历史的 Tape -increment SDD 已合并到本文，详细实施顺序从 Git 历史查询。 +stored manifest validation、legacy `hashVersion` normalization、entry-id collection 和 replay slice hash +属于 `src/main/tape/domain/replay.ts` 的纯逻辑；SQLite row parsing、message trace 和 terminal evidence +读取仍属于 `TapeViewReplayService`，不能反向放进 domain。 + +关键行为测试位于 `test/main/session/data/tape*.test.ts`，分层守护位于 +`test/main/tape/layerBoundaries.test.ts`；runtime 和 tool 契约继续位于 +`test/main/agent/deepchat/` 与 `test/main/tool/`。历史的 Tape increment SDD 已合并到本文，详细实施 +顺序从 Git 历史查询。 diff --git a/scripts/check-memory-test-scope.mjs b/scripts/check-memory-test-scope.mjs index 993f806530..1ce061efc7 100644 --- a/scripts/check-memory-test-scope.mjs +++ b/scripts/check-memory-test-scope.mjs @@ -6,6 +6,9 @@ const CATEGORY_NAMES = ['behavior', 'native', 'eval', 'perf'] const MEMORY_IMPORT = /from ['"][^'"]*(?:\/memory(?:\/|['"])|agent-memory|memory\.routes|agentMemory|recallKeyword)[^'"]*['"]/ const TEST_FILE = /\.(?:test|spec)\.[cm]?[jt]sx?$|\.perf\.[cm]?[jt]sx?$/ +const NATIVE_TAPE_TEST = + /^test\/main\/session\/data\/(?:tape[^/]*|tables\/deepchatTapeEntriesTable)\.(?:test|spec)\.ts$/ +const NATIVE_SQLITE_GATE = /\b(?:itIfSqlite|describeIfSqlite)\b/ function walkTestFiles(rootDir) { const testRoot = join(rootDir, 'test', 'main') @@ -30,11 +33,24 @@ function isMemoryOwned(path, readContent) { return MEMORY_IMPORT.test(readContent(path)) } +export function findRequiredNativeTapeTests(paths, readContent) { + return paths.filter( + (path) => NATIVE_TAPE_TEST.test(path) && NATIVE_SQLITE_GATE.test(readContent(path)) + ) +} + +function discoverRequiredNativeTapeTests(rootDir) { + return findRequiredNativeTapeTests(walkTestFiles(rootDir), (path) => + readFileSync(join(rootDir, path), 'utf8') + ) +} + export function validateMemoryTestScope({ rootDir, manifest, existingPaths, discoveredPaths, + requiredNativePaths = [], fileContents, readFile }) { @@ -101,6 +117,12 @@ export function validateMemoryTestScope({ } } + for (const path of requiredNativePaths) { + if (ownerByPath.get(path) !== 'native') { + errors.push(`Required native Tape test is not classified as native: ${path}`) + } + } + return errors.sort() } @@ -108,7 +130,8 @@ function runCli() { const scriptDirectory = dirname(fileURLToPath(import.meta.url)) const rootDir = resolve(scriptDirectory, '..') const manifest = JSON.parse(readFileSync(join(rootDir, 'test/memory-test-scope.json'), 'utf8')) - const errors = validateMemoryTestScope({ rootDir, manifest }) + const requiredNativePaths = discoverRequiredNativeTapeTests(rootDir) + const errors = validateMemoryTestScope({ rootDir, manifest, requiredNativePaths }) if (errors.length > 0) { console.error(['Memory test scope validation failed:', ...errors.map((error) => `- ${error}`)].join('\n')) process.exitCode = 1 diff --git a/scripts/generate-architecture-baseline.mjs b/scripts/generate-architecture-baseline.mjs index 2172da73c4..ec66e5d901 100644 --- a/scripts/generate-architecture-baseline.mjs +++ b/scripts/generate-architecture-baseline.mjs @@ -24,7 +24,8 @@ const AGENT_SYSTEM_RUNTIME_BOUNDARY_FILES = [ 'src/main/agent/deepchat/runtime/process.ts', 'src/main/agent/deepchat/runtime/dispatch.ts', 'src/main/session/data/transcript.ts', - 'src/main/session/data/tape.ts', + 'src/main/tape/application/sessionTape.ts', + 'src/main/tape/ports/capabilities.ts', 'src/main/provider/providers/acpProvider.ts' ] const AGENT_SYSTEM_EXPECTED_FILES = [ @@ -73,7 +74,11 @@ const AGENT_SYSTEM_OWNER_EVIDENCE = [ 'src/main/agent/deepchat/loop/deepChatLoopEngine.ts', /\bclass DeepChatLoopEngine\b/g ], - ['tapeRecorder', 'src/main/agent/deepchat/loop/ports.ts', /\binterface TapeRecorder\b/g], + [ + 'tapeToolFactWriter', + 'src/main/tape/ports/capabilities.ts', + /\binterface TapeToolFactWriter\b/g + ], [ 'memoryRuntimeCoordinator', 'src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts', diff --git a/src/main/agent/acp/compatibility/adapters.ts b/src/main/agent/acp/compatibility/adapters.ts index 3bfc85a76d..fb1b5c1956 100644 --- a/src/main/agent/acp/compatibility/adapters.ts +++ b/src/main/agent/acp/compatibility/adapters.ts @@ -24,8 +24,8 @@ import { type IoParams, type StreamState } from '@/agent/deepchat/runtime/types' -import { SessionTranscript } from '@/session/data/transcript' -import { SessionTape } from '@/session/data/tape' +import type { SessionTranscript } from '@/session/data/transcript' +import type { TapeReconciliationPort } from '@/tape/ports/capabilities' import { buildPersistableMessageTracePayload } from '@/agent/deepchat/runtime/messageTracePayload' interface ProjectionState { @@ -36,7 +36,7 @@ interface ProjectionState { export interface AcpCompatibilityProjectionAdapterOptions { messageStore: SessionTranscript - tapeService: SessionTape + tapeReconciliation: TapeReconciliationPort writeViewManifest: (input: AcpViewManifestInput) => void | Promise setStatus: (status: 'generating' | 'idle' | 'error') => void publishEvent: DeepChatEventPublisher @@ -56,8 +56,8 @@ export class AcpCompatibilityProjectionAdapter implements AcpCompatibilityProjec sessionId: AcpViewManifestInput['sessionId'] userContent: Parameters[2] }): AcpProjectionHandle { - const { messageStore, tapeService } = this.options - tapeService.ensureSessionTapeReady(input.sessionId, messageStore) + const { messageStore, tapeReconciliation } = this.options + tapeReconciliation.ensureSessionTapeReady(input.sessionId, messageStore) const userMessageId = messageStore.createUserMessage( input.sessionId, messageStore.getNextOrderSeq(input.sessionId), diff --git a/src/main/agent/acp/compatibility/dependencies.ts b/src/main/agent/acp/compatibility/dependencies.ts index 32b6b1908d..42755c8e51 100644 --- a/src/main/agent/acp/compatibility/dependencies.ts +++ b/src/main/agent/acp/compatibility/dependencies.ts @@ -17,7 +17,7 @@ import { createUserChatMessage } from '@/agent/deepchat/runtime/contextBuilder' import { resolveEffectiveActiveSkillNames } from '@/agent/deepchat/resources/systemPromptBuilder' import type { SessionSettingsStore } from '@/session/data/settings' import type { SessionTranscript } from '@/session/data/transcript' -import type { SessionTape } from '@/session/data/tape' +import type { TapeReconciliationPort } from '@/tape/ports/capabilities' import type { DeepChatToolResolver } from '@/agent/deepchat/runtime/toolResolver' import type { DeepChatEventPublisher, @@ -34,7 +34,7 @@ export interface AcpCompatibilityDependencyBuilderDependencies { providerRuntime: Pick sessionStore: SessionSettingsStore messageStore: SessionTranscript - tapeService: SessionTape + tapeReconciliation: TapeReconciliationPort toolResolver: DeepChatToolResolver appendViewManifest(input: AcpViewManifestInput): void setStatus(sessionId: string, status: DeepChatSessionState['status']): void @@ -84,7 +84,7 @@ export function createAcpCompatibilityDependencies( publishEvent: dependencies.publishEvent, publishSessionUpdate: dependencies.publishSessionUpdate, messageStore: dependencies.messageStore, - tapeService: dependencies.tapeService, + tapeReconciliation: dependencies.tapeReconciliation, writeViewManifest: async (manifest) => dependencies.appendViewManifest(manifest), setStatus: (status) => dependencies.setStatus(sessionId, status) }) diff --git a/src/main/agent/deepchat/loop/ports.ts b/src/main/agent/deepchat/loop/ports.ts index 69a939dce1..42bb831725 100644 --- a/src/main/agent/deepchat/loop/ports.ts +++ b/src/main/agent/deepchat/loop/ports.ts @@ -1,11 +1,10 @@ import type { AppSessionId } from '@/agent/shared/agentSessionIds' -import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' +import type { AssistantMessageBlock } from '@shared/types/agent-interface' import type { ChatMessage } from '@shared/types/core/chat-message' import type { LLMCoreStreamEvent } from '@shared/types/core/llm-events' import type { MCPToolCall, MCPToolDefinition, MCPToolResponse } from '@shared/types/core/mcp' import type { ToolCallOptions, ToolPermissionPreCheckResult } from '@shared/types/tool' import type { ModelConfig } from '@shared/types/provider' -import type { DeepChatTapeViewManifest } from '@shared/types/tape-view-manifest' import type { MemorySessionHandle } from '@/agent/deepchat/memory/memoryPromptContributor' export interface ProviderRequest { @@ -162,54 +161,6 @@ export interface ToolResultPort { }): Promise } -export interface TapeEntryRef { - sessionId: AppSessionId - entryId: number -} - -export interface TapeHead { - sessionId: AppSessionId - latestEntryId: number -} - -export interface TapeFactProvenance { - source: 'message' | 'tool_call' | 'tool_result' | 'runtime_event' - sourceId: string - sequence: number -} - -export interface TapeToolFactInput { - sessionId: AppSessionId - messageId: string - orderSeq: number - blockIndex: number - block: AssistantMessageBlock - provenance: TapeFactProvenance -} - -export interface TapeAnchorInput { - sessionId: AppSessionId - name: string - state: Readonly> - meta: Readonly> - provenance: TapeFactProvenance -} - -export interface TapeEffectiveView { - sessionId: AppSessionId - records: ChatMessageRecord[] -} - -export interface TapeRecorder { - ensureSession(input: { sessionId: AppSessionId }): Promise - appendUserMessage(input: { record: ChatMessageRecord }): Promise - appendViewManifest(manifest: DeepChatTapeViewManifest): Promise - appendAssistantFact(input: { record: ChatMessageRecord }): Promise - appendToolFact(input: TapeToolFactInput): Promise - appendAnchor(input: TapeAnchorInput): Promise - readEffectiveView(input: { sessionId: AppSessionId }): Promise -} - export interface OutputSink { update(input: { runId: string diff --git a/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts b/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts index 75fecf0aa7..59719edcc0 100644 --- a/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts +++ b/src/main/agent/deepchat/memory/memoryRuntimeCoordinator.ts @@ -4,13 +4,14 @@ import { appendMemorySectionWithManifest } from '@/memory/injection' import type { MemoryExecutionToken, MemoryRuntimePort } from '@/memory/injection' import { BUILTIN_DEEPCHAT_AGENT_ID } from '@/agent/repository' import { withSoftDeadline } from '@/memory/core/asyncDeadline' -import { buildEffectiveTapeView } from '@/session/data/tapeEffectiveView' +import { buildEffectiveTapeView } from '@/tape/domain/effectiveView' import type { DeepChatMemoryIngestionCurrentRange, DeepChatMemoryIngestionProjectionInput, DeepChatMemoryIngestionProjectionRow } from '@/memory/data/tables/deepchatMemoryIngestionProjection' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeAnchorWriter, TapeRawEntryReader } from '@/tape/ports/capabilities' import { MEMORY_EXTRACTION_CHUNKS_PER_QUEUE_TASK, buildMemoryExtractionChunks, @@ -68,13 +69,8 @@ export interface MemoryRuntimeCoordinatorDependencies { getMemoryCursorOrderSeq(sessionId: string): number | null updateMemoryCursorOrderSeq(sessionId: string, orderSeq: number): void rewindMemoryCursorOrderSeq(sessionId: string, orderSeq: number): void - getTapeRows(sessionId: string): DeepChatTapeEntryRow[] - appendTapeAnchor(input: { - sessionId: string - name: string - state: Record - meta?: Record - }): void + tapeReader: TapeRawEntryReader + tapeAnchorWriter: TapeAnchorWriter getIngestionProjection(): MemoryIngestionProjection | undefined } @@ -193,7 +189,7 @@ export class MemoryRuntimeCoordinator implements MemoryPromptContributor, Memory } if (this.canContinueExecution(sessionId, executionToken)) { try { - this.deps.appendTapeAnchor({ + this.deps.tapeAnchorWriter.appendAnchor({ sessionId, name: 'memory/view_assembled', state: assembled.manifest as unknown as Record, @@ -406,7 +402,7 @@ export class MemoryRuntimeCoordinator implements MemoryPromptContributor, Memory this.deps.updateMemoryCursorOrderSeq(sessionId, chunk.cursorCommitOrderSeq) } if (result.createdIds.length > 0) { - this.deps.appendTapeAnchor({ + this.deps.tapeAnchorWriter.appendAnchor({ sessionId, name: 'memory/extract', state: { @@ -678,7 +674,7 @@ export class MemoryRuntimeCoordinator implements MemoryPromptContributor, Memory maxEntryId: number, projection: MemoryIngestionProjection ): { rows: DeepChatMemoryIngestionProjectionRow[]; cursorCommitAllowed: boolean } { - const view = buildEffectiveTapeView(this.deps.getTapeRows(sessionId)) + const view = buildEffectiveTapeView(this.deps.tapeReader.getBySession(sessionId)) const projectionRows = this.projectionRowsFromEffectiveView(sessionId, view) try { projection.replaceSession(sessionId, projectionRows, maxEntryId) @@ -732,7 +728,7 @@ export class MemoryRuntimeCoordinator implements MemoryPromptContributor, Memory cursorCommitAllowed: boolean ): { rows: DeepChatMemoryIngestionProjectionRow[]; cursorCommitAllowed: boolean } | null { try { - const view = buildEffectiveTapeView(this.deps.getTapeRows(sessionId)) + const view = buildEffectiveTapeView(this.deps.tapeReader.getBySession(sessionId)) const rows = this.projectionRowsFromEffectiveView(sessionId, view) return { rows: this.filterIngestionRange(rows, fromOrderSeqExclusive, toOrderSeqInclusive), diff --git a/src/main/agent/deepchat/runtime/contextBuilder.ts b/src/main/agent/deepchat/runtime/contextBuilder.ts index 705316c019..cd2deb0e7d 100644 --- a/src/main/agent/deepchat/runtime/contextBuilder.ts +++ b/src/main/agent/deepchat/runtime/contextBuilder.ts @@ -14,7 +14,7 @@ import { estimateMessageTokens, estimateMessagesTokens } from '@shared/utils/messageTokens' -import { isCompactionRecord } from '@/session/data/tapeViewManifest' +import { isCompactionRecord } from '@/tape/domain/viewManifest' export { estimateMessagesTokens } from '@shared/utils/messageTokens' diff --git a/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts b/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts index 99257feccd..7f1773be95 100644 --- a/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts +++ b/src/main/agent/deepchat/runtime/deepChatLoopRunner.ts @@ -66,8 +66,13 @@ import { createTapeViewManifest, resolveTapeViewManifestPolicy, type TapeViewContextSelection -} from '@/session/data/tapeViewManifest' -import type { SessionTape } from '@/session/data/tape' +} from '@/tape/domain/viewManifest' +import type { + TapeReconciliationPort, + TapeToolFactWriter, + TapeViewManifestReader, + TapeViewManifestWriter +} from '@/tape/ports/capabilities' import type { AgentTraceSettingsPort } from '@/agent/traceSettings' import type { DeepChatToolResolver } from '@/agent/deepchat/runtime/toolResolver' import type { @@ -179,7 +184,10 @@ export interface DeepChatLoopRunnerPorts { traceSettings: AgentTraceSettingsPort sessionStore: SessionSettingsStore messageStore: SessionTranscript - tapeService: SessionTape + tapeReconciliation: TapeReconciliationPort + tapeViewManifestReader: TapeViewManifestReader + tapeViewManifestWriter: TapeViewManifestWriter + tapeToolFactWriter: TapeToolFactWriter pendingInputCoordinator: SessionPendingInputs toolResolver: DeepChatToolResolver providerPermissionCoordinator: ProviderPermissionCoordinator @@ -391,7 +399,8 @@ export class DeepChatLoopRunner { const traceEnabled = this.ports.traceSettings.isEnabled() const initialRequestSeq = Math.max( - this.ports.tapeService.listViewManifestsByMessage(sessionId, messageId)[0]?.requestSeq ?? 0, + this.ports.tapeViewManifestReader.listViewManifestsByMessage(sessionId, messageId)[0] + ?.requestSeq ?? 0, this.ports.messageStore.getMaxMessageTraceRequestSeq(messageId) ) const temperature = generationSettings.temperature @@ -754,7 +763,7 @@ export class DeepChatLoopRunner { }, io: { messageStore: this.ports.messageStore, - tapeRecorder: this.ports.tapeService, + tapeToolFactWriter: this.ports.tapeToolFactWriter, publishEvent: this.ports.publishEvent, publishSessionUpdate: this.ports.publishSessionUpdate } @@ -770,7 +779,7 @@ export class DeepChatLoopRunner { } appendTapeViewManifest(params: AppendTapeViewManifestInput): void { - const sourceMaps = this.ports.tapeService.getViewManifestSourceMaps( + const sourceMaps = this.ports.tapeViewManifestReader.getViewManifestSourceMaps( params.sessionId, params.messageId ) @@ -799,7 +808,7 @@ export class DeepChatLoopRunner { supportsAudioInput: params.supportsAudioInput, traceDebugEnabled: params.traceDebugEnabled }) - this.ports.tapeService.appendViewManifest(manifest) + this.ports.tapeViewManifestWriter.appendViewManifest(manifest) } emitRateLimitWaitingMessage( @@ -893,7 +902,10 @@ export class DeepChatLoopRunner { prepareCompaction: async (systemPrompt) => { const prepared = await this.ports.inputPreparationCoordinator.prepareExisting({ ensureHistory: () => - this.ports.tapeService.ensureSessionTapeReady(params.sessionId, this.ports.messageStore) + this.ports.tapeReconciliation.ensureSessionTapeReady( + params.sessionId, + this.ports.messageStore + ) .historyRecords, prepareIntent: async (historyRecords) => await this.ports.compactionService.prepareForContextPressureRecovery({ diff --git a/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts b/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts index d4eee9c9b7..8c1b365a6f 100644 --- a/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts +++ b/src/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.ts @@ -329,11 +329,8 @@ export class DeepChatRuntimeCoordinator { this.sqlitePresenter.deepchatSessionsTable.updateMemoryCursorOrderSeq(sessionId, orderSeq), rewindMemoryCursorOrderSeq: (sessionId, orderSeq) => this.sqlitePresenter.deepchatSessionsTable.rewindMemoryCursorOrderSeq(sessionId, orderSeq), - getTapeRows: (sessionId) => - this.sqlitePresenter.deepchatTapeEntriesTable.getBySession(sessionId), - appendTapeAnchor: (input) => { - this.sqlitePresenter.deepchatTapeEntriesTable.appendAnchor(input) - }, + tapeReader: this.tapeService, + tapeAnchorWriter: this.tapeService, getIngestionProjection: runtimePorts.getMemoryIngestionProjection }) this.memoryPromptContributor = this.memoryCoordinator @@ -413,7 +410,10 @@ export class DeepChatRuntimeCoordinator { traceSettings: this.traceSettings, sessionStore: this.sessionStore, messageStore: this.messageStore, - tapeService: this.tapeService, + tapeReconciliation: this.tapeService, + tapeViewManifestReader: this.tapeService, + tapeViewManifestWriter: this.tapeService, + tapeToolFactWriter: this.tapeService, pendingInputCoordinator: this.pendingInputCoordinator, toolResolver: this.toolResolver, providerPermissionCoordinator: this.providerPermissionCoordinator, @@ -460,7 +460,7 @@ export class DeepChatRuntimeCoordinator { toolService: this.toolService, sessionStore: this.sessionStore, messageStore: this.messageStore, - tapeService: this.tapeService, + tapeReconciliation: this.tapeService, pendingInputCoordinator: this.pendingInputCoordinator, toolResolver: this.toolResolver, compactionService: this.compactionService, @@ -578,7 +578,7 @@ export class DeepChatRuntimeCoordinator { providerRuntime: this.providerRuntime, sessionStore: this.sessionStore, messageStore: this.messageStore, - tapeService: this.tapeService, + tapeReconciliation: this.tapeService, toolResolver: this.toolResolver, appendViewManifest: (manifest) => { this.loopRunner.appendTapeViewManifest({ diff --git a/src/main/agent/deepchat/runtime/process.ts b/src/main/agent/deepchat/runtime/process.ts index 8f0c6679f5..b75acefbc0 100644 --- a/src/main/agent/deepchat/runtime/process.ts +++ b/src/main/agent/deepchat/runtime/process.ts @@ -31,7 +31,7 @@ import { } from '@/agent/deepchat/loop/deepChatLoopEngine' import { emitDeepChatLoopNotification } from '@/agent/deepchat/loop/notificationObserver' import type { OutputSink } from '@/agent/deepchat/loop/ports' -import { buildTapeToolFactInputs } from '@/session/data/tapeFacts' +import { buildTapeToolFactInputs } from '@/tape/application/factPersistence' const UNKNOWN_CONTEXT_LIMIT = Number.MAX_SAFE_INTEGER const USER_CANCELED_GENERATION_ERROR = 'common.error.userCanceledGeneration' @@ -739,7 +739,7 @@ export async function processStream(params: ProcessParams): Promise sessionStore: SessionSettingsStore messageStore: SessionTranscript - tapeService: SessionTape + tapeReconciliation: TapeReconciliationPort pendingInputCoordinator: SessionPendingInputs toolResolver: DeepChatToolResolver compactionService: CompactionService @@ -352,7 +352,10 @@ export class TurnCoordinator { ensureHistory: () => this.ports.runSynchronousPreStreamStep(sessionId, 'tape-ready', () => getTapeContextHistoryRecords( - this.ports.tapeService.ensureSessionTapeReady(sessionId, this.ports.messageStore) + this.ports.tapeReconciliation.ensureSessionTapeReady( + sessionId, + this.ports.messageStore + ) .historyRecords ) ), @@ -840,7 +843,10 @@ export class TurnCoordinator { sessionId, 'tape-ready', () => - this.ports.tapeService.ensureSessionTapeReady(sessionId, this.ports.messageStore) + this.ports.tapeReconciliation.ensureSessionTapeReady( + sessionId, + this.ports.messageStore + ) .historyRecords ), refreshHistory: () => @@ -848,7 +854,10 @@ export class TurnCoordinator { sessionId, 'tape-ready', () => - this.ports.tapeService.ensureSessionTapeReady(sessionId, this.ports.messageStore) + this.ports.tapeReconciliation.ensureSessionTapeReady( + sessionId, + this.ports.messageStore + ) .historyRecords ), prepareIntent: async (historyRecords) => { diff --git a/src/main/agent/deepchat/runtime/types.ts b/src/main/agent/deepchat/runtime/types.ts index 176f20c674..7986eef96c 100644 --- a/src/main/agent/deepchat/runtime/types.ts +++ b/src/main/agent/deepchat/runtime/types.ts @@ -17,11 +17,11 @@ import type { DeepChatLoopNotificationObserver, PendingToolInteractionOrigin, PersistedToolBatchState, - TapeRecorder, ToolCatalogPort, ToolExecutionPort, ToolResultPort } from '@/agent/deepchat/loop/ports' +import type { TapeToolFactWriter } from '@/tape/ports/capabilities' export interface InterleavedReasoningConfig { preserveReasoningContent: boolean @@ -91,7 +91,7 @@ export type ProcessIoParams = Pick< IoParams, 'messageStore' | 'publishEvent' | 'publishSessionUpdate' > & { - tapeRecorder: Pick + tapeToolFactWriter: TapeToolFactWriter } export interface ProcessControlCollaborators { diff --git a/src/main/app/composition.ts b/src/main/app/composition.ts index c2eef8f0ab..747bfe152a 100644 --- a/src/main/app/composition.ts +++ b/src/main/app/composition.ts @@ -386,7 +386,8 @@ export async function createMainProcessControl(dependencies: { appDatabase, sessionData.database, projectDatabase, - memoryDatabase + memoryDatabase, + sessionData.tapeStore ) usageStatsService = new UsageStatsService( sessionData.database, @@ -986,7 +987,8 @@ export async function createMainProcessControl(dependencies: { transcript: sessionData.transcript, settings: sessionData.settings, pendingInputs: sessionData.pendingInputs, - runtime: deepChatRuntimeCoordinator + runtime: deepChatRuntimeCoordinator, + runInTransaction: (operation) => sessionData.database.getDatabase().transaction(operation)() }) memoryIngestionObserver = deepChatRuntimeCoordinator.memoryIngestionObserver acpAgentRuntime = new AcpAgentRuntime( @@ -1611,7 +1613,7 @@ export async function createMainProcessControl(dependencies: { const memoryRoutes = createMemoryRoutes({ memoryService, getAgentType: (agentId) => agentSettings.getAgentType(agentId), - getTapeEntries: () => sessionData.database.deepchatTapeEntriesTable, + getTapeInspection: () => sessionData.tapeStore, getAuditEntries: () => memoryDatabase.agentMemoryAuditTable }) const desktopRoutes = createDesktopRoutes({ diff --git a/src/main/app/startupMigrations/legacyChatImportService.ts b/src/main/app/startupMigrations/legacyChatImportService.ts index 89904f61f7..acd72273f5 100644 --- a/src/main/app/startupMigrations/legacyChatImportService.ts +++ b/src/main/app/startupMigrations/legacyChatImportService.ts @@ -12,6 +12,7 @@ import { isReasoningEffort } from '@shared/types/model-db' import { resolveAcpAgentAlias } from '@shared/utils/acpAgentAlias' import { SessionTranscript } from '@/session/data/transcript' import { SessionDatabase } from '@/session/data/database' +import type { TapeMessageFactWriter } from '@/tape/ports/capabilities' import type { ProjectDatabase } from '@/project/data/database' import type { AppDatabase } from '@/app/data/database' import type { MemoryDatabase } from '@/memory/data/database' @@ -44,13 +45,14 @@ export class LegacyChatImportService { sessionDatabase: SessionDatabase, projectDatabase: ProjectDatabase, memoryDatabase: MemoryDatabase, + tapeFacts: TapeMessageFactWriter, sourceDbPath?: string ) { this.appDatabase = appDatabase this.sessionDatabase = sessionDatabase this.projectDatabase = projectDatabase this.memoryDatabase = memoryDatabase - this.messageStore = new SessionTranscript(this.sessionDatabase) + this.messageStore = new SessionTranscript(this.sessionDatabase, tapeFacts) this.sourceDbPath = sourceDbPath ?? path.join(app.getPath('userData'), 'app_db', 'chat.db') } @@ -157,6 +159,8 @@ export class LegacyChatImportService { private async clearImportedSessionData(): Promise { const db = this.sessionDatabase.getDatabase() db.transaction(() => { + // Migration-only exception: overwrite import intentionally clears all legacy-owned tables + // in one transaction, including Tape and its projections. db.exec(` DELETE FROM deepchat_message_search_results; DELETE FROM deepchat_search_documents; diff --git a/src/main/data/schemaCatalog.ts b/src/main/data/schemaCatalog.ts index 5922b76c10..55f4a11899 100644 --- a/src/main/data/schemaCatalog.ts +++ b/src/main/data/schemaCatalog.ts @@ -19,9 +19,9 @@ import { DeepChatMessageSearchResultsTable } from '@/session/data/tables/deepcha import { DeepChatSearchDocumentsTable } from '@/session/data/tables/deepchatSearchDocuments' import { DeepChatPendingInputsTable } from '@/session/data/tables/deepchatPendingInputs' import { DeepChatUsageStatsTable } from '@/session/data/tables/deepchatUsageStats' -import { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' +import { DeepChatTapeEntriesTable } from '@/tape/infrastructure/sqlite/tapeEntryStore' import { DeepChatMemoryIngestionProjectionTable } from '@/memory/data/tables/deepchatMemoryIngestionProjection' -import { DeepChatTapeSearchProjectionTable } from '@/session/data/tables/deepchatTapeSearchProjection' +import { DeepChatTapeSearchProjectionTable } from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' import { DeepChatSessionMetadataTable } from '@/session/data/tables/deepchatSessionMetadata' import { LegacyImportStatusTable } from '@/app/data/tables/legacyImportStatus' import { AgentsTable } from '@/agent/data/tables/agents' diff --git a/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts b/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts index 557247e3ba..c59dfc857d 100644 --- a/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts +++ b/src/main/memory/data/tables/deepchatMemoryIngestionProjection.ts @@ -1,6 +1,7 @@ import type Database from 'better-sqlite3-multiple-ciphers' import { BaseTable } from '@/data/baseTable' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeMutationProjection } from '@/tape/ports/storage' import type { MemoryPerfObserver } from '../../../memory/ports' import { readTapeMessageRetractionId, @@ -9,7 +10,7 @@ import { tapeEntryToMessageRecord, tapeMessageRank, tapeToolRank -} from '@/session/data/tables/deepchatTapeEffectiveSemantics' +} from '@/tape/domain/effectiveSemantics' export const DEEPCHAT_MEMORY_INGESTION_PROJECTION_VERSION = 1 @@ -92,7 +93,10 @@ const MEMORY_INGESTION_PROJECTION_SQL = ` ); ` -export class DeepChatMemoryIngestionProjectionTable extends BaseTable { +export class DeepChatMemoryIngestionProjectionTable + extends BaseTable + implements TapeMutationProjection +{ constructor( db: Database.Database, private readonly perfObserver?: MemoryPerfObserver @@ -267,6 +271,8 @@ export class DeepChatMemoryIngestionProjectionTable extends BaseTable { toOrderSeqInclusive: number ): DeepChatMemoryIngestionCurrentRange { this.perfObserver?.increment('repositoryCalls') + // Read-only infrastructure exception: the Tape head and projection head must be observed by + // one SQL statement so concurrent appends cannot create a false-current projection window. const records = this.db .prepare( `WITH state AS ( diff --git a/src/main/memory/routes.ts b/src/main/memory/routes.ts index 055b5a20c5..c9f8f5b4d9 100644 --- a/src/main/memory/routes.ts +++ b/src/main/memory/routes.ts @@ -40,7 +40,7 @@ import { type MemoryUpdateResult } from '@shared/contracts/routes/memory.routes' import { createRouteMap, type DeepchatRouteMap } from '@/routes/routeRegistry' -import { buildEffectiveTapeView } from '@/session/data/tapeEffectiveView' +import type { TapeInspectionReader } from '@/tape/ports/capabilities' import type { MemoryConflictPair, MemoryConflictResolution, @@ -57,7 +57,9 @@ const MEMORY_PERSONA_STATES = ['draft', 'active', 'superseded', 'rejected'] as c type MemoryPersonaState = (typeof MEMORY_PERSONA_STATES)[number] const MEMORY_PERSONA_STATE_SET: ReadonlySet = new Set(MEMORY_PERSONA_STATES) -export function formatMemorySourceRecordContent(record: ChatMessageRecord): string { +export function formatMemorySourceRecordContent( + record: Pick +): string { try { const parsed = JSON.parse(record.content) as unknown if (record.role === 'user') { @@ -180,81 +182,6 @@ function toMemoryAuditEventDto(row: AgentMemoryAuditRow) { } } -function deriveSelectedMemoryIds(value: unknown): string[] | null { - if (!Array.isArray(value)) return null - const ids = new Set() - for (const item of value) { - const id = - typeof item === 'string' - ? item - : item && typeof item === 'object' && !Array.isArray(item) - ? (item as Record).id - : null - if (typeof id === 'string' && id.length > 0) ids.add(id) - } - return [...ids] -} - -interface MemoryTapeEntryRow { - session_id: string - entry_id: number - kind: 'event' | 'anchor' | 'message' | 'tool_call' | 'tool_result' - name: string | null - source_type: - | 'session' - | 'message' - | 'assistant_block' - | 'tool_call' - | 'tool_result' - | 'runtime_event' - | 'migration' - | 'summary' - | 'fork' - | 'subagent' - | null - source_id: string | null - source_seq: number | null - provenance_key: string | null - payload_json: string - meta_json: string - created_at: number -} - -function toMemoryViewManifestDto(row: MemoryTapeEntryRow) { - const payload = parseJsonRecord(row.payload_json) - const meta = parseJsonRecord(row.meta_json) - const manifest = - payload.state && typeof payload.state === 'object' && !Array.isArray(payload.state) - ? (payload.state as Record) - : null - if (!manifest) return null - const readNumber = (value: unknown): number => - typeof value === 'number' && Number.isFinite(value) ? value : 0 - return { - sessionId: row.session_id, - messageId: typeof meta.messageId === 'string' ? meta.messageId : null, - entryId: row.entry_id, - policyVersion: - typeof manifest.policyVersion === 'number' && Number.isFinite(manifest.policyVersion) - ? manifest.policyVersion - : null, - tokenBudget: readNumber(manifest.tokenBudget), - estimatedTokens: readNumber(manifest.estimatedTokens), - selectedCount: Array.isArray(manifest.selected) ? manifest.selected.length : 0, - selectedIds: deriveSelectedMemoryIds(manifest.selected), - droppedCount: Array.isArray(manifest.dropped) ? manifest.dropped.length : 0, - queryHash: typeof manifest.queryHash === 'string' ? manifest.queryHash : null, - createdAt: row.created_at - } -} - -type MemoryTapeEntries = { - getBySession(sessionId: string): MemoryTapeEntryRow[] - listMemoryViewManifestAnchorsByAgent( - agentId: string, - options: { sessionId?: string; limit?: number; messageId?: string } - ): MemoryTapeEntryRow[] -} type MemoryAuditEntries = { listByAgent(agentId: string, options: MemoryAuditListOptions): AgentMemoryAuditRow[] } @@ -316,7 +243,7 @@ interface MemoryRouteService { export function createMemoryRoutes(deps: { memoryService: MemoryRouteService getAgentType(agentId: string): Promise - getTapeEntries(): MemoryTapeEntries + getTapeInspection(): TapeInspectionReader getAuditEntries(): MemoryAuditEntries }): DeepchatRouteMap { const { memoryService } = deps @@ -325,9 +252,9 @@ export function createMemoryRoutes(deps: { if (!row || row.agent_id !== agentId || !row.source_session) return null const sourceEntryIds = parseAgentMemorySourceEntryIds(row.source_entry_ids) if (!sourceEntryIds?.length) return null - const sourceSet = new Set(sourceEntryIds) - const entries = buildEffectiveTapeView(deps.getTapeEntries().getBySession(row.source_session)) - .messageEntries.filter((entry) => sourceSet.has(entry.entryId)) + const entries = deps + .getTapeInspection() + .getEffectiveMessageSourceSpan(row.source_session, sourceEntryIds) .map((entry) => ({ entryId: entry.entryId, role: entry.record.role, @@ -519,15 +446,12 @@ export function createMemoryRoutes(deps: { } const limit = input.limit ?? 100 const manifests = deps - .getTapeEntries() - .listMemoryViewManifestAnchorsByAgent(input.agentId, { + .getTapeInspection() + .listMemoryViewManifestsByAgent(input.agentId, { sessionId: input.sessionId, limit, messageId: input.messageId }) - .map(toMemoryViewManifestDto) - .filter((manifest): manifest is NonNullable => Boolean(manifest)) - .filter((manifest) => !input.messageId || manifest.messageId === input.messageId) .slice(0, limit) return memoryListViewManifestsRoute.output.parse({ manifests }) } diff --git a/src/main/session/data/database.ts b/src/main/session/data/database.ts index 9e6d48024b..a28c7c4c59 100644 --- a/src/main/session/data/database.ts +++ b/src/main/session/data/database.ts @@ -15,11 +15,10 @@ import { DeepChatMessageSearchResultsTable } from './tables/deepchatMessageSearc import { DeepChatSearchDocumentsTable } from './tables/deepchatSearchDocuments' import { DeepChatPendingInputsTable } from './tables/deepchatPendingInputs' import { DeepChatUsageStatsTable } from './tables/deepchatUsageStats' -import { - DeepChatTapeEntriesTable, - type DeepChatTapeMutationProjection -} from './tables/deepchatTapeEntries' -import { DeepChatTapeSearchProjectionTable } from './tables/deepchatTapeSearchProjection' +import { DeepChatTapeEntriesTable } from '@/tape/infrastructure/sqlite/tapeEntryStore' +import type { TapeMutationProjection } from '@/tape/ports/storage' +import { SqliteTapeLifecycleAdapter } from '@/tape/infrastructure/sqlite/tapeLifecycleAdapter' +import { DeepChatTapeSearchProjectionTable } from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' import { DeepChatSessionMetadataTable } from './tables/deepchatSessionMetadata' import { NewSessionActiveSkillsTable } from './tables/newSessionActiveSkills' import { NewSessionDisabledAgentToolsTable } from './tables/newSessionDisabledAgentTools' @@ -27,7 +26,7 @@ import { NewSessionDisabledAgentToolsTable } from './tables/newSessionDisabledAg export class SessionDatabase { constructor( private readonly connection: DatabaseConnectionProvider, - private readonly getTapeMutationProjection?: () => DeepChatTapeMutationProjection + private readonly getTapeMutationProjection?: () => TapeMutationProjection ) {} getDatabase() { @@ -98,6 +97,10 @@ export class SessionDatabase { return new DeepChatTapeEntriesTable(this.getDatabase(), this.getTapeMutationProjection?.()) } + get tapeLifecycle() { + return new SqliteTapeLifecycleAdapter(this.getDatabase(), this.getTapeMutationProjection?.()) + } + get deepchatTapeSearchProjectionTable() { return new DeepChatTapeSearchProjectionTable(this.getDatabase()) } diff --git a/src/main/session/data/index.ts b/src/main/session/data/index.ts index bf528a77a3..a64f168ae5 100644 --- a/src/main/session/data/index.ts +++ b/src/main/session/data/index.ts @@ -1,17 +1,17 @@ import type { DatabaseConnectionProvider } from '@/data/databaseConnection' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeMutationProjection } from '@/tape/ports/storage' import type { SessionTapePort } from './contracts' import { SessionPendingInputStore } from './pendingInputStore' import { SessionPendingInputs } from './pendingInputs' import { SessionSettingsStore } from './settings' -import { normalizeTapeHandoffState, SessionTape } from './tape' +import { normalizeTapeHandoffState, SessionTape } from '@/tape/application/sessionTape' import { SessionTranscript } from './transcript' import { SessionDatabase } from './database' -import type { DeepChatTapeMutationProjection } from './tables/deepchatTapeEntries' export function createSessionData( connection: DatabaseConnectionProvider, - getTapeMutationProjection: (() => DeepChatTapeMutationProjection) | undefined, + getTapeMutationProjection: (() => TapeMutationProjection) | undefined, events: SessionDataEvents ) { const database = new SessionDatabase(connection, getTapeMutationProjection) @@ -26,8 +26,8 @@ export function createSessionDataFromDatabase( database: SessionDatabase, events: SessionDataEvents ) { - const transcript = new SessionTranscript(database) const tapeStore = new SessionTape(database) + const transcript = new SessionTranscript(database, tapeStore) const pendingInputStore = new SessionPendingInputStore(database) const ensureTape = (sessionId: string) => tapeStore.ensureSessionTapeReady(sessionId, transcript) const toTapeAnchor = (row: DeepChatTapeEntryRow) => ({ @@ -80,7 +80,7 @@ export function createSessionDataFromDatabase( return { database, - settings: new SessionSettingsStore(database), + settings: new SessionSettingsStore(database, tapeStore), transcript, tape, tapeStore, diff --git a/src/main/session/data/settings.ts b/src/main/session/data/settings.ts index 344edcd8b3..c6831d1821 100644 --- a/src/main/session/data/settings.ts +++ b/src/main/session/data/settings.ts @@ -1,7 +1,12 @@ import { SessionDatabase } from './database' import type { PermissionMode, SessionGenerationSettings } from '@shared/types/agent-interface' import type { DeepChatSessionSummaryRow } from '@/session/data/tables/deepchatSessions' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { + TapeAnchorReader, + TapeAnchorWriter, + TapeLifecycleAdmin +} from '@/tape/ports/capabilities' export type SessionSummaryState = { summaryText: string | null @@ -131,9 +136,14 @@ function summaryStatesEqual(left: SessionSummaryState, right: SessionSummaryStat export class SessionSettingsStore { private database: SessionDatabase + private readonly tape: TapeAnchorReader & TapeAnchorWriter & TapeLifecycleAdmin - constructor(database: SessionDatabase) { + constructor( + database: SessionDatabase, + tape: TapeAnchorReader & TapeAnchorWriter & TapeLifecycleAdmin + ) { this.database = database + this.tape = tape } create( @@ -150,7 +160,7 @@ export class SessionSettingsStore { permissionMode, generationSettings ) - this.database.deepchatTapeEntriesTable.ensureBootstrapAnchor(id) + this.tape.initializeSessionTape(id) } get(id: string) { @@ -158,8 +168,7 @@ export class SessionSettingsStore { } delete(id: string): void { - this.database.deepchatTapeEntriesTable.deleteBySession(id) - this.database.deepchatTapeSearchProjectionTable.deleteBySession(id) + this.tape.deleteSessionTape(id) this.database.deepchatSessionsTable.delete(id) } @@ -198,8 +207,7 @@ export class SessionSettingsStore { } getSummaryState(id: string): SessionSummaryState { - const tapeTable = this.database.deepchatTapeEntriesTable - const tapeState = summaryStateFromTapeAnchor(tapeTable.getLatestReconstructionAnchor(id)) + const tapeState = summaryStateFromTapeAnchor(this.tape.getLatestReconstructionAnchor(id)) if (tapeState) { return tapeState } @@ -208,9 +216,7 @@ export class SessionSettingsStore { } getReconstructionAnchorPromptState(id: string): ReconstructionAnchorPromptState | null { - return reconstructionAnchorPromptStateFromRow( - this.database.deepchatTapeEntriesTable.getLatestReconstructionAnchor(id) - ) + return reconstructionAnchorPromptStateFromRow(this.tape.getLatestReconstructionAnchor(id)) } updateSummaryState(id: string, state: SessionSummaryState): void { @@ -224,8 +230,7 @@ export class SessionSettingsStore { tapeAnchor?: SummaryTapeAnchorInput ): SummaryStateCompareAndSetResult { const applyUpdate = (): boolean => { - const tapeTable = this.database.deepchatTapeEntriesTable - const latestTapeAnchor = tapeTable.getLatestReconstructionAnchor(id) + const latestTapeAnchor = this.tape.getLatestReconstructionAnchor(id) const currentState = this.getSummaryState(id) if (!summaryStatesEqual(currentState, expectedState)) { return false @@ -236,7 +241,7 @@ export class SessionSettingsStore { this.database.deepchatSessionsTable.updateSummaryState(id, nextState) if (tapeAnchor) { - tapeTable.appendAnchor({ + this.tape.appendAnchor({ sessionId: id, name: tapeAnchor.name, state: tapeAnchor.state, @@ -265,7 +270,7 @@ export class SessionSettingsStore { resetSummaryState(id: string): void { const reset = (): void => { this.database.deepchatSessionsTable.resetSummaryState(id) - this.database.deepchatTapeEntriesTable.appendAnchor({ + this.tape.appendAnchor({ sessionId: id, name: 'summary/reset', state: { @@ -278,8 +283,6 @@ export class SessionSettingsStore { } resetTape(id: string): void { - this.database.deepchatTapeEntriesTable.deleteBySession(id) - this.database.deepchatTapeSearchProjectionTable.deleteBySession(id) - this.database.deepchatTapeEntriesTable.ensureBootstrapAnchor(id) + this.tape.resetSessionTape(id) } } diff --git a/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts b/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts index 932234b62f..3223a637b4 100644 --- a/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts +++ b/src/main/session/data/tables/deepchatTapeEffectiveSemantics.ts @@ -1,161 +1,16 @@ -import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' -import type { DeepChatTapeEntryRow } from './deepchatTapeEntries' - -const TERMINAL_TAPE_TOOL_STATUSES = new Set(['success', 'error']) - -export interface DeepChatTapeToolIdentity { - key: string - messageId: string -} - -export function parseTapeJsonObject(raw: string): Record { - try { - const parsed = JSON.parse(raw) as unknown - if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { - return parsed as Record - } - } catch {} - return {} -} - -export function parseNestedTapeJsonObject(value: unknown): Record { - if (typeof value === 'string') { - return parseTapeJsonObject(value) - } - if (value && typeof value === 'object' && !Array.isArray(value)) { - return value as Record - } - return {} -} - -export function parseAssistantBlocks(rawContent: string): AssistantMessageBlock[] { - try { - const parsed = JSON.parse(rawContent) as unknown - return Array.isArray(parsed) ? (parsed as AssistantMessageBlock[]) : [] - } catch { - return [] - } -} - -export function messageRecordHasFinalToolUse(record: ChatMessageRecord): boolean { - if (record.role !== 'assistant' || (record.status !== 'sent' && record.status !== 'error')) { - return false - } - const blocks = parseAssistantBlocks(record.content) - const pendingInteractionToolIds = new Set( - blocks.flatMap((block) => - block.type === 'action' && - (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && - block.status === 'pending' && - typeof block.tool_call?.id === 'string' - ? [block.tool_call.id] - : [] - ) - ) - return blocks.some( - (block) => - block.type === 'tool_call' && - (block.status === 'success' || block.status === 'error') && - typeof block.tool_call?.id === 'string' && - !pendingInteractionToolIds.has(block.tool_call.id) - ) -} - -function isMessageStatus(value: unknown): value is ChatMessageRecord['status'] { - return value === 'pending' || value === 'sent' || value === 'error' -} - -export function tapeEntryToMessageRecord(row: DeepChatTapeEntryRow): ChatMessageRecord | null { - if (row.kind !== 'message') { - return null - } - - const payload = parseTapeJsonObject(row.payload_json) - const record = payload.record - if (!record || typeof record !== 'object' || Array.isArray(record)) { - return null - } - - const candidate = record as Partial - if ( - typeof candidate.id !== 'string' || - typeof candidate.sessionId !== 'string' || - typeof candidate.orderSeq !== 'number' || - (candidate.role !== 'user' && candidate.role !== 'assistant') || - typeof candidate.content !== 'string' - ) { - return null - } - - return { - id: candidate.id, - sessionId: candidate.sessionId, - orderSeq: candidate.orderSeq, - role: candidate.role, - content: candidate.content, - status: isMessageStatus(candidate.status) ? candidate.status : 'sent', - isContextEdge: typeof candidate.isContextEdge === 'number' ? candidate.isContextEdge : 0, - metadata: typeof candidate.metadata === 'string' ? candidate.metadata : '{}', - traceCount: typeof candidate.traceCount === 'number' ? candidate.traceCount : 0, - createdAt: typeof candidate.createdAt === 'number' ? candidate.createdAt : row.created_at, - updatedAt: typeof candidate.updatedAt === 'number' ? candidate.updatedAt : row.created_at - } -} - -export function tapeMessageRank(record: ChatMessageRecord, includePending: boolean): number { - if (record.status === 'sent' || record.status === 'error') { - return 2 - } - return includePending && record.status === 'pending' ? 1 : 0 -} - -export function readTapeMessageRetractionId(row: DeepChatTapeEntryRow): string | null { - if (row.kind !== 'event' || row.name !== 'message/retracted') { - return null - } - - const payload = parseTapeJsonObject(row.payload_json) - const data = parseNestedTapeJsonObject(payload.data) - return typeof data.messageId === 'string' ? data.messageId : null -} - -export function readTapeToolStatus(row: DeepChatTapeEntryRow): string | null { - const meta = parseTapeJsonObject(row.meta_json) - return typeof meta.status === 'string' ? meta.status : null -} - -export function tapeToolRank(row: DeepChatTapeEntryRow, includePending: boolean): number { - const status = readTapeToolStatus(row) - if (status === 'pending') { - return includePending ? 1 : 0 - } - return status !== null && TERMINAL_TAPE_TOOL_STATUSES.has(status) ? 2 : 0 -} - -export function readTapeToolIdentity(row: DeepChatTapeEntryRow): DeepChatTapeToolIdentity | null { - if (row.kind !== 'tool_call' && row.kind !== 'tool_result') { - return null - } - - const payload = parseTapeJsonObject(row.payload_json) - const messageId = payload.messageId - if (typeof messageId !== 'string' || messageId.length === 0) { - return null - } - - let toolCallId: unknown - if (row.kind === 'tool_call') { - toolCallId = parseNestedTapeJsonObject(payload.toolCall).id - } else { - toolCallId = payload.toolCallId - } - - if (typeof toolCallId !== 'string' || toolCallId.length === 0) { - return null - } - - return { - key: `${row.kind}:${messageId}:${toolCallId}`, - messageId - } -} +/** @deprecated Import Tape semantics from `@/tape/domain/effectiveSemantics`. */ +export { + messageRecordHasFinalToolUse, + parseAssistantBlocks, + parseNestedTapeJsonObject, + parseTapeJsonObject, + readTapeMessageRetractionId, + readTapeToolIdentity, + readTapeToolStatus, + tapeEntryToMessageRecord, + tapeMessageRank, + tapeToolRank +} from '@/tape/domain/effectiveSemantics' + +/** @deprecated Import Tape semantic types from `@/tape/domain/effectiveSemantics`. */ +export type { DeepChatTapeToolIdentity } from '@/tape/domain/effectiveSemantics' diff --git a/src/main/session/data/tables/deepchatTapeEntries.ts b/src/main/session/data/tables/deepchatTapeEntries.ts index 2987280341..c34220272c 100644 --- a/src/main/session/data/tables/deepchatTapeEntries.ts +++ b/src/main/session/data/tables/deepchatTapeEntries.ts @@ -1,1158 +1,22 @@ -import Database from 'better-sqlite3-multiple-ciphers' -import logger from '@shared/logger' -import { BaseTable } from '@/data/baseTable' -import { randomUUID } from 'crypto' - -export type DeepChatTapeEntryKind = 'event' | 'anchor' | 'message' | 'tool_call' | 'tool_result' - -export const TAPE_INCARNATION_META_KEY = 'tapeIncarnationId' - -export type DeepChatTapeSourceType = - | 'session' - | 'message' - | 'assistant_block' - | 'tool_call' - | 'tool_result' - | 'runtime_event' - | 'migration' - | 'summary' - | 'fork' - | 'subagent' - -export interface DeepChatTapeEntryRow { - session_id: string - entry_id: number - kind: DeepChatTapeEntryKind - name: string | null - source_type: DeepChatTapeSourceType | null - source_id: string | null - source_seq: number | null - provenance_key: string | null - payload_json: string - meta_json: string - created_at: number -} - -export interface DeepChatTapeSourceInput { - type: DeepChatTapeSourceType - id: string - seq?: number | null -} - -export interface DeepChatTapeAppendInput { - sessionId: string - kind: DeepChatTapeEntryKind - name?: string | null - source?: DeepChatTapeSourceInput | null - provenanceKey?: string | null - payload: Record - meta?: Record - createdAt?: number - idempotent?: boolean -} - -export interface DeepChatTapeSearchInput { - limit?: number - kinds?: DeepChatTapeEntryKind[] - startCreatedAt?: number - endCreatedAt?: number -} - -export interface DeepChatTapeReadSource { - sessionId: string - maxEntryId: number -} - -export interface DeepChatTapeMutationProjection { - applyAppendedEntry(row: DeepChatTapeEntryRow, previousSessionMaxEntryId: number): boolean - invalidateSession(sessionId: string): void - deleteBySession(sessionId: string): void -} - -export const SUMMARY_ANCHOR_NAMES = [ - 'compaction/auto', - 'compaction/manual', - 'compaction/context_pressure', - 'compaction/resume', - 'compaction/migrated_summary', - 'auto_handoff/context_overflow', - 'summary/reset' -] as const - -const RECONSTRUCTION_ANCHOR_NAMES = SUMMARY_ANCHOR_NAMES - -const TAPE_ENTRY_INDEX_SQL = ` - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_kind - ON deepchat_tape_entries(session_id, kind, entry_id); - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_name - ON deepchat_tape_entries(session_id, name, entry_id); - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_source - ON deepchat_tape_entries(session_id, source_type, source_id, source_seq); - CREATE UNIQUE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_provenance - ON deepchat_tape_entries(session_id, provenance_key) - WHERE provenance_key IS NOT NULL; -` - -function safeJsonStringify(value: Record | undefined): string { - return JSON.stringify(value ?? {}) -} - -function buildProvenanceKey(input: DeepChatTapeAppendInput): string | null { - if (input.provenanceKey !== undefined) { - return input.provenanceKey - } - if (!input.source?.type || !input.source.id) { - return null - } - return [ - input.source.type, - input.source.id, - input.source.seq ?? 0, - input.kind, - input.name ?? '' - ].join(':') -} - -function escapeLikePattern(value: string): string { - return value.replace(/[\\%_]/g, (character) => `\\${character}`) -} - -// Three LIKE parameters per field group stay below SQLite's portable 999-variable floor after -// source, filter, and limit bindings. -const MAX_TAPE_SEARCH_TOKEN_CLAUSES = 256 - -function tokenizeDeepChatTapeSearchQuery(value: string): string[] { - return value - .split(/\s+/) - .map((token) => token.trim()) - .filter(Boolean) -} - -export function buildDeepChatTapeFtsMatch(value: string): string { - const tokens = tokenizeDeepChatTapeSearchQuery(value) - const values = - tokens.length > 1 && tokens.length <= MAX_TAPE_SEARCH_TOKEN_CLAUSES ? tokens : [value] - return values.map((token) => `"${token.replace(/"/g, '""')}"`).join(' AND ') -} - -export function buildDeepChatTapeLikeSearchPredicate( - fieldExpressions: readonly [string, ...string[]], - normalizedQuery: string -): { sql: string; params: string[] } { - const fieldClause = `(${fieldExpressions - .map((field) => `${field} LIKE ? ESCAPE '\\'`) - .join(' OR ')})` - const queryClauses = [fieldClause] - const queryPattern = `%${escapeLikePattern(normalizedQuery)}%` - const params = fieldExpressions.map(() => queryPattern) - const tokens = tokenizeDeepChatTapeSearchQuery(normalizedQuery) - - if (tokens.length > 1 && tokens.length <= MAX_TAPE_SEARCH_TOKEN_CLAUSES) { - queryClauses.push(`(${tokens.map(() => fieldClause).join(' AND ')})`) - for (const token of tokens) { - const tokenPattern = `%${escapeLikePattern(token)}%` - params.push(...fieldExpressions.map(() => tokenPattern)) - } - } - - return { - sql: `(${queryClauses.join(' OR ')})`, - params - } -} - -export function normalizeDeepChatTapeReadSources( - sources: readonly DeepChatTapeReadSource[] -): DeepChatTapeReadSource[] { - const maxEntryIdBySession = new Map() - for (const source of sources) { - const sessionId = source.sessionId.trim() - if (!sessionId || !Number.isSafeInteger(source.maxEntryId) || source.maxEntryId < 0) { - continue - } - maxEntryIdBySession.set( - sessionId, - Math.max(maxEntryIdBySession.get(sessionId) ?? 0, source.maxEntryId) - ) - } - return [...maxEntryIdBySession.entries()] - .map(([sessionId, maxEntryId]) => ({ sessionId, maxEntryId })) - .sort((left, right) => - left.sessionId < right.sessionId ? -1 : left.sessionId > right.sessionId ? 1 : 0 - ) -} - -export function serializeDeepChatTapeReadSources( - sources: readonly DeepChatTapeReadSource[] -): string { - return JSON.stringify(normalizeDeepChatTapeReadSources(sources)) -} - -const AUTHORIZED_TAPE_SOURCES_CTE_SQL = ` - authorized_sources(session_id, max_entry_id) AS ( - SELECT - json_extract(value, '$.sessionId'), - CAST(json_extract(value, '$.maxEntryId') AS INTEGER) - FROM json_each(?) - ) -` - -function effectiveTapeMessagePredicateSql(alias: string): string { - return ` - json_type(${alias}.payload_json, '$.record') = 'object' - AND typeof(json_extract(${alias}.payload_json, '$.record.id')) = 'text' - AND typeof(json_extract(${alias}.payload_json, '$.record.sessionId')) = 'text' - AND typeof(json_extract(${alias}.payload_json, '$.record.orderSeq')) IN ('integer', 'real') - AND json_extract(${alias}.payload_json, '$.record.role') IN ('user', 'assistant') - AND typeof(json_extract(${alias}.payload_json, '$.record.content')) = 'text' - AND ( - json_extract(${alias}.payload_json, '$.record.status') IS NULL - OR json_extract(${alias}.payload_json, '$.record.status') != 'pending' - ) - ` -} - -function tapeRetractionMessageIdSql(alias: string): string { - return ` - CASE - WHEN json_type(${alias}.payload_json, '$.data') = 'object' - THEN json_extract(${alias}.payload_json, '$.data.messageId') - WHEN json_type(${alias}.payload_json, '$.data') = 'text' - AND json_valid(json_extract(${alias}.payload_json, '$.data')) - THEN json_extract(json_extract(${alias}.payload_json, '$.data'), '$.messageId') - ELSE NULL - END - ` -} - -function tapeToolCallIdSql(alias: string): string { - return ` - CASE - WHEN ${alias}.kind = 'tool_result' - THEN json_extract(${alias}.payload_json, '$.toolCallId') - WHEN json_type(${alias}.payload_json, '$.toolCall') = 'object' - THEN json_extract(${alias}.payload_json, '$.toolCall.id') - WHEN json_type(${alias}.payload_json, '$.toolCall') = 'text' - AND json_valid(json_extract(${alias}.payload_json, '$.toolCall')) - THEN json_extract(json_extract(${alias}.payload_json, '$.toolCall'), '$.id') - ELSE NULL - END - ` -} - -// These read-only SQL forms mirror deepchatTapeEffectiveSemantics. Search uses correlated -// candidate validation to avoid materializing a whole linked Tape; context uses the complete -// effective CTE because it needs stable neighboring-row positions. Native tests cover parity for -// replacement, retraction, head cutoff, and window ordering. -const EFFECTIVE_TAPE_ROWS_CTE_SQL = ` - bounded_rows AS ( - SELECT tape.* - FROM deepchat_tape_entries AS tape - INNER JOIN authorized_sources AS source - ON source.session_id = tape.session_id - AND tape.entry_id <= source.max_entry_id - ), - raw_message_candidates AS ( - SELECT - bounded_rows.*, - json_extract(payload_json, '$.record.id') AS message_id - FROM bounded_rows - WHERE kind = 'message' - ), - message_candidates AS ( - SELECT * - FROM raw_message_candidates - WHERE ${effectiveTapeMessagePredicateSql('raw_message_candidates')} - ), - ranked_messages AS ( - SELECT - message_candidates.*, - ROW_NUMBER() OVER ( - PARTITION BY session_id, message_id - ORDER BY entry_id DESC - ) AS candidate_rank - FROM message_candidates - ), - effective_message_rows AS ( - SELECT ranked_messages.* - FROM ranked_messages - WHERE candidate_rank = 1 - AND NOT EXISTS ( - SELECT 1 - FROM bounded_rows AS retraction - WHERE retraction.session_id = ranked_messages.session_id - AND retraction.kind = 'event' - AND retraction.name = 'message/retracted' - AND retraction.entry_id > ranked_messages.entry_id - AND (${tapeRetractionMessageIdSql('retraction')}) = ranked_messages.message_id - ) - ), - raw_tool_candidates AS ( - SELECT - bounded_rows.*, - json_extract(payload_json, '$.messageId') AS message_id, - (${tapeToolCallIdSql('bounded_rows')}) AS tool_call_id, - json_extract(meta_json, '$.status') AS tool_status - FROM bounded_rows - WHERE kind IN ('tool_call', 'tool_result') - ), - tool_candidates AS ( - SELECT * - FROM raw_tool_candidates - WHERE typeof(message_id) = 'text' - AND length(message_id) > 0 - AND typeof(tool_call_id) = 'text' - AND length(tool_call_id) > 0 - AND tool_status IN ('success', 'error') - ), - ranked_tools AS ( - SELECT - tool_candidates.*, - ROW_NUMBER() OVER ( - PARTITION BY session_id, kind, message_id, tool_call_id - ORDER BY entry_id DESC - ) AS candidate_rank - FROM tool_candidates - ), - effective_rows AS ( - SELECT - session_id, entry_id, kind, name, source_type, source_id, source_seq, - provenance_key, payload_json, meta_json, created_at - FROM bounded_rows - WHERE kind = 'anchor' - UNION ALL - SELECT - session_id, entry_id, kind, name, source_type, source_id, source_seq, - provenance_key, payload_json, meta_json, created_at - FROM bounded_rows - WHERE kind = 'event' - AND (name IS NULL OR name NOT IN ( - 'message/retracted', - 'message/compaction_indicator', - 'migration/backfill' - )) - UNION ALL - SELECT - session_id, entry_id, kind, name, source_type, source_id, source_seq, - provenance_key, payload_json, meta_json, created_at - FROM effective_message_rows - UNION ALL - SELECT - ranked_tools.session_id, - ranked_tools.entry_id, - ranked_tools.kind, - ranked_tools.name, - ranked_tools.source_type, - ranked_tools.source_id, - ranked_tools.source_seq, - ranked_tools.provenance_key, - ranked_tools.payload_json, - ranked_tools.meta_json, - ranked_tools.created_at - FROM ranked_tools - INNER JOIN effective_message_rows AS message - ON message.session_id = ranked_tools.session_id - AND message.message_id = ranked_tools.message_id - WHERE ranked_tools.candidate_rank = 1 - ) -` - -const EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL = ` - candidate.kind = 'anchor' - OR ( - candidate.kind = 'event' - AND (candidate.name IS NULL OR candidate.name NOT IN ( - 'message/retracted', - 'message/compaction_indicator', - 'migration/backfill' - )) - ) - OR ( - candidate.kind = 'message' - AND ${effectiveTapeMessagePredicateSql('candidate')} - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS later_message - WHERE later_message.session_id = candidate.session_id - AND later_message.entry_id > candidate.entry_id - AND later_message.entry_id <= source.max_entry_id - AND later_message.kind = 'message' - AND ${effectiveTapeMessagePredicateSql('later_message')} - AND json_extract(later_message.payload_json, '$.record.id') = - json_extract(candidate.payload_json, '$.record.id') - ) - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS retraction - WHERE retraction.session_id = candidate.session_id - AND retraction.entry_id > candidate.entry_id - AND retraction.entry_id <= source.max_entry_id - AND retraction.kind = 'event' - AND retraction.name = 'message/retracted' - AND (${tapeRetractionMessageIdSql('retraction')}) = - json_extract(candidate.payload_json, '$.record.id') - ) - ) - OR ( - candidate.kind IN ('tool_call', 'tool_result') - AND json_extract(candidate.meta_json, '$.status') IN ('success', 'error') - AND typeof(json_extract(candidate.payload_json, '$.messageId')) = 'text' - AND length(json_extract(candidate.payload_json, '$.messageId')) > 0 - AND typeof((${tapeToolCallIdSql('candidate')})) = 'text' - AND length((${tapeToolCallIdSql('candidate')})) > 0 - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS later_tool - WHERE later_tool.session_id = candidate.session_id - AND later_tool.entry_id > candidate.entry_id - AND later_tool.entry_id <= source.max_entry_id - AND later_tool.kind = candidate.kind - AND json_extract(later_tool.meta_json, '$.status') IN ('success', 'error') - AND json_extract(later_tool.payload_json, '$.messageId') = - json_extract(candidate.payload_json, '$.messageId') - AND (${tapeToolCallIdSql('later_tool')}) = (${tapeToolCallIdSql('candidate')}) - ) - AND EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS message - WHERE message.session_id = candidate.session_id - AND message.entry_id <= source.max_entry_id - AND message.kind = 'message' - AND ${effectiveTapeMessagePredicateSql('message')} - AND json_extract(message.payload_json, '$.record.id') = - json_extract(candidate.payload_json, '$.messageId') - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS later_message - WHERE later_message.session_id = message.session_id - AND later_message.entry_id > message.entry_id - AND later_message.entry_id <= source.max_entry_id - AND later_message.kind = 'message' - AND ${effectiveTapeMessagePredicateSql('later_message')} - AND json_extract(later_message.payload_json, '$.record.id') = - json_extract(message.payload_json, '$.record.id') - ) - AND NOT EXISTS ( - SELECT 1 - FROM deepchat_tape_entries AS retraction - WHERE retraction.session_id = message.session_id - AND retraction.entry_id > message.entry_id - AND retraction.entry_id <= source.max_entry_id - AND retraction.kind = 'event' - AND retraction.name = 'message/retracted' - AND (${tapeRetractionMessageIdSql('retraction')}) = - json_extract(message.payload_json, '$.record.id') - ) - ) - ) -` - -export class DeepChatTapeEntriesTable extends BaseTable { - constructor( - db: Database.Database, - private readonly mutationProjection?: DeepChatTapeMutationProjection - ) { - super(db, 'deepchat_tape_entries') - } - - getCreateTableSQL(): string { - return ` - CREATE TABLE IF NOT EXISTS deepchat_tape_entries ( - session_id TEXT NOT NULL, - entry_id INTEGER NOT NULL, - kind TEXT NOT NULL, - name TEXT, - source_type TEXT, - source_id TEXT, - source_seq INTEGER, - provenance_key TEXT, - payload_json TEXT NOT NULL DEFAULT '{}', - meta_json TEXT NOT NULL DEFAULT '{}', - created_at INTEGER NOT NULL, - PRIMARY KEY (session_id, entry_id) - ); - ${TAPE_ENTRY_INDEX_SQL} - ` - } - - public createTable(): void { - if (!this.tableExists()) { - this.db.exec(this.getCreateTableSQL()) - return - } - this.ensureProvenanceColumns() - this.db.exec(TAPE_ENTRY_INDEX_SQL) - } - - getMigrationSQL(_version: number): string | null { - return null - } - - getLatestVersion(): number { - return 0 - } - - runInTransaction(operation: () => T): T { - return this.db.transaction(operation)() - } - - append(input: DeepChatTapeAppendInput): DeepChatTapeEntryRow { - const append = this.db.transaction(() => { - const provenanceKey = buildProvenanceKey(input) - if (input.idempotent && provenanceKey) { - const existing = this.getByProvenanceKey(input.sessionId, provenanceKey) - if (existing) { - return existing - } - } - - const createdAt = input.createdAt ?? Date.now() - const previousSessionMaxEntryId = this.getMaxEntryId(input.sessionId) - const row = { - session_id: input.sessionId, - entry_id: previousSessionMaxEntryId + 1, - kind: input.kind, - name: input.name ?? null, - source_type: input.source?.type ?? null, - source_id: input.source?.id ?? null, - source_seq: input.source?.seq ?? null, - provenance_key: provenanceKey, - payload_json: safeJsonStringify(input.payload), - meta_json: safeJsonStringify(input.meta), - created_at: createdAt - } satisfies DeepChatTapeEntryRow - - try { - this.db - .prepare( - `INSERT INTO deepchat_tape_entries ( - session_id, - entry_id, - kind, - name, - source_type, - source_id, - source_seq, - provenance_key, - payload_json, - meta_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - .run( - row.session_id, - row.entry_id, - row.kind, - row.name, - row.source_type, - row.source_id, - row.source_seq, - row.provenance_key, - row.payload_json, - row.meta_json, - row.created_at - ) - } catch (error) { - if (input.idempotent && provenanceKey) { - const existing = this.getByProvenanceKey(input.sessionId, provenanceKey) - if (existing) { - return existing - } - } - throw error - } - - if (this.mutationProjection) { - try { - const applyProjection = this.db.transaction(() => - this.mutationProjection?.applyAppendedEntry(row, previousSessionMaxEntryId) - ) - applyProjection() - } catch (error) { - this.mutationProjection.invalidateSession(row.session_id) - logger.warn( - `[Tape] memory ingestion projection append failed; session marked stale: ${String(error)}` - ) - } - } - - return row - }) - - return append() - } - - appendAnchor(input: { - sessionId: string - name: string - state: Record - meta?: Record - source?: DeepChatTapeSourceInput | null - provenanceKey?: string | null - createdAt?: number - idempotent?: boolean - }): DeepChatTapeEntryRow { - return this.append({ - sessionId: input.sessionId, - kind: 'anchor', - name: input.name, - source: input.source, - provenanceKey: input.provenanceKey, - payload: { - name: input.name, - state: input.state - }, - meta: input.meta, - createdAt: input.createdAt, - idempotent: input.idempotent - }) - } - - appendEvent(input: { - sessionId: string - name: string - data: Record - meta?: Record - source?: DeepChatTapeSourceInput | null - provenanceKey?: string | null - createdAt?: number - idempotent?: boolean - }): DeepChatTapeEntryRow { - return this.append({ - sessionId: input.sessionId, - kind: 'event', - name: input.name, - source: input.source, - provenanceKey: input.provenanceKey, - payload: { - name: input.name, - data: input.data - }, - meta: input.meta, - createdAt: input.createdAt, - idempotent: input.idempotent - }) - } - - ensureBootstrapAnchor(sessionId: string): void { - const existing = this.db - .prepare( - `SELECT entry_id - FROM deepchat_tape_entries - WHERE session_id = ? AND kind = 'anchor' - ORDER BY entry_id ASC - LIMIT 1` - ) - .get(sessionId) as { entry_id: number } | undefined - - if (existing) { - return - } - - this.appendAnchor({ - sessionId, - name: 'session/start', - source: { - type: 'session', - id: sessionId, - seq: 0 - }, - state: { - owner: 'human' - }, - meta: { - [TAPE_INCARNATION_META_KEY]: randomUUID() - }, - idempotent: true - }) - } - - getBySession(sessionId: string): DeepChatTapeEntryRow[] { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - ORDER BY entry_id ASC` - ) - .all(sessionId) as DeepChatTapeEntryRow[] - } - - getSubagentLineageEvents(sessionId: string): DeepChatTapeEntryRow[] { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - AND kind = 'event' - AND name IN ('subagent/tape_linked', 'fork/merge') - ORDER BY entry_id ASC` - ) - .all(sessionId) as DeepChatTapeEntryRow[] - } - - getFirstEntriesBySessions(sessionIds: string[]): DeepChatTapeEntryRow[] { - const ids = [...new Set(sessionIds.map((id) => id.trim()).filter(Boolean))] - if (ids.length === 0) { - return [] - } - return this.db - .prepare( - `WITH requested_sessions(session_id) AS ( - SELECT value FROM json_each(?) - ), - first_entries(session_id, entry_id) AS ( - SELECT tape.session_id, MIN(tape.entry_id) - FROM deepchat_tape_entries AS tape - INNER JOIN requested_sessions AS requested - ON requested.session_id = tape.session_id - GROUP BY tape.session_id - ) - SELECT tape.* - FROM deepchat_tape_entries AS tape - INNER JOIN first_entries AS first - ON first.session_id = tape.session_id - AND first.entry_id = tape.entry_id - ORDER BY tape.session_id ASC` - ) - .all(JSON.stringify(ids)) as DeepChatTapeEntryRow[] - } - - getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND entry_id <= ? - ORDER BY entry_id ASC` - ) - .all(sessionId, maxEntryId) as DeepChatTapeEntryRow[] - } - - listMemoryViewManifestAnchorsBySessions( - sessionIds: string[], - optionsOrLimit: number | { limit?: number; messageId?: string } = 100 - ): DeepChatTapeEntryRow[] { - const uniqueSessionIds = [...new Set(sessionIds.filter((id) => id.trim().length > 0))] - if (uniqueSessionIds.length === 0) { - return [] - } - const options = typeof optionsOrLimit === 'number' ? { limit: optionsOrLimit } : optionsOrLimit - const cappedLimit = Math.min(Math.max(Math.floor(options.limit ?? 100), 1), 500) - const placeholders = uniqueSessionIds.map(() => '?').join(', ') - const whereClauses = [ - `session_id IN (${placeholders})`, - "kind = 'anchor'", - "name = 'memory/view_assembled'" - ] - const params: Array = [...uniqueSessionIds] - if (options.messageId) { - whereClauses.push("json_extract(meta_json, '$.messageId') = ?") - params.push(options.messageId) - } - params.push(cappedLimit) - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE ${whereClauses.join(' AND ')} - ORDER BY created_at DESC, entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeEntryRow[] - } - - listMemoryViewManifestAnchorsByAgent( - agentId: string, - options: { sessionId?: string; limit?: number; messageId?: string } = {} - ): DeepChatTapeEntryRow[] { - const cappedLimit = Math.min(Math.max(Math.floor(options.limit ?? 100), 1), 500) - const whereClauses = [ - 'sessions.agent_id = ?', - "tape.kind = 'anchor'", - "tape.name = 'memory/view_assembled'" - ] - const params: Array = [agentId] - if (options.sessionId) { - whereClauses.push('tape.session_id = ?') - params.push(options.sessionId) - } - if (options.messageId) { - whereClauses.push("json_extract(tape.meta_json, '$.messageId') = ?") - params.push(options.messageId) - } - params.push(cappedLimit) - return this.db - .prepare( - `SELECT tape.* - FROM deepchat_tape_entries AS tape - INNER JOIN new_sessions AS sessions - ON sessions.id = tape.session_id - WHERE ${whereClauses.join(' AND ')} - ORDER BY tape.created_at DESC, tape.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeEntryRow[] - } - - getEntriesAfter(sessionId: string, entryId: number): DeepChatTapeEntryRow[] { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND entry_id > ? - ORDER BY entry_id ASC` - ) - .all(sessionId, entryId) as DeepChatTapeEntryRow[] - } - - getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND kind = 'anchor' - ORDER BY entry_id DESC - LIMIT 1` - ) - .get(sessionId) as DeepChatTapeEntryRow | undefined - } - - getAnchors(sessionId: string, limit: number = 20): DeepChatTapeEntryRow[] { - const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) - const rows = this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND kind = 'anchor' - ORDER BY entry_id DESC - LIMIT ?` - ) - .all(sessionId, cappedLimit) as DeepChatTapeEntryRow[] - - return rows.reverse() - } - - getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { - const placeholders = SUMMARY_ANCHOR_NAMES.map(() => '?').join(', ') - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - AND kind = 'anchor' - AND name IN (${placeholders}) - ORDER BY entry_id DESC - LIMIT 1` - ) - .get(sessionId, ...SUMMARY_ANCHOR_NAMES) as DeepChatTapeEntryRow | undefined - } - - getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { - const placeholders = RECONSTRUCTION_ANCHOR_NAMES.map(() => '?').join(', ') - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - AND kind = 'anchor' - AND ( - name IN (${placeholders}) - OR name LIKE 'handoff/%' - OR name LIKE 'auto_handoff/%' - ) - ORDER BY entry_id DESC - LIMIT 1` - ) - .get(sessionId, ...RECONSTRUCTION_ANCHOR_NAMES) as DeepChatTapeEntryRow | undefined - } - - getByProvenanceKey(sessionId: string, provenanceKey: string): DeepChatTapeEntryRow | undefined { - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? AND provenance_key = ? - LIMIT 1` - ) - .get(sessionId, provenanceKey) as DeepChatTapeEntryRow | undefined - } - - getMaxEntryId(sessionId: string): number { - const row = this.db - .prepare( - `SELECT MAX(entry_id) AS max_entry_id - FROM deepchat_tape_entries - WHERE session_id = ?` - ) - .get(sessionId) as { max_entry_id: number | null } | undefined - return row?.max_entry_id ?? 0 - } - - getMaxEntryIdsBySessions(sessionIds: string[]): Map { - const ids = [...new Set(sessionIds.map((id) => id.trim()).filter(Boolean))] - const maxEntryIdBySession = new Map(ids.map((id) => [id, 0])) - if (ids.length === 0) { - return maxEntryIdBySession - } - const rows = this.db - .prepare( - `WITH requested_sessions(session_id) AS ( - SELECT value FROM json_each(?) - ) - SELECT tape.session_id, MAX(tape.entry_id) AS max_entry_id - FROM deepchat_tape_entries AS tape - INNER JOIN requested_sessions AS requested - ON requested.session_id = tape.session_id - GROUP BY tape.session_id` - ) - .all(JSON.stringify(ids)) as Array<{ session_id: string; max_entry_id: number }> - for (const row of rows) { - maxEntryIdBySession.set(row.session_id, row.max_entry_id) - } - return maxEntryIdBySession - } - - countAnchorsBySession(sessionId: string): number { - const row = this.db - .prepare( - `SELECT COUNT(*) AS count - FROM deepchat_tape_entries - WHERE session_id = ? AND kind = 'anchor'` - ) - .get(sessionId) as { count: number } | undefined - return row?.count ?? 0 - } - - countEntriesAfter(sessionId: string, entryId: number): number { - const row = this.db - .prepare( - `SELECT COUNT(*) AS count - FROM deepchat_tape_entries - WHERE session_id = ? AND entry_id > ?` - ) - .get(sessionId, entryId) as { count: number } | undefined - return row?.count ?? 0 - } - - countBySession(sessionId: string): number { - const row = this.db - .prepare( - `SELECT COUNT(*) AS count - FROM deepchat_tape_entries - WHERE session_id = ?` - ) - .get(sessionId) as { count: number } | undefined - return row?.count ?? 0 - } - - search( - sessionId: string, - query: string, - options: DeepChatTapeSearchInput = {} - ): DeepChatTapeEntryRow[] { - const normalizedQuery = query.trim() - if (!normalizedQuery) { - return [] - } - const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 - const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) - const queryPredicate = buildDeepChatTapeLikeSearchPredicate( - ['payload_json', 'meta_json', 'name'], - normalizedQuery - ) - const whereClauses = ['session_id = ?', queryPredicate.sql] - const params: Array = [sessionId, ...queryPredicate.params] - - if (options.kinds?.length) { - whereClauses.push(`kind IN (${options.kinds.map(() => '?').join(', ')})`) - params.push(...options.kinds) - } - - if (Number.isFinite(options.startCreatedAt)) { - whereClauses.push('created_at >= ?') - params.push(options.startCreatedAt as number) - } - - if (Number.isFinite(options.endCreatedAt)) { - whereClauses.push('created_at <= ?') - params.push(options.endCreatedAt as number) - } - - params.push(cappedLimit) - - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE ${whereClauses.join(' AND ')} - ORDER BY entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeEntryRow[] - } - - searchEffectiveSourcesAtHeads( - sources: readonly DeepChatTapeReadSource[], - query: string, - options: DeepChatTapeSearchInput = {} - ): DeepChatTapeEntryRow[] { - const normalizedSources = normalizeDeepChatTapeReadSources(sources) - const normalizedQuery = query.trim() - if (normalizedSources.length === 0 || !normalizedQuery) { - return [] - } - - const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 - const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) - const queryPredicate = buildDeepChatTapeLikeSearchPredicate( - ['candidate.payload_json', 'candidate.meta_json', 'candidate.name'], - normalizedQuery - ) - const whereClauses = [queryPredicate.sql] - const params: Array = [ - serializeDeepChatTapeReadSources(normalizedSources), - ...queryPredicate.params - ] - - if (options.kinds?.length) { - whereClauses.push(`candidate.kind IN (${options.kinds.map(() => '?').join(', ')})`) - params.push(...options.kinds) - } - if (Number.isFinite(options.startCreatedAt)) { - whereClauses.push('candidate.created_at >= ?') - params.push(options.startCreatedAt as number) - } - if (Number.isFinite(options.endCreatedAt)) { - whereClauses.push('candidate.created_at <= ?') - params.push(options.endCreatedAt as number) - } - params.push(cappedLimit) - - return this.db - .prepare( - `WITH - ${AUTHORIZED_TAPE_SOURCES_CTE_SQL} - SELECT candidate.* - FROM deepchat_tape_entries AS candidate - INNER JOIN authorized_sources AS source - ON source.session_id = candidate.session_id - AND candidate.entry_id <= source.max_entry_id - WHERE ${whereClauses.join(' AND ')} - AND (${EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL}) - ORDER BY candidate.created_at DESC, candidate.session_id ASC, candidate.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeEntryRow[] - } - - getEffectiveContextRowsAtHead( - source: DeepChatTapeReadSource, - entryIds: number[], - options: { before: number; after: number; limit: number } - ): DeepChatTapeEntryRow[] { - const normalizedSource = normalizeDeepChatTapeReadSources([source])[0] - const requestedEntryIds = [ - ...new Set(entryIds.filter((entryId) => Number.isSafeInteger(entryId) && entryId > 0)) - ].sort((left, right) => left - right) - if (!normalizedSource || requestedEntryIds.length === 0) { - return [] - } - const before = Math.min(Math.max(Math.floor(options.before), 0), 20) - const after = Math.min(Math.max(Math.floor(options.after), 0), 20) - const limit = Math.min(Math.max(Math.floor(options.limit), 1), 100) - - return this.db - .prepare( - `WITH - ${AUTHORIZED_TAPE_SOURCES_CTE_SQL}, - ${EFFECTIVE_TAPE_ROWS_CTE_SQL}, - ordered_rows AS ( - SELECT - effective_rows.*, - ROW_NUMBER() OVER (ORDER BY entry_id ASC) AS row_position - FROM effective_rows - ), - requested_ids(entry_id, request_ordinal) AS ( - SELECT CAST(value AS INTEGER), CAST(key AS INTEGER) - FROM json_each(?) - ), - requested_positions AS ( - SELECT - requested_ids.request_ordinal, - ordered_rows.entry_id, - ordered_rows.row_position - FROM requested_ids - INNER JOIN ordered_rows - ON ordered_rows.entry_id = requested_ids.entry_id - ), - context_candidates AS ( - SELECT - ordered_rows.*, - 0 AS priority_group, - requested_positions.request_ordinal, - 0 AS neighbor_position - FROM requested_positions - INNER JOIN ordered_rows - ON ordered_rows.entry_id = requested_positions.entry_id - UNION ALL - SELECT - ordered_rows.*, - 1 AS priority_group, - requested_positions.request_ordinal, - ordered_rows.row_position AS neighbor_position - FROM requested_positions - INNER JOIN ordered_rows - ON ordered_rows.row_position BETWEEN requested_positions.row_position - ? - AND requested_positions.row_position + ? - AND ordered_rows.entry_id != requested_positions.entry_id - ), - ranked_context_candidates AS ( - SELECT - context_candidates.*, - ROW_NUMBER() OVER ( - PARTITION BY session_id, entry_id - ORDER BY priority_group, request_ordinal, neighbor_position - ) AS duplicate_rank - FROM context_candidates - ) - SELECT - session_id, entry_id, kind, name, source_type, source_id, source_seq, - provenance_key, payload_json, meta_json, created_at - FROM ranked_context_candidates - WHERE duplicate_rank = 1 - ORDER BY priority_group, request_ordinal, neighbor_position - LIMIT ?` - ) - .all( - serializeDeepChatTapeReadSources([normalizedSource]), - JSON.stringify(requestedEntryIds), - before, - after, - limit - ) as DeepChatTapeEntryRow[] - } - - deleteBySession(sessionId: string): void { - const remove = this.db.transaction(() => { - this.db.prepare('DELETE FROM deepchat_tape_entries WHERE session_id = ?').run(sessionId) - this.mutationProjection?.deleteBySession(sessionId) - }) - remove() - } - - private ensureProvenanceColumns(): void { - const columns: Array<[string, string]> = [ - ['source_type', 'TEXT'], - ['source_id', 'TEXT'], - ['source_seq', 'INTEGER'], - ['provenance_key', 'TEXT'] - ] - for (const [columnName, columnType] of columns) { - if (!this.hasColumn(columnName)) { - this.db.exec(`ALTER TABLE deepchat_tape_entries ADD COLUMN ${columnName} ${columnType}`) - } - } - } -} +/** @deprecated Import the SQLite Tape store from `@/tape/infrastructure/sqlite/tapeEntryStore`. */ +export { + buildDeepChatTapeFtsMatch, + buildDeepChatTapeLikeSearchPredicate, + DeepChatTapeEntriesTable, + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources, + SUMMARY_ANCHOR_NAMES, + TAPE_INCARNATION_META_KEY +} from '@/tape/infrastructure/sqlite/tapeEntryStore' + +/** @deprecated Import SQLite Tape types from `@/tape/infrastructure/sqlite/tapeEntryStore`. */ +export type { + DeepChatTapeAppendInput, + DeepChatTapeEntryKind, + DeepChatTapeEntryRow, + DeepChatTapeMutationProjection, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + DeepChatTapeSourceInput, + DeepChatTapeSourceType +} from '@/tape/infrastructure/sqlite/tapeEntryStore' diff --git a/src/main/session/data/tables/deepchatTapeSearchProjection.ts b/src/main/session/data/tables/deepchatTapeSearchProjection.ts index 46660b03f3..6869f22cea 100644 --- a/src/main/session/data/tables/deepchatTapeSearchProjection.ts +++ b/src/main/session/data/tables/deepchatTapeSearchProjection.ts @@ -1,926 +1,14 @@ -import Database from 'better-sqlite3-multiple-ciphers' -import { BaseTable } from '@/data/baseTable' -import { - buildDeepChatTapeFtsMatch, - buildDeepChatTapeLikeSearchPredicate, - normalizeDeepChatTapeReadSources, - serializeDeepChatTapeReadSources -} from './deepchatTapeEntries' -import type { - DeepChatTapeReadSource, - DeepChatTapeEntryKind, - DeepChatTapeSearchInput, - DeepChatTapeSourceType -} from './deepchatTapeEntries' - -export const DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION = 2 - -export interface DeepChatTapeSearchProjectionInput { - sessionId: string - entryId: number - kind: DeepChatTapeEntryKind - name: string | null - sourceType: DeepChatTapeSourceType | null - sourceId: string | null - sourceSeq: number | null - searchText: string - summaryText: string - refs: Record - createdAt: number -} - -export interface DeepChatTapeSearchProjectionRow { - session_id: string - entry_id: number - kind: DeepChatTapeEntryKind - name: string | null - source_type: DeepChatTapeSourceType | null - source_id: string | null - source_seq: number | null - search_text: string - summary_text: string - refs_json: string - created_at: number -} - -export interface DeepChatTapeSearchProjectionResultRow extends DeepChatTapeSearchProjectionRow { - score: number | null -} - -export interface DeepChatTapeSearchProjectionMeta { - projectionVersion: number - maxEntryId: number -} - -export interface DeepChatTapeSearchProjectionReadResult { - rows: DeepChatTapeSearchProjectionResultRow[] - coveredSources: DeepChatTapeReadSource[] -} - -type FtsCapability = { available: boolean; tokenizer: 'trigram' | 'unicode61' } - -const TAPE_SEARCH_PROJECTION_INDEX_SQL = ` - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_kind - ON deepchat_tape_search_projection(session_id, kind, entry_id); - CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_created - ON deepchat_tape_search_projection(session_id, created_at, entry_id); -` - -function normalizeLimit(limit: number | undefined): number { - return Math.min(Math.max(Math.floor(Number.isFinite(limit) ? (limit as number) : 20), 1), 100) -} - -function safeJsonStringify(value: Record): string { - return JSON.stringify(value ?? {}) -} - -function parseRefs(value: string): Record { - try { - const parsed = JSON.parse(value) - return parsed && typeof parsed === 'object' && !Array.isArray(parsed) - ? (parsed as Record) - : {} - } catch { - return {} - } -} - -export class DeepChatTapeSearchProjectionTable extends BaseTable { - private ftsCapability: FtsCapability | undefined - private ftsReady = false - - constructor(db: Database.Database) { - super(db, 'deepchat_tape_search_projection') - } - - getCreateTableSQL(): string { - return ` - CREATE TABLE IF NOT EXISTS deepchat_tape_search_projection ( - session_id TEXT NOT NULL, - entry_id INTEGER NOT NULL, - kind TEXT NOT NULL, - name TEXT, - source_type TEXT, - source_id TEXT, - source_seq INTEGER, - search_text TEXT NOT NULL, - summary_text TEXT NOT NULL, - refs_json TEXT NOT NULL DEFAULT '{}', - created_at INTEGER NOT NULL, - PRIMARY KEY (session_id, entry_id) - ); - CREATE TABLE IF NOT EXISTS deepchat_tape_search_projection_meta ( - session_id TEXT PRIMARY KEY, - projection_version INTEGER NOT NULL, - max_entry_id INTEGER NOT NULL, - updated_at INTEGER NOT NULL - ); - CREATE TABLE IF NOT EXISTS deepchat_tape_search_fts_meta ( - session_id TEXT PRIMARY KEY, - projection_version INTEGER NOT NULL, - max_entry_id INTEGER NOT NULL, - updated_at INTEGER NOT NULL - ); - ${TAPE_SEARCH_PROJECTION_INDEX_SQL} - ` - } - - override createTable(): void { - this.db.exec(this.getCreateTableSQL()) - if (!this.ftsReady) { - this.ensureFtsIndex() - } - } - - getMigrationSQL(_version: number): string | null { - return null - } - - getLatestVersion(): number { - return 0 - } - - getSessionMeta(sessionId: string): DeepChatTapeSearchProjectionMeta | null { - const row = this.db - .prepare( - `SELECT projection_version, max_entry_id - FROM deepchat_tape_search_projection_meta - WHERE session_id = ?` - ) - .get(sessionId) as - | { - projection_version: number - max_entry_id: number - } - | undefined - if (!row) return null - return { - projectionVersion: row.projection_version, - maxEntryId: row.max_entry_id - } - } - - isCurrent( - sessionId: string, - maxEntryId: number, - projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - ): boolean { - const row = this.getSessionMeta(sessionId) - return row?.projectionVersion === projectionVersion && row.maxEntryId === maxEntryId - } - - getProjectedEntryIds(sessionId: string): number[] { - return ( - this.db - .prepare( - `SELECT entry_id - FROM deepchat_tape_search_projection - WHERE session_id = ? - ORDER BY entry_id ASC` - ) - .all(sessionId) as Array<{ entry_id: number }> - ).map((row) => row.entry_id) - } - - appendSession( - sessionId: string, - rows: DeepChatTapeSearchProjectionInput[], - maxEntryId: number, - projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - ): void { - const previousMeta = this.getSessionMeta(sessionId) - try { - this.db.transaction(() => { - if (!this.ftsReady) { - this.clearSessionFtsForBaseWrite(sessionId) - } - this.insertProjectionRows(rows) - if (this.ftsReady) { - if (previousMeta && this.isFtsCurrent(sessionId, previousMeta)) { - this.insertFtsRows(rows) - } else { - this.replaceSessionFtsRows(sessionId, this.getSessionProjectionInputs(sessionId)) - } - this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) - } - this.upsertMeta(sessionId, projectionVersion, maxEntryId) - })() - } catch (error) { - if (this.ftsReady) { - this.ftsReady = false - } - throw error - } - } - - replaceSession( - sessionId: string, - rows: DeepChatTapeSearchProjectionInput[], - maxEntryId: number, - projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - ): void { - try { - this.db.transaction(() => { - if (this.ftsReady) { - this.db - .prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?') - .run(sessionId) - this.db - .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') - .run(sessionId) - } else { - this.clearSessionFtsForBaseWrite(sessionId) - } - this.db - .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') - .run(sessionId) - this.db - .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') - .run(sessionId) - this.insertProjectionRows(rows) - if (this.ftsReady) { - this.insertFtsRows(rows) - this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) - } - this.upsertMeta(sessionId, projectionVersion, maxEntryId) - })() - } catch (error) { - if (this.ftsReady) { - this.ftsReady = false - } - throw error - } - } - - private insertProjectionRows(rows: DeepChatTapeSearchProjectionInput[]): void { - if (!rows.length) return - const insertProjection = this.db.prepare( - `INSERT OR REPLACE INTO deepchat_tape_search_projection ( - session_id, - entry_id, - kind, - name, - source_type, - source_id, - source_seq, - search_text, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - for (const row of rows) { - insertProjection.run( - row.sessionId, - row.entryId, - row.kind, - row.name, - row.sourceType, - row.sourceId, - row.sourceSeq, - row.searchText, - row.summaryText, - safeJsonStringify(row.refs), - row.createdAt - ) - } - } - - private upsertMeta(sessionId: string, projectionVersion: number, maxEntryId: number): void { - this.db - .prepare( - `INSERT INTO deepchat_tape_search_projection_meta ( - session_id, - projection_version, - max_entry_id, - updated_at - ) - VALUES (?, ?, ?, ?) - ON CONFLICT(session_id) DO UPDATE SET - projection_version = excluded.projection_version, - max_entry_id = excluded.max_entry_id, - updated_at = excluded.updated_at` - ) - .run(sessionId, projectionVersion, maxEntryId, Date.now()) - } - - private getFtsMeta(sessionId: string): DeepChatTapeSearchProjectionMeta | null { - const row = this.db - .prepare( - `SELECT projection_version, max_entry_id - FROM deepchat_tape_search_fts_meta - WHERE session_id = ?` - ) - .get(sessionId) as - | { - projection_version: number - max_entry_id: number - } - | undefined - if (!row) return null - return { - projectionVersion: row.projection_version, - maxEntryId: row.max_entry_id - } - } - - private isFtsCurrent(sessionId: string, meta: DeepChatTapeSearchProjectionMeta): boolean { - const row = this.getFtsMeta(sessionId) - return row?.projectionVersion === meta.projectionVersion && row.maxEntryId === meta.maxEntryId - } - - private upsertFtsMeta(sessionId: string, projectionVersion: number, maxEntryId: number): void { - this.db - .prepare( - `INSERT INTO deepchat_tape_search_fts_meta ( - session_id, - projection_version, - max_entry_id, - updated_at - ) - VALUES (?, ?, ?, ?) - ON CONFLICT(session_id) DO UPDATE SET - projection_version = excluded.projection_version, - max_entry_id = excluded.max_entry_id, - updated_at = excluded.updated_at` - ) - .run(sessionId, projectionVersion, maxEntryId, Date.now()) - } - - getByEntryIds(sessionId: string, entryIds: number[]): DeepChatTapeSearchProjectionRow[] { - const ids = [...new Set(entryIds.filter((id) => Number.isInteger(id) && id > 0))] - if (!ids.length) return [] - const placeholders = ids.map(() => '?').join(', ') - return this.db - .prepare( - `SELECT * - FROM deepchat_tape_search_projection - WHERE session_id = ? AND entry_id IN (${placeholders}) - ORDER BY entry_id ASC` - ) - .all(sessionId, ...ids) as DeepChatTapeSearchProjectionRow[] - } - - searchSourcesReadOnly( - sources: readonly DeepChatTapeReadSource[], - query: string, - options: DeepChatTapeSearchInput = {} - ): DeepChatTapeSearchProjectionReadResult { - const normalizedSources = normalizeDeepChatTapeReadSources(sources) - const coveredSources = this.getCurrentReadSources( - normalizedSources, - 'deepchat_tape_search_projection_meta' - ) - const normalizedQuery = query.trim() - if (coveredSources.length === 0 || !normalizedQuery) { - return { rows: [], coveredSources } - } - if (coveredSources.length !== normalizedSources.length) { - return { rows: [], coveredSources } - } - - const limit = normalizeLimit(options.limit) - const ordered: DeepChatTapeSearchProjectionResultRow[] = [] - const seen = new Set() - const collect = (rows: DeepChatTapeSearchProjectionResultRow[]): void => { - for (const row of rows) { - const key = `${row.session_id}:${row.entry_id}` - if (seen.has(key)) continue - seen.add(key) - ordered.push(row) - } - } - - if (this.ftsReady) { - const ftsSources = this.getCurrentReadSources(coveredSources, 'deepchat_tape_search_fts_meta') - if (ftsSources.length === coveredSources.length) { - try { - collect(this.searchFtsSourcesReadOnly(ftsSources, normalizedQuery, options, limit)) - } catch { - // The base projection remains a complete read-only fallback. - } - } - } - if (ordered.length < limit) { - collect(this.searchLikeSourcesReadOnly(coveredSources, normalizedQuery, options, limit)) - } - - return { rows: ordered.slice(0, limit), coveredSources } - } - - search( - sessionId: string, - query: string, - options: DeepChatTapeSearchInput = {} - ): DeepChatTapeSearchProjectionResultRow[] { - const normalized = query.trim() - if (!normalized) return [] - const limit = normalizeLimit(options.limit) - const ordered: DeepChatTapeSearchProjectionResultRow[] = [] - const seen = new Set() - const collect = (rows: DeepChatTapeSearchProjectionResultRow[]): void => { - for (const row of rows) { - if (seen.has(row.entry_id)) continue - seen.add(row.entry_id) - ordered.push(row) - } - } - this.recoverSessionFts(sessionId) - if (this.ftsReady) { - collect(this.searchFts(sessionId, normalized, options, limit)) - } - if (!this.ftsReady || ordered.length < limit) { - collect(this.searchLike(sessionId, normalized, options, limit)) - } - return ordered.slice(0, limit) - } - - deleteBySession(sessionId: string): void { - this.db - .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') - .run(sessionId) - this.db - .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') - .run(sessionId) - this.deleteSessionFts(sessionId) - } - - clearAll(): void { - this.db.prepare('DELETE FROM deepchat_tape_search_projection').run() - this.db.prepare('DELETE FROM deepchat_tape_search_projection_meta').run() - if (this.ftsMetaTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() - } - this.clearFts() - } - - private detectFtsCapability(): FtsCapability { - if (this.ftsCapability) return this.ftsCapability - const probe = (tokenizer: string): boolean => { - const name = `temp.tape_search_fts_probe_${tokenizer}` - try { - this.db.exec( - `CREATE VIRTUAL TABLE IF NOT EXISTS ${name} USING fts5(c, tokenize='${tokenizer}');` - ) - this.db.exec(`DROP TABLE IF EXISTS ${name};`) - return true - } catch { - return false - } - } - if (probe('trigram')) this.ftsCapability = { available: true, tokenizer: 'trigram' } - else if (probe('unicode61')) this.ftsCapability = { available: true, tokenizer: 'unicode61' } - else this.ftsCapability = { available: false, tokenizer: 'unicode61' } - return this.ftsCapability - } - - private ensureFtsIndex(): void { - const capability = this.detectFtsCapability() - if (!capability.available) { - this.ftsReady = false - return - } - try { - this.db.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS deepchat_tape_search_fts USING fts5( - search_text, - name, - session_id UNINDEXED, - entry_id UNINDEXED, - kind UNINDEXED, - source_type UNINDEXED, - source_id UNINDEXED, - source_seq UNINDEXED, - summary_text UNINDEXED, - refs_json UNINDEXED, - created_at UNINDEXED, - tokenize='${capability.tokenizer}' - ); - `) - this.ftsReady = true - } catch { - this.ftsReady = false - } - } - - private toProjectionInput( - row: DeepChatTapeSearchProjectionRow - ): DeepChatTapeSearchProjectionInput { - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - sourceType: row.source_type, - sourceId: row.source_id, - sourceSeq: row.source_seq, - searchText: row.search_text, - summaryText: row.summary_text, - refs: parseRefs(row.refs_json), - createdAt: row.created_at - } - } - - private getSessionProjectionInputs(sessionId: string): DeepChatTapeSearchProjectionInput[] { - return ( - this.db - .prepare( - `SELECT * - FROM deepchat_tape_search_projection - WHERE session_id = ? - ORDER BY entry_id ASC` - ) - .all(sessionId) as DeepChatTapeSearchProjectionRow[] - ).map((row) => this.toProjectionInput(row)) - } - - private ftsTableExists(): boolean { - const row = this.db - .prepare( - `SELECT name - FROM sqlite_master - WHERE type = 'table' AND name = 'deepchat_tape_search_fts' - LIMIT 1` - ) - .get() as { name: string } | undefined - return Boolean(row) - } - - private ftsMetaTableExists(): boolean { - const row = this.db - .prepare( - `SELECT name - FROM sqlite_master - WHERE type = 'table' AND name = 'deepchat_tape_search_fts_meta' - LIMIT 1` - ) - .get() as { name: string } | undefined - return Boolean(row) - } - - private clearSessionFtsForBaseWrite(sessionId: string): void { - if (this.ftsMetaTableExists()) { - this.db - .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') - .run(sessionId) - } - if (this.ftsTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) - } - } - - private getProjectionRowId(sessionId: string, entryId: number): number { - const row = this.db - .prepare( - `SELECT rowid - FROM deepchat_tape_search_projection - WHERE session_id = ? AND entry_id = ?` - ) - .get(sessionId, entryId) as { rowid: number } | undefined - if (!row) { - throw new Error(`Missing tape search projection row for ${sessionId}:${entryId}`) - } - return row.rowid - } - - private insertFtsRows(rows: DeepChatTapeSearchProjectionInput[]): void { - if (!rows.length) return - const insertFts = this.db.prepare( - `INSERT INTO deepchat_tape_search_fts ( - rowid, - search_text, - name, - session_id, - entry_id, - kind, - source_type, - source_id, - source_seq, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ) - for (const row of rows) { - this.db - .prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ? AND entry_id = ?') - .run(row.sessionId, row.entryId) - insertFts.run( - this.getProjectionRowId(row.sessionId, row.entryId), - row.searchText, - row.name ?? '', - row.sessionId, - row.entryId, - row.kind, - row.sourceType, - row.sourceId, - row.sourceSeq, - row.summaryText, - safeJsonStringify(row.refs), - row.createdAt - ) - } - } - - private replaceSessionFtsRows( - sessionId: string, - rows: DeepChatTapeSearchProjectionInput[] - ): void { - this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) - this.insertFtsRows(rows) - } - - private replaceSessionFts( - sessionId: string, - rows: DeepChatTapeSearchProjectionInput[], - projectionVersion: number, - maxEntryId: number - ): boolean { - if (!this.ftsReady) return false - try { - this.db.transaction(() => { - this.replaceSessionFtsRows(sessionId, rows) - this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) - })() - return true - } catch { - this.ftsReady = false - return false - } - } - - private recoverSessionFts(sessionId: string): void { - const meta = this.getSessionMeta(sessionId) - if (!meta) { - this.deleteSessionFts(sessionId) - return - } - if (this.ftsReady && this.isFtsCurrent(sessionId, meta)) return - if (!this.ftsReady) { - this.ensureFtsIndex() - } - if (!this.ftsReady) return - if (this.isFtsCurrent(sessionId, meta)) return - this.replaceSessionFts( - sessionId, - this.getSessionProjectionInputs(sessionId), - meta.projectionVersion, - meta.maxEntryId - ) - } - - hasFtsReadyForTesting(): boolean { - return this.ftsReady - } - - disableFtsForTesting(): void { - this.ftsReady = false - } - - dropFtsForTesting(): void { - this.db.exec('DROP TABLE IF EXISTS deepchat_tape_search_fts') - if (this.ftsMetaTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() - } - this.ftsReady = false - } - - private deleteSessionFts(sessionId: string): void { - if (this.ftsMetaTableExists()) { - this.db - .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') - .run(sessionId) - } - if (!this.ftsTableExists()) return - try { - this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) - } catch { - this.ftsReady = false - } - } - - private clearFts(): void { - if (this.ftsMetaTableExists()) { - this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() - } - if (!this.ftsTableExists()) return - try { - this.db.prepare('DELETE FROM deepchat_tape_search_fts').run() - } catch { - this.ftsReady = false - } - } - - private searchFts( - sessionId: string, - normalized: string, - options: DeepChatTapeSearchInput, - limit: number - ): DeepChatTapeSearchProjectionResultRow[] { - const match = buildDeepChatTapeFtsMatch(normalized) - const whereClauses = [ - 'deepchat_tape_search_fts MATCH ?', - 'deepchat_tape_search_fts.session_id = ?', - 'projection.session_id = ?' - ] - const params: Array = [match, sessionId, sessionId] - this.addFilters(whereClauses, params, options, true, 'projection') - params.push(limit) - try { - return this.db - .prepare( - `SELECT - projection.session_id, - projection.entry_id, - projection.kind, - projection.name, - projection.source_type, - projection.source_id, - projection.source_seq, - projection.search_text, - projection.summary_text, - projection.refs_json, - projection.created_at, - bm25(deepchat_tape_search_fts) AS score - FROM deepchat_tape_search_fts - INNER JOIN deepchat_tape_search_projection AS projection - ON projection.session_id = deepchat_tape_search_fts.session_id - AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) - AND projection.search_text = deepchat_tape_search_fts.search_text - WHERE ${whereClauses.join(' AND ')} - ORDER BY score ASC, projection.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeSearchProjectionResultRow[] - } catch { - return [] - } - } - - private getCurrentReadSources( - sources: readonly DeepChatTapeReadSource[], - metaTable: 'deepchat_tape_search_projection_meta' | 'deepchat_tape_search_fts_meta' - ): DeepChatTapeReadSource[] { - const normalizedSources = normalizeDeepChatTapeReadSources(sources) - if (normalizedSources.length === 0) { - return [] - } - const rows = this.db - .prepare( - `WITH requested_sources(session_id, max_entry_id) AS ( - SELECT - json_extract(value, '$.sessionId'), - CAST(json_extract(value, '$.maxEntryId') AS INTEGER) - FROM json_each(?) - ) - SELECT requested.session_id, requested.max_entry_id - FROM requested_sources AS requested - INNER JOIN ${metaTable} AS meta - ON meta.session_id = requested.session_id - AND meta.max_entry_id = requested.max_entry_id - AND meta.projection_version = ? - ORDER BY requested.session_id ASC` - ) - .all( - serializeDeepChatTapeReadSources(normalizedSources), - DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - ) as Array<{ session_id: string; max_entry_id: number }> - return rows.map((row) => ({ sessionId: row.session_id, maxEntryId: row.max_entry_id })) - } - - private searchFtsSourcesReadOnly( - sources: readonly DeepChatTapeReadSource[], - normalized: string, - options: DeepChatTapeSearchInput, - limit: number - ): DeepChatTapeSearchProjectionResultRow[] { - const match = buildDeepChatTapeFtsMatch(normalized) - const whereClauses = ['deepchat_tape_search_fts MATCH ?'] - const params: Array = [serializeDeepChatTapeReadSources(sources), match] - this.addFilters(whereClauses, params, options, true, 'projection') - params.push(limit) - return this.db - .prepare( - `WITH authorized_sources(session_id, max_entry_id) AS ( - SELECT - json_extract(value, '$.sessionId'), - CAST(json_extract(value, '$.maxEntryId') AS INTEGER) - FROM json_each(?) - ) - SELECT - projection.session_id, - projection.entry_id, - projection.kind, - projection.name, - projection.source_type, - projection.source_id, - projection.source_seq, - projection.search_text, - projection.summary_text, - projection.refs_json, - projection.created_at, - bm25(deepchat_tape_search_fts) AS score - FROM deepchat_tape_search_fts - INNER JOIN deepchat_tape_search_projection AS projection - ON projection.session_id = deepchat_tape_search_fts.session_id - AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) - AND projection.search_text = deepchat_tape_search_fts.search_text - INNER JOIN authorized_sources AS source - ON source.session_id = projection.session_id - AND projection.entry_id <= source.max_entry_id - WHERE ${whereClauses.join(' AND ')} - ORDER BY score ASC, projection.created_at DESC, projection.session_id ASC, - projection.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeSearchProjectionResultRow[] - } - - private searchLikeSourcesReadOnly( - sources: readonly DeepChatTapeReadSource[], - normalized: string, - options: DeepChatTapeSearchInput, - limit: number - ): DeepChatTapeSearchProjectionResultRow[] { - const queryPredicate = buildDeepChatTapeLikeSearchPredicate( - ['projection.search_text', 'projection.summary_text', 'projection.name'], - normalized - ) - const whereClauses = [queryPredicate.sql] - const params: Array = [serializeDeepChatTapeReadSources(sources)] - params.push(...queryPredicate.params) - this.addFilters(whereClauses, params, options, false, 'projection') - params.push(limit) - return this.db - .prepare( - `WITH authorized_sources(session_id, max_entry_id) AS ( - SELECT - json_extract(value, '$.sessionId'), - CAST(json_extract(value, '$.maxEntryId') AS INTEGER) - FROM json_each(?) - ) - SELECT projection.*, NULL AS score - FROM deepchat_tape_search_projection AS projection - INNER JOIN authorized_sources AS source - ON source.session_id = projection.session_id - AND projection.entry_id <= source.max_entry_id - WHERE ${whereClauses.join(' AND ')} - ORDER BY projection.created_at DESC, projection.session_id ASC, projection.entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeSearchProjectionResultRow[] - } - - private searchLike( - sessionId: string, - normalized: string, - options: DeepChatTapeSearchInput, - limit: number - ): DeepChatTapeSearchProjectionResultRow[] { - const queryPredicate = buildDeepChatTapeLikeSearchPredicate( - ['search_text', 'summary_text', 'name'], - normalized - ) - const whereClauses = ['session_id = ?', queryPredicate.sql] - const params: Array = [sessionId] - params.push(...queryPredicate.params) - this.addFilters(whereClauses, params, options) - params.push(limit) - return this.db - .prepare( - `SELECT *, NULL AS score - FROM deepchat_tape_search_projection - WHERE ${whereClauses.join(' AND ')} - ORDER BY entry_id DESC - LIMIT ?` - ) - .all(...params) as DeepChatTapeSearchProjectionResultRow[] - } - - private addFilters( - whereClauses: string[], - params: Array, - options: DeepChatTapeSearchInput, - castCreatedAt = false, - tableAlias?: string - ): void { - const column = (name: string) => (tableAlias ? `${tableAlias}.${name}` : name) - if (options.kinds?.length) { - whereClauses.push(`${column('kind')} IN (${options.kinds.map(() => '?').join(', ')})`) - params.push(...options.kinds) - } - const createdAtColumn = castCreatedAt - ? `CAST(${column('created_at')} AS INTEGER)` - : column('created_at') - if (Number.isFinite(options.startCreatedAt)) { - whereClauses.push(`${createdAtColumn} >= ?`) - params.push(options.startCreatedAt as number) - } - if (Number.isFinite(options.endCreatedAt)) { - whereClauses.push(`${createdAtColumn} <= ?`) - params.push(options.endCreatedAt as number) - } - } -} +/** @deprecated Import the SQLite projection from `@/tape/infrastructure/sqlite/tapeSearchProjectionStore`. */ +export { + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, + DeepChatTapeSearchProjectionTable +} from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' + +/** @deprecated Import projection types from `@/tape/infrastructure/sqlite/tapeSearchProjectionStore`. */ +export type { + DeepChatTapeSearchProjectionInput, + DeepChatTapeSearchProjectionMeta, + DeepChatTapeSearchProjectionReadResult, + DeepChatTapeSearchProjectionResultRow, + DeepChatTapeSearchProjectionRow +} from '@/tape/infrastructure/sqlite/tapeSearchProjectionStore' diff --git a/src/main/session/data/tape.ts b/src/main/session/data/tape.ts index a0908c9d42..6915099d38 100644 --- a/src/main/session/data/tape.ts +++ b/src/main/session/data/tape.ts @@ -1,2631 +1,19 @@ -import { SessionDatabase } from './database' -import { nanoid } from 'nanoid' -import { createHash } from 'crypto' -import logger from 'electron-log' -import type { - AgentTapeAnchorResult, - AgentTapeAnchorsOptions, - AgentTapeContextEntry, - AgentTapeContextOptions, - AgentTapeContextResult, - AgentTapeHandoffState, - AgentTapeSearchOptions, - AgentTapeViewScope, - ChatMessageRecord, - SubagentTapeLinkInput, - SubagentTapeLinkOutcome, - SubagentTapeLinkReceipt -} from '@shared/types/agent-interface' -import type { - DeepChatTapeViewExcludedRange, - DeepChatTapeViewManifest, - DeepChatTapeViewManifestRecord -} from '@shared/types/tape-view-manifest' -import type { - DeepChatCausalObservationReadOptions, - DeepChatCausalObservationRequest, - DeepChatCausalObservationSlice, - DeepChatTapeReplayEntrySnapshot, - DeepChatTapeReplayExportOptions, - DeepChatTapeReplaySlice, - DeepChatTapeReplayTraceSnapshot -} from '@shared/types/tape-replay' -import type { SessionTranscript } from '@/session/data/transcript' -import { - SUMMARY_ANCHOR_NAMES, - TAPE_INCARNATION_META_KEY, - type DeepChatTapeReadSource, - type DeepChatTapeEntryRow, - type DeepChatTapeSearchInput -} from '@/session/data/tables/deepchatTapeEntries' -import type { - DeepChatTapeSearchProjectionInput, - DeepChatTapeSearchProjectionResultRow, - DeepChatTapeSearchProjectionRow -} from '@/session/data/tables/deepchatTapeSearchProjection' -import type { DeepChatMessageTraceRow } from '@/session/data/tables/deepchatMessageTraces' -import { appendMessageRecordToTape, appendTapeToolFact } from '@/session/data/tapeFacts' -import type { TapeEntryRef, TapeRecorder, TapeToolFactInput } from '@/agent/deepchat/loop/ports' -import { - buildEffectiveTapeView, - getLastEffectiveTokenUsage, - searchEffectiveTapeRows -} from '@/session/data/tapeEffectiveView' -import { - hashJson, - TAPE_VIEW_MANIFEST_EVENT_NAME, - verifyTapeViewManifestHash -} from '@/session/data/tapeViewManifest' - -export type TapeMigrationState = 'none' | 'ready' - -export type TapeBackfillResult = { - sessionId: string - migrationState: TapeMigrationState - messageCount: number - maxOrderSeq: number - appendedFactCount: number - historyRecords: ChatMessageRecord[] -} - -export type TapeInfo = { - sessionId: string - entries: number - anchors: number - lastAnchor: string | null - lastAnchorEntryId: number | null - entriesSinceLastAnchor: number - lastTokenUsage: number | null - migrationState: TapeMigrationState -} - -export type TapeSearchResult = { - sessionId: string - entryId: number - kind: string - name: string | null - createdAt: number - summary?: string - refs?: Record - score?: number -} - -export type TapeAnchorResult = AgentTapeAnchorResult - -export type TapeForkHandle = { - parentSessionId: string - forkId: string - forkSessionId: string - parentHeadEntryId: number -} - -export type TapeViewManifestSourceMaps = { - latestEntryId: number - anchorEntryIds: number[] - reconstructionAnchorEntryIds: number[] - reconstructionAnchorEntryId: number | null - entryIdByMessageId: Map - toolCallEntryIdByToolId: Map - toolResultEntryIdByToolId: Map -} - -const BOOTSTRAP_ANCHOR_NAME = 'session/start' -const DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY = 2048 -const DEFAULT_CONTEXT_MAX_TOTAL_BYTES = 16384 -const MAX_CONTEXT_MAX_BYTES_PER_ENTRY = 8192 -const MAX_CONTEXT_MAX_TOTAL_BYTES = 65536 - -// Mirrors getLatestReconstructionAnchor: the anchor set that owns the summary -// cursor and prompt-visible reconstruction state (summaries, handoffs). -function isReconstructionAnchorName(name: string | null): boolean { - if (name === null) { - return false - } - return ( - (SUMMARY_ANCHOR_NAMES as readonly string[]).includes(name) || - name.startsWith('handoff/') || - name.startsWith('auto_handoff/') - ) -} - -function parseJsonObject(raw: string): Record { - try { - const parsed = JSON.parse(raw) as unknown - if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { - return parsed as Record - } - } catch {} - return {} -} - -function parseJsonValue(raw: string): unknown { - try { - return JSON.parse(raw) as unknown - } catch { - return null - } -} - -const SUBAGENT_TAPE_LINK_EVENT_NAME = 'subagent/tape_linked' -const SUBAGENT_TAPE_LINK_VERSION = 2 -const TAPE_IDENTITY_PATTERN = /^[a-f0-9]{64}$/ -const SUBAGENT_TAPE_LINK_OUTCOMES = new Set([ - 'completed', - 'error', - 'cancelled' -]) - -type SubagentTapeLinkSnapshot = { - linkEntryId: number - childSessionId: string - childHeadEntryId: number - childEntryCount: number - outcome: SubagentTapeLinkOutcome - childTapeIdentity: string | null -} - -type ParsedSubagentTapeLink = { - snapshot: SubagentTapeLinkSnapshot - frozenInput: SubagentTapeLinkInput -} - -type LinkedTapeSourceResolution = { - sources: DeepChatTapeReadSource[] - unavailableSourceIds: Set -} - -export type AgentTapeViewErrorCode = - | 'current_tape_unavailable' - | 'linked_tape_unavailable' - | 'linked_tape_unauthorized' - -export class AgentTapeViewError extends Error { - readonly name = 'AgentTapeViewError' - - constructor( - readonly code: AgentTapeViewErrorCode, - readonly parentSessionId: string, - readonly sourceSessionId: string, - message: string - ) { - super(message) - } -} - -export function normalizeSubagentTapeLinkInput( - input: SubagentTapeLinkInput -): SubagentTapeLinkInput { - const normalized = { - parentSessionId: input.parentSessionId.trim(), - childSessionId: input.childSessionId.trim(), - runId: input.runId.trim(), - taskId: input.taskId.trim(), - slotId: input.slotId.trim(), - taskTitle: compactText(input.taskTitle, 500), - outcome: input.outcome, - resultSummary: input.resultSummary?.trim() ? compactText(input.resultSummary, 2000) : null - } - for (const [name, value] of Object.entries(normalized)) { - if (name === 'resultSummary' || name === 'outcome') continue - if (typeof value !== 'string' || !value) { - throw new Error(`Subagent Tape link ${name} is required.`) - } - } - if (normalized.parentSessionId === normalized.childSessionId) { - throw new Error('Subagent Tape link child must differ from its parent.') - } - if (!SUBAGENT_TAPE_LINK_OUTCOMES.has(normalized.outcome)) { - throw new Error(`Invalid subagent Tape link outcome: ${String(normalized.outcome)}`) - } - return normalized -} - -function isUnmarkedLegacyTape(row: DeepChatTapeEntryRow): boolean { - const meta = parseJsonValue(row.meta_json) - return ( - meta !== null && - typeof meta === 'object' && - !Array.isArray(meta) && - !Object.prototype.hasOwnProperty.call(meta, TAPE_INCARNATION_META_KEY) - ) -} - -function computeTapeIdentity(row: DeepChatTapeEntryRow): string { - return createHash('sha256') - .update( - JSON.stringify([ - row.session_id, - row.entry_id, - row.kind, - row.name, - row.source_type, - row.source_id, - row.source_seq, - row.provenance_key, - row.payload_json, - row.meta_json, - row.created_at - ]) - ) - .digest('hex') -} - -function subagentTapeLinkProvenanceKey(input: SubagentTapeLinkInput): string { - // This version belongs to the stable task-identity key, independently of the evolving event - // payload's linkVersion. - const identityHash = createHash('sha256') - .update( - JSON.stringify([input.parentSessionId, input.childSessionId, input.runId, input.taskId]) - ) - .digest('hex') - return `subagent:tape-link:v1:${identityHash}` -} - -function parseSubagentTapeLink(row: DeepChatTapeEntryRow): ParsedSubagentTapeLink | null { - const payload = parseJsonObject(row.payload_json) - const data = - payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) - ? (payload.data as Record) - : {} - const childSessionId = data.childSessionId - const childHeadEntryId = data.childHeadEntryId - const childEntryCount = data.childEntryCount - const outcome = data.outcome - const runId = data.runId - const taskId = data.taskId - const slotId = data.slotId - const taskTitle = data.taskTitle - const resultSummary = data.resultSummary - const linkVersion = data.linkVersion - const childTapeIdentity = data.childTapeIdentity - const hasValidLinkVersion = - (linkVersion === 1 && childTapeIdentity === undefined) || - (linkVersion === SUBAGENT_TAPE_LINK_VERSION && - typeof childTapeIdentity === 'string' && - TAPE_IDENTITY_PATTERN.test(childTapeIdentity)) - if ( - row.kind !== 'event' || - row.name !== SUBAGENT_TAPE_LINK_EVENT_NAME || - typeof childSessionId !== 'string' || - !childSessionId || - typeof childHeadEntryId !== 'number' || - !Number.isSafeInteger(childHeadEntryId) || - childHeadEntryId < 0 || - typeof childEntryCount !== 'number' || - !Number.isSafeInteger(childEntryCount) || - childEntryCount < 0 || - typeof outcome !== 'string' || - !SUBAGENT_TAPE_LINK_OUTCOMES.has(outcome as SubagentTapeLinkOutcome) || - typeof runId !== 'string' || - typeof taskId !== 'string' || - typeof slotId !== 'string' || - typeof taskTitle !== 'string' || - (resultSummary !== null && typeof resultSummary !== 'string') || - !hasValidLinkVersion || - row.source_type !== 'subagent' || - row.source_id !== childSessionId || - row.source_seq !== childHeadEntryId || - childEntryCount > childHeadEntryId - ) { - return null - } - - let normalizedInput: SubagentTapeLinkInput - try { - normalizedInput = normalizeSubagentTapeLinkInput({ - parentSessionId: row.session_id, - childSessionId, - runId, - taskId, - slotId, - taskTitle, - outcome: outcome as SubagentTapeLinkOutcome, - resultSummary - }) - } catch { - return null - } - if ( - normalizedInput.parentSessionId !== row.session_id || - normalizedInput.childSessionId !== childSessionId || - normalizedInput.runId !== runId || - normalizedInput.taskId !== taskId || - normalizedInput.slotId !== slotId || - normalizedInput.taskTitle !== taskTitle || - normalizedInput.resultSummary !== resultSummary || - row.provenance_key !== subagentTapeLinkProvenanceKey(normalizedInput) - ) { - return null - } - - return { - snapshot: { - linkEntryId: row.entry_id, - childSessionId, - childHeadEntryId, - childEntryCount, - outcome: outcome as SubagentTapeLinkOutcome, - childTapeIdentity: - linkVersion === SUBAGENT_TAPE_LINK_VERSION ? (childTapeIdentity as string) : null - }, - frozenInput: normalizedInput - } -} - -function parseSubagentTapeLinkSnapshot(row: DeepChatTapeEntryRow): SubagentTapeLinkSnapshot | null { - return parseSubagentTapeLink(row)?.snapshot ?? null -} - -function parseLegacyExternalTapeLinkSnapshot( - row: DeepChatTapeEntryRow -): SubagentTapeLinkSnapshot | null { - if (row.kind !== 'event' || row.name !== 'fork/merge' || row.source_type !== 'fork') { - return null - } - const payload = parseJsonObject(row.payload_json) - const data = - payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) - ? (payload.data as Record) - : {} - const childSessionId = data.forkSessionId - const forkId = data.forkId - const referencedEntryCount = data.referencedEntryCount - if ( - typeof childSessionId !== 'string' || - !childSessionId || - forkId !== childSessionId || - row.source_id !== childSessionId || - row.source_seq !== 0 || - row.provenance_key !== `fork:${row.session_id}:${childSessionId}:external-merge:event` || - typeof referencedEntryCount !== 'number' || - !Number.isSafeInteger(referencedEntryCount) || - referencedEntryCount <= 0 - ) { - return null - } - return { - linkEntryId: row.entry_id, - childSessionId, - childHeadEntryId: referencedEntryCount, - childEntryCount: referencedEntryCount, - outcome: 'completed', - childTapeIdentity: null - } -} - -function readForkMergeReceiptCount( - row: DeepChatTapeEntryRow, - parentSessionId: string, - forkId: string, - forkSessionIdValue: string -): number { - const payload = parseJsonObject(row.payload_json) - const data = - payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) - ? (payload.data as Record) - : {} - const mergedCount = data.mergedCount - const forkHeadEntryId = data.forkHeadEntryId - const hasValidLegacyOrCurrentHead = - forkHeadEntryId === undefined || - (typeof forkHeadEntryId === 'number' && - Number.isSafeInteger(forkHeadEntryId) && - forkHeadEntryId >= 0) - if ( - row.session_id !== parentSessionId || - row.kind !== 'event' || - row.name !== 'fork/merge' || - row.source_type !== 'fork' || - row.source_id !== forkId || - row.source_seq !== 0 || - row.provenance_key !== `fork:${parentSessionId}:${forkId}:merge:event` || - data.forkId !== forkId || - data.forkSessionId !== forkSessionIdValue || - typeof mergedCount !== 'number' || - !Number.isSafeInteger(mergedCount) || - mergedCount < 0 || - !hasValidLegacyOrCurrentHead || - (typeof forkHeadEntryId === 'number' && mergedCount > forkHeadEntryId) - ) { - throw new Error(`Stored fork merge receipt is malformed: ${row.entry_id}`) - } - return mergedCount -} - -function assertValidForkStart( - row: DeepChatTapeEntryRow | undefined, - parentSessionId: string, - forkId: string, - forkSessionIdValue: string -): void { - if (!row) { - throw new Error(`Fork ${forkId} does not exist or has been discarded.`) - } - const payload = parseJsonObject(row.payload_json) - const state = - payload.state && typeof payload.state === 'object' && !Array.isArray(payload.state) - ? (payload.state as Record) - : {} - const parentHeadEntryId = state.parentHeadEntryId - const hasValidLegacyOrCurrentHead = - parentHeadEntryId === undefined || - (typeof parentHeadEntryId === 'number' && - Number.isSafeInteger(parentHeadEntryId) && - parentHeadEntryId >= 0) - if ( - row.session_id !== forkSessionIdValue || - row.kind !== 'anchor' || - row.name !== 'fork/start' || - row.source_type !== 'fork' || - row.source_id !== forkId || - row.source_seq !== 0 || - row.provenance_key !== `fork:${parentSessionId}:${forkId}:start` || - state.parentSessionId !== parentSessionId || - !hasValidLegacyOrCurrentHead - ) { - throw new Error(`Stored fork start is malformed: ${row.entry_id}`) - } -} - -function toSubagentTapeLinkReceipt(row: DeepChatTapeEntryRow): SubagentTapeLinkReceipt { - const snapshot = parseSubagentTapeLinkSnapshot(row) - if (!snapshot) { - throw new Error(`Stored subagent Tape link receipt is malformed: ${row.entry_id}`) - } - return { - linkEntry: { - sessionId: row.session_id, - entryId: snapshot.linkEntryId - }, - childSessionId: snapshot.childSessionId, - childHeadEntryId: snapshot.childHeadEntryId, - childEntryCount: snapshot.childEntryCount, - outcome: snapshot.outcome - } -} - -function assertSubagentTapeLinkMatchesInput( - row: DeepChatTapeEntryRow, - input: SubagentTapeLinkInput -): void { - const parsed = parseSubagentTapeLink(row) - if (!parsed) { - throw new Error(`Stored subagent Tape link receipt is malformed: ${row.entry_id}`) - } - const storedInput = parsed.frozenInput - const storedKeys = Object.keys(storedInput) as Array - if ( - storedKeys.length !== Object.keys(input).length || - storedKeys.some((key) => storedInput[key] !== input[key]) - ) { - throw new Error( - `Subagent Tape link conflicts with finalized task ${input.runId}/${input.taskId}.` - ) - } -} - -function compactText(value: string, maxLength = 1000): string { - const normalized = value.replace(/\s+/g, ' ').trim() - if (normalized.length <= maxLength) return normalized - return `${normalized.slice(0, Math.max(0, maxLength - 3)).trimEnd()}...` -} - -function stringifyForSummary(value: unknown, maxLength = 1000): string { - if (typeof value === 'string') return compactText(value, maxLength) - if (value === null || value === undefined) return '' - try { - return compactText(JSON.stringify(value), maxLength) - } catch { - return compactText(String(value), maxLength) - } -} - -function truncateToUtf8Bytes(text: string, maxBytes: number): { text: string; truncated: boolean } { - const normalized = text.trim() - if (maxBytes <= 0) { - return { text: '', truncated: normalized.length > 0 } - } - if (maxBytes < 3) { - return { text: '', truncated: normalized.length > 0 } - } - if (Buffer.byteLength(normalized, 'utf8') <= maxBytes) { - return { text: normalized, truncated: false } - } - let bytes = 0 - let output = '' - for (const character of normalized) { - const nextBytes = Buffer.byteLength(character, 'utf8') - if (bytes + nextBytes > Math.max(0, maxBytes - 3)) break - output += character - bytes += nextBytes - } - return { text: `${output.trimEnd()}...`, truncated: true } -} - -function normalizeContextByteLimit( - value: number | undefined, - fallback: number, - max: number -): number { - if (!Number.isFinite(value)) return fallback - return Math.min(Math.max(Math.floor(value as number), 0), max) -} - -function uniqueStrings(values: string[], limit = 10): string[] { - const seen = new Set() - const result: string[] = [] - for (const value of values) { - const normalized = value.trim() - if (!normalized || seen.has(normalized)) continue - seen.add(normalized) - result.push(normalized) - if (result.length >= limit) break - } - return result -} - -function extractFilePaths(text: string): string[] { - const matches = [ - ...text.matchAll( - /(?:^|[\s"'`([{<])((?:[A-Za-z]:\\|\/|\.{1,2}\/|[\w.-]+\/)[^\s"'`<>{}(),;!?]+(?:[/\\][^\s"'`<>{}(),;!?]+)*)/g - ) - ].map((match) => match[1].replace(/[.:]+$/g, '')) - return uniqueStrings(matches ?? []) -} - -function extractErrorCodes(text: string): string[] { - const matches = text.match(/\b(?:E[A-Z0-9_]{3,}|[A-Z][A-Z0-9_]*Error)\b/g) - return uniqueStrings(matches ?? []) -} - -function collectKeyedStrings( - value: unknown, - keys: Set, - output: string[] = [], - depth = 0 -): string[] { - if (depth > 4 || output.length >= 10 || !value || typeof value !== 'object') return output - if (Array.isArray(value)) { - for (const item of value) collectKeyedStrings(item, keys, output, depth + 1) - return output - } - for (const [key, nested] of Object.entries(value as Record)) { - if (keys.has(key) && typeof nested === 'string' && nested.trim()) { - output.push(compactText(nested, 500)) - if (output.length >= 10) return output - } - collectKeyedStrings(nested, keys, output, depth + 1) - if (output.length >= 10) return output - } - return output -} - -function collectUserMessageAttachmentRefs(files: unknown): { - searchText: string[] - filePaths: string[] - fileNames: string[] -} { - const searchText: string[] = [] - const filePaths: string[] = [] - const fileNames: string[] = [] - if (!Array.isArray(files)) { - return { searchText, filePaths, fileNames } - } - for (const file of files) { - if (!isRecordObject(file)) continue - const path = typeof file.path === 'string' ? file.path : '' - const name = typeof file.name === 'string' ? file.name : '' - const metadata = isRecordObject(file.metadata) ? file.metadata : null - const metadataFileName = - metadata && typeof metadata.fileName === 'string' ? metadata.fileName : '' - if (path) { - filePaths.push(compactText(path, 500)) - searchText.push(compactText(path, 500)) - } - for (const value of [name, metadataFileName]) { - if (!value) continue - fileNames.push(compactText(value, 500)) - searchText.push(compactText(value, 500)) - } - } - return { - searchText: uniqueStrings(searchText, 20), - filePaths: uniqueStrings(filePaths, 20), - fileNames: uniqueStrings(fileNames, 20) - } -} - -type UserMessageProjectionText = { - text: string - attachmentRefs: { - searchText: string[] - filePaths: string[] - fileNames: string[] - } -} - -function emptyUserMessageAttachmentRefs(): UserMessageProjectionText['attachmentRefs'] { - return { searchText: [], filePaths: [], fileNames: [] } -} - -function parseUserMessageProjectionText(content: string): UserMessageProjectionText { - const parsed = parseJsonValue(content) - if (isRecordObject(parsed) && typeof parsed.text === 'string') { - const attachmentRefs = collectUserMessageAttachmentRefs(parsed.files) - const parts = [parsed.text] - if (Array.isArray(parsed.files) && parsed.files.length > 0) { - parts.push(`files:${parsed.files.length}`) - parts.push(...attachmentRefs.searchText) - } - if (Array.isArray(parsed.links) && parsed.links.length > 0) { - parts.push(`links:${parsed.links.length}`) - } - return { text: parts.join(' '), attachmentRefs } - } - return { text: content, attachmentRefs: emptyUserMessageAttachmentRefs() } -} - -function getUserMessageProjectionText( - row: DeepChatTapeEntryRow, - payload: Record -): UserMessageProjectionText | null { - if (row.kind !== 'message' || !isRecordObject(payload.record)) return null - const record = payload.record - const role = typeof record.role === 'string' ? record.role : 'message' - if (role === 'assistant') return null - const content = typeof record.content === 'string' ? record.content : '' - return parseUserMessageProjectionText(content) -} - -function collectUserMessageAttachmentRefsFromPayload(payload: Record): { - searchText: string[] - filePaths: string[] - fileNames: string[] -} { - if (!isRecordObject(payload.record)) { - return { searchText: [], filePaths: [], fileNames: [] } - } - const content = typeof payload.record.content === 'string' ? payload.record.content : '' - const parsed = parseJsonValue(content) - return isRecordObject(parsed) - ? collectUserMessageAttachmentRefs(parsed.files) - : { searchText: [], filePaths: [], fileNames: [] } -} - -function readUserMessageText(content: string, parsed?: UserMessageProjectionText): string { - return parsed?.text ?? parseUserMessageProjectionText(content).text -} - -function readAssistantMessageText(content: string): string { - const parsed = parseJsonValue(content) - if (!Array.isArray(parsed)) return content - const parts: string[] = [] - for (const block of parsed) { - if (!isRecordObject(block)) continue - if (typeof block.content === 'string' && block.content.trim()) { - parts.push(block.content) - continue - } - const toolCall = block.tool_call - if (isRecordObject(toolCall)) { - const name = typeof toolCall.name === 'string' ? toolCall.name : 'unknown' - const params = typeof toolCall.params === 'string' ? toolCall.params : '' - const response = typeof toolCall.response === 'string' ? toolCall.response : '' - parts.push(`tool ${name} ${params} ${response}`.trim()) - } - } - return parts.join(' ') -} - -function summarizeTapeRow( - row: DeepChatTapeEntryRow, - payload: Record, - userMessage?: UserMessageProjectionText | null -): string { - if (row.kind === 'message') { - const record = payload.record - if (isRecordObject(record)) { - const role = typeof record.role === 'string' ? record.role : 'message' - const content = typeof record.content === 'string' ? record.content : '' - const text = - role === 'assistant' - ? readAssistantMessageText(content) - : readUserMessageText(content, userMessage ?? undefined) - return compactText(`${role}: ${text}`, 1200) - } - } - - if (row.kind === 'tool_call') { - const toolCall = payload.toolCall - if (isRecordObject(toolCall)) { - const name = typeof toolCall.name === 'string' ? toolCall.name : (row.name ?? 'unknown') - const params = typeof toolCall.params === 'string' ? toolCall.params : '' - return compactText(`tool_call ${name}: ${params}`, 1200) - } - } - - if (row.kind === 'tool_result') { - const response = typeof payload.response === 'string' ? payload.response : payload - return compactText( - `tool_result ${row.name ?? 'unknown'}: ${stringifyForSummary(response)}`, - 1200 - ) - } - - if (row.kind === 'anchor') { - const state = isRecordObject(payload.state) ? payload.state : payload - const summary = typeof state.summary === 'string' ? state.summary : stringifyForSummary(state) - return compactText(`anchor ${row.name ?? 'unknown'}: ${summary}`, 1200) - } - - if (row.kind === 'event') { - const data = isRecordObject(payload.data) ? payload.data : payload - return compactText(`event ${row.name ?? 'unknown'}: ${stringifyForSummary(data)}`, 1200) - } - - return compactText(`${row.kind} ${row.name ?? ''}`.trim(), 1200) -} - -function buildTapeRowEvidenceText( - row: DeepChatTapeEntryRow, - payload: Record, - meta: Record, - userMessage?: UserMessageProjectionText | null -): string { - const parts: string[] = [] - if (row.kind === 'message' && isRecordObject(payload.record)) { - const record = payload.record - const content = typeof record.content === 'string' ? record.content : '' - const role = typeof record.role === 'string' ? record.role : 'message' - parts.push( - role === 'assistant' - ? readAssistantMessageText(content) - : readUserMessageText(content, userMessage ?? undefined) - ) - } else if (row.kind === 'tool_call' && isRecordObject(payload.toolCall)) { - const toolCall = payload.toolCall - parts.push(stringifyForSummary(toolCall.name, 200)) - parts.push(stringifyForSummary(toolCall.params, 3000)) - } else if (row.kind === 'tool_result') { - parts.push(stringifyForSummary(payload.response ?? payload, 4000)) - } else if (row.kind === 'anchor') { - parts.push(stringifyForSummary(isRecordObject(payload.state) ? payload.state : payload, 4000)) - } else if (row.kind === 'event') { - parts.push(stringifyForSummary(isRecordObject(payload.data) ? payload.data : payload, 4000)) - } else { - parts.push(stringifyForSummary(payload, 4000)) - } - if (typeof meta.status === 'string') parts.push(`status:${meta.status}`) - return compactText(parts.filter(Boolean).join('\n'), 5000) -} - -function setRef(target: Record, key: string, value: unknown): void { - if (value !== null && value !== undefined && value !== '') { - target[key] = value - } -} - -function enrichTapeRowRefs( - refs: Record, - payload: Record, - meta: Record, - evidenceText: string, - userMessage?: UserMessageProjectionText | null -): void { - const attachmentRefs = - userMessage?.attachmentRefs ?? collectUserMessageAttachmentRefsFromPayload(payload) - const filePaths = uniqueStrings( - [...extractFilePaths(evidenceText), ...attachmentRefs.filePaths], - 20 - ) - const errorCodes = extractErrorCodes(evidenceText) - const commands = uniqueStrings( - [ - ...collectKeyedStrings(payload, new Set(['command', 'cmd', 'script', 'shellCommand'])), - ...collectKeyedStrings(meta, new Set(['command', 'cmd', 'script', 'shellCommand'])) - ], - 10 - ) - setRef(refs, 'filePaths', filePaths.length ? filePaths : null) - setRef(refs, 'fileNames', attachmentRefs.fileNames.length ? attachmentRefs.fileNames : null) - setRef(refs, 'commands', commands.length ? commands : null) - setRef(refs, 'errorCodes', errorCodes.length ? errorCodes : null) - for (const key of ['exitCode', 'exitStatus', 'code']) { - const value = payload[key] ?? meta[key] - if (typeof value === 'number' || typeof value === 'string') { - setRef(refs, key, value) - } - } -} - -function buildTapeRowRefs( - row: DeepChatTapeEntryRow, - payload: Record, - meta: Record, - userMessage?: UserMessageProjectionText | null, - evidenceText?: string -): Record { - const refs: Record = {} - setRef(refs, 'sourceType', row.source_type) - setRef(refs, 'sourceId', row.source_id) - setRef(refs, 'sourceSeq', row.source_seq) - setRef(refs, 'status', typeof meta.status === 'string' ? meta.status : null) - - if (row.kind === 'message' && isRecordObject(payload.record)) { - const record = payload.record - setRef(refs, 'messageId', record.id) - setRef(refs, 'orderSeq', record.orderSeq) - setRef(refs, 'role', record.role) - setRef(refs, 'messageStatus', record.status) - } else if (row.kind === 'tool_call' && isRecordObject(payload.toolCall)) { - const toolCall = payload.toolCall - setRef(refs, 'messageId', payload.messageId) - setRef(refs, 'orderSeq', payload.orderSeq) - setRef(refs, 'toolCallId', toolCall.id) - setRef(refs, 'toolName', toolCall.name ?? row.name) - setRef(refs, 'serverName', toolCall.serverName) - } else if (row.kind === 'tool_result') { - setRef(refs, 'messageId', payload.messageId) - setRef(refs, 'orderSeq', payload.orderSeq) - setRef(refs, 'toolCallId', payload.toolCallId) - setRef(refs, 'toolName', row.name) - } else if (row.kind === 'anchor') { - setRef(refs, 'anchorName', row.name) - } else if (row.kind === 'event') { - setRef(refs, 'eventName', row.name) - } - - enrichTapeRowRefs( - refs, - payload, - meta, - evidenceText ?? buildTapeRowEvidenceText(row, payload, meta, userMessage), - userMessage - ) - return refs -} - -function parseProjectionRefs(raw: string): Record { - const parsed = parseJsonObject(raw) - return parsed -} - -function normalizeContextWindowValue(value: number | undefined, fallback: number): number { - if (!Number.isFinite(value)) return fallback - return Math.min(Math.max(Math.floor(value as number), 0), 20) -} - -function normalizeContextLimit(value: number | undefined): number { - if (!Number.isFinite(value)) return 50 - return Math.min(Math.max(Math.floor(value as number), 1), 100) -} - -function readToolFactStatus(row: DeepChatTapeEntryRow): string | null { - const status = parseJsonObject(row.meta_json).status - return typeof status === 'string' ? status : null -} - -function readToolFactToolCallId(row: DeepChatTapeEntryRow): string | null { - const payload = parseJsonObject(row.payload_json) - if (row.kind === 'tool_call') { - const toolCall = payload.toolCall - if (toolCall && typeof toolCall === 'object' && !Array.isArray(toolCall)) { - const id = (toolCall as Record).id - return typeof id === 'string' && id.length > 0 ? id : null - } - return null - } - const toolCallId = payload.toolCallId - return typeof toolCallId === 'string' && toolCallId.length > 0 ? toolCallId : null -} - -function readToolFactMessageId(row: DeepChatTapeEntryRow): string | null { - const messageId = parseJsonObject(row.payload_json).messageId - return typeof messageId === 'string' && messageId.length > 0 ? messageId : null -} - -function parseSearchBoundary(value: string | undefined, name: string): number | undefined { - const trimmed = value?.trim() - if (!trimmed) { - return undefined - } - - const numericValue = Number(trimmed) - if (Number.isFinite(numericValue)) { - return numericValue - } - - const parsedDate = Date.parse(trimmed) - if (Number.isFinite(parsedDate)) { - return parsedDate - } - - throw new Error(`${name} must be an ISO date/time or millisecond timestamp.`) -} - -function toTapeSearchInput(options: AgentTapeSearchOptions | undefined): DeepChatTapeSearchInput { - return { - limit: options?.limit, - kinds: options?.kinds, - startCreatedAt: parseSearchBoundary(options?.start, 'start'), - endCreatedAt: parseSearchBoundary(options?.end, 'end') - } -} - -function normalizeTapeViewScope(scope: AgentTapeViewScope | undefined): AgentTapeViewScope { - if (scope === undefined || scope === 'current') return 'current' - if (scope === 'linked_subagents' || scope === 'current_and_linked') return scope - throw new Error(`Invalid Tape view scope: ${String(scope)}`) -} - -function normalizeTapeSearchLimit(value: number | undefined): number { - if (!Number.isFinite(value)) return 20 - return Math.min(Math.max(Math.floor(value as number), 1), 100) -} - -function compareTapeSearchResults(left: TapeSearchResult, right: TapeSearchResult): number { - const leftHasScore = typeof left.score === 'number' && Number.isFinite(left.score) - const rightHasScore = typeof right.score === 'number' && Number.isFinite(right.score) - if (leftHasScore && rightHasScore && left.score !== right.score) { - return (left.score as number) - (right.score as number) - } - if (leftHasScore !== rightHasScore) { - return leftHasScore ? -1 : 1 - } - if (left.createdAt !== right.createdAt) { - return right.createdAt - left.createdAt - } - if (left.sessionId !== right.sessionId) { - return left.sessionId < right.sessionId ? -1 : 1 - } - return right.entryId - left.entryId -} - -function migrationProvenanceKey(sessionId: string): string { - return `migration:${sessionId}:message-backfill:v1` -} - -function legacySummaryProvenanceKey(sessionId: string): string { - return `summary:${sessionId}:legacy-summary:v1` -} - -function normalizeHandoffName(name: string): string { - const trimmed = name.trim() - if (!trimmed) { - return 'handoff/manual' - } - if (trimmed.startsWith('handoff/') || trimmed.startsWith('auto_handoff/')) { - return trimmed - } - return `handoff/${trimmed}` -} - -function normalizePositiveInteger(value: unknown): number | null { - if (typeof value === 'number' && Number.isFinite(value)) { - return Math.max(1, Math.floor(value)) - } - return null -} - -function hasOwnKey(value: Record, key: string): boolean { - return Object.prototype.hasOwnProperty.call(value, key) -} - -function buildOrderSeqRange(records: ChatMessageRecord[]): Record | null { - if (records.length === 0) { - return null - } - - return { - fromOrderSeq: records[0].orderSeq, - toOrderSeq: records[records.length - 1].orderSeq - } -} - -function enrichHandoffState( - state: Record, - historyRecords: ChatMessageRecord[] -): Record { - const maxOrderSeq = historyRecords.reduce( - (currentMax, record) => Math.max(currentMax, record.orderSeq), - 0 - ) - const cursorOrderSeq = - normalizePositiveInteger(state.cursorOrderSeq ?? state.summaryCursorOrderSeq) ?? maxOrderSeq + 1 - const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) - const enrichedState: Record = { - ...state, - cursorOrderSeq - } - - if (!hasOwnKey(enrichedState, 'range')) { - enrichedState.range = buildOrderSeqRange(sourceRecords) - } - - const sourceMessageIds = enrichedState.sourceMessageIds - if (!Array.isArray(sourceMessageIds) || sourceMessageIds.some((id) => typeof id !== 'string')) { - enrichedState.sourceMessageIds = sourceRecords.map((record) => record.id) - } - - return enrichedState -} - -export function normalizeTapeHandoffState(state: unknown): AgentTapeHandoffState { - if (!state || typeof state !== 'object' || Array.isArray(state)) { - throw new Error('Tape handoff requires a non-empty summary.') - } - - const summary = (state as Record).summary - if (typeof summary !== 'string' || !summary.trim()) { - throw new Error('Tape handoff requires a non-empty summary.') - } - - return { - ...(state as Record), - summary: summary.trim() - } -} - -function forkSessionId(parentSessionId: string, forkId: string): string { - return `${parentSessionId}::fork::${forkId}` -} - -function isEntryIdPrefix(prefix: number[], values: number[]): boolean { - if (prefix.length > values.length) return false - for (let index = 0; index < prefix.length; index += 1) { - if (prefix[index] !== values[index]) return false - } - return true -} - -function hashString(value: string): string { - return createHash('sha256').update(value).digest('hex') -} - -function isPositiveInteger(value: number): boolean { - return Number.isInteger(value) && value > 0 -} - -function collectEntryIds(values: Array): number[] { - return [...new Set(values.filter((value): value is number => typeof value === 'number'))].sort( - (left, right) => left - right - ) -} - -const VIEW_POLICIES = new Set([ - 'legacy_context_v1', - 'legacy_context_shadow', - 'resume_shadow', - 'tool_loop_shadow', - 'context_pressure_recovery_shadow' -]) - -const VIEW_ENTRY_REASONS = new Set([ - 'system_prompt', - 'selected_history', - 'new_user_input', - 'resume_target', - 'tool_loop_message' -]) - -const VIEW_EXCLUDED_REASONS = new Set([ - 'before_summary_cursor', - 'compaction_indicator', - 'pending_not_context_history', - 'out_of_budget', - 'empty_after_formatting', - 'superseded', - 'retracted' -]) - -function isRecordObject(value: unknown): value is Record { - return Boolean(value && typeof value === 'object' && !Array.isArray(value)) -} - -function isNullableString(value: unknown): value is string | null { - return value === null || typeof value === 'string' -} - -function isNullableNumber(value: unknown): value is number | null { - return value === null || typeof value === 'number' -} - -function isViewEntryRef(value: unknown): value is DeepChatTapeViewManifest['included'][number] { - if (!isRecordObject(value)) { - return false - } - - return ( - isNullableNumber(value.entryId) && - isNullableString(value.messageId) && - isNullableNumber(value.orderSeq) && - (value.role === 'system' || - value.role === 'user' || - value.role === 'assistant' || - value.role === 'tool' || - value.role === null) && - (value.source === 'tape' || value.source === 'synthetic') && - typeof value.reason === 'string' && - VIEW_ENTRY_REASONS.has(value.reason) - ) -} - -function isViewExcludedRef(value: unknown): value is DeepChatTapeViewManifest['excluded'][number] { - if (!isRecordObject(value)) { - return false - } - - return ( - isNullableNumber(value.entryId) && - isNullableString(value.messageId) && - isNullableNumber(value.orderSeq) && - typeof value.reason === 'string' && - VIEW_EXCLUDED_REASONS.has(value.reason) - ) -} - -function isViewExcludedRange(value: unknown): value is DeepChatTapeViewExcludedRange { - if (!isRecordObject(value)) { - return false - } - - return ( - typeof value.fromOrderSeq === 'number' && - typeof value.toOrderSeq === 'number' && - typeof value.count === 'number' && - typeof value.reason === 'string' && - VIEW_EXCLUDED_REASONS.has(value.reason) - ) -} - -function hasNumberFields(value: unknown, fields: string[]): value is Record { - if (!isRecordObject(value)) { - return false - } - - return fields.every((field) => typeof value[field] === 'number') -} - -function hasStringFields(value: unknown, fields: string[]): value is Record { - if (!isRecordObject(value)) { - return false - } - - return fields.every((field) => typeof value[field] === 'string') -} - -function isViewManifestMeta(value: unknown): value is DeepChatTapeViewManifest['meta'] { - if (!isRecordObject(value)) { - return false - } - - return ( - typeof value.providerId === 'string' && - typeof value.modelId === 'string' && - typeof value.summaryCursorOrderSeq === 'number' && - typeof value.supportsVision === 'boolean' && - typeof value.supportsAudioInput === 'boolean' && - typeof value.traceDebugEnabled === 'boolean' - ) -} - -function isViewManifest(value: unknown, sessionId: string): value is DeepChatTapeViewManifest { - if (!isRecordObject(value)) { - return false - } - - return ( - (value.schemaVersion === 1 || value.schemaVersion === 2) && - typeof value.hashVersion === 'number' && - value.sessionId === sessionId && - typeof value.viewId === 'string' && - typeof value.messageId === 'string' && - typeof value.requestSeq === 'number' && - (value.taskType === 'chat' || value.taskType === 'resume' || value.taskType === 'tool_loop') && - typeof value.policy === 'string' && - VIEW_POLICIES.has(value.policy) && - (typeof value.policyVersion === 'number' || value.policyVersion === null) && - value.contextBuilderVersion === 'legacy-v1' && - typeof value.latestEntryId === 'number' && - Array.isArray(value.anchorEntryIds) && - value.anchorEntryIds.every((entryId) => typeof entryId === 'number') && - (value.reconstructionAnchorEntryId === undefined || - isNullableNumber(value.reconstructionAnchorEntryId)) && - (value.excludedRanges === undefined || - (Array.isArray(value.excludedRanges) && value.excludedRanges.every(isViewExcludedRange))) && - Array.isArray(value.included) && - value.included.every(isViewEntryRef) && - Array.isArray(value.excluded) && - value.excluded.every(isViewExcludedRef) && - hasNumberFields(value.tokenBudget, [ - 'contextLength', - 'requestedMaxTokens', - 'effectiveMaxTokens', - 'reserveTokens', - 'toolReserveTokens', - 'estimatedPromptTokens' - ]) && - hasStringFields(value.hashes, ['promptHash', 'toolDefinitionsHash', 'manifestHash']) && - isViewManifestMeta(value.meta) && - typeof value.assembledAt === 'number' - ) -} - -function withReplaySliceHash( - slice: Omit & { - hashes: Omit & { sliceHash: '' } - } -): DeepChatTapeReplaySlice { - const sliceForHash = { ...slice } as Partial - delete sliceForHash.createdAt - delete sliceForHash.integrity - return { - ...slice, - hashes: { - ...slice.hashes, - sliceHash: hashJson(sliceForHash) - } - } -} - -export class SessionTape implements Pick { - constructor(private readonly database: SessionDatabase) {} - - private get table(): SessionDatabase['deepchatTapeEntriesTable'] { - return this.database.deepchatTapeEntriesTable - } - - private get searchProjectionTable(): SessionDatabase['deepchatTapeSearchProjectionTable'] { - return this.database.deepchatTapeSearchProjectionTable - } - - ensureSessionTapeReady(sessionId: string, messageStore: SessionTranscript): TapeBackfillResult { - const table = this.table - const historyRecords = messageStore - .getMessages(sessionId) - .sort((left, right) => left.orderSeq - right.orderSeq) - const maxOrderSeq = historyRecords.reduce( - (currentMax, record) => Math.max(currentMax, record.orderSeq), - 0 - ) - - table.ensureBootstrapAnchor(sessionId) - - let appendedFactCount = 0 - for (const record of historyRecords) { - appendedFactCount += appendMessageRecordToTape(table, record, 'backfill') - } - - this.backfillLegacySummaryAnchor(sessionId, historyRecords) - - table.appendEvent({ - sessionId, - name: 'migration/backfill', - source: { - type: 'migration', - id: 'message-backfill', - seq: 1 - }, - provenanceKey: migrationProvenanceKey(sessionId), - data: { - source: 'deepchat_messages', - messageCount: historyRecords.length, - maxOrderSeq - }, - idempotent: true - }) - - return { - sessionId, - migrationState: 'ready', - messageCount: historyRecords.length, - maxOrderSeq, - appendedFactCount, - historyRecords: this.getMessageRecords(sessionId) - } - } - - appendMessageRecord(record: ChatMessageRecord): number { - return appendMessageRecordToTape(this.table, record, 'live') - } - - async appendToolFact(input: TapeToolFactInput): Promise { - const row = appendTapeToolFact(this.table, input, 'live', 'tool_loop') - if (!row) throw new Error('Tape tool fact was not appendable.') - return { sessionId: input.sessionId, entryId: row.entry_id } - } - - getMessageRecords(sessionId: string): ChatMessageRecord[] { - return buildEffectiveTapeView(this.table.getBySession(sessionId), { includePending: true }) - .messageRecords - } - - info(sessionId: string): TapeInfo { - const table = this.table - const lastAnchor = table.getLatestAnchor(sessionId) - const rows = table.getBySession(sessionId) - return { - sessionId, - entries: table.countBySession(sessionId), - anchors: table.countAnchorsBySession(sessionId), - lastAnchor: lastAnchor?.name ?? null, - lastAnchorEntryId: lastAnchor?.entry_id ?? null, - entriesSinceLastAnchor: lastAnchor - ? table.countEntriesAfter(sessionId, lastAnchor.entry_id) - : 0, - lastTokenUsage: getLastEffectiveTokenUsage(rows), - migrationState: table.getByProvenanceKey(sessionId, migrationProvenanceKey(sessionId)) - ? 'ready' - : 'none' - } - } - - search(sessionId: string, query: string, options?: AgentTapeSearchOptions): TapeSearchResult[] { - const scope = normalizeTapeViewScope(options?.scope) - if (!query.trim()) { - return [] - } - if (scope === 'current') { - return this.searchCurrentTape(sessionId, query, options) - } - - const resolution = this.resolveLinkedTapeSources(sessionId) - if (resolution.unavailableSourceIds.size > 0) { - const sourceSessionId = [...resolution.unavailableSourceIds].sort()[0] - throw new AgentTapeViewError( - 'linked_tape_unavailable', - sessionId, - sourceSessionId, - `Linked Tape ${sourceSessionId} is unavailable.` - ) - } - const sources = [...resolution.sources] - if (scope === 'current_and_linked') { - sources.push({ sessionId, maxEntryId: this.table?.getMaxEntryId(sessionId) ?? 0 }) - } - return this.searchTapeSourcesReadOnly(sources, query, options) - } - - private searchCurrentTape( - sessionId: string, - query: string, - options?: AgentTapeSearchOptions - ): TapeSearchResult[] { - const table = this.table - const searchInput = toTapeSearchInput(options) - const projectionTable = this.searchProjectionTable - let skipProjectionSearch = false - - if (projectionTable) { - try { - const maxEntryId = table.getMaxEntryId(sessionId) - if (projectionTable.isCurrent(sessionId, maxEntryId)) { - return projectionTable - .search(sessionId, query, searchInput) - .map((row) => this.toProjectedSearchResult(row, undefined)) - } - } catch (error) { - skipProjectionSearch = true - logger.warn( - `[Tape] projection fast-path search failed; falling back to effective search: ${String(error)}` - ) - } - } - - const rows = table.getBySession(sessionId) - const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows - const preparedProjectionTable = skipProjectionSearch - ? null - : this.ensureSearchProjection(sessionId, rows, effectiveRows) - if (!preparedProjectionTable) { - return searchEffectiveTapeRows(rows, query, searchInput).map((row) => - this.toSearchResult(row) - ) - } - - const rowByEntryId = new Map(effectiveRows.map((row) => [row.entry_id, row])) - try { - return preparedProjectionTable - .search(sessionId, query, searchInput) - .map((row) => this.toProjectedSearchResult(row, rowByEntryId.get(row.entry_id))) - } catch (error) { - logger.warn( - `[Tape] projection search failed; falling back to effective search: ${String(error)}` - ) - return searchEffectiveTapeRows(rows, query, searchInput).map((row) => - this.toSearchResult(row) - ) - } - } - - getContext( - sessionId: string, - entryIds: number[], - options: AgentTapeContextOptions = {} - ): AgentTapeContextResult { - const sourceSessionId = options.sourceSessionId?.trim() || sessionId - if (sourceSessionId !== sessionId) { - return this.getLinkedTapeContext(sessionId, sourceSessionId, entryIds, options) - } - - const requestedEntryIds = [ - ...new Set(entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0)) - ].sort((left, right) => left - right) - const table = this.table - if (!table || requestedEntryIds.length === 0) { - return { - sessionId, - sourceSessionId, - requestedEntryIds, - matchedEntryIds: [], - entries: [] - } - } - - const rows = table.getBySession(sessionId) - const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows - const indexByEntryId = new Map(effectiveRows.map((row, index) => [row.entry_id, index])) - const before = normalizeContextWindowValue(options.before, 2) - const after = normalizeContextWindowValue(options.after, 2) - const limit = normalizeContextLimit(options.limit) - const maxBytesPerEntry = normalizeContextByteLimit( - options.maxBytesPerEntry, - DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY, - MAX_CONTEXT_MAX_BYTES_PER_ENTRY - ) - const maxTotalBytes = normalizeContextByteLimit( - options.maxTotalBytes, - DEFAULT_CONTEXT_MAX_TOTAL_BYTES, - MAX_CONTEXT_MAX_TOTAL_BYTES - ) - const selectedIndexes = new Set() - const requestedIndexes: number[] = [] - const windowIndexes: number[] = [] - - for (const entryId of requestedEntryIds) { - const index = indexByEntryId.get(entryId) - if (index === undefined) continue - requestedIndexes.push(index) - for ( - let cursor = Math.max(0, index - before); - cursor <= Math.min(effectiveRows.length - 1, index + after); - cursor += 1 - ) { - if (cursor === index) continue - windowIndexes.push(cursor) - } - } - - for (const index of requestedIndexes) { - if (selectedIndexes.size >= limit) break - selectedIndexes.add(index) - } - for (const index of windowIndexes) { - if (selectedIndexes.size >= limit) break - selectedIndexes.add(index) - } - - const selectedRows = [...selectedIndexes] - .sort((left, right) => left - right) - .map((index) => effectiveRows[index]) - let projectionRows = new Map() - try { - projectionRows = new Map( - this.searchProjectionTable - .getByEntryIds( - sessionId, - selectedRows.map((row) => row.entry_id) - ) - .map((row) => [row.entry_id, row]) - ) - } catch { - projectionRows = new Map() - } - let usedBytes = 0 - const entries: AgentTapeContextEntry[] = [] - const priorityIndexes = [...requestedIndexes, ...windowIndexes].filter( - (index, offset, indexes) => { - return selectedIndexes.has(index) && indexes.indexOf(index) === offset - } - ) - for (const index of priorityIndexes) { - const row = effectiveRows[index] - const remaining = Math.max(0, maxTotalBytes - usedBytes) - if (remaining <= 0) break - const maxEntryBytes = Math.min(maxBytesPerEntry, remaining) - if (maxEntryBytes <= 0) break - const entry = this.toContextEntry(row, projectionRows.get(row.entry_id), maxEntryBytes) - if (entry.evidence.bytes <= 0) continue - usedBytes += entry.evidence.bytes - entries.push(entry) - } - entries.sort((left, right) => left.entryId - right.entryId) - const returnedEntryIds = new Set(entries.map((entry) => entry.entryId)) - - return { - sessionId, - sourceSessionId, - requestedEntryIds, - matchedEntryIds: requestedEntryIds.filter((entryId) => returnedEntryIds.has(entryId)), - entries - } - } - - private resolveLinkedTapeSources(parentSessionId: string): LinkedTapeSourceResolution { - const table = this.table - const sessionTable = this.database.newSessionsTable - if (!table || !sessionTable?.get(parentSessionId)) { - throw new AgentTapeViewError( - 'current_tape_unavailable', - parentSessionId, - parentSessionId, - `Current Tape ${parentSessionId} is unavailable.` - ) - } - - const parsedSnapshots = table - .getSubagentLineageEvents(parentSessionId) - .map((row) => parseSubagentTapeLinkSnapshot(row) ?? parseLegacyExternalTapeLinkSnapshot(row)) - .filter((snapshot): snapshot is SubagentTapeLinkSnapshot => snapshot !== null) - const latestSnapshotByChild = new Map() - for (const snapshot of parsedSnapshots) { - const current = latestSnapshotByChild.get(snapshot.childSessionId) - if (!current || snapshot.linkEntryId > current.linkEntryId) { - latestSnapshotByChild.set(snapshot.childSessionId, snapshot) - } - } - const snapshots = [...latestSnapshotByChild.values()] - const childSessionIds = [...new Set(snapshots.map((snapshot) => snapshot.childSessionId))] - const childById = new Map(sessionTable.getMany(childSessionIds).map((row) => [row.id, row])) - const unavailableSourceIds = new Set() - const snapshotBySource = new Map() - - for (const snapshot of snapshots) { - const child = childById.get(snapshot.childSessionId) - if (!child) { - unavailableSourceIds.add(snapshot.childSessionId) - continue - } - if (child.session_kind !== 'subagent' || child.parent_session_id !== parentSessionId) { - continue - } - snapshotBySource.set(snapshot.childSessionId, snapshot) - } - - const authorizedSourceIds = [...snapshotBySource.keys()] - const firstEntryBySource = new Map( - table.getFirstEntriesBySessions(authorizedSourceIds).map((row) => [row.session_id, row]) - ) - const liveHeads = table.getMaxEntryIdsBySessions(authorizedSourceIds) - const availableSources: DeepChatTapeReadSource[] = [] - for (const [sourceSessionId, snapshot] of snapshotBySource) { - const firstEntry = firstEntryBySource.get(sourceSessionId) - const identityMatches = snapshot.childTapeIdentity - ? firstEntry !== undefined && computeTapeIdentity(firstEntry) === snapshot.childTapeIdentity - : firstEntry !== undefined && isUnmarkedLegacyTape(firstEntry) - if (!identityMatches || (liveHeads.get(sourceSessionId) ?? 0) < snapshot.childHeadEntryId) { - unavailableSourceIds.add(sourceSessionId) - continue - } - availableSources.push({ - sessionId: sourceSessionId, - maxEntryId: snapshot.childHeadEntryId - }) - } - - return { - sources: availableSources.sort((left, right) => - left.sessionId < right.sessionId ? -1 : left.sessionId > right.sessionId ? 1 : 0 - ), - unavailableSourceIds - } - } - - private searchTapeSourcesReadOnly( - sources: DeepChatTapeReadSource[], - query: string, - options: AgentTapeSearchOptions | undefined - ): TapeSearchResult[] { - const table = this.table - if (!table || !query.trim() || sources.length === 0) { - return [] - } - const searchInput = toTapeSearchInput(options) - const limit = normalizeTapeSearchLimit(options?.limit) - const results: TapeSearchResult[] = [] - let uncoveredSources = sources - - try { - const projected = this.searchProjectionTable?.searchSourcesReadOnly( - sources, - query, - searchInput - ) - if (projected) { - const coveredHeadBySource = new Map( - projected.coveredSources.map((source) => [source.sessionId, source.maxEntryId]) - ) - const hasCompleteProjection = - coveredHeadBySource.size === sources.length && - sources.every((source) => coveredHeadBySource.get(source.sessionId) === source.maxEntryId) - if (hasCompleteProjection) { - results.push(...projected.rows.map((row) => this.toProjectedSearchResult(row, undefined))) - uncoveredSources = [] - } - } - } catch (error) { - logger.warn( - `[Tape] linked projection search failed; using read-only Tape fallback: ${String(error)}` - ) - uncoveredSources = sources - } - - if (uncoveredSources.length > 0) { - results.push( - ...table - .searchEffectiveSourcesAtHeads(uncoveredSources, query, searchInput) - .map((row) => this.toSearchResult(row)) - ) - } - - const seen = new Set() - return results - .sort(compareTapeSearchResults) - .filter((result) => { - const key = `${result.sessionId}:${result.entryId}` - if (seen.has(key)) return false - seen.add(key) - return true - }) - .slice(0, limit) - } - - private getLinkedTapeContext( - parentSessionId: string, - sourceSessionId: string, - entryIds: number[], - options: AgentTapeContextOptions - ): AgentTapeContextResult { - const resolution = this.resolveLinkedTapeSources(parentSessionId) - if (resolution.unavailableSourceIds.has(sourceSessionId)) { - throw new AgentTapeViewError( - 'linked_tape_unavailable', - parentSessionId, - sourceSessionId, - `Linked Tape ${sourceSessionId} is unavailable.` - ) - } - const source = resolution.sources.find((candidate) => candidate.sessionId === sourceSessionId) - if (!source) { - throw new AgentTapeViewError( - 'linked_tape_unauthorized', - parentSessionId, - sourceSessionId, - `Tape ${sourceSessionId} is not an authorized direct child of ${parentSessionId}.` - ) - } - - const requestedEntryIds = [ - ...new Set(entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0)) - ].sort((left, right) => left - right) - const table = this.table - if (!table || requestedEntryIds.length === 0) { - return { - sessionId: parentSessionId, - sourceSessionId, - requestedEntryIds, - matchedEntryIds: [], - entries: [] - } - } - - const before = normalizeContextWindowValue(options.before, 2) - const after = normalizeContextWindowValue(options.after, 2) - const limit = normalizeContextLimit(options.limit) - const maxBytesPerEntry = normalizeContextByteLimit( - options.maxBytesPerEntry, - DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY, - MAX_CONTEXT_MAX_BYTES_PER_ENTRY - ) - const maxTotalBytes = normalizeContextByteLimit( - options.maxTotalBytes, - DEFAULT_CONTEXT_MAX_TOTAL_BYTES, - MAX_CONTEXT_MAX_TOTAL_BYTES - ) - const rows = table.getEffectiveContextRowsAtHead(source, requestedEntryIds, { - before, - after, - limit - }) - - let usedBytes = 0 - const entries: AgentTapeContextEntry[] = [] - for (const row of rows) { - const remaining = Math.max(0, maxTotalBytes - usedBytes) - if (remaining <= 0) break - const maxEntryBytes = Math.min(maxBytesPerEntry, remaining) - const entry = this.toContextEntry(row, undefined, maxEntryBytes) - if (entry.evidence.bytes <= 0) continue - usedBytes += entry.evidence.bytes - entries.push(entry) - } - entries.sort((left, right) => left.entryId - right.entryId) - const returnedEntryIds = new Set(entries.map((entry) => entry.entryId)) - - return { - sessionId: parentSessionId, - sourceSessionId, - requestedEntryIds, - matchedEntryIds: requestedEntryIds.filter((entryId) => returnedEntryIds.has(entryId)), - entries - } - } - - anchors(sessionId: string, options: AgentTapeAnchorsOptions = {}): TapeAnchorResult[] { - return this.table.getAnchors(sessionId, options.limit).map((row) => this.toAnchorResult(row)) - } - - getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestSourceMaps { - const table = this.table - const rows = table.getBySession(sessionId) - const entryIdByMessageId = new Map() - const toolCallEntryIdByToolId = new Map() - const toolResultEntryIdByToolId = new Map() - let latestEntryId = 0 - const anchorEntryIds: number[] = [] - let reconstructionAnchorEntryId: number | null = null - let bootstrapAnchorEntryId: number | null = null - - for (const row of rows) { - latestEntryId = Math.max(latestEntryId, row.entry_id) - if (row.kind === 'anchor') { - anchorEntryIds.push(row.entry_id) - if (isReconstructionAnchorName(row.name)) { - if (reconstructionAnchorEntryId === null || row.entry_id > reconstructionAnchorEntryId) { - reconstructionAnchorEntryId = row.entry_id - } - } else if (row.name === BOOTSTRAP_ANCHOR_NAME) { - bootstrapAnchorEntryId = row.entry_id - } - continue - } - if (row.kind === 'message' && row.source_type === 'message' && row.source_id) { - entryIdByMessageId.set(row.source_id, row.entry_id) - continue - } - if (row.kind === 'tool_call' || row.kind === 'tool_result') { - if (messageId && readToolFactMessageId(row) !== messageId) { - continue - } - const toolCallId = readToolFactToolCallId(row) - if (!toolCallId || readToolFactStatus(row) === 'pending') { - continue - } - const target = - row.kind === 'tool_call' ? toolCallEntryIdByToolId : toolResultEntryIdByToolId - target.set(toolCallId, row.entry_id) - } - } - - const reconstructionAnchorEntryIds = - reconstructionAnchorEntryId !== null - ? [reconstructionAnchorEntryId] - : bootstrapAnchorEntryId !== null - ? [bootstrapAnchorEntryId] - : [] - - return { - latestEntryId, - anchorEntryIds, - reconstructionAnchorEntryIds, - reconstructionAnchorEntryId, - entryIdByMessageId, - toolCallEntryIdByToolId, - toolResultEntryIdByToolId - } - } - - appendViewManifest(manifest: DeepChatTapeViewManifest): DeepChatTapeEntryRow { - const table = this.table - table.ensureBootstrapAnchor(manifest.sessionId) - return table.appendEvent({ - sessionId: manifest.sessionId, - name: TAPE_VIEW_MANIFEST_EVENT_NAME, - source: { - type: 'runtime_event', - id: manifest.messageId, - seq: manifest.requestSeq - }, - provenanceKey: `view:${manifest.sessionId}:${manifest.messageId}:${manifest.requestSeq}:${manifest.hashes.manifestHash}`, - data: { - manifest - }, - meta: { - viewId: manifest.viewId, - requestSeq: manifest.requestSeq, - taskType: manifest.taskType, - policy: manifest.policy, - policyVersion: manifest.policyVersion - }, - createdAt: manifest.assembledAt, - idempotent: true - }) - } - - listViewManifestsByMessage( - sessionId: string, - messageId: string - ): DeepChatTapeViewManifestRecord[] { - const table = this.table - return table - .getBySession(sessionId) - .filter( - (row) => - row.kind === 'event' && - row.name === TAPE_VIEW_MANIFEST_EVENT_NAME && - row.source_type === 'runtime_event' && - row.source_id === messageId - ) - .map((row) => this.toViewManifestRecord(row)) - .filter((record): record is DeepChatTapeViewManifestRecord => Boolean(record)) - .sort((left, right) => right.requestSeq - left.requestSeq || right.entryId - left.entryId) - } - - exportReplaySlice( - sessionId: string, - messageId: string, - options: DeepChatTapeReplayExportOptions = {} - ): DeepChatTapeReplaySlice | null { - if (options.requestSeq !== undefined && !isPositiveInteger(options.requestSeq)) { - throw new Error('requestSeq must be a positive integer.') - } - - const manifests = this.listViewManifestsByMessage(sessionId, messageId) - const manifestRecord = - options.requestSeq === undefined - ? manifests[0] - : manifests.find((record) => record.requestSeq === options.requestSeq) - if (!manifestRecord) { - return null - } - - return this.buildReplaySlice(sessionId, messageId, manifestRecord, options) - } - - readCausalObservationSlice( - sessionId: string, - messageId: string, - options: DeepChatCausalObservationReadOptions = {} - ): DeepChatCausalObservationSlice { - if (options.requestSeq !== undefined && !isPositiveInteger(options.requestSeq)) { - throw new Error('requestSeq must be a positive integer.') - } - - const rows = this.table.getBySession(sessionId) - const manifestRows = rows.filter( - (row) => - row.kind === 'event' && - row.name === TAPE_VIEW_MANIFEST_EVENT_NAME && - row.source_type === 'runtime_event' && - row.source_id === messageId - ) - const traces = this.database.deepchatMessageTracesTable - .listByMessageId(messageId) - .filter( - (row) => - row.session_id === sessionId && - row.message_id === messageId && - isPositiveInteger(row.request_seq) - ) - - const requestSeq = - options.requestSeq ?? - [...manifestRows.map((row) => row.source_seq), ...traces.map((row) => row.request_seq)] - .filter((value): value is number => typeof value === 'number' && isPositiveInteger(value)) - .reduce((latest, value) => Math.max(latest ?? value, value), null) - - let request: DeepChatCausalObservationRequest - if (requestSeq === null) { - request = { state: 'request_unavailable', requestSeq: null, trace: null } - } else { - const selectedManifestRows = manifestRows.filter((row) => row.source_seq === requestSeq) - const manifestRecord = selectedManifestRows - .map((row) => this.toViewManifestRecord(row)) - .find((record) => record?.messageId === messageId && record.requestSeq === requestSeq) - const trace = traces.find((row) => row.request_seq === requestSeq) ?? null - - if (manifestRecord) { - request = { - state: 'manifest_bound', - requestSeq, - replay: this.buildReplaySlice(sessionId, messageId, manifestRecord, options) - } - } else { - request = { - state: selectedManifestRows.length > 0 ? 'manifest_malformed' : 'manifest_missing', - requestSeq, - trace: trace - ? this.toReplayTraceSnapshot(trace, options.includeTracePayload === true) - : null - } - } - } - - const outputEntries = buildEffectiveTapeView(rows, { includePending: false }) - .rows.filter( - (row) => - (row.kind === 'message' && - row.source_type === 'message' && - row.source_id === messageId) || - ((row.kind === 'tool_call' || row.kind === 'tool_result') && - readToolFactMessageId(row) === messageId) - ) - .map((row) => this.toReplayEntrySnapshot(row, options.includeTapePayloads === true)) - const message = this.database.deepchatMessagesTable.get(messageId) - const terminalMessage = - message?.session_id === sessionId && - message.role === 'assistant' && - (message.status === 'sent' || message.status === 'error') - ? { - status: message.status, - orderSeq: message.order_seq, - createdAt: message.created_at, - updatedAt: message.updated_at, - contentHash: hashString(message.content), - metadataHash: hashString(message.metadata) - } - : null - - return { - schemaVersion: 1, - sessionId, - messageId, - request, - output: { - correlation: 'message_only', - entries: outputEntries, - terminalMessage - }, - runtime: - options.currentRuntimeStatus === undefined - ? { scope: 'unavailable', status: null, eventHistory: 'not_persisted' } - : { - scope: 'current_only', - status: options.currentRuntimeStatus, - eventHistory: 'not_persisted' - } - } - } - - private buildReplaySlice( - sessionId: string, - messageId: string, - manifestRecord: DeepChatTapeViewManifestRecord, - options: DeepChatTapeReplayExportOptions - ): DeepChatTapeReplaySlice { - const table = this.table - const manifest = manifestRecord.manifest - const includedEntryIds = collectEntryIds(manifest.included.map((ref) => ref.entryId)) - const excludedEntryIds = collectEntryIds(manifest.excluded.map((ref) => ref.entryId)) - const anchorEntryIds = collectEntryIds(manifest.anchorEntryIds) - const selectedEntryIds = new Set([ - manifestRecord.entryId, - ...includedEntryIds, - ...excludedEntryIds, - ...anchorEntryIds - ]) - const entries = table - .getBySession(sessionId) - .filter((row) => selectedEntryIds.has(row.entry_id)) - .map((row) => this.toReplayEntrySnapshot(row, options.includeTapePayloads === true)) - - const trace = this.findReplayTrace(sessionId, messageId, manifestRecord.requestSeq) - const createdAt = Date.now() - const sliceBase: Omit & { - hashes: Omit & { sliceHash: '' } - } = { - schemaVersion: 1 as const, - sliceId: `replay_${hashJson({ - sessionId, - messageId, - requestSeq: manifestRecord.requestSeq, - manifestHash: manifest.hashes.manifestHash - }).slice(0, 16)}`, - sessionId, - messageId, - requestSeq: manifestRecord.requestSeq, - mode: trace ? 'trace_bound' : 'manifest_only', - manifestRecord, - trace: trace ? this.toReplayTraceSnapshot(trace, options.includeTracePayload === true) : null, - entries, - refs: { - manifestEntryId: manifestRecord.entryId, - includedEntryIds, - excludedEntryIds, - anchorEntryIds - }, - hashes: { - manifestHash: manifest.hashes.manifestHash, - sliceHash: '' - }, - integrity: manifestRecord.integrity, - createdAt - } - - return withReplaySliceHash(sliceBase) - } - - handoff( - sessionId: string, - name: string, - state: AgentTapeHandoffState, - meta: Record = {} - ): DeepChatTapeEntryRow { - const normalizedState = normalizeTapeHandoffState(state) - const table = this.table - table.ensureBootstrapAnchor(sessionId) - const handoffState = enrichHandoffState(normalizedState, this.getMessageRecords(sessionId)) - return table.appendAnchor({ - sessionId, - name: normalizeHandoffName(name), - source: { - type: 'runtime_event', - id: `handoff:${Date.now()}`, - seq: 0 - }, - state: handoffState, - meta: { - ...meta, - handoff: true - } - }) - } - - handoffResult( - sessionId: string, - name: string, - state: AgentTapeHandoffState, - meta: Record = {} - ): TapeAnchorResult { - return this.toAnchorResult(this.handoff(sessionId, name, state, meta)) - } - - createFork(parentSessionId: string, forkId: string = nanoid()): TapeForkHandle { - const table = this.table - const forkIdValue = forkId.trim() || nanoid() - const forkSessionIdValue = forkSessionId(parentSessionId, forkIdValue) - const parentHeadEntryId = table.getMaxEntryId(parentSessionId) - table.ensureBootstrapAnchor(forkSessionIdValue) - const parentAnchor = table.getLatestAnchor(parentSessionId) - const forkStart = table.appendAnchor({ - sessionId: forkSessionIdValue, - name: 'fork/start', - source: { - type: 'fork', - id: forkIdValue, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkIdValue}:start`, - state: { - parentSessionId, - parentHeadEntryId, - parentLastAnchorEntryId: parentAnchor?.entry_id ?? null, - parentLastAnchorName: parentAnchor?.name ?? null - }, - idempotent: true - }) - const forkStartPayload = parseJsonObject(forkStart.payload_json) - const persistedState = - forkStartPayload.state && - typeof forkStartPayload.state === 'object' && - !Array.isArray(forkStartPayload.state) - ? (forkStartPayload.state as Record) - : {} - const persistedParentHeadEntryId = persistedState.parentHeadEntryId - return { - parentSessionId, - forkId: forkIdValue, - forkSessionId: forkSessionIdValue, - parentHeadEntryId: - typeof persistedParentHeadEntryId === 'number' && - Number.isSafeInteger(persistedParentHeadEntryId) && - persistedParentHeadEntryId >= 0 - ? persistedParentHeadEntryId - : parentHeadEntryId - } - } - - appendForkMessageRecord(handle: TapeForkHandle, record: ChatMessageRecord): number { - return appendMessageRecordToTape( - this.table, - { - ...record, - sessionId: handle.forkSessionId - }, - 'live' - ) - } - - mergeFork(parentSessionId: string, forkId: string): number { - const table = this.table - const forkSessionIdValue = forkSessionId(parentSessionId, forkId) - const mergeProvenanceKey = `fork:${parentSessionId}:${forkId}:merge:event` - - return table.runInTransaction(() => { - const existingReceipt = table.getByProvenanceKey(parentSessionId, mergeProvenanceKey) - if (existingReceipt) { - return readForkMergeReceiptCount( - existingReceipt, - parentSessionId, - forkId, - forkSessionIdValue - ) - } - - assertValidForkStart( - table.getByProvenanceKey(forkSessionIdValue, `fork:${parentSessionId}:${forkId}:start`), - parentSessionId, - forkId, - forkSessionIdValue - ) - - const forkHeadEntryId = table.getMaxEntryId(forkSessionIdValue) - const forkEntries = table - .getBySessionUpToEntryId(forkSessionIdValue, forkHeadEntryId) - .filter( - (entry) => - !( - entry.kind === 'anchor' && - (entry.name === 'session/start' || entry.name === 'fork/start') - ) - ) - - for (const entry of forkEntries) { - table.append({ - sessionId: parentSessionId, - kind: entry.kind, - name: entry.name, - source: { - type: 'fork', - id: forkId, - seq: entry.entry_id - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:merge:${entry.entry_id}`, - payload: parseJsonObject(entry.payload_json), - meta: { - ...parseJsonObject(entry.meta_json), - forkId, - forkSessionId: forkSessionIdValue, - mergedFromEntryId: entry.entry_id - }, - createdAt: entry.created_at, - idempotent: true - }) - } - - table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/merge', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: mergeProvenanceKey, - data: { - forkId, - forkSessionId: forkSessionIdValue, - forkHeadEntryId, - mergedCount: forkEntries.length - }, - idempotent: true - }) - - return forkEntries.length - }) - } - - discardFork(parentSessionId: string, forkId: string): void { - const table = this.table - const forkSessionIdValue = forkSessionId(parentSessionId, forkId) - table.deleteBySession(forkSessionIdValue) - try { - this.searchProjectionTable.deleteBySession(forkSessionIdValue) - } catch (error) { - logger.warn(`[Tape] failed to delete fork search projection: ${String(error)}`) - } - table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/discard', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:discard:event`, - data: { - forkId, - forkSessionId: forkSessionIdValue - }, - idempotent: true - }) - } - - recordExternalForkMerge( - parentSessionId: string, - forkSessionIdValue: string, - forkId: string, - meta: Record = {} - ): DeepChatTapeEntryRow { - const table = this.table - const referencedEntryCount = table.countBySession(forkSessionIdValue) - return table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/merge', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:external-merge:event`, - data: { - forkId, - forkSessionId: forkSessionIdValue, - referencedEntryCount, - ...meta - }, - idempotent: true - }) - } - - recordExternalForkDiscard( - parentSessionId: string, - forkSessionIdValue: string, - forkId: string, - meta: Record = {} - ): DeepChatTapeEntryRow { - const table = this.table - return table.appendEvent({ - sessionId: parentSessionId, - name: 'fork/discard', - source: { - type: 'fork', - id: forkId, - seq: 0 - }, - provenanceKey: `fork:${parentSessionId}:${forkId}:external-discard:event`, - data: { - forkId, - forkSessionId: forkSessionIdValue, - ...meta - }, - idempotent: true - }) - } - - linkSubagentTape(input: SubagentTapeLinkInput): SubagentTapeLinkReceipt { - const table = this.table - const normalized = normalizeSubagentTapeLinkInput(input) - const provenanceKey = subagentTapeLinkProvenanceKey(normalized) - return table.runInTransaction(() => { - const existing = table.getByProvenanceKey(normalized.parentSessionId, provenanceKey) - if (existing) { - assertSubagentTapeLinkMatchesInput(existing, normalized) - const receipt = toSubagentTapeLinkReceipt(existing) - return receipt - } - - const childFirstEntry = table.getFirstEntriesBySessions([normalized.childSessionId])[0] - if (!childFirstEntry || childFirstEntry.session_id !== normalized.childSessionId) { - throw new Error(`Subagent Tape ${normalized.childSessionId} is unavailable.`) - } - const childTapeIdentity = computeTapeIdentity(childFirstEntry) - const childHeadEntryId = table.getMaxEntryId(normalized.childSessionId) - const childEntryCount = table.countBySession(normalized.childSessionId) - const row = table.appendEvent({ - sessionId: normalized.parentSessionId, - name: SUBAGENT_TAPE_LINK_EVENT_NAME, - source: { - type: 'subagent', - id: normalized.childSessionId, - seq: childHeadEntryId - }, - provenanceKey, - data: { - linkVersion: SUBAGENT_TAPE_LINK_VERSION, - childSessionId: normalized.childSessionId, - childHeadEntryId, - childEntryCount, - childTapeIdentity, - runId: normalized.runId, - taskId: normalized.taskId, - slotId: normalized.slotId, - taskTitle: normalized.taskTitle, - outcome: normalized.outcome, - resultSummary: normalized.resultSummary - }, - idempotent: true - }) - assertSubagentTapeLinkMatchesInput(row, normalized) - const receipt = toSubagentTapeLinkReceipt(row) - return receipt - }) - } - - private backfillLegacySummaryAnchor( - sessionId: string, - historyRecords: ChatMessageRecord[] - ): void { - const table = this.table - if (table.getLatestSummaryAnchor(sessionId)) { - return - } - - const legacyState = this.database.deepchatSessionsTable.getSummaryState(sessionId) - if (!legacyState) { - return - } - - const summary = legacyState.summary_text?.trim() - if (!summary) { - return - } - - const cursorOrderSeq = Math.max(1, legacyState.summary_cursor_order_seq ?? 1) - const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) - table.appendAnchor({ - sessionId, - name: 'compaction/migrated_summary', - source: { - type: 'summary', - id: 'legacy-summary', - seq: 1 - }, - provenanceKey: legacySummaryProvenanceKey(sessionId), - state: { - summary, - cursorOrderSeq, - range: - sourceRecords.length > 0 - ? { - fromOrderSeq: sourceRecords[0].orderSeq, - toOrderSeq: sourceRecords[sourceRecords.length - 1].orderSeq - } - : null, - sourceMessageIds: sourceRecords.map((record) => record.id), - migratedFrom: 'deepchat_sessions.summary_text' - }, - idempotent: true, - createdAt: legacyState.summary_updated_at ?? undefined - }) - } - - private ensureSearchProjection( - sessionId: string, - rows: DeepChatTapeEntryRow[], - effectiveRows: DeepChatTapeEntryRow[] - ): SessionDatabase['deepchatTapeSearchProjectionTable'] | null { - const projectionTable = this.searchProjectionTable - const maxEntryId = rows.reduce((max, row) => Math.max(max, row.entry_id), 0) - try { - if (!projectionTable.isCurrent(sessionId, maxEntryId)) { - const meta = projectionTable.getSessionMeta(sessionId) - const projectedEntryIds = projectionTable.getProjectedEntryIds(sessionId) - const effectiveEntryIds = effectiveRows.map((row) => row.entry_id) - const canAppend = - !!meta && - projectionTable.isCurrent(sessionId, meta.maxEntryId) && - meta.maxEntryId <= maxEntryId && - isEntryIdPrefix(projectedEntryIds, effectiveEntryIds) - if (canAppend) { - projectionTable.appendSession( - sessionId, - effectiveRows.slice(projectedEntryIds.length).map((row) => this.toProjectionInput(row)), - maxEntryId - ) - } else { - projectionTable.replaceSession( - sessionId, - effectiveRows.map((row) => this.toProjectionInput(row)), - maxEntryId - ) - } - } - return projectionTable - } catch { - return null - } - } - - private toProjectionInput(row: DeepChatTapeEntryRow): DeepChatTapeSearchProjectionInput { - const payload = parseJsonObject(row.payload_json) - const meta = parseJsonObject(row.meta_json) - const userMessage = getUserMessageProjectionText(row, payload) - const summaryText = summarizeTapeRow(row, payload, userMessage) - const evidenceText = buildTapeRowEvidenceText(row, payload, meta, userMessage) - const refs = buildTapeRowRefs(row, payload, meta, userMessage, evidenceText) - const searchText = [ - row.kind, - row.name ?? '', - summaryText, - evidenceText, - Object.values(refs) - .map((value) => stringifyForSummary(value)) - .join(' ') - ] - .filter(Boolean) - .join('\n') - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - sourceType: row.source_type, - sourceId: row.source_id, - sourceSeq: row.source_seq, - searchText, - summaryText, - refs, - createdAt: row.created_at - } - } - - private toProjectedSearchResult( - row: DeepChatTapeSearchProjectionResultRow, - _sourceRow: DeepChatTapeEntryRow | undefined - ): TapeSearchResult { - const score = - typeof row.score === 'number' && Number.isFinite(row.score) ? row.score : undefined - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - createdAt: row.created_at, - summary: row.summary_text, - refs: parseProjectionRefs(row.refs_json), - ...(score === undefined ? {} : { score }) - } - } - - private toContextEntry( - row: DeepChatTapeEntryRow, - projectionRow: DeepChatTapeSearchProjectionRow | undefined, - maxBytes: number - ): AgentTapeContextEntry { - const fallbackProjection = projectionRow ? null : this.toProjectionInput(row) - const payload = parseJsonObject(row.payload_json) - const meta = parseJsonObject(row.meta_json) - const evidenceSource = buildTapeRowEvidenceText(row, payload, meta) - const clipped = truncateToUtf8Bytes(evidenceSource, maxBytes) - const bytes = Buffer.byteLength(clipped.text, 'utf8') - return { - entryId: row.entry_id, - kind: row.kind, - name: row.name, - summary: projectionRow?.summary_text ?? fallbackProjection?.summaryText ?? '', - refs: projectionRow - ? parseProjectionRefs(projectionRow.refs_json) - : (fallbackProjection?.refs ?? {}), - evidence: { - text: clipped.text, - truncated: clipped.truncated, - bytes - }, - createdAt: row.created_at - } - } - - private toSearchResult(row: DeepChatTapeEntryRow): TapeSearchResult { - const projection = this.toProjectionInput(row) - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - createdAt: row.created_at, - summary: projection.summaryText, - refs: projection.refs - } - } - - private toAnchorResult(row: DeepChatTapeEntryRow): TapeAnchorResult { - return { - sessionId: row.session_id, - entryId: row.entry_id, - kind: row.kind, - name: row.name, - payload: parseJsonObject(row.payload_json), - meta: parseJsonObject(row.meta_json), - createdAt: row.created_at - } - } - - private toViewManifestRecord(row: DeepChatTapeEntryRow): DeepChatTapeViewManifestRecord | null { - const payload = parseJsonObject(row.payload_json) - const data = payload.data - const rawManifest = - data && typeof data === 'object' && !Array.isArray(data) - ? (data as Record).manifest - : undefined - const manifest = - isRecordObject(rawManifest) && rawManifest.hashVersion === undefined - ? { ...rawManifest, hashVersion: 1 } - : rawManifest - if (!isViewManifest(manifest, row.session_id)) { - return null - } - - return { - sessionId: row.session_id, - messageId: manifest.messageId, - requestSeq: manifest.requestSeq, - entryId: row.entry_id, - createdAt: row.created_at, - integrity: verifyTapeViewManifestHash(manifest), - manifest - } - } - - private findReplayTrace( - sessionId: string, - messageId: string, - requestSeq: number - ): DeepChatMessageTraceRow | null { - const traceTable = this.database.deepchatMessageTracesTable - return ( - traceTable - .listByMessageId(messageId) - .find((row) => row.session_id === sessionId && row.request_seq === requestSeq) ?? null - ) - } - - private toReplayEntrySnapshot( - row: DeepChatTapeEntryRow, - includePayloads: boolean - ): DeepChatTapeReplayEntrySnapshot { - const snapshot: DeepChatTapeReplayEntrySnapshot = { - entryId: row.entry_id, - kind: row.kind, - name: row.name, - sourceType: row.source_type, - sourceId: row.source_id, - sourceSeq: row.source_seq, - provenanceKey: row.provenance_key, - payloadHash: hashString(row.payload_json), - metaHash: hashString(row.meta_json), - createdAt: row.created_at - } - - if (includePayloads) { - snapshot.payload = parseJsonObject(row.payload_json) - snapshot.meta = parseJsonObject(row.meta_json) - } - - return snapshot - } - - private toReplayTraceSnapshot( - row: DeepChatMessageTraceRow, - includePayload: boolean - ): DeepChatTapeReplayTraceSnapshot { - const snapshot: DeepChatTapeReplayTraceSnapshot = { - id: row.id, - requestSeq: row.request_seq, - providerId: row.provider_id, - modelId: row.model_id, - endpoint: row.endpoint, - headersHash: hashString(row.headers_json), - bodyHash: hashString(row.body_json), - truncated: row.truncated === 1, - createdAt: row.created_at - } - - if (includePayload) { - snapshot.headersJson = row.headers_json - snapshot.bodyJson = row.body_json - } - - return snapshot - } -} +/** @deprecated Import Tape application APIs from `@/tape/application/sessionTape`. */ +export { + AgentTapeViewError, + normalizeSubagentTapeLinkInput, + normalizeTapeHandoffState, + SessionTape +} from '@/tape/application/sessionTape' + +/** @deprecated Import Tape application types from `@/tape/application/sessionTape`. */ +export type { + AgentTapeViewErrorCode, + TapeAnchorResult, + TapeBackfillResult, + TapeForkHandle, + TapeInfo, + TapeMigrationState, + TapeSearchResult, + TapeViewManifestAssemblySources as TapeViewManifestSourceMaps +} from '@/tape/application/sessionTape' diff --git a/src/main/session/data/tapeEffectiveView.ts b/src/main/session/data/tapeEffectiveView.ts index b49135e2bc..a8ee4faa81 100644 --- a/src/main/session/data/tapeEffectiveView.ts +++ b/src/main/session/data/tapeEffectiveView.ts @@ -1,255 +1,9 @@ -import type { ChatMessageRecord } from '@shared/types/agent-interface' -import type { - DeepChatTapeEntryKind, - DeepChatTapeEntryRow, - DeepChatTapeSearchInput -} from '@/session/data/tables/deepchatTapeEntries' -import { - parseNestedTapeJsonObject, - readTapeMessageRetractionId, - readTapeToolIdentity, - tapeEntryToMessageRecord, - tapeMessageRank, - tapeToolRank -} from '@/session/data/tables/deepchatTapeEffectiveSemantics' - -export interface EffectiveMessageEntry { - entryId: number - record: ChatMessageRecord -} - -export interface EffectiveTapeView { - rows: DeepChatTapeEntryRow[] - messageRecords: ChatMessageRecord[] - /** Effective messages paired with their tape entry_id, ordered by orderSeq (for lineage). */ - messageEntries: EffectiveMessageEntry[] -} - -interface EffectiveTapeViewOptions { - includePending?: boolean - includeAuditEvents?: boolean -} - -type EffectiveMessageCandidate = { - row: DeepChatTapeEntryRow - record: ChatMessageRecord -} - -function compareSqliteBinaryText(left: string, right: string): number { - return Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')) -} - -function toNonNegativeInteger(value: unknown): number | null { - if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { - return null - } - return Math.floor(value) -} - -function readTokenUsage(metadata: Record): number | null { - const totalTokens = toNonNegativeInteger(metadata.totalTokens ?? metadata.total_tokens) - if (totalTokens !== null) { - return totalTokens - } - - const inputTokens = toNonNegativeInteger(metadata.inputTokens ?? metadata.input_tokens) - const outputTokens = toNonNegativeInteger(metadata.outputTokens ?? metadata.output_tokens) - if (inputTokens !== null || outputTokens !== null) { - return (inputTokens ?? 0) + (outputTokens ?? 0) - } - - return null -} - -function shouldReplaceMessage( - current: EffectiveMessageCandidate | undefined, - next: EffectiveMessageCandidate, - includePending: boolean -): boolean { - if (!current) { - return true - } - - const currentRank = tapeMessageRank(current.record, includePending) - const nextRank = tapeMessageRank(next.record, includePending) - if (nextRank > currentRank) { - return true - } - if (nextRank < currentRank) { - return false - } - return next.row.entry_id > current.row.entry_id -} - -function isAuditEvent(row: DeepChatTapeEntryRow): boolean { - return ( - row.name === 'message/retracted' || - row.name === 'message/compaction_indicator' || - row.name === 'migration/backfill' - ) -} - -function shouldReplaceToolRow( - current: DeepChatTapeEntryRow | undefined, - next: DeepChatTapeEntryRow, - includePending: boolean -): boolean { - if (!current) { - return true - } - - const currentRank = tapeToolRank(current, includePending) - const nextRank = tapeToolRank(next, includePending) - if (nextRank > currentRank) { - return true - } - if (nextRank < currentRank) { - return false - } - return next.entry_id > current.entry_id -} - -function matchesKinds( - row: DeepChatTapeEntryRow, - kinds: DeepChatTapeEntryKind[] | undefined -): boolean { - return !kinds?.length || kinds.includes(row.kind) -} - -function matchesCreatedAt(row: DeepChatTapeEntryRow, options: DeepChatTapeSearchInput): boolean { - if ( - Number.isFinite(options.startCreatedAt) && - row.created_at < (options.startCreatedAt as number) - ) { - return false - } - if (Number.isFinite(options.endCreatedAt) && row.created_at > (options.endCreatedAt as number)) { - return false - } - return true -} - -function matchesQuery(row: DeepChatTapeEntryRow, normalizedQuery: string): boolean { - const haystack = `${row.payload_json}\n${row.meta_json}\n${row.name ?? ''}`.toLowerCase() - return haystack.includes(normalizedQuery) -} - -export function buildEffectiveTapeView( - rows: DeepChatTapeEntryRow[], - options: EffectiveTapeViewOptions = {} -): EffectiveTapeView { - const includePending = options.includePending === true - const includeAuditEvents = options.includeAuditEvents === true - const messageCandidates = new Map() - const retractedMessageIds = new Set() - const toolRows = new Map() - const anchorRows: DeepChatTapeEntryRow[] = [] - const eventRows: DeepChatTapeEntryRow[] = [] - - for (const row of [...rows].sort((left, right) => left.entry_id - right.entry_id)) { - if (row.kind === 'anchor') { - anchorRows.push(row) - continue - } - - if (row.kind === 'event') { - const retractedMessageId = readTapeMessageRetractionId(row) - if (retractedMessageId) { - messageCandidates.delete(retractedMessageId) - retractedMessageIds.add(retractedMessageId) - } - if (includeAuditEvents || !isAuditEvent(row)) { - eventRows.push(row) - } - continue - } - - if (row.kind === 'message') { - const record = tapeEntryToMessageRecord(row) - if (!record) { - continue - } - const rank = tapeMessageRank(record, includePending) - if (rank === 0) { - continue - } - const candidate = { row, record } - if (shouldReplaceMessage(messageCandidates.get(record.id), candidate, includePending)) { - messageCandidates.set(record.id, candidate) - retractedMessageIds.delete(record.id) - } - continue - } - - const identity = readTapeToolIdentity(row) - if (!identity || tapeToolRank(row, includePending) === 0) { - continue - } - const current = toolRows.get(identity.key)?.row - if (shouldReplaceToolRow(current, row, includePending)) { - toolRows.set(identity.key, { row, messageId: identity.messageId }) - } - } - - const messageRows = [...messageCandidates.values()] - .filter((candidate) => !retractedMessageIds.has(candidate.record.id)) - .sort( - (left, right) => - left.record.orderSeq - right.record.orderSeq || - compareSqliteBinaryText(left.record.id, right.record.id) - ) - const effectiveMessageIds = new Set(messageRows.map((candidate) => candidate.record.id)) - const effectiveToolRows = [...toolRows.values()] - .filter((candidate) => effectiveMessageIds.has(candidate.messageId)) - .map((candidate) => candidate.row) - const effectiveRows = [ - ...anchorRows, - ...eventRows, - ...messageRows.map((candidate) => candidate.row), - ...effectiveToolRows - ].sort((left, right) => left.entry_id - right.entry_id) - - return { - rows: effectiveRows, - messageRecords: messageRows.map((candidate) => candidate.record), - messageEntries: messageRows.map((candidate) => ({ - entryId: candidate.row.entry_id, - record: candidate.record - })) - } -} - -export function searchEffectiveTapeRows( - rows: DeepChatTapeEntryRow[], - query: string, - options: DeepChatTapeSearchInput = {} -): DeepChatTapeEntryRow[] { - const normalizedQuery = query.trim().toLowerCase() - if (!normalizedQuery) { - return [] - } - - const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 - const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) - return buildEffectiveTapeView(rows, { includePending: false }) - .rows.filter((row) => matchesKinds(row, options.kinds)) - .filter((row) => matchesCreatedAt(row, options)) - .filter((row) => matchesQuery(row, normalizedQuery)) - .sort((left, right) => right.entry_id - left.entry_id) - .slice(0, cappedLimit) -} - -export function getLastEffectiveTokenUsage(rows: DeepChatTapeEntryRow[]): number | null { - const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows - for (let index = effectiveRows.length - 1; index >= 0; index -= 1) { - const record = tapeEntryToMessageRecord(effectiveRows[index]) - if (!record || record.role !== 'assistant') { - continue - } - const usage = readTokenUsage(parseNestedTapeJsonObject(record.metadata)) - if (usage !== null) { - return usage - } - } - return null -} +/** @deprecated Import effective Tape helpers from `@/tape/domain/effectiveView`. */ +export { + buildEffectiveTapeView, + getLastEffectiveTokenUsage, + searchEffectiveTapeRows +} from '@/tape/domain/effectiveView' + +/** @deprecated Import effective Tape types from `@/tape/domain/effectiveView`. */ +export type { EffectiveMessageEntry, EffectiveTapeView } from '@/tape/domain/effectiveView' diff --git a/src/main/session/data/tapeFacts.ts b/src/main/session/data/tapeFacts.ts index e8e4028124..9d06afbce1 100644 --- a/src/main/session/data/tapeFacts.ts +++ b/src/main/session/data/tapeFacts.ts @@ -1,352 +1,14 @@ -import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' -import type { TapeToolFactInput } from '@/agent/deepchat/loop/ports' -import { toAppSessionId } from '@/agent/shared/agentSessionIds' -import type { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' -import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' -import { buildEffectiveTapeView } from './tapeEffectiveView' -import { hashJson } from './tapeViewManifest' -import { parseAssistantBlocks } from '@/session/data/tables/deepchatTapeEffectiveSemantics' - -export { tapeEntryToMessageRecord } from '@/session/data/tables/deepchatTapeEffectiveSemantics' - -export type TapeFactSource = 'live' | 'backfill' | 'repair' - -function readCompactionStatus(record: ChatMessageRecord): string | null { - try { - const parsed = JSON.parse(record.metadata) as { - messageType?: string - compactionStatus?: unknown - } - if (parsed.messageType !== 'compaction') { - return null - } - return typeof parsed.compactionStatus === 'string' ? parsed.compactionStatus : record.status - } catch { - return null - } -} - -function shouldUseRevisionProvenance(record: ChatMessageRecord, source: TapeFactSource): boolean { - return source === 'repair' || record.status !== 'sent' -} - -function buildMessageProvenanceKey( - record: ChatMessageRecord, - source: TapeFactSource -): string | undefined { - if (!shouldUseRevisionProvenance(record, source)) { - return undefined - } - return `message:${record.id}:revision:${record.status}:${record.updatedAt}` -} - -function buildToolFactProvenanceKey( - kind: 'tool_call' | 'tool_result', - messageId: string, - toolCallId: string, - payload: Record -): string { - return `${kind}:${messageId}:${toolCallId}:${hashJson(payload)}` -} - -function collectPendingInteractionToolIds(blocks: AssistantMessageBlock[]): Set { - const ids = new Set() - for (const block of blocks) { - if ( - block.type === 'action' && - (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && - block.status === 'pending' && - typeof block.tool_call?.id === 'string' && - block.tool_call.id.length > 0 - ) { - ids.add(block.tool_call.id) - } - } - return ids -} - -export function buildTapeToolFactInputs(record: ChatMessageRecord): TapeToolFactInput[] { - if (record.role !== 'assistant') return [] - - const inputs: TapeToolFactInput[] = [] - const blocks = parseAssistantBlocks(record.content) - const pendingInteractionToolIds = collectPendingInteractionToolIds(blocks) - blocks.forEach((block, blockIndex) => { - if (block.type !== 'tool_call' || !block.tool_call) return - if (block.status !== 'success' && block.status !== 'error') return - - const toolCallId = block.tool_call.id - if (!toolCallId || pendingInteractionToolIds.has(toolCallId)) return - const sourceId = `${record.id}:${toolCallId}` - const factBlock = - block.timestamp === undefined ? { ...block, timestamp: record.updatedAt } : block - inputs.push({ - sessionId: toAppSessionId(record.sessionId), - messageId: record.id, - orderSeq: record.orderSeq, - blockIndex, - block: factBlock, - provenance: { source: 'tool_call', sourceId, sequence: blockIndex } - }) - if (typeof block.tool_call.response === 'string' && block.tool_call.response.length > 0) { - inputs.push({ - sessionId: toAppSessionId(record.sessionId), - messageId: record.id, - orderSeq: record.orderSeq, - blockIndex, - block: factBlock, - provenance: { source: 'tool_result', sourceId, sequence: blockIndex } - }) - } - }) - return inputs -} - -export function appendTapeToolFact( - table: DeepChatTapeEntriesTable, - input: TapeToolFactInput, - source: TapeFactSource, - reason?: string -): DeepChatTapeEntryRow | null { - const block = input.block - const toolCall = block.type === 'tool_call' ? block.tool_call : undefined - if ( - !toolCall?.id || - (block.status !== 'success' && block.status !== 'error') || - (input.provenance.source !== 'tool_call' && input.provenance.source !== 'tool_result') - ) { - return null - } - - table.ensureBootstrapAnchor(input.sessionId) - const meta = reason - ? { source, role: 'assistant', status: block.status, reason } - : { source, role: 'assistant', status: block.status } - if (input.provenance.source === 'tool_call') { - const payload = { - messageId: input.messageId, - orderSeq: input.orderSeq, - toolCall: { - id: toolCall.id, - name: toolCall.name, - params: toolCall.params, - serverName: toolCall.server_name, - serverIcons: toolCall.server_icons, - serverDescription: toolCall.server_description - } - } - return table.append({ - sessionId: input.sessionId, - kind: 'tool_call', - name: toolCall.name || 'unknown', - source: { - type: input.provenance.source, - id: input.provenance.sourceId, - seq: input.provenance.sequence - }, - provenanceKey: buildToolFactProvenanceKey('tool_call', input.messageId, toolCall.id, payload), - payload, - meta, - createdAt: block.timestamp, - idempotent: true - }) - } - - if (typeof toolCall.response !== 'string' || toolCall.response.length === 0) return null - const payload = { - messageId: input.messageId, - orderSeq: input.orderSeq, - toolCallId: toolCall.id, - response: toolCall.response, - rtkApplied: toolCall.rtkApplied, - rtkMode: toolCall.rtkMode, - rtkFallbackReason: toolCall.rtkFallbackReason, - imagePreviews: toolCall.imagePreviews - } - return table.append({ - sessionId: input.sessionId, - kind: 'tool_result', - name: toolCall.name || 'unknown', - source: { - type: input.provenance.source, - id: input.provenance.sourceId, - seq: input.provenance.sequence - }, - provenanceKey: buildToolFactProvenanceKey('tool_result', input.messageId, toolCall.id, payload), - payload, - meta, - createdAt: block.timestamp, - idempotent: true - }) -} - -export function appendToolFactsToTape( - table: DeepChatTapeEntriesTable, - record: ChatMessageRecord, - source: TapeFactSource, - reason?: string -): number { - if (record.role !== 'assistant') { - return 0 - } - - return buildTapeToolFactInputs(record).reduce( - (appended, input) => appended + (appendTapeToolFact(table, input, source, reason) ? 1 : 0), - 0 - ) -} - -export function appendMessageRecordToTape( - table: DeepChatTapeEntriesTable, - record: ChatMessageRecord, - source: TapeFactSource -): number { - table.ensureBootstrapAnchor(record.sessionId) - - const compactionStatus = readCompactionStatus(record) - if (compactionStatus) { - table.appendEvent({ - sessionId: record.sessionId, - name: 'message/compaction_indicator', - source: { - type: 'message', - id: record.id, - seq: record.updatedAt - }, - provenanceKey: `message:${record.id}:compaction_indicator:${compactionStatus}:${record.updatedAt}`, - data: { - messageId: record.id, - orderSeq: record.orderSeq, - status: compactionStatus, - metadata: record.metadata - }, - meta: { - source, - status: compactionStatus - }, - createdAt: record.updatedAt, - idempotent: true - }) - return 1 - } - - table.append({ - sessionId: record.sessionId, - kind: 'message', - name: `message/${record.role}`, - source: { - type: 'message', - id: record.id, - seq: 0 - }, - provenanceKey: buildMessageProvenanceKey(record, source), - payload: { - record: { - id: record.id, - sessionId: record.sessionId, - orderSeq: record.orderSeq, - role: record.role, - content: record.content, - status: record.status, - isContextEdge: record.isContextEdge, - metadata: record.metadata, - traceCount: record.traceCount, - createdAt: record.createdAt, - updatedAt: record.updatedAt - } - }, - meta: { - source, - orderSeq: record.orderSeq, - role: record.role, - status: record.status - }, - createdAt: record.createdAt, - idempotent: true - }) - - return 1 + appendToolFactsToTape(table, record, source) -} - -export function appendMessageReplacementToTape( - table: DeepChatTapeEntriesTable, - record: ChatMessageRecord, - reason: string -): number { - table.ensureBootstrapAnchor(record.sessionId) - table.append({ - sessionId: record.sessionId, - kind: 'message', - name: `message/${record.role}`, - source: { - type: 'message', - id: record.id, - seq: record.updatedAt - }, - provenanceKey: `message:${record.id}:revision:${record.updatedAt}`, - payload: { - record: { - id: record.id, - sessionId: record.sessionId, - orderSeq: record.orderSeq, - role: record.role, - content: record.content, - status: record.status, - isContextEdge: record.isContextEdge, - metadata: record.metadata, - traceCount: record.traceCount, - createdAt: record.createdAt, - updatedAt: record.updatedAt - } - }, - meta: { - source: 'live', - correction: true, - reason, - orderSeq: record.orderSeq, - role: record.role, - status: record.status - }, - createdAt: record.updatedAt, - idempotent: true - }) - - return 1 + appendToolFactsToTape(table, record, 'repair') -} - -export function appendMessageRetractionToTape( - table: DeepChatTapeEntriesTable, - record: ChatMessageRecord, - reason: string -): number { - table.ensureBootstrapAnchor(record.sessionId) - table.appendEvent({ - sessionId: record.sessionId, - name: 'message/retracted', - source: { - type: 'message', - id: record.id, - seq: Date.now() - }, - provenanceKey: null, - data: { - messageId: record.id, - orderSeq: record.orderSeq, - role: record.role, - reason - }, - meta: { - source: 'live', - correction: true - }, - idempotent: false - }) - - return 1 -} - -export function tapeEntriesToEffectiveMessageRecords( - rows: DeepChatTapeEntryRow[] -): ChatMessageRecord[] { - return buildEffectiveTapeView(rows, { includePending: true }).messageRecords -} +/** @deprecated Import Tape fact helpers from `@/tape/application/factPersistence`. */ +export { + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + appendTapeToolFact, + appendToolFactsToTape, + buildTapeToolFactInputs, + tapeEntriesToEffectiveMessageRecords, + tapeEntryToMessageRecord +} from '@/tape/application/factPersistence' + +/** @deprecated Import Tape fact types from `@/tape/application/factPersistence`. */ +export type { TapeFactSource } from '@/tape/application/factPersistence' diff --git a/src/main/session/data/tapeViewManifest.ts b/src/main/session/data/tapeViewManifest.ts index 95794a955d..529d6f7707 100644 --- a/src/main/session/data/tapeViewManifest.ts +++ b/src/main/session/data/tapeViewManifest.ts @@ -1,362 +1,25 @@ -import { createHash } from 'crypto' -import type { ChatMessage } from '@shared/types/core/chat-message' -import type { MCPToolDefinition } from '@shared/types/core/mcp' -import type { ChatMessageRecord } from '@shared/types/agent-interface' -import type { - DeepChatTapeViewEntryRef, - DeepChatTapeViewExcludedRange, - DeepChatTapeViewExcludedRef, - DeepChatTapeViewManifest, - DeepChatTapeViewManifestIntegrity, - DeepChatTapeViewPolicy, - DeepChatTapeViewTaskType, - DeepChatTapeViewTokenBudget -} from '@shared/types/tape-view-manifest' -import { estimateMessagesTokens } from '@shared/utils/messageTokens' - -export type ContextSummaryCursorMetadata = { - summaryCursorOrderSeq: number - preCursorOrderSeqMin: number | null - preCursorOrderSeqMax: number | null - preCursorCount: number -} - -export function isCompactionRecord(record: ChatMessageRecord): boolean { - try { - const metadata = JSON.parse(record.metadata) as { messageType?: string } - return metadata.messageType === 'compaction' - } catch { - return false - } -} - -export const TAPE_VIEW_MANIFEST_EVENT_NAME = 'view/assembled' -export const TAPE_VIEW_CONTEXT_BUILDER_VERSION = 'legacy-v1' as const - -export type TapeViewManifestSourceMaps = { - entryIdByMessageId?: Map - toolCallEntryIdByToolId?: Map - toolResultEntryIdByToolId?: Map -} - -export type TapeViewManifestBuildInput = { - sessionId: string - messageId: string - requestSeq: number - taskType: DeepChatTapeViewTaskType - policy: DeepChatTapeViewPolicy - policyVersion?: number | null - messages: ChatMessage[] - tools: MCPToolDefinition[] - latestEntryId: number - anchorEntryIds: number[] - reconstructionAnchorEntryId?: number | null - included: DeepChatTapeViewEntryRef[] - excluded: DeepChatTapeViewExcludedRef[] - summaryCursor?: ContextSummaryCursorMetadata - tokenBudget: Omit - providerId: string - modelId: string - summaryCursorOrderSeq: number - supportsVision: boolean - supportsAudioInput: boolean - traceDebugEnabled: boolean - assembledAt?: number -} - -export type TapeViewManifestPolicyInput = { - recoveredFromContextPressure: boolean - isInitialViewRequest: boolean - viewPolicy?: DeepChatTapeViewPolicy - viewPolicyVersion?: number | null -} - -export type TapeViewManifestPolicyResult = { - policy: DeepChatTapeViewPolicy - policyVersion: number | null -} - -export type TapeViewContextSelection = { - includedRecords: Array<{ - record: ChatMessageRecord - reason: DeepChatTapeViewEntryRef['reason'] - }> - excludedRecords: Array<{ - record: ChatMessageRecord - reason: DeepChatTapeViewExcludedRef['reason'] - }> - summaryCursor?: ContextSummaryCursorMetadata - includesSystemPrompt: boolean - newUserMessageId?: string | null -} - -export function resolveTapeViewManifestPolicy( - input: TapeViewManifestPolicyInput -): TapeViewManifestPolicyResult { - if (input.recoveredFromContextPressure) { - return { - policy: 'context_pressure_recovery_shadow', - policyVersion: null - } - } - - if (input.isInitialViewRequest && input.viewPolicy) { - return { - policy: input.viewPolicy, - policyVersion: input.viewPolicyVersion ?? null - } - } - - return { - policy: 'tool_loop_shadow', - policyVersion: null - } -} - -function normalizeForStableJson(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map(normalizeForStableJson) - } - - if (!value || typeof value !== 'object') { - return value - } - - const record = value as Record - return Object.keys(record) - .sort() - .reduce>((result, key) => { - const nested = record[key] - if (nested !== undefined) { - result[key] = normalizeForStableJson(nested) - } - return result - }, {}) -} - -export function stableJsonStringify(value: unknown): string { - return JSON.stringify(normalizeForStableJson(value)) -} - -export function hashJson(value: unknown): string { - return createHash('sha256').update(stableJsonStringify(value)).digest('hex') -} - -export const TAPE_VIEW_MANIFEST_HASH_VERSION = 2 - -function buildManifestHash(manifest: DeepChatTapeViewManifest): string { - const hashable: Record = { ...manifest } - delete hashable.assembledAt - delete hashable.viewId - hashable.hashes = { - promptHash: manifest.hashes.promptHash, - toolDefinitionsHash: manifest.hashes.toolDefinitionsHash - } - return hashJson(hashable) -} - -function buildExcludedRanges( - summaryCursor?: ContextSummaryCursorMetadata -): DeepChatTapeViewExcludedRange[] { - if ( - !summaryCursor || - summaryCursor.preCursorCount === 0 || - summaryCursor.preCursorOrderSeqMin === null || - summaryCursor.preCursorOrderSeqMax === null - ) { - return [] - } - return [ - { - fromOrderSeq: summaryCursor.preCursorOrderSeqMin, - toOrderSeq: summaryCursor.preCursorOrderSeqMax, - count: summaryCursor.preCursorCount, - reason: 'before_summary_cursor' - } - ] -} - -export function verifyTapeViewManifestHash( - manifest: DeepChatTapeViewManifest -): DeepChatTapeViewManifestIntegrity { - if (manifest.hashVersion !== TAPE_VIEW_MANIFEST_HASH_VERSION) { - return 'unverified' - } - return buildManifestHash(manifest) === manifest.hashes.manifestHash ? 'valid' : 'invalid' -} - -export function createTapeViewManifest( - input: TapeViewManifestBuildInput -): DeepChatTapeViewManifest { - const assembledAt = input.assembledAt ?? Date.now() - const excludedRanges = buildExcludedRanges(input.summaryCursor) - const draft: DeepChatTapeViewManifest = { - schemaVersion: 2, - hashVersion: TAPE_VIEW_MANIFEST_HASH_VERSION, - viewId: '', - sessionId: input.sessionId, - messageId: input.messageId, - requestSeq: input.requestSeq, - taskType: input.taskType, - policy: input.policy, - policyVersion: input.policyVersion ?? null, - contextBuilderVersion: TAPE_VIEW_CONTEXT_BUILDER_VERSION, - latestEntryId: input.latestEntryId, - anchorEntryIds: [...input.anchorEntryIds], - ...(input.reconstructionAnchorEntryId !== undefined - ? { reconstructionAnchorEntryId: input.reconstructionAnchorEntryId } - : {}), - included: input.included.map((entry) => ({ ...entry })), - excluded: input.excluded.map((entry) => ({ ...entry })), - ...(excludedRanges.length > 0 ? { excludedRanges } : {}), - tokenBudget: { - ...input.tokenBudget, - estimatedPromptTokens: estimateMessagesTokens(input.messages) - }, - hashes: { - promptHash: hashJson(input.messages), - toolDefinitionsHash: hashJson(input.tools), - manifestHash: '' - }, - meta: { - providerId: input.providerId, - modelId: input.modelId, - summaryCursorOrderSeq: input.summaryCursorOrderSeq, - supportsVision: input.supportsVision, - supportsAudioInput: input.supportsAudioInput, - traceDebugEnabled: input.traceDebugEnabled - }, - assembledAt - } - - const manifestHash = buildManifestHash(draft) - return { - ...draft, - viewId: `view_${manifestHash.slice(0, 16)}`, - hashes: { ...draft.hashes, manifestHash } - } -} - -export function buildIncludedRefs( - selection: TapeViewContextSelection, - sourceMaps: TapeViewManifestSourceMaps = {} -): DeepChatTapeViewEntryRef[] { - const refs: DeepChatTapeViewEntryRef[] = [] - - if (selection.includesSystemPrompt) { - refs.push({ - entryId: null, - messageId: null, - orderSeq: null, - role: 'system', - source: 'synthetic', - reason: 'system_prompt' - }) - } - - for (const item of selection.includedRecords) { - refs.push({ - entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, - messageId: item.record.id, - orderSeq: item.record.orderSeq, - role: item.record.role, - source: sourceMaps.entryIdByMessageId?.has(item.record.id) ? 'tape' : 'synthetic', - reason: item.reason - }) - } - - if (selection.newUserMessageId) { - refs.push({ - entryId: sourceMaps.entryIdByMessageId?.get(selection.newUserMessageId) ?? null, - messageId: selection.newUserMessageId, - orderSeq: null, - role: 'user', - source: sourceMaps.entryIdByMessageId?.has(selection.newUserMessageId) ? 'tape' : 'synthetic', - reason: 'new_user_input' - }) - } - - return refs -} - -export function buildExcludedRefs( - selection: TapeViewContextSelection, - sourceMaps: TapeViewManifestSourceMaps = {} -): DeepChatTapeViewExcludedRef[] { - return selection.excludedRecords.map((item) => ({ - entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, - messageId: item.record.id, - orderSeq: item.record.orderSeq, - reason: item.reason - })) -} - -export function buildRequestRefs( - messages: ChatMessage[], - sourceMaps: TapeViewManifestSourceMaps = {} -): DeepChatTapeViewEntryRef[] { - const lastToolCallIndex = new Map() - const lastToolResultIndex = new Map() - messages.forEach((message, index) => { - if (message.role === 'assistant' && message.tool_calls?.length) { - for (const toolCall of message.tool_calls) { - lastToolCallIndex.set(toolCall.id, index) - } - } else if (message.role === 'tool' && message.tool_call_id) { - lastToolResultIndex.set(message.tool_call_id, index) - } - }) - - const refs: DeepChatTapeViewEntryRef[] = [] - messages.forEach((message, index) => { - if (message.role === 'assistant' && message.tool_calls?.length) { - for (const toolCall of message.tool_calls) { - const entryId = - lastToolCallIndex.get(toolCall.id) === index - ? (sourceMaps.toolCallEntryIdByToolId?.get(toolCall.id) ?? null) - : null - refs.push({ - entryId, - messageId: null, - orderSeq: null, - role: 'assistant', - source: entryId === null ? 'synthetic' : 'tape', - reason: 'tool_loop_message' - }) - } - return - } - - if (message.role === 'tool' && message.tool_call_id) { - const entryId = - lastToolResultIndex.get(message.tool_call_id) === index - ? (sourceMaps.toolResultEntryIdByToolId?.get(message.tool_call_id) ?? null) - : null - refs.push({ - entryId, - messageId: null, - orderSeq: null, - role: 'tool', - source: entryId === null ? 'synthetic' : 'tape', - reason: 'tool_loop_message' - }) - return - } - - refs.push({ - entryId: null, - messageId: null, - orderSeq: null, - role: message.role, - source: 'synthetic', - reason: - message.role === 'system' - ? 'system_prompt' - : message.role === 'tool' - ? 'tool_loop_message' - : 'selected_history' - }) - }) - - return refs -} +/** @deprecated Import ViewManifest helpers from `@/tape/domain/viewManifest`. */ +export { + buildExcludedRefs, + buildIncludedRefs, + buildRequestRefs, + createTapeViewManifest, + hashJson, + isCompactionRecord, + resolveTapeViewManifestPolicy, + stableJsonStringify, + TAPE_VIEW_CONTEXT_BUILDER_VERSION, + TAPE_VIEW_MANIFEST_EVENT_NAME, + TAPE_VIEW_MANIFEST_HASH_VERSION, + verifyTapeViewManifestHash +} from '@/tape/domain/viewManifest' + +/** @deprecated Import ViewManifest types from `@/tape/domain/viewManifest`. */ +export type { + ContextSummaryCursorMetadata, + TapeViewContextSelection, + TapeViewManifestBuildInput, + TapeViewManifestLookupMaps as TapeViewManifestSourceMaps, + TapeViewManifestPolicyInput, + TapeViewManifestPolicyResult +} from '@/tape/domain/viewManifest' diff --git a/src/main/session/data/transcript.ts b/src/main/session/data/transcript.ts index 4e986d9e3d..f1d9764af3 100644 --- a/src/main/session/data/transcript.ts +++ b/src/main/session/data/transcript.ts @@ -23,11 +23,7 @@ import { resolveUsageModelId, resolveUsageProviderId } from '@/session/usageStats' -import { - appendMessageRecordToTape, - appendMessageReplacementToTape, - appendMessageRetractionToTape -} from '@/session/data/tapeFacts' +import type { TapeMessageFactWriter } from '@/tape/ports/capabilities' function shouldConvertPendingBlockToError( status: AssistantMessageBlock['status'] @@ -128,9 +124,11 @@ function extractSearchableMessageContent(rawContent: string): string { export class SessionTranscript { private database: SessionDatabase + private readonly tapeFacts: TapeMessageFactWriter - constructor(database: SessionDatabase) { + constructor(database: SessionDatabase, tapeFacts: TapeMessageFactWriter) { this.database = database + this.tapeFacts = tapeFacts } private runInDatabaseTransaction(operation: () => T): T { @@ -372,11 +370,7 @@ export class SessionTranscript { } const updated = this.getMessage(messageId) if (updated) { - appendMessageReplacementToTape( - this.database.deepchatTapeEntriesTable, - updated, - 'message_content_updated' - ) + this.tapeFacts.appendMessageReplacement(updated, 'message_content_updated') } return } @@ -394,11 +388,7 @@ export class SessionTranscript { } const updated = this.getMessage(messageId) if (updated) { - appendMessageReplacementToTape( - this.database.deepchatTapeEntriesTable, - updated, - 'message_content_updated' - ) + this.tapeFacts.appendMessageReplacement(updated, 'message_content_updated') } } @@ -421,11 +411,7 @@ export class SessionTranscript { this.runInDatabaseTransaction(() => { const record = this.getMessage(messageId) if (record) { - appendMessageRetractionToTape( - this.database.deepchatTapeEntriesTable, - record, - 'message_deleted' - ) + this.tapeFacts.appendMessageRetraction(record, 'message_deleted') } this.database.deepchatSearchDocumentsTable.delete(`message:${messageId}`) this.database.deepchatAssistantBlocksTable.delete(messageId) @@ -444,11 +430,7 @@ export class SessionTranscript { (record) => record.orderSeq >= fromOrderSeq ) for (const record of records) { - appendMessageRetractionToTape( - this.database.deepchatTapeEntriesTable, - record, - 'messages_deleted_from_order_seq' - ) + this.tapeFacts.appendMessageRetraction(record, 'messages_deleted_from_order_seq') } const messageIds = records.map((record) => record.id) if (messageIds.length > 0) { @@ -685,7 +667,7 @@ export class SessionTranscript { if (!record) { return } - appendMessageRecordToTape(this.database.deepchatTapeEntriesTable, record, 'live') + this.tapeFacts.appendMessageRecord(record) } private toRecord(row: DeepChatMessageRow): ChatMessageRecord { diff --git a/src/main/session/transcriptMutations.ts b/src/main/session/transcriptMutations.ts index eb15255dee..c3bbf21d27 100644 --- a/src/main/session/transcriptMutations.ts +++ b/src/main/session/transcriptMutations.ts @@ -20,6 +20,7 @@ export interface SessionTranscriptMutationDependencies { settings: SessionSettingsStore pendingInputs: SessionPendingInputs runtime: SessionTranscriptRuntimePort + runInTransaction(operation: () => T): T } export class SessionTranscriptMutations { @@ -27,9 +28,11 @@ export class SessionTranscriptMutations { async clearMessages(sessionId: string): Promise { await this.dependencies.runtime.prepareClearMessages(sessionId) - this.dependencies.pendingInputs.deleteBySession(sessionId) - this.dependencies.transcript.deleteBySession(sessionId) - this.dependencies.settings.resetTape(sessionId) + this.dependencies.runInTransaction(() => { + this.dependencies.pendingInputs.deleteBySession(sessionId) + this.dependencies.transcript.deleteBySession(sessionId) + this.dependencies.settings.resetTape(sessionId) + }) this.dependencies.runtime.finishClearMessages(sessionId) } diff --git a/src/main/tape/application/common.ts b/src/main/tape/application/common.ts new file mode 100644 index 0000000000..5eb330afa3 --- /dev/null +++ b/src/main/tape/application/common.ts @@ -0,0 +1,29 @@ +export function parseJsonObject(raw: string): Record { + try { + const parsed = JSON.parse(raw) as unknown + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record + } + } catch {} + return {} +} + +export function parseJsonValue(raw: string): unknown { + try { + return JSON.parse(raw) as unknown + } catch { + return null + } +} + +export function isEntryIdPrefix(prefix: number[], values: number[]): boolean { + if (prefix.length > values.length) return false + for (let index = 0; index < prefix.length; index += 1) { + if (prefix[index] !== values[index]) return false + } + return true +} + +export function migrationProvenanceKey(sessionId: string): string { + return `migration:${sessionId}:message-backfill:v1` +} diff --git a/src/main/tape/application/contracts.ts b/src/main/tape/application/contracts.ts new file mode 100644 index 0000000000..4c537623cb --- /dev/null +++ b/src/main/tape/application/contracts.ts @@ -0,0 +1,39 @@ +import type { AgentTapeAnchorResult } from '@shared/types/agent-interface' +import type { TapeMigrationState } from '../ports/capabilities' + +export type { + TapeBackfillResult, + TapeMigrationState, + TapeViewManifestAssemblySources +} from '../ports/capabilities' + +export type TapeInfo = { + sessionId: string + entries: number + anchors: number + lastAnchor: string | null + lastAnchorEntryId: number | null + entriesSinceLastAnchor: number + lastTokenUsage: number | null + migrationState: TapeMigrationState +} + +export type TapeSearchResult = { + sessionId: string + entryId: number + kind: string + name: string | null + createdAt: number + summary?: string + refs?: Record + score?: number +} + +export type TapeAnchorResult = AgentTapeAnchorResult + +export type TapeForkHandle = { + parentSessionId: string + forkId: string + forkSessionId: string + parentHeadEntryId: number +} diff --git a/src/main/tape/application/factPersistence.ts b/src/main/tape/application/factPersistence.ts new file mode 100644 index 0000000000..a320777677 --- /dev/null +++ b/src/main/tape/application/factPersistence.ts @@ -0,0 +1,352 @@ +import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' +import { toTapeSessionId, type TapeFactSource, type TapeToolFactInput } from '@/tape/domain/facts' +import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' +import type { TapeBootstrapStore, TapeEntryStore } from '@/tape/ports/storage' +import { buildEffectiveTapeView } from '@/tape/domain/effectiveView' +import { parseAssistantBlocks } from '@/tape/domain/effectiveSemantics' +import { hashJson } from '@/tape/domain/viewManifest' + +export { tapeEntryToMessageRecord } from '@/tape/domain/effectiveSemantics' +export type { TapeFactSource } from '@/tape/domain/facts' + +type TapeFactStore = Pick & TapeBootstrapStore + +function readCompactionStatus(record: ChatMessageRecord): string | null { + try { + const parsed = JSON.parse(record.metadata) as { + messageType?: string + compactionStatus?: unknown + } + if (parsed.messageType !== 'compaction') { + return null + } + return typeof parsed.compactionStatus === 'string' ? parsed.compactionStatus : record.status + } catch { + return null + } +} + +function shouldUseRevisionProvenance(record: ChatMessageRecord, source: TapeFactSource): boolean { + return source === 'repair' || record.status !== 'sent' +} + +function buildMessageProvenanceKey( + record: ChatMessageRecord, + source: TapeFactSource +): string | undefined { + if (!shouldUseRevisionProvenance(record, source)) { + return undefined + } + return `message:${record.id}:revision:${record.status}:${record.updatedAt}` +} + +function buildToolFactProvenanceKey( + kind: 'tool_call' | 'tool_result', + messageId: string, + toolCallId: string, + payload: Record +): string { + return `${kind}:${messageId}:${toolCallId}:${hashJson(payload)}` +} + +function collectPendingInteractionToolIds(blocks: AssistantMessageBlock[]): Set { + const ids = new Set() + for (const block of blocks) { + if ( + block?.type === 'action' && + (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && + block.status === 'pending' && + typeof block.tool_call?.id === 'string' && + block.tool_call.id.length > 0 + ) { + ids.add(block.tool_call.id) + } + } + return ids +} + +export function buildTapeToolFactInputs(record: ChatMessageRecord): TapeToolFactInput[] { + if (record.role !== 'assistant') return [] + + const inputs: TapeToolFactInput[] = [] + const blocks = parseAssistantBlocks(record.content) + const pendingInteractionToolIds = collectPendingInteractionToolIds(blocks) + blocks.forEach((block, blockIndex) => { + if (block?.type !== 'tool_call' || !block.tool_call) return + if (block.status !== 'success' && block.status !== 'error') return + + const toolCallId = block.tool_call.id + if (!toolCallId || pendingInteractionToolIds.has(toolCallId)) return + const sourceId = `${record.id}:${toolCallId}` + const factBlock = + block.timestamp === undefined ? { ...block, timestamp: record.updatedAt } : block + inputs.push({ + sessionId: toTapeSessionId(record.sessionId), + messageId: record.id, + orderSeq: record.orderSeq, + blockIndex, + block: factBlock, + provenance: { source: 'tool_call', sourceId, sequence: blockIndex } + }) + if (typeof block.tool_call.response === 'string' && block.tool_call.response.length > 0) { + inputs.push({ + sessionId: toTapeSessionId(record.sessionId), + messageId: record.id, + orderSeq: record.orderSeq, + blockIndex, + block: factBlock, + provenance: { source: 'tool_result', sourceId, sequence: blockIndex } + }) + } + }) + return inputs +} + +export function appendTapeToolFact( + table: TapeFactStore, + input: TapeToolFactInput, + source: TapeFactSource, + reason?: string +): DeepChatTapeEntryRow | null { + const block = input.block + const toolCall = block.type === 'tool_call' ? block.tool_call : undefined + if ( + !toolCall?.id || + (block.status !== 'success' && block.status !== 'error') || + (input.provenance.source !== 'tool_call' && input.provenance.source !== 'tool_result') + ) { + return null + } + + table.ensureBootstrapAnchor(input.sessionId) + const meta = reason + ? { source, role: 'assistant', status: block.status, reason } + : { source, role: 'assistant', status: block.status } + if (input.provenance.source === 'tool_call') { + const payload = { + messageId: input.messageId, + orderSeq: input.orderSeq, + toolCall: { + id: toolCall.id, + name: toolCall.name, + params: toolCall.params, + serverName: toolCall.server_name, + serverIcons: toolCall.server_icons, + serverDescription: toolCall.server_description + } + } + return table.append({ + sessionId: input.sessionId, + kind: 'tool_call', + name: toolCall.name || 'unknown', + source: { + type: input.provenance.source, + id: input.provenance.sourceId, + seq: input.provenance.sequence + }, + provenanceKey: buildToolFactProvenanceKey('tool_call', input.messageId, toolCall.id, payload), + payload, + meta, + createdAt: block.timestamp, + idempotent: true + }) + } + + if (typeof toolCall.response !== 'string' || toolCall.response.length === 0) return null + const payload = { + messageId: input.messageId, + orderSeq: input.orderSeq, + toolCallId: toolCall.id, + response: toolCall.response, + rtkApplied: toolCall.rtkApplied, + rtkMode: toolCall.rtkMode, + rtkFallbackReason: toolCall.rtkFallbackReason, + imagePreviews: toolCall.imagePreviews + } + return table.append({ + sessionId: input.sessionId, + kind: 'tool_result', + name: toolCall.name || 'unknown', + source: { + type: input.provenance.source, + id: input.provenance.sourceId, + seq: input.provenance.sequence + }, + provenanceKey: buildToolFactProvenanceKey('tool_result', input.messageId, toolCall.id, payload), + payload, + meta, + createdAt: block.timestamp, + idempotent: true + }) +} + +export function appendToolFactsToTape( + table: TapeFactStore, + record: ChatMessageRecord, + source: TapeFactSource, + reason?: string +): number { + if (record.role !== 'assistant') { + return 0 + } + + return buildTapeToolFactInputs(record).reduce( + (appended, input) => appended + (appendTapeToolFact(table, input, source, reason) ? 1 : 0), + 0 + ) +} + +export function appendMessageRecordToTape( + table: TapeFactStore, + record: ChatMessageRecord, + source: TapeFactSource +): number { + table.ensureBootstrapAnchor(record.sessionId) + + const compactionStatus = readCompactionStatus(record) + if (compactionStatus) { + table.appendEvent({ + sessionId: record.sessionId, + name: 'message/compaction_indicator', + source: { + type: 'message', + id: record.id, + seq: record.updatedAt + }, + provenanceKey: `message:${record.id}:compaction_indicator:${compactionStatus}:${record.updatedAt}`, + data: { + messageId: record.id, + orderSeq: record.orderSeq, + status: compactionStatus, + metadata: record.metadata + }, + meta: { + source, + status: compactionStatus + }, + createdAt: record.updatedAt, + idempotent: true + }) + return 1 + } + + table.append({ + sessionId: record.sessionId, + kind: 'message', + name: `message/${record.role}`, + source: { + type: 'message', + id: record.id, + seq: 0 + }, + provenanceKey: buildMessageProvenanceKey(record, source), + payload: { + record: { + id: record.id, + sessionId: record.sessionId, + orderSeq: record.orderSeq, + role: record.role, + content: record.content, + status: record.status, + isContextEdge: record.isContextEdge, + metadata: record.metadata, + traceCount: record.traceCount, + createdAt: record.createdAt, + updatedAt: record.updatedAt + } + }, + meta: { + source, + orderSeq: record.orderSeq, + role: record.role, + status: record.status + }, + createdAt: record.createdAt, + idempotent: true + }) + + return 1 + appendToolFactsToTape(table, record, source) +} + +export function appendMessageReplacementToTape( + table: TapeFactStore, + record: ChatMessageRecord, + reason: string +): number { + table.ensureBootstrapAnchor(record.sessionId) + table.append({ + sessionId: record.sessionId, + kind: 'message', + name: `message/${record.role}`, + source: { + type: 'message', + id: record.id, + seq: record.updatedAt + }, + provenanceKey: `message:${record.id}:revision:${record.updatedAt}`, + payload: { + record: { + id: record.id, + sessionId: record.sessionId, + orderSeq: record.orderSeq, + role: record.role, + content: record.content, + status: record.status, + isContextEdge: record.isContextEdge, + metadata: record.metadata, + traceCount: record.traceCount, + createdAt: record.createdAt, + updatedAt: record.updatedAt + } + }, + meta: { + source: 'live', + correction: true, + reason, + orderSeq: record.orderSeq, + role: record.role, + status: record.status + }, + createdAt: record.updatedAt, + idempotent: true + }) + + return 1 + appendToolFactsToTape(table, record, 'repair') +} + +export function appendMessageRetractionToTape( + table: TapeFactStore, + record: ChatMessageRecord, + reason: string +): number { + table.ensureBootstrapAnchor(record.sessionId) + table.appendEvent({ + sessionId: record.sessionId, + name: 'message/retracted', + source: { + type: 'message', + id: record.id, + seq: Date.now() + }, + provenanceKey: null, + data: { + messageId: record.id, + orderSeq: record.orderSeq, + role: record.role, + reason + }, + meta: { + source: 'live', + correction: true + }, + idempotent: false + }) + + return 1 +} + +export function tapeEntriesToEffectiveMessageRecords( + rows: DeepChatTapeEntryRow[] +): ChatMessageRecord[] { + return buildEffectiveTapeView(rows, { includePending: true }).messageRecords +} diff --git a/src/main/tape/application/factService.ts b/src/main/tape/application/factService.ts new file mode 100644 index 0000000000..36684e5a76 --- /dev/null +++ b/src/main/tape/application/factService.ts @@ -0,0 +1,166 @@ +import type { AgentTapeHandoffState, ChatMessageRecord } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryRow, TapeAnchorAppendInput } from '../domain/entry' +import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' +import { buildEffectiveTapeView } from '../domain/effectiveView' +import type { + TapeAnchorWriter, + TapeMessageFactWriter, + TapeToolFactWriter +} from '../ports/capabilities' +import type { TapeApplicationProviders } from '../ports/application' +import { + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + appendTapeToolFact +} from './factPersistence' +import { parseJsonObject } from './common' +import type { TapeAnchorResult } from './contracts' + +type TapeFactProviders = Pick + +function normalizeHandoffName(name: string): string { + const trimmed = name.trim() + if (!trimmed) return 'handoff/manual' + if (trimmed.startsWith('handoff/') || trimmed.startsWith('auto_handoff/')) return trimmed + return `handoff/${trimmed}` +} + +function normalizePositiveInteger(value: unknown): number | null { + if (typeof value !== 'number' || !Number.isFinite(value)) return null + return Math.max(1, Math.floor(value)) +} + +function buildOrderSeqRange(records: ChatMessageRecord[]): Record | null { + if (records.length === 0) return null + return { + fromOrderSeq: records[0].orderSeq, + toOrderSeq: records[records.length - 1].orderSeq + } +} + +function enrichHandoffState( + state: Record, + historyRecords: ChatMessageRecord[] +): Record { + const maxOrderSeq = historyRecords.reduce( + (currentMax, record) => Math.max(currentMax, record.orderSeq), + 0 + ) + const cursorOrderSeq = + normalizePositiveInteger(state.cursorOrderSeq ?? state.summaryCursorOrderSeq) ?? maxOrderSeq + 1 + const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) + const enrichedState: Record = { ...state, cursorOrderSeq } + + if (!Object.prototype.hasOwnProperty.call(enrichedState, 'range')) { + enrichedState.range = buildOrderSeqRange(sourceRecords) + } + + const sourceMessageIds = enrichedState.sourceMessageIds + if (!Array.isArray(sourceMessageIds) || sourceMessageIds.some((id) => typeof id !== 'string')) { + enrichedState.sourceMessageIds = sourceRecords.map((record) => record.id) + } + + return enrichedState +} + +export function normalizeTapeHandoffState(state: unknown): AgentTapeHandoffState { + if (!state || typeof state !== 'object' || Array.isArray(state)) { + throw new Error('Tape handoff requires a non-empty summary.') + } + + const summary = (state as Record).summary + if (typeof summary !== 'string' || !summary.trim()) { + throw new Error('Tape handoff requires a non-empty summary.') + } + + return { + ...(state as Record), + summary: summary.trim() + } +} + +export class TapeFactService + implements TapeToolFactWriter, TapeMessageFactWriter, TapeAnchorWriter +{ + constructor(private readonly providers: TapeFactProviders) {} + + private get table() { + return this.providers.getEntryStore() + } + + appendMessageRecord(record: ChatMessageRecord): number { + return appendMessageRecordToTape(this.table, record, 'live') + } + + appendMessageRecordForSession(sessionId: string, record: ChatMessageRecord): number { + return appendMessageRecordToTape(this.table, { ...record, sessionId }, 'live') + } + + appendMessageReplacement(record: ChatMessageRecord, reason: string): number { + return appendMessageReplacementToTape(this.table, record, reason) + } + + appendMessageRetraction(record: ChatMessageRecord, reason: string): number { + return appendMessageRetractionToTape(this.table, record, reason) + } + + async appendToolFact(input: TapeToolFactInput): Promise { + const row = appendTapeToolFact(this.table, input, 'live', 'tool_loop') + if (!row) throw new Error('Tape tool fact was not appendable.') + return { sessionId: input.sessionId, entryId: row.entry_id } + } + + getMessageRecords(sessionId: string): ChatMessageRecord[] { + return buildEffectiveTapeView(this.table.getBySession(sessionId), { includePending: true }) + .messageRecords + } + + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow { + return this.table.appendAnchor(input) + } + + handoff( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): DeepChatTapeEntryRow { + const normalizedState = normalizeTapeHandoffState(state) + const table = this.table + table.ensureBootstrapAnchor(sessionId) + const handoffState = enrichHandoffState(normalizedState, this.getMessageRecords(sessionId)) + return table.appendAnchor({ + sessionId, + name: normalizeHandoffName(name), + source: { + type: 'runtime_event', + id: `handoff:${Date.now()}`, + seq: 0 + }, + state: handoffState, + meta: { + ...meta, + handoff: true + } + }) + } + + handoffResult( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): TapeAnchorResult { + const row = this.handoff(sessionId, name, state, meta) + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + payload: parseJsonObject(row.payload_json), + meta: parseJsonObject(row.meta_json), + createdAt: row.created_at + } + } +} diff --git a/src/main/tape/application/forkService.ts b/src/main/tape/application/forkService.ts new file mode 100644 index 0000000000..3dd69371a5 --- /dev/null +++ b/src/main/tape/application/forkService.ts @@ -0,0 +1,335 @@ +import { nanoid } from 'nanoid' +import logger from 'electron-log' +import type { DeepChatTapeEntryRow } from '../domain/entry' +import type { TapeApplicationProviders } from '../ports/application' +import { deleteTapeGeneration } from './generationLifecycle' +import { parseJsonObject } from './common' +import type { TapeForkHandle } from './contracts' + +type TapeForkProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getEntryLifecycleStore' | 'getSearchProjectionStore' +> + +function readForkMergeReceiptCount( + row: DeepChatTapeEntryRow, + parentSessionId: string, + forkId: string, + forkSessionIdValue: string +): number { + const payload = parseJsonObject(row.payload_json) + const data = + payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) + ? (payload.data as Record) + : {} + const mergedCount = data.mergedCount + const forkHeadEntryId = data.forkHeadEntryId + const hasValidLegacyOrCurrentHead = + forkHeadEntryId === undefined || + (typeof forkHeadEntryId === 'number' && + Number.isSafeInteger(forkHeadEntryId) && + forkHeadEntryId >= 0) + if ( + row.session_id !== parentSessionId || + row.kind !== 'event' || + row.name !== 'fork/merge' || + row.source_type !== 'fork' || + row.source_id !== forkId || + row.source_seq !== 0 || + row.provenance_key !== `fork:${parentSessionId}:${forkId}:merge:event` || + data.forkId !== forkId || + data.forkSessionId !== forkSessionIdValue || + typeof mergedCount !== 'number' || + !Number.isSafeInteger(mergedCount) || + mergedCount < 0 || + !hasValidLegacyOrCurrentHead || + (typeof forkHeadEntryId === 'number' && mergedCount > forkHeadEntryId) + ) { + throw new Error(`Stored fork merge receipt is malformed: ${row.entry_id}`) + } + return mergedCount +} + +function assertValidForkStart( + row: DeepChatTapeEntryRow | undefined, + parentSessionId: string, + forkId: string, + forkSessionIdValue: string +): void { + if (!row) { + throw new Error(`Fork ${forkId} does not exist or has been discarded.`) + } + const payload = parseJsonObject(row.payload_json) + const state = + payload.state && typeof payload.state === 'object' && !Array.isArray(payload.state) + ? (payload.state as Record) + : {} + const parentHeadEntryId = state.parentHeadEntryId + const hasValidLegacyOrCurrentHead = + parentHeadEntryId === undefined || + (typeof parentHeadEntryId === 'number' && + Number.isSafeInteger(parentHeadEntryId) && + parentHeadEntryId >= 0) + if ( + row.session_id !== forkSessionIdValue || + row.kind !== 'anchor' || + row.name !== 'fork/start' || + row.source_type !== 'fork' || + row.source_id !== forkId || + row.source_seq !== 0 || + row.provenance_key !== `fork:${parentSessionId}:${forkId}:start` || + state.parentSessionId !== parentSessionId || + !hasValidLegacyOrCurrentHead + ) { + throw new Error(`Stored fork start is malformed: ${row.entry_id}`) + } +} + +function forkSessionId(parentSessionId: string, forkId: string): string { + return `${parentSessionId}::fork::${forkId}` +} + +function forkDiscardProvenanceKey(parentSessionId: string, forkId: string): string { + return `fork:${parentSessionId}:${forkId}:discard:event` +} + +function forkMergeProvenanceKey(parentSessionId: string, forkId: string): string { + return `fork:${parentSessionId}:${forkId}:merge:event` +} + +export class TapeForkService { + constructor(private readonly providers: TapeForkProviders) {} + + private get table() { + return this.providers.getEntryStore() + } + + createFork(parentSessionId: string, forkId: string = nanoid()): TapeForkHandle { + const table = this.table + const forkIdValue = forkId.trim() || nanoid() + if ( + table.getByProvenanceKey( + parentSessionId, + forkMergeProvenanceKey(parentSessionId, forkIdValue) + ) + ) { + throw new Error(`Fork ${forkIdValue} has already been merged and cannot be reused.`) + } + if ( + table.getByProvenanceKey( + parentSessionId, + forkDiscardProvenanceKey(parentSessionId, forkIdValue) + ) + ) { + throw new Error(`Fork ${forkIdValue} has been discarded and cannot be reused.`) + } + const forkSessionIdValue = forkSessionId(parentSessionId, forkIdValue) + const parentHeadEntryId = table.getMaxEntryId(parentSessionId) + table.ensureBootstrapAnchor(forkSessionIdValue) + const parentAnchor = table.getLatestAnchor(parentSessionId) + const forkStart = table.appendAnchor({ + sessionId: forkSessionIdValue, + name: 'fork/start', + source: { + type: 'fork', + id: forkIdValue, + seq: 0 + }, + provenanceKey: `fork:${parentSessionId}:${forkIdValue}:start`, + state: { + parentSessionId, + parentHeadEntryId, + parentLastAnchorEntryId: parentAnchor?.entry_id ?? null, + parentLastAnchorName: parentAnchor?.name ?? null + }, + idempotent: true + }) + const forkStartPayload = parseJsonObject(forkStart.payload_json) + const persistedState = + forkStartPayload.state && + typeof forkStartPayload.state === 'object' && + !Array.isArray(forkStartPayload.state) + ? (forkStartPayload.state as Record) + : {} + const persistedParentHeadEntryId = persistedState.parentHeadEntryId + return { + parentSessionId, + forkId: forkIdValue, + forkSessionId: forkSessionIdValue, + parentHeadEntryId: + typeof persistedParentHeadEntryId === 'number' && + Number.isSafeInteger(persistedParentHeadEntryId) && + persistedParentHeadEntryId >= 0 + ? persistedParentHeadEntryId + : parentHeadEntryId + } + } + + mergeFork(parentSessionId: string, forkId: string): number { + const table = this.table + const forkSessionIdValue = forkSessionId(parentSessionId, forkId) + const mergeProvenanceKey = forkMergeProvenanceKey(parentSessionId, forkId) + + return table.runInTransaction(() => { + const existingReceipt = table.getByProvenanceKey(parentSessionId, mergeProvenanceKey) + if (existingReceipt) { + return readForkMergeReceiptCount( + existingReceipt, + parentSessionId, + forkId, + forkSessionIdValue + ) + } + + if ( + table.getByProvenanceKey(parentSessionId, forkDiscardProvenanceKey(parentSessionId, forkId)) + ) { + throw new Error(`Fork ${forkId} does not exist or has been discarded.`) + } + + assertValidForkStart( + table.getByProvenanceKey(forkSessionIdValue, `fork:${parentSessionId}:${forkId}:start`), + parentSessionId, + forkId, + forkSessionIdValue + ) + + const forkHeadEntryId = table.getMaxEntryId(forkSessionIdValue) + const forkEntries = table + .getBySessionUpToEntryId(forkSessionIdValue, forkHeadEntryId) + .filter( + (entry) => + !( + entry.kind === 'anchor' && + (entry.name === 'session/start' || entry.name === 'fork/start') + ) + ) + + for (const entry of forkEntries) { + table.append({ + sessionId: parentSessionId, + kind: entry.kind, + name: entry.name, + source: { + type: 'fork', + id: forkId, + seq: entry.entry_id + }, + provenanceKey: `fork:${parentSessionId}:${forkId}:merge:${entry.entry_id}`, + payload: parseJsonObject(entry.payload_json), + meta: { + ...parseJsonObject(entry.meta_json), + forkId, + forkSessionId: forkSessionIdValue, + mergedFromEntryId: entry.entry_id + }, + createdAt: entry.created_at, + idempotent: true + }) + } + + table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/merge', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: mergeProvenanceKey, + data: { + forkId, + forkSessionId: forkSessionIdValue, + forkHeadEntryId, + mergedCount: forkEntries.length + }, + idempotent: true + }) + + return forkEntries.length + }) + } + + discardFork(parentSessionId: string, forkId: string): void { + const table = this.table + const forkSessionIdValue = forkSessionId(parentSessionId, forkId) + let cleanupError: unknown + table.runInTransaction(() => { + try { + deleteTapeGeneration(this.providers, forkSessionIdValue) + } catch (error) { + cleanupError = error + } + table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/discard', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: forkDiscardProvenanceKey(parentSessionId, forkId), + data: { + forkId, + forkSessionId: forkSessionIdValue + }, + idempotent: true + }) + }) + if (cleanupError) { + logger.warn(`[Tape] failed to delete fork Tape generation: ${String(cleanupError)}`) + } + } + + recordExternalForkMerge( + parentSessionId: string, + forkSessionIdValue: string, + forkId: string, + meta: Record = {} + ): DeepChatTapeEntryRow { + const table = this.table + const referencedEntryCount = table.countBySession(forkSessionIdValue) + return table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/merge', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: `fork:${parentSessionId}:${forkId}:external-merge:event`, + data: { + ...meta, + forkId, + forkSessionId: forkSessionIdValue, + referencedEntryCount + }, + idempotent: true + }) + } + + recordExternalForkDiscard( + parentSessionId: string, + forkSessionIdValue: string, + forkId: string, + meta: Record = {} + ): DeepChatTapeEntryRow { + const table = this.table + return table.appendEvent({ + sessionId: parentSessionId, + name: 'fork/discard', + source: { + type: 'fork', + id: forkId, + seq: 0 + }, + provenanceKey: `fork:${parentSessionId}:${forkId}:external-discard:event`, + data: { + ...meta, + forkId, + forkSessionId: forkSessionIdValue + }, + idempotent: true + }) + } +} diff --git a/src/main/tape/application/generationLifecycle.ts b/src/main/tape/application/generationLifecycle.ts new file mode 100644 index 0000000000..54f4dac3f0 --- /dev/null +++ b/src/main/tape/application/generationLifecycle.ts @@ -0,0 +1,29 @@ +import type { TapeApplicationProviders } from '../ports/application' + +type TapeGenerationLifecycleProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getEntryLifecycleStore' | 'getSearchProjectionStore' +> + +export function deleteTapeGeneration( + providers: TapeGenerationLifecycleProviders, + sessionId: string +): void { + const table = providers.getEntryStore() + table.runInTransaction(() => { + providers.getEntryLifecycleStore().deleteBySession(sessionId) + providers.getSearchProjectionStore().deleteBySession(sessionId) + }) +} + +export function resetTapeGeneration( + providers: TapeGenerationLifecycleProviders, + sessionId: string +): void { + const table = providers.getEntryStore() + table.runInTransaction(() => { + providers.getEntryLifecycleStore().deleteBySession(sessionId) + providers.getSearchProjectionStore().deleteBySession(sessionId) + table.ensureBootstrapAnchor(sessionId) + }) +} diff --git a/src/main/tape/application/lineageService.ts b/src/main/tape/application/lineageService.ts new file mode 100644 index 0000000000..42aba422c7 --- /dev/null +++ b/src/main/tape/application/lineageService.ts @@ -0,0 +1,458 @@ +import { createHash } from 'crypto' +import type { + SubagentTapeLinkInput, + SubagentTapeLinkOutcome, + SubagentTapeLinkReceipt +} from '@shared/types/agent-interface' +import { + TAPE_INCARNATION_META_KEY, + type DeepChatTapeEntryRow, + type DeepChatTapeReadSource +} from '../domain/entry' +import type { TapeApplicationProviders } from '../ports/application' +import { parseJsonObject, parseJsonValue } from './common' + +type TapeLineageProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getLineageSessionReader' +> + +function compactText(value: string, maxLength = 1000): string { + const normalized = value.replace(/\s+/g, ' ').trim() + if (normalized.length <= maxLength) return normalized + return `${normalized.slice(0, Math.max(0, maxLength - 3)).trimEnd()}...` +} + +const SUBAGENT_TAPE_LINK_EVENT_NAME = 'subagent/tape_linked' + +const SUBAGENT_TAPE_LINK_VERSION = 2 + +const TAPE_IDENTITY_PATTERN = /^[a-f0-9]{64}$/ + +const SUBAGENT_TAPE_LINK_OUTCOMES = new Set([ + 'completed', + 'error', + 'cancelled' +]) + +type SubagentTapeLinkSnapshot = { + linkEntryId: number + childSessionId: string + childHeadEntryId: number + childEntryCount: number + outcome: SubagentTapeLinkOutcome + childTapeIdentity: string | null +} + +type ParsedSubagentTapeLink = { + snapshot: SubagentTapeLinkSnapshot + frozenInput: SubagentTapeLinkInput +} + +export type LinkedTapeSourceResolution = { + sources: DeepChatTapeReadSource[] + unavailableSourceIds: Set +} + +export type AgentTapeViewErrorCode = + | 'current_tape_unavailable' + | 'linked_tape_unavailable' + | 'linked_tape_unauthorized' + +export class AgentTapeViewError extends Error { + readonly name = 'AgentTapeViewError' + + constructor( + readonly code: AgentTapeViewErrorCode, + readonly parentSessionId: string, + readonly sourceSessionId: string, + message: string + ) { + super(message) + } +} + +export function normalizeSubagentTapeLinkInput( + input: SubagentTapeLinkInput +): SubagentTapeLinkInput { + const normalized = { + parentSessionId: input.parentSessionId.trim(), + childSessionId: input.childSessionId.trim(), + runId: input.runId.trim(), + taskId: input.taskId.trim(), + slotId: input.slotId.trim(), + taskTitle: compactText(input.taskTitle, 500), + outcome: input.outcome, + resultSummary: input.resultSummary?.trim() ? compactText(input.resultSummary, 2000) : null + } + for (const [name, value] of Object.entries(normalized)) { + if (name === 'resultSummary' || name === 'outcome') continue + if (typeof value !== 'string' || !value) { + throw new Error(`Subagent Tape link ${name} is required.`) + } + } + if (normalized.parentSessionId === normalized.childSessionId) { + throw new Error('Subagent Tape link child must differ from its parent.') + } + if (!SUBAGENT_TAPE_LINK_OUTCOMES.has(normalized.outcome)) { + throw new Error(`Invalid subagent Tape link outcome: ${String(normalized.outcome)}`) + } + return normalized +} + +function isUnmarkedLegacyTape(row: DeepChatTapeEntryRow): boolean { + const meta = parseJsonValue(row.meta_json) + return ( + meta !== null && + typeof meta === 'object' && + !Array.isArray(meta) && + !Object.prototype.hasOwnProperty.call(meta, TAPE_INCARNATION_META_KEY) + ) +} + +function computeTapeIdentity(row: DeepChatTapeEntryRow): string { + return createHash('sha256') + .update( + JSON.stringify([ + row.session_id, + row.entry_id, + row.kind, + row.name, + row.source_type, + row.source_id, + row.source_seq, + row.provenance_key, + row.payload_json, + row.meta_json, + row.created_at + ]) + ) + .digest('hex') +} + +function subagentTapeLinkProvenanceKey(input: SubagentTapeLinkInput): string { + // This version belongs to the stable task-identity key, independently of the evolving event + // payload's linkVersion. + const identityHash = createHash('sha256') + .update( + JSON.stringify([input.parentSessionId, input.childSessionId, input.runId, input.taskId]) + ) + .digest('hex') + return `subagent:tape-link:v1:${identityHash}` +} + +function parseSubagentTapeLink(row: DeepChatTapeEntryRow): ParsedSubagentTapeLink | null { + const payload = parseJsonObject(row.payload_json) + const data = + payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) + ? (payload.data as Record) + : {} + const childSessionId = data.childSessionId + const childHeadEntryId = data.childHeadEntryId + const childEntryCount = data.childEntryCount + const outcome = data.outcome + const runId = data.runId + const taskId = data.taskId + const slotId = data.slotId + const taskTitle = data.taskTitle + const resultSummary = data.resultSummary + const linkVersion = data.linkVersion + const childTapeIdentity = data.childTapeIdentity + const hasValidLinkVersion = + (linkVersion === 1 && childTapeIdentity === undefined) || + (linkVersion === SUBAGENT_TAPE_LINK_VERSION && + typeof childTapeIdentity === 'string' && + TAPE_IDENTITY_PATTERN.test(childTapeIdentity)) + if ( + row.kind !== 'event' || + row.name !== SUBAGENT_TAPE_LINK_EVENT_NAME || + typeof childSessionId !== 'string' || + !childSessionId || + typeof childHeadEntryId !== 'number' || + !Number.isSafeInteger(childHeadEntryId) || + childHeadEntryId < 0 || + typeof childEntryCount !== 'number' || + !Number.isSafeInteger(childEntryCount) || + childEntryCount < 0 || + typeof outcome !== 'string' || + !SUBAGENT_TAPE_LINK_OUTCOMES.has(outcome as SubagentTapeLinkOutcome) || + typeof runId !== 'string' || + typeof taskId !== 'string' || + typeof slotId !== 'string' || + typeof taskTitle !== 'string' || + (resultSummary !== null && typeof resultSummary !== 'string') || + !hasValidLinkVersion || + row.source_type !== 'subagent' || + row.source_id !== childSessionId || + row.source_seq !== childHeadEntryId || + childEntryCount > childHeadEntryId + ) { + return null + } + + let normalizedInput: SubagentTapeLinkInput + try { + normalizedInput = normalizeSubagentTapeLinkInput({ + parentSessionId: row.session_id, + childSessionId, + runId, + taskId, + slotId, + taskTitle, + outcome: outcome as SubagentTapeLinkOutcome, + resultSummary + }) + } catch { + return null + } + if ( + normalizedInput.parentSessionId !== row.session_id || + normalizedInput.childSessionId !== childSessionId || + normalizedInput.runId !== runId || + normalizedInput.taskId !== taskId || + normalizedInput.slotId !== slotId || + normalizedInput.taskTitle !== taskTitle || + normalizedInput.resultSummary !== resultSummary || + row.provenance_key !== subagentTapeLinkProvenanceKey(normalizedInput) + ) { + return null + } + + return { + snapshot: { + linkEntryId: row.entry_id, + childSessionId, + childHeadEntryId, + childEntryCount, + outcome: outcome as SubagentTapeLinkOutcome, + childTapeIdentity: + linkVersion === SUBAGENT_TAPE_LINK_VERSION ? (childTapeIdentity as string) : null + }, + frozenInput: normalizedInput + } +} + +function parseSubagentTapeLinkSnapshot(row: DeepChatTapeEntryRow): SubagentTapeLinkSnapshot | null { + return parseSubagentTapeLink(row)?.snapshot ?? null +} + +function parseLegacyExternalTapeLinkSnapshot( + row: DeepChatTapeEntryRow +): SubagentTapeLinkSnapshot | null { + if (row.kind !== 'event' || row.name !== 'fork/merge' || row.source_type !== 'fork') { + return null + } + const payload = parseJsonObject(row.payload_json) + const data = + payload.data && typeof payload.data === 'object' && !Array.isArray(payload.data) + ? (payload.data as Record) + : {} + const childSessionId = data.forkSessionId + const forkId = data.forkId + const referencedEntryCount = data.referencedEntryCount + if ( + typeof childSessionId !== 'string' || + !childSessionId || + forkId !== childSessionId || + row.source_id !== childSessionId || + row.source_seq !== 0 || + row.provenance_key !== `fork:${row.session_id}:${childSessionId}:external-merge:event` || + typeof referencedEntryCount !== 'number' || + !Number.isSafeInteger(referencedEntryCount) || + referencedEntryCount <= 0 + ) { + return null + } + return { + linkEntryId: row.entry_id, + childSessionId, + childHeadEntryId: referencedEntryCount, + childEntryCount: referencedEntryCount, + outcome: 'completed', + childTapeIdentity: null + } +} + +function toSubagentTapeLinkReceipt(row: DeepChatTapeEntryRow): SubagentTapeLinkReceipt { + const snapshot = parseSubagentTapeLinkSnapshot(row) + if (!snapshot) { + throw new Error(`Stored subagent Tape link receipt is malformed: ${row.entry_id}`) + } + return { + linkEntry: { + sessionId: row.session_id, + entryId: snapshot.linkEntryId + }, + childSessionId: snapshot.childSessionId, + childHeadEntryId: snapshot.childHeadEntryId, + childEntryCount: snapshot.childEntryCount, + outcome: snapshot.outcome + } +} + +function assertSubagentTapeLinkMatchesInput( + row: DeepChatTapeEntryRow, + input: SubagentTapeLinkInput +): void { + const parsed = parseSubagentTapeLink(row) + if (!parsed) { + throw new Error(`Stored subagent Tape link receipt is malformed: ${row.entry_id}`) + } + const storedInput = parsed.frozenInput + const storedKeys = Object.keys(storedInput) as Array + if ( + storedKeys.length !== Object.keys(input).length || + storedKeys.some((key) => storedInput[key] !== input[key]) + ) { + throw new Error( + `Subagent Tape link conflicts with finalized task ${input.runId}/${input.taskId}.` + ) + } +} + +export class TapeLineageService { + constructor(private readonly providers: TapeLineageProviders) {} + + private get table() { + return this.providers.getEntryStore() + } + + resolveLinkedTapeSources(parentSessionId: string): LinkedTapeSourceResolution { + const table = this.table + const sessionTable = this.providers.getLineageSessionReader() + if (!table || !sessionTable?.get(parentSessionId)) { + throw new AgentTapeViewError( + 'current_tape_unavailable', + parentSessionId, + parentSessionId, + `Current Tape ${parentSessionId} is unavailable.` + ) + } + + const parsedSnapshots = table + .getSubagentLineageEvents(parentSessionId) + .map((row) => parseSubagentTapeLinkSnapshot(row) ?? parseLegacyExternalTapeLinkSnapshot(row)) + .filter((snapshot): snapshot is SubagentTapeLinkSnapshot => snapshot !== null) + const latestSnapshotByChild = new Map() + for (const snapshot of parsedSnapshots) { + const current = latestSnapshotByChild.get(snapshot.childSessionId) + if (!current || snapshot.linkEntryId > current.linkEntryId) { + latestSnapshotByChild.set(snapshot.childSessionId, snapshot) + } + } + const snapshots = [...latestSnapshotByChild.values()] + const childSessionIds = [...new Set(snapshots.map((snapshot) => snapshot.childSessionId))] + const childById = new Map(sessionTable.getMany(childSessionIds).map((row) => [row.id, row])) + const unavailableSourceIds = new Set() + const snapshotBySource = new Map() + + for (const snapshot of snapshots) { + const child = childById.get(snapshot.childSessionId) + if (!child) { + unavailableSourceIds.add(snapshot.childSessionId) + continue + } + if (child.session_kind !== 'subagent' || child.parent_session_id !== parentSessionId) { + continue + } + snapshotBySource.set(snapshot.childSessionId, snapshot) + } + + const authorizedSourceIds = [...snapshotBySource.keys()] + const firstEntryBySource = new Map( + table.getFirstEntriesBySessions(authorizedSourceIds).map((row) => [row.session_id, row]) + ) + const liveHeads = table.getMaxEntryIdsBySessions(authorizedSourceIds) + const availableSources: DeepChatTapeReadSource[] = [] + for (const [sourceSessionId, snapshot] of snapshotBySource) { + const firstEntry = firstEntryBySource.get(sourceSessionId) + const identityMatches = snapshot.childTapeIdentity + ? firstEntry !== undefined && computeTapeIdentity(firstEntry) === snapshot.childTapeIdentity + : firstEntry !== undefined && isUnmarkedLegacyTape(firstEntry) + if (!identityMatches || (liveHeads.get(sourceSessionId) ?? 0) < snapshot.childHeadEntryId) { + unavailableSourceIds.add(sourceSessionId) + continue + } + availableSources.push({ + sessionId: sourceSessionId, + maxEntryId: snapshot.childHeadEntryId + }) + } + + return { + sources: availableSources.sort((left, right) => + left.sessionId < right.sessionId ? -1 : left.sessionId > right.sessionId ? 1 : 0 + ), + unavailableSourceIds + } + } + + linkSubagentTape(input: SubagentTapeLinkInput): SubagentTapeLinkReceipt { + const table = this.table + const sessionTable = this.providers.getLineageSessionReader() + const normalized = normalizeSubagentTapeLinkInput(input) + const provenanceKey = subagentTapeLinkProvenanceKey(normalized) + return table.runInTransaction(() => { + const existing = table.getByProvenanceKey(normalized.parentSessionId, provenanceKey) + if (existing) { + assertSubagentTapeLinkMatchesInput(existing, normalized) + const receipt = toSubagentTapeLinkReceipt(existing) + return receipt + } + + const sessionById = new Map( + sessionTable + .getMany([normalized.parentSessionId, normalized.childSessionId]) + .map((session) => [session.id, session]) + ) + const parent = sessionById.get(normalized.parentSessionId) + const child = sessionById.get(normalized.childSessionId) + if ( + !parent || + !child || + child.session_kind !== 'subagent' || + child.parent_session_id !== normalized.parentSessionId + ) { + throw new Error( + `Session ${normalized.childSessionId} is not a direct subagent child of ` + + `${normalized.parentSessionId}.` + ) + } + + const childFirstEntry = table.getFirstEntriesBySessions([normalized.childSessionId])[0] + if (!childFirstEntry || childFirstEntry.session_id !== normalized.childSessionId) { + throw new Error(`Subagent Tape ${normalized.childSessionId} is unavailable.`) + } + const childTapeIdentity = computeTapeIdentity(childFirstEntry) + const childHeadEntryId = table.getMaxEntryId(normalized.childSessionId) + const childEntryCount = table.countBySession(normalized.childSessionId) + const row = table.appendEvent({ + sessionId: normalized.parentSessionId, + name: SUBAGENT_TAPE_LINK_EVENT_NAME, + source: { + type: 'subagent', + id: normalized.childSessionId, + seq: childHeadEntryId + }, + provenanceKey, + data: { + linkVersion: SUBAGENT_TAPE_LINK_VERSION, + childSessionId: normalized.childSessionId, + childHeadEntryId, + childEntryCount, + childTapeIdentity, + runId: normalized.runId, + taskId: normalized.taskId, + slotId: normalized.slotId, + taskTitle: normalized.taskTitle, + outcome: normalized.outcome, + resultSummary: normalized.resultSummary + }, + idempotent: true + }) + assertSubagentTapeLinkMatchesInput(row, normalized) + const receipt = toSubagentTapeLinkReceipt(row) + return receipt + }) + } +} diff --git a/src/main/tape/application/recallProjection.ts b/src/main/tape/application/recallProjection.ts new file mode 100644 index 0000000000..e5336b5519 --- /dev/null +++ b/src/main/tape/application/recallProjection.ts @@ -0,0 +1,466 @@ +import type { AgentTapeSearchOptions, AgentTapeViewScope } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryRow, DeepChatTapeSearchInput } from '../domain/entry' +import { parseJsonObject, parseJsonValue } from './common' +import type { TapeSearchResult } from './contracts' + +function isRecordObject(value: unknown): value is Record { + return Boolean(value && typeof value === 'object' && !Array.isArray(value)) +} + +function compactText(value: string, maxLength = 1000): string { + const normalized = value.replace(/\s+/g, ' ').trim() + if (normalized.length <= maxLength) return normalized + return `${normalized.slice(0, Math.max(0, maxLength - 3)).trimEnd()}...` +} + +export function stringifyForSummary(value: unknown, maxLength = 1000): string { + if (typeof value === 'string') return compactText(value, maxLength) + if (value === null || value === undefined) return '' + try { + return compactText(JSON.stringify(value), maxLength) + } catch { + return compactText(String(value), maxLength) + } +} + +export function truncateToUtf8Bytes( + text: string, + maxBytes: number +): { text: string; truncated: boolean } { + const normalized = text.trim() + if (maxBytes <= 0) { + return { text: '', truncated: normalized.length > 0 } + } + if (maxBytes < 3) { + return { text: '', truncated: normalized.length > 0 } + } + if (Buffer.byteLength(normalized, 'utf8') <= maxBytes) { + return { text: normalized, truncated: false } + } + let bytes = 0 + let output = '' + for (const character of normalized) { + const nextBytes = Buffer.byteLength(character, 'utf8') + if (bytes + nextBytes > Math.max(0, maxBytes - 3)) break + output += character + bytes += nextBytes + } + return { text: `${output.trimEnd()}...`, truncated: true } +} + +export function normalizeContextByteLimit( + value: number | undefined, + fallback: number, + max: number +): number { + if (!Number.isFinite(value)) return fallback + return Math.min(Math.max(Math.floor(value as number), 0), max) +} + +function uniqueStrings(values: string[], limit = 10): string[] { + const seen = new Set() + const result: string[] = [] + for (const value of values) { + const normalized = value.trim() + if (!normalized || seen.has(normalized)) continue + seen.add(normalized) + result.push(normalized) + if (result.length >= limit) break + } + return result +} + +function extractFilePaths(text: string): string[] { + const matches = [ + ...text.matchAll( + /(?:^|[\s"'`([{<])((?:[A-Za-z]:\\|\/|\.{1,2}\/|[\w.-]+\/)[^\s"'`<>{}(),;!?]+(?:[/\\][^\s"'`<>{}(),;!?]+)*)/g + ) + ].map((match) => match[1].replace(/[.:]+$/g, '')) + return uniqueStrings(matches ?? []) +} + +function extractErrorCodes(text: string): string[] { + const matches = text.match(/\b(?:E[A-Z0-9_]{3,}|[A-Z][A-Z0-9_]*Error)\b/g) + return uniqueStrings(matches ?? []) +} + +function collectKeyedStrings( + value: unknown, + keys: Set, + output: string[] = [], + depth = 0 +): string[] { + if (depth > 4 || output.length >= 10 || !value || typeof value !== 'object') return output + if (Array.isArray(value)) { + for (const item of value) collectKeyedStrings(item, keys, output, depth + 1) + return output + } + for (const [key, nested] of Object.entries(value as Record)) { + if (keys.has(key) && typeof nested === 'string' && nested.trim()) { + output.push(compactText(nested, 500)) + if (output.length >= 10) return output + } + collectKeyedStrings(nested, keys, output, depth + 1) + if (output.length >= 10) return output + } + return output +} + +function collectUserMessageAttachmentRefs(files: unknown): { + searchText: string[] + filePaths: string[] + fileNames: string[] +} { + const searchText: string[] = [] + const filePaths: string[] = [] + const fileNames: string[] = [] + if (!Array.isArray(files)) { + return { searchText, filePaths, fileNames } + } + for (const file of files) { + if (!isRecordObject(file)) continue + const path = typeof file.path === 'string' ? file.path : '' + const name = typeof file.name === 'string' ? file.name : '' + const metadata = isRecordObject(file.metadata) ? file.metadata : null + const metadataFileName = + metadata && typeof metadata.fileName === 'string' ? metadata.fileName : '' + if (path) { + filePaths.push(compactText(path, 500)) + searchText.push(compactText(path, 500)) + } + for (const value of [name, metadataFileName]) { + if (!value) continue + fileNames.push(compactText(value, 500)) + searchText.push(compactText(value, 500)) + } + } + return { + searchText: uniqueStrings(searchText, 20), + filePaths: uniqueStrings(filePaths, 20), + fileNames: uniqueStrings(fileNames, 20) + } +} + +type UserMessageProjectionText = { + text: string + attachmentRefs: { + searchText: string[] + filePaths: string[] + fileNames: string[] + } +} + +function emptyUserMessageAttachmentRefs(): UserMessageProjectionText['attachmentRefs'] { + return { searchText: [], filePaths: [], fileNames: [] } +} + +function parseUserMessageProjectionText(content: string): UserMessageProjectionText { + const parsed = parseJsonValue(content) + if (isRecordObject(parsed) && typeof parsed.text === 'string') { + const attachmentRefs = collectUserMessageAttachmentRefs(parsed.files) + const parts = [parsed.text] + if (Array.isArray(parsed.files) && parsed.files.length > 0) { + parts.push(`files:${parsed.files.length}`) + parts.push(...attachmentRefs.searchText) + } + if (Array.isArray(parsed.links) && parsed.links.length > 0) { + parts.push(`links:${parsed.links.length}`) + } + return { text: parts.join(' '), attachmentRefs } + } + return { text: content, attachmentRefs: emptyUserMessageAttachmentRefs() } +} + +export function getUserMessageProjectionText( + row: DeepChatTapeEntryRow, + payload: Record +): UserMessageProjectionText | null { + if (row.kind !== 'message' || !isRecordObject(payload.record)) return null + const record = payload.record + const role = typeof record.role === 'string' ? record.role : 'message' + if (role === 'assistant') return null + const content = typeof record.content === 'string' ? record.content : '' + return parseUserMessageProjectionText(content) +} + +function collectUserMessageAttachmentRefsFromPayload(payload: Record): { + searchText: string[] + filePaths: string[] + fileNames: string[] +} { + if (!isRecordObject(payload.record)) { + return { searchText: [], filePaths: [], fileNames: [] } + } + const content = typeof payload.record.content === 'string' ? payload.record.content : '' + const parsed = parseJsonValue(content) + return isRecordObject(parsed) + ? collectUserMessageAttachmentRefs(parsed.files) + : { searchText: [], filePaths: [], fileNames: [] } +} + +function readUserMessageText(content: string, parsed?: UserMessageProjectionText): string { + return parsed?.text ?? parseUserMessageProjectionText(content).text +} + +function readAssistantMessageText(content: string): string { + const parsed = parseJsonValue(content) + if (!Array.isArray(parsed)) return content + const parts: string[] = [] + for (const block of parsed) { + if (!isRecordObject(block)) continue + if (typeof block.content === 'string' && block.content.trim()) { + parts.push(block.content) + continue + } + const toolCall = block.tool_call + if (isRecordObject(toolCall)) { + const name = typeof toolCall.name === 'string' ? toolCall.name : 'unknown' + const params = typeof toolCall.params === 'string' ? toolCall.params : '' + const response = typeof toolCall.response === 'string' ? toolCall.response : '' + parts.push(`tool ${name} ${params} ${response}`.trim()) + } + } + return parts.join(' ') +} + +export function summarizeTapeRow( + row: DeepChatTapeEntryRow, + payload: Record, + userMessage?: UserMessageProjectionText | null +): string { + if (row.kind === 'message') { + const record = payload.record + if (isRecordObject(record)) { + const role = typeof record.role === 'string' ? record.role : 'message' + const content = typeof record.content === 'string' ? record.content : '' + const text = + role === 'assistant' + ? readAssistantMessageText(content) + : readUserMessageText(content, userMessage ?? undefined) + return compactText(`${role}: ${text}`, 1200) + } + } + + if (row.kind === 'tool_call') { + const toolCall = payload.toolCall + if (isRecordObject(toolCall)) { + const name = typeof toolCall.name === 'string' ? toolCall.name : (row.name ?? 'unknown') + const params = typeof toolCall.params === 'string' ? toolCall.params : '' + return compactText(`tool_call ${name}: ${params}`, 1200) + } + } + + if (row.kind === 'tool_result') { + const response = typeof payload.response === 'string' ? payload.response : payload + return compactText( + `tool_result ${row.name ?? 'unknown'}: ${stringifyForSummary(response)}`, + 1200 + ) + } + + if (row.kind === 'anchor') { + const state = isRecordObject(payload.state) ? payload.state : payload + const summary = typeof state.summary === 'string' ? state.summary : stringifyForSummary(state) + return compactText(`anchor ${row.name ?? 'unknown'}: ${summary}`, 1200) + } + + if (row.kind === 'event') { + const data = isRecordObject(payload.data) ? payload.data : payload + return compactText(`event ${row.name ?? 'unknown'}: ${stringifyForSummary(data)}`, 1200) + } + + return compactText(`${row.kind} ${row.name ?? ''}`.trim(), 1200) +} + +export function buildTapeRowEvidenceText( + row: DeepChatTapeEntryRow, + payload: Record, + meta: Record, + userMessage?: UserMessageProjectionText | null +): string { + const parts: string[] = [] + if (row.kind === 'message' && isRecordObject(payload.record)) { + const record = payload.record + const content = typeof record.content === 'string' ? record.content : '' + const role = typeof record.role === 'string' ? record.role : 'message' + parts.push( + role === 'assistant' + ? readAssistantMessageText(content) + : readUserMessageText(content, userMessage ?? undefined) + ) + } else if (row.kind === 'tool_call' && isRecordObject(payload.toolCall)) { + const toolCall = payload.toolCall + parts.push(stringifyForSummary(toolCall.name, 200)) + parts.push(stringifyForSummary(toolCall.params, 3000)) + } else if (row.kind === 'tool_result') { + parts.push(stringifyForSummary(payload.response ?? payload, 4000)) + } else if (row.kind === 'anchor') { + parts.push(stringifyForSummary(isRecordObject(payload.state) ? payload.state : payload, 4000)) + } else if (row.kind === 'event') { + parts.push(stringifyForSummary(isRecordObject(payload.data) ? payload.data : payload, 4000)) + } else { + parts.push(stringifyForSummary(payload, 4000)) + } + if (typeof meta.status === 'string') parts.push(`status:${meta.status}`) + return compactText(parts.filter(Boolean).join('\n'), 5000) +} + +function setRef(target: Record, key: string, value: unknown): void { + if (value !== null && value !== undefined && value !== '') { + target[key] = value + } +} + +function enrichTapeRowRefs( + refs: Record, + payload: Record, + meta: Record, + evidenceText: string, + userMessage?: UserMessageProjectionText | null +): void { + const attachmentRefs = + userMessage?.attachmentRefs ?? collectUserMessageAttachmentRefsFromPayload(payload) + const filePaths = uniqueStrings( + [...extractFilePaths(evidenceText), ...attachmentRefs.filePaths], + 20 + ) + const errorCodes = extractErrorCodes(evidenceText) + const commands = uniqueStrings( + [ + ...collectKeyedStrings(payload, new Set(['command', 'cmd', 'script', 'shellCommand'])), + ...collectKeyedStrings(meta, new Set(['command', 'cmd', 'script', 'shellCommand'])) + ], + 10 + ) + setRef(refs, 'filePaths', filePaths.length ? filePaths : null) + setRef(refs, 'fileNames', attachmentRefs.fileNames.length ? attachmentRefs.fileNames : null) + setRef(refs, 'commands', commands.length ? commands : null) + setRef(refs, 'errorCodes', errorCodes.length ? errorCodes : null) + for (const key of ['exitCode', 'exitStatus', 'code']) { + const value = payload[key] ?? meta[key] + if (typeof value === 'number' || typeof value === 'string') { + setRef(refs, key, value) + } + } +} + +export function buildTapeRowRefs( + row: DeepChatTapeEntryRow, + payload: Record, + meta: Record, + userMessage?: UserMessageProjectionText | null, + evidenceText?: string +): Record { + const refs: Record = {} + setRef(refs, 'sourceType', row.source_type) + setRef(refs, 'sourceId', row.source_id) + setRef(refs, 'sourceSeq', row.source_seq) + setRef(refs, 'status', typeof meta.status === 'string' ? meta.status : null) + + if (row.kind === 'message' && isRecordObject(payload.record)) { + const record = payload.record + setRef(refs, 'messageId', record.id) + setRef(refs, 'orderSeq', record.orderSeq) + setRef(refs, 'role', record.role) + setRef(refs, 'messageStatus', record.status) + } else if (row.kind === 'tool_call' && isRecordObject(payload.toolCall)) { + const toolCall = payload.toolCall + setRef(refs, 'messageId', payload.messageId) + setRef(refs, 'orderSeq', payload.orderSeq) + setRef(refs, 'toolCallId', toolCall.id) + setRef(refs, 'toolName', toolCall.name ?? row.name) + setRef(refs, 'serverName', toolCall.serverName) + } else if (row.kind === 'tool_result') { + setRef(refs, 'messageId', payload.messageId) + setRef(refs, 'orderSeq', payload.orderSeq) + setRef(refs, 'toolCallId', payload.toolCallId) + setRef(refs, 'toolName', row.name) + } else if (row.kind === 'anchor') { + setRef(refs, 'anchorName', row.name) + } else if (row.kind === 'event') { + setRef(refs, 'eventName', row.name) + } + + enrichTapeRowRefs( + refs, + payload, + meta, + evidenceText ?? buildTapeRowEvidenceText(row, payload, meta, userMessage), + userMessage + ) + return refs +} + +export function parseProjectionRefs(raw: string): Record { + const parsed = parseJsonObject(raw) + return parsed +} + +export function normalizeContextWindowValue(value: number | undefined, fallback: number): number { + if (!Number.isFinite(value)) return fallback + return Math.min(Math.max(Math.floor(value as number), 0), 20) +} + +export function normalizeContextLimit(value: number | undefined): number { + if (!Number.isFinite(value)) return 50 + return Math.min(Math.max(Math.floor(value as number), 1), 100) +} + +function parseSearchBoundary(value: string | undefined, name: string): number | undefined { + const trimmed = value?.trim() + if (!trimmed) { + return undefined + } + + const numericValue = Number(trimmed) + if (Number.isFinite(numericValue)) { + return numericValue + } + + const parsedDate = Date.parse(trimmed) + if (Number.isFinite(parsedDate)) { + return parsedDate + } + + throw new Error(`${name} must be an ISO date/time or millisecond timestamp.`) +} + +export function toTapeSearchInput( + options: AgentTapeSearchOptions | undefined +): DeepChatTapeSearchInput { + return { + limit: options?.limit, + kinds: options?.kinds, + startCreatedAt: parseSearchBoundary(options?.start, 'start'), + endCreatedAt: parseSearchBoundary(options?.end, 'end') + } +} + +export function normalizeTapeViewScope(scope: AgentTapeViewScope | undefined): AgentTapeViewScope { + if (scope === undefined || scope === 'current') return 'current' + if (scope === 'linked_subagents' || scope === 'current_and_linked') return scope + throw new Error(`Invalid Tape view scope: ${String(scope)}`) +} + +export function normalizeTapeSearchLimit(value: number | undefined): number { + if (!Number.isFinite(value)) return 20 + return Math.min(Math.max(Math.floor(value as number), 1), 100) +} + +export function compareTapeSearchResults(left: TapeSearchResult, right: TapeSearchResult): number { + const leftHasScore = typeof left.score === 'number' && Number.isFinite(left.score) + const rightHasScore = typeof right.score === 'number' && Number.isFinite(right.score) + if (leftHasScore && rightHasScore && left.score !== right.score) { + return (left.score as number) - (right.score as number) + } + if (leftHasScore !== rightHasScore) { + return leftHasScore ? -1 : 1 + } + if (left.createdAt !== right.createdAt) { + return right.createdAt - left.createdAt + } + if (left.sessionId !== right.sessionId) { + return left.sessionId < right.sessionId ? -1 : 1 + } + return right.entryId - left.entryId +} diff --git a/src/main/tape/application/recallService.ts b/src/main/tape/application/recallService.ts new file mode 100644 index 0000000000..93a4e74f0f --- /dev/null +++ b/src/main/tape/application/recallService.ts @@ -0,0 +1,584 @@ +import logger from 'electron-log' +import type { + AgentTapeAnchorsOptions, + AgentTapeContextEntry, + AgentTapeContextOptions, + AgentTapeContextResult, + AgentTapeSearchOptions +} from '@shared/types/agent-interface' +import { + buildEffectiveTapeView, + getLastEffectiveTokenUsage, + searchEffectiveTapeRows +} from '../domain/effectiveView' +import type { DeepChatTapeEntryRow, DeepChatTapeReadSource } from '../domain/entry' +import type { + TapeApplicationProviders, + TapeSearchProjectionInput as DeepChatTapeSearchProjectionInput, + TapeSearchProjectionResultRow as DeepChatTapeSearchProjectionResultRow, + TapeSearchProjectionRow as DeepChatTapeSearchProjectionRow, + TapeSearchProjectionStore +} from '../ports/application' +import type { TapeEffectiveMessageSourceEntry } from '../ports/capabilities' +import { isEntryIdPrefix, migrationProvenanceKey, parseJsonObject } from './common' +import type { TapeAnchorResult, TapeInfo, TapeSearchResult } from './contracts' +import { AgentTapeViewError, type TapeLineageService } from './lineageService' +import { + buildTapeRowEvidenceText, + buildTapeRowRefs, + compareTapeSearchResults, + getUserMessageProjectionText, + normalizeContextByteLimit, + normalizeContextLimit, + normalizeContextWindowValue, + normalizeTapeSearchLimit, + normalizeTapeViewScope, + parseProjectionRefs, + stringifyForSummary, + summarizeTapeRow, + toTapeSearchInput, + truncateToUtf8Bytes +} from './recallProjection' + +type TapeRecallProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getSearchProjectionStore' +> + +const DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY = 2048 +const DEFAULT_CONTEXT_MAX_TOTAL_BYTES = 16384 +const MAX_CONTEXT_MAX_BYTES_PER_ENTRY = 8192 +const MAX_CONTEXT_MAX_TOTAL_BYTES = 65536 + +export class TapeRecallService { + constructor( + private readonly providers: TapeRecallProviders, + private readonly lineage: TapeLineageService + ) {} + + private get table() { + return this.providers.getEntryStore() + } + + private get searchProjectionTable() { + return this.providers.getSearchProjectionStore() + } + + info(sessionId: string): TapeInfo { + const table = this.table + const lastAnchor = table.getLatestAnchor(sessionId) + const rows = table.getBySession(sessionId) + return { + sessionId, + entries: table.countBySession(sessionId), + anchors: table.countAnchorsBySession(sessionId), + lastAnchor: lastAnchor?.name ?? null, + lastAnchorEntryId: lastAnchor?.entry_id ?? null, + entriesSinceLastAnchor: lastAnchor + ? table.countEntriesAfter(sessionId, lastAnchor.entry_id) + : 0, + lastTokenUsage: getLastEffectiveTokenUsage(rows), + migrationState: table.getByProvenanceKey(sessionId, migrationProvenanceKey(sessionId)) + ? 'ready' + : 'none' + } + } + + getEffectiveMessageSourceSpan( + sessionId: string, + entryIds: number[] + ): TapeEffectiveMessageSourceEntry[] { + const requestedEntryIds = new Set( + entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0) + ) + if (requestedEntryIds.size === 0) return [] + return buildEffectiveTapeView(this.table.getBySession(sessionId), { includePending: false }) + .messageEntries.filter((entry) => requestedEntryIds.has(entry.entryId)) + .map((entry) => ({ + entryId: entry.entryId, + record: { + role: entry.record.role, + content: entry.record.content, + orderSeq: entry.record.orderSeq + } + })) + } + + search(sessionId: string, query: string, options?: AgentTapeSearchOptions): TapeSearchResult[] { + const scope = normalizeTapeViewScope(options?.scope) + if (!query.trim()) { + return [] + } + if (scope === 'current') { + return this.searchCurrentTape(sessionId, query, options) + } + + const resolution = this.lineage.resolveLinkedTapeSources(sessionId) + if (resolution.unavailableSourceIds.size > 0) { + const sourceSessionId = [...resolution.unavailableSourceIds].sort()[0] + throw new AgentTapeViewError( + 'linked_tape_unavailable', + sessionId, + sourceSessionId, + `Linked Tape ${sourceSessionId} is unavailable.` + ) + } + const sources = [...resolution.sources] + if (scope === 'current_and_linked') { + sources.push({ sessionId, maxEntryId: this.table?.getMaxEntryId(sessionId) ?? 0 }) + } + return this.searchTapeSourcesReadOnly(sources, query, options) + } + + private searchCurrentTape( + sessionId: string, + query: string, + options?: AgentTapeSearchOptions + ): TapeSearchResult[] { + const table = this.table + const searchInput = toTapeSearchInput(options) + const projectionTable = this.searchProjectionTable + let skipProjectionSearch = false + + if (projectionTable) { + try { + const maxEntryId = table.getMaxEntryId(sessionId) + if (projectionTable.isCurrent(sessionId, maxEntryId)) { + return projectionTable + .search(sessionId, query, searchInput) + .map((row) => this.toProjectedSearchResult(row, undefined)) + } + } catch (error) { + skipProjectionSearch = true + logger.warn( + `[Tape] projection fast-path search failed; falling back to effective search: ${String(error)}` + ) + } + } + + const rows = table.getBySession(sessionId) + const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows + const preparedProjectionTable = skipProjectionSearch + ? null + : this.ensureSearchProjection(sessionId, rows, effectiveRows) + if (!preparedProjectionTable) { + return searchEffectiveTapeRows(rows, query, searchInput).map((row) => + this.toSearchResult(row) + ) + } + + const rowByEntryId = new Map(effectiveRows.map((row) => [row.entry_id, row])) + try { + return preparedProjectionTable + .search(sessionId, query, searchInput) + .map((row) => this.toProjectedSearchResult(row, rowByEntryId.get(row.entry_id))) + } catch (error) { + logger.warn( + `[Tape] projection search failed; falling back to effective search: ${String(error)}` + ) + return searchEffectiveTapeRows(rows, query, searchInput).map((row) => + this.toSearchResult(row) + ) + } + } + + getContext( + sessionId: string, + entryIds: number[], + options: AgentTapeContextOptions = {} + ): AgentTapeContextResult { + const sourceSessionId = options.sourceSessionId?.trim() || sessionId + if (sourceSessionId !== sessionId) { + return this.getLinkedTapeContext(sessionId, sourceSessionId, entryIds, options) + } + + const requestedEntryIds = [ + ...new Set(entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0)) + ].sort((left, right) => left - right) + const table = this.table + if (!table || requestedEntryIds.length === 0) { + return { + sessionId, + sourceSessionId, + requestedEntryIds, + matchedEntryIds: [], + entries: [] + } + } + + const rows = table.getBySession(sessionId) + const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows + const indexByEntryId = new Map(effectiveRows.map((row, index) => [row.entry_id, index])) + const before = normalizeContextWindowValue(options.before, 2) + const after = normalizeContextWindowValue(options.after, 2) + const limit = normalizeContextLimit(options.limit) + const maxBytesPerEntry = normalizeContextByteLimit( + options.maxBytesPerEntry, + DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY, + MAX_CONTEXT_MAX_BYTES_PER_ENTRY + ) + const maxTotalBytes = normalizeContextByteLimit( + options.maxTotalBytes, + DEFAULT_CONTEXT_MAX_TOTAL_BYTES, + MAX_CONTEXT_MAX_TOTAL_BYTES + ) + const selectedIndexes = new Set() + const requestedIndexes: number[] = [] + const windowIndexes: number[] = [] + + for (const entryId of requestedEntryIds) { + const index = indexByEntryId.get(entryId) + if (index === undefined) continue + requestedIndexes.push(index) + for ( + let cursor = Math.max(0, index - before); + cursor <= Math.min(effectiveRows.length - 1, index + after); + cursor += 1 + ) { + if (cursor === index) continue + windowIndexes.push(cursor) + } + } + + for (const index of requestedIndexes) { + if (selectedIndexes.size >= limit) break + selectedIndexes.add(index) + } + for (const index of windowIndexes) { + if (selectedIndexes.size >= limit) break + selectedIndexes.add(index) + } + + const selectedRows = [...selectedIndexes] + .sort((left, right) => left - right) + .map((index) => effectiveRows[index]) + let projectionRows = new Map() + try { + const maxEntryId = rows.reduce((max, row) => Math.max(max, row.entry_id), 0) + projectionRows = new Map( + this.searchProjectionTable + .getByEntryIdsIfCurrent( + sessionId, + maxEntryId, + selectedRows.map((row) => row.entry_id) + ) + .map((row) => [row.entry_id, row]) + ) + } catch { + projectionRows = new Map() + } + let usedBytes = 0 + const entries: AgentTapeContextEntry[] = [] + const priorityIndexes = [...requestedIndexes, ...windowIndexes].filter( + (index, offset, indexes) => { + return selectedIndexes.has(index) && indexes.indexOf(index) === offset + } + ) + for (const index of priorityIndexes) { + const row = effectiveRows[index] + const remaining = Math.max(0, maxTotalBytes - usedBytes) + if (remaining <= 0) break + const maxEntryBytes = Math.min(maxBytesPerEntry, remaining) + if (maxEntryBytes <= 0) break + const entry = this.toContextEntry(row, projectionRows.get(row.entry_id), maxEntryBytes) + if (entry.evidence.bytes <= 0) continue + usedBytes += entry.evidence.bytes + entries.push(entry) + } + entries.sort((left, right) => left.entryId - right.entryId) + const returnedEntryIds = new Set(entries.map((entry) => entry.entryId)) + + return { + sessionId, + sourceSessionId, + requestedEntryIds, + matchedEntryIds: requestedEntryIds.filter((entryId) => returnedEntryIds.has(entryId)), + entries + } + } + + private searchTapeSourcesReadOnly( + sources: DeepChatTapeReadSource[], + query: string, + options: AgentTapeSearchOptions | undefined + ): TapeSearchResult[] { + const table = this.table + if (!table || !query.trim() || sources.length === 0) { + return [] + } + const searchInput = toTapeSearchInput(options) + const limit = normalizeTapeSearchLimit(options?.limit) + const results: TapeSearchResult[] = [] + let uncoveredSources = sources + + try { + const projected = this.searchProjectionTable?.searchSourcesReadOnly( + sources, + query, + searchInput + ) + if (projected) { + const coveredHeadBySource = new Map( + projected.coveredSources.map((source) => [source.sessionId, source.maxEntryId]) + ) + const hasCompleteProjection = + coveredHeadBySource.size === sources.length && + sources.every((source) => coveredHeadBySource.get(source.sessionId) === source.maxEntryId) + if (hasCompleteProjection) { + results.push(...projected.rows.map((row) => this.toProjectedSearchResult(row, undefined))) + uncoveredSources = [] + } + } + } catch (error) { + logger.warn( + `[Tape] linked projection search failed; using read-only Tape fallback: ${String(error)}` + ) + uncoveredSources = sources + } + + if (uncoveredSources.length > 0) { + results.push( + ...table + .searchEffectiveSourcesAtHeads(uncoveredSources, query, searchInput) + .map((row) => this.toSearchResult(row)) + ) + } + + const seen = new Set() + return results + .sort(compareTapeSearchResults) + .filter((result) => { + const key = `${result.sessionId}:${result.entryId}` + if (seen.has(key)) return false + seen.add(key) + return true + }) + .slice(0, limit) + } + + private getLinkedTapeContext( + parentSessionId: string, + sourceSessionId: string, + entryIds: number[], + options: AgentTapeContextOptions + ): AgentTapeContextResult { + const resolution = this.lineage.resolveLinkedTapeSources(parentSessionId) + if (resolution.unavailableSourceIds.has(sourceSessionId)) { + throw new AgentTapeViewError( + 'linked_tape_unavailable', + parentSessionId, + sourceSessionId, + `Linked Tape ${sourceSessionId} is unavailable.` + ) + } + const source = resolution.sources.find((candidate) => candidate.sessionId === sourceSessionId) + if (!source) { + throw new AgentTapeViewError( + 'linked_tape_unauthorized', + parentSessionId, + sourceSessionId, + `Tape ${sourceSessionId} is not an authorized direct child of ${parentSessionId}.` + ) + } + + const requestedEntryIds = [ + ...new Set(entryIds.filter((entryId) => Number.isInteger(entryId) && entryId > 0)) + ].sort((left, right) => left - right) + const table = this.table + if (!table || requestedEntryIds.length === 0) { + return { + sessionId: parentSessionId, + sourceSessionId, + requestedEntryIds, + matchedEntryIds: [], + entries: [] + } + } + + const before = normalizeContextWindowValue(options.before, 2) + const after = normalizeContextWindowValue(options.after, 2) + const limit = normalizeContextLimit(options.limit) + const maxBytesPerEntry = normalizeContextByteLimit( + options.maxBytesPerEntry, + DEFAULT_CONTEXT_MAX_BYTES_PER_ENTRY, + MAX_CONTEXT_MAX_BYTES_PER_ENTRY + ) + const maxTotalBytes = normalizeContextByteLimit( + options.maxTotalBytes, + DEFAULT_CONTEXT_MAX_TOTAL_BYTES, + MAX_CONTEXT_MAX_TOTAL_BYTES + ) + const rows = table.getEffectiveContextRowsAtHead(source, requestedEntryIds, { + before, + after, + limit + }) + + let usedBytes = 0 + const entries: AgentTapeContextEntry[] = [] + for (const row of rows) { + const remaining = Math.max(0, maxTotalBytes - usedBytes) + if (remaining <= 0) break + const maxEntryBytes = Math.min(maxBytesPerEntry, remaining) + const entry = this.toContextEntry(row, undefined, maxEntryBytes) + if (entry.evidence.bytes <= 0) continue + usedBytes += entry.evidence.bytes + entries.push(entry) + } + entries.sort((left, right) => left.entryId - right.entryId) + const returnedEntryIds = new Set(entries.map((entry) => entry.entryId)) + + return { + sessionId: parentSessionId, + sourceSessionId, + requestedEntryIds, + matchedEntryIds: requestedEntryIds.filter((entryId) => returnedEntryIds.has(entryId)), + entries + } + } + + anchors(sessionId: string, options: AgentTapeAnchorsOptions = {}): TapeAnchorResult[] { + return this.table.getAnchors(sessionId, options.limit).map((row) => this.toAnchorResult(row)) + } + + private ensureSearchProjection( + sessionId: string, + rows: DeepChatTapeEntryRow[], + effectiveRows: DeepChatTapeEntryRow[] + ): TapeSearchProjectionStore | null { + const projectionTable = this.searchProjectionTable + const maxEntryId = rows.reduce((max, row) => Math.max(max, row.entry_id), 0) + try { + if (!projectionTable.isCurrent(sessionId, maxEntryId)) { + const meta = projectionTable.getSessionMeta(sessionId) + const projectedEntryIds = projectionTable.getProjectedEntryIds(sessionId) + const effectiveEntryIds = effectiveRows.map((row) => row.entry_id) + const canAppend = + !!meta && + projectionTable.isCurrent(sessionId, meta.maxEntryId) && + meta.maxEntryId <= maxEntryId && + isEntryIdPrefix(projectedEntryIds, effectiveEntryIds) + if (canAppend) { + projectionTable.appendSession( + sessionId, + effectiveRows.slice(projectedEntryIds.length).map((row) => this.toProjectionInput(row)), + maxEntryId + ) + } else { + projectionTable.replaceSession( + sessionId, + effectiveRows.map((row) => this.toProjectionInput(row)), + maxEntryId + ) + } + } + return projectionTable + } catch { + return null + } + } + + private toProjectionInput(row: DeepChatTapeEntryRow): DeepChatTapeSearchProjectionInput { + const payload = parseJsonObject(row.payload_json) + const meta = parseJsonObject(row.meta_json) + const userMessage = getUserMessageProjectionText(row, payload) + const summaryText = summarizeTapeRow(row, payload, userMessage) + const evidenceText = buildTapeRowEvidenceText(row, payload, meta, userMessage) + const refs = buildTapeRowRefs(row, payload, meta, userMessage, evidenceText) + const searchText = [ + row.kind, + row.name ?? '', + summaryText, + evidenceText, + Object.values(refs) + .map((value) => stringifyForSummary(value)) + .join(' ') + ] + .filter(Boolean) + .join('\n') + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + sourceType: row.source_type, + sourceId: row.source_id, + sourceSeq: row.source_seq, + searchText, + summaryText, + refs, + createdAt: row.created_at + } + } + + private toProjectedSearchResult( + row: DeepChatTapeSearchProjectionResultRow, + _sourceRow: DeepChatTapeEntryRow | undefined + ): TapeSearchResult { + const score = + typeof row.score === 'number' && Number.isFinite(row.score) ? row.score : undefined + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + createdAt: row.created_at, + summary: row.summary_text, + refs: parseProjectionRefs(row.refs_json), + ...(score === undefined ? {} : { score }) + } + } + + private toContextEntry( + row: DeepChatTapeEntryRow, + projectionRow: DeepChatTapeSearchProjectionRow | undefined, + maxBytes: number + ): AgentTapeContextEntry { + const fallbackProjection = projectionRow ? null : this.toProjectionInput(row) + const payload = parseJsonObject(row.payload_json) + const meta = parseJsonObject(row.meta_json) + const evidenceSource = buildTapeRowEvidenceText(row, payload, meta) + const clipped = truncateToUtf8Bytes(evidenceSource, maxBytes) + const bytes = Buffer.byteLength(clipped.text, 'utf8') + return { + entryId: row.entry_id, + kind: row.kind, + name: row.name, + summary: projectionRow?.summary_text ?? fallbackProjection?.summaryText ?? '', + refs: projectionRow + ? parseProjectionRefs(projectionRow.refs_json) + : (fallbackProjection?.refs ?? {}), + evidence: { + text: clipped.text, + truncated: clipped.truncated, + bytes + }, + createdAt: row.created_at + } + } + + private toSearchResult(row: DeepChatTapeEntryRow): TapeSearchResult { + const projection = this.toProjectionInput(row) + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + createdAt: row.created_at, + summary: projection.summaryText, + refs: projection.refs + } + } + + private toAnchorResult(row: DeepChatTapeEntryRow): TapeAnchorResult { + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + payload: parseJsonObject(row.payload_json), + meta: parseJsonObject(row.meta_json), + createdAt: row.created_at + } + } +} diff --git a/src/main/tape/application/reconcilerService.ts b/src/main/tape/application/reconcilerService.ts new file mode 100644 index 0000000000..b8379cbda2 --- /dev/null +++ b/src/main/tape/application/reconcilerService.ts @@ -0,0 +1,123 @@ +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { TapeApplicationProviders } from '../ports/application' +import type { TapeBackfillResult, TapeTranscriptReader } from '../ports/capabilities' +import { appendMessageRecordToTape } from './factPersistence' +import type { TapeFactService } from './factService' +import { migrationProvenanceKey } from './common' + +type TapeReconcilerProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getLegacySummaryReader' +> + +function legacySummaryProvenanceKey(sessionId: string): string { + return `summary:${sessionId}:legacy-summary:v1` +} + +export class TapeReconcilerService { + constructor( + private readonly providers: TapeReconcilerProviders, + private readonly facts: TapeFactService + ) {} + + private get table() { + return this.providers.getEntryStore() + } + + ensureSessionTapeReady( + sessionId: string, + messageStore: TapeTranscriptReader + ): TapeBackfillResult { + const table = this.table + const historyRecords = [...messageStore.getMessages(sessionId)].sort( + (left, right) => left.orderSeq - right.orderSeq + ) + const maxOrderSeq = historyRecords.reduce( + (currentMax, record) => Math.max(currentMax, record.orderSeq), + 0 + ) + + table.ensureBootstrapAnchor(sessionId) + + let appendedFactCount = 0 + for (const record of historyRecords) { + appendedFactCount += appendMessageRecordToTape(table, record, 'backfill') + } + + this.backfillLegacySummaryAnchor(sessionId, historyRecords) + + table.appendEvent({ + sessionId, + name: 'migration/backfill', + source: { + type: 'migration', + id: 'message-backfill', + seq: 1 + }, + provenanceKey: migrationProvenanceKey(sessionId), + data: { + source: 'deepchat_messages', + messageCount: historyRecords.length, + maxOrderSeq + }, + idempotent: true + }) + + return { + sessionId, + migrationState: 'ready', + messageCount: historyRecords.length, + maxOrderSeq, + appendedFactCount, + historyRecords: this.facts.getMessageRecords(sessionId) + } + } + + private backfillLegacySummaryAnchor( + sessionId: string, + historyRecords: ChatMessageRecord[] + ): void { + const table = this.table + if (table.getLatestSummaryAnchor(sessionId)) { + return + } + + const legacyState = this.providers.getLegacySummaryReader().getSummaryState(sessionId) + if (!legacyState) { + return + } + + const summary = legacyState.summary_text?.trim() + if (!summary) { + return + } + + const cursorOrderSeq = Math.max(1, legacyState.summary_cursor_order_seq ?? 1) + const sourceRecords = historyRecords.filter((record) => record.orderSeq < cursorOrderSeq) + table.appendAnchor({ + sessionId, + name: 'compaction/migrated_summary', + source: { + type: 'summary', + id: 'legacy-summary', + seq: 1 + }, + provenanceKey: legacySummaryProvenanceKey(sessionId), + state: { + summary, + cursorOrderSeq, + range: + sourceRecords.length > 0 + ? { + fromOrderSeq: sourceRecords[0].orderSeq, + toOrderSeq: sourceRecords[sourceRecords.length - 1].orderSeq + } + : null, + sourceMessageIds: sourceRecords.map((record) => record.id), + migratedFrom: 'deepchat_sessions.summary_text' + }, + idempotent: true, + createdAt: legacyState.summary_updated_at ?? undefined + }) + } +} diff --git a/src/main/tape/application/sessionTape.ts b/src/main/tape/application/sessionTape.ts new file mode 100644 index 0000000000..8a1fccd651 --- /dev/null +++ b/src/main/tape/application/sessionTape.ts @@ -0,0 +1,282 @@ +import type { + AgentTapeAnchorsOptions, + AgentTapeContextOptions, + AgentTapeContextResult, + AgentTapeHandoffState, + AgentTapeSearchOptions, + ChatMessageRecord, + SubagentTapeLinkInput, + SubagentTapeLinkReceipt +} from '@shared/types/agent-interface' +import type { + DeepChatTapeViewManifest, + DeepChatTapeViewManifestRecord +} from '@shared/types/tape-view-manifest' +import type { + DeepChatCausalObservationReadOptions, + DeepChatCausalObservationSlice, + DeepChatTapeReplayExportOptions, + DeepChatTapeReplaySlice +} from '@shared/types/tape-replay' +import type { DeepChatTapeEntryRow, TapeAnchorAppendInput } from '../domain/entry' +import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' +import type { + TapeAnchorReader, + TapeAnchorWriter, + TapeEffectiveMessageSourceEntry, + TapeInspectionReader, + TapeLifecycleAdmin, + TapeMessageFactWriter, + TapeRawEntryReader, + TapeReconciliationPort, + TapeToolFactWriter, + TapeTranscriptReader, + TapeMemoryViewManifestInspection, + TapeViewManifestReader, + TapeViewManifestWriter +} from '../ports/capabilities' +import { + createTapeApplicationProviders, + type TapeApplicationDatabase, + type TapeApplicationProviders +} from '../ports/application' +import type { + TapeAnchorResult, + TapeBackfillResult, + TapeForkHandle, + TapeInfo, + TapeMigrationState, + TapeSearchResult, + TapeViewManifestAssemblySources +} from './contracts' +import { normalizeTapeHandoffState, TapeFactService } from './factService' +import { TapeForkService } from './forkService' +import { deleteTapeGeneration, resetTapeGeneration } from './generationLifecycle' +import { + AgentTapeViewError, + normalizeSubagentTapeLinkInput, + TapeLineageService, + type AgentTapeViewErrorCode +} from './lineageService' +import { TapeRecallService } from './recallService' +import { TapeReconcilerService } from './reconcilerService' +import { TapeViewReplayService } from './viewReplayService' + +export type { + AgentTapeViewErrorCode, + TapeAnchorResult, + TapeBackfillResult, + TapeForkHandle, + TapeInfo, + TapeMigrationState, + TapeSearchResult, + TapeViewManifestAssemblySources +} +export { AgentTapeViewError, normalizeSubagentTapeLinkInput, normalizeTapeHandoffState } + +export class SessionTape + implements + TapeToolFactWriter, + TapeMessageFactWriter, + TapeRawEntryReader, + TapeReconciliationPort, + TapeViewManifestReader, + TapeViewManifestWriter, + TapeAnchorReader, + TapeAnchorWriter, + TapeInspectionReader, + TapeLifecycleAdmin +{ + private readonly providers: TapeApplicationProviders + private readonly facts: TapeFactService + private readonly reconciler: TapeReconcilerService + private readonly recall: TapeRecallService + private readonly lineage: TapeLineageService + private readonly viewReplay: TapeViewReplayService + private readonly forks: TapeForkService + + constructor(database: TapeApplicationDatabase) { + this.providers = createTapeApplicationProviders(database) + this.facts = new TapeFactService(this.providers) + this.lineage = new TapeLineageService(this.providers) + this.reconciler = new TapeReconcilerService(this.providers, this.facts) + this.recall = new TapeRecallService(this.providers, this.lineage) + this.viewReplay = new TapeViewReplayService(this.providers) + this.forks = new TapeForkService(this.providers) + } + + ensureSessionTapeReady( + sessionId: string, + messageStore: TapeTranscriptReader + ): TapeBackfillResult { + return this.reconciler.ensureSessionTapeReady(sessionId, messageStore) + } + + appendMessageRecord(record: ChatMessageRecord): number { + return this.facts.appendMessageRecord(record) + } + + appendMessageReplacement(record: ChatMessageRecord, reason: string): number { + return this.facts.appendMessageReplacement(record, reason) + } + + appendMessageRetraction(record: ChatMessageRecord, reason: string): number { + return this.facts.appendMessageRetraction(record, reason) + } + + appendToolFact(input: TapeToolFactInput): Promise { + return this.facts.appendToolFact(input) + } + + getMessageRecords(sessionId: string): ChatMessageRecord[] { + return this.facts.getMessageRecords(sessionId) + } + + info(sessionId: string): TapeInfo { + return this.recall.info(sessionId) + } + + search(sessionId: string, query: string, options?: AgentTapeSearchOptions): TapeSearchResult[] { + return this.recall.search(sessionId, query, options) + } + + getContext( + sessionId: string, + entryIds: number[], + options: AgentTapeContextOptions = {} + ): AgentTapeContextResult { + return this.recall.getContext(sessionId, entryIds, options) + } + + anchors(sessionId: string, options: AgentTapeAnchorsOptions = {}): TapeAnchorResult[] { + return this.recall.anchors(sessionId, options) + } + + getViewManifestSourceMaps( + sessionId: string, + messageId?: string + ): TapeViewManifestAssemblySources { + return this.viewReplay.getViewManifestSourceMaps(sessionId, messageId) + } + + appendViewManifest(manifest: DeepChatTapeViewManifest): DeepChatTapeEntryRow { + return this.viewReplay.appendViewManifest(manifest) + } + + listViewManifestsByMessage( + sessionId: string, + messageId: string + ): DeepChatTapeViewManifestRecord[] { + return this.viewReplay.listViewManifestsByMessage(sessionId, messageId) + } + + exportReplaySlice( + sessionId: string, + messageId: string, + options: DeepChatTapeReplayExportOptions = {} + ): DeepChatTapeReplaySlice | null { + return this.viewReplay.exportReplaySlice(sessionId, messageId, options) + } + + readCausalObservationSlice( + sessionId: string, + messageId: string, + options: DeepChatCausalObservationReadOptions = {} + ): DeepChatCausalObservationSlice { + return this.viewReplay.readCausalObservationSlice(sessionId, messageId, options) + } + + handoff( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): DeepChatTapeEntryRow { + return this.facts.handoff(sessionId, name, state, meta) + } + + handoffResult( + sessionId: string, + name: string, + state: AgentTapeHandoffState, + meta: Record = {} + ): TapeAnchorResult { + return this.facts.handoffResult(sessionId, name, state, meta) + } + + createFork(parentSessionId: string, forkId?: string): TapeForkHandle { + return this.forks.createFork(parentSessionId, forkId) + } + + appendForkMessageRecord(handle: TapeForkHandle, record: ChatMessageRecord): number { + return this.facts.appendMessageRecordForSession(handle.forkSessionId, record) + } + + mergeFork(parentSessionId: string, forkId: string): number { + return this.forks.mergeFork(parentSessionId, forkId) + } + + discardFork(parentSessionId: string, forkId: string): void { + this.forks.discardFork(parentSessionId, forkId) + } + + recordExternalForkMerge( + parentSessionId: string, + forkSessionId: string, + forkId: string, + meta: Record = {} + ): DeepChatTapeEntryRow { + return this.forks.recordExternalForkMerge(parentSessionId, forkSessionId, forkId, meta) + } + + recordExternalForkDiscard( + parentSessionId: string, + forkSessionId: string, + forkId: string, + meta: Record = {} + ): DeepChatTapeEntryRow { + return this.forks.recordExternalForkDiscard(parentSessionId, forkSessionId, forkId, meta) + } + + linkSubagentTape(input: SubagentTapeLinkInput): SubagentTapeLinkReceipt { + return this.lineage.linkSubagentTape(input) + } + + getBySession(sessionId: string): DeepChatTapeEntryRow[] { + return this.providers.getEntryStore().getBySession(sessionId) + } + + getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + return this.providers.getEntryStore().getLatestReconstructionAnchor(sessionId) + } + + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow { + return this.facts.appendAnchor(input) + } + + getEffectiveMessageSourceSpan( + sessionId: string, + entryIds: number[] + ): TapeEffectiveMessageSourceEntry[] { + return this.recall.getEffectiveMessageSourceSpan(sessionId, entryIds) + } + + listMemoryViewManifestsByAgent( + agentId: string, + options?: { sessionId?: string; limit?: number; messageId?: string } + ): TapeMemoryViewManifestInspection[] { + return this.viewReplay.listMemoryViewManifestsByAgent(agentId, options) + } + + initializeSessionTape(sessionId: string): void { + this.providers.getEntryStore().ensureBootstrapAnchor(sessionId) + } + + deleteSessionTape(sessionId: string): void { + deleteTapeGeneration(this.providers, sessionId) + } + + resetSessionTape(sessionId: string): void { + resetTapeGeneration(this.providers, sessionId) + } +} diff --git a/src/main/tape/application/viewReplayService.ts b/src/main/tape/application/viewReplayService.ts new file mode 100644 index 0000000000..1e1f8dbff0 --- /dev/null +++ b/src/main/tape/application/viewReplayService.ts @@ -0,0 +1,523 @@ +import type { + DeepChatTapeViewManifest, + DeepChatTapeViewManifestRecord +} from '@shared/types/tape-view-manifest' +import type { + DeepChatCausalObservationReadOptions, + DeepChatCausalObservationRequest, + DeepChatCausalObservationSlice, + DeepChatTapeReplayEntrySnapshot, + DeepChatTapeReplayExportOptions, + DeepChatTapeReplaySlice, + DeepChatTapeReplayTraceSnapshot +} from '@shared/types/tape-replay' +import { SUMMARY_ANCHOR_NAMES, type DeepChatTapeEntryRow } from '../domain/entry' +import { buildEffectiveTapeView } from '../domain/effectiveView' +import { + collectEntryIds, + hashString, + isPositiveInteger, + normalizeStoredTapeViewManifest, + withReplaySliceHash +} from '../domain/replay' +import { + hashJson, + TAPE_VIEW_MANIFEST_EVENT_NAME, + verifyTapeViewManifestHash +} from '../domain/viewManifest' +import type { + TapeApplicationProviders, + TapeMessageTraceRow as DeepChatMessageTraceRow +} from '../ports/application' +import type { TapeMemoryViewManifestInspection } from '../ports/capabilities' +import { parseJsonObject } from './common' +import type { TapeViewManifestAssemblySources } from './contracts' + +type TapeViewReplayProviders = Pick< + TapeApplicationProviders, + 'getEntryStore' | 'getMessageTraceReader' | 'getTerminalMessageReader' +> + +const BOOTSTRAP_ANCHOR_NAME = 'session/start' + +function isReconstructionAnchorName(name: string | null): boolean { + if (name === null) { + return false + } + return ( + (SUMMARY_ANCHOR_NAMES as readonly string[]).includes(name) || + name.startsWith('handoff/') || + name.startsWith('auto_handoff/') + ) +} + +function readToolFactStatus(row: DeepChatTapeEntryRow): string | null { + const status = parseJsonObject(row.meta_json).status + return typeof status === 'string' ? status : null +} + +function readToolFactToolCallId(row: DeepChatTapeEntryRow): string | null { + const payload = parseJsonObject(row.payload_json) + if (row.kind === 'tool_call') { + const toolCall = payload.toolCall + if (toolCall && typeof toolCall === 'object' && !Array.isArray(toolCall)) { + const id = (toolCall as Record).id + return typeof id === 'string' && id.length > 0 ? id : null + } + return null + } + const toolCallId = payload.toolCallId + return typeof toolCallId === 'string' && toolCallId.length > 0 ? toolCallId : null +} + +function readToolFactMessageId(row: DeepChatTapeEntryRow): string | null { + const messageId = parseJsonObject(row.payload_json).messageId + return typeof messageId === 'string' && messageId.length > 0 ? messageId : null +} + +function deriveSelectedMemoryIds(value: unknown): string[] | null { + if (!Array.isArray(value)) return null + const ids = new Set() + for (const item of value) { + const id = + typeof item === 'string' + ? item + : item && typeof item === 'object' && !Array.isArray(item) + ? (item as Record).id + : null + if (typeof id === 'string' && id.length > 0) ids.add(id) + } + return [...ids] +} + +function toMemoryViewManifestInspection( + row: DeepChatTapeEntryRow +): TapeMemoryViewManifestInspection | null { + const payload = parseJsonObject(row.payload_json) + const meta = parseJsonObject(row.meta_json) + const manifest = + payload.state && typeof payload.state === 'object' && !Array.isArray(payload.state) + ? (payload.state as Record) + : null + if (!manifest) return null + const readNumber = (value: unknown): number => + typeof value === 'number' && Number.isFinite(value) ? value : 0 + return { + sessionId: row.session_id, + messageId: typeof meta.messageId === 'string' ? meta.messageId : null, + entryId: row.entry_id, + policyVersion: + typeof manifest.policyVersion === 'number' && Number.isFinite(manifest.policyVersion) + ? manifest.policyVersion + : null, + tokenBudget: readNumber(manifest.tokenBudget), + estimatedTokens: readNumber(manifest.estimatedTokens), + selectedCount: Array.isArray(manifest.selected) ? manifest.selected.length : 0, + selectedIds: deriveSelectedMemoryIds(manifest.selected), + droppedCount: Array.isArray(manifest.dropped) ? manifest.dropped.length : 0, + queryHash: typeof manifest.queryHash === 'string' ? manifest.queryHash : null, + createdAt: row.created_at + } +} + +export class TapeViewReplayService { + constructor(private readonly providers: TapeViewReplayProviders) {} + + private get table() { + return this.providers.getEntryStore() + } + + getViewManifestSourceMaps( + sessionId: string, + messageId?: string + ): TapeViewManifestAssemblySources { + const table = this.table + const rows = table.getBySession(sessionId) + const entryIdByMessageId = new Map() + const toolCallEntryIdByToolId = new Map() + const toolResultEntryIdByToolId = new Map() + let latestEntryId = 0 + const anchorEntryIds: number[] = [] + let reconstructionAnchorEntryId: number | null = null + let bootstrapAnchorEntryId: number | null = null + + for (const row of rows) { + latestEntryId = Math.max(latestEntryId, row.entry_id) + if (row.kind === 'anchor') { + anchorEntryIds.push(row.entry_id) + if (isReconstructionAnchorName(row.name)) { + if (reconstructionAnchorEntryId === null || row.entry_id > reconstructionAnchorEntryId) { + reconstructionAnchorEntryId = row.entry_id + } + } else if (row.name === BOOTSTRAP_ANCHOR_NAME) { + bootstrapAnchorEntryId = row.entry_id + } + continue + } + if (row.kind === 'message' && row.source_type === 'message' && row.source_id) { + entryIdByMessageId.set(row.source_id, row.entry_id) + continue + } + if (row.kind === 'tool_call' || row.kind === 'tool_result') { + if (messageId && readToolFactMessageId(row) !== messageId) { + continue + } + const toolCallId = readToolFactToolCallId(row) + if (!toolCallId || readToolFactStatus(row) === 'pending') { + continue + } + const target = + row.kind === 'tool_call' ? toolCallEntryIdByToolId : toolResultEntryIdByToolId + target.set(toolCallId, row.entry_id) + } + } + + const reconstructionAnchorEntryIds = + reconstructionAnchorEntryId !== null + ? [reconstructionAnchorEntryId] + : bootstrapAnchorEntryId !== null + ? [bootstrapAnchorEntryId] + : [] + + return { + latestEntryId, + anchorEntryIds, + reconstructionAnchorEntryIds, + reconstructionAnchorEntryId, + entryIdByMessageId, + toolCallEntryIdByToolId, + toolResultEntryIdByToolId + } + } + + listMemoryViewManifestsByAgent( + agentId: string, + options?: { sessionId?: string; limit?: number; messageId?: string } + ): TapeMemoryViewManifestInspection[] { + return this.table + .listMemoryViewManifestAnchorsByAgent(agentId, options) + .map(toMemoryViewManifestInspection) + .filter((manifest): manifest is TapeMemoryViewManifestInspection => manifest !== null) + .filter((manifest) => !options?.messageId || manifest.messageId === options.messageId) + } + + appendViewManifest(manifest: DeepChatTapeViewManifest): DeepChatTapeEntryRow { + const table = this.table + table.ensureBootstrapAnchor(manifest.sessionId) + return table.appendEvent({ + sessionId: manifest.sessionId, + name: TAPE_VIEW_MANIFEST_EVENT_NAME, + source: { + type: 'runtime_event', + id: manifest.messageId, + seq: manifest.requestSeq + }, + provenanceKey: `view:${manifest.sessionId}:${manifest.messageId}:${manifest.requestSeq}:${manifest.hashes.manifestHash}`, + data: { + manifest + }, + meta: { + viewId: manifest.viewId, + requestSeq: manifest.requestSeq, + taskType: manifest.taskType, + policy: manifest.policy, + policyVersion: manifest.policyVersion + }, + createdAt: manifest.assembledAt, + idempotent: true + }) + } + + listViewManifestsByMessage( + sessionId: string, + messageId: string + ): DeepChatTapeViewManifestRecord[] { + const table = this.table + return table + .getBySession(sessionId) + .filter( + (row) => + row.kind === 'event' && + row.name === TAPE_VIEW_MANIFEST_EVENT_NAME && + row.source_type === 'runtime_event' && + row.source_id === messageId + ) + .map((row) => this.toViewManifestRecord(row)) + .filter((record): record is DeepChatTapeViewManifestRecord => Boolean(record)) + .sort((left, right) => right.requestSeq - left.requestSeq || right.entryId - left.entryId) + } + + exportReplaySlice( + sessionId: string, + messageId: string, + options: DeepChatTapeReplayExportOptions = {} + ): DeepChatTapeReplaySlice | null { + if (options.requestSeq !== undefined && !isPositiveInteger(options.requestSeq)) { + throw new Error('requestSeq must be a positive integer.') + } + + const manifests = this.listViewManifestsByMessage(sessionId, messageId) + const manifestRecord = + options.requestSeq === undefined + ? manifests[0] + : manifests.find((record) => record.requestSeq === options.requestSeq) + if (!manifestRecord) { + return null + } + + return this.buildReplaySlice(sessionId, messageId, manifestRecord, options) + } + + readCausalObservationSlice( + sessionId: string, + messageId: string, + options: DeepChatCausalObservationReadOptions = {} + ): DeepChatCausalObservationSlice { + if (options.requestSeq !== undefined && !isPositiveInteger(options.requestSeq)) { + throw new Error('requestSeq must be a positive integer.') + } + + const rows = this.table.getBySession(sessionId) + const manifestRows = rows.filter( + (row) => + row.kind === 'event' && + row.name === TAPE_VIEW_MANIFEST_EVENT_NAME && + row.source_type === 'runtime_event' && + row.source_id === messageId + ) + const traces = this.providers + .getMessageTraceReader() + .listByMessageId(messageId) + .filter( + (row) => + row.session_id === sessionId && + row.message_id === messageId && + isPositiveInteger(row.request_seq) + ) + + const requestSeq = + options.requestSeq ?? + [...manifestRows.map((row) => row.source_seq), ...traces.map((row) => row.request_seq)] + .filter((value): value is number => typeof value === 'number' && isPositiveInteger(value)) + .reduce((latest, value) => Math.max(latest ?? value, value), null) + + let request: DeepChatCausalObservationRequest + if (requestSeq === null) { + request = { state: 'request_unavailable', requestSeq: null, trace: null } + } else { + const selectedManifestRows = manifestRows.filter((row) => row.source_seq === requestSeq) + const manifestRecord = selectedManifestRows + .map((row) => this.toViewManifestRecord(row)) + .find((record) => record?.messageId === messageId && record.requestSeq === requestSeq) + const trace = traces.find((row) => row.request_seq === requestSeq) ?? null + + if (manifestRecord) { + request = { + state: 'manifest_bound', + requestSeq, + replay: this.buildReplaySlice(sessionId, messageId, manifestRecord, options) + } + } else { + request = { + state: selectedManifestRows.length > 0 ? 'manifest_malformed' : 'manifest_missing', + requestSeq, + trace: trace + ? this.toReplayTraceSnapshot(trace, options.includeTracePayload === true) + : null + } + } + } + + const outputEntries = buildEffectiveTapeView(rows, { includePending: false }) + .rows.filter( + (row) => + (row.kind === 'message' && + row.source_type === 'message' && + row.source_id === messageId) || + ((row.kind === 'tool_call' || row.kind === 'tool_result') && + readToolFactMessageId(row) === messageId) + ) + .map((row) => this.toReplayEntrySnapshot(row, options.includeTapePayloads === true)) + const message = this.providers.getTerminalMessageReader().get(messageId) + const terminalMessage = + message?.session_id === sessionId && + message.role === 'assistant' && + (message.status === 'sent' || message.status === 'error') + ? { + status: message.status, + orderSeq: message.order_seq, + createdAt: message.created_at, + updatedAt: message.updated_at, + contentHash: hashString(message.content), + metadataHash: hashString(message.metadata) + } + : null + + return { + schemaVersion: 1, + sessionId, + messageId, + request, + output: { + correlation: 'message_only', + entries: outputEntries, + terminalMessage + }, + runtime: + options.currentRuntimeStatus === undefined + ? { scope: 'unavailable', status: null, eventHistory: 'not_persisted' } + : { + scope: 'current_only', + status: options.currentRuntimeStatus, + eventHistory: 'not_persisted' + } + } + } + + private buildReplaySlice( + sessionId: string, + messageId: string, + manifestRecord: DeepChatTapeViewManifestRecord, + options: DeepChatTapeReplayExportOptions + ): DeepChatTapeReplaySlice { + const table = this.table + const manifest = manifestRecord.manifest + const includedEntryIds = collectEntryIds(manifest.included.map((ref) => ref.entryId)) + const excludedEntryIds = collectEntryIds(manifest.excluded.map((ref) => ref.entryId)) + const anchorEntryIds = collectEntryIds(manifest.anchorEntryIds) + const selectedEntryIds = new Set([ + manifestRecord.entryId, + ...includedEntryIds, + ...excludedEntryIds, + ...anchorEntryIds + ]) + const entries = table + .getBySession(sessionId) + .filter((row) => selectedEntryIds.has(row.entry_id)) + .map((row) => this.toReplayEntrySnapshot(row, options.includeTapePayloads === true)) + + const trace = this.findReplayTrace(sessionId, messageId, manifestRecord.requestSeq) + const createdAt = Date.now() + const sliceBase: Omit & { + hashes: Omit & { sliceHash: '' } + } = { + schemaVersion: 1 as const, + sliceId: `replay_${hashJson({ + sessionId, + messageId, + requestSeq: manifestRecord.requestSeq, + manifestHash: manifest.hashes.manifestHash + }).slice(0, 16)}`, + sessionId, + messageId, + requestSeq: manifestRecord.requestSeq, + mode: trace ? 'trace_bound' : 'manifest_only', + manifestRecord, + trace: trace ? this.toReplayTraceSnapshot(trace, options.includeTracePayload === true) : null, + entries, + refs: { + manifestEntryId: manifestRecord.entryId, + includedEntryIds, + excludedEntryIds, + anchorEntryIds + }, + hashes: { + manifestHash: manifest.hashes.manifestHash, + sliceHash: '' + }, + integrity: manifestRecord.integrity, + createdAt + } + + return withReplaySliceHash(sliceBase) + } + + private toViewManifestRecord(row: DeepChatTapeEntryRow): DeepChatTapeViewManifestRecord | null { + const payload = parseJsonObject(row.payload_json) + const data = payload.data + const rawManifest = + data && typeof data === 'object' && !Array.isArray(data) + ? (data as Record).manifest + : undefined + const manifest = normalizeStoredTapeViewManifest(rawManifest, row.session_id) + if ( + !manifest || + manifest.messageId !== row.source_id || + manifest.requestSeq !== row.source_seq + ) { + return null + } + + return { + sessionId: row.session_id, + messageId: manifest.messageId, + requestSeq: manifest.requestSeq, + entryId: row.entry_id, + createdAt: row.created_at, + integrity: verifyTapeViewManifestHash(manifest), + manifest + } + } + + private findReplayTrace( + sessionId: string, + messageId: string, + requestSeq: number + ): DeepChatMessageTraceRow | null { + const traceTable = this.providers.getMessageTraceReader() + return ( + traceTable + .listByMessageId(messageId) + .find((row) => row.session_id === sessionId && row.request_seq === requestSeq) ?? null + ) + } + + private toReplayEntrySnapshot( + row: DeepChatTapeEntryRow, + includePayloads: boolean + ): DeepChatTapeReplayEntrySnapshot { + const snapshot: DeepChatTapeReplayEntrySnapshot = { + entryId: row.entry_id, + kind: row.kind, + name: row.name, + sourceType: row.source_type, + sourceId: row.source_id, + sourceSeq: row.source_seq, + provenanceKey: row.provenance_key, + payloadHash: hashString(row.payload_json), + metaHash: hashString(row.meta_json), + createdAt: row.created_at + } + + if (includePayloads) { + snapshot.payload = parseJsonObject(row.payload_json) + snapshot.meta = parseJsonObject(row.meta_json) + } + + return snapshot + } + + private toReplayTraceSnapshot( + row: DeepChatMessageTraceRow, + includePayload: boolean + ): DeepChatTapeReplayTraceSnapshot { + const snapshot: DeepChatTapeReplayTraceSnapshot = { + id: row.id, + requestSeq: row.request_seq, + providerId: row.provider_id, + modelId: row.model_id, + endpoint: row.endpoint, + headersHash: hashString(row.headers_json), + bodyHash: hashString(row.body_json), + truncated: row.truncated === 1, + createdAt: row.created_at + } + + if (includePayload) { + snapshot.headersJson = row.headers_json + snapshot.bodyJson = row.body_json + } + + return snapshot + } +} diff --git a/src/main/tape/domain/effectiveSemantics.ts b/src/main/tape/domain/effectiveSemantics.ts new file mode 100644 index 0000000000..a50091b1a9 --- /dev/null +++ b/src/main/tape/domain/effectiveSemantics.ts @@ -0,0 +1,161 @@ +import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryRow } from './entry' + +const TERMINAL_TAPE_TOOL_STATUSES = new Set(['success', 'error']) + +export interface DeepChatTapeToolIdentity { + key: string + messageId: string +} + +export function parseTapeJsonObject(raw: string): Record { + try { + const parsed = JSON.parse(raw) as unknown + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record + } + } catch {} + return {} +} + +export function parseNestedTapeJsonObject(value: unknown): Record { + if (typeof value === 'string') { + return parseTapeJsonObject(value) + } + if (value && typeof value === 'object' && !Array.isArray(value)) { + return value as Record + } + return {} +} + +export function parseAssistantBlocks(rawContent: string): AssistantMessageBlock[] { + try { + const parsed = JSON.parse(rawContent) as unknown + return Array.isArray(parsed) ? (parsed as AssistantMessageBlock[]) : [] + } catch { + return [] + } +} + +export function messageRecordHasFinalToolUse(record: ChatMessageRecord): boolean { + if (record.role !== 'assistant' || (record.status !== 'sent' && record.status !== 'error')) { + return false + } + const blocks = parseAssistantBlocks(record.content) + const pendingInteractionToolIds = new Set( + blocks.flatMap((block) => + block?.type === 'action' && + (block.action_type === 'tool_call_permission' || block.action_type === 'question_request') && + block.status === 'pending' && + typeof block.tool_call?.id === 'string' + ? [block.tool_call.id] + : [] + ) + ) + return blocks.some( + (block) => + block?.type === 'tool_call' && + (block.status === 'success' || block.status === 'error') && + typeof block.tool_call?.id === 'string' && + !pendingInteractionToolIds.has(block.tool_call.id) + ) +} + +function isMessageStatus(value: unknown): value is ChatMessageRecord['status'] { + return value === 'pending' || value === 'sent' || value === 'error' +} + +export function tapeEntryToMessageRecord(row: DeepChatTapeEntryRow): ChatMessageRecord | null { + if (row.kind !== 'message') { + return null + } + + const payload = parseTapeJsonObject(row.payload_json) + const record = payload.record + if (!record || typeof record !== 'object' || Array.isArray(record)) { + return null + } + + const candidate = record as Partial + if ( + typeof candidate.id !== 'string' || + typeof candidate.sessionId !== 'string' || + typeof candidate.orderSeq !== 'number' || + (candidate.role !== 'user' && candidate.role !== 'assistant') || + typeof candidate.content !== 'string' + ) { + return null + } + + return { + id: candidate.id, + sessionId: candidate.sessionId, + orderSeq: candidate.orderSeq, + role: candidate.role, + content: candidate.content, + status: isMessageStatus(candidate.status) ? candidate.status : 'sent', + isContextEdge: typeof candidate.isContextEdge === 'number' ? candidate.isContextEdge : 0, + metadata: typeof candidate.metadata === 'string' ? candidate.metadata : '{}', + traceCount: typeof candidate.traceCount === 'number' ? candidate.traceCount : 0, + createdAt: typeof candidate.createdAt === 'number' ? candidate.createdAt : row.created_at, + updatedAt: typeof candidate.updatedAt === 'number' ? candidate.updatedAt : row.created_at + } +} + +export function tapeMessageRank(record: ChatMessageRecord, includePending: boolean): number { + if (record.status === 'sent' || record.status === 'error') { + return 2 + } + return includePending && record.status === 'pending' ? 1 : 0 +} + +export function readTapeMessageRetractionId(row: DeepChatTapeEntryRow): string | null { + if (row.kind !== 'event' || row.name !== 'message/retracted') { + return null + } + + const payload = parseTapeJsonObject(row.payload_json) + const data = parseNestedTapeJsonObject(payload.data) + return typeof data.messageId === 'string' ? data.messageId : null +} + +export function readTapeToolStatus(row: DeepChatTapeEntryRow): string | null { + const meta = parseTapeJsonObject(row.meta_json) + return typeof meta.status === 'string' ? meta.status : null +} + +export function tapeToolRank(row: DeepChatTapeEntryRow, includePending: boolean): number { + const status = readTapeToolStatus(row) + if (status === 'pending') { + return includePending ? 1 : 0 + } + return status !== null && TERMINAL_TAPE_TOOL_STATUSES.has(status) ? 2 : 0 +} + +export function readTapeToolIdentity(row: DeepChatTapeEntryRow): DeepChatTapeToolIdentity | null { + if (row.kind !== 'tool_call' && row.kind !== 'tool_result') { + return null + } + + const payload = parseTapeJsonObject(row.payload_json) + const messageId = payload.messageId + if (typeof messageId !== 'string' || messageId.length === 0) { + return null + } + + let toolCallId: unknown + if (row.kind === 'tool_call') { + toolCallId = parseNestedTapeJsonObject(payload.toolCall).id + } else { + toolCallId = payload.toolCallId + } + + if (typeof toolCallId !== 'string' || toolCallId.length === 0) { + return null + } + + return { + key: `${row.kind}:${messageId}:${toolCallId}`, + messageId + } +} diff --git a/src/main/tape/domain/effectiveView.ts b/src/main/tape/domain/effectiveView.ts new file mode 100644 index 0000000000..377341ace1 --- /dev/null +++ b/src/main/tape/domain/effectiveView.ts @@ -0,0 +1,251 @@ +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { DeepChatTapeEntryKind, DeepChatTapeEntryRow, DeepChatTapeSearchInput } from './entry' +import { + parseNestedTapeJsonObject, + readTapeMessageRetractionId, + readTapeToolIdentity, + tapeEntryToMessageRecord, + tapeMessageRank, + tapeToolRank +} from './effectiveSemantics' + +export interface EffectiveMessageEntry { + entryId: number + record: ChatMessageRecord +} + +export interface EffectiveTapeView { + rows: DeepChatTapeEntryRow[] + messageRecords: ChatMessageRecord[] + /** Effective messages paired with their tape entry_id, ordered by orderSeq (for lineage). */ + messageEntries: EffectiveMessageEntry[] +} + +interface EffectiveTapeViewOptions { + includePending?: boolean + includeAuditEvents?: boolean +} + +type EffectiveMessageCandidate = { + row: DeepChatTapeEntryRow + record: ChatMessageRecord +} + +function compareSqliteBinaryText(left: string, right: string): number { + return Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')) +} + +function toNonNegativeInteger(value: unknown): number | null { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + return null + } + return Math.floor(value) +} + +function readTokenUsage(metadata: Record): number | null { + const totalTokens = toNonNegativeInteger(metadata.totalTokens ?? metadata.total_tokens) + if (totalTokens !== null) { + return totalTokens + } + + const inputTokens = toNonNegativeInteger(metadata.inputTokens ?? metadata.input_tokens) + const outputTokens = toNonNegativeInteger(metadata.outputTokens ?? metadata.output_tokens) + if (inputTokens !== null || outputTokens !== null) { + return (inputTokens ?? 0) + (outputTokens ?? 0) + } + + return null +} + +function shouldReplaceMessage( + current: EffectiveMessageCandidate | undefined, + next: EffectiveMessageCandidate, + includePending: boolean +): boolean { + if (!current) { + return true + } + + const currentRank = tapeMessageRank(current.record, includePending) + const nextRank = tapeMessageRank(next.record, includePending) + if (nextRank > currentRank) { + return true + } + if (nextRank < currentRank) { + return false + } + return next.row.entry_id > current.row.entry_id +} + +function isAuditEvent(row: DeepChatTapeEntryRow): boolean { + return ( + row.name === 'message/retracted' || + row.name === 'message/compaction_indicator' || + row.name === 'migration/backfill' + ) +} + +function shouldReplaceToolRow( + current: DeepChatTapeEntryRow | undefined, + next: DeepChatTapeEntryRow, + includePending: boolean +): boolean { + if (!current) { + return true + } + + const currentRank = tapeToolRank(current, includePending) + const nextRank = tapeToolRank(next, includePending) + if (nextRank > currentRank) { + return true + } + if (nextRank < currentRank) { + return false + } + return next.entry_id > current.entry_id +} + +function matchesKinds( + row: DeepChatTapeEntryRow, + kinds: DeepChatTapeEntryKind[] | undefined +): boolean { + return !kinds?.length || kinds.includes(row.kind) +} + +function matchesCreatedAt(row: DeepChatTapeEntryRow, options: DeepChatTapeSearchInput): boolean { + if ( + Number.isFinite(options.startCreatedAt) && + row.created_at < (options.startCreatedAt as number) + ) { + return false + } + if (Number.isFinite(options.endCreatedAt) && row.created_at > (options.endCreatedAt as number)) { + return false + } + return true +} + +function matchesQuery(row: DeepChatTapeEntryRow, normalizedQuery: string): boolean { + const haystack = `${row.payload_json}\n${row.meta_json}\n${row.name ?? ''}`.toLowerCase() + return haystack.includes(normalizedQuery) +} + +export function buildEffectiveTapeView( + rows: DeepChatTapeEntryRow[], + options: EffectiveTapeViewOptions = {} +): EffectiveTapeView { + const includePending = options.includePending === true + const includeAuditEvents = options.includeAuditEvents === true + const messageCandidates = new Map() + const retractedMessageIds = new Set() + const toolRows = new Map() + const anchorRows: DeepChatTapeEntryRow[] = [] + const eventRows: DeepChatTapeEntryRow[] = [] + + for (const row of [...rows].sort((left, right) => left.entry_id - right.entry_id)) { + if (row.kind === 'anchor') { + anchorRows.push(row) + continue + } + + if (row.kind === 'event') { + const retractedMessageId = readTapeMessageRetractionId(row) + if (retractedMessageId) { + messageCandidates.delete(retractedMessageId) + retractedMessageIds.add(retractedMessageId) + } + if (includeAuditEvents || !isAuditEvent(row)) { + eventRows.push(row) + } + continue + } + + if (row.kind === 'message') { + const record = tapeEntryToMessageRecord(row) + if (!record) { + continue + } + const rank = tapeMessageRank(record, includePending) + if (rank === 0) { + continue + } + const candidate = { row, record } + if (shouldReplaceMessage(messageCandidates.get(record.id), candidate, includePending)) { + messageCandidates.set(record.id, candidate) + retractedMessageIds.delete(record.id) + } + continue + } + + const identity = readTapeToolIdentity(row) + if (!identity || tapeToolRank(row, includePending) === 0) { + continue + } + const current = toolRows.get(identity.key)?.row + if (shouldReplaceToolRow(current, row, includePending)) { + toolRows.set(identity.key, { row, messageId: identity.messageId }) + } + } + + const messageRows = [...messageCandidates.values()] + .filter((candidate) => !retractedMessageIds.has(candidate.record.id)) + .sort( + (left, right) => + left.record.orderSeq - right.record.orderSeq || + compareSqliteBinaryText(left.record.id, right.record.id) + ) + const effectiveMessageIds = new Set(messageRows.map((candidate) => candidate.record.id)) + const effectiveToolRows = [...toolRows.values()] + .filter((candidate) => effectiveMessageIds.has(candidate.messageId)) + .map((candidate) => candidate.row) + const effectiveRows = [ + ...anchorRows, + ...eventRows, + ...messageRows.map((candidate) => candidate.row), + ...effectiveToolRows + ].sort((left, right) => left.entry_id - right.entry_id) + + return { + rows: effectiveRows, + messageRecords: messageRows.map((candidate) => candidate.record), + messageEntries: messageRows.map((candidate) => ({ + entryId: candidate.row.entry_id, + record: candidate.record + })) + } +} + +export function searchEffectiveTapeRows( + rows: DeepChatTapeEntryRow[], + query: string, + options: DeepChatTapeSearchInput = {} +): DeepChatTapeEntryRow[] { + const normalizedQuery = query.trim().toLowerCase() + if (!normalizedQuery) { + return [] + } + + const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 + const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) + return buildEffectiveTapeView(rows, { includePending: false }) + .rows.filter((row) => matchesKinds(row, options.kinds)) + .filter((row) => matchesCreatedAt(row, options)) + .filter((row) => matchesQuery(row, normalizedQuery)) + .sort((left, right) => right.entry_id - left.entry_id) + .slice(0, cappedLimit) +} + +export function getLastEffectiveTokenUsage(rows: DeepChatTapeEntryRow[]): number | null { + const effectiveRows = buildEffectiveTapeView(rows, { includePending: false }).rows + for (let index = effectiveRows.length - 1; index >= 0; index -= 1) { + const record = tapeEntryToMessageRecord(effectiveRows[index]) + if (!record || record.role !== 'assistant') { + continue + } + const usage = readTokenUsage(parseNestedTapeJsonObject(record.metadata)) + if (usage !== null) { + return usage + } + } + return null +} diff --git a/src/main/tape/domain/entry.ts b/src/main/tape/domain/entry.ts new file mode 100644 index 0000000000..2bfbaf30e3 --- /dev/null +++ b/src/main/tape/domain/entry.ts @@ -0,0 +1,118 @@ +export type DeepChatTapeEntryKind = 'event' | 'anchor' | 'message' | 'tool_call' | 'tool_result' + +export const TAPE_INCARNATION_META_KEY = 'tapeIncarnationId' + +export type DeepChatTapeSourceType = + | 'session' + | 'message' + | 'assistant_block' + | 'tool_call' + | 'tool_result' + | 'runtime_event' + | 'migration' + | 'summary' + | 'fork' + | 'subagent' + +export interface DeepChatTapeEntryRow { + session_id: string + entry_id: number + kind: DeepChatTapeEntryKind + name: string | null + source_type: DeepChatTapeSourceType | null + source_id: string | null + source_seq: number | null + provenance_key: string | null + payload_json: string + meta_json: string + created_at: number +} + +export interface DeepChatTapeSourceInput { + type: DeepChatTapeSourceType + id: string + seq?: number | null +} + +export interface DeepChatTapeAppendInput { + sessionId: string + kind: DeepChatTapeEntryKind + name?: string | null + source?: DeepChatTapeSourceInput | null + provenanceKey?: string | null + payload: Record + meta?: Record + createdAt?: number + idempotent?: boolean +} + +export interface TapeAnchorAppendInput { + sessionId: string + name: string + state: Record + meta?: Record + source?: DeepChatTapeSourceInput | null + provenanceKey?: string | null + createdAt?: number + idempotent?: boolean +} + +export interface TapeEventAppendInput { + sessionId: string + name: string + data: Record + meta?: Record + source?: DeepChatTapeSourceInput | null + provenanceKey?: string | null + createdAt?: number + idempotent?: boolean +} + +export interface DeepChatTapeSearchInput { + limit?: number + kinds?: DeepChatTapeEntryKind[] + startCreatedAt?: number + endCreatedAt?: number +} + +export interface DeepChatTapeReadSource { + sessionId: string + maxEntryId: number +} + +export const SUMMARY_ANCHOR_NAMES = [ + 'compaction/auto', + 'compaction/manual', + 'compaction/context_pressure', + 'compaction/resume', + 'compaction/migrated_summary', + 'auto_handoff/context_overflow', + 'summary/reset' +] as const + +export function normalizeDeepChatTapeReadSources( + sources: readonly DeepChatTapeReadSource[] +): DeepChatTapeReadSource[] { + const maxEntryIdBySession = new Map() + for (const source of sources) { + const sessionId = source.sessionId.trim() + if (!sessionId || !Number.isSafeInteger(source.maxEntryId) || source.maxEntryId < 0) { + continue + } + maxEntryIdBySession.set( + sessionId, + Math.max(maxEntryIdBySession.get(sessionId) ?? 0, source.maxEntryId) + ) + } + return [...maxEntryIdBySession.entries()] + .map(([sessionId, maxEntryId]) => ({ sessionId, maxEntryId })) + .sort((left, right) => + left.sessionId < right.sessionId ? -1 : left.sessionId > right.sessionId ? 1 : 0 + ) +} + +export function serializeDeepChatTapeReadSources( + sources: readonly DeepChatTapeReadSource[] +): string { + return JSON.stringify(normalizeDeepChatTapeReadSources(sources)) +} diff --git a/src/main/tape/domain/facts.ts b/src/main/tape/domain/facts.ts new file mode 100644 index 0000000000..2c30b6abbe --- /dev/null +++ b/src/main/tape/domain/facts.ts @@ -0,0 +1,29 @@ +import type { AssistantMessageBlock } from '@shared/types/agent-interface' + +declare const tapeSessionIdBrand: unique symbol + +export type TapeSessionId = string & { readonly [tapeSessionIdBrand]: 'TapeSessionId' } + +export const toTapeSessionId = (value: string): TapeSessionId => value as TapeSessionId + +export interface TapeEntryRef { + sessionId: TapeSessionId + entryId: number +} + +export interface TapeFactProvenance { + source: 'message' | 'tool_call' | 'tool_result' | 'runtime_event' + sourceId: string + sequence: number +} + +export interface TapeToolFactInput { + sessionId: TapeSessionId + messageId: string + orderSeq: number + blockIndex: number + block: AssistantMessageBlock + provenance: TapeFactProvenance +} + +export type TapeFactSource = 'live' | 'backfill' | 'repair' diff --git a/src/main/tape/domain/replay.ts b/src/main/tape/domain/replay.ts new file mode 100644 index 0000000000..c9b5e75b37 --- /dev/null +++ b/src/main/tape/domain/replay.ts @@ -0,0 +1,193 @@ +import { createHash } from 'crypto' +import type { + DeepChatTapeViewExcludedRange, + DeepChatTapeViewManifest +} from '@shared/types/tape-view-manifest' +import type { DeepChatTapeReplaySlice } from '@shared/types/tape-replay' +import { hashJson } from './viewManifest' + +const VIEW_POLICIES = new Set([ + 'legacy_context_v1', + 'legacy_context_shadow', + 'resume_shadow', + 'tool_loop_shadow', + 'context_pressure_recovery_shadow' +]) + +const VIEW_ENTRY_REASONS = new Set([ + 'system_prompt', + 'selected_history', + 'new_user_input', + 'resume_target', + 'tool_loop_message' +]) + +const VIEW_EXCLUDED_REASONS = new Set([ + 'before_summary_cursor', + 'compaction_indicator', + 'pending_not_context_history', + 'out_of_budget', + 'empty_after_formatting', + 'superseded', + 'retracted' +]) + +function isRecordObject(value: unknown): value is Record { + return Boolean(value && typeof value === 'object' && !Array.isArray(value)) +} + +function isNullableString(value: unknown): value is string | null { + return value === null || typeof value === 'string' +} + +function isNullableNumber(value: unknown): value is number | null { + return value === null || typeof value === 'number' +} + +function isViewEntryRef(value: unknown): value is DeepChatTapeViewManifest['included'][number] { + if (!isRecordObject(value)) return false + + return ( + isNullableNumber(value.entryId) && + isNullableString(value.messageId) && + isNullableNumber(value.orderSeq) && + (value.role === 'system' || + value.role === 'user' || + value.role === 'assistant' || + value.role === 'tool' || + value.role === null) && + (value.source === 'tape' || value.source === 'synthetic') && + typeof value.reason === 'string' && + VIEW_ENTRY_REASONS.has(value.reason) + ) +} + +function isViewExcludedRef(value: unknown): value is DeepChatTapeViewManifest['excluded'][number] { + if (!isRecordObject(value)) return false + + return ( + isNullableNumber(value.entryId) && + isNullableString(value.messageId) && + isNullableNumber(value.orderSeq) && + typeof value.reason === 'string' && + VIEW_EXCLUDED_REASONS.has(value.reason) + ) +} + +function isViewExcludedRange(value: unknown): value is DeepChatTapeViewExcludedRange { + if (!isRecordObject(value)) return false + + return ( + typeof value.fromOrderSeq === 'number' && + typeof value.toOrderSeq === 'number' && + typeof value.count === 'number' && + typeof value.reason === 'string' && + VIEW_EXCLUDED_REASONS.has(value.reason) + ) +} + +function hasNumberFields(value: unknown, fields: string[]): value is Record { + if (!isRecordObject(value)) return false + return fields.every((field) => typeof value[field] === 'number') +} + +function hasStringFields(value: unknown, fields: string[]): value is Record { + if (!isRecordObject(value)) return false + return fields.every((field) => typeof value[field] === 'string') +} + +function isViewManifestMeta(value: unknown): value is DeepChatTapeViewManifest['meta'] { + if (!isRecordObject(value)) return false + + return ( + typeof value.providerId === 'string' && + typeof value.modelId === 'string' && + typeof value.summaryCursorOrderSeq === 'number' && + typeof value.supportsVision === 'boolean' && + typeof value.supportsAudioInput === 'boolean' && + typeof value.traceDebugEnabled === 'boolean' + ) +} + +export function isTapeViewManifest( + value: unknown, + sessionId: string +): value is DeepChatTapeViewManifest { + if (!isRecordObject(value)) return false + + return ( + (value.schemaVersion === 1 || value.schemaVersion === 2) && + typeof value.hashVersion === 'number' && + value.sessionId === sessionId && + typeof value.viewId === 'string' && + typeof value.messageId === 'string' && + typeof value.requestSeq === 'number' && + (value.taskType === 'chat' || value.taskType === 'resume' || value.taskType === 'tool_loop') && + typeof value.policy === 'string' && + VIEW_POLICIES.has(value.policy) && + (typeof value.policyVersion === 'number' || value.policyVersion === null) && + value.contextBuilderVersion === 'legacy-v1' && + typeof value.latestEntryId === 'number' && + Array.isArray(value.anchorEntryIds) && + value.anchorEntryIds.every((entryId) => typeof entryId === 'number') && + (value.reconstructionAnchorEntryId === undefined || + isNullableNumber(value.reconstructionAnchorEntryId)) && + (value.excludedRanges === undefined || + (Array.isArray(value.excludedRanges) && value.excludedRanges.every(isViewExcludedRange))) && + Array.isArray(value.included) && + value.included.every(isViewEntryRef) && + Array.isArray(value.excluded) && + value.excluded.every(isViewExcludedRef) && + hasNumberFields(value.tokenBudget, [ + 'contextLength', + 'requestedMaxTokens', + 'effectiveMaxTokens', + 'reserveTokens', + 'toolReserveTokens', + 'estimatedPromptTokens' + ]) && + hasStringFields(value.hashes, ['promptHash', 'toolDefinitionsHash', 'manifestHash']) && + isViewManifestMeta(value.meta) && + typeof value.assembledAt === 'number' + ) +} + +export function normalizeStoredTapeViewManifest( + value: unknown, + sessionId: string +): DeepChatTapeViewManifest | null { + const candidate = + isRecordObject(value) && value.hashVersion === undefined ? { ...value, hashVersion: 1 } : value + return isTapeViewManifest(candidate, sessionId) ? candidate : null +} + +export function hashString(value: string): string { + return createHash('sha256').update(value).digest('hex') +} + +export function isPositiveInteger(value: number): boolean { + return Number.isInteger(value) && value > 0 +} + +export function collectEntryIds(values: Array): number[] { + return [...new Set(values.filter((value): value is number => typeof value === 'number'))].sort( + (left, right) => left - right + ) +} + +export function withReplaySliceHash( + slice: Omit & { + hashes: Omit & { sliceHash: '' } + } +): DeepChatTapeReplaySlice { + const sliceForHash = { ...slice } as Partial + delete sliceForHash.createdAt + delete sliceForHash.integrity + return { + ...slice, + hashes: { + ...slice.hashes, + sliceHash: hashJson(sliceForHash) + } + } +} diff --git a/src/main/tape/domain/viewManifest.ts b/src/main/tape/domain/viewManifest.ts new file mode 100644 index 0000000000..9f4ac268f0 --- /dev/null +++ b/src/main/tape/domain/viewManifest.ts @@ -0,0 +1,363 @@ +import { createHash } from 'crypto' +import type { ChatMessage } from '@shared/types/core/chat-message' +import type { MCPToolDefinition } from '@shared/types/core/mcp' +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { + DeepChatTapeViewEntryRef, + DeepChatTapeViewExcludedRange, + DeepChatTapeViewExcludedRef, + DeepChatTapeViewManifest, + DeepChatTapeViewManifestIntegrity, + DeepChatTapeViewPolicy, + DeepChatTapeViewTaskType, + DeepChatTapeViewTokenBudget +} from '@shared/types/tape-view-manifest' +import { estimateMessagesTokens } from '@shared/utils/messageTokens' + +export type ContextSummaryCursorMetadata = { + summaryCursorOrderSeq: number + preCursorOrderSeqMin: number | null + preCursorOrderSeqMax: number | null + preCursorCount: number +} + +export function isCompactionRecord(record: ChatMessageRecord): boolean { + try { + const metadata = JSON.parse(record.metadata) as { messageType?: string } + return metadata.messageType === 'compaction' + } catch { + return false + } +} + +/** Stable event name persisted for deterministic view reconstruction. */ +export const TAPE_VIEW_MANIFEST_EVENT_NAME = 'view/assembled' +export const TAPE_VIEW_CONTEXT_BUILDER_VERSION = 'legacy-v1' as const + +export type TapeViewManifestLookupMaps = { + entryIdByMessageId?: Map + toolCallEntryIdByToolId?: Map + toolResultEntryIdByToolId?: Map +} + +export type TapeViewManifestBuildInput = { + sessionId: string + messageId: string + requestSeq: number + taskType: DeepChatTapeViewTaskType + policy: DeepChatTapeViewPolicy + policyVersion?: number | null + messages: ChatMessage[] + tools: MCPToolDefinition[] + latestEntryId: number + anchorEntryIds: number[] + reconstructionAnchorEntryId?: number | null + included: DeepChatTapeViewEntryRef[] + excluded: DeepChatTapeViewExcludedRef[] + summaryCursor?: ContextSummaryCursorMetadata + tokenBudget: Omit + providerId: string + modelId: string + summaryCursorOrderSeq: number + supportsVision: boolean + supportsAudioInput: boolean + traceDebugEnabled: boolean + assembledAt?: number +} + +export type TapeViewManifestPolicyInput = { + recoveredFromContextPressure: boolean + isInitialViewRequest: boolean + viewPolicy?: DeepChatTapeViewPolicy + viewPolicyVersion?: number | null +} + +export type TapeViewManifestPolicyResult = { + policy: DeepChatTapeViewPolicy + policyVersion: number | null +} + +export type TapeViewContextSelection = { + includedRecords: Array<{ + record: ChatMessageRecord + reason: DeepChatTapeViewEntryRef['reason'] + }> + excludedRecords: Array<{ + record: ChatMessageRecord + reason: DeepChatTapeViewExcludedRef['reason'] + }> + summaryCursor?: ContextSummaryCursorMetadata + includesSystemPrompt: boolean + newUserMessageId?: string | null +} + +export function resolveTapeViewManifestPolicy( + input: TapeViewManifestPolicyInput +): TapeViewManifestPolicyResult { + if (input.recoveredFromContextPressure) { + return { + policy: 'context_pressure_recovery_shadow', + policyVersion: null + } + } + + if (input.isInitialViewRequest && input.viewPolicy) { + return { + policy: input.viewPolicy, + policyVersion: input.viewPolicyVersion ?? null + } + } + + return { + policy: 'tool_loop_shadow', + policyVersion: null + } +} + +function normalizeForStableJson(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(normalizeForStableJson) + } + + if (!value || typeof value !== 'object') { + return value + } + + const record = value as Record + return Object.keys(record) + .sort() + .reduce>((result, key) => { + const nested = record[key] + if (nested !== undefined) { + result[key] = normalizeForStableJson(nested) + } + return result + }, {}) +} + +export function stableJsonStringify(value: unknown): string { + return JSON.stringify(normalizeForStableJson(value)) +} + +export function hashJson(value: unknown): string { + return createHash('sha256').update(stableJsonStringify(value)).digest('hex') +} + +export const TAPE_VIEW_MANIFEST_HASH_VERSION = 2 + +function buildManifestHash(manifest: DeepChatTapeViewManifest): string { + const hashable: Record = { ...manifest } + delete hashable.assembledAt + delete hashable.viewId + hashable.hashes = { + promptHash: manifest.hashes.promptHash, + toolDefinitionsHash: manifest.hashes.toolDefinitionsHash + } + return hashJson(hashable) +} + +function buildExcludedRanges( + summaryCursor?: ContextSummaryCursorMetadata +): DeepChatTapeViewExcludedRange[] { + if ( + !summaryCursor || + summaryCursor.preCursorCount === 0 || + summaryCursor.preCursorOrderSeqMin === null || + summaryCursor.preCursorOrderSeqMax === null + ) { + return [] + } + return [ + { + fromOrderSeq: summaryCursor.preCursorOrderSeqMin, + toOrderSeq: summaryCursor.preCursorOrderSeqMax, + count: summaryCursor.preCursorCount, + reason: 'before_summary_cursor' + } + ] +} + +export function verifyTapeViewManifestHash( + manifest: DeepChatTapeViewManifest +): DeepChatTapeViewManifestIntegrity { + if (manifest.hashVersion !== TAPE_VIEW_MANIFEST_HASH_VERSION) { + return 'unverified' + } + return buildManifestHash(manifest) === manifest.hashes.manifestHash ? 'valid' : 'invalid' +} + +export function createTapeViewManifest( + input: TapeViewManifestBuildInput +): DeepChatTapeViewManifest { + const assembledAt = input.assembledAt ?? Date.now() + const excludedRanges = buildExcludedRanges(input.summaryCursor) + const draft: DeepChatTapeViewManifest = { + schemaVersion: 2, + hashVersion: TAPE_VIEW_MANIFEST_HASH_VERSION, + viewId: '', + sessionId: input.sessionId, + messageId: input.messageId, + requestSeq: input.requestSeq, + taskType: input.taskType, + policy: input.policy, + policyVersion: input.policyVersion ?? null, + contextBuilderVersion: TAPE_VIEW_CONTEXT_BUILDER_VERSION, + latestEntryId: input.latestEntryId, + anchorEntryIds: [...input.anchorEntryIds], + ...(input.reconstructionAnchorEntryId !== undefined + ? { reconstructionAnchorEntryId: input.reconstructionAnchorEntryId } + : {}), + included: input.included.map((entry) => ({ ...entry })), + excluded: input.excluded.map((entry) => ({ ...entry })), + ...(excludedRanges.length > 0 ? { excludedRanges } : {}), + tokenBudget: { + ...input.tokenBudget, + estimatedPromptTokens: estimateMessagesTokens(input.messages) + }, + hashes: { + promptHash: hashJson(input.messages), + toolDefinitionsHash: hashJson(input.tools), + manifestHash: '' + }, + meta: { + providerId: input.providerId, + modelId: input.modelId, + summaryCursorOrderSeq: input.summaryCursorOrderSeq, + supportsVision: input.supportsVision, + supportsAudioInput: input.supportsAudioInput, + traceDebugEnabled: input.traceDebugEnabled + }, + assembledAt + } + + const manifestHash = buildManifestHash(draft) + return { + ...draft, + viewId: `view_${manifestHash.slice(0, 16)}`, + hashes: { ...draft.hashes, manifestHash } + } +} + +export function buildIncludedRefs( + selection: TapeViewContextSelection, + sourceMaps: TapeViewManifestLookupMaps = {} +): DeepChatTapeViewEntryRef[] { + const refs: DeepChatTapeViewEntryRef[] = [] + + if (selection.includesSystemPrompt) { + refs.push({ + entryId: null, + messageId: null, + orderSeq: null, + role: 'system', + source: 'synthetic', + reason: 'system_prompt' + }) + } + + for (const item of selection.includedRecords) { + refs.push({ + entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, + messageId: item.record.id, + orderSeq: item.record.orderSeq, + role: item.record.role, + source: sourceMaps.entryIdByMessageId?.has(item.record.id) ? 'tape' : 'synthetic', + reason: item.reason + }) + } + + if (selection.newUserMessageId) { + refs.push({ + entryId: sourceMaps.entryIdByMessageId?.get(selection.newUserMessageId) ?? null, + messageId: selection.newUserMessageId, + orderSeq: null, + role: 'user', + source: sourceMaps.entryIdByMessageId?.has(selection.newUserMessageId) ? 'tape' : 'synthetic', + reason: 'new_user_input' + }) + } + + return refs +} + +export function buildExcludedRefs( + selection: TapeViewContextSelection, + sourceMaps: TapeViewManifestLookupMaps = {} +): DeepChatTapeViewExcludedRef[] { + return selection.excludedRecords.map((item) => ({ + entryId: sourceMaps.entryIdByMessageId?.get(item.record.id) ?? null, + messageId: item.record.id, + orderSeq: item.record.orderSeq, + reason: item.reason + })) +} + +export function buildRequestRefs( + messages: ChatMessage[], + sourceMaps: TapeViewManifestLookupMaps = {} +): DeepChatTapeViewEntryRef[] { + const lastToolCallIndex = new Map() + const lastToolResultIndex = new Map() + messages.forEach((message, index) => { + if (message.role === 'assistant' && message.tool_calls?.length) { + for (const toolCall of message.tool_calls) { + lastToolCallIndex.set(toolCall.id, index) + } + } else if (message.role === 'tool' && message.tool_call_id) { + lastToolResultIndex.set(message.tool_call_id, index) + } + }) + + const refs: DeepChatTapeViewEntryRef[] = [] + messages.forEach((message, index) => { + if (message.role === 'assistant' && message.tool_calls?.length) { + for (const toolCall of message.tool_calls) { + const entryId = + lastToolCallIndex.get(toolCall.id) === index + ? (sourceMaps.toolCallEntryIdByToolId?.get(toolCall.id) ?? null) + : null + refs.push({ + entryId, + messageId: null, + orderSeq: null, + role: 'assistant', + source: entryId === null ? 'synthetic' : 'tape', + reason: 'tool_loop_message' + }) + } + return + } + + if (message.role === 'tool' && message.tool_call_id) { + const entryId = + lastToolResultIndex.get(message.tool_call_id) === index + ? (sourceMaps.toolResultEntryIdByToolId?.get(message.tool_call_id) ?? null) + : null + refs.push({ + entryId, + messageId: null, + orderSeq: null, + role: 'tool', + source: entryId === null ? 'synthetic' : 'tape', + reason: 'tool_loop_message' + }) + return + } + + refs.push({ + entryId: null, + messageId: null, + orderSeq: null, + role: message.role, + source: 'synthetic', + reason: + message.role === 'system' + ? 'system_prompt' + : message.role === 'tool' + ? 'tool_loop_message' + : 'selected_history' + }) + }) + + return refs +} diff --git a/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts b/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts new file mode 100644 index 0000000000..1965e58aaf --- /dev/null +++ b/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts @@ -0,0 +1,1070 @@ +import Database from 'better-sqlite3-multiple-ciphers' +import logger from '@shared/logger' +import { BaseTable } from '@/data/baseTable' +import { randomUUID } from 'crypto' +import { + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources, + SUMMARY_ANCHOR_NAMES, + TAPE_INCARNATION_META_KEY, + type DeepChatTapeAppendInput, + type DeepChatTapeEntryRow, + type DeepChatTapeReadSource, + type DeepChatTapeSearchInput, + type TapeAnchorAppendInput, + type TapeEventAppendInput +} from '@/tape/domain/entry' +import type { + TapeBootstrapStore, + TapeEntryStore, + TapeMutationProjection, + TapeTransactionRunner +} from '@/tape/ports/storage' + +export { + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources, + SUMMARY_ANCHOR_NAMES, + TAPE_INCARNATION_META_KEY +} from '@/tape/domain/entry' +export type { + DeepChatTapeAppendInput, + DeepChatTapeEntryKind, + DeepChatTapeEntryRow, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + DeepChatTapeSourceInput, + DeepChatTapeSourceType, + TapeAnchorAppendInput, + TapeEventAppendInput +} from '@/tape/domain/entry' + +export type DeepChatTapeMutationProjection = TapeMutationProjection + +const RECONSTRUCTION_ANCHOR_NAMES = SUMMARY_ANCHOR_NAMES + +const TAPE_ENTRY_INDEX_SQL = ` + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_kind + ON deepchat_tape_entries(session_id, kind, entry_id); + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_name + ON deepchat_tape_entries(session_id, name, entry_id); + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_source + ON deepchat_tape_entries(session_id, source_type, source_id, source_seq); + CREATE UNIQUE INDEX IF NOT EXISTS idx_deepchat_tape_entries_session_provenance + ON deepchat_tape_entries(session_id, provenance_key) + WHERE provenance_key IS NOT NULL; +` + +function safeJsonStringify(value: Record | undefined): string { + return JSON.stringify(value ?? {}) +} + +function buildProvenanceKey(input: DeepChatTapeAppendInput): string | null { + if (input.provenanceKey !== undefined) { + return input.provenanceKey + } + if (!input.source?.type || !input.source.id) { + return null + } + return [ + input.source.type, + input.source.id, + input.source.seq ?? 0, + input.kind, + input.name ?? '' + ].join(':') +} + +function escapeLikePattern(value: string): string { + return value.replace(/[\\%_]/g, (character) => `\\${character}`) +} + +// Three LIKE parameters per field group stay below SQLite's portable 999-variable floor after +// source, filter, and limit bindings. +const MAX_TAPE_SEARCH_TOKEN_CLAUSES = 256 + +function tokenizeDeepChatTapeSearchQuery(value: string): string[] { + return value + .split(/\s+/) + .map((token) => token.trim()) + .filter(Boolean) +} + +export function buildDeepChatTapeFtsMatch(value: string): string { + const tokens = tokenizeDeepChatTapeSearchQuery(value) + const values = + tokens.length > 1 && tokens.length <= MAX_TAPE_SEARCH_TOKEN_CLAUSES ? tokens : [value] + return values.map((token) => `"${token.replace(/"/g, '""')}"`).join(' AND ') +} + +export function buildDeepChatTapeLikeSearchPredicate( + fieldExpressions: readonly [string, ...string[]], + normalizedQuery: string +): { sql: string; params: string[] } { + const fieldClause = `(${fieldExpressions + .map((field) => `${field} LIKE ? ESCAPE '\\'`) + .join(' OR ')})` + const queryClauses = [fieldClause] + const queryPattern = `%${escapeLikePattern(normalizedQuery)}%` + const params = fieldExpressions.map(() => queryPattern) + const tokens = tokenizeDeepChatTapeSearchQuery(normalizedQuery) + + if (tokens.length > 1 && tokens.length <= MAX_TAPE_SEARCH_TOKEN_CLAUSES) { + queryClauses.push(`(${tokens.map(() => fieldClause).join(' AND ')})`) + for (const token of tokens) { + const tokenPattern = `%${escapeLikePattern(token)}%` + params.push(...fieldExpressions.map(() => tokenPattern)) + } + } + + return { + sql: `(${queryClauses.join(' OR ')})`, + params + } +} + +const AUTHORIZED_TAPE_SOURCES_CTE_SQL = ` + authorized_sources(session_id, max_entry_id) AS ( + SELECT + json_extract(value, '$.sessionId'), + CAST(json_extract(value, '$.maxEntryId') AS INTEGER) + FROM json_each(?) + ) +` + +function effectiveTapeMessagePredicateSql(alias: string): string { + return ` + json_type(${alias}.payload_json, '$.record') = 'object' + AND typeof(json_extract(${alias}.payload_json, '$.record.id')) = 'text' + AND typeof(json_extract(${alias}.payload_json, '$.record.sessionId')) = 'text' + AND typeof(json_extract(${alias}.payload_json, '$.record.orderSeq')) IN ('integer', 'real') + AND json_extract(${alias}.payload_json, '$.record.role') IN ('user', 'assistant') + AND typeof(json_extract(${alias}.payload_json, '$.record.content')) = 'text' + AND ( + json_extract(${alias}.payload_json, '$.record.status') IS NULL + OR json_extract(${alias}.payload_json, '$.record.status') != 'pending' + ) + ` +} + +function tapeRetractionMessageIdSql(alias: string): string { + return ` + CASE + WHEN json_type(${alias}.payload_json, '$.data') = 'object' + THEN json_extract(${alias}.payload_json, '$.data.messageId') + WHEN json_type(${alias}.payload_json, '$.data') = 'text' + AND json_valid(json_extract(${alias}.payload_json, '$.data')) + THEN json_extract(json_extract(${alias}.payload_json, '$.data'), '$.messageId') + ELSE NULL + END + ` +} + +function tapeToolCallIdSql(alias: string): string { + return ` + CASE + WHEN ${alias}.kind = 'tool_result' + THEN json_extract(${alias}.payload_json, '$.toolCallId') + WHEN json_type(${alias}.payload_json, '$.toolCall') = 'object' + THEN json_extract(${alias}.payload_json, '$.toolCall.id') + WHEN json_type(${alias}.payload_json, '$.toolCall') = 'text' + AND json_valid(json_extract(${alias}.payload_json, '$.toolCall')) + THEN json_extract(json_extract(${alias}.payload_json, '$.toolCall'), '$.id') + ELSE NULL + END + ` +} + +// These read-only SQL forms mirror deepchatTapeEffectiveSemantics. Search uses correlated +// candidate validation to avoid materializing a whole linked Tape; context uses the complete +// effective CTE because it needs stable neighboring-row positions. Native tests cover parity for +// replacement, retraction, head cutoff, and window ordering. +const EFFECTIVE_TAPE_ROWS_CTE_SQL = ` + bounded_rows AS ( + SELECT tape.* + FROM deepchat_tape_entries AS tape + INNER JOIN authorized_sources AS source + ON source.session_id = tape.session_id + AND tape.entry_id <= source.max_entry_id + ), + raw_message_candidates AS ( + SELECT + bounded_rows.*, + json_extract(payload_json, '$.record.id') AS message_id + FROM bounded_rows + WHERE kind = 'message' + ), + message_candidates AS ( + SELECT * + FROM raw_message_candidates + WHERE ${effectiveTapeMessagePredicateSql('raw_message_candidates')} + ), + ranked_messages AS ( + SELECT + message_candidates.*, + ROW_NUMBER() OVER ( + PARTITION BY session_id, message_id + ORDER BY entry_id DESC + ) AS candidate_rank + FROM message_candidates + ), + effective_message_rows AS ( + SELECT ranked_messages.* + FROM ranked_messages + WHERE candidate_rank = 1 + AND NOT EXISTS ( + SELECT 1 + FROM bounded_rows AS retraction + WHERE retraction.session_id = ranked_messages.session_id + AND retraction.kind = 'event' + AND retraction.name = 'message/retracted' + AND retraction.entry_id > ranked_messages.entry_id + AND (${tapeRetractionMessageIdSql('retraction')}) = ranked_messages.message_id + ) + ), + raw_tool_candidates AS ( + SELECT + bounded_rows.*, + json_extract(payload_json, '$.messageId') AS message_id, + (${tapeToolCallIdSql('bounded_rows')}) AS tool_call_id, + json_extract(meta_json, '$.status') AS tool_status + FROM bounded_rows + WHERE kind IN ('tool_call', 'tool_result') + ), + tool_candidates AS ( + SELECT * + FROM raw_tool_candidates + WHERE typeof(message_id) = 'text' + AND length(message_id) > 0 + AND typeof(tool_call_id) = 'text' + AND length(tool_call_id) > 0 + AND tool_status IN ('success', 'error') + ), + ranked_tools AS ( + SELECT + tool_candidates.*, + ROW_NUMBER() OVER ( + PARTITION BY session_id, kind, message_id, tool_call_id + ORDER BY entry_id DESC + ) AS candidate_rank + FROM tool_candidates + ), + effective_rows AS ( + SELECT + session_id, entry_id, kind, name, source_type, source_id, source_seq, + provenance_key, payload_json, meta_json, created_at + FROM bounded_rows + WHERE kind = 'anchor' + UNION ALL + SELECT + session_id, entry_id, kind, name, source_type, source_id, source_seq, + provenance_key, payload_json, meta_json, created_at + FROM bounded_rows + WHERE kind = 'event' + AND (name IS NULL OR name NOT IN ( + 'message/retracted', + 'message/compaction_indicator', + 'migration/backfill' + )) + UNION ALL + SELECT + session_id, entry_id, kind, name, source_type, source_id, source_seq, + provenance_key, payload_json, meta_json, created_at + FROM effective_message_rows + UNION ALL + SELECT + ranked_tools.session_id, + ranked_tools.entry_id, + ranked_tools.kind, + ranked_tools.name, + ranked_tools.source_type, + ranked_tools.source_id, + ranked_tools.source_seq, + ranked_tools.provenance_key, + ranked_tools.payload_json, + ranked_tools.meta_json, + ranked_tools.created_at + FROM ranked_tools + INNER JOIN effective_message_rows AS message + ON message.session_id = ranked_tools.session_id + AND message.message_id = ranked_tools.message_id + WHERE ranked_tools.candidate_rank = 1 + ) +` + +const EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL = ` + candidate.kind = 'anchor' + OR ( + candidate.kind = 'event' + AND (candidate.name IS NULL OR candidate.name NOT IN ( + 'message/retracted', + 'message/compaction_indicator', + 'migration/backfill' + )) + ) + OR ( + candidate.kind = 'message' + AND ${effectiveTapeMessagePredicateSql('candidate')} + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS later_message + WHERE later_message.session_id = candidate.session_id + AND later_message.entry_id > candidate.entry_id + AND later_message.entry_id <= source.max_entry_id + AND later_message.kind = 'message' + AND ${effectiveTapeMessagePredicateSql('later_message')} + AND json_extract(later_message.payload_json, '$.record.id') = + json_extract(candidate.payload_json, '$.record.id') + ) + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS retraction + WHERE retraction.session_id = candidate.session_id + AND retraction.entry_id > candidate.entry_id + AND retraction.entry_id <= source.max_entry_id + AND retraction.kind = 'event' + AND retraction.name = 'message/retracted' + AND (${tapeRetractionMessageIdSql('retraction')}) = + json_extract(candidate.payload_json, '$.record.id') + ) + ) + OR ( + candidate.kind IN ('tool_call', 'tool_result') + AND json_extract(candidate.meta_json, '$.status') IN ('success', 'error') + AND typeof(json_extract(candidate.payload_json, '$.messageId')) = 'text' + AND length(json_extract(candidate.payload_json, '$.messageId')) > 0 + AND typeof((${tapeToolCallIdSql('candidate')})) = 'text' + AND length((${tapeToolCallIdSql('candidate')})) > 0 + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS later_tool + WHERE later_tool.session_id = candidate.session_id + AND later_tool.entry_id > candidate.entry_id + AND later_tool.entry_id <= source.max_entry_id + AND later_tool.kind = candidate.kind + AND json_extract(later_tool.meta_json, '$.status') IN ('success', 'error') + AND json_extract(later_tool.payload_json, '$.messageId') = + json_extract(candidate.payload_json, '$.messageId') + AND (${tapeToolCallIdSql('later_tool')}) = (${tapeToolCallIdSql('candidate')}) + ) + AND EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS message + WHERE message.session_id = candidate.session_id + AND message.entry_id <= source.max_entry_id + AND message.kind = 'message' + AND ${effectiveTapeMessagePredicateSql('message')} + AND json_extract(message.payload_json, '$.record.id') = + json_extract(candidate.payload_json, '$.messageId') + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS later_message + WHERE later_message.session_id = message.session_id + AND later_message.entry_id > message.entry_id + AND later_message.entry_id <= source.max_entry_id + AND later_message.kind = 'message' + AND ${effectiveTapeMessagePredicateSql('later_message')} + AND json_extract(later_message.payload_json, '$.record.id') = + json_extract(message.payload_json, '$.record.id') + ) + AND NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_entries AS retraction + WHERE retraction.session_id = message.session_id + AND retraction.entry_id > message.entry_id + AND retraction.entry_id <= source.max_entry_id + AND retraction.kind = 'event' + AND retraction.name = 'message/retracted' + AND (${tapeRetractionMessageIdSql('retraction')}) = + json_extract(message.payload_json, '$.record.id') + ) + ) + ) +` + +export class DeepChatTapeEntriesTable + extends BaseTable + implements TapeEntryStore, TapeTransactionRunner, TapeBootstrapStore +{ + constructor( + db: Database.Database, + private readonly mutationProjection?: DeepChatTapeMutationProjection + ) { + super(db, 'deepchat_tape_entries') + } + + getCreateTableSQL(): string { + return ` + CREATE TABLE IF NOT EXISTS deepchat_tape_entries ( + session_id TEXT NOT NULL, + entry_id INTEGER NOT NULL, + kind TEXT NOT NULL, + name TEXT, + source_type TEXT, + source_id TEXT, + source_seq INTEGER, + provenance_key TEXT, + payload_json TEXT NOT NULL DEFAULT '{}', + meta_json TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, entry_id) + ); + ${TAPE_ENTRY_INDEX_SQL} + ` + } + + public createTable(): void { + if (!this.tableExists()) { + this.db.exec(this.getCreateTableSQL()) + return + } + this.ensureProvenanceColumns() + this.db.exec(TAPE_ENTRY_INDEX_SQL) + } + + getMigrationSQL(_version: number): string | null { + return null + } + + getLatestVersion(): number { + return 0 + } + + runInTransaction(operation: () => T): T { + return this.db.transaction(operation)() + } + + append(input: DeepChatTapeAppendInput): DeepChatTapeEntryRow { + const append = this.db.transaction(() => { + const provenanceKey = buildProvenanceKey(input) + if (input.idempotent && provenanceKey) { + const existing = this.getByProvenanceKey(input.sessionId, provenanceKey) + if (existing) { + return existing + } + } + + const createdAt = input.createdAt ?? Date.now() + const previousSessionMaxEntryId = this.getMaxEntryId(input.sessionId) + const row = { + session_id: input.sessionId, + entry_id: previousSessionMaxEntryId + 1, + kind: input.kind, + name: input.name ?? null, + source_type: input.source?.type ?? null, + source_id: input.source?.id ?? null, + source_seq: input.source?.seq ?? null, + provenance_key: provenanceKey, + payload_json: safeJsonStringify(input.payload), + meta_json: safeJsonStringify(input.meta), + created_at: createdAt + } satisfies DeepChatTapeEntryRow + + try { + this.db + .prepare( + `INSERT INTO deepchat_tape_entries ( + session_id, + entry_id, + kind, + name, + source_type, + source_id, + source_seq, + provenance_key, + payload_json, + meta_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run( + row.session_id, + row.entry_id, + row.kind, + row.name, + row.source_type, + row.source_id, + row.source_seq, + row.provenance_key, + row.payload_json, + row.meta_json, + row.created_at + ) + } catch (error) { + if (input.idempotent && provenanceKey) { + const existing = this.getByProvenanceKey(input.sessionId, provenanceKey) + if (existing) { + return existing + } + } + throw error + } + + if (this.mutationProjection) { + try { + const applyProjection = this.db.transaction(() => + this.mutationProjection?.applyAppendedEntry(row, previousSessionMaxEntryId) + ) + applyProjection() + } catch (error) { + this.mutationProjection.invalidateSession(row.session_id) + logger.warn( + `[Tape] memory ingestion projection append failed; session marked stale: ${String(error)}` + ) + } + } + + return row + }) + + return append() + } + + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow { + return this.append({ + sessionId: input.sessionId, + kind: 'anchor', + name: input.name, + source: input.source, + provenanceKey: input.provenanceKey, + payload: { + name: input.name, + state: input.state + }, + meta: input.meta, + createdAt: input.createdAt, + idempotent: input.idempotent + }) + } + + appendEvent(input: TapeEventAppendInput): DeepChatTapeEntryRow { + return this.append({ + sessionId: input.sessionId, + kind: 'event', + name: input.name, + source: input.source, + provenanceKey: input.provenanceKey, + payload: { + name: input.name, + data: input.data + }, + meta: input.meta, + createdAt: input.createdAt, + idempotent: input.idempotent + }) + } + + ensureBootstrapAnchor(sessionId: string): void { + const existing = this.db + .prepare( + `SELECT entry_id + FROM deepchat_tape_entries + WHERE session_id = ? AND kind = 'anchor' + ORDER BY entry_id ASC + LIMIT 1` + ) + .get(sessionId) as { entry_id: number } | undefined + + if (existing) { + return + } + + this.appendAnchor({ + sessionId, + name: 'session/start', + source: { + type: 'session', + id: sessionId, + seq: 0 + }, + state: { + owner: 'human' + }, + meta: { + [TAPE_INCARNATION_META_KEY]: randomUUID() + }, + idempotent: true + }) + } + + getBySession(sessionId: string): DeepChatTapeEntryRow[] { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? + ORDER BY entry_id ASC` + ) + .all(sessionId) as DeepChatTapeEntryRow[] + } + + getSubagentLineageEvents(sessionId: string): DeepChatTapeEntryRow[] { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? + AND kind = 'event' + AND name IN ('subagent/tape_linked', 'fork/merge') + ORDER BY entry_id ASC` + ) + .all(sessionId) as DeepChatTapeEntryRow[] + } + + getFirstEntriesBySessions(sessionIds: string[]): DeepChatTapeEntryRow[] { + const ids = [...new Set(sessionIds.map((id) => id.trim()).filter(Boolean))] + if (ids.length === 0) { + return [] + } + return this.db + .prepare( + `WITH requested_sessions(session_id) AS ( + SELECT value FROM json_each(?) + ), + first_entries(session_id, entry_id) AS ( + SELECT tape.session_id, MIN(tape.entry_id) + FROM deepchat_tape_entries AS tape + INNER JOIN requested_sessions AS requested + ON requested.session_id = tape.session_id + GROUP BY tape.session_id + ) + SELECT tape.* + FROM deepchat_tape_entries AS tape + INNER JOIN first_entries AS first + ON first.session_id = tape.session_id + AND first.entry_id = tape.entry_id + ORDER BY tape.session_id ASC` + ) + .all(JSON.stringify(ids)) as DeepChatTapeEntryRow[] + } + + getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND entry_id <= ? + ORDER BY entry_id ASC` + ) + .all(sessionId, maxEntryId) as DeepChatTapeEntryRow[] + } + + listMemoryViewManifestAnchorsBySessions( + sessionIds: string[], + optionsOrLimit: number | { limit?: number; messageId?: string } = 100 + ): DeepChatTapeEntryRow[] { + const uniqueSessionIds = [...new Set(sessionIds.filter((id) => id.trim().length > 0))] + if (uniqueSessionIds.length === 0) { + return [] + } + const options = typeof optionsOrLimit === 'number' ? { limit: optionsOrLimit } : optionsOrLimit + const cappedLimit = Math.min(Math.max(Math.floor(options.limit ?? 100), 1), 500) + const placeholders = uniqueSessionIds.map(() => '?').join(', ') + const whereClauses = [ + `session_id IN (${placeholders})`, + "kind = 'anchor'", + "name = 'memory/view_assembled'" + ] + const params: Array = [...uniqueSessionIds] + if (options.messageId) { + whereClauses.push("json_extract(meta_json, '$.messageId') = ?") + params.push(options.messageId) + } + params.push(cappedLimit) + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE ${whereClauses.join(' AND ')} + ORDER BY created_at DESC, entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeEntryRow[] + } + + listMemoryViewManifestAnchorsByAgent( + agentId: string, + options: { sessionId?: string; limit?: number; messageId?: string } = {} + ): DeepChatTapeEntryRow[] { + const cappedLimit = Math.min(Math.max(Math.floor(options.limit ?? 100), 1), 500) + const whereClauses = [ + 'sessions.agent_id = ?', + "tape.kind = 'anchor'", + "tape.name = 'memory/view_assembled'" + ] + const params: Array = [agentId] + if (options.sessionId) { + whereClauses.push('tape.session_id = ?') + params.push(options.sessionId) + } + if (options.messageId) { + whereClauses.push("json_extract(tape.meta_json, '$.messageId') = ?") + params.push(options.messageId) + } + params.push(cappedLimit) + return this.db + .prepare( + `SELECT tape.* + FROM deepchat_tape_entries AS tape + INNER JOIN new_sessions AS sessions + ON sessions.id = tape.session_id + WHERE ${whereClauses.join(' AND ')} + ORDER BY tape.created_at DESC, tape.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeEntryRow[] + } + + getEntriesAfter(sessionId: string, entryId: number): DeepChatTapeEntryRow[] { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND entry_id > ? + ORDER BY entry_id ASC` + ) + .all(sessionId, entryId) as DeepChatTapeEntryRow[] + } + + getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND kind = 'anchor' + ORDER BY entry_id DESC + LIMIT 1` + ) + .get(sessionId) as DeepChatTapeEntryRow | undefined + } + + getAnchors(sessionId: string, limit: number = 20): DeepChatTapeEntryRow[] { + const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) + const rows = this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND kind = 'anchor' + ORDER BY entry_id DESC + LIMIT ?` + ) + .all(sessionId, cappedLimit) as DeepChatTapeEntryRow[] + + return rows.reverse() + } + + getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + const placeholders = SUMMARY_ANCHOR_NAMES.map(() => '?').join(', ') + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? + AND kind = 'anchor' + AND name IN (${placeholders}) + ORDER BY entry_id DESC + LIMIT 1` + ) + .get(sessionId, ...SUMMARY_ANCHOR_NAMES) as DeepChatTapeEntryRow | undefined + } + + getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined { + const placeholders = RECONSTRUCTION_ANCHOR_NAMES.map(() => '?').join(', ') + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? + AND kind = 'anchor' + AND ( + name IN (${placeholders}) + OR name LIKE 'handoff/%' + OR name LIKE 'auto_handoff/%' + ) + ORDER BY entry_id DESC + LIMIT 1` + ) + .get(sessionId, ...RECONSTRUCTION_ANCHOR_NAMES) as DeepChatTapeEntryRow | undefined + } + + getByProvenanceKey(sessionId: string, provenanceKey: string): DeepChatTapeEntryRow | undefined { + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE session_id = ? AND provenance_key = ? + LIMIT 1` + ) + .get(sessionId, provenanceKey) as DeepChatTapeEntryRow | undefined + } + + getMaxEntryId(sessionId: string): number { + const row = this.db + .prepare( + `SELECT MAX(entry_id) AS max_entry_id + FROM deepchat_tape_entries + WHERE session_id = ?` + ) + .get(sessionId) as { max_entry_id: number | null } | undefined + return row?.max_entry_id ?? 0 + } + + getMaxEntryIdsBySessions(sessionIds: string[]): Map { + const ids = [...new Set(sessionIds.map((id) => id.trim()).filter(Boolean))] + const maxEntryIdBySession = new Map(ids.map((id) => [id, 0])) + if (ids.length === 0) { + return maxEntryIdBySession + } + const rows = this.db + .prepare( + `WITH requested_sessions(session_id) AS ( + SELECT value FROM json_each(?) + ) + SELECT tape.session_id, MAX(tape.entry_id) AS max_entry_id + FROM deepchat_tape_entries AS tape + INNER JOIN requested_sessions AS requested + ON requested.session_id = tape.session_id + GROUP BY tape.session_id` + ) + .all(JSON.stringify(ids)) as Array<{ session_id: string; max_entry_id: number }> + for (const row of rows) { + maxEntryIdBySession.set(row.session_id, row.max_entry_id) + } + return maxEntryIdBySession + } + + countAnchorsBySession(sessionId: string): number { + const row = this.db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_entries + WHERE session_id = ? AND kind = 'anchor'` + ) + .get(sessionId) as { count: number } | undefined + return row?.count ?? 0 + } + + countEntriesAfter(sessionId: string, entryId: number): number { + const row = this.db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_entries + WHERE session_id = ? AND entry_id > ?` + ) + .get(sessionId, entryId) as { count: number } | undefined + return row?.count ?? 0 + } + + countBySession(sessionId: string): number { + const row = this.db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_entries + WHERE session_id = ?` + ) + .get(sessionId) as { count: number } | undefined + return row?.count ?? 0 + } + + search( + sessionId: string, + query: string, + options: DeepChatTapeSearchInput = {} + ): DeepChatTapeEntryRow[] { + const normalizedQuery = query.trim() + if (!normalizedQuery) { + return [] + } + const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 + const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) + const queryPredicate = buildDeepChatTapeLikeSearchPredicate( + ['payload_json', 'meta_json', 'name'], + normalizedQuery + ) + const whereClauses = ['session_id = ?', queryPredicate.sql] + const params: Array = [sessionId, ...queryPredicate.params] + + if (options.kinds?.length) { + whereClauses.push(`kind IN (${options.kinds.map(() => '?').join(', ')})`) + params.push(...options.kinds) + } + + if (Number.isFinite(options.startCreatedAt)) { + whereClauses.push('created_at >= ?') + params.push(options.startCreatedAt as number) + } + + if (Number.isFinite(options.endCreatedAt)) { + whereClauses.push('created_at <= ?') + params.push(options.endCreatedAt as number) + } + + params.push(cappedLimit) + + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_entries + WHERE ${whereClauses.join(' AND ')} + ORDER BY entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeEntryRow[] + } + + searchEffectiveSourcesAtHeads( + sources: readonly DeepChatTapeReadSource[], + query: string, + options: DeepChatTapeSearchInput = {} + ): DeepChatTapeEntryRow[] { + const normalizedSources = normalizeDeepChatTapeReadSources(sources) + const normalizedQuery = query.trim() + if (normalizedSources.length === 0 || !normalizedQuery) { + return [] + } + + const limit = Number.isFinite(options.limit) ? (options.limit as number) : 20 + const cappedLimit = Math.min(Math.max(Math.floor(limit), 1), 100) + const queryPredicate = buildDeepChatTapeLikeSearchPredicate( + ['candidate.payload_json', 'candidate.meta_json', 'candidate.name'], + normalizedQuery + ) + const whereClauses = [queryPredicate.sql] + const params: Array = [ + serializeDeepChatTapeReadSources(normalizedSources), + ...queryPredicate.params + ] + + if (options.kinds?.length) { + whereClauses.push(`candidate.kind IN (${options.kinds.map(() => '?').join(', ')})`) + params.push(...options.kinds) + } + if (Number.isFinite(options.startCreatedAt)) { + whereClauses.push('candidate.created_at >= ?') + params.push(options.startCreatedAt as number) + } + if (Number.isFinite(options.endCreatedAt)) { + whereClauses.push('candidate.created_at <= ?') + params.push(options.endCreatedAt as number) + } + params.push(cappedLimit) + + return this.db + .prepare( + `WITH + ${AUTHORIZED_TAPE_SOURCES_CTE_SQL} + SELECT candidate.* + FROM deepchat_tape_entries AS candidate + INNER JOIN authorized_sources AS source + ON source.session_id = candidate.session_id + AND candidate.entry_id <= source.max_entry_id + WHERE ${whereClauses.join(' AND ')} + AND (${EFFECTIVE_TAPE_SEARCH_ROW_PREDICATE_SQL}) + ORDER BY candidate.created_at DESC, candidate.session_id ASC, candidate.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeEntryRow[] + } + + getEffectiveContextRowsAtHead( + source: DeepChatTapeReadSource, + entryIds: number[], + options: { before: number; after: number; limit: number } + ): DeepChatTapeEntryRow[] { + const normalizedSource = normalizeDeepChatTapeReadSources([source])[0] + const requestedEntryIds = [ + ...new Set(entryIds.filter((entryId) => Number.isSafeInteger(entryId) && entryId > 0)) + ].sort((left, right) => left - right) + if (!normalizedSource || requestedEntryIds.length === 0) { + return [] + } + const before = Math.min(Math.max(Math.floor(options.before), 0), 20) + const after = Math.min(Math.max(Math.floor(options.after), 0), 20) + const limit = Math.min(Math.max(Math.floor(options.limit), 1), 100) + + return this.db + .prepare( + `WITH + ${AUTHORIZED_TAPE_SOURCES_CTE_SQL}, + ${EFFECTIVE_TAPE_ROWS_CTE_SQL}, + ordered_rows AS ( + SELECT + effective_rows.*, + ROW_NUMBER() OVER (ORDER BY entry_id ASC) AS row_position + FROM effective_rows + ), + requested_ids(entry_id, request_ordinal) AS ( + SELECT CAST(value AS INTEGER), CAST(key AS INTEGER) + FROM json_each(?) + ), + requested_positions AS ( + SELECT + requested_ids.request_ordinal, + ordered_rows.entry_id, + ordered_rows.row_position + FROM requested_ids + INNER JOIN ordered_rows + ON ordered_rows.entry_id = requested_ids.entry_id + ), + context_candidates AS ( + SELECT + ordered_rows.*, + 0 AS priority_group, + requested_positions.request_ordinal, + 0 AS neighbor_position + FROM requested_positions + INNER JOIN ordered_rows + ON ordered_rows.entry_id = requested_positions.entry_id + UNION ALL + SELECT + ordered_rows.*, + 1 AS priority_group, + requested_positions.request_ordinal, + ordered_rows.row_position AS neighbor_position + FROM requested_positions + INNER JOIN ordered_rows + ON ordered_rows.row_position BETWEEN requested_positions.row_position - ? + AND requested_positions.row_position + ? + AND ordered_rows.entry_id != requested_positions.entry_id + ), + ranked_context_candidates AS ( + SELECT + context_candidates.*, + ROW_NUMBER() OVER ( + PARTITION BY session_id, entry_id + ORDER BY priority_group, request_ordinal, neighbor_position + ) AS duplicate_rank + FROM context_candidates + ) + SELECT + session_id, entry_id, kind, name, source_type, source_id, source_seq, + provenance_key, payload_json, meta_json, created_at + FROM ranked_context_candidates + WHERE duplicate_rank = 1 + ORDER BY priority_group, request_ordinal, neighbor_position + LIMIT ?` + ) + .all( + serializeDeepChatTapeReadSources([normalizedSource]), + JSON.stringify(requestedEntryIds), + before, + after, + limit + ) as DeepChatTapeEntryRow[] + } + + private ensureProvenanceColumns(): void { + const columns: Array<[string, string]> = [ + ['source_type', 'TEXT'], + ['source_id', 'TEXT'], + ['source_seq', 'INTEGER'], + ['provenance_key', 'TEXT'] + ] + for (const [columnName, columnType] of columns) { + if (!this.hasColumn(columnName)) { + this.db.exec(`ALTER TABLE deepchat_tape_entries ADD COLUMN ${columnName} ${columnType}`) + } + } + } +} diff --git a/src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts b/src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts new file mode 100644 index 0000000000..f245c8d9aa --- /dev/null +++ b/src/main/tape/infrastructure/sqlite/tapeLifecycleAdapter.ts @@ -0,0 +1,17 @@ +import type Database from 'better-sqlite3-multiple-ciphers' +import type { TapeEntryLifecycleStore, TapeMutationProjection } from '../../ports/storage' + +export class SqliteTapeLifecycleAdapter implements TapeEntryLifecycleStore { + constructor( + private readonly db: Database.Database, + private readonly mutationProjection?: TapeMutationProjection + ) {} + + deleteBySession(sessionId: string): void { + const remove = this.db.transaction(() => { + this.db.prepare('DELETE FROM deepchat_tape_entries WHERE session_id = ?').run(sessionId) + this.mutationProjection?.deleteBySession(sessionId) + }) + remove() + } +} diff --git a/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts new file mode 100644 index 0000000000..3c4d8db625 --- /dev/null +++ b/src/main/tape/infrastructure/sqlite/tapeSearchProjectionStore.ts @@ -0,0 +1,1009 @@ +import Database from 'better-sqlite3-multiple-ciphers' +import { BaseTable } from '@/data/baseTable' +import { buildDeepChatTapeFtsMatch, buildDeepChatTapeLikeSearchPredicate } from './tapeEntryStore' +import { + normalizeDeepChatTapeReadSources, + serializeDeepChatTapeReadSources +} from '@/tape/domain/entry' +import type { DeepChatTapeReadSource, DeepChatTapeSearchInput } from '@/tape/domain/entry' +import type { + TapeSearchProjectionInput, + TapeSearchProjectionMeta, + TapeSearchProjectionReadResult, + TapeSearchProjectionResultRow, + TapeSearchProjectionRow, + TapeSearchProjectionStore +} from '@/tape/ports/application' + +// Version 3 invalidates projections that may predate atomic Tape generation transitions. A +// matching entry-id head alone cannot prove that a version 2 row belongs to the current +// incarnation after a previously interrupted reset. +export const DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION = 3 + +export type DeepChatTapeSearchProjectionInput = TapeSearchProjectionInput +export type DeepChatTapeSearchProjectionRow = TapeSearchProjectionRow +export type DeepChatTapeSearchProjectionResultRow = TapeSearchProjectionResultRow +export type DeepChatTapeSearchProjectionMeta = TapeSearchProjectionMeta +export type DeepChatTapeSearchProjectionReadResult = TapeSearchProjectionReadResult + +type FtsCapability = { available: boolean; tokenizer: 'trigram' | 'unicode61' } + +const FTS_CAPABILITY_BY_DATABASE = new WeakMap() + +const TAPE_SEARCH_PROJECTION_INDEX_SQL = ` + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_kind + ON deepchat_tape_search_projection(session_id, kind, entry_id); + CREATE INDEX IF NOT EXISTS idx_deepchat_tape_search_projection_session_created + ON deepchat_tape_search_projection(session_id, created_at, entry_id); +` + +function normalizeLimit(limit: number | undefined): number { + return Math.min(Math.max(Math.floor(Number.isFinite(limit) ? (limit as number) : 20), 1), 100) +} + +function safeJsonStringify(value: Record): string { + return JSON.stringify(value ?? {}) +} + +function parseRefs(value: string): Record { + try { + const parsed = JSON.parse(value) + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : {} + } catch { + return {} + } +} + +export class DeepChatTapeSearchProjectionTable + extends BaseTable + implements TapeSearchProjectionStore +{ + private ftsCapability: FtsCapability | undefined + private ftsReady = false + + constructor(db: Database.Database) { + super(db, 'deepchat_tape_search_projection') + } + + getCreateTableSQL(): string { + return ` + CREATE TABLE IF NOT EXISTS deepchat_tape_search_projection ( + session_id TEXT NOT NULL, + entry_id INTEGER NOT NULL, + kind TEXT NOT NULL, + name TEXT, + source_type TEXT, + source_id TEXT, + source_seq INTEGER, + search_text TEXT NOT NULL, + summary_text TEXT NOT NULL, + refs_json TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, entry_id) + ); + CREATE TABLE IF NOT EXISTS deepchat_tape_search_projection_meta ( + session_id TEXT PRIMARY KEY, + projection_version INTEGER NOT NULL, + max_entry_id INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS deepchat_tape_search_fts_meta ( + session_id TEXT PRIMARY KEY, + projection_version INTEGER NOT NULL, + max_entry_id INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + ${TAPE_SEARCH_PROJECTION_INDEX_SQL} + ` + } + + override createTable(): void { + this.db.exec(this.getCreateTableSQL()) + this.pruneInvalidProjectionRows() + if (!this.ftsReady) { + this.ensureFtsIndex() + } + } + + getMigrationSQL(_version: number): string | null { + return null + } + + getLatestVersion(): number { + return 0 + } + + getSessionMeta(sessionId: string): DeepChatTapeSearchProjectionMeta | null { + const row = this.db + .prepare( + `SELECT projection_version, max_entry_id + FROM deepchat_tape_search_projection_meta + WHERE session_id = ?` + ) + .get(sessionId) as + | { + projection_version: number + max_entry_id: number + } + | undefined + if (!row) return null + return { + projectionVersion: row.projection_version, + maxEntryId: row.max_entry_id + } + } + + isCurrent( + sessionId: string, + maxEntryId: number, + projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ): boolean { + const row = this.getSessionMeta(sessionId) + return row?.projectionVersion === projectionVersion && row.maxEntryId === maxEntryId + } + + getProjectedEntryIds(sessionId: string): number[] { + return ( + this.db + .prepare( + `SELECT entry_id + FROM deepchat_tape_search_projection + WHERE session_id = ? + ORDER BY entry_id ASC` + ) + .all(sessionId) as Array<{ entry_id: number }> + ).map((row) => row.entry_id) + } + + appendSession( + sessionId: string, + rows: DeepChatTapeSearchProjectionInput[], + maxEntryId: number, + projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ): void { + const previousMeta = this.getSessionMeta(sessionId) + try { + this.db.transaction(() => { + if (!this.ftsReady) { + this.clearSessionFtsForBaseWrite(sessionId) + } + this.insertProjectionRows(rows) + if (this.ftsReady) { + if (previousMeta && this.isFtsCurrent(sessionId, previousMeta)) { + this.insertFtsRows(rows) + } else { + this.replaceSessionFtsRows(sessionId, this.getSessionProjectionInputs(sessionId)) + } + this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) + } + this.upsertMeta(sessionId, projectionVersion, maxEntryId) + })() + } catch (error) { + if (this.ftsReady) { + this.ftsReady = false + } + throw error + } + } + + replaceSession( + sessionId: string, + rows: DeepChatTapeSearchProjectionInput[], + maxEntryId: number, + projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ): void { + try { + this.db.transaction(() => { + if (this.ftsReady) { + this.db + .prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?') + .run(sessionId) + this.db + .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') + .run(sessionId) + } else { + this.clearSessionFtsForBaseWrite(sessionId) + } + this.db + .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') + .run(sessionId) + this.db + .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') + .run(sessionId) + this.insertProjectionRows(rows) + if (this.ftsReady) { + this.insertFtsRows(rows) + this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) + } + this.upsertMeta(sessionId, projectionVersion, maxEntryId) + })() + } catch (error) { + if (this.ftsReady) { + this.ftsReady = false + } + throw error + } + } + + private insertProjectionRows(rows: DeepChatTapeSearchProjectionInput[]): void { + if (!rows.length) return + const insertProjection = this.db.prepare( + `INSERT OR REPLACE INTO deepchat_tape_search_projection ( + session_id, + entry_id, + kind, + name, + source_type, + source_id, + source_seq, + search_text, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + for (const row of rows) { + insertProjection.run( + row.sessionId, + row.entryId, + row.kind, + row.name, + row.sourceType, + row.sourceId, + row.sourceSeq, + row.searchText, + row.summaryText, + safeJsonStringify(row.refs), + row.createdAt + ) + } + } + + private upsertMeta(sessionId: string, projectionVersion: number, maxEntryId: number): void { + this.db + .prepare( + `INSERT INTO deepchat_tape_search_projection_meta ( + session_id, + projection_version, + max_entry_id, + updated_at + ) + VALUES (?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + projection_version = excluded.projection_version, + max_entry_id = excluded.max_entry_id, + updated_at = excluded.updated_at` + ) + .run(sessionId, projectionVersion, maxEntryId, Date.now()) + } + + private getFtsMeta(sessionId: string): DeepChatTapeSearchProjectionMeta | null { + const row = this.db + .prepare( + `SELECT projection_version, max_entry_id + FROM deepchat_tape_search_fts_meta + WHERE session_id = ?` + ) + .get(sessionId) as + | { + projection_version: number + max_entry_id: number + } + | undefined + if (!row) return null + return { + projectionVersion: row.projection_version, + maxEntryId: row.max_entry_id + } + } + + private isFtsCurrent(sessionId: string, meta: DeepChatTapeSearchProjectionMeta): boolean { + const row = this.getFtsMeta(sessionId) + return row?.projectionVersion === meta.projectionVersion && row.maxEntryId === meta.maxEntryId + } + + private upsertFtsMeta(sessionId: string, projectionVersion: number, maxEntryId: number): void { + this.db + .prepare( + `INSERT INTO deepchat_tape_search_fts_meta ( + session_id, + projection_version, + max_entry_id, + updated_at + ) + VALUES (?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + projection_version = excluded.projection_version, + max_entry_id = excluded.max_entry_id, + updated_at = excluded.updated_at` + ) + .run(sessionId, projectionVersion, maxEntryId, Date.now()) + } + + getByEntryIds(sessionId: string, entryIds: number[]): DeepChatTapeSearchProjectionRow[] { + const ids = [...new Set(entryIds.filter((id) => Number.isInteger(id) && id > 0))] + if (!ids.length) return [] + const placeholders = ids.map(() => '?').join(', ') + return this.db + .prepare( + `SELECT * + FROM deepchat_tape_search_projection + WHERE session_id = ? AND entry_id IN (${placeholders}) + ORDER BY entry_id ASC` + ) + .all(sessionId, ...ids) as DeepChatTapeSearchProjectionRow[] + } + + getByEntryIdsIfCurrent( + sessionId: string, + maxEntryId: number, + entryIds: number[], + projectionVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ): DeepChatTapeSearchProjectionRow[] { + const ids = [...new Set(entryIds.filter((id) => Number.isInteger(id) && id > 0))] + if (!ids.length) return [] + const placeholders = ids.map(() => '?').join(', ') + return this.db + .prepare( + `SELECT projection.* + FROM deepchat_tape_search_projection AS projection + INNER JOIN deepchat_tape_search_projection_meta AS meta + ON meta.session_id = projection.session_id + AND meta.projection_version = ? + AND meta.max_entry_id = ? + WHERE projection.session_id = ? AND projection.entry_id IN (${placeholders}) + ORDER BY projection.entry_id ASC` + ) + .all(projectionVersion, maxEntryId, sessionId, ...ids) as DeepChatTapeSearchProjectionRow[] + } + + searchSourcesReadOnly( + sources: readonly DeepChatTapeReadSource[], + query: string, + options: DeepChatTapeSearchInput = {} + ): DeepChatTapeSearchProjectionReadResult { + const normalizedSources = normalizeDeepChatTapeReadSources(sources) + const coveredSources = this.getCurrentReadSources( + normalizedSources, + 'deepchat_tape_search_projection_meta' + ) + const normalizedQuery = query.trim() + if (coveredSources.length === 0 || !normalizedQuery) { + return { rows: [], coveredSources } + } + if (coveredSources.length !== normalizedSources.length) { + return { rows: [], coveredSources } + } + + const limit = normalizeLimit(options.limit) + const ordered: DeepChatTapeSearchProjectionResultRow[] = [] + const seen = new Set() + const collect = (rows: DeepChatTapeSearchProjectionResultRow[]): void => { + for (const row of rows) { + const key = `${row.session_id}:${row.entry_id}` + if (seen.has(key)) continue + seen.add(key) + ordered.push(row) + } + } + + if (this.ftsReady) { + const ftsSources = this.getCurrentReadSources(coveredSources, 'deepchat_tape_search_fts_meta') + if (ftsSources.length === coveredSources.length) { + try { + collect(this.searchFtsSourcesReadOnly(ftsSources, normalizedQuery, options, limit)) + } catch { + // The base projection remains a complete read-only fallback. + } + } + } + if (ordered.length < limit) { + collect(this.searchLikeSourcesReadOnly(coveredSources, normalizedQuery, options, limit)) + } + + return { rows: ordered.slice(0, limit), coveredSources } + } + + search( + sessionId: string, + query: string, + options: DeepChatTapeSearchInput = {} + ): DeepChatTapeSearchProjectionResultRow[] { + const normalized = query.trim() + if (!normalized) return [] + const limit = normalizeLimit(options.limit) + const ordered: DeepChatTapeSearchProjectionResultRow[] = [] + const seen = new Set() + const collect = (rows: DeepChatTapeSearchProjectionResultRow[]): void => { + for (const row of rows) { + if (seen.has(row.entry_id)) continue + seen.add(row.entry_id) + ordered.push(row) + } + } + this.recoverSessionFts(sessionId) + if (this.ftsReady) { + collect(this.searchFts(sessionId, normalized, options, limit)) + } + if (!this.ftsReady || ordered.length < limit) { + collect(this.searchLike(sessionId, normalized, options, limit)) + } + return ordered.slice(0, limit) + } + + deleteBySession(sessionId: string): void { + this.db.transaction(() => { + this.db + .prepare('DELETE FROM deepchat_tape_search_projection WHERE session_id = ?') + .run(sessionId) + this.db + .prepare('DELETE FROM deepchat_tape_search_projection_meta WHERE session_id = ?') + .run(sessionId) + this.deleteSessionFts(sessionId, true) + })() + } + + clearAll(): void { + this.db.transaction(() => { + this.db.prepare('DELETE FROM deepchat_tape_search_projection').run() + this.db.prepare('DELETE FROM deepchat_tape_search_projection_meta').run() + if (this.ftsMetaTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() + } + this.clearFts(true) + })() + } + + private detectFtsCapability(): FtsCapability { + if (this.ftsCapability) return this.ftsCapability + const cached = FTS_CAPABILITY_BY_DATABASE.get(this.db) + if (cached) { + this.ftsCapability = cached + return cached + } + const probe = (tokenizer: string): boolean => { + const name = `temp.tape_search_fts_probe_${tokenizer}` + try { + this.db.exec( + `CREATE VIRTUAL TABLE IF NOT EXISTS ${name} USING fts5(c, tokenize='${tokenizer}');` + ) + this.db.exec(`DROP TABLE IF EXISTS ${name};`) + return true + } catch { + return false + } + } + if (probe('trigram')) this.ftsCapability = { available: true, tokenizer: 'trigram' } + else if (probe('unicode61')) this.ftsCapability = { available: true, tokenizer: 'unicode61' } + else this.ftsCapability = { available: false, tokenizer: 'unicode61' } + FTS_CAPABILITY_BY_DATABASE.set(this.db, this.ftsCapability) + return this.ftsCapability + } + + private ensureFtsIndex(): void { + const capability = this.detectFtsCapability() + if (!capability.available) { + this.ftsReady = false + return + } + try { + this.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS deepchat_tape_search_fts USING fts5( + search_text, + name, + session_id UNINDEXED, + entry_id UNINDEXED, + kind UNINDEXED, + source_type UNINDEXED, + source_id UNINDEXED, + source_seq UNINDEXED, + summary_text UNINDEXED, + refs_json UNINDEXED, + created_at UNINDEXED, + tokenize='${capability.tokenizer}' + ); + `) + this.ftsReady = true + } catch { + this.ftsReady = false + } + } + + private toProjectionInput( + row: DeepChatTapeSearchProjectionRow + ): DeepChatTapeSearchProjectionInput { + return { + sessionId: row.session_id, + entryId: row.entry_id, + kind: row.kind, + name: row.name, + sourceType: row.source_type, + sourceId: row.source_id, + sourceSeq: row.source_seq, + searchText: row.search_text, + summaryText: row.summary_text, + refs: parseRefs(row.refs_json), + createdAt: row.created_at + } + } + + private getSessionProjectionInputs(sessionId: string): DeepChatTapeSearchProjectionInput[] { + return ( + this.db + .prepare( + `SELECT * + FROM deepchat_tape_search_projection + WHERE session_id = ? + ORDER BY entry_id ASC` + ) + .all(sessionId) as DeepChatTapeSearchProjectionRow[] + ).map((row) => this.toProjectionInput(row)) + } + + private ftsTableExists(): boolean { + const row = this.db + .prepare( + `SELECT name + FROM sqlite_master + WHERE type = 'table' AND name = 'deepchat_tape_search_fts' + LIMIT 1` + ) + .get() as { name: string } | undefined + return Boolean(row) + } + + private ftsMetaTableExists(): boolean { + const row = this.db + .prepare( + `SELECT name + FROM sqlite_master + WHERE type = 'table' AND name = 'deepchat_tape_search_fts_meta' + LIMIT 1` + ) + .get() as { name: string } | undefined + return Boolean(row) + } + + private clearSessionFtsForBaseWrite(sessionId: string): void { + if (this.ftsMetaTableExists()) { + this.db + .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') + .run(sessionId) + } + if (this.ftsTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) + } + } + + private getProjectionRowId(sessionId: string, entryId: number): number { + const row = this.db + .prepare( + `SELECT rowid + FROM deepchat_tape_search_projection + WHERE session_id = ? AND entry_id = ?` + ) + .get(sessionId, entryId) as { rowid: number } | undefined + if (!row) { + throw new Error(`Missing tape search projection row for ${sessionId}:${entryId}`) + } + return row.rowid + } + + private insertFtsRows(rows: DeepChatTapeSearchProjectionInput[]): void { + if (!rows.length) return + const insertFts = this.db.prepare( + `INSERT INTO deepchat_tape_search_fts ( + rowid, + search_text, + name, + session_id, + entry_id, + kind, + source_type, + source_id, + source_seq, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + for (const row of rows) { + this.db + .prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ? AND entry_id = ?') + .run(row.sessionId, row.entryId) + insertFts.run( + this.getProjectionRowId(row.sessionId, row.entryId), + row.searchText, + row.name ?? '', + row.sessionId, + row.entryId, + row.kind, + row.sourceType, + row.sourceId, + row.sourceSeq, + row.summaryText, + safeJsonStringify(row.refs), + row.createdAt + ) + } + } + + private replaceSessionFtsRows( + sessionId: string, + rows: DeepChatTapeSearchProjectionInput[] + ): void { + this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) + this.insertFtsRows(rows) + } + + private replaceSessionFts( + sessionId: string, + rows: DeepChatTapeSearchProjectionInput[], + projectionVersion: number, + maxEntryId: number + ): boolean { + if (!this.ftsReady) return false + try { + this.db.transaction(() => { + this.replaceSessionFtsRows(sessionId, rows) + this.upsertFtsMeta(sessionId, projectionVersion, maxEntryId) + })() + return true + } catch { + this.ftsReady = false + return false + } + } + + private recoverSessionFts(sessionId: string): void { + const meta = this.getSessionMeta(sessionId) + if (!meta) { + this.deleteSessionFts(sessionId) + return + } + if (this.ftsReady && this.isFtsCurrent(sessionId, meta)) return + if (!this.ftsReady) { + this.ensureFtsIndex() + } + if (!this.ftsReady) return + if (this.isFtsCurrent(sessionId, meta)) return + this.replaceSessionFts( + sessionId, + this.getSessionProjectionInputs(sessionId), + meta.projectionVersion, + meta.maxEntryId + ) + } + + hasFtsReadyForTesting(): boolean { + return this.ftsReady + } + + disableFtsForTesting(): void { + this.ftsReady = false + } + + dropFtsForTesting(): void { + this.dropFtsIndex() + } + + private deleteSessionFts(sessionId: string, dropOnFailure = false): void { + if (this.ftsMetaTableExists()) { + this.db + .prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?') + .run(sessionId) + } + if (!this.ftsTableExists()) return + try { + this.db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run(sessionId) + } catch { + this.ftsReady = false + if (dropOnFailure) { + this.dropFtsIndex() + } + } + } + + private clearFts(dropOnFailure = false): void { + if (this.ftsMetaTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() + } + if (!this.ftsTableExists()) return + try { + this.db.prepare('DELETE FROM deepchat_tape_search_fts').run() + } catch { + this.ftsReady = false + if (dropOnFailure) { + this.dropFtsIndex() + } + } + } + + private dropFtsIndex(): void { + this.ftsReady = false + this.db.exec('DROP TABLE IF EXISTS deepchat_tape_search_fts') + if (this.ftsMetaTableExists()) { + this.db.prepare('DELETE FROM deepchat_tape_search_fts_meta').run() + } + } + + private pruneInvalidProjectionRows(): void { + this.db.transaction(() => { + this.db + .prepare( + `DELETE FROM deepchat_tape_search_projection + WHERE NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_search_projection_meta AS meta + WHERE meta.session_id = deepchat_tape_search_projection.session_id + AND meta.projection_version >= ? + )` + ) + .run(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION) + this.db + .prepare( + `DELETE FROM deepchat_tape_search_projection_meta + WHERE projection_version < ?` + ) + .run(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION) + + if (this.ftsTableExists()) { + try { + this.db + .prepare( + `DELETE FROM deepchat_tape_search_fts + WHERE NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_search_fts_meta AS meta + WHERE meta.session_id = deepchat_tape_search_fts.session_id + AND meta.projection_version >= ? + ) + OR NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_search_projection_meta AS meta + WHERE meta.session_id = deepchat_tape_search_fts.session_id + AND meta.projection_version >= ? + )` + ) + .run(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION) + } catch { + this.dropFtsIndex() + } + } + if (this.ftsMetaTableExists()) { + this.db + .prepare( + `DELETE FROM deepchat_tape_search_fts_meta + WHERE projection_version < ? + OR NOT EXISTS ( + SELECT 1 + FROM deepchat_tape_search_projection_meta AS projection_meta + WHERE projection_meta.session_id = deepchat_tape_search_fts_meta.session_id + AND projection_meta.projection_version >= ? + )` + ) + .run(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION) + } + })() + } + + private searchFts( + sessionId: string, + normalized: string, + options: DeepChatTapeSearchInput, + limit: number + ): DeepChatTapeSearchProjectionResultRow[] { + const match = buildDeepChatTapeFtsMatch(normalized) + const whereClauses = [ + 'deepchat_tape_search_fts MATCH ?', + 'deepchat_tape_search_fts.session_id = ?', + 'projection.session_id = ?' + ] + const params: Array = [match, sessionId, sessionId] + this.addFilters(whereClauses, params, options, true, 'projection') + params.push(limit) + try { + return this.db + .prepare( + `SELECT + projection.session_id, + projection.entry_id, + projection.kind, + projection.name, + projection.source_type, + projection.source_id, + projection.source_seq, + projection.search_text, + projection.summary_text, + projection.refs_json, + projection.created_at, + bm25(deepchat_tape_search_fts) AS score + FROM deepchat_tape_search_fts + INNER JOIN deepchat_tape_search_projection AS projection + ON projection.session_id = deepchat_tape_search_fts.session_id + AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) + AND projection.search_text = deepchat_tape_search_fts.search_text + WHERE ${whereClauses.join(' AND ')} + ORDER BY score ASC, projection.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeSearchProjectionResultRow[] + } catch { + try { + this.dropFtsIndex() + } catch { + this.ftsReady = false + } + return [] + } + } + + private getCurrentReadSources( + sources: readonly DeepChatTapeReadSource[], + metaTable: 'deepchat_tape_search_projection_meta' | 'deepchat_tape_search_fts_meta' + ): DeepChatTapeReadSource[] { + const normalizedSources = normalizeDeepChatTapeReadSources(sources) + if (normalizedSources.length === 0) { + return [] + } + const rows = this.db + .prepare( + `WITH requested_sources(session_id, max_entry_id) AS ( + SELECT + json_extract(value, '$.sessionId'), + CAST(json_extract(value, '$.maxEntryId') AS INTEGER) + FROM json_each(?) + ) + SELECT requested.session_id, requested.max_entry_id + FROM requested_sources AS requested + INNER JOIN ${metaTable} AS meta + ON meta.session_id = requested.session_id + AND meta.max_entry_id = requested.max_entry_id + AND meta.projection_version = ? + ORDER BY requested.session_id ASC` + ) + .all( + serializeDeepChatTapeReadSources(normalizedSources), + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + ) as Array<{ session_id: string; max_entry_id: number }> + return rows.map((row) => ({ sessionId: row.session_id, maxEntryId: row.max_entry_id })) + } + + private searchFtsSourcesReadOnly( + sources: readonly DeepChatTapeReadSource[], + normalized: string, + options: DeepChatTapeSearchInput, + limit: number + ): DeepChatTapeSearchProjectionResultRow[] { + const match = buildDeepChatTapeFtsMatch(normalized) + const whereClauses = ['deepchat_tape_search_fts MATCH ?'] + const params: Array = [serializeDeepChatTapeReadSources(sources), match] + this.addFilters(whereClauses, params, options, true, 'projection') + params.push(limit) + return this.db + .prepare( + `WITH authorized_sources(session_id, max_entry_id) AS ( + SELECT + json_extract(value, '$.sessionId'), + CAST(json_extract(value, '$.maxEntryId') AS INTEGER) + FROM json_each(?) + ) + SELECT + projection.session_id, + projection.entry_id, + projection.kind, + projection.name, + projection.source_type, + projection.source_id, + projection.source_seq, + projection.search_text, + projection.summary_text, + projection.refs_json, + projection.created_at, + bm25(deepchat_tape_search_fts) AS score + FROM deepchat_tape_search_fts + INNER JOIN deepchat_tape_search_projection AS projection + ON projection.session_id = deepchat_tape_search_fts.session_id + AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) + AND projection.search_text = deepchat_tape_search_fts.search_text + INNER JOIN authorized_sources AS source + ON source.session_id = projection.session_id + AND projection.entry_id <= source.max_entry_id + WHERE ${whereClauses.join(' AND ')} + ORDER BY score ASC, projection.created_at DESC, projection.session_id ASC, + projection.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeSearchProjectionResultRow[] + } + + private searchLikeSourcesReadOnly( + sources: readonly DeepChatTapeReadSource[], + normalized: string, + options: DeepChatTapeSearchInput, + limit: number + ): DeepChatTapeSearchProjectionResultRow[] { + const queryPredicate = buildDeepChatTapeLikeSearchPredicate( + ['projection.search_text', 'projection.summary_text', 'projection.name'], + normalized + ) + const whereClauses = [queryPredicate.sql] + const params: Array = [serializeDeepChatTapeReadSources(sources)] + params.push(...queryPredicate.params) + this.addFilters(whereClauses, params, options, false, 'projection') + params.push(limit) + return this.db + .prepare( + `WITH authorized_sources(session_id, max_entry_id) AS ( + SELECT + json_extract(value, '$.sessionId'), + CAST(json_extract(value, '$.maxEntryId') AS INTEGER) + FROM json_each(?) + ) + SELECT projection.*, NULL AS score + FROM deepchat_tape_search_projection AS projection + INNER JOIN authorized_sources AS source + ON source.session_id = projection.session_id + AND projection.entry_id <= source.max_entry_id + WHERE ${whereClauses.join(' AND ')} + ORDER BY projection.created_at DESC, projection.session_id ASC, projection.entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeSearchProjectionResultRow[] + } + + private searchLike( + sessionId: string, + normalized: string, + options: DeepChatTapeSearchInput, + limit: number + ): DeepChatTapeSearchProjectionResultRow[] { + const queryPredicate = buildDeepChatTapeLikeSearchPredicate( + ['search_text', 'summary_text', 'name'], + normalized + ) + const whereClauses = ['session_id = ?', queryPredicate.sql] + const params: Array = [sessionId] + params.push(...queryPredicate.params) + this.addFilters(whereClauses, params, options) + params.push(limit) + return this.db + .prepare( + `SELECT *, NULL AS score + FROM deepchat_tape_search_projection + WHERE ${whereClauses.join(' AND ')} + ORDER BY entry_id DESC + LIMIT ?` + ) + .all(...params) as DeepChatTapeSearchProjectionResultRow[] + } + + private addFilters( + whereClauses: string[], + params: Array, + options: DeepChatTapeSearchInput, + castCreatedAt = false, + tableAlias?: string + ): void { + const column = (name: string) => (tableAlias ? `${tableAlias}.${name}` : name) + if (options.kinds?.length) { + whereClauses.push(`${column('kind')} IN (${options.kinds.map(() => '?').join(', ')})`) + params.push(...options.kinds) + } + const createdAtColumn = castCreatedAt + ? `CAST(${column('created_at')} AS INTEGER)` + : column('created_at') + if (Number.isFinite(options.startCreatedAt)) { + whereClauses.push(`${createdAtColumn} >= ?`) + params.push(options.startCreatedAt as number) + } + if (Number.isFinite(options.endCreatedAt)) { + whereClauses.push(`${createdAtColumn} <= ?`) + params.push(options.endCreatedAt as number) + } + } +} diff --git a/src/main/tape/ports/application.ts b/src/main/tape/ports/application.ts new file mode 100644 index 0000000000..c1b220f029 --- /dev/null +++ b/src/main/tape/ports/application.ts @@ -0,0 +1,179 @@ +import type { + DeepChatTapeEntryKind, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + DeepChatTapeSourceType +} from '../domain/entry' +import type { + TapeBootstrapStore, + TapeEntryLifecycleStore, + TapeEntryStore, + TapeTransactionRunner +} from './storage' + +export interface TapeSearchProjectionInput { + sessionId: string + entryId: number + kind: DeepChatTapeEntryKind + name: string | null + sourceType: DeepChatTapeSourceType | null + sourceId: string | null + sourceSeq: number | null + searchText: string + summaryText: string + refs: Record + createdAt: number +} + +export interface TapeSearchProjectionRow { + session_id: string + entry_id: number + kind: DeepChatTapeEntryKind + name: string | null + source_type: DeepChatTapeSourceType | null + source_id: string | null + source_seq: number | null + search_text: string + summary_text: string + refs_json: string + created_at: number +} + +export interface TapeSearchProjectionResultRow extends TapeSearchProjectionRow { + score: number | null +} + +export interface TapeSearchProjectionMeta { + projectionVersion: number + maxEntryId: number +} + +export interface TapeSearchProjectionReadResult { + rows: TapeSearchProjectionResultRow[] + coveredSources: DeepChatTapeReadSource[] +} + +export interface TapeSearchProjectionStore { + getSessionMeta(sessionId: string): TapeSearchProjectionMeta | null + isCurrent(sessionId: string, maxEntryId: number, projectionVersion?: number): boolean + getProjectedEntryIds(sessionId: string): number[] + appendSession( + sessionId: string, + rows: TapeSearchProjectionInput[], + maxEntryId: number, + projectionVersion?: number + ): void + replaceSession( + sessionId: string, + rows: TapeSearchProjectionInput[], + maxEntryId: number, + projectionVersion?: number + ): void + getByEntryIdsIfCurrent( + sessionId: string, + maxEntryId: number, + entryIds: number[], + projectionVersion?: number + ): TapeSearchProjectionRow[] + searchSourcesReadOnly( + sources: readonly DeepChatTapeReadSource[], + query: string, + options?: DeepChatTapeSearchInput + ): TapeSearchProjectionReadResult + search( + sessionId: string, + query: string, + options?: DeepChatTapeSearchInput + ): TapeSearchProjectionResultRow[] + deleteBySession(sessionId: string): void +} + +export interface TapeLineageSessionRow { + id: string + session_kind: 'regular' | 'subagent' + parent_session_id: string | null +} + +export interface TapeLineageSessionReader { + get(sessionId: string): TapeLineageSessionRow | undefined + getMany(sessionIds: string[]): TapeLineageSessionRow[] +} + +export interface TapeLegacySummaryState { + summary_text: string | null + summary_cursor_order_seq: number | null + summary_updated_at: number | null +} + +export interface TapeLegacySummaryReader { + getSummaryState(sessionId: string): TapeLegacySummaryState | null +} + +export interface TapeMessageTraceRow { + id: string + message_id: string + session_id: string + provider_id: string + model_id: string + request_seq: number + endpoint: string + headers_json: string + body_json: string + truncated: number + created_at: number +} + +export interface TapeMessageTraceReader { + listByMessageId(messageId: string): TapeMessageTraceRow[] +} + +export interface TapeTerminalMessageRow { + session_id: string + order_seq: number + role: 'user' | 'assistant' + content: string + status: 'pending' | 'sent' | 'error' + metadata: string + created_at: number + updated_at: number +} + +export interface TapeTerminalMessageReader { + get(messageId: string): TapeTerminalMessageRow | undefined +} + +export type TapeApplicationEntryStore = TapeEntryStore & TapeTransactionRunner & TapeBootstrapStore + +export interface TapeApplicationDatabase { + readonly deepchatTapeEntriesTable: TapeApplicationEntryStore + readonly tapeLifecycle: TapeEntryLifecycleStore + readonly deepchatTapeSearchProjectionTable: TapeSearchProjectionStore + readonly newSessionsTable: TapeLineageSessionReader + readonly deepchatSessionsTable: TapeLegacySummaryReader + readonly deepchatMessageTracesTable: TapeMessageTraceReader + readonly deepchatMessagesTable: TapeTerminalMessageReader +} + +export interface TapeApplicationProviders { + getEntryStore(): TapeApplicationEntryStore + getEntryLifecycleStore(): TapeEntryLifecycleStore + getSearchProjectionStore(): TapeSearchProjectionStore + getLineageSessionReader(): TapeLineageSessionReader + getLegacySummaryReader(): TapeLegacySummaryReader + getMessageTraceReader(): TapeMessageTraceReader + getTerminalMessageReader(): TapeTerminalMessageReader +} + +export function createTapeApplicationProviders( + database: TapeApplicationDatabase +): TapeApplicationProviders { + return { + getEntryStore: () => database.deepchatTapeEntriesTable, + getEntryLifecycleStore: () => database.tapeLifecycle, + getSearchProjectionStore: () => database.deepchatTapeSearchProjectionTable, + getLineageSessionReader: () => database.newSessionsTable, + getLegacySummaryReader: () => database.deepchatSessionsTable, + getMessageTraceReader: () => database.deepchatMessageTracesTable, + getTerminalMessageReader: () => database.deepchatMessagesTable + } +} diff --git a/src/main/tape/ports/capabilities.ts b/src/main/tape/ports/capabilities.ts new file mode 100644 index 0000000000..68e5898aa7 --- /dev/null +++ b/src/main/tape/ports/capabilities.ts @@ -0,0 +1,108 @@ +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import type { + DeepChatTapeViewManifest, + DeepChatTapeViewManifestRecord +} from '@shared/types/tape-view-manifest' +import type { DeepChatTapeEntryRow, TapeAnchorAppendInput } from '../domain/entry' +import type { TapeEntryRef, TapeToolFactInput } from '../domain/facts' + +export type TapeMigrationState = 'none' | 'ready' + +export type TapeBackfillResult = { + sessionId: string + migrationState: TapeMigrationState + messageCount: number + maxOrderSeq: number + appendedFactCount: number + historyRecords: ChatMessageRecord[] +} + +export interface TapeTranscriptReader { + getMessages(sessionId: string): ChatMessageRecord[] +} + +export interface TapeReconciliationPort { + ensureSessionTapeReady(sessionId: string, messageStore: TapeTranscriptReader): TapeBackfillResult +} + +export type TapeViewManifestAssemblySources = { + latestEntryId: number + anchorEntryIds: number[] + reconstructionAnchorEntryIds: number[] + reconstructionAnchorEntryId: number | null + entryIdByMessageId: Map + toolCallEntryIdByToolId: Map + toolResultEntryIdByToolId: Map +} + +export interface TapeViewManifestReader { + getViewManifestSourceMaps(sessionId: string, messageId?: string): TapeViewManifestAssemblySources + listViewManifestsByMessage(sessionId: string, messageId: string): DeepChatTapeViewManifestRecord[] +} + +export interface TapeViewManifestWriter { + appendViewManifest(manifest: DeepChatTapeViewManifest): void +} + +export interface TapeToolFactWriter { + appendToolFact(input: TapeToolFactInput): Promise +} + +export interface TapeMessageFactWriter { + appendMessageRecord(record: ChatMessageRecord): number + appendMessageReplacement(record: ChatMessageRecord, reason: string): number + appendMessageRetraction(record: ChatMessageRecord, reason: string): number +} + +export interface TapeRawEntryReader { + getBySession(sessionId: string): DeepChatTapeEntryRow[] +} + +export interface TapeAnchorReader { + getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined +} + +export interface TapeAnchorWriter { + /** + * Compound Session settings updates require this writer to share the caller's synchronous + * SQLite connection and transaction context. Cross-connection or asynchronous adapters cannot + * preserve summary-and-anchor atomicity. + */ + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow +} + +export interface TapeInspectionReader { + getEffectiveMessageSourceSpan( + sessionId: string, + entryIds: number[] + ): TapeEffectiveMessageSourceEntry[] + listMemoryViewManifestsByAgent( + agentId: string, + options?: { sessionId?: string; limit?: number; messageId?: string } + ): TapeMemoryViewManifestInspection[] +} + +export interface TapeEffectiveMessageSourceEntry { + entryId: number + record: Pick +} + +export interface TapeMemoryViewManifestInspection { + sessionId: string + messageId: string | null + entryId: number + policyVersion: number | null + tokenBudget: number + estimatedTokens: number + selectedCount: number + selectedIds: string[] | null + droppedCount: number + queryHash: string | null + createdAt: number +} + +export interface TapeLifecycleAdmin { + initializeSessionTape(sessionId: string): void + deleteSessionTape(sessionId: string): void + resetSessionTape(sessionId: string): void +} diff --git a/src/main/tape/ports/storage.ts b/src/main/tape/ports/storage.ts new file mode 100644 index 0000000000..1a33d390a0 --- /dev/null +++ b/src/main/tape/ports/storage.ts @@ -0,0 +1,67 @@ +import type { + DeepChatTapeAppendInput, + DeepChatTapeEntryRow, + DeepChatTapeReadSource, + DeepChatTapeSearchInput, + TapeAnchorAppendInput, + TapeEventAppendInput +} from '../domain/entry' + +export interface TapeMutationProjection { + applyAppendedEntry(row: DeepChatTapeEntryRow, previousSessionMaxEntryId: number): boolean + invalidateSession(sessionId: string): void + deleteBySession(sessionId: string): void +} + +/** Append/read/query persistence only. Physical deletion belongs to TapeEntryLifecycleStore. */ +export interface TapeEntryStore { + append(input: DeepChatTapeAppendInput): DeepChatTapeEntryRow + appendAnchor(input: TapeAnchorAppendInput): DeepChatTapeEntryRow + appendEvent(input: TapeEventAppendInput): DeepChatTapeEntryRow + getBySession(sessionId: string): DeepChatTapeEntryRow[] + getSubagentLineageEvents(sessionId: string): DeepChatTapeEntryRow[] + getFirstEntriesBySessions(sessionIds: string[]): DeepChatTapeEntryRow[] + getBySessionUpToEntryId(sessionId: string, maxEntryId: number): DeepChatTapeEntryRow[] + listMemoryViewManifestAnchorsByAgent( + agentId: string, + options?: { sessionId?: string; limit?: number; messageId?: string } + ): DeepChatTapeEntryRow[] + getLatestAnchor(sessionId: string): DeepChatTapeEntryRow | undefined + getAnchors(sessionId: string, limit?: number): DeepChatTapeEntryRow[] + getLatestSummaryAnchor(sessionId: string): DeepChatTapeEntryRow | undefined + getLatestReconstructionAnchor(sessionId: string): DeepChatTapeEntryRow | undefined + getByProvenanceKey(sessionId: string, provenanceKey: string): DeepChatTapeEntryRow | undefined + getMaxEntryId(sessionId: string): number + getMaxEntryIdsBySessions(sessionIds: string[]): Map + countAnchorsBySession(sessionId: string): number + countEntriesAfter(sessionId: string, entryId: number): number + countBySession(sessionId: string): number + search( + sessionId: string, + query: string, + options?: DeepChatTapeSearchInput + ): DeepChatTapeEntryRow[] + searchEffectiveSourcesAtHeads( + sources: readonly DeepChatTapeReadSource[], + query: string, + options?: DeepChatTapeSearchInput + ): DeepChatTapeEntryRow[] + getEffectiveContextRowsAtHead( + source: DeepChatTapeReadSource, + entryIds: number[], + options: { before: number; after: number; limit: number } + ): DeepChatTapeEntryRow[] +} + +export interface TapeTransactionRunner { + runInTransaction(operation: () => T): T +} + +/** Transitional bootstrap capability until bootstrap orchestration lives in application services. */ +export interface TapeBootstrapStore { + ensureBootstrapAnchor(sessionId: string): void +} + +export interface TapeEntryLifecycleStore { + deleteBySession(sessionId: string): void +} diff --git a/test/main/agent/acp/compatibility/adapters.test.ts b/test/main/agent/acp/compatibility/adapters.test.ts index 9388d1a4f5..62d7a732b9 100644 --- a/test/main/agent/acp/compatibility/adapters.test.ts +++ b/test/main/agent/acp/compatibility/adapters.test.ts @@ -182,13 +182,13 @@ function createProjectionHarness() { ) } } as unknown as MainDatabase - const messageStore = new SessionTranscript(sqlitePresenter) const tapeService = new SessionTape(sqlitePresenter) + const messageStore = new SessionTranscript(sqlitePresenter, tapeService) const adapter = new AcpCompatibilityProjectionAdapter({ publishEvent: publishDeepchatEvent, publishSessionUpdate: vi.fn(), messageStore, - tapeService, + tapeReconciliation: tapeService, writeViewManifest: vi.fn(), setStatus: vi.fn() }) diff --git a/test/main/agent/deepchat/memory/memoryRuntimeCoordinator.test.ts b/test/main/agent/deepchat/memory/memoryRuntimeCoordinator.test.ts index ee02cefd3a..2b724c9616 100644 --- a/test/main/agent/deepchat/memory/memoryRuntimeCoordinator.test.ts +++ b/test/main/agent/deepchat/memory/memoryRuntimeCoordinator.test.ts @@ -95,6 +95,8 @@ function createHarness() { replaceSession: vi.fn(), invalidateSession: vi.fn() } + const getTapeRows = vi.fn(() => tapeRows) + const appendTapeAnchor = vi.fn() const deps = { memoryPort: port as any, getSessionAgentId: vi.fn(() => 'agent-a'), @@ -116,8 +118,16 @@ function createHarness() { rewindMemoryCursorOrderSeq: vi.fn((_sessionId: string, orderSeq: number) => { cursor = orderSeq }), - getTapeRows: vi.fn(() => tapeRows), - appendTapeAnchor: vi.fn(), + tapeReader: { + getBySession: getTapeRows, + getBySessionUpToEntryId: vi.fn((_sessionId: string, maxEntryId: number) => + tapeRows.filter((row) => row.entry_id <= maxEntryId) + ), + getMaxEntryId: vi.fn(() => tapeRows.at(-1)?.entry_id ?? 0) + }, + tapeAnchorWriter: { appendAnchor: appendTapeAnchor }, + getTapeRows, + appendTapeAnchor, getIngestionProjection: vi.fn(() => projection) } const coordinator = new MemoryRuntimeCoordinator(deps) diff --git a/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts b/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts index 16498a5563..c85b15a1ca 100644 --- a/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts +++ b/test/main/agent/deepchat/runtime/deepChatRuntimeCoordinator.test.ts @@ -322,6 +322,7 @@ function createMockSqlitePresenter() { delete: vi.fn() }, deepchatTapeEntriesTable: (deepchatTapeEntriesTable = { + runInTransaction: vi.fn((operation: () => unknown) => operation()), ensureBootstrapAnchor: vi.fn(), append: vi.fn((input: any) => { const provenanceKey = @@ -443,10 +444,12 @@ function createMockSqlitePresenter() { memoryIngestionProjectionCurrent = false }) }), + tapeLifecycle: deepchatTapeEntriesTable, deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn(), isCurrent: vi.fn().mockReturnValue(false), - getByEntryIds: vi.fn().mockReturnValue([]) + getByEntryIds: vi.fn().mockReturnValue([]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) }, deepchatMemoryIngestionProjectionTable: { readCurrentRange: vi.fn( @@ -874,7 +877,9 @@ describe('DeepChatRuntimeCoordinator', () => { transcript: sessionData.transcript, settings: sessionData.settings, pendingInputs: sessionData.pendingInputs, - runtime: agent + runtime: agent, + runInTransaction: (operation) => + sessionData.database.getDatabase().transaction(operation)() }) }) diff --git a/test/main/agent/deepchat/runtime/process.test.ts b/test/main/agent/deepchat/runtime/process.test.ts index 6c1b7a5c90..75de6fe02b 100644 --- a/test/main/agent/deepchat/runtime/process.test.ts +++ b/test/main/agent/deepchat/runtime/process.test.ts @@ -160,7 +160,7 @@ function makeStreamEvents(...events: LLMCoreStreamEvent[]): LLMCoreStreamEvent[] describe('processStream', () => { let messageStore: ReturnType - let tapeRecorder: { appendToolFact: ReturnType } + let tapeToolFactWriter: { appendToolFact: ReturnType } let tempHome: string | null = null let homedirSpy: ReturnType | null = null @@ -168,7 +168,7 @@ describe('processStream', () => { vi.useFakeTimers() vi.clearAllMocks() messageStore = createMockMessageStore() - tapeRecorder = { + tapeToolFactWriter = { appendToolFact: vi.fn(async (input) => ({ sessionId: input.sessionId, entryId: 1 @@ -241,7 +241,7 @@ describe('processStream', () => { permissionMode: 'full_access', io: { messageStore, - tapeRecorder, + tapeToolFactWriter, publishEvent: publishDeepchatEventMock, publishSessionUpdate: vi.fn() }, @@ -273,7 +273,7 @@ describe('processStream', () => { messageStore.setMessageError.mockImplementation(() => { order.push('message:error') }) - tapeRecorder.appendToolFact.mockImplementation(async (input) => { + tapeToolFactWriter.appendToolFact.mockImplementation(async (input) => { order.push(`tape:${input.provenance.source}`) return { sessionId: input.sessionId, entryId: 1 } }) @@ -374,7 +374,7 @@ describe('processStream', () => { 'renderer:update', 'renderer:complete' ]) - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() expect(JSON.parse(messageStore.finalizeAssistantMessage.mock.calls[0][2])).toMatchObject({ provider: 'openai', model: 'gpt-4' @@ -410,7 +410,7 @@ describe('processStream', () => { ]) expect(messageStore.finalizeAssistantMessage).toHaveBeenCalledTimes(1) expect(messageStore.setMessageError).toHaveBeenCalledTimes(1) - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() }) it('persists each tool round before entering the next provider round', async () => { @@ -470,9 +470,9 @@ describe('processStream', () => { 'renderer:update', 'renderer:complete' ]) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(6) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(6) expect( - tapeRecorder.appendToolFact.mock.calls.map(([input]) => [ + tapeToolFactWriter.appendToolFact.mock.calls.map(([input]) => [ input.provenance.source, input.provenance.sourceId, input.provenance.sequence @@ -487,8 +487,8 @@ describe('processStream', () => { ]) }) - it('keeps the tool loop fail-open when TapeRecorder rejects a fact', async () => { - tapeRecorder.appendToolFact.mockRejectedValue(new Error('tape unavailable')) + it('keeps the tool loop fail-open when TapeToolFactWriter rejects a fact', async () => { + tapeToolFactWriter.appendToolFact.mockRejectedValue(new Error('tape unavailable')) const coreStream = createToolThenCompleteStream('action') const result = await processStream( @@ -501,7 +501,7 @@ describe('processStream', () => { expect(result.status).toBe('completed') expect(coreStream).toHaveBeenCalledTimes(2) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(1) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(1) }) it('persists a paused tool round before its terminal projection', async () => { @@ -553,7 +553,7 @@ describe('processStream', () => { 'renderer:update', 'renderer:complete' ]) - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() }) it('accounts for executed tools before pausing a mixed tool batch', async () => { @@ -705,7 +705,7 @@ describe('processStream', () => { ]) expect(messageStore.setMessageError).toHaveBeenCalled() expect(messageStore.finalizeAssistantMessage).not.toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() expectDeepchatEvent('chat.stream.failed', { sessionId: 's1', messageId: 'm1', @@ -738,7 +738,7 @@ describe('processStream', () => { const abortMetadata = JSON.parse(messageStore.setMessageError.mock.calls[0][2]) expect(abortMetadata).toMatchObject({ provider: 'openai', model: 'gpt-4' }) expect(messageStore.finalizeAssistantMessage).not.toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() expectDeepchatEvent('chat.stream.failed', { sessionId: 's1', messageId: 'm1', @@ -767,7 +767,7 @@ describe('processStream', () => { }) expect(order).toEqual([...TOOL_ROUND_COMMIT_ORDER, ...ERROR_TERMINAL_COMMIT_ORDER]) expect(coreStream).toHaveBeenCalledTimes(1) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) it('does not snapshot an oversized tool batch that never executes', async () => { @@ -807,7 +807,7 @@ describe('processStream', () => { ...COMPLETED_TERMINAL_COMMIT_ORDER ]) expect(toolService.callTool).not.toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() }) it('persists a terminal tool-output error before the failed projection', async () => { @@ -831,7 +831,7 @@ describe('processStream', () => { expect(order).toEqual([...TOOL_ROUND_COMMIT_ORDER, ...ERROR_TERMINAL_COMMIT_ORDER]) expect(coreStream).toHaveBeenCalledTimes(1) expect(messageStore.setMessageError).toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) it('settles a post-stream abort without a tool Tape snapshot', async () => { @@ -848,7 +848,7 @@ describe('processStream', () => { expect(result.status).toBe('aborted') expect(order).toEqual(['renderer:update', 'message:update', ...ERROR_TERMINAL_COMMIT_ORDER]) - expect(tapeRecorder.appendToolFact).not.toHaveBeenCalled() + expect(tapeToolFactWriter.appendToolFact).not.toHaveBeenCalled() }) it('persists the completed tool batch before a post-tool abort', async () => { @@ -878,7 +878,7 @@ describe('processStream', () => { expect(toolService.callTool).toHaveBeenCalledTimes(1) expect(messageStore.setMessageError).toHaveBeenCalled() expect(messageStore.finalizeAssistantMessage).not.toHaveBeenCalled() - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) it('keeps a tool-local AbortError as a tool failure while the run remains active', async () => { @@ -929,7 +929,7 @@ describe('processStream', () => { }) ]) ) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) it('persists the completed batch before settling for pending input', async () => { @@ -953,7 +953,7 @@ describe('processStream', () => { }) expect(order).toEqual([...TOOL_ROUND_COMMIT_ORDER, ...COMPLETED_TERMINAL_COMMIT_ORDER]) expect(coreStream).toHaveBeenCalledTimes(1) - expect(tapeRecorder.appendToolFact).toHaveBeenCalledTimes(2) + expect(tapeToolFactWriter.appendToolFact).toHaveBeenCalledTimes(2) }) }) diff --git a/test/main/app/compositionBoundaries.test.ts b/test/main/app/compositionBoundaries.test.ts index 95af980b7e..6222d0e172 100644 --- a/test/main/app/compositionBoundaries.test.ts +++ b/test/main/app/compositionBoundaries.test.ts @@ -10,6 +10,9 @@ describe('session boundary composition', () => { ) expect(compositionSource.match(/new LegacyChatImportService\(/g)).toHaveLength(1) + expect(compositionSource).toMatch( + /new LegacyChatImportService\([^)]*memoryDatabase,\s*sessionData\.tapeStore/ + ) expect(compositionSource).toContain( 'legacyChatImportService.repairImportedLegacySessionSkills(conversationId)' ) diff --git a/test/main/app/startupMigrations/legacyChatImportService.test.ts b/test/main/app/startupMigrations/legacyChatImportService.test.ts index de24962edc..f2d083b8a1 100644 --- a/test/main/app/startupMigrations/legacyChatImportService.test.ts +++ b/test/main/app/startupMigrations/legacyChatImportService.test.ts @@ -123,6 +123,11 @@ describe('LegacyChatImportService', () => { sqlitePresenter as any, sqlitePresenter as any, sqlitePresenter as any, + { + appendMessageRecord: vi.fn(() => 0), + appendMessageReplacement: vi.fn(() => 0), + appendMessageRetraction: vi.fn(() => 0) + }, '/mock/legacy.db' ) }) diff --git a/test/main/evals/nativeAgent/harness.ts b/test/main/evals/nativeAgent/harness.ts index 7b8ce7fc05..3656bd979c 100644 --- a/test/main/evals/nativeAgent/harness.ts +++ b/test/main/evals/nativeAgent/harness.ts @@ -499,7 +499,7 @@ export async function runNativeAgentEvalScenario( messageStore, publishEvent: vi.fn(), publishSessionUpdate: vi.fn(), - tapeRecorder: { + tapeToolFactWriter: { appendToolFact: async () => ({ sessionId, entryId: 1 }) } } diff --git a/test/main/memory/deepchatMemoryIngestionProjection.test.ts b/test/main/memory/deepchatMemoryIngestionProjection.test.ts index 5549ae0550..564875c303 100644 --- a/test/main/memory/deepchatMemoryIngestionProjection.test.ts +++ b/test/main/memory/deepchatMemoryIngestionProjection.test.ts @@ -1,5 +1,6 @@ import { describe, expect, vi } from 'vitest' import { buildEffectiveTapeView } from '@/session/data/tapeEffectiveView' +import { SqliteTapeLifecycleAdapter } from '@/tape/infrastructure/sqlite/tapeLifecycleAdapter' import { Database, nativeSqliteItIf } from '../nativeSqliteHarness' const entriesModule = Database ? await import('@/session/data/tables/deepchatTapeEntries') : null @@ -25,7 +26,8 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { projection.createTable() const tape = new TapeTableCtor(db, projection) tape.createTable() - return { db, projection, tape } + const lifecycle = new SqliteTapeLifecycleAdapter(db, projection) + return { db, lifecycle, projection, tape } } function appendMessage( @@ -243,7 +245,7 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { itIfSqlite( 'marks retractions stale and keeps a final tool-before-message sequence current', () => { - const { db, projection, tape } = createTables() + const { db, lifecycle, projection, tape } = createTables() try { appendMessage(tape, { id: 'm1', @@ -259,8 +261,7 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { }) expect(projection.isCurrent('s1', tape.getMaxEntryId('s1'))).toBe(false) - projection.deleteBySession('s1') - tape.deleteBySession('s1') + lifecycle.deleteBySession('s1') appendToolCall(tape, { messageId: 'future-message', toolCallId: 'tool-before-message', @@ -434,7 +435,7 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { ) itIfSqlite('cleans projection rows and meta with the authoritative session delete', () => { - const { db, projection, tape } = createTables() + const { db, lifecycle, projection, tape } = createTables() try { appendMessage(tape, { id: 'm1', @@ -442,7 +443,7 @@ describe('DeepChatMemoryIngestionProjectionTable', () => { status: 'sent', content: 'first' }) - tape.deleteBySession('s1') + lifecycle.deleteBySession('s1') expect(tape.getBySession('s1')).toEqual([]) expect(projection.listRange('s1', 0, 10)).toEqual([]) diff --git a/test/main/routes/dispatcher.test.ts b/test/main/routes/dispatcher.test.ts index a788bcc968..5c79a6e5e2 100644 --- a/test/main/routes/dispatcher.test.ts +++ b/test/main/routes/dispatcher.test.ts @@ -14,6 +14,7 @@ import type { IYoBrowserPresenter } from '@shared/types/desktop' import type { MainDatabase } from '@/data/mainDatabase' +import type { TapeInspectionReader } from '@/tape/ports/capabilities' import type { OAuthServicePort } from '@shared/types/oauth' import type { DialogServicePort } from '@shared/types/dialog' import type { DeviceServicePort } from '@shared/types/device' @@ -1271,12 +1272,12 @@ function createRuntime() { repairSchema: vi.fn().mockResolvedValue(databaseRepairReport), agentMemoryAuditTable: { listByAgent: vi.fn(() => []) - }, - deepchatTapeEntriesTable: { - getBySession: vi.fn(() => []), - listMemoryViewManifestAnchorsByAgent: vi.fn(() => []) } } as unknown as MainDatabase + const tapeInspection: TapeInspectionReader = { + getEffectiveMessageSourceSpan: vi.fn(() => []), + listMemoryViewManifestsByAgent: vi.fn(() => []) + } const cronJob = { id: 'cron-1', name: 'Cron smoke', @@ -1515,7 +1516,7 @@ function createRuntime() { const memoryRoutes = createMemoryRoutes({ memoryService, getAgentType: (agentId) => providerSettings.getAgentType(agentId), - getTapeEntries: () => (sqlitePresenter as any).deepchatTapeEntriesTable, + getTapeInspection: () => tapeInspection, getAuditEntries: () => (sqlitePresenter as any).agentMemoryAuditTable }) const desktopRoutes = createDesktopRoutes({ @@ -1754,6 +1755,7 @@ function createRuntime() { remoteService, shortcutPresenter, sqlitePresenter, + tapeInspection, windowPresenter, deviceService, appDataReset, @@ -2380,61 +2382,29 @@ describe('dispatchDeepchatRoute', () => { }) it('filters memory view manifests by message before applying the requested limit', async () => { - const { runtime, providerSettings } = createRuntime() + const { runtime, providerSettings, tapeInspection } = createRuntime() vi.mocked(providerSettings.getAgentType).mockResolvedValueOnce('deepchat') const listSessions = vi.fn() - const listMemoryViewManifestAnchorsByAgent = vi.fn().mockReturnValue([ - { - session_id: 's1', - entry_id: 20, - kind: 'anchor', - name: 'memory/view_assembled', - source_type: 'memory', - source_id: 'msg-new', - source_seq: 0, - provenance_key: null, - payload_json: JSON.stringify({ - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 10, - selected: ['new'], - dropped: [], - queryHash: 'newhash' - } - }), - meta_json: JSON.stringify({ messageId: 'msg-new' }), - created_at: 200 - }, - { - session_id: 's1', - entry_id: 10, - kind: 'anchor', - name: 'memory/view_assembled', - source_type: 'memory', - source_id: 'msg-old', - source_seq: 0, - provenance_key: null, - payload_json: JSON.stringify({ - state: { - policyVersion: 1, - tokenBudget: 900, - estimatedTokens: 9, - selected: ['old'], - dropped: ['drop'], - queryHash: 'oldhash' - } - }), - meta_json: JSON.stringify({ messageId: 'msg-old' }), - created_at: 100 - } - ]) + const listMemoryViewManifestsByAgent = vi + .mocked(tapeInspection.listMemoryViewManifestsByAgent) + .mockReturnValue([ + { + sessionId: 's1', + messageId: 'msg-old', + entryId: 10, + policyVersion: 1, + tokenBudget: 900, + estimatedTokens: 9, + selectedCount: 1, + selectedIds: ['old'], + droppedCount: 1, + queryHash: 'oldhash', + createdAt: 100 + } + ]) ;(runtime as any).sqlitePresenter = { newSessionsTable: { list: listSessions - }, - deepchatTapeEntriesTable: { - listMemoryViewManifestAnchorsByAgent } } @@ -2446,7 +2416,7 @@ describe('dispatchDeepchatRoute', () => { ) expect(listSessions).not.toHaveBeenCalled() - expect(listMemoryViewManifestAnchorsByAgent).toHaveBeenCalledWith('a', { + expect(listMemoryViewManifestsByAgent).toHaveBeenCalledWith('a', { sessionId: 's1', limit: 1, messageId: 'msg-old' @@ -2465,46 +2435,24 @@ describe('dispatchDeepchatRoute', () => { }) }) - it('derives selected memory ids from string and object manifest selections', async () => { - const { runtime, providerSettings } = createRuntime() + it('returns Tape inspection manifest DTOs without exposing raw rows', async () => { + const { runtime, providerSettings, tapeInspection } = createRuntime() vi.mocked(providerSettings.getAgentType).mockResolvedValueOnce('deepchat') - const listMemoryViewManifestAnchorsByAgent = vi.fn().mockReturnValue([ - { - session_id: 's1', - entry_id: 30, - kind: 'anchor', - name: 'memory/view_assembled', - source_type: 'memory', - source_id: 'msg-1', - source_seq: 0, - provenance_key: null, - payload_json: JSON.stringify({ - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 10, - selected: [ - 'm-string', - { id: 'm-object' }, - 'm-string', - { id: 'm-object' }, - { nope: 'ignored' }, - 3 - ], - dropped: [], - queryHash: 'hash' - } - }), - meta_json: JSON.stringify({ messageId: 'msg-1' }), - created_at: 300 + vi.mocked(tapeInspection.listMemoryViewManifestsByAgent).mockReturnValue([ + { + sessionId: 's1', + messageId: 'msg-1', + entryId: 30, + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: 10, + selectedCount: 6, + selectedIds: ['m-string', 'm-object'], + droppedCount: 0, + queryHash: 'hash', + createdAt: 300 } ]) - ;(runtime as any).sqlitePresenter = { - deepchatTapeEntriesTable: { - listMemoryViewManifestAnchorsByAgent - } - } - const result = await dispatchDeepchatRoute( runtime, 'memory.listViewManifests', @@ -2520,6 +2468,48 @@ describe('dispatchDeepchatRoute', () => { }) ] }) + expect(result.manifests[0]).not.toHaveProperty('payload_json') + expect(result.manifests[0]).not.toHaveProperty('meta_json') + }) + + it('reads memory source spans through the DTO-only Tape inspection port', async () => { + const { runtime, tapeInspection } = createRuntime() + const getManagementVisibleByIds = vi.fn().mockReturnValue([ + { + id: 'memory-1', + agent_id: 'deepchat', + source_session: 's1', + source_entry_ids: '[2,3]' + } + ]) + const getEffectiveMessageSourceSpan = vi + .mocked(tapeInspection.getEffectiveMessageSourceSpan) + .mockReturnValue([ + { + entryId: 2, + record: { + role: 'user', + orderSeq: 1, + content: JSON.stringify({ text: 'source context' }) + } + } + ]) + ;(runtime as any).memoryService = { getManagementVisibleByIds } + + const result = await dispatchDeepchatRoute( + runtime, + 'memory.getSourceSpan', + { agentId: 'deepchat', memoryId: 'memory-1' }, + { webContentsId: 42, windowId: 7 } + ) + + expect(getEffectiveMessageSourceSpan).toHaveBeenCalledWith('s1', [2, 3]) + expect(result).toEqual({ + span: { + sessionId: 's1', + entries: [{ entryId: 2, role: 'user', content: 'source context', orderSeq: 1 }] + } + }) }) it('dispatches memory.getByIds with deepchat guard and input order projection', async () => { @@ -2702,13 +2692,8 @@ describe('dispatchDeepchatRoute', () => { }) it('returns no memory view manifests for missing or non-DeepChat agents', async () => { - const { runtime, providerSettings } = createRuntime() - const listMemoryViewManifestAnchorsByAgent = vi.fn() - ;(runtime as any).sqlitePresenter = { - deepchatTapeEntriesTable: { - listMemoryViewManifestAnchorsByAgent - } - } + const { runtime, providerSettings, tapeInspection } = createRuntime() + const listMemoryViewManifestsByAgent = vi.mocked(tapeInspection.listMemoryViewManifestsByAgent) vi.mocked(providerSettings.getAgentType) .mockResolvedValueOnce(null) .mockResolvedValueOnce('acp') @@ -2729,7 +2714,7 @@ describe('dispatchDeepchatRoute', () => { { webContentsId: 42, windowId: 7 } ) ).resolves.toEqual({ manifests: [] }) - expect(listMemoryViewManifestAnchorsByAgent).not.toHaveBeenCalled() + expect(listMemoryViewManifestsByAgent).not.toHaveBeenCalled() }) it('dispatches bounded memory pages and returns an opaque keyset cursor', async () => { @@ -2803,41 +2788,31 @@ describe('dispatchDeepchatRoute', () => { }) it('does not expand all sessions when listing memory view manifests', async () => { - const { runtime, providerSettings } = createRuntime() + const { runtime, providerSettings, tapeInspection } = createRuntime() vi.mocked(providerSettings.getAgentType).mockResolvedValueOnce('deepchat') const listSessions = vi.fn(() => Array.from({ length: 1200 }, (_, index) => ({ id: `s-${index}` })) ) - const listMemoryViewManifestAnchorsByAgent = vi.fn().mockReturnValue([ - { - session_id: 's-1199', - entry_id: 1, - kind: 'anchor', - name: 'memory/view_assembled', - source_type: 'memory', - source_id: 'msg-1', - source_seq: 0, - provenance_key: null, - payload_json: JSON.stringify({ - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 10, - selected: ['m1'], - dropped: [], - queryHash: 'hash' - } - }), - meta_json: JSON.stringify({ messageId: 'msg-1' }), - created_at: 100 - } - ]) + const listMemoryViewManifestsByAgent = vi + .mocked(tapeInspection.listMemoryViewManifestsByAgent) + .mockReturnValue([ + { + sessionId: 's-1199', + messageId: 'msg-1', + entryId: 1, + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: 10, + selectedCount: 1, + selectedIds: ['m1'], + droppedCount: 0, + queryHash: 'hash', + createdAt: 100 + } + ]) ;(runtime as any).sqlitePresenter = { newSessionsTable: { list: listSessions - }, - deepchatTapeEntriesTable: { - listMemoryViewManifestAnchorsByAgent } } @@ -2849,7 +2824,7 @@ describe('dispatchDeepchatRoute', () => { ) expect(listSessions).not.toHaveBeenCalled() - expect(listMemoryViewManifestAnchorsByAgent).toHaveBeenCalledWith('a', { + expect(listMemoryViewManifestsByAgent).toHaveBeenCalledWith('a', { sessionId: undefined, limit: 100, messageId: undefined diff --git a/test/main/scripts/memoryTestScope.test.ts b/test/main/scripts/memoryTestScope.test.ts index d7c6d1604b..4c5117c4e0 100644 --- a/test/main/scripts/memoryTestScope.test.ts +++ b/test/main/scripts/memoryTestScope.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it, vi } from 'vitest' -import { validateMemoryTestScope } from '../../../scripts/check-memory-test-scope.mjs' +import { + findRequiredNativeTapeTests, + validateMemoryTestScope +} from '../../../scripts/check-memory-test-scope.mjs' const rootDir = process.cwd() const baseManifest = { @@ -120,4 +123,66 @@ describe('memory test scope guard', () => { ).toEqual([]) expect(readFile).not.toHaveBeenCalled() }) + + it('requires discovered native Tape tests to stay in the native scope', () => { + const requiredPath = 'tape-native.test.ts' + const paths = new Set([...existingPaths, requiredPath]) + const contents = new Map([...fileContents, [requiredPath, 'export {}']]) + + expect( + validateMemoryTestScope({ + rootDir, + manifest: baseManifest, + existingPaths: paths, + discoveredPaths: [], + requiredNativePaths: [requiredPath], + fileContents: contents + }) + ).toContainEqual(expect.stringContaining('not classified as native')) + + const manifest = structuredClone(baseManifest) + manifest.native.push(requiredPath) + expect( + validateMemoryTestScope({ + rootDir, + manifest, + existingPaths: paths, + discoveredPaths: [], + requiredNativePaths: [requiredPath], + fileContents: contents + }) + ).toEqual([]) + }) + + it('discovers only split Tape suites with native SQLite gates', () => { + const paths = [ + 'test/main/session/data/tapeRecall.test.ts', + 'test/main/session/data/tapeReplay.spec.ts', + 'test/main/session/data/tapeReconciler.test.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.spec.ts', + 'test/main/memory/agentMemoryTable.test.ts' + ] + const contents = new Map([ + ['test/main/session/data/tapeRecall.test.ts', 'itIfSqlite(\'uses SQLite\', () => {})'], + ['test/main/session/data/tapeReplay.spec.ts', 'describeIfSqlite(\'uses SQLite\', () => {})'], + ['test/main/session/data/tapeReconciler.test.ts', 'it(\'stays portable\', () => {})'], + [ + 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts', + 'describeIfSqlite(\'uses SQLite\', () => {})' + ], + [ + 'test/main/session/data/tables/deepchatTapeEntriesTable.spec.ts', + 'itIfSqlite(\'uses SQLite\', () => {})' + ], + ['test/main/memory/agentMemoryTable.test.ts', 'itIfSqlite(\'outside Tape\', () => {})'] + ]) + + expect(findRequiredNativeTapeTests(paths, (path) => contents.get(path) ?? '')).toEqual([ + 'test/main/session/data/tapeRecall.test.ts', + 'test/main/session/data/tapeReplay.spec.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.test.ts', + 'test/main/session/data/tables/deepchatTapeEntriesTable.spec.ts' + ]) + }) }) diff --git a/test/main/session/data/settings.test.ts b/test/main/session/data/settings.test.ts index 357e85514e..76f9394c57 100644 --- a/test/main/session/data/settings.test.ts +++ b/test/main/session/data/settings.test.ts @@ -7,12 +7,22 @@ const sqlitePresenterModule = sqliteModule const sessionStoreModule = sqliteModule ? await import('../../../../src/main/session/data/settings') : null +const sessionDatabaseModule = sqliteModule + ? await import('../../../../src/main/session/data/database') + : null +const sessionTapeModule = sqliteModule + ? await import('../../../../src/main/tape/application/sessionTape') + : null const Database = sqliteModule?.default const MainDatabase = sqlitePresenterModule?.MainDatabase const SessionSettingsStore = sessionStoreModule?.SessionSettingsStore +const SessionDatabase = sessionDatabaseModule?.SessionDatabase +const SessionTape = sessionTapeModule?.SessionTape const MainDatabaseCtor = MainDatabase! const SessionSettingsStoreCtor = SessionSettingsStore! +const SessionDatabaseCtor = SessionDatabase! +const SessionTapeCtor = SessionTape! let sqliteAvailable = false if (Database) { @@ -29,18 +39,20 @@ const describeIfSqlite = sqliteAvailable ? describe : describe.skip describeIfSqlite('SessionSettingsStore tape summary state', () => { function createStore() { - const sqlitePresenter = new MainDatabaseCtor(':memory:') - const store = new SessionSettingsStoreCtor(sqlitePresenter) - return { sqlitePresenter, store } + const connection = new MainDatabaseCtor(':memory:') + const database = new SessionDatabaseCtor(connection) + const tape = new SessionTapeCtor(database) + const store = new SessionSettingsStoreCtor(database, tape) + return { connection, database, store } } it('creates a bootstrap anchor for each session', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') - store.create('s2', 'openai', 'gpt-4o-mini') + store.create('s1', 'openai', 'gpt-4o', 'full_access') + store.create('s2', 'openai', 'gpt-4o-mini', 'full_access') - expect(sqlitePresenter.deepchatTapeEntriesTable.getBySession('s1')).toMatchObject([ + expect(database.deepchatTapeEntriesTable.getBySession('s1')).toMatchObject([ { session_id: 's1', entry_id: 1, @@ -48,7 +60,7 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { name: 'session/start' } ]) - expect(sqlitePresenter.deepchatTapeEntriesTable.getBySession('s2')).toMatchObject([ + expect(database.deepchatTapeEntriesTable.getBySession('s2')).toMatchObject([ { session_id: 's2', entry_id: 1, @@ -57,13 +69,13 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { } ]) - sqlitePresenter.close() + connection.close() }) it('prefers compaction summary anchors over legacy summary columns', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', { summaryText: 'legacy summary', summaryCursorOrderSeq: 2, @@ -101,24 +113,24 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { } }) expect(store.getSummaryState('s1')).toEqual(result.currentState) - expect(sqlitePresenter.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toMatchObject({ + expect(database.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toMatchObject({ name: 'compaction/manual', created_at: 100 }) - sqlitePresenter.close() + connection.close() }) it('uses handoff anchors as context reconstruction state', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', { summaryText: 'legacy summary', summaryCursorOrderSeq: 2, summaryUpdatedAt: 50 }) - sqlitePresenter.deepchatTapeEntriesTable.appendAnchor({ + database.deepchatTapeEntriesTable.appendAnchor({ sessionId: 's1', name: 'handoff/manual', state: { @@ -134,14 +146,14 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryUpdatedAt: 120 }) - sqlitePresenter.close() + connection.close() }) it('uses handoff cursor even when handoff state has no summary', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') - sqlitePresenter.deepchatTapeEntriesTable.appendAnchor({ + store.create('s1', 'openai', 'gpt-4o', 'full_access') + database.deepchatTapeEntriesTable.appendAnchor({ sessionId: 's1', name: 'handoff/manual', state: { @@ -157,19 +169,19 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryUpdatedAt: null }) - sqlitePresenter.close() + connection.close() }) it('compares summary state against tape reconstruction anchors before writing compaction anchors', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', { summaryText: 'legacy summary', summaryCursorOrderSeq: 2, summaryUpdatedAt: 50 }) - sqlitePresenter.deepchatTapeEntriesTable.appendAnchor({ + database.deepchatTapeEntriesTable.appendAnchor({ sessionId: 's1', name: 'handoff/manual', state: { @@ -208,21 +220,19 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryUpdatedAt: 200 } }) - expect( - sqlitePresenter.deepchatTapeEntriesTable.getLatestReconstructionAnchor('s1') - ).toMatchObject({ + expect(database.deepchatTapeEntriesTable.getLatestReconstructionAnchor('s1')).toMatchObject({ name: 'compaction/auto', created_at: 200 }) - sqlitePresenter.close() + connection.close() }) it('does not apply no-anchor summary updates over tape-backed state', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') - sqlitePresenter.deepchatTapeEntriesTable.appendAnchor({ + store.create('s1', 'openai', 'gpt-4o', 'full_access') + database.deepchatTapeEntriesTable.appendAnchor({ sessionId: 's1', name: 'handoff/manual', state: { @@ -256,13 +266,13 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { }) expect(store.getSummaryState('s1')).toEqual(result.currentState) - sqlitePresenter.close() + connection.close() }) it('does not write a stale anchor when summary compare-and-set fails', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.updateSummaryState('s1', { summaryText: 'newer summary', summaryCursorOrderSeq: 5, @@ -298,15 +308,59 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryUpdatedAt: 200 } }) - expect(sqlitePresenter.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toBeUndefined() + expect(database.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toBeUndefined() + + connection.close() + }) + + it('rolls back summary state when the matching tape anchor cannot be appended', () => { + const { connection, database, store } = createStore() + const originalState = { + summaryText: 'stable summary', + summaryCursorOrderSeq: 3, + summaryUpdatedAt: 50 + } + + store.create('s1', 'openai', 'gpt-4o', 'full_access') + store.updateSummaryState('s1', originalState) + database.getDatabase().exec(` + CREATE TRIGGER fail_compaction_anchor + BEFORE INSERT ON deepchat_tape_entries + WHEN NEW.name = 'compaction/failing' + BEGIN + SELECT RAISE(ABORT, 'forced tape anchor failure'); + END; + `) + + expect(() => + store.compareAndSetSummaryState( + 's1', + originalState, + { + summaryText: 'must roll back', + summaryCursorOrderSeq: 6, + summaryUpdatedAt: 100 + }, + { + name: 'compaction/failing', + state: { + summary: 'must roll back', + cursorOrderSeq: 6 + } + } + ) + ).toThrow('forced tape anchor failure') + + expect(store.getSummaryState('s1')).toEqual(originalState) + expect(database.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toBeUndefined() - sqlitePresenter.close() + connection.close() }) it('uses reset anchors to invalidate older compaction anchors', () => { - const { sqlitePresenter, store } = createStore() + const { connection, database, store } = createStore() - store.create('s1', 'openai', 'gpt-4o') + store.create('s1', 'openai', 'gpt-4o', 'full_access') store.compareAndSetSummaryState( 's1', { @@ -335,10 +389,10 @@ describeIfSqlite('SessionSettingsStore tape summary state', () => { summaryCursorOrderSeq: 1, summaryUpdatedAt: null }) - expect(sqlitePresenter.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toMatchObject({ + expect(database.deepchatTapeEntriesTable.getLatestSummaryAnchor('s1')).toMatchObject({ name: 'summary/reset' }) - sqlitePresenter.close() + connection.close() }) }) diff --git a/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts b/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts index 13e7c9e9d3..aa8efc4a2c 100644 --- a/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts +++ b/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts @@ -1,12 +1,17 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' const sqliteModule = await import('better-sqlite3-multiple-ciphers').catch(() => null) const tableModule = sqliteModule ? await import('@/session/data/tables/deepchatTapeEntries') : null +const lifecycleModule = sqliteModule + ? await import('@/tape/infrastructure/sqlite/tapeLifecycleAdapter') + : null const Database = sqliteModule?.default const DeepChatTapeEntriesTable = tableModule?.DeepChatTapeEntriesTable +const SqliteTapeLifecycleAdapter = lifecycleModule?.SqliteTapeLifecycleAdapter const DatabaseCtor = Database! const DeepChatTapeEntriesTableCtor = DeepChatTapeEntriesTable! +const SqliteTapeLifecycleAdapterCtor = SqliteTapeLifecycleAdapter! let sqliteAvailable = false if (Database) { @@ -26,7 +31,8 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { const db = new DatabaseCtor(':memory:') const table = new DeepChatTapeEntriesTableCtor(db) table.createTable() - return { db, table } + const lifecycle = new SqliteTapeLifecycleAdapterCtor(db) + return { db, table, lifecycle } } it('keeps memory/persona anchors out of context reconstruction (C7, AC-7.3)', () => { @@ -86,7 +92,7 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { }) it('assigns a new Tape incarnation when a session Tape is rebuilt', () => { - const { db, table } = createTable() + const { db, table, lifecycle } = createTable() table.ensureBootstrapAnchor('s1') table.ensureBootstrapAnchor('s2') @@ -101,7 +107,7 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { expect(secondIncarnation).toEqual(expect.any(String)) expect(secondIncarnation).not.toBe(firstIncarnation) - table.deleteBySession('s1') + lifecycle.deleteBySession('s1') table.ensureBootstrapAnchor('s1') const rebuiltIncarnation = JSON.parse( table.getFirstEntriesBySessions(['s1'])[0].meta_json @@ -112,6 +118,43 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { db.close() }) + it('deletes Tape and its mutation projection through the lifecycle adapter', () => { + const db = new DatabaseCtor(':memory:') + const projection = { + applyAppendedEntry: vi.fn(() => true), + invalidateSession: vi.fn(), + deleteBySession: vi.fn() + } + const table = new DeepChatTapeEntriesTableCtor(db, projection) + table.createTable() + const lifecycle = new SqliteTapeLifecycleAdapterCtor(db, projection) + table.ensureBootstrapAnchor('s1') + + lifecycle.deleteBySession('s1') + + expect(table.getBySession('s1')).toEqual([]) + expect(projection.deleteBySession).toHaveBeenCalledWith('s1') + db.close() + }) + + it('rolls back Tape deletion when mutation projection cleanup fails', () => { + const db = new DatabaseCtor(':memory:') + const table = new DeepChatTapeEntriesTableCtor(db) + table.createTable() + table.ensureBootstrapAnchor('s1') + const lifecycle = new SqliteTapeLifecycleAdapterCtor(db, { + applyAppendedEntry: vi.fn(() => true), + invalidateSession: vi.fn(), + deleteBySession: vi.fn(() => { + throw new Error('projection cleanup failed') + }) + }) + + expect(() => lifecycle.deleteBySession('s1')).toThrow('projection cleanup failed') + expect(table.getBySession('s1')).toHaveLength(1) + db.close() + }) + it('tracks the latest summary-related anchor only within the requested session', () => { const { db, table } = createTable() diff --git a/test/main/session/data/tape.test.ts b/test/main/session/data/tape.test.ts deleted file mode 100644 index cb5521f7cb..0000000000 --- a/test/main/session/data/tape.test.ts +++ /dev/null @@ -1,5531 +0,0 @@ -import { performance } from 'node:perf_hooks' -import { describe, expect, it, vi } from 'vitest' -import { buildContext } from '@/agent/deepchat/runtime/contextBuilder' -import { toAppSessionId } from '@/agent/shared/agentSessionIds' -import { SessionTape } from '@/session/data/tape' -import { buildEffectiveTapeView, searchEffectiveTapeRows } from '@/session/data/tapeEffectiveView' -import { - createTapeViewManifest, - type TapeViewManifestBuildInput -} from '@/session/data/tapeViewManifest' -import { - appendMessageRecordToTape, - appendMessageReplacementToTape, - appendMessageRetractionToTape, - appendToolFactsToTape -} from '@/session/data/tapeFacts' -import { buildRequestRefs } from '@/session/data/tapeViewManifest' -import { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' -import { - DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, - DeepChatTapeSearchProjectionTable -} from '@/session/data/tables/deepchatTapeSearchProjection' -import { DeepChatMemoryIngestionProjectionTable } from '@/memory/data/tables/deepchatMemoryIngestionProjection' -import { DeepChatMessagesTable } from '@/session/data/tables/deepchatMessages' -import { DeepChatMessageTracesTable } from '@/session/data/tables/deepchatMessageTraces' -import { DeepChatSessionsTable } from '@/session/data/tables/deepchatSessions' -import { NewSessionsTable } from '@/session/data/tables/newSessions' -import type { ChatMessageRecord } from '@shared/types/agent-interface' - -const sqliteModule = await import('better-sqlite3-multiple-ciphers').catch(() => null) -const Database = sqliteModule?.default -const DatabaseCtor = Database! -const sqliteSkipReason = 'skipped: better-sqlite3-multiple-ciphers is unavailable' -const requireNativeSqlite = process.env.DEEPCHAT_REQUIRE_NATIVE_SQLITE === '1' - -let sqliteAvailable = false -if (Database) { - try { - const smokeDb = new Database(':memory:') - smokeDb.close() - sqliteAvailable = true - } catch { - sqliteAvailable = false - } -} - -const itIfSqlite = sqliteAvailable - ? it - : requireNativeSqlite - ? (name: string, _test: () => unknown, timeout?: number) => - it( - name, - () => { - throw new Error(sqliteSkipReason) - }, - timeout - ) - : it.skip - -function createTapeTableMock() { - const entries: any[] = [] - let tapeIncarnationSequence = 0 - const table = { - ensureBootstrapAnchor: vi.fn((sessionId: string) => { - if ( - entries.some((entry) => entry.session_id === sessionId && entry.name === 'session/start') - ) { - return - } - table.appendAnchor({ - sessionId, - name: 'session/start', - source: { type: 'session', id: sessionId, seq: 0 }, - state: { owner: 'human' }, - meta: { tapeIncarnationId: `test-tape-${++tapeIncarnationSequence}` }, - idempotent: true - }) - }), - append: vi.fn((input: any) => { - const provenanceKey = - input.provenanceKey !== undefined - ? input.provenanceKey - : input.source - ? [ - input.source.type, - input.source.id, - input.source.seq ?? 0, - input.kind, - input.name ?? '' - ].join(':') - : null - const existing = input.idempotent - ? entries.find( - (entry) => - entry.session_id === input.sessionId && entry.provenance_key === provenanceKey - ) - : null - if (existing) { - return existing - } - const row = { - session_id: input.sessionId, - entry_id: - Math.max( - 0, - ...entries - .filter((entry) => entry.session_id === input.sessionId) - .map((entry) => entry.entry_id) - ) + 1, - kind: input.kind, - name: input.name ?? null, - source_type: input.source?.type ?? null, - source_id: input.source?.id ?? null, - source_seq: input.source?.seq ?? null, - provenance_key: provenanceKey, - payload_json: JSON.stringify(input.payload ?? {}), - meta_json: JSON.stringify(input.meta ?? {}), - created_at: input.createdAt ?? Date.now() - } - entries.push(row) - return row - }), - appendAnchor: vi.fn((input: any) => - table.append({ - ...input, - kind: 'anchor', - payload: { name: input.name, state: input.state } - }) - ), - appendEvent: vi.fn((input: any) => - table.append({ - ...input, - kind: 'event', - payload: { name: input.name, data: input.data } - }) - ), - runInTransaction: vi.fn((operation: () => unknown) => { - const snapshot = entries.map((entry) => ({ ...entry })) - try { - return operation() - } catch (error) { - entries.splice(0, entries.length, ...snapshot) - throw error - } - }), - getBySession: vi.fn((sessionId: string) => - entries.filter((entry) => entry.session_id === sessionId) - ), - getSubagentLineageEvents: vi.fn((sessionId: string) => - entries.filter( - (entry) => - entry.session_id === sessionId && - entry.kind === 'event' && - (entry.name === 'subagent/tape_linked' || entry.name === 'fork/merge') - ) - ), - getFirstEntriesBySessions: vi.fn((sessionIds: string[]) => - [...new Set(sessionIds)] - .flatMap((sessionId) => { - const first = entries - .filter((entry) => entry.session_id === sessionId) - .sort((left, right) => left.entry_id - right.entry_id)[0] - return first ? [first] : [] - }) - .sort((left, right) => left.session_id.localeCompare(right.session_id)) - ), - getBySessionUpToEntryId: vi.fn((sessionId: string, maxEntryId: number) => - entries.filter((entry) => entry.session_id === sessionId && entry.entry_id <= maxEntryId) - ), - getMaxEntryId: vi.fn((sessionId: string) => - Math.max( - 0, - ...entries.filter((entry) => entry.session_id === sessionId).map((entry) => entry.entry_id) - ) - ), - getMaxEntryIdsBySessions: vi.fn( - (sessionIds: string[]) => - new Map( - sessionIds.map((sessionId) => [ - sessionId, - Math.max( - 0, - ...entries - .filter((entry) => entry.session_id === sessionId) - .map((entry) => entry.entry_id) - ) - ]) - ) - ), - getLatestAnchor: vi.fn( - (sessionId: string) => - entries - .filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor') - .sort((left, right) => right.entry_id - left.entry_id)[0] - ), - getAnchors: vi.fn((sessionId: string, limit: number = 20) => - entries - .filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor') - .sort((left, right) => right.entry_id - left.entry_id) - .slice(0, Math.min(Math.max(Math.floor(limit), 1), 100)) - .reverse() - ), - getLatestSummaryAnchor: vi.fn( - (sessionId: string) => - entries - .filter( - (entry) => - entry.session_id === sessionId && - entry.kind === 'anchor' && - ['compaction/migrated_summary', 'compaction/manual', 'summary/reset'].includes( - entry.name - ) - ) - .sort((left, right) => right.entry_id - left.entry_id)[0] - ), - getByProvenanceKey: vi.fn((sessionId: string, provenanceKey: string) => - entries.find( - (entry) => entry.session_id === sessionId && entry.provenance_key === provenanceKey - ) - ), - countBySession: vi.fn( - (sessionId: string) => entries.filter((entry) => entry.session_id === sessionId).length - ), - countAnchorsBySession: vi.fn( - (sessionId: string) => - entries.filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor').length - ), - countEntriesAfter: vi.fn( - (sessionId: string, entryId: number) => - entries.filter((entry) => entry.session_id === sessionId && entry.entry_id > entryId).length - ), - search: vi.fn((sessionId: string, query: string, options: any = {}) => { - const normalizedQuery = query.trim() - if (!normalizedQuery) { - return [] - } - const limit = Number.isFinite(options.limit) ? Math.floor(options.limit) : 20 - return entries - .filter((entry) => entry.session_id === sessionId) - .filter( - (entry) => - entry.payload_json.includes(normalizedQuery) || - entry.meta_json.includes(normalizedQuery) || - entry.name?.includes(normalizedQuery) - ) - .filter((entry) => !options.kinds?.length || options.kinds.includes(entry.kind)) - .filter( - (entry) => - !Number.isFinite(options.startCreatedAt) || entry.created_at >= options.startCreatedAt - ) - .filter( - (entry) => - !Number.isFinite(options.endCreatedAt) || entry.created_at <= options.endCreatedAt - ) - .sort((left, right) => right.entry_id - left.entry_id) - .slice(0, Math.min(Math.max(limit, 1), 100)) - }), - searchEffectiveSourcesAtHeads: vi.fn((sources: any[], query: string, options: any = {}) => - sources - .flatMap((source) => - searchEffectiveTapeRows( - entries.filter( - (entry) => - entry.session_id === source.sessionId && entry.entry_id <= source.maxEntryId - ), - query, - { ...options, limit: 100 } - ) - ) - .sort( - (left, right) => - right.created_at - left.created_at || - left.session_id.localeCompare(right.session_id) || - right.entry_id - left.entry_id - ) - .slice(0, Math.min(Math.max(Math.floor(options.limit ?? 20), 1), 100)) - ), - getEffectiveContextRowsAtHead: vi.fn( - ( - source: any, - entryIds: number[], - options: { before: number; after: number; limit: number } - ) => { - const effectiveRows = buildEffectiveTapeView( - entries.filter( - (entry) => entry.session_id === source.sessionId && entry.entry_id <= source.maxEntryId - ), - { includePending: false } - ).rows - const indexesByEntryId = new Map( - effectiveRows.map((entry, index) => [entry.entry_id, index]) - ) - const indexes: number[] = [] - for (const entryId of entryIds) { - const index = indexesByEntryId.get(entryId) - if (index !== undefined) indexes.push(index) - } - for (const entryId of entryIds) { - const index = indexesByEntryId.get(entryId) - if (index === undefined) continue - for ( - let cursor = Math.max(0, index - options.before); - cursor <= Math.min(effectiveRows.length - 1, index + options.after); - cursor += 1 - ) { - if (cursor !== index) indexes.push(cursor) - } - } - return [...new Set(indexes)].slice(0, options.limit).map((index) => effectiveRows[index]) - } - ), - deleteBySession: vi.fn((sessionId: string) => { - for (let index = entries.length - 1; index >= 0; index -= 1) { - if (entries[index].session_id === sessionId) { - entries.splice(index, 1) - } - } - }) - } - return { table, entries } -} - -function createRecord(overrides: Partial): ChatMessageRecord { - return { - id: 'm1', - sessionId: 's1', - orderSeq: 1, - role: 'user', - content: JSON.stringify({ text: 'hello', files: [], links: [], search: false, think: false }), - status: 'sent', - isContextEdge: 0, - metadata: '{}', - traceCount: 0, - createdAt: 100, - updatedAt: 100, - ...overrides - } -} - -function createTraceRow(overrides: Record = {}) { - return { - id: 'trace-1', - message_id: 'a1', - session_id: 's1', - provider_id: 'openai', - model_id: 'gpt-4o', - request_seq: 1, - endpoint: 'https://api.openai.test/v1/chat/completions', - headers_json: '{"authorization":"[redacted]"}', - body_json: '{"messages":[{"role":"user","content":"hello"}]}', - truncated: 0, - created_at: 300, - ...overrides - } -} - -function createMessageRow(overrides: Record = {}) { - return { - id: 'a1', - session_id: 's1', - order_seq: 2, - role: 'assistant', - content: '[{"type":"content","content":"done","status":"success"}]', - status: 'sent', - is_context_edge: 0, - metadata: '{"totalTokens":10}', - created_at: 200, - updated_at: 300, - ...overrides - } -} - -function createObservationManifest( - overrides: Partial[0]> = {} -) { - return createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user', content: 'hello' }], - tools: [], - latestEntryId: 0, - anchorEntryIds: [], - included: [], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: true, - assembledAt: 200, - ...overrides - }) -} - -function createTapeService( - table: unknown, - traceRows: Array> = [], - messageRows: Array> = [] -) { - return new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: { - deleteBySession: vi.fn(), - getByEntryIds: vi.fn().mockReturnValue([]) - }, - deepchatMessageTracesTable: { - listByMessageId: vi.fn((messageId: string) => - traceRows.filter((row) => row.message_id === messageId) - ) - }, - deepchatMessagesTable: { - get: vi.fn((messageId: string) => messageRows.find((row) => row.id === messageId)) - }, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) -} - -function createLinkedTapeService( - table: unknown, - sessions: Array<{ - id: string - session_kind: 'regular' | 'subagent' - parent_session_id: string | null - }>, - projectionTable?: unknown -) { - const sessionById = new Map(sessions.map((session) => [session.id, session])) - return { - service: new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - newSessionsTable: { - get: vi.fn((sessionId: string) => sessionById.get(sessionId)), - getMany: vi.fn((sessionIds: string[]) => - sessionIds.flatMap((sessionId) => { - const session = sessionById.get(sessionId) - return session ? [session] : [] - }) - ) - }, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any), - sessionById - } -} - -function createSubagentLinkInput(parentSessionId: string, childSessionId: string) { - return { - parentSessionId, - childSessionId, - runId: `run-${childSessionId}`, - taskId: `task-${childSessionId}`, - slotId: 'reviewer', - taskTitle: `Review ${childSessionId}`, - outcome: 'completed' as const, - resultSummary: 'Done' - } -} - -function appendObservationIsolationFacts(table: unknown) { - const original = createRecord({ id: 'u1', orderSeq: 1, createdAt: 100, updatedAt: 100 }) - const edited = createRecord({ - id: 'u1', - orderSeq: 1, - content: JSON.stringify({ - text: 'edited', - files: [], - links: [], - search: false, - think: false - }), - createdAt: 100, - updatedAt: 150 - }) - const retracted = createRecord({ id: 'u2', orderSeq: 2, createdAt: 160, updatedAt: 160 }) - const pending = createRecord({ - id: 'a1', - orderSeq: 3, - role: 'assistant', - status: 'pending', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'pending', - timestamp: 200, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}' } - } - ]), - createdAt: 200, - updatedAt: 200 - }) - const final = createRecord({ - id: 'a1', - orderSeq: 3, - role: 'assistant', - status: 'sent', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 300, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'tape-result-secret' - } - } - ]), - metadata: '{"totalTokens":12}', - createdAt: 200, - updatedAt: 300 - }) - - appendMessageRecordToTape(table as any, original, 'live') - appendMessageReplacementToTape(table as any, edited, 'test_edit') - appendMessageRecordToTape(table as any, retracted, 'live') - appendMessageRetractionToTape(table as any, retracted, 'test_delete') - appendMessageRecordToTape(table as any, pending, 'live') - appendMessageRecordToTape(table as any, final, 'live') - - return { edited, final } -} - -function stripObservationPayloadOptIns(value: T): T { - const copy = structuredClone(value) as any - const stripEntryPayloads = (entries: any[] | undefined) => { - for (const entry of entries ?? []) { - delete entry.payload - delete entry.meta - } - } - - if (copy.request?.state === 'manifest_bound') { - stripEntryPayloads(copy.request.replay.entries) - delete copy.request.replay.hashes.sliceHash - if (copy.request.replay.trace) { - delete copy.request.replay.trace.headersJson - delete copy.request.replay.trace.bodyJson - } - } else if (copy.request?.trace) { - delete copy.request.trace.headersJson - delete copy.request.trace.bodyJson - } - stripEntryPayloads(copy.output?.entries) - return copy -} - -function createSpies(names: string[]) { - return Object.fromEntries(names.map((name) => [name, vi.fn()])) as Record< - string, - ReturnType - > -} - -function trackMemoryPropertyAccess(target: T) { - const memoryPropertyAccess = vi.fn() - return { - memoryPropertyAccess, - presenter: new Proxy(target, { - get(value, property, receiver) { - if (typeof property === 'string' && /memory/i.test(property)) { - memoryPropertyAccess(property) - } - return Reflect.get(value, property, receiver) - } - }) - } -} - -function readObservationMatrix(service: SessionTape) { - return { - defaultObservation: service.readCausalObservationSlice('s1', 'a1'), - repeatedObservation: service.readCausalObservationSlice('s1', 'a1'), - explicitObservation: service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }), - optInObservation: service.readCausalObservationSlice('s1', 'a1', { - includeTapePayloads: true, - includeTracePayload: true - }), - traceOnlyObservation: service.readCausalObservationSlice('s1', 'a-trace') - } -} - -describe('SessionTape', () => { - it('backfills message and tool facts idempotently before returning tape records', () => { - const { table, entries } = createTapeTableMock() - const assistantBlocks = [ - { - type: 'tool_call', - status: 'success', - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } - } - ] - const records = [ - createRecord({ id: 'u1', orderSeq: 1 }), - createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify(assistantBlocks), - createdAt: 120, - updatedAt: 120 - }) - ] - const messageStore = { - getMessages: vi.fn().mockReturnValue(records) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const first = service.ensureSessionTapeReady('s1', messageStore as any) - const second = service.ensureSessionTapeReady('s1', messageStore as any) - - expect(first.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) - expect(second.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) - expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) - expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) - expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) - expect(entries.filter((entry) => entry.name === 'migration/backfill')).toHaveLength(1) - }) - - it('appends live tool facts through the stable recorder port idempotently', async () => { - const { table, entries } = createTapeTableMock() - const service = createTapeService(table) - const input = { - sessionId: toAppSessionId('s1'), - messageId: 'a1', - orderSeq: 2, - blockIndex: 0, - block: { - type: 'tool_call' as const, - status: 'success' as const, - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } - }, - provenance: { source: 'tool_call' as const, sourceId: 'a1:tc1', sequence: 0 } - } - - const first = await service.appendToolFact(input) - const second = await service.appendToolFact(input) - - expect(second).toEqual(first) - expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) - expect(JSON.parse(entries.find((entry) => entry.kind === 'tool_call').meta_json)).toEqual({ - source: 'live', - role: 'assistant', - status: 'success', - reason: 'tool_loop' - }) - }) - - it('reports info, search, and handoff within one session scope', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([ - createRecord({ id: 'u1' }), - createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { type: 'content', content: 'answer', status: 'success', timestamp: 101 } - ]), - metadata: JSON.stringify({ totalTokens: 9 }), - createdAt: 101, - updatedAt: 101 - }) - ]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - service.handoff('s1', 'phase_done', { summary: ' done ' }) - const handoffAnchor = entries.find((entry) => entry.name === 'handoff/phase_done') - - expect(service.info('s1')).toMatchObject({ - sessionId: 's1', - anchors: 2, - lastAnchor: 'handoff/phase_done', - lastTokenUsage: 9, - migrationState: 'ready' - }) - expect(JSON.parse(handoffAnchor.payload_json).state).toMatchObject({ - summary: 'done', - cursorOrderSeq: 3, - range: { - fromOrderSeq: 1, - toOrderSeq: 2 - }, - sourceMessageIds: ['u1', 'a1'] - }) - expect(service.search('s1', 'hello')).toHaveLength(1) - expect( - service.search('s1', 'hello', { kinds: ['message'], start: '1970-01-01T00:00:00.000Z' }) - ).toHaveLength(1) - expect(service.search('s1', 'hello', { kinds: ['anchor'] })).toHaveLength(0) - expect(service.search('s1', 'hello', { end: '99' })).toHaveLength(0) - expect(() => service.search('s1', 'hello', { start: 'not-a-date' })).toThrow( - 'start must be an ISO date/time or millisecond timestamp.' - ) - expect(service.anchors('s1')).toMatchObject([ - { sessionId: 's1', name: 'session/start' }, - { sessionId: 's1', name: 'handoff/phase_done' } - ]) - expect(service.anchors('s1', { limit: 1 })).toMatchObject([ - { sessionId: 's1', name: 'handoff/phase_done' } - ]) - expect(service.search('s2', 'hello')).toHaveLength(0) - }) - - it('projects fallback tape search into compact results and bounded context', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'Run the dev server', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.append({ - sessionId: 's1', - kind: 'tool_result', - name: 'shell', - source: { type: 'tool_result', id: 'm1:tc1', seq: 0 }, - payload: { - messageId: 'm1', - orderSeq: 2, - toolCallId: 'tc1', - command: 'pnpm run dev', - exitStatus: 1, - response: 'Command failed with EADDRINUSE in /tmp/deepchat.log' - }, - meta: { source: 'live', status: 'error' }, - createdAt: 110 - }) - - const hits = service.search('s1', 'pnpm run dev', { limit: 5 }) - expect(hits).toHaveLength(1) - expect(hits[0]).toMatchObject({ - kind: 'tool_result', - summary: expect.stringContaining('EADDRINUSE'), - refs: { - toolCallId: 'tc1', - commands: expect.arrayContaining(['pnpm run dev']), - filePaths: expect.arrayContaining(['/tmp/deepchat.log']), - errorCodes: expect.arrayContaining(['EADDRINUSE']), - exitStatus: 1 - } - }) - expect(hits[0]).not.toHaveProperty('payload') - expect(hits[0]).not.toHaveProperty('meta') - - const context = service.getContext('s1', [hits[0].entryId], { - before: 0, - after: 0, - maxBytesPerEntry: 12, - maxTotalBytes: 12 - }) - expect(context.matchedEntryIds).toEqual([hits[0].entryId]) - expect(context.entries[0]).toMatchObject({ - entryId: hits[0].entryId, - evidence: { truncated: true } - }) - expect(context.entries[0].evidence.bytes).toBeLessThanOrEqual(12) - expect(context.entries[0]).not.toHaveProperty('payload') - expect(context.entries[0]).not.toHaveProperty('meta') - - const exhaustedContext = service.getContext('s1', [hits[0].entryId], { - before: 0, - after: 0, - maxTotalBytes: 0 - }) - expect(exhaustedContext.entries).toEqual([]) - expect(exhaustedContext.matchedEntryIds).toEqual([]) - }) - - it('projects user message attachment metadata into search text and refs', () => { - const { table } = createTapeTableMock() - const projectionTable = { - isCurrent: vi.fn().mockReturnValue(false), - getSessionMeta: vi.fn().mockReturnValue(null), - getProjectedEntryIds: vi.fn().mockReturnValue([]), - appendSession: vi.fn(), - replaceSession: vi.fn(), - search: vi.fn().mockReturnValue([]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm-file', seq: 0 }, - payload: { - record: createRecord({ - id: 'm-file', - content: JSON.stringify({ - text: 'Please review the attachment', - files: [ - { - name: 'a.md', - path: '/tmp/a.md', - content: 'raw attachment body should not be projected', - metadata: { fileName: 'workspace-a.md' } - } - ], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - - service.search('s1', '/tmp/a.md', { limit: 5 }) - - expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) - const projectedRows = projectionTable.replaceSession.mock.calls[0][1] - expect(projectedRows[0]).toMatchObject({ - entryId: 1, - refs: { - filePaths: expect.arrayContaining(['/tmp/a.md']), - fileNames: expect.arrayContaining(['a.md', 'workspace-a.md']) - } - }) - expect(projectedRows[0].searchText).toContain('/tmp/a.md') - expect(projectedRows[0].searchText).toContain('a.md') - expect(projectedRows[0].searchText).toContain('workspace-a.md') - expect(projectedRows[0].searchText).not.toContain('raw attachment body should not be projected') - }) - - it('preserves relative file paths in projected refs', () => { - const { table } = createTapeTableMock() - let projectedRows: any[] = [] - const projectionTable = { - isCurrent: vi.fn().mockReturnValue(false), - getSessionMeta: vi.fn().mockReturnValue(null), - getProjectedEntryIds: vi.fn().mockReturnValue([]), - appendSession: vi.fn(), - replaceSession: vi.fn((_sessionId: string, rows: any[]) => { - projectedRows = rows - }), - search: vi.fn().mockReturnValue([]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm-relative', seq: 0 }, - payload: { - record: createRecord({ - id: 'm-relative', - content: JSON.stringify({ - text: 'Touched src/main/index.ts, ./lib/util.ts, ../shared/types.ts, test/main/foo/bar.ts, /usr/local/bin/deploy, and https://example.com/not-a-file', - files: [], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - - service.search('s1', 'src/main/index.ts', { limit: 5 }) - - expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) - const filePaths = projectedRows[0].refs.filePaths - expect(filePaths).toEqual( - expect.arrayContaining([ - 'src/main/index.ts', - './lib/util.ts', - '../shared/types.ts', - 'test/main/foo/bar.ts', - '/usr/local/bin/deploy' - ]) - ) - expect(filePaths).not.toContain('/main/index.ts') - expect(filePaths).not.toContain('/lib/util.ts') - expect(filePaths).not.toContain('/main/foo/bar.ts') - expect(filePaths).not.toContain('example.com/not-a-file') - }) - - it('rebuilds old tape projection versions before attachment path search', () => { - const { table } = createTapeTableMock() - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm-file', seq: 0 }, - payload: { - record: createRecord({ - id: 'm-file', - content: JSON.stringify({ - text: 'Please review the migrated attachment', - files: [ - { - name: 'legacy.md', - path: '/tmp/legacy.md', - content: 'raw migrated body must stay out of projection', - metadata: { fileName: 'legacy-workspace.md' } - } - ], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - let storedVersion = 1 - let storedMaxEntryId = 1 - let projectedRows: any[] = [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm-file', - sourceSeq: 0, - searchText: 'message/user Please review the migrated attachment', - summaryText: 'user: Please review the migrated attachment', - refs: { messageId: 'm-file' }, - createdAt: 100 - } - ] - const projectionTable = { - isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => { - return ( - storedVersion === DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION && - storedMaxEntryId === maxEntryId - ) - }), - getSessionMeta: vi.fn(() => ({ - projectionVersion: storedVersion, - maxEntryId: storedMaxEntryId - })), - getProjectedEntryIds: vi.fn().mockReturnValue([1]), - appendSession: vi.fn(), - replaceSession: vi.fn((_sessionId: string, rows: any[], maxEntryId: number) => { - projectedRows = rows - storedVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION - storedMaxEntryId = maxEntryId - }), - search: vi.fn((_sessionId: string, query: string) => { - return projectedRows - .filter((row) => row.searchText.includes(query)) - .map((row) => ({ - session_id: row.sessionId, - entry_id: row.entryId, - kind: row.kind, - name: row.name, - source_type: row.sourceType, - source_id: row.sourceId, - source_seq: row.sourceSeq, - search_text: row.searchText, - summary_text: row.summaryText, - refs_json: JSON.stringify(row.refs), - created_at: row.createdAt, - score: 1 - })) - }) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const hits = service.search('s1', '/tmp/legacy.md', { limit: 5 }) - - expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) - expect(projectionTable.appendSession).not.toHaveBeenCalled() - expect(hits).toHaveLength(1) - expect(hits[0].refs).toMatchObject({ - filePaths: expect.arrayContaining(['/tmp/legacy.md']), - fileNames: expect.arrayContaining(['legacy.md', 'legacy-workspace.md']) - }) - expect(projectedRows[0].searchText).not.toContain( - 'raw migrated body must stay out of projection' - ) - }) - - it('prioritizes requested tape context entries before window entries under byte caps', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'before-1', seq: 0 }, - payload: { - record: createRecord({ - id: 'before-1', - content: JSON.stringify({ - text: 'before entry consumes the tiny byte budget first', - files: [], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'before-2', seq: 0 }, - payload: { - record: createRecord({ - id: 'before-2', - content: JSON.stringify({ - text: 'second before entry also appears earlier', - files: [], - links: [] - }), - createdAt: 110, - updatedAt: 110 - }) - }, - meta: { source: 'live', orderSeq: 2, role: 'user' }, - createdAt: 110 - }) - const target = table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'target', seq: 0 }, - payload: { - record: createRecord({ - id: 'target', - content: JSON.stringify({ - text: 'target-ok consumes the available budget', - files: [], - links: [] - }), - createdAt: 120, - updatedAt: 120 - }) - }, - meta: { source: 'live', orderSeq: 3, role: 'user' }, - createdAt: 120 - }) - - const context = service.getContext('s1', [target.entry_id], { - before: 2, - after: 0, - limit: 3, - maxBytesPerEntry: 18, - maxTotalBytes: 18 - }) - - expect(context.matchedEntryIds).toEqual([target.entry_id]) - expect(context.entries.map((entry) => entry.entryId)).toEqual([target.entry_id]) - expect(context.entries[0].evidence.text).toContain('target-ok') - }) - - it('binds tape projection LIKE fallback params for single and multi-term queries', () => { - const all = vi.fn().mockReturnValue([]) - const db = { - prepare: vi.fn((sql: string) => ({ - all: (...params: unknown[]) => all(sql, params), - get: vi.fn().mockReturnValue(undefined), - run: vi.fn() - })) - } - const projectionTable = new DeepChatTapeSearchProjectionTable(db as any) - ;(projectionTable as any).recoverSessionFts = vi.fn() - - projectionTable.search('s1', 'Redis', { limit: 5 }) - expect(all).toHaveBeenLastCalledWith( - expect.stringContaining('FROM deepchat_tape_search_projection'), - ['s1', '%Redis%', '%Redis%', '%Redis%', 5] - ) - - projectionTable.search('s1', 'Redis TTL', { limit: 5 }) - expect(all).toHaveBeenLastCalledWith( - expect.stringContaining('FROM deepchat_tape_search_projection'), - [ - 's1', - '%Redis TTL%', - '%Redis TTL%', - '%Redis TTL%', - '%Redis%', - '%Redis%', - '%Redis%', - '%TTL%', - '%TTL%', - '%TTL%', - 5 - ] - ) - - const oversizedQuery = Array.from({ length: 257 }, (_, index) => `term-${index}`).join(' ') - projectionTable.search('s1', oversizedQuery, { limit: 5 }) - expect(all).toHaveBeenLastCalledWith( - expect.stringContaining('FROM deepchat_tape_search_projection'), - ['s1', `%${oversizedQuery}%`, `%${oversizedQuery}%`, `%${oversizedQuery}%`, 5] - ) - }) - - it('uses current tape projection without loading full session rows', () => { - const { table } = createTapeTableMock() - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'Redis compact marker', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.getBySession.mockClear() - const projectionTable = { - isCurrent: vi.fn().mockReturnValue(true), - getSessionMeta: vi.fn(), - getProjectedEntryIds: vi.fn(), - appendSession: vi.fn(), - replaceSession: vi.fn(), - search: vi.fn().mockReturnValue([ - { - session_id: 's1', - entry_id: 1, - kind: 'message', - name: 'message/user', - source_type: 'message', - source_id: 'm1', - source_seq: 0, - search_text: 'Redis compact marker', - summary_text: 'Redis compact marker', - refs_json: '{"messageId":"m1"}', - created_at: 100, - score: -2 - } - ]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const hits = service.search('s1', 'Redis compact', { limit: 5 }) - - expect(table.getMaxEntryId).toHaveBeenCalledWith('s1') - expect(table.getBySession).not.toHaveBeenCalled() - expect(projectionTable.search).toHaveBeenCalledWith( - 's1', - 'Redis compact', - expect.objectContaining({ limit: 5 }) - ) - expect(projectionTable.appendSession).not.toHaveBeenCalled() - expect(projectionTable.replaceSession).not.toHaveBeenCalled() - expect(hits[0]).toMatchObject({ - entryId: 1, - kind: 'message', - summary: 'Redis compact marker', - refs: { messageId: 'm1' }, - score: -2 - }) - expect(hits[0]).not.toHaveProperty('payload') - expect(hits[0]).not.toHaveProperty('meta') - }) - - it('falls back to effective tape search when projection search throws', () => { - const { table } = createTapeTableMock() - const projectionTable = { - isCurrent: vi.fn().mockReturnValue(true), - search: vi.fn(() => { - throw new Error('projection failed') - }) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'Redis fallback marker', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - - const hits = service.search('s1', 'Redis fallback', { limit: 5 }) - expect(hits).toHaveLength(1) - expect(hits[0]).toMatchObject({ - kind: 'message', - summary: expect.stringContaining('Redis fallback') - }) - expect(hits[0]).not.toHaveProperty('payload') - expect(hits[0]).not.toHaveProperty('meta') - }) - - it('appends tape projection rows when the previous projection is an effective prefix', () => { - const { table } = createTapeTableMock() - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'first redis', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm2', seq: 0 }, - payload: { - record: createRecord({ - id: 'm2', - content: JSON.stringify({ text: 'second vue', files: [], links: [] }), - createdAt: 110, - updatedAt: 110 - }) - }, - meta: { source: 'live', orderSeq: 2, role: 'user' }, - createdAt: 110 - }) - const projectionTable = { - isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => maxEntryId === 1), - getSessionMeta: vi.fn().mockReturnValue({ projectionVersion: 1, maxEntryId: 1 }), - getProjectedEntryIds: vi.fn().mockReturnValue([1]), - appendSession: vi.fn(), - replaceSession: vi.fn(), - search: vi.fn().mockReturnValue([]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.search('s1', 'vue', { limit: 5 }) - - expect(projectionTable.appendSession).toHaveBeenCalledTimes(1) - expect(projectionTable.appendSession.mock.calls[0][1].map((row: any) => row.entryId)).toEqual([ - 2 - ]) - expect(projectionTable.replaceSession).not.toHaveBeenCalled() - }) - - it('rebuilds tape projection when projected entry ids are not an effective prefix', () => { - const { table } = createTapeTableMock() - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'm1', seq: 0 }, - payload: { - record: createRecord({ - id: 'm1', - content: JSON.stringify({ text: 'redis', files: [], links: [] }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - const projectionTable = { - isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => maxEntryId === 0), - getSessionMeta: vi.fn().mockReturnValue({ projectionVersion: 1, maxEntryId: 0 }), - getProjectedEntryIds: vi.fn().mockReturnValue([99]), - appendSession: vi.fn(), - replaceSession: vi.fn(), - search: vi.fn().mockReturnValue([]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.search('s1', 'redis', { limit: 5 }) - - expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) - expect(projectionTable.appendSession).not.toHaveBeenCalled() - }) - - it('does not run LIKE fallback when FTS fills the tape projection search limit', () => { - const all = vi.fn((sql: string, _params: unknown[]) => - sql.includes('deepchat_tape_search_fts') - ? [ - { - session_id: 's1', - entry_id: 1, - kind: 'message', - name: 'message/user', - source_type: 'message', - source_id: 'm1', - source_seq: 0, - search_text: 'Redis TTL', - summary_text: 'Redis TTL', - refs_json: '{}', - created_at: 100, - score: -1 - } - ] - : [] - ) - const db = { - exec: vi.fn(() => { - throw new Error('unexpected FTS ensure') - }), - prepare: vi.fn((sql: string) => ({ - all: (...params: unknown[]) => all(sql, params), - get: (..._params: unknown[]) => { - if ( - sql.includes('deepchat_tape_search_projection_meta') || - sql.includes('deepchat_tape_search_fts_meta') - ) { - return { projection_version: 1, max_entry_id: 1 } - } - return undefined - }, - run: vi.fn() - })), - transaction: vi.fn((callback: () => void) => callback) - } - const projectionTable = new DeepChatTapeSearchProjectionTable(db as any) - ;(projectionTable as any).ftsReady = true - - const hits = projectionTable.search('s1', 'Redis', { limit: 1 }) - - expect(hits).toHaveLength(1) - expect(db.exec).not.toHaveBeenCalled() - const ftsCall = all.mock.calls.find(([sql]) => String(sql).includes('deepchat_tape_search_fts')) - expect(String(ftsCall?.[0])).toContain('deepchat_tape_search_fts.session_id = ?') - expect((ftsCall?.[1] as unknown[]).filter((param) => param === 's1')).toHaveLength(2) - expect( - vi.mocked(db.prepare).mock.calls.some(([sql]) => String(sql).includes('NULL AS score')) - ).toBe(false) - }) - - it('queries memory view manifests by agent without expanding session ids', () => { - const all = vi.fn().mockReturnValue([]) - const db = { - prepare: vi.fn((sql: string) => ({ - all: (...params: unknown[]) => all(sql, params) - })) - } - const table = new DeepChatTapeEntriesTable(db as any) - - table.listMemoryViewManifestAnchorsByAgent('agent-a', { - sessionId: 's-1', - messageId: 'msg-1', - limit: 7 - }) - - expect(all).toHaveBeenCalledWith( - expect.stringContaining('INNER JOIN new_sessions AS sessions'), - ['agent-a', 's-1', 'msg-1', 7] - ) - const sql = String(all.mock.calls[0][0]) - expect(sql).not.toContain(' IN (') - expect(sql).toContain('sessions.agent_id = ?') - expect(sql).toContain('tape.session_id = ?') - expect(sql).toContain("json_extract(tape.meta_json, '$.messageId') = ?") - }) - - it('keeps linked raw search candidate-bounded instead of materializing complete Tapes', () => { - const all = vi.fn().mockReturnValue([]) - const db = { - prepare: vi.fn((sql: string) => ({ - all: (...params: unknown[]) => all(sql, params) - })) - } - const table = new DeepChatTapeEntriesTable(db as any) - - table.searchEffectiveSourcesAtHeads( - [ - { sessionId: 'child-b', maxEntryId: 20 }, - { sessionId: 'child-a', maxEntryId: 10 } - ], - 'needle', - { limit: 5 } - ) - - const sql = String(all.mock.calls[0][0]) - const params = all.mock.calls[0][1] - expect(sql).toContain('FROM deepchat_tape_entries AS candidate') - expect(sql).toContain('candidate.entry_id <= source.max_entry_id') - expect(sql).toContain('FROM deepchat_tape_entries AS later_message') - expect(sql).toContain( - "typeof(json_extract(later_message.payload_json, '$.record.content')) = 'text'" - ) - expect(sql).not.toContain('bounded_rows AS') - expect(params[0]).toBe( - JSON.stringify([ - { sessionId: 'child-a', maxEntryId: 10 }, - { sessionId: 'child-b', maxEntryId: 20 } - ]) - ) - expect(params.at(-1)).toBe(5) - }) - - it('reads exact linked projections without invoking FTS recovery or writes', () => { - const run = vi.fn() - const exec = vi.fn() - const reads: Array<{ sql: string; params: unknown[] }> = [] - const prepare = vi.fn((sql: string) => ({ - all: (...params: unknown[]) => { - reads.push({ sql, params }) - if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { - return [{ session_id: 'child', max_entry_id: 2 }] - } - if (sql.includes('SELECT projection.*, NULL AS score')) { - return [ - { - session_id: 'child', - entry_id: 2, - kind: 'event', - name: 'child/result', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'projection needle', - summary_text: 'projection needle', - refs_json: '{}', - created_at: 100, - score: null - } - ] - } - return [] - }, - run - })) - const projectionTable = new DeepChatTapeSearchProjectionTable({ prepare, exec } as any) - - const result = projectionTable.searchSourcesReadOnly( - [{ sessionId: 'child', maxEntryId: 2 }], - 'projection needle', - { limit: 5 } - ) - - expect(result).toMatchObject({ - coveredSources: [{ sessionId: 'child', maxEntryId: 2 }], - rows: [{ session_id: 'child', entry_id: 2 }] - }) - expect(exec).not.toHaveBeenCalled() - expect(run).not.toHaveBeenCalled() - expect(prepare.mock.calls.map(([sql]) => String(sql)).join('\n')).not.toContain( - 'deepchat_tape_search_fts_meta' - ) - expect( - reads.find((read) => read.sql.includes('SELECT projection.*, NULL AS score'))?.params.at(-1) - ).toBe(5) - }) - - it('returns no ranked rows when linked projection coverage is only partial', () => { - const reads: string[] = [] - const prepare = vi.fn((sql: string) => ({ - all: () => { - reads.push(sql) - if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { - return [{ session_id: 'child-a', max_entry_id: 2 }] - } - return [] - } - })) - const projectionTable = new DeepChatTapeSearchProjectionTable({ - prepare, - exec: vi.fn() - } as any) - - const result = projectionTable.searchSourcesReadOnly( - [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 3 } - ], - 'projection needle', - { limit: 5 } - ) - - expect(result).toEqual({ - coveredSources: [{ sessionId: 'child-a', maxEntryId: 2 }], - rows: [] - }) - expect( - reads.some((sql) => sql.includes('FROM deepchat_tape_search_projection AS projection')) - ).toBe(false) - expect(reads.some((sql) => sql.includes('FROM deepchat_tape_search_fts'))).toBe(false) - }) - - it('uses one LIKE ranking when linked FTS freshness is only partial', () => { - const reads: Array<{ sql: string; params: unknown[] }> = [] - const prepare = vi.fn((sql: string) => ({ - all: (...params: unknown[]) => { - reads.push({ sql, params }) - if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { - return [ - { session_id: 'child-a', max_entry_id: 2 }, - { session_id: 'child-b', max_entry_id: 3 } - ] - } - if (sql.includes('deepchat_tape_search_fts_meta AS meta')) { - return [{ session_id: 'child-a', max_entry_id: 2 }] - } - if (sql.includes('SELECT projection.*, NULL AS score')) { - return [ - { - session_id: 'child-b', - entry_id: 3, - kind: 'event', - name: 'child/result', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'projection needle', - summary_text: 'projection needle', - refs_json: '{}', - created_at: 200, - score: null - } - ] - } - return [] - } - })) - const projectionTable = new DeepChatTapeSearchProjectionTable({ - prepare, - exec: vi.fn() - } as any) - ;(projectionTable as any).ftsReady = true - - const sources = [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 3 } - ] - const result = projectionTable.searchSourcesReadOnly(sources, 'projection needle', { limit: 5 }) - - expect(result.rows).toMatchObject([{ session_id: 'child-b', entry_id: 3, score: null }]) - expect(reads.some(({ sql }) => sql.includes('bm25(deepchat_tape_search_fts)'))).toBe(false) - expect( - reads.find(({ sql }) => sql.includes('SELECT projection.*, NULL AS score'))?.params[0] - ).toBe(JSON.stringify(sources)) - }) - - itIfSqlite( - `keeps projected and raw linked multi-term search aligned${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - table.createTable() - projectionTable.createTable() - table.ensureBootstrapAnchor('child') - const entry = table.appendEvent({ - sessionId: 'child', - name: 'child/result', - data: { text: 'alpha separated by several words before beta' }, - createdAt: 100 - }) - const sources = [{ sessionId: 'child', maxEntryId: entry.entry_id }] - projectionTable.replaceSession( - 'child', - [ - { - sessionId: 'child', - entryId: entry.entry_id, - kind: 'event', - name: 'child/result', - sourceType: null, - sourceId: null, - sourceSeq: null, - searchText: 'alpha separated by several words before beta', - summaryText: 'alpha separated by several words before beta', - refs: {}, - createdAt: 100 - } - ], - entry.entry_id - ) - - const projected = projectionTable.searchSourcesReadOnly(sources, 'alpha beta', { limit: 5 }) - const raw = table.searchEffectiveSourcesAtHeads(sources, 'alpha beta', { limit: 5 }) - - expect(projected.coveredSources).toEqual(sources) - expect(raw.map((row) => [row.session_id, row.entry_id])).toEqual( - projected.rows.map((row) => [row.session_id, row.entry_id]) - ) - expect(raw).toHaveLength(1) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `queries effective linked sources and context at frozen heads${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - table.ensureBootstrapAnchor('child-a') - table.ensureBootstrapAnchor('child-b') - table.appendEvent({ - sessionId: 'child-a', - name: 'child/result', - data: { text: 'native linked needle A' }, - createdAt: 100 - }) - table.appendEvent({ - sessionId: 'child-b', - name: 'child/result', - data: { text: 'native linked needle B' }, - createdAt: 200 - }) - table.appendEvent({ - sessionId: 'child-a', - name: 'child/late', - data: { text: 'native linked needle late' }, - createdAt: 300 - }) - - const sources = [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 2 } - ] - expect( - table.searchEffectiveSourcesAtHeads(sources, 'native linked needle', { limit: 1 }) - ).toMatchObject([{ session_id: 'child-b', entry_id: 2 }]) - expect( - table - .searchEffectiveSourcesAtHeads(sources, 'native linked needle', { limit: 10 }) - .map((row) => [row.session_id, row.entry_id]) - ).toEqual([ - ['child-b', 2], - ['child-a', 2] - ]) - expect( - table - .getEffectiveContextRowsAtHead({ sessionId: 'child-a', maxEntryId: 2 }, [2], { - before: 1, - after: 5, - limit: 10 - }) - .map((row) => row.entry_id) - ).toEqual([2, 1]) - - table.ensureBootstrapAnchor('child-message') - const original = createRecord({ - id: 'linked-message', - sessionId: 'child-message', - content: JSON.stringify({ text: 'old linked marker', files: [], links: [] }) - }) - const replacement = { - ...original, - content: JSON.stringify({ text: 'new linked marker', files: [], links: [] }), - updatedAt: 200 - } - appendMessageRecordToTape(table, original, 'live') - appendMessageReplacementToTape(table, replacement, 'native_edit') - const replacementHead = table.getMaxEntryId('child-message') - expect( - table.searchEffectiveSourcesAtHeads( - [{ sessionId: 'child-message', maxEntryId: replacementHead }], - 'old linked marker' - ) - ).toEqual([]) - const replacementHits = table.searchEffectiveSourcesAtHeads( - [{ sessionId: 'child-message', maxEntryId: replacementHead }], - 'new linked marker' - ) - expect(replacementHits).toHaveLength(1) - expect( - table - .getEffectiveContextRowsAtHead( - { sessionId: 'child-message', maxEntryId: replacementHead }, - [replacementHits[0].entry_id], - { before: 0, after: 0, limit: 5 } - ) - .map((row) => row.entry_id) - ).toEqual([replacementHits[0].entry_id]) - - table.append({ - sessionId: 'child-message', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: original.id, seq: 0 }, - provenanceKey: null, - payload: { - record: { - id: original.id, - sessionId: original.sessionId, - orderSeq: original.orderSeq, - role: original.role, - status: 'sent' - } - }, - meta: { source: 'malformed_import' } - }) - expect( - table.searchEffectiveSourcesAtHeads( - [ - { - sessionId: 'child-message', - maxEntryId: table.getMaxEntryId('child-message') - } - ], - 'new linked marker' - ) - ).toMatchObject([{ entry_id: replacementHits[0].entry_id }]) - - appendMessageRetractionToTape(table, replacement, 'native_delete') - expect( - table.searchEffectiveSourcesAtHeads( - [ - { - sessionId: 'child-message', - maxEntryId: table.getMaxEntryId('child-message') - } - ], - 'new linked marker' - ) - ).toEqual([]) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `searches exact frozen projections without repairing a later child tail${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - projectionTable.replaceSession( - 'child', - [ - { - sessionId: 'child', - entryId: 2, - kind: 'event', - name: 'child/result', - sourceType: null, - sourceId: null, - sourceSeq: null, - searchText: 'native frozen projection needle', - summaryText: 'native frozen projection needle', - refs: {}, - createdAt: 100 - } - ], - 2 - ) - const before = db - .prepare( - `SELECT projection_version, max_entry_id, updated_at - FROM deepchat_tape_search_projection_meta - WHERE session_id = ?` - ) - .get('child') - - const result = projectionTable.searchSourcesReadOnly( - [{ sessionId: 'child', maxEntryId: 2 }], - 'native frozen projection needle', - { limit: 5 } - ) - - expect(result.coveredSources).toEqual([{ sessionId: 'child', maxEntryId: 2 }]) - expect(result.rows).toMatchObject([{ session_id: 'child', entry_id: 2 }]) - expect( - projectionTable.searchSourcesReadOnly( - [{ sessionId: 'child', maxEntryId: 3 }], - 'native frozen projection needle', - { limit: 5 } - ) - ).toEqual({ rows: [], coveredSources: [] }) - expect( - db - .prepare( - `SELECT projection_version, max_entry_id, updated_at - FROM deepchat_tape_search_projection_meta - WHERE session_id = ?` - ) - .get('child') - ).toEqual(before) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `filters stale FTS rows through the base projection after restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 2, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'current', - sourceSeq: 0, - searchText: 'current Redis marker', - summaryText: 'current Redis marker', - refs: { messageId: 'current' }, - createdAt: 200 - } - ], - 2 - ) - db.prepare( - `INSERT INTO deepchat_tape_search_fts ( - search_text, - name, - session_id, - entry_id, - kind, - source_type, - source_id, - source_seq, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - 'stale removed marker', - 'message/user', - 's1', - 1, - 'message', - 'message', - 'old', - 0, - 'stale removed marker', - '{"messageId":"old"}', - 100 - ) - - const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - restartedProjectionTable.createTable() - - expect(restartedProjectionTable.isCurrent('s1', 2)).toBe(true) - expect(restartedProjectionTable.search('s1', 'stale removed marker', { limit: 5 })).toEqual( - [] - ) - expect( - restartedProjectionTable.search('s1', 'current Redis marker', { limit: 5 })[0] - ).toMatchObject({ - entry_id: 2, - refs_json: '{"messageId":"current"}' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `recovers same-entry stale FTS after a base-only projection write and restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm1', - sourceSeq: 0, - searchText: 'old durable marker', - summaryText: 'old durable marker', - refs: { messageId: 'm1' }, - createdAt: 100 - } - ], - 1 - ) - - projectionTable.disableFtsForTesting() - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm1', - sourceSeq: 0, - searchText: 'new durable marker', - summaryText: 'new durable marker', - refs: { messageId: 'm1' }, - createdAt: 100 - } - ], - 1 - ) - db.prepare( - `INSERT INTO deepchat_tape_search_fts ( - search_text, - name, - session_id, - entry_id, - kind, - source_type, - source_id, - source_seq, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - 'old durable marker', - 'message/user', - 's1', - 1, - 'message', - 'message', - 'm1', - 0, - 'old durable marker', - '{"messageId":"m1"}', - 100 - ) - - const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - restartedProjectionTable.createTable() - - expect(restartedProjectionTable.isCurrent('s1', 1)).toBe(true) - expect(restartedProjectionTable.search('s1', 'old durable marker', { limit: 5 })).toEqual( - [] - ) - expect( - restartedProjectionTable.search('s1', 'new durable marker', { limit: 5 })[0] - ).toMatchObject({ - entry_id: 1, - search_text: 'new durable marker' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `rebuilds FTS during append when previous FTS meta is missing after restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.disableFtsForTesting() - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'old', - sourceSeq: 0, - searchText: 'old append marker', - summaryText: 'old append marker', - refs: { messageId: 'old' }, - createdAt: 100 - } - ], - 1 - ) - - const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - restartedProjectionTable.createTable() - restartedProjectionTable.appendSession( - 's1', - [ - { - sessionId: 's1', - entryId: 2, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'new', - sourceSeq: 0, - searchText: 'new append marker', - summaryText: 'new append marker', - refs: { messageId: 'new' }, - createdAt: 200 - } - ], - 2 - ) - - expect(restartedProjectionTable.isCurrent('s1', 2)).toBe(true) - expect( - restartedProjectionTable.search('s1', 'old append marker', { limit: 1 })[0] - ).toMatchObject({ - entry_id: 1, - refs_json: '{"messageId":"old"}' - }) - expect( - restartedProjectionTable.search('s1', 'new append marker', { limit: 1 })[0] - ).toMatchObject({ - entry_id: 2, - refs_json: '{"messageId":"new"}' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `rebuilds migrated tape FTS when freshness meta is excluded${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'old', - sourceSeq: 0, - searchText: 'old migrated marker', - summaryText: 'old migrated marker', - refs: { messageId: 'old' }, - createdAt: 100 - } - ], - 1 - ) - db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run('s1') - db.prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?').run('s1') - - const migratedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - migratedProjectionTable.createTable() - migratedProjectionTable.appendSession( - 's1', - [ - { - sessionId: 's1', - entryId: 2, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'new', - sourceSeq: 0, - searchText: 'new migrated marker', - summaryText: 'new migrated marker', - refs: { messageId: 'new' }, - createdAt: 200 - } - ], - 2 - ) - - expect(migratedProjectionTable.isCurrent('s1', 2)).toBe(true) - expect( - migratedProjectionTable.search('s1', 'old migrated marker', { limit: 1 })[0] - ).toMatchObject({ - entry_id: 1, - refs_json: '{"messageId":"old"}' - }) - expect( - migratedProjectionTable.search('s1', 'new migrated marker', { limit: 1 })[0] - ).toMatchObject({ - entry_id: 2, - refs_json: '{"messageId":"new"}' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `keeps common-term FTS searches scoped and bounded on large session sets${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - - for (let index = 0; index < 180; index += 1) { - const sessionId = `s-${index}` - const rows = Array.from({ length: 8 }, (_, offset) => ({ - sessionId, - entryId: offset + 1, - kind: 'message' as const, - name: 'message/user', - sourceType: 'message' as const, - sourceId: `m-${index}-${offset}`, - sourceSeq: offset, - searchText: `sharedcommon marker session-${index} row-${offset}`, - summaryText: `sharedcommon marker session-${index} row-${offset}`, - refs: { messageId: `m-${index}-${offset}` }, - createdAt: index * 10 + offset - })) - projectionTable.replaceSession(sessionId, rows, rows.length) - } - - const planRows = db - .prepare( - `EXPLAIN QUERY PLAN - SELECT projection.session_id, - projection.entry_id, - projection.kind, - projection.name, - projection.source_type, - projection.source_id, - projection.source_seq, - projection.search_text, - projection.summary_text, - projection.refs_json, - projection.created_at, - bm25(deepchat_tape_search_fts) AS score - FROM deepchat_tape_search_fts - INNER JOIN deepchat_tape_search_projection AS projection - ON projection.session_id = deepchat_tape_search_fts.session_id - AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) - AND projection.search_text = deepchat_tape_search_fts.search_text - WHERE deepchat_tape_search_fts MATCH ? - AND deepchat_tape_search_fts.session_id = ? - AND projection.session_id = ? - ORDER BY score ASC, projection.entry_id DESC - LIMIT ?` - ) - .all('"sharedcommon"', 's-42', 's-42', 5) as Array<{ detail: string }> - const plan = planRows.map((row) => row.detail).join('\n') - - expect(plan).toMatch(/VIRTUAL TABLE INDEX/i) - expect(plan).toMatch(/SEARCH projection USING (?:COVERING )?INDEX/i) - expect(plan).not.toMatch(/\bSCAN projection\b/i) - - const startedAt = performance.now() - const hits = projectionTable.search('s-42', 'sharedcommon', { limit: 5 }) - const elapsedMs = performance.now() - startedAt - - expect(hits).toHaveLength(5) - expect(hits.every((hit) => hit.session_id === 's-42')).toBe(true) - expect(elapsedMs).toBeLessThan(1500) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `does not trust same-entry stale FTS text even when stale FTS meta is current${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm1', - sourceSeq: 0, - searchText: 'new guarded marker', - summaryText: 'new guarded marker', - refs: { messageId: 'm1' }, - createdAt: 100 - } - ], - 1 - ) - db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run('s1') - db.prepare( - `INSERT INTO deepchat_tape_search_fts ( - search_text, - name, - session_id, - entry_id, - kind, - source_type, - source_id, - source_seq, - summary_text, - refs_json, - created_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - 'old guarded marker', - 'message/user', - 's1', - 1, - 'message', - 'message', - 'm1', - 0, - 'old guarded marker', - '{"messageId":"m1"}', - 100 - ) - - const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) - restartedProjectionTable.createTable() - - expect(restartedProjectionTable.isCurrent('s1', 1)).toBe(true) - expect(restartedProjectionTable.search('s1', 'old guarded marker', { limit: 5 })).toEqual( - [] - ) - expect( - restartedProjectionTable.search('s1', 'new guarded marker', { limit: 5 })[0] - ).toMatchObject({ - entry_id: 1, - search_text: 'new guarded marker' - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `does not mark a tape projection current when FTS DML fails${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - projectionTable.createTable() - if (!projectionTable.hasFtsReadyForTesting()) { - return - } - projectionTable.dropFtsForTesting() - ;(projectionTable as any).ftsReady = true - - expect(() => - projectionTable.replaceSession( - 's1', - [ - { - sessionId: 's1', - entryId: 1, - kind: 'message', - name: 'message/user', - sourceType: 'message', - sourceId: 'm1', - sourceSeq: 0, - searchText: 'Redis TTL', - summaryText: 'Redis TTL', - refs: { messageId: 'm1' }, - createdAt: 100 - } - ], - 1 - ) - ).toThrow() - expect(projectionTable.isCurrent('s1', 1)).toBe(false) - expect(projectionTable.getProjectedEntryIds('s1')).toEqual([]) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `filters memory view manifests by message in SQLite before limit${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - data: { ignored: true }, - meta: { messageId: 'msg-old' }, - createdAt: 999 - }) - for (let index = 0; index < 505; index += 1) { - table.appendAnchor({ - sessionId: 's1', - name: 'memory/view_assembled', - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: index, - selected: [`m-${index}`], - dropped: [], - queryHash: `hash-${index}` - }, - meta: { messageId: `msg-${index}` }, - createdAt: index - }) - } - - const rows = table.listMemoryViewManifestAnchorsBySessions(['s1'], { - limit: 1, - messageId: 'msg-0' - }) - - expect(rows).toHaveLength(1) - expect(rows[0]).toMatchObject({ - kind: 'anchor', - name: 'memory/view_assembled', - created_at: 0 - }) - expect(JSON.parse(rows[0].meta_json)).toEqual({ messageId: 'msg-0' }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `queries memory view manifests for large agents without expanding session parameters${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const sessionTable = new NewSessionsTable(db) - const tapeTable = new DeepChatTapeEntriesTable(db) - sessionTable.createTable() - tapeTable.createTable() - for (let index = 0; index < 1200; index += 1) { - const sessionId = `s-${index}` - db.prepare( - `INSERT INTO new_sessions ( - id, - agent_id, - title, - project_dir, - is_pinned, - is_draft, - active_skills, - disabled_agent_tools, - subagent_enabled, - session_kind, - parent_session_id, - subagent_meta_json, - created_at, - updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - sessionId, - 'agent-a', - `Session ${index}`, - null, - 0, - 0, - '[]', - '[]', - index % 2 === 0 ? 0 : 1, - index % 2 === 0 ? 'regular' : 'subagent', - null, - null, - index, - index - ) - tapeTable.appendAnchor({ - sessionId, - name: 'memory/view_assembled', - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: index, - selected: [`m-${index}`], - dropped: [], - queryHash: `hash-${index}` - }, - meta: { messageId: `msg-${index}` }, - createdAt: index - }) - } - db.prepare( - `INSERT INTO new_sessions ( - id, - agent_id, - title, - project_dir, - is_pinned, - is_draft, - active_skills, - disabled_agent_tools, - subagent_enabled, - session_kind, - parent_session_id, - subagent_meta_json, - created_at, - updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - 'other-session', - 'other-agent', - 'Other', - null, - 0, - 0, - '[]', - '[]', - 0, - 'regular', - null, - null, - 9999, - 9999 - ) - tapeTable.appendAnchor({ - sessionId: 'other-session', - name: 'memory/view_assembled', - state: { - policyVersion: 1, - tokenBudget: 1000, - estimatedTokens: 9999, - selected: ['other'], - dropped: [], - queryHash: 'other' - }, - meta: { messageId: 'msg-0' }, - createdAt: 9999 - }) - - const rows = tapeTable.listMemoryViewManifestAnchorsByAgent('agent-a', { - messageId: 'msg-0', - limit: 1 - }) - - expect(rows).toHaveLength(1) - expect(rows[0]).toMatchObject({ - session_id: 's-0', - kind: 'anchor', - name: 'memory/view_assembled', - created_at: 0 - }) - } finally { - db.close() - } - } - ) - - itIfSqlite( - `searches a SQLite tape projection and expands compact context without raw payloads${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - const projectionTable = new DeepChatTapeSearchProjectionTable(db) - table.createTable() - projectionTable.createTable() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'u1', seq: 0 }, - payload: { - record: createRecord({ - id: 'u1', - content: JSON.stringify({ - text: 'Check Redis TTL with /usr/local/bin/deploy --flag and error 42.', - files: [], - links: [] - }), - createdAt: 100, - updatedAt: 100 - }) - }, - meta: { source: 'live', orderSeq: 1, role: 'user' }, - createdAt: 100 - }) - table.append({ - sessionId: 's1', - kind: 'tool_result', - name: 'shell', - source: { type: 'tool_result', id: 'u1:tc1', seq: 0 }, - payload: { - messageId: 'u1', - orderSeq: 2, - toolCallId: 'tc1', - exitStatus: 42, - response: 'Exit code 42 in /tmp/deploy.log' - }, - meta: { source: 'live', status: 'error' }, - createdAt: 110 - }) - - const pathHits = service.search('s1', '/usr/local/bin/deploy', { limit: 5 }) - expect(pathHits).toHaveLength(1) - expect(pathHits[0]).toMatchObject({ - kind: 'message', - summary: expect.stringContaining('Redis TTL'), - refs: { - messageId: 'u1', - role: 'user', - filePaths: expect.arrayContaining(['/usr/local/bin/deploy']) - } - }) - expect(pathHits[0]).not.toHaveProperty('payload') - expect(pathHits[0]).not.toHaveProperty('meta') - expect(service.search('s1', 'Redis TTL', { limit: 5 }).map((hit) => hit.entryId)).toContain( - pathHits[0].entryId - ) - const errorHits = service.search('s1', '42', { kinds: ['tool_result'], limit: 5 }) - expect(errorHits[0]).toMatchObject({ - refs: { - toolCallId: 'tc1', - exitStatus: 42 - } - }) - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) - - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'u2', seq: 0 }, - payload: { - record: createRecord({ - id: 'u2', - orderSeq: 3, - content: JSON.stringify({ text: 'zoxide marker 简洁', files: [], links: [] }), - createdAt: 120, - updatedAt: 120 - }) - }, - meta: { source: 'live', orderSeq: 3, role: 'user' }, - createdAt: 120 - }) - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(false) - const rebuiltHits = service.search('s1', '简洁', { limit: 5 }) - expect(rebuiltHits.map((hit) => hit.refs?.messageId)).toContain('u2') - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) - if (projectionTable.hasFtsReadyForTesting()) { - projectionTable.dropFtsForTesting() - ;(projectionTable as any).ftsReady = true - table.append({ - sessionId: 's1', - kind: 'message', - name: 'message/user', - source: { type: 'message', id: 'u3', seq: 0 }, - payload: { - record: createRecord({ - id: 'u3', - orderSeq: 4, - content: JSON.stringify({ - text: 'fts recovery marker', - files: [], - links: [] - }), - createdAt: 130, - updatedAt: 130 - }) - }, - meta: { source: 'live', orderSeq: 4, role: 'user' }, - createdAt: 130 - }) - const recoveryHits = service.search('s1', 'fts recovery marker', { limit: 5 }) - expect(recoveryHits.map((hit) => hit.refs?.messageId)).toContain('u3') - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(false) - expect(projectionTable.hasFtsReadyForTesting()).toBe(false) - - const restoredHits = service.search('s1', 'fts recovery marker', { limit: 5 }) - expect(restoredHits.map((hit) => hit.refs?.messageId)).toContain('u3') - expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) - expect(projectionTable.hasFtsReadyForTesting()).toBe(true) - } - - const context = service.getContext('s1', [pathHits[0].entryId], { - before: 0, - after: 1, - limit: 2, - maxBytesPerEntry: 24, - maxTotalBytes: 24 - }) - expect(context.matchedEntryIds).toEqual([pathHits[0].entryId]) - expect(context.entries[0]).toMatchObject({ - entryId: pathHits[0].entryId, - summary: expect.stringContaining('Redis TTL'), - evidence: { - truncated: true - } - }) - expect(context.entries[0].evidence.bytes).toBeLessThanOrEqual(24) - expect(context.entries[0]).not.toHaveProperty('payload') - expect(context.entries[0]).not.toHaveProperty('meta') - const limitedContext = service.getContext( - 's1', - [pathHits[0].entryId, errorHits[0].entryId], - { - before: 0, - after: 0, - limit: 1 - } - ) - expect(limitedContext.entries.map((entry) => entry.entryId)).toEqual([pathHits[0].entryId]) - expect(limitedContext.matchedEntryIds).toEqual([pathHits[0].entryId]) - } finally { - db.close() - } - } - ) - - it('keeps legacy context builder output stable after tape backfill projection', () => { - const { table } = createTapeTableMock() - const records = [ - createRecord({ id: 'u1', orderSeq: 1 }), - createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { type: 'content', content: 'Tool finished', status: 'success', timestamp: 120 }, - { - type: 'tool_call', - status: 'success', - timestamp: 121, - tool_call: { - id: 'tc1', - name: 'example_tool', - params: '{"foo":"bar"}', - response: 'All good' - } - } - ]), - createdAt: 120, - updatedAt: 121 - }) - ] - const legacyMessageStore = { - getMessages: vi.fn().mockReturnValue(records) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const legacyContext = buildContext( - 's1', - { text: 'next', files: [] }, - 'System', - 10000, - 4096, - legacyMessageStore as any - ) - const tapeReady = service.ensureSessionTapeReady('s1', legacyMessageStore as any) - const tapeOnlyStore = { - getMessages: vi.fn(() => { - throw new Error('buildContext must use provided tape history records') - }) - } - const tapeContext = buildContext( - 's1', - { text: 'next', files: [] }, - 'System', - 10000, - 4096, - tapeOnlyStore as any, - false, - { - historyRecords: tapeReady.historyRecords - } - ) - - expect(tapeContext).toEqual(legacyContext) - expect(tapeOnlyStore.getMessages).not.toHaveBeenCalled() - }) - - it('rejects handoff anchors without a non-empty summary before writing Tape state', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - expect(() => service.handoff('s1', 'phase_done', { summary: ' ' })).toThrow( - 'Tape handoff requires a non-empty summary.' - ) - expect(() => service.handoff('s1', 'phase_done', { reason: 'phase complete' } as any)).toThrow( - 'Tape handoff requires a non-empty summary.' - ) - - expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() - expect(table.appendAnchor).not.toHaveBeenCalled() - expect(entries).toEqual([]) - }) - - it('migrates legacy session summary into a tape anchor during backfill', () => { - const { table, entries } = createTapeTableMock() - const messageStore = { - getMessages: vi.fn().mockReturnValue([ - createRecord({ id: 'u1', orderSeq: 1 }), - createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([{ type: 'content', content: 'answer', status: 'success' }]) - }) - ]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { - getSummaryState: vi.fn().mockReturnValue({ - summary_text: 'legacy compacted state', - summary_cursor_order_seq: 3, - summary_updated_at: 200 - }) - } - } as any) - - service.ensureSessionTapeReady('s1', messageStore as any) - - const summaryAnchor = entries.find((entry) => entry.name === 'compaction/migrated_summary') - expect(summaryAnchor).toMatchObject({ - kind: 'anchor', - source_type: 'summary', - source_id: 'legacy-summary', - created_at: 200 - }) - expect(JSON.parse(summaryAnchor.payload_json).state).toMatchObject({ - summary: 'legacy compacted state', - cursorOrderSeq: 3, - sourceMessageIds: ['u1', 'a1'] - }) - }) - - it('stores and lists view manifests as idempotent tape events', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user', - source: 'tape', - reason: 'selected_history' - } - ], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - - const first = service.appendViewManifest(manifest) - const second = service.appendViewManifest(manifest) - - expect(second.entry_id).toBe(first.entry_id) - expect(entries.filter((entry) => entry.name === 'view/assembled')).toHaveLength(1) - expect(JSON.parse(first.meta_json)).toMatchObject({ - policy: 'legacy_context_v1', - policyVersion: 1 - }) - expect(service.listViewManifestsByMessage('s1', 'a1')).toMatchObject([ - { - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - entryId: first.entry_id, - manifest: { - hashes: { - manifestHash: manifest.hashes.manifestHash - }, - policy: 'legacy_context_v1', - policyVersion: 1, - included: [ - { - messageId: 'u1', - entryId: sourceMaps.entryIdByMessageId.get('u1') - } - ] - } - } - ]) - }) - - it('indexes effective tool facts so tool-loop manifests reference real entries', () => { - const { table } = createTapeTableMock() - const assistantRecord = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } - } - ]) - }) - appendToolFactsToTape(table as any, assistantRecord, 'live', 'tool_loop') - - const service = createTapeService(table) - const sourceMaps = service.getViewManifestSourceMaps('s1') - expect(sourceMaps.toolCallEntryIdByToolId.get('tc1')).toBeGreaterThan(0) - expect(sourceMaps.toolResultEntryIdByToolId.get('tc1')).toBeGreaterThan(0) - - const refs = buildRequestRefs( - [ - { role: 'system', content: 'system' }, - { - role: 'assistant', - content: '', - tool_calls: [ - { id: 'tc1', type: 'function', function: { name: 'search', arguments: '{"q":"x"}' } } - ] - }, - { role: 'tool', content: 'result', tool_call_id: 'tc1' } - ], - sourceMaps - ) - expect(refs).toMatchObject([ - { role: 'system', source: 'synthetic' }, - { - role: 'assistant', - source: 'tape', - reason: 'tool_loop_message', - entryId: sourceMaps.toolCallEntryIdByToolId.get('tc1') - }, - { - role: 'tool', - source: 'tape', - reason: 'tool_loop_message', - entryId: sourceMaps.toolResultEntryIdByToolId.get('tc1') - } - ]) - }) - - it('scopes tool source maps to the in-flight message so reused tool ids do not collide', () => { - const { table } = createTapeTableMock() - const blocks = (response: string) => - JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response } - } - ]) - appendToolFactsToTape( - table as any, - createRecord({ id: 'a1', orderSeq: 2, role: 'assistant', content: blocks('first') }), - 'live', - 'tool_loop' - ) - appendToolFactsToTape( - table as any, - createRecord({ id: 'a2', orderSeq: 4, role: 'assistant', content: blocks('second') }), - 'live', - 'tool_loop' - ) - - const service = createTapeService(table) - const scopedToA1 = service.getViewManifestSourceMaps('s1', 'a1') - const scopedToA2 = service.getViewManifestSourceMaps('s1', 'a2') - - expect(scopedToA1.toolCallEntryIdByToolId.get('tc1')).toBeLessThan( - scopedToA2.toolCallEntryIdByToolId.get('tc1')! - ) - expect(scopedToA1.toolResultEntryIdByToolId.get('tc1')).not.toBe( - scopedToA2.toolResultEntryIdByToolId.get('tc1') - ) - }) - - it('exports tool_call and tool_result entries in a tool-loop replay slice', () => { - const { table } = createTapeTableMock() - const assistantRecord = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 120, - tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } - } - ]) - }) - appendToolFactsToTape(table as any, assistantRecord, 'live', 'tool_loop') - - const service = createTapeService(table) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const messages = [ - { role: 'system' as const, content: 'system' }, - { - role: 'assistant' as const, - content: '', - tool_calls: [ - { id: 'tc1', type: 'function' as const, function: { name: 'search', arguments: '{}' } } - ] - }, - { role: 'tool' as const, content: 'result', tool_call_id: 'tc1' } - ] - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: 1, - messages, - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: buildRequestRefs(messages, sourceMaps), - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - service.appendViewManifest(manifest) - - const slice = service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }) - const kinds = slice?.entries.map((entry) => entry.kind) ?? [] - expect(kinds).toContain('tool_call') - expect(kinds).toContain('tool_result') - }) - - it('filters malformed view manifest rows when listing by message', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { - type: 'runtime_event', - id: 'a1', - seq: 1 - }, - data: { - manifest: { - schemaVersion: 1, - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - included: 'not-an-array' - } - } - }) - - expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) - }) - - it('normalizes legacy manifests without hashVersion to hashVersion 1', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - const legacyManifest: Record = { ...manifest } - delete legacyManifest.hashVersion - - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 99 }, - data: { manifest: legacyManifest } - }) - - const [record] = service.listViewManifestsByMessage('s1', 'a1') - expect(record.manifest.hashVersion).toBe(1) - }) - - it('filters manifests whose hashVersion is not a number', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 99 }, - data: { manifest: { ...manifest, hashVersion: '2' } } - }) - - expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) - }) - - it('annotates read records with hash integrity without dropping tampered manifests', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const baseInput = { - sessionId: 's1', - taskType: 'chat' as const, - policy: 'legacy_context_v1' as const, - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - } - const validManifest = createTapeViewManifest({ ...baseInput, messageId: 'a1', requestSeq: 1 }) - service.appendViewManifest(validManifest) - - const tamperedManifest = createTapeViewManifest({ - ...baseInput, - messageId: 'a2', - requestSeq: 1 - }) - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a2', seq: 99 }, - data: { manifest: { ...tamperedManifest, latestEntryId: tamperedManifest.latestEntryId + 1 } } - }) - - const [validRecord] = service.listViewManifestsByMessage('s1', 'a1') - const [tamperedRecord] = service.listViewManifestsByMessage('s1', 'a2') - expect(validRecord.integrity).toBe('valid') - expect(tamperedRecord).toBeDefined() - expect(tamperedRecord.integrity).toBe('invalid') - }) - - it('binds reconstruction lineage to the latest reconstruction anchor including handoffs', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('s1') - table.appendAnchor({ - sessionId: 's1', - name: 'compaction/manual', - source: { type: 'summary', id: 's1', seq: 1 }, - state: {} - }) - table.appendAnchor({ - sessionId: 's1', - name: 'handoff/phase_done', - source: { type: 'handoff', id: 's1', seq: 2 }, - state: {} - }) - table.appendAnchor({ - sessionId: 's1', - name: 'fork/merge', - source: { type: 'fork', id: 'child', seq: 3 }, - state: {} - }) - - const sourceMaps = service.getViewManifestSourceMaps('s1') - const entryIdByName = (name: string) => - table.getBySession('s1').find((entry: any) => entry.name === name)?.entry_id - - expect(sourceMaps.anchorEntryIds).toHaveLength(4) - expect(sourceMaps.reconstructionAnchorEntryId).toBe(entryIdByName('handoff/phase_done')) - expect(sourceMaps.reconstructionAnchorEntryIds).toEqual([ - sourceMaps.reconstructionAnchorEntryId - ]) - expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain( - entryIdByName('compaction/manual') - ) - expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('fork/merge')) - }) - - it('keeps memory anchors off the reconstruction lineage', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('s1') - table.appendAnchor({ - sessionId: 's1', - name: 'compaction/manual', - source: { type: 'summary', id: 's1', seq: 1 }, - state: {} - }) - table.appendAnchor({ - sessionId: 's1', - name: 'memory/extract', - source: { type: 'runtime_event', id: 's1', seq: 2 }, - state: { memoryIds: ['m1'], count: 1, reason: 'episodic' } - }) - table.appendAnchor({ - sessionId: 's1', - name: 'memory/reflect', - source: { type: 'runtime_event', id: 's1', seq: 3 }, - state: { reflectionIds: ['r1'], sourceMemoryIds: ['m1'], count: 1 } - }) - - const sourceMaps = service.getViewManifestSourceMaps('s1') - const entryIdByName = (name: string) => - table.getBySession('s1').find((entry: any) => entry.name === name)?.entry_id - - // Memory anchors are recorded on the tape for observability... - expect(sourceMaps.anchorEntryIds).toContain(entryIdByName('memory/extract')) - expect(sourceMaps.anchorEntryIds).toContain(entryIdByName('memory/reflect')) - // ...but never own the reconstruction cursor; only the summary anchor does. - expect(sourceMaps.reconstructionAnchorEntryId).toBe(entryIdByName('compaction/manual')) - expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('memory/extract')) - expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('memory/reflect')) - }) - - it('bounds replay slices to the selected view instead of pre-cursor history', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.reconstructionAnchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user', - source: 'tape', - reason: 'selected_history' - } - ], - excluded: [], - summaryCursor: { - summaryCursorOrderSeq: 100, - preCursorOrderSeqMin: 1, - preCursorOrderSeqMax: 99, - preCursorCount: 99 - }, - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 100, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: false, - assembledAt: 200 - }) - service.appendViewManifest(manifest) - - const slice = service.exportReplaySlice('s1', 'a1') - - expect(slice?.refs.excludedEntryIds).toEqual([]) - expect(slice?.refs.anchorEntryIds).toEqual(sourceMaps.reconstructionAnchorEntryIds) - expect(slice?.refs.anchorEntryIds).toHaveLength(1) - expect(slice?.manifestRecord.manifest.excludedRanges).toEqual([ - { fromOrderSeq: 1, toOrderSeq: 99, count: 99, reason: 'before_summary_cursor' } - ]) - expect(slice?.entries).toHaveLength(3) - }) - - it('exports replay slices with metadata-only payloads by default', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [createTraceRow()]) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const manifest = createTapeViewManifest({ - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat', - policy: 'legacy_context_v1', - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user', - source: 'tape', - reason: 'selected_history' - } - ], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: true, - assembledAt: 200 - }) - const manifestEntry = service.appendViewManifest(manifest) - - const nowSpy = vi.spyOn(Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000) - const slice = service.exportReplaySlice('s1', 'a1') - const secondSlice = service.exportReplaySlice('s1', 'a1') - nowSpy.mockRestore() - - expect(slice).toMatchObject({ - schemaVersion: 1, - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - mode: 'trace_bound', - refs: { - manifestEntryId: manifestEntry.entry_id, - includedEntryIds: [sourceMaps.entryIdByMessageId.get('u1')], - anchorEntryIds: sourceMaps.anchorEntryIds - }, - hashes: { - manifestHash: manifest.hashes.manifestHash - } - }) - expect(slice?.hashes.sliceHash).toHaveLength(64) - expect(secondSlice?.hashes.sliceHash).toBe(slice?.hashes.sliceHash) - expect(secondSlice?.createdAt).toBe(2000) - expect(slice?.trace?.bodyHash).toHaveLength(64) - expect(slice?.trace?.bodyJson).toBeUndefined() - expect(slice?.entries.some((entry) => entry.entryId === manifestEntry.entry_id)).toBe(true) - expect( - slice?.entries.every((entry) => entry.payload === undefined && entry.meta === undefined) - ).toBe(true) - }) - - it('exports explicit replay request sequences with opt-in payloads', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [ - createTraceRow({ id: 'trace-1', request_seq: 1 }), - createTraceRow({ - id: 'trace-2', - request_seq: 2, - body_json: '{"messages":[{"role":"tool","content":"done"}]}' - }) - ]) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const baseManifestInput: TapeViewManifestBuildInput = { - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat' as const, - policy: 'legacy_context_v1' as const, - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user', - source: 'tape', - reason: 'selected_history' - } - ], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: true, - assembledAt: 200 - } - const firstManifest = createTapeViewManifest(baseManifestInput) - const secondManifest = createTapeViewManifest({ - ...baseManifestInput, - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 250 - }) - service.appendViewManifest(firstManifest) - service.appendViewManifest(secondManifest) - - const latest = service.exportReplaySlice('s1', 'a1') - const first = service.exportReplaySlice('s1', 'a1', { - requestSeq: 1, - includeTapePayloads: true, - includeTracePayload: true - }) - - expect(latest?.requestSeq).toBe(2) - expect(first?.requestSeq).toBe(1) - expect(first?.trace?.bodyJson).toContain('"hello"') - expect(first?.entries.some((entry) => entry.payload?.record)).toBe(true) - expect(first?.entries.some((entry) => entry.meta?.source === 'backfill')).toBe(true) - }) - - it('binds each replay slice to its own request seq, ignoring sentinel gap traces', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [ - createTraceRow({ - id: 'trace-req-1', - request_seq: 1, - body_json: '{"messages":[{"role":"user","content":"first-request"}]}' - }), - createTraceRow({ - id: 'trace-gap', - request_seq: 0, - endpoint: 'deepchat://interleaved-reasoning-gap', - body_json: '{"providerId":"openai"}' - }), - createTraceRow({ - id: 'trace-req-2', - request_seq: 2, - body_json: '{"messages":[{"role":"tool","content":"second-request"}]}' - }) - ]) - const messageStore = { - getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) - } - - service.ensureSessionTapeReady('s1', messageStore as any) - const sourceMaps = service.getViewManifestSourceMaps('s1') - const baseManifestInput = { - sessionId: 's1', - messageId: 'a1', - requestSeq: 1, - taskType: 'chat' as const, - policy: 'legacy_context_v1' as const, - policyVersion: 1, - messages: [{ role: 'user' as const, content: 'hello' }], - tools: [], - latestEntryId: sourceMaps.latestEntryId, - anchorEntryIds: sourceMaps.anchorEntryIds, - included: [ - { - entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, - messageId: 'u1', - orderSeq: 1, - role: 'user' as const, - source: 'tape' as const, - reason: 'selected_history' as const - } - ], - excluded: [], - tokenBudget: { - contextLength: 1000, - requestedMaxTokens: 100, - effectiveMaxTokens: 100, - reserveTokens: 100, - toolReserveTokens: 0 - }, - providerId: 'openai', - modelId: 'gpt-4o', - summaryCursorOrderSeq: 1, - supportsVision: true, - supportsAudioInput: false, - traceDebugEnabled: true, - assembledAt: 200 - } - service.appendViewManifest(createTapeViewManifest(baseManifestInput)) - service.appendViewManifest( - createTapeViewManifest({ - ...baseManifestInput, - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 250 - }) - ) - - const first = service.exportReplaySlice('s1', 'a1', { - requestSeq: 1, - includeTracePayload: true - }) - const second = service.exportReplaySlice('s1', 'a1', { - requestSeq: 2, - includeTracePayload: true - }) - - expect(first?.trace?.bodyJson).toContain('first-request') - expect(second?.trace?.bodyJson).toContain('second-request') - }) - - it('returns null when exporting a replay slice without a manifest', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [createTraceRow()]) - - expect(service.exportReplaySlice('s1', 'a1')).toBeNull() - }) - - it('rejects non-positive replay request sequences', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [createTraceRow()]) - - expect(() => service.exportReplaySlice('s1', 'a1', { requestSeq: 0 })).toThrow( - 'requestSeq must be a positive integer.' - ) - }) - - it('joins an exact manifest and trace into a causal observation slice', () => { - const { table } = createTapeTableMock() - const assistant = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: '[{"type":"content","content":"done","status":"success"}]', - status: 'sent', - metadata: '{"totalTokens":10}', - createdAt: 200, - updatedAt: 300 - }) - appendMessageRecordToTape(table as any, assistant, 'live') - const service = createTapeService( - table, - [createTraceRow(), createTraceRow({ id: 'other-session', session_id: 's2', request_seq: 9 })], - [createMessageRow()] - ) - service.appendViewManifest(createObservationManifest()) - - const slice = service.readCausalObservationSlice('s1', 'a1', { - currentRuntimeStatus: 'idle' - }) - - expect(slice).toMatchObject({ - schemaVersion: 1, - sessionId: 's1', - messageId: 'a1', - request: { - state: 'manifest_bound', - requestSeq: 1, - replay: { - requestSeq: 1, - mode: 'trace_bound', - trace: { id: 'trace-1', requestSeq: 1 } - } - }, - output: { - correlation: 'message_only', - terminalMessage: { - status: 'sent', - orderSeq: 2, - createdAt: 200, - updatedAt: 300 - } - }, - runtime: { scope: 'current_only', status: 'idle', eventHistory: 'not_persisted' } - }) - expect(slice.output.entries.map((entry) => entry.kind)).toContain('message') - expect(slice.output.terminalMessage?.contentHash).toHaveLength(64) - expect(slice.output.terminalMessage?.metadataHash).toHaveLength(64) - }) - - it('prefers an explicit causal request sequence and otherwise selects the latest raw sequence', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 100 })) - service.appendViewManifest( - createObservationManifest({ - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 200 - }) - ) - - expect(service.readCausalObservationSlice('s1', 'a1').request).toMatchObject({ - state: 'manifest_bound', - requestSeq: 2 - }) - expect(service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }).request).toMatchObject( - { state: 'manifest_bound', requestSeq: 1 } - ) - }) - - it('does not fall back to an older manifest when a later traced request has no manifest', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [ - createTraceRow({ id: 'trace-1', request_seq: 1 }), - createTraceRow({ id: 'trace-2', request_seq: 2 }) - ]) - service.appendViewManifest(createObservationManifest({ requestSeq: 1 })) - - expect(service.readCausalObservationSlice('s1', 'a1').request).toMatchObject({ - state: 'manifest_missing', - requestSeq: 2, - trace: { id: 'trace-2', requestSeq: 2 } - }) - }) - - it('returns a trace-only request without manufacturing a view manifest', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table, [createTraceRow({ request_seq: 4 })]) - - const request = service.readCausalObservationSlice('s1', 'a1').request - - expect(request).toMatchObject({ - state: 'manifest_missing', - requestSeq: 4, - trace: { requestSeq: 4 } - }) - expect( - request.state === 'manifest_missing' ? request.trace?.bodyJson : undefined - ).toBeUndefined() - }) - - it('distinguishes malformed manifests from hash-invalid but readable manifests', () => { - const { table } = createTapeTableMock() - const service = createTapeService(table) - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'bad', seq: 2 }, - data: { manifest: { schemaVersion: 99, requestSeq: 2 } } - }) - const tamperedManifest = createObservationManifest({ messageId: 'a1', requestSeq: 1 }) - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 1 }, - data: { manifest: { ...tamperedManifest, latestEntryId: 1 } } - }) - - expect(service.readCausalObservationSlice('s1', 'bad').request).toEqual({ - state: 'manifest_malformed', - requestSeq: 2, - trace: null - }) - const invalid = service.readCausalObservationSlice('s1', 'a1').request - expect(invalid.state).toBe('manifest_bound') - expect(invalid.state === 'manifest_bound' ? invalid.replay.integrity : undefined).toBe( - 'invalid' - ) - }) - - it('ignores request sequence zero interleaved-reasoning sentinels', () => { - const { table } = createTapeTableMock() - table.appendEvent({ - sessionId: 's1', - name: 'view/assembled', - source: { type: 'runtime_event', id: 'a1', seq: 0 }, - data: { manifest: { schemaVersion: 99, requestSeq: 0 } } - }) - const service = createTapeService(table, [ - createTraceRow({ - id: 'trace-gap', - request_seq: 0, - endpoint: 'deepchat://interleaved-reasoning-gap' - }) - ]) - - expect(service.readCausalObservationSlice('s1', 'a1').request).toEqual({ - state: 'request_unavailable', - requestSeq: null, - trace: null - }) - }) - - it('keeps multi-round assistant and tool output correlated at message scope only', () => { - const { table } = createTapeTableMock() - const assistant = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: JSON.stringify([ - { - type: 'tool_call', - status: 'success', - timestamp: 220, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'result' - } - } - ]), - status: 'sent', - createdAt: 200, - updatedAt: 300 - }) - appendMessageRecordToTape(table as any, assistant, 'live') - const service = createTapeService(table, [], [createMessageRow({ content: assistant.content })]) - service.appendViewManifest(createObservationManifest({ requestSeq: 1 })) - service.appendViewManifest( - createObservationManifest({ - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null - }) - ) - - const firstOutput = service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }).output - const latestOutput = service.readCausalObservationSlice('s1', 'a1').output - - expect(latestOutput.correlation).toBe('message_only') - expect(latestOutput.entries.map((entry) => entry.kind)).toEqual([ - 'message', - 'tool_call', - 'tool_result' - ]) - expect(latestOutput.entries.map((entry) => entry.entryId)).toEqual( - firstOutput.entries.map((entry) => entry.entryId) - ) - }) - - it('does not infer a completed event from a paused pending assistant message', () => { - const { table } = createTapeTableMock() - const service = createTapeService( - table, - [], - [createMessageRow({ status: 'pending', content: '[{"type":"action","status":"pending"}]' })] - ) - - const slice = service.readCausalObservationSlice('s1', 'a1', { - currentRuntimeStatus: 'generating' - }) - - expect(slice.output.entries).toEqual([]) - expect(slice.output.terminalMessage).toBeNull() - expect(slice.runtime).toEqual({ - scope: 'current_only', - status: 'generating', - eventHistory: 'not_persisted' - }) - expect(JSON.stringify(slice)).not.toContain('completed') - }) - - it('reads an old message without bootstrapping or backfilling Tape', () => { - const { table, entries } = createTapeTableMock() - const messageInsert = vi.fn() - const traceInsert = vi.fn() - const projectionApply = vi.fn() - const projectionReplace = vi.fn() - const cursorWrite = vi.fn() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatMessageTracesTable: { - listByMessageId: vi.fn().mockReturnValue([]), - insert: traceInsert - }, - deepchatMessagesTable: { - get: vi.fn().mockReturnValue(createMessageRow()), - insert: messageInsert - }, - deepchatMemoryIngestionProjectionTable: { - applyAppendedEntry: projectionApply, - replaceSession: projectionReplace - }, - deepchatSessionsTable: { - getSummaryState: vi.fn().mockReturnValue(null), - updateMemoryCursorOrderSeq: cursorWrite - } - } as any) - - const slice = service.readCausalObservationSlice('s1', 'a1') - - expect(slice.request).toEqual({ - state: 'request_unavailable', - requestSeq: null, - trace: null - }) - expect(slice.output.terminalMessage?.status).toBe('sent') - expect(slice.runtime).toEqual({ - scope: 'unavailable', - status: null, - eventHistory: 'not_persisted' - }) - expect(entries).toEqual([]) - expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() - expect(table.append).not.toHaveBeenCalled() - expect(table.appendEvent).not.toHaveBeenCalled() - expect(messageInsert).not.toHaveBeenCalled() - expect(traceInsert).not.toHaveBeenCalled() - expect(projectionApply).not.toHaveBeenCalled() - expect(projectionReplace).not.toHaveBeenCalled() - expect(cursorWrite).not.toHaveBeenCalled() - }) - - it('keeps causal observations metadata-only by default and reuses replay payload opt-ins', () => { - const { table } = createTapeTableMock() - const assistant = createRecord({ - id: 'a1', - orderSeq: 2, - role: 'assistant', - content: 'tape-secret-output', - status: 'sent', - metadata: '{"secret":"tape-metadata"}', - createdAt: 200, - updatedAt: 300 - }) - appendMessageRecordToTape(table as any, assistant, 'live') - const service = createTapeService( - table, - [ - createTraceRow({ - headers_json: '{"authorization":"secret-header"}', - body_json: '{"prompt":"secret-request"}' - }) - ], - [ - createMessageRow({ - content: 'projection-only-content-secret', - metadata: '{"secret":"projection-only-metadata"}', - blocks_json: '["projection-only-blocks"]', - error: 'projection-only-error' - }) - ] - ) - service.appendViewManifest(createObservationManifest()) - - const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) - const metadataOnly = service.readCausalObservationSlice('s1', 'a1') - const withPayloads = service.readCausalObservationSlice('s1', 'a1', { - includeTapePayloads: true, - includeTracePayload: true - }) - now.mockRestore() - - expect(JSON.stringify(metadataOnly)).not.toContain('tape-secret-output') - expect(JSON.stringify(metadataOnly)).not.toContain('tape-metadata') - expect(JSON.stringify(metadataOnly)).not.toContain('secret-header') - expect(JSON.stringify(metadataOnly)).not.toContain('secret-request') - expect(JSON.stringify(metadataOnly)).not.toContain('projection-only') - expect(withPayloads.output.entries.some((entry) => entry.payload?.record)).toBe(true) - expect(withPayloads.request.state).toBe('manifest_bound') - expect( - withPayloads.request.state === 'manifest_bound' - ? withPayloads.request.replay.trace?.headersJson - : undefined - ).toContain('secret-header') - expect(JSON.stringify(withPayloads)).toContain('tape-secret-output') - expect(JSON.stringify(withPayloads)).not.toContain('projection-only') - expect(metadataOnly.output.terminalMessage).not.toHaveProperty('content') - expect(metadataOnly.output.terminalMessage).not.toHaveProperty('metadata') - expect(metadataOnly.output.terminalMessage).not.toHaveProperty('blocks') - expect(metadataOnly.output.terminalMessage).not.toHaveProperty('error') - expect(stripObservationPayloadOptIns(withPayloads)).toEqual( - stripObservationPayloadOptIns(metadataOnly) - ) - }) - - it('keeps storage and Memory seams unchanged across every causal observation read mode', () => { - const { table, entries } = createTapeTableMock() - const { final } = appendObservationIsolationFacts(table) - const traceRows = [ - createTraceRow({ id: 'trace-1', request_seq: 1 }), - createTraceRow({ id: 'trace-2', request_seq: 2 }), - createTraceRow({ id: 'trace-only', message_id: 'a-trace', request_seq: 7 }) - ] - const messageRows = [createMessageRow({ order_seq: 3 })] - const projectionState = { - meta: { session_id: 's1', projection_version: 1, max_entry_id: entries.length }, - range: [{ session_id: 's1', message_id: 'a1', order_seq: 3, entry_id: entries.length }] - } - const memoryCursorOrderSeq = 3 - const memoryProjectionWrites = createSpies([ - 'applyAppendedEntry', - 'replaceSession', - 'invalidateSession' - ]) - const memoryProjectionTable = { - ...memoryProjectionWrites, - readCurrentRange: vi.fn(() => structuredClone(projectionState.range)), - getSessionMeta: vi.fn(() => structuredClone(projectionState.meta)) - } - const cursorWrites = createSpies(['updateMemoryCursorOrderSeq', 'rewindMemoryCursorOrderSeq']) - const sessionTable = { - ...cursorWrites, - getSummaryState: vi.fn().mockReturnValue(null), - getMemoryCursorOrderSeq: vi.fn(() => memoryCursorOrderSeq) - } - const messageWrites = createSpies([ - 'insert', - 'updateContent', - 'updateStatus', - 'updateContentAndStatus' - ]) - const messageTable = { - ...messageWrites, - get: vi.fn((messageId: string) => messageRows.find((row) => row.id === messageId)) - } - const traceWrites = createSpies(['insert', 'deleteByMessageId', 'deleteBySessionId']) - const traceTable = { - ...traceWrites, - listByMessageId: vi.fn((messageId: string) => - traceRows.filter((row) => row.message_id === messageId) - ) - } - const { presenter, memoryPropertyAccess } = trackMemoryPropertyAccess({ - deepchatTapeEntriesTable: table, - deepchatMessageTracesTable: traceTable, - deepchatMessagesTable: messageTable, - deepchatSessionsTable: sessionTable, - deepchatMemoryIngestionProjectionTable: memoryProjectionTable - }) - const service = new SessionTape(presenter as any) - service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 210 })) - service.appendViewManifest( - createObservationManifest({ - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 310 - }) - ) - - for (const write of [ - table.ensureBootstrapAnchor, - table.append, - table.appendAnchor, - table.appendEvent, - table.deleteBySession - ]) { - write.mockClear() - } - const captureState = () => ({ - tapeRows: structuredClone(entries), - maxEntryId: table.getMaxEntryId('s1'), - entryOrder: entries.map((entry) => entry.entry_id), - maxMessageOrder: Math.max(0, ...service.getMessageRecords('s1').map((row) => row.orderSeq)), - messageRows: structuredClone(messageRows), - traceRows: structuredClone(traceRows), - viewManifestRows: structuredClone( - entries.filter((entry) => entry.kind === 'event' && entry.name === 'view/assembled') - ), - effectiveView: service.getMessageRecords('s1'), - replay: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }), - replayHash: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 })?.hashes.sliceHash, - projectionMeta: memoryProjectionTable.getSessionMeta(), - projectionRange: memoryProjectionTable.readCurrentRange(), - memoryCursorOrderSeq: sessionTable.getMemoryCursorOrderSeq() - }) - const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) - - try { - const before = captureState() - const { defaultObservation, repeatedObservation, explicitObservation, traceOnlyObservation } = - readObservationMatrix(service) - const after = captureState() - - expect(after).toEqual(before) - expect(repeatedObservation).toEqual(defaultObservation) - expect(defaultObservation.request).toMatchObject({ - state: 'manifest_bound', - requestSeq: 2 - }) - expect(explicitObservation.request).toMatchObject({ - state: 'manifest_bound', - requestSeq: 1 - }) - expect(traceOnlyObservation.request).toMatchObject({ - state: 'manifest_missing', - requestSeq: 7, - trace: { id: 'trace-only' } - }) - expect(defaultObservation.output).toMatchObject({ - correlation: 'message_only', - entries: [{ kind: 'message' }, { kind: 'tool_call' }, { kind: 'tool_result' }] - }) - expect(before.effectiveView.map((record) => record.id)).toEqual(['u1', 'a1']) - expect(JSON.parse(before.effectiveView[0].content).text).toBe('edited') - expect(before.effectiveView[1]).toMatchObject({ status: 'sent', content: final.content }) - } finally { - now.mockRestore() - } - - for (const write of [ - table.ensureBootstrapAnchor, - table.append, - table.appendAnchor, - table.appendEvent, - table.deleteBySession, - ...Object.values(messageWrites), - ...Object.values(traceWrites), - ...Object.values(memoryProjectionWrites), - ...Object.values(cursorWrites), - memoryPropertyAccess - ]) { - expect(write).not.toHaveBeenCalled() - } - }) - - itIfSqlite( - `preserves real Tape, message, trace, Memory projection and schema state while observing${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, - () => { - const db = new DatabaseCtor(':memory:') - try { - const memoryProjectionTable = new DeepChatMemoryIngestionProjectionTable(db) - const tapeTable = new DeepChatTapeEntriesTable(db, memoryProjectionTable) - const messageTable = new DeepChatMessagesTable(db) - const traceTable = new DeepChatMessageTracesTable(db) - const sessionTable = new DeepChatSessionsTable(db) - memoryProjectionTable.createTable() - tapeTable.createTable() - messageTable.createTable() - traceTable.createTable() - sessionTable.createTable() - sessionTable.create('s1', 'openai', 'gpt-4o', 'full_access') - sessionTable.updateMemoryCursorOrderSeq('s1', 3) - - appendObservationIsolationFacts(tapeTable) - messageTable.insert({ - id: 'a1', - sessionId: 's1', - orderSeq: 3, - role: 'assistant', - content: 'projection-only-content-secret', - status: 'sent', - metadata: '{"secret":"projection-only-metadata"}', - createdAt: 200, - updatedAt: 300 - }) - for (const trace of [ - { id: 'trace-1', messageId: 'a1', requestSeq: 1 }, - { id: 'trace-2', messageId: 'a1', requestSeq: 2 }, - { id: 'trace-only', messageId: 'a-trace', requestSeq: 7 } - ]) { - traceTable.insert({ - ...trace, - sessionId: 's1', - providerId: 'openai', - modelId: 'gpt-4o', - endpoint: 'https://api.openai.test/v1/chat/completions', - headersJson: `{"authorization":"${trace.id}-header"}`, - bodyJson: `{"prompt":"${trace.id}-body"}`, - truncated: false, - createdAt: 300 + trace.requestSeq - }) - } - - const service = new SessionTape({ - deepchatTapeEntriesTable: tapeTable, - deepchatMessagesTable: messageTable, - deepchatMessageTracesTable: traceTable, - deepchatSessionsTable: sessionTable, - deepchatMemoryIngestionProjectionTable: memoryProjectionTable - } as any) - service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 210 })) - service.appendViewManifest( - createObservationManifest({ - requestSeq: 2, - taskType: 'tool_loop', - policy: 'tool_loop_shadow', - policyVersion: null, - assembledAt: 310 - }) - ) - const effectiveRecords = service.getMessageRecords('s1') - const sourceMaps = service.getViewManifestSourceMaps('s1') - memoryProjectionTable.replaceSession( - 's1', - effectiveRecords - .filter( - (record): record is ChatMessageRecord & { status: 'sent' | 'error' } => - record.status === 'sent' || record.status === 'error' - ) - .map((record) => ({ - sessionId: 's1', - messageId: record.id, - orderSeq: record.orderSeq, - entryId: sourceMaps.entryIdByMessageId.get(record.id)!, - role: record.role, - content: record.content, - status: record.status, - hadToolUse: record.id === 'a1' - })), - tapeTable.getMaxEntryId('s1') - ) - - const captureState = () => { - const tapeRows = tapeTable.getBySession('s1') - return { - tapeRows, - maxEntryId: tapeTable.getMaxEntryId('s1'), - entryOrder: tapeRows.map((row) => row.entry_id), - messageRow: messageTable.get('a1'), - traceRows: db - .prepare( - `SELECT * - FROM deepchat_message_traces - ORDER BY session_id, message_id, request_seq, id` - ) - .all(), - viewManifestRows: tapeRows.filter( - (row) => row.kind === 'event' && row.name === 'view/assembled' - ), - effectiveView: service.getMessageRecords('s1'), - replay: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }), - replayHash: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 })?.hashes.sliceHash, - projectionMeta: memoryProjectionTable.getSessionMeta('s1'), - projectionRange: memoryProjectionTable.readCurrentRange('s1', 0, 10), - memoryCursorOrderSeq: sessionTable.getMemoryCursorOrderSeq('s1'), - schema: db - .prepare( - `SELECT type, name, tbl_name, sql - FROM sqlite_master - WHERE name NOT LIKE 'sqlite_%' - ORDER BY type, name, tbl_name` - ) - .all() - } - } - const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) - - try { - const before = captureState() - readObservationMatrix(service) - const after = captureState() - - expect(after).toEqual(before) - } finally { - now.mockRestore() - } - } finally { - db.close() - } - } - ) - - it('keeps pending message records for resume but hides pending tool facts from search', () => { - const { table } = createTapeTableMock() - const pendingBlocks = [ - { - type: 'tool_call', - status: 'pending', - timestamp: 100, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'pending result' - } - } - ] - const messageStore = { - getMessages: vi.fn().mockReturnValue([ - createRecord({ - id: 'a1', - orderSeq: 1, - role: 'assistant', - status: 'pending', - content: JSON.stringify(pendingBlocks), - updatedAt: 100 - }) - ]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.ensureSessionTapeReady('s1', messageStore as any) - - expect(service.getMessageRecords('s1')).toMatchObject([{ id: 'a1', status: 'pending' }]) - expect(service.search('s1', 'pending result', { kinds: ['tool_result'] })).toEqual([]) - }) - - it('lets final assistant facts supersede earlier pending tape facts', () => { - const { table, entries } = createTapeTableMock() - const pendingBlocks = [ - { - type: 'tool_call', - status: 'pending', - timestamp: 100, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'pending result' - } - } - ] - const finalBlocks = [ - { - type: 'tool_call', - status: 'success', - timestamp: 200, - tool_call: { - id: 'tc1', - name: 'search', - params: '{"q":"x"}', - response: 'final result' - } - } - ] - const messageStore = { - getMessages: vi - .fn() - .mockReturnValueOnce([ - createRecord({ - id: 'a1', - orderSeq: 1, - role: 'assistant', - status: 'pending', - content: JSON.stringify(pendingBlocks), - metadata: JSON.stringify({ totalTokens: 1 }), - updatedAt: 100 - }) - ]) - .mockReturnValue([ - createRecord({ - id: 'a1', - orderSeq: 1, - role: 'assistant', - status: 'sent', - content: JSON.stringify(finalBlocks), - metadata: JSON.stringify({ totalTokens: 7 }), - updatedAt: 200 - }) - ]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.ensureSessionTapeReady('s1', messageStore as any) - service.ensureSessionTapeReady('s1', messageStore as any) - - expect(service.getMessageRecords('s1')).toMatchObject([ - { - id: 'a1', - status: 'sent' - } - ]) - const effectiveRecord = service.getMessageRecords('s1')[0]! - expect(JSON.parse(effectiveRecord.content)[0].tool_call.response).toBe('final result') - expect( - entries.filter((entry) => entry.kind === 'message' && entry.name === 'message/assistant') - ).toHaveLength(2) - expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) - const finalToolResult = entries.filter((entry) => entry.kind === 'tool_result').at(-1)! - expect(JSON.parse(finalToolResult.payload_json).response).toBe('final result') - expect(service.info('s1').lastTokenUsage).toBe(7) - expect(service.search('s1', 'pending result', { kinds: ['tool_result'] })).toEqual([]) - expect(service.search('s1', 'final result', { kinds: ['tool_result'] })).toHaveLength(1) - }) - - it('keeps fork writes isolated until merge and discards fork entries on discard', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const fork = service.createFork('s1', 'fork-1') - service.appendForkMessageRecord(fork, createRecord({ id: 'fu1', sessionId: 'ignored' })) - - expect( - entries.some((entry) => entry.session_id === 's1' && entry.name === 'message/user') - ).toBe(false) - - const mergedCount = service.mergeFork('s1', 'fork-1') - - expect(mergedCount).toBe(1) - expect( - entries.some((entry) => entry.session_id === 's1' && entry.name === 'message/user') - ).toBe(true) - expect(entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/merge')).toBe( - true - ) - expect(entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/start')).toBe( - false - ) - - const discardFork = service.createFork('s1', 'fork-2') - service.appendForkMessageRecord(discardFork, createRecord({ id: 'fu2', sessionId: 'ignored' })) - service.discardFork('s1', 'fork-2') - - expect(entries.some((entry) => entry.session_id === discardFork.forkSessionId)).toBe(false) - expect( - entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') - ).toBe(true) - }) - - it('records exact fork heads and keeps retries bounded to the first merge', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('parent') - table.appendEvent({ sessionId: 'parent', name: 'parent/tail', data: { value: 1 } }) - const fork = service.createFork('parent', 'bounded') - table.appendEvent({ sessionId: 'parent', name: 'parent/later', data: { value: 2 } }) - const retriedFork = service.createFork('parent', 'bounded') - service.appendForkMessageRecord(fork, createRecord({ id: 'first', sessionId: 'ignored' })) - - const forkStart = entries.find( - (entry) => entry.session_id === fork.forkSessionId && entry.name === 'fork/start' - ) - expect(fork.parentHeadEntryId).toBe(2) - expect(retriedFork.parentHeadEntryId).toBe(2) - expect(JSON.parse(forkStart.payload_json).state.parentHeadEntryId).toBe(2) - - const getBoundedEntries = table.getBySessionUpToEntryId.getMockImplementation()! - table.getBySessionUpToEntryId.mockImplementationOnce( - (sessionId: string, maxEntryId: number) => { - table.appendEvent({ - sessionId: fork.forkSessionId, - name: 'fork/late-tail', - data: { value: 2 } - }) - return getBoundedEntries(sessionId, maxEntryId) - } - ) - - expect(service.mergeFork('parent', 'bounded')).toBe(1) - expect(service.mergeFork('parent', 'bounded')).toBe(1) - - const parentEntries = entries.filter((entry) => entry.session_id === 'parent') - expect(parentEntries.filter((entry) => entry.source_type === 'fork')).toHaveLength(2) - expect(parentEntries.some((entry) => entry.name === 'fork/late-tail')).toBe(false) - const receipt = parentEntries.find((entry) => entry.name === 'fork/merge')! - expect(JSON.parse(receipt.payload_json).data).toMatchObject({ - forkHeadEntryId: 3, - mergedCount: 1 - }) - }) - - it('rolls back mocked fork merge writes when a copied entry fails', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const fork = service.createFork('parent', 'mock-atomic') - service.appendForkMessageRecord( - fork, - createRecord({ id: 'first', orderSeq: 1, sessionId: 'ignored' }) - ) - service.appendForkMessageRecord( - fork, - createRecord({ id: 'second', orderSeq: 2, sessionId: 'ignored' }) - ) - - const append = table.append.getMockImplementation()! - let copiedEntryCount = 0 - table.append.mockImplementation((input: any) => { - if (input.sessionId === 'parent' && input.source?.type === 'fork') { - copiedEntryCount += 1 - if (copiedEntryCount === 2) { - throw new Error('injected merge failure') - } - } - return append(input) - }) - - expect(() => service.mergeFork('parent', 'mock-atomic')).toThrow('injected merge failure') - expect( - entries.filter( - (entry) => - entry.session_id === 'parent' && - (entry.source_type === 'fork' || entry.name === 'fork/merge') - ) - ).toEqual([]) - }) - - it('rolls back copied fork entries when the merge receipt fails', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const fork = service.createFork('parent', 'receipt-failure') - service.appendForkMessageRecord(fork, createRecord({ id: 'first', sessionId: 'ignored' })) - - const appendEvent = table.appendEvent.getMockImplementation()! - table.appendEvent.mockImplementation((input: any) => { - if (input.sessionId === 'parent' && input.name === 'fork/merge') { - throw new Error('injected receipt failure') - } - return appendEvent(input) - }) - - expect(() => service.mergeFork('parent', 'receipt-failure')).toThrow('injected receipt failure') - expect( - entries.filter( - (entry) => - entry.session_id === 'parent' && - (entry.source_type === 'fork' || entry.name === 'fork/merge') - ) - ).toEqual([]) - }) - - it('commits one idempotent receipt for an empty fork', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - service.createFork('parent', 'empty') - - expect(service.mergeFork('parent', 'empty')).toBe(0) - expect(service.mergeFork('parent', 'empty')).toBe(0) - const parentEntries = entries.filter((entry) => entry.session_id === 'parent') - expect(parentEntries).toHaveLength(1) - expect(parentEntries[0].name).toBe('fork/merge') - expect(JSON.parse(parentEntries[0].payload_json).data).toMatchObject({ - forkHeadEntryId: 2, - mergedCount: 0 - }) - }) - - it('rejects missing and discarded forks without committing an empty merge receipt', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - expect(() => service.mergeFork('parent', 'missing')).toThrow( - 'Fork missing does not exist or has been discarded.' - ) - - service.createFork('parent', 'discarded') - service.discardFork('parent', 'discarded') - expect(() => service.mergeFork('parent', 'discarded')).toThrow( - 'Fork discarded does not exist or has been discarded.' - ) - expect( - entries.filter((entry) => entry.session_id === 'parent' && entry.name === 'fork/merge') - ).toEqual([]) - }) - - it('returns an existing merge receipt after the merged fork Tape is cleaned up', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const fork = service.createFork('parent', 'cleanup-after-merge') - service.appendForkMessageRecord(fork, createRecord({ id: 'merged', sessionId: 'ignored' })) - - expect(service.mergeFork('parent', 'cleanup-after-merge')).toBe(1) - table.deleteBySession(fork.forkSessionId) - expect(service.mergeFork('parent', 'cleanup-after-merge')).toBe(1) - }) - - it('merges a legacy fork start that predates the parent head field', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - const fork = service.createFork('parent', 'legacy-start') - const start = entries.find( - (entry) => entry.session_id === fork.forkSessionId && entry.name === 'fork/start' - )! - const payload = JSON.parse(start.payload_json) - delete payload.state.parentHeadEntryId - start.payload_json = JSON.stringify(payload) - service.appendForkMessageRecord(fork, createRecord({ id: 'legacy', sessionId: 'ignored' })) - - expect(service.mergeFork('parent', 'legacy-start')).toBe(1) - }) - - it('accepts a valid legacy fork merge receipt without a frozen head', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/merge', - source: { type: 'fork', id: 'legacy-receipt', seq: 0 }, - provenanceKey: 'fork:parent:legacy-receipt:merge:event', - data: { - forkId: 'legacy-receipt', - forkSessionId: 'parent::fork::legacy-receipt', - mergedCount: 2 - } - }) - - expect(service.mergeFork('parent', 'legacy-receipt')).toBe(2) - }) - - it('rejects a malformed stored fork merge receipt instead of reporting an empty merge', () => { - const { table } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/merge', - source: { type: 'fork', id: 'malformed-receipt', seq: 0 }, - provenanceKey: 'fork:parent:malformed-receipt:merge:event', - data: { - forkId: 'malformed-receipt', - forkSessionId: 'parent::fork::malformed-receipt', - mergedCount: 'two' - } - }) - - expect(() => service.mergeFork('parent', 'malformed-receipt')).toThrow( - 'Stored fork merge receipt is malformed' - ) - }) - - itIfSqlite('rolls back copied fork entries and the receipt when merge fails', () => { - const db = new DatabaseCtor(':memory:') - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('parent') - const fork = service.createFork('parent', 'atomic') - service.appendForkMessageRecord( - fork, - createRecord({ id: 'first', orderSeq: 1, sessionId: 'ignored' }) - ) - service.appendForkMessageRecord( - fork, - createRecord({ id: 'second', orderSeq: 2, sessionId: 'ignored' }) - ) - - const append = table.append.bind(table) - let copiedEntryCount = 0 - const appendSpy = vi.spyOn(table, 'append').mockImplementation((input) => { - if (input.sessionId === 'parent' && input.source?.type === 'fork') { - copiedEntryCount += 1 - if (copiedEntryCount === 2) { - throw new Error('injected merge failure') - } - } - return append(input) - }) - - expect(() => service.mergeFork('parent', 'atomic')).toThrow('injected merge failure') - expect( - table - .getBySession('parent') - .filter((entry) => entry.source_type === 'fork' || entry.name === 'fork/merge') - ).toEqual([]) - - appendSpy.mockRestore() - expect(service.mergeFork('parent', 'atomic')).toBe(2) - expect(service.mergeFork('parent', 'atomic')).toBe(2) - expect( - table.getBySession('parent').filter((entry) => entry.source_type === 'fork') - ).toHaveLength(3) - - db.close() - }) - - itIfSqlite('rejects missing and discarded forks in SQLite', () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - expect(() => service.mergeFork('parent', 'missing-native')).toThrow( - 'Fork missing-native does not exist or has been discarded.' - ) - service.createFork('parent', 'discarded-native') - service.discardFork('parent', 'discarded-native') - expect(() => service.mergeFork('parent', 'discarded-native')).toThrow( - 'Fork discarded-native does not exist or has been discarded.' - ) - expect(table.getBySession('parent').some((entry) => entry.name === 'fork/merge')).toBe(false) - } finally { - db.close() - } - }) - - it('cleans fork search projection on discard without blocking the discard event', () => { - const { table, entries } = createTapeTableMock() - const projectionTable = { - deleteBySession: vi.fn(() => { - throw new Error('projection cleanup failed') - }) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatTapeSearchProjectionTable: projectionTable, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - const fork = service.createFork('s1', 'fork-cleanup') - service.appendForkMessageRecord(fork, createRecord({ id: 'fu-cleanup', sessionId: 'ignored' })) - service.discardFork('s1', 'fork-cleanup') - - expect(table.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) - expect(projectionTable.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) - expect(entries.some((entry) => entry.session_id === fork.forkSessionId)).toBe(false) - expect( - entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') - ).toBe(true) - }) - - it('links a frozen subagent Tape without copying child entries and retries idempotently', () => { - const { table, entries } = createTapeTableMock() - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ sessionId: 'child', name: 'child/result', data: { text: 'done' } }) - const input = { - parentSessionId: 'parent', - childSessionId: 'child', - runId: 'run-1', - taskId: 'task-1', - slotId: 'reviewer', - taskTitle: 'Review', - outcome: 'completed' as const, - resultSummary: 'Done' - } - const first = service.linkSubagentTape(input) - table.appendEvent({ sessionId: 'child', name: 'child/late', data: { text: 'late' } }) - const retry = service.linkSubagentTape(input) - const normalizedRetry = service.linkSubagentTape({ - ...input, - slotId: ' reviewer ', - taskTitle: ' Review ', - resultSummary: ' Done ' - }) - - expect(first).toEqual({ - linkEntry: { sessionId: 'parent', entryId: 2 }, - childSessionId: 'child', - childHeadEntryId: 2, - childEntryCount: 2, - outcome: 'completed' - }) - expect(retry).toEqual(first) - expect(normalizedRetry).toEqual(first) - const links = entries.filter( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - ) - expect(links).toHaveLength(1) - expect(links[0]).toMatchObject({ - source_type: 'subagent', - source_id: 'child', - source_seq: 2 - }) - expect(JSON.parse(links[0].payload_json).data).toMatchObject({ - linkVersion: 2, - childTapeIdentity: expect.stringMatching(/^[a-f0-9]{64}$/) - }) - expect( - entries.some((entry) => entry.session_id === 'parent' && entry.name === 'child/result') - ).toBe(false) - expect(() => service.linkSubagentTape({ ...input, outcome: 'error' })).toThrow( - 'Subagent Tape link conflicts with finalized task run-1/task-1.' - ) - expect(() => service.linkSubagentTape({ ...input, slotId: 'writer' })).toThrow( - 'Subagent Tape link conflicts with finalized task run-1/task-1.' - ) - expect(() => service.linkSubagentTape({ ...input, taskTitle: 'Write' })).toThrow( - 'Subagent Tape link conflicts with finalized task run-1/task-1.' - ) - expect(() => service.linkSubagentTape({ ...input, resultSummary: 'Changed' })).toThrow( - 'Subagent Tape link conflicts with finalized task run-1/task-1.' - ) - - table.ensureBootstrapAnchor('child-2') - expect( - service.linkSubagentTape({ - ...input, - childSessionId: 'child-2', - outcome: 'error' - }) - ).toMatchObject({ - childSessionId: 'child-2', - outcome: 'error' - }) - expect( - entries.filter( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - ) - ).toHaveLength(2) - }) - - it('rejects a stored subagent link whose task identity no longer matches its provenance', () => { - const { table, entries } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - const input = createSubagentLinkInput('parent', 'child') - service.linkSubagentTape(input) - - const link = entries.find( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - ) - if (!link) { - throw new Error('Expected subagent Tape link fixture.') - } - const payload = JSON.parse(link.payload_json) as { - data?: Record - } - link.payload_json = JSON.stringify({ - ...payload, - data: { ...payload.data, runId: 'different-run' } - }) - - expect(() => service.linkSubagentTape(input)).toThrow( - /Stored subagent Tape link receipt is malformed/ - ) - expect(() => service.getContext('parent', [1], { sourceSessionId: 'child' })).toThrowError( - /not an authorized direct child/ - ) - }) - - it('keeps a version-one link readable while its original unmarked Tape remains present', () => { - const { table, entries } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/result', - data: { text: 'version one compatibility marker' } - }) - const input = createSubagentLinkInput('parent', 'child') - const receipt = service.linkSubagentTape(input) - const link = entries.find( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - )! - const payload = JSON.parse(link.payload_json) - payload.data.linkVersion = 1 - delete payload.data.childTapeIdentity - link.payload_json = JSON.stringify(payload) - entries.find((entry) => entry.session_id === 'child' && entry.entry_id === 1)!.meta_json = '{}' - - expect(service.linkSubagentTape(input)).toEqual(receipt) - expect( - service.search('parent', 'version one compatibility marker', { - scope: 'linked_subagents' - }) - ).toMatchObject([{ sessionId: 'child', entryId: 2 }]) - }) - - it('fails closed when a version-one link points at malformed incarnation metadata', () => { - const { table, entries } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/result', - data: { text: 'malformed incarnation metadata marker' } - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - const link = entries.find( - (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' - )! - const payload = JSON.parse(link.payload_json) - payload.data.linkVersion = 1 - delete payload.data.childTapeIdentity - link.payload_json = JSON.stringify(payload) - entries.find((entry) => entry.session_id === 'child' && entry.entry_id === 1)!.meta_json = '{' - - expect(() => - service.search('parent', 'malformed incarnation metadata marker', { - scope: 'linked_subagents' - }) - ).toThrowError(/Linked Tape child is unavailable/) - }) - - it('searches direct linked children at frozen heads with one global limit', () => { - const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child-a', session_kind: 'subagent', parent_session_id: 'parent' }, - { id: 'child-b', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child-a') - table.ensureBootstrapAnchor('child-b') - table.appendEvent({ - sessionId: 'parent', - name: 'parent/note', - data: { text: 'shared needle from parent' }, - createdAt: 150 - }) - table.appendEvent({ - sessionId: 'child-a', - name: 'child/result', - data: { text: 'shared needle from A' }, - createdAt: 100 - }) - table.appendEvent({ - sessionId: 'child-b', - name: 'child/result', - data: { text: 'shared needle from B' }, - createdAt: 200 - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child-a')) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child-b')) - table.appendEvent({ - sessionId: 'child-a', - name: 'child/late', - data: { text: 'shared needle after cutoff' }, - createdAt: 300 - }) - - table.ensureBootstrapAnchor.mockClear() - table.append.mockClear() - const limited = service.search('parent', 'shared needle', { - scope: 'linked_subagents', - limit: 1 - }) - const all = service.search('parent', 'shared needle', { - scope: 'linked_subagents', - limit: 10 - }) - const combined = service.search('parent', 'shared needle', { - scope: 'current_and_linked', - limit: 10 - }) - - expect(limited).toMatchObject([{ sessionId: 'child-b', entryId: 2 }]) - expect(all.map((result) => [result.sessionId, result.entryId])).toEqual([ - ['child-b', 2], - ['child-a', 2] - ]) - expect(combined.map((result) => [result.sessionId, result.entryId])).toEqual([ - ['child-b', 2], - ['parent', 2], - ['child-a', 2] - ]) - expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( - [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 2 } - ], - 'shared needle', - expect.objectContaining({ limit: 1 }) - ) - expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() - expect(table.append).not.toHaveBeenCalled() - }) - - it('falls back all linked sources together when projection coverage is partial', () => { - const { table } = createTapeTableMock() - const projectionTable = { - searchSourcesReadOnly: vi.fn(() => ({ - coveredSources: [{ sessionId: 'child-a', maxEntryId: 2 }], - rows: [ - { - session_id: 'child-a', - entry_id: 2, - kind: 'event', - name: 'child/result', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'shared partial needle', - summary_text: 'shared partial needle from A', - refs_json: '{}', - created_at: 100, - score: -100 - } - ] - })) - } - const { service } = createLinkedTapeService( - table, - [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child-a', session_kind: 'subagent', parent_session_id: 'parent' }, - { id: 'child-b', session_kind: 'subagent', parent_session_id: 'parent' } - ], - projectionTable - ) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child-a') - table.ensureBootstrapAnchor('child-b') - table.appendEvent({ - sessionId: 'child-a', - name: 'child/result', - data: { text: 'shared partial needle from A' }, - createdAt: 100 - }) - table.appendEvent({ - sessionId: 'child-b', - name: 'child/result', - data: { text: 'shared partial needle from B' }, - createdAt: 200 - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child-a')) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child-b')) - table.searchEffectiveSourcesAtHeads.mockClear() - - const hits = service.search('parent', 'shared partial needle', { - scope: 'linked_subagents', - limit: 1 - }) - - expect(hits).toMatchObject([{ sessionId: 'child-b', entryId: 2 }]) - expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( - [ - { sessionId: 'child-a', maxEntryId: 2 }, - { sessionId: 'child-b', maxEntryId: 2 } - ], - 'shared partial needle', - expect.objectContaining({ limit: 1 }) - ) - }) - - it('uses an exact frozen projection read-only after the child appends a late tail', () => { - const { table } = createTapeTableMock() - const projectionTable = { - searchSourcesReadOnly: vi.fn((sources: any[]) => ({ - coveredSources: sources, - rows: [ - { - session_id: 'child', - entry_id: 2, - kind: 'event', - name: 'child/result', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'frozen projection needle', - summary_text: 'frozen projection needle', - refs_json: '{}', - created_at: 100, - score: -1 - } - ] - })), - replaceSession: vi.fn(), - appendSession: vi.fn(), - getByEntryIds: vi.fn().mockReturnValue([]) - } - const { service } = createLinkedTapeService( - table, - [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ], - projectionTable - ) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/result', - data: { text: 'frozen projection needle' }, - createdAt: 100 - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - table.appendEvent({ - sessionId: 'child', - name: 'child/late', - data: { text: 'frozen projection needle late' }, - createdAt: 200 - }) - table.searchEffectiveSourcesAtHeads.mockClear() - - const hits = service.search('parent', 'frozen projection needle', { - scope: 'linked_subagents' - }) - - expect(projectionTable.searchSourcesReadOnly).toHaveBeenCalledWith( - [{ sessionId: 'child', maxEntryId: 2 }], - 'frozen projection needle', - expect.any(Object) - ) - expect(hits).toMatchObject([{ sessionId: 'child', entryId: 2 }]) - expect(table.searchEffectiveSourcesAtHeads).not.toHaveBeenCalled() - expect(projectionTable.replaceSession).not.toHaveBeenCalled() - expect(projectionTable.appendSession).not.toHaveBeenCalled() - }) - - it('deduplicates repeated child links at the newest finalized snapshot', () => { - const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/first', - data: { text: 'first snapshot marker' } - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - table.appendEvent({ - sessionId: 'child', - name: 'child/second', - data: { text: 'second snapshot marker' } - }) - service.linkSubagentTape({ - ...createSubagentLinkInput('parent', 'child'), - runId: 'run-child-second', - taskId: 'task-child-second' - }) - - expect( - service.search('parent', 'second snapshot marker', { scope: 'linked_subagents' }) - ).toMatchObject([{ sessionId: 'child', entryId: 3 }]) - expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( - [{ sessionId: 'child', maxEntryId: 3 }], - 'second snapshot marker', - expect.any(Object) - ) - }) - - it('expands linked context within one source and never crosses the frozen head', () => { - const { table } = createTapeTableMock() - const projectionTable = { - getByEntryIds: vi.fn(() => [ - { - session_id: 'child', - entry_id: 2, - kind: 'event', - name: 'child/target', - source_type: null, - source_id: null, - source_seq: null, - search_text: 'stale projection text', - summary_text: 'stale projection summary', - refs_json: '{"stale":true}', - created_at: 100 - } - ]) - } - const { service } = createLinkedTapeService( - table, - [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ], - projectionTable - ) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/target', - data: { text: 'target evidence' }, - createdAt: 100 - }) - table.appendEvent({ - sessionId: 'child', - name: 'child/neighbor', - data: { text: 'neighbor evidence' }, - createdAt: 110 - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - table.appendEvent({ - sessionId: 'child', - name: 'child/late', - data: { text: 'late evidence' }, - createdAt: 120 - }) - table.append.mockClear() - - const context = service.getContext('parent', [2], { - sourceSessionId: 'child', - before: 0, - after: 5 - }) - - expect(context).toMatchObject({ - sessionId: 'parent', - sourceSessionId: 'child', - requestedEntryIds: [2], - matchedEntryIds: [2] - }) - expect(context.entries.map((entry) => entry.entryId)).toEqual([2, 3]) - expect(context.entries[0].summary).toContain('target evidence') - expect(context.entries[0].summary).not.toContain('stale projection') - expect(table.getEffectiveContextRowsAtHead).toHaveBeenCalledWith( - { sessionId: 'child', maxEntryId: 3 }, - [2], - expect.objectContaining({ before: 0, after: 5 }) - ) - expect(projectionTable.getByEntryIds).not.toHaveBeenCalled() - expect(table.append).not.toHaveBeenCalled() - }) - - it('rejects sibling and unlinked source ids even when a forged link event exists', () => { - const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'other-parent', session_kind: 'regular', parent_session_id: null }, - { id: 'sibling', session_kind: 'subagent', parent_session_id: 'other-parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('sibling') - service.linkSubagentTape(createSubagentLinkInput('parent', 'sibling')) - - let error: unknown - try { - service.getContext('parent', [1], { sourceSessionId: 'sibling' }) - } catch (caught) { - error = caught - } - expect(error).toMatchObject({ - code: 'linked_tape_unauthorized', - parentSessionId: 'parent', - sourceSessionId: 'sibling' - }) - }) - - it('reports a finalized linked Tape as unavailable after its durable session is deleted', () => { - const { table } = createTapeTableMock() - const { service, sessionById } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - sessionById.delete('child') - - let error: unknown - try { - service.search('parent', 'anything', { scope: 'linked_subagents' }) - } catch (caught) { - error = caught - } - expect(error).toMatchObject({ - code: 'linked_tape_unavailable', - parentSessionId: 'parent', - sourceSessionId: 'child' - }) - }) - - it('rejects a rebuilt child Tape until a new incarnation is explicitly linked', () => { - const { table } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/original', - data: { text: 'original incarnation marker' } - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - - table.deleteBySession('child') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/rebuilt', - data: { text: 'rebuilt incarnation marker' } - }) - expect( - service.search('child', 'rebuilt incarnation marker', { scope: 'current' }) - ).toHaveLength(1) - expect(() => - service.search('parent', 'rebuilt incarnation marker', { scope: 'linked_subagents' }) - ).toThrowError(/Linked Tape child is unavailable/) - - service.linkSubagentTape({ - ...createSubagentLinkInput('parent', 'child'), - runId: 'run-rebuilt-child', - taskId: 'task-rebuilt-child' - }) - expect( - service.search('parent', 'rebuilt incarnation marker', { scope: 'linked_subagents' }) - ).toMatchObject([{ sessionId: 'child', entryId: 2 }]) - }) - - itIfSqlite('detects linked Tape replacement after entry ids are reused in SQLite', () => { - const db = new DatabaseCtor(':memory:') - try { - const table = new DeepChatTapeEntriesTable(db) - table.createTable() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/original', - data: { text: 'native original incarnation' } - }) - service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) - - table.deleteBySession('child') - table.ensureBootstrapAnchor('child') - table.appendEvent({ - sessionId: 'child', - name: 'child/rebuilt', - data: { text: 'native rebuilt incarnation' } - }) - - expect(() => - service.search('parent', 'native rebuilt incarnation', { scope: 'linked_subagents' }) - ).toThrowError(/Linked Tape child is unavailable/) - } finally { - db.close() - } - }) - - it('reads legacy external merge links and keeps legacy discard audit-only', () => { - const { table, entries } = createTapeTableMock() - const { service } = createLinkedTapeService(table, [ - { id: 'parent', session_kind: 'regular', parent_session_id: null }, - { id: 'legacy-child', session_kind: 'subagent', parent_session_id: 'parent' }, - { id: 'malformed-child', session_kind: 'subagent', parent_session_id: 'parent' }, - { id: 'discarded-child', session_kind: 'subagent', parent_session_id: 'parent' } - ]) - table.ensureBootstrapAnchor('parent') - table.ensureBootstrapAnchor('legacy-child') - table.ensureBootstrapAnchor('malformed-child') - table.ensureBootstrapAnchor('discarded-child') - const legacyBootstrap = entries.find( - (entry) => entry.session_id === 'legacy-child' && entry.entry_id === 1 - )! - legacyBootstrap.meta_json = '{}' - table.appendEvent({ - sessionId: 'legacy-child', - name: 'child/result', - data: { text: 'legacy link needle' } - }) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/merge', - source: { type: 'fork', id: 'legacy-child', seq: 0 }, - provenanceKey: 'fork:parent:legacy-child:external-merge:event', - data: { - forkId: 'legacy-child', - forkSessionId: 'legacy-child', - referencedEntryCount: 2, - status: 'completed' - } - }) - table.appendEvent({ - sessionId: 'malformed-child', - name: 'child/result', - data: { text: 'malformed legacy link needle' } - }) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/merge', - source: { type: 'fork', id: 'malformed-child', seq: 1 }, - provenanceKey: 'fork:parent:malformed-child:external-merge:event', - data: { - forkId: 'malformed-child', - forkSessionId: 'malformed-child', - referencedEntryCount: 2, - status: 'completed' - } - }) - table.appendEvent({ - sessionId: 'parent', - name: 'fork/discard', - source: { type: 'fork', id: 'discarded-child', seq: 0 }, - provenanceKey: 'fork:parent:discarded-child:external-discard:event', - data: { - forkId: 'discarded-child', - forkSessionId: 'discarded-child', - status: 'cancelled' - } - }) - - expect( - service.search('parent', 'legacy link needle', { scope: 'linked_subagents' }) - ).toMatchObject([{ sessionId: 'legacy-child', entryId: 2 }]) - expect( - service.search('parent', 'malformed legacy link needle', { scope: 'linked_subagents' }) - ).toEqual([]) - let error: unknown - try { - service.getContext('parent', [1], { sourceSessionId: 'discarded-child' }) - } catch (caught) { - error = caught - } - expect(error).toMatchObject({ code: 'linked_tape_unauthorized' }) - }) - - it('uses effective message facts after replacement and retraction events', () => { - const { table, entries } = createTapeTableMock() - const original = createRecord({ id: 'u1', orderSeq: 1 }) - const messageStore = { - getMessages: vi.fn().mockReturnValue([original]) - } - const service = new SessionTape({ - deepchatTapeEntriesTable: table, - deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } - } as any) - - service.ensureSessionTapeReady('s1', messageStore as any) - appendMessageReplacementToTape( - table as any, - createRecord({ - id: 'u1', - orderSeq: 1, - content: JSON.stringify({ - text: 'edited', - files: [], - links: [], - search: false, - think: false - }), - updatedAt: 300 - }), - 'test_edit' - ) - - expect(JSON.parse(service.getMessageRecords('s1')[0].content).text).toBe('edited') - expect(service.search('s1', 'hello', { kinds: ['message'] })).toEqual([]) - expect(service.search('s1', 'edited', { kinds: ['message'] })).toHaveLength(1) - expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) - - appendMessageRetractionToTape(table as any, service.getMessageRecords('s1')[0], 'test_delete') - - expect(service.getMessageRecords('s1')).toEqual([]) - expect(service.search('s1', 'edited', { kinds: ['message'] })).toEqual([]) - }) - - it('appends non-idempotent retractions without generated provenance keys', () => { - const { table, entries } = createTapeTableMock() - const record = createRecord({ id: 'u1' }) - - appendMessageRetractionToTape(table as any, record, 'first_delete') - appendMessageRetractionToTape(table as any, record, 'second_delete') - - const retractions = entries.filter((entry) => entry.name === 'message/retracted') - expect(retractions).toHaveLength(2) - expect(retractions.map((entry) => entry.provenance_key)).toEqual([null, null]) - }) -}) diff --git a/test/main/session/data/tapeFacts.test.ts b/test/main/session/data/tapeFacts.test.ts index 6dbfcf63c5..98e10024c6 100644 --- a/test/main/session/data/tapeFacts.test.ts +++ b/test/main/session/data/tapeFacts.test.ts @@ -2,7 +2,10 @@ import { describe, expect, it } from 'vitest' import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' import { appendMessageRecordToTape, appendToolFactsToTape } from '@/session/data/tapeFacts' import { buildEffectiveTapeView } from '@/session/data/tapeEffectiveView' -import { tapeToolRank } from '@/session/data/tables/deepchatTapeEffectiveSemantics' +import { + messageRecordHasFinalToolUse, + tapeToolRank +} from '@/session/data/tables/deepchatTapeEffectiveSemantics' import type { DeepChatTapeEntryRow } from '@/session/data/tables/deepchatTapeEntries' function createTable() { @@ -162,6 +165,26 @@ describe('appendToolFactsToTape', () => { expect(toolCallIds).toEqual(['done1']) }) + it('ignores malformed block elements without shifting valid tool provenance', () => { + const table = createTable() + const record = assistantRecord([ + null, + 'malformed', + toolCallBlock('success', 'valid', 'done') + ] as unknown as AssistantMessageBlock[]) + + expect(messageRecordHasFinalToolUse(record)).toBe(true) + expect(appendToolFactsToTape(table as any, record, 'live', 'tool_loop')).toBe(2) + expect(table.rows.filter((row) => row.kind === 'tool_call')).toMatchObject([ + { source_id: 'a1:valid', source_seq: 2 } + ]) + expect( + messageRecordHasFinalToolUse( + assistantRecord([null, 42] as unknown as AssistantMessageBlock[]) + ) + ).toBe(false) + }) + it('dedupes a mid-loop snapshot and the finalize write to one effective tool fact', () => { const table = createTable() const pending = assistantRecord([toolCallBlock('success', 'tc1', 'result')], { diff --git a/test/main/session/data/tapeFork.test.ts b/test/main/session/data/tapeFork.test.ts new file mode 100644 index 0000000000..be7da20c8a --- /dev/null +++ b/test/main/session/data/tapeFork.test.ts @@ -0,0 +1,459 @@ +import { + describe, + expect, + it, + vi, + SessionTape, + DeepChatTapeEntriesTable, + SqliteTapeLifecycleAdapter, + DatabaseCtor, + itIfSqlite, + createTapeTableMock, + createRecord +} from './tapeTestHarness' + +describe('SessionTape forks', () => { + it('keeps fork writes isolated until merge and discards fork entries on discard', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn() }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const fork = service.createFork('s1', 'fork-1') + service.appendForkMessageRecord(fork, createRecord({ id: 'fu1', sessionId: 'ignored' })) + + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'message/user') + ).toBe(false) + + const mergedCount = service.mergeFork('s1', 'fork-1') + + expect(mergedCount).toBe(1) + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'message/user') + ).toBe(true) + expect(entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/merge')).toBe( + true + ) + expect(entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/start')).toBe( + false + ) + + const discardFork = service.createFork('s1', 'fork-2') + service.appendForkMessageRecord(discardFork, createRecord({ id: 'fu2', sessionId: 'ignored' })) + service.discardFork('s1', 'fork-2') + + expect(entries.some((entry) => entry.session_id === discardFork.forkSessionId)).toBe(false) + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') + ).toBe(true) + }) + + it('records exact fork heads and keeps retries bounded to the first merge', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('parent') + table.appendEvent({ sessionId: 'parent', name: 'parent/tail', data: { value: 1 } }) + const fork = service.createFork('parent', 'bounded') + table.appendEvent({ sessionId: 'parent', name: 'parent/later', data: { value: 2 } }) + const retriedFork = service.createFork('parent', 'bounded') + service.appendForkMessageRecord(fork, createRecord({ id: 'first', sessionId: 'ignored' })) + + const forkStart = entries.find( + (entry) => entry.session_id === fork.forkSessionId && entry.name === 'fork/start' + ) + expect(fork.parentHeadEntryId).toBe(2) + expect(retriedFork.parentHeadEntryId).toBe(2) + expect(JSON.parse(forkStart.payload_json).state.parentHeadEntryId).toBe(2) + + const getBoundedEntries = table.getBySessionUpToEntryId.getMockImplementation()! + table.getBySessionUpToEntryId.mockImplementationOnce( + (sessionId: string, maxEntryId: number) => { + table.appendEvent({ + sessionId: fork.forkSessionId, + name: 'fork/late-tail', + data: { value: 2 } + }) + return getBoundedEntries(sessionId, maxEntryId) + } + ) + + expect(service.mergeFork('parent', 'bounded')).toBe(1) + expect(service.mergeFork('parent', 'bounded')).toBe(1) + expect(() => service.createFork('parent', 'bounded')).toThrow( + 'Fork bounded has already been merged and cannot be reused.' + ) + + const parentEntries = entries.filter((entry) => entry.session_id === 'parent') + expect(parentEntries.filter((entry) => entry.source_type === 'fork')).toHaveLength(2) + expect(parentEntries.some((entry) => entry.name === 'fork/late-tail')).toBe(false) + const receipt = parentEntries.find((entry) => entry.name === 'fork/merge')! + expect(JSON.parse(receipt.payload_json).data).toMatchObject({ + forkHeadEntryId: 3, + mergedCount: 1 + }) + }) + + it('rolls back mocked fork merge writes when a copied entry fails', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('parent', 'mock-atomic') + service.appendForkMessageRecord( + fork, + createRecord({ id: 'first', orderSeq: 1, sessionId: 'ignored' }) + ) + service.appendForkMessageRecord( + fork, + createRecord({ id: 'second', orderSeq: 2, sessionId: 'ignored' }) + ) + + const append = table.append.getMockImplementation()! + let copiedEntryCount = 0 + table.append.mockImplementation((input: any) => { + if (input.sessionId === 'parent' && input.source?.type === 'fork') { + copiedEntryCount += 1 + if (copiedEntryCount === 2) { + throw new Error('injected merge failure') + } + } + return append(input) + }) + + expect(() => service.mergeFork('parent', 'mock-atomic')).toThrow('injected merge failure') + expect( + entries.filter( + (entry) => + entry.session_id === 'parent' && + (entry.source_type === 'fork' || entry.name === 'fork/merge') + ) + ).toEqual([]) + }) + + it('rolls back copied fork entries when the merge receipt fails', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('parent', 'receipt-failure') + service.appendForkMessageRecord(fork, createRecord({ id: 'first', sessionId: 'ignored' })) + + const appendEvent = table.appendEvent.getMockImplementation()! + table.appendEvent.mockImplementation((input: any) => { + if (input.sessionId === 'parent' && input.name === 'fork/merge') { + throw new Error('injected receipt failure') + } + return appendEvent(input) + }) + + expect(() => service.mergeFork('parent', 'receipt-failure')).toThrow('injected receipt failure') + expect( + entries.filter( + (entry) => + entry.session_id === 'parent' && + (entry.source_type === 'fork' || entry.name === 'fork/merge') + ) + ).toEqual([]) + }) + + it('commits one idempotent receipt for an empty fork', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + service.createFork('parent', 'empty') + + expect(service.mergeFork('parent', 'empty')).toBe(0) + expect(service.mergeFork('parent', 'empty')).toBe(0) + const parentEntries = entries.filter((entry) => entry.session_id === 'parent') + expect(parentEntries).toHaveLength(1) + expect(parentEntries[0].name).toBe('fork/merge') + expect(JSON.parse(parentEntries[0].payload_json).data).toMatchObject({ + forkHeadEntryId: 2, + mergedCount: 0 + }) + }) + + it('rejects missing and discarded forks without committing an empty merge receipt', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn() }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + expect(() => service.mergeFork('parent', 'missing')).toThrow( + 'Fork missing does not exist or has been discarded.' + ) + + service.createFork('parent', 'discarded') + service.discardFork('parent', 'discarded') + expect(() => service.mergeFork('parent', 'discarded')).toThrow( + 'Fork discarded does not exist or has been discarded.' + ) + expect( + entries.filter((entry) => entry.session_id === 'parent' && entry.name === 'fork/merge') + ).toEqual([]) + }) + + it('returns an existing merge receipt after the merged fork Tape is cleaned up', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('parent', 'cleanup-after-merge') + service.appendForkMessageRecord(fork, createRecord({ id: 'merged', sessionId: 'ignored' })) + + expect(service.mergeFork('parent', 'cleanup-after-merge')).toBe(1) + table.deleteBySession(fork.forkSessionId) + expect(service.mergeFork('parent', 'cleanup-after-merge')).toBe(1) + }) + + it('merges a legacy fork start that predates the parent head field', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('parent', 'legacy-start') + const start = entries.find( + (entry) => entry.session_id === fork.forkSessionId && entry.name === 'fork/start' + )! + const payload = JSON.parse(start.payload_json) + delete payload.state.parentHeadEntryId + start.payload_json = JSON.stringify(payload) + service.appendForkMessageRecord(fork, createRecord({ id: 'legacy', sessionId: 'ignored' })) + + expect(service.mergeFork('parent', 'legacy-start')).toBe(1) + }) + + it('accepts a valid legacy fork merge receipt without a frozen head', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/merge', + source: { type: 'fork', id: 'legacy-receipt', seq: 0 }, + provenanceKey: 'fork:parent:legacy-receipt:merge:event', + data: { + forkId: 'legacy-receipt', + forkSessionId: 'parent::fork::legacy-receipt', + mergedCount: 2 + } + }) + + expect(service.mergeFork('parent', 'legacy-receipt')).toBe(2) + }) + + it('rejects a malformed stored fork merge receipt instead of reporting an empty merge', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/merge', + source: { type: 'fork', id: 'malformed-receipt', seq: 0 }, + provenanceKey: 'fork:parent:malformed-receipt:merge:event', + data: { + forkId: 'malformed-receipt', + forkSessionId: 'parent::fork::malformed-receipt', + mergedCount: 'two' + } + }) + + expect(() => service.mergeFork('parent', 'malformed-receipt')).toThrow( + 'Stored fork merge receipt is malformed' + ) + }) + + it('keeps external fork receipt identity fields authoritative over metadata', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.ensureBootstrapAnchor('external-child') + + const merge = service.recordExternalForkMerge('parent', 'external-child', 'external-child', { + forkId: 'spoofed', + forkSessionId: 'spoofed-session', + referencedEntryCount: 999, + taskId: 'task-1' + }) + const discard = service.recordExternalForkDiscard( + 'parent', + 'external-child', + 'external-child', + { + forkId: 'spoofed', + forkSessionId: 'spoofed-session', + taskId: 'task-1' + } + ) + + expect(JSON.parse(merge.payload_json).data).toEqual({ + forkId: 'external-child', + forkSessionId: 'external-child', + referencedEntryCount: 1, + taskId: 'task-1' + }) + expect(JSON.parse(discard.payload_json).data).toEqual({ + forkId: 'external-child', + forkSessionId: 'external-child', + taskId: 'task-1' + }) + }) + + itIfSqlite('rolls back copied fork entries and the receipt when merge fails', () => { + const db = new DatabaseCtor(':memory:') + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('parent') + const fork = service.createFork('parent', 'atomic') + service.appendForkMessageRecord( + fork, + createRecord({ id: 'first', orderSeq: 1, sessionId: 'ignored' }) + ) + service.appendForkMessageRecord( + fork, + createRecord({ id: 'second', orderSeq: 2, sessionId: 'ignored' }) + ) + + const append = table.append.bind(table) + let copiedEntryCount = 0 + const appendSpy = vi.spyOn(table, 'append').mockImplementation((input) => { + if (input.sessionId === 'parent' && input.source?.type === 'fork') { + copiedEntryCount += 1 + if (copiedEntryCount === 2) { + throw new Error('injected merge failure') + } + } + return append(input) + }) + + expect(() => service.mergeFork('parent', 'atomic')).toThrow('injected merge failure') + expect( + table + .getBySession('parent') + .filter((entry) => entry.source_type === 'fork' || entry.name === 'fork/merge') + ).toEqual([]) + + appendSpy.mockRestore() + expect(service.mergeFork('parent', 'atomic')).toBe(2) + expect(service.mergeFork('parent', 'atomic')).toBe(2) + expect( + table.getBySession('parent').filter((entry) => entry.source_type === 'fork') + ).toHaveLength(3) + + db.close() + }) + + itIfSqlite('rejects missing and discarded forks in SQLite', () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn() }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + expect(() => service.mergeFork('parent', 'missing-native')).toThrow( + 'Fork missing-native does not exist or has been discarded.' + ) + service.createFork('parent', 'discarded-native') + service.discardFork('parent', 'discarded-native') + expect(() => service.mergeFork('parent', 'discarded-native')).toThrow( + 'Fork discarded-native does not exist or has been discarded.' + ) + expect(table.getBySession('parent').some((entry) => entry.name === 'fork/merge')).toBe(false) + } finally { + db.close() + } + }) + + it('keeps a failed fork cleanup isolated and makes its discard receipt fail closed', () => { + const { table, entries } = createTapeTableMock() + const projectionTable = { + deleteBySession: vi.fn(() => { + throw new Error('projection cleanup failed') + }) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const fork = service.createFork('s1', 'fork-cleanup') + service.appendForkMessageRecord(fork, createRecord({ id: 'fu-cleanup', sessionId: 'ignored' })) + service.discardFork('s1', 'fork-cleanup') + + expect(table.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) + expect(projectionTable.deleteBySession).toHaveBeenCalledWith(fork.forkSessionId) + expect(entries.some((entry) => entry.session_id === fork.forkSessionId)).toBe(true) + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') + ).toBe(true) + expect(() => service.mergeFork('s1', 'fork-cleanup')).toThrow( + 'Fork fork-cleanup does not exist or has been discarded.' + ) + expect(() => service.createFork('s1', 'fork-cleanup')).toThrow( + 'Fork fork-cleanup has been discarded and cannot be reused.' + ) + }) + + it('restores fork entries when the discard receipt cannot be appended', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn() }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const fork = service.createFork('s1', 'receipt-cleanup') + service.appendForkMessageRecord( + fork, + createRecord({ id: 'receipt-message', sessionId: 'ignored' }) + ) + const appendEvent = table.appendEvent.getMockImplementation()! + table.appendEvent.mockImplementation((input: any) => { + if (input.sessionId === 's1' && input.name === 'fork/discard') { + throw new Error('discard receipt failed') + } + return appendEvent(input) + }) + + expect(() => service.discardFork('s1', 'receipt-cleanup')).toThrow('discard receipt failed') + expect(entries.some((entry) => entry.session_id === fork.forkSessionId)).toBe(true) + expect( + entries.some((entry) => entry.session_id === 's1' && entry.name === 'fork/discard') + ).toBe(false) + }) +}) diff --git a/test/main/session/data/tapeLifecycle.test.ts b/test/main/session/data/tapeLifecycle.test.ts new file mode 100644 index 0000000000..87a3493837 --- /dev/null +++ b/test/main/session/data/tapeLifecycle.test.ts @@ -0,0 +1,429 @@ +import { + DatabaseCtor, + DeepChatTapeEntriesTable, + DeepChatTapeSearchProjectionTable, + describe, + expect, + it, + itIfSqlite, + SessionTape, + SqliteTapeLifecycleAdapter, + vi +} from './tapeTestHarness' +import { SessionTranscriptMutations } from '@/session/transcriptMutations' + +function createHarness() { + const calls: string[] = [] + const state = { entriesPresent: true, searchPresent: true, bootstrapCount: 0 } + const entryStore = { + runInTransaction: vi.fn((operation: () => unknown) => { + const snapshot = { ...state } + try { + return operation() + } catch (error) { + Object.assign(state, snapshot) + throw error + } + }), + ensureBootstrapAnchor: vi.fn((sessionId: string) => { + calls.push(`bootstrap:${sessionId}`) + state.entriesPresent = true + state.bootstrapCount += 1 + }) + } + const lifecycle = { + deleteBySession: vi.fn((sessionId: string) => { + calls.push(`entries:${sessionId}`) + state.entriesPresent = false + }) + } + const searchProjection = { + deleteBySession: vi.fn((sessionId: string) => { + calls.push(`search:${sessionId}`) + state.searchPresent = false + }) + } + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: lifecycle, + deepchatTapeSearchProjectionTable: searchProjection + } as any) + + return { calls, entryStore, lifecycle, searchProjection, state, tape } +} + +describe('SessionTape lifecycle administration', () => { + it('deletes entries before the search projection', () => { + const { calls, entryStore, tape } = createHarness() + + tape.deleteSessionTape('s1') + + expect(calls).toEqual(['entries:s1', 'search:s1']) + expect(entryStore.runInTransaction).toHaveBeenCalledOnce() + }) + + it('rolls back final Tape deletion when search cleanup fails', () => { + const { searchProjection, state, tape } = createHarness() + searchProjection.deleteBySession.mockImplementationOnce(() => { + throw new Error('search cleanup failed') + }) + + expect(() => tape.deleteSessionTape('s1')).toThrow('search cleanup failed') + expect(state).toEqual({ entriesPresent: true, searchPresent: true, bootstrapCount: 0 }) + }) + + it('rebuilds the bootstrap only after both destructive stores are cleared', () => { + const { calls, entryStore, tape } = createHarness() + + tape.resetSessionTape('s1') + + expect(calls).toEqual(['entries:s1', 'search:s1', 'bootstrap:s1']) + expect(entryStore.runInTransaction).toHaveBeenCalledOnce() + }) + + it('does not create a mixed-generation Tape when projection cleanup fails', () => { + const { entryStore, searchProjection, state, tape } = createHarness() + searchProjection.deleteBySession.mockImplementationOnce(() => { + throw new Error('search cleanup failed') + }) + + expect(() => tape.resetSessionTape('s1')).toThrow('search cleanup failed') + expect(entryStore.ensureBootstrapAnchor).not.toHaveBeenCalled() + expect(state).toEqual({ entriesPresent: true, searchPresent: true, bootstrapCount: 0 }) + }) + + it('rolls a reset back when the replacement bootstrap fails', () => { + const { entryStore, state, tape } = createHarness() + entryStore.ensureBootstrapAnchor.mockImplementationOnce(() => { + throw new Error('bootstrap failed') + }) + + expect(() => tape.resetSessionTape('s1')).toThrow('bootstrap failed') + expect(state).toEqual({ entriesPresent: true, searchPresent: true, bootstrapCount: 0 }) + }) + + itIfSqlite('rolls back a failed reset and creates a fresh incarnation only on retry', () => { + const db = new DatabaseCtor(':memory:') + try { + const entryStore = new DeepChatTapeEntriesTable(db) + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + entryStore.createTable() + searchProjection.createTable() + entryStore.ensureBootstrapAnchor('s1') + const oldBootstrap = entryStore.getBySession('s1')[0] + entryStore.appendEvent({ + sessionId: 's1', + name: 'old/generation', + data: { marker: 'old generation' } + }) + searchProjection.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'event', + name: 'old/generation', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'old generation', + summaryText: 'old generation', + refs: { generation: 'old' }, + createdAt: 100 + } + ], + 2 + ) + const deleteProjection = searchProjection.deleteBySession.bind(searchProjection) + const deleteProjectionSpy = vi + .spyOn(searchProjection, 'deleteBySession') + .mockImplementationOnce(() => { + throw new Error('search cleanup failed') + }) + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: searchProjection + } as any) + + expect(() => tape.resetSessionTape('s1')).toThrow('search cleanup failed') + expect(entryStore.getBySession('s1').map((entry) => entry.name)).toEqual([ + 'session/start', + 'old/generation' + ]) + expect(entryStore.getBySession('s1')[0].meta_json).toBe(oldBootstrap.meta_json) + expect(searchProjection.isCurrent('s1', 2)).toBe(true) + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([2]) + + deleteProjectionSpy.mockImplementation(deleteProjection) + tape.resetSessionTape('s1') + + const newEntries = entryStore.getBySession('s1') + expect(newEntries).toHaveLength(1) + expect(newEntries[0]).toMatchObject({ entry_id: 1, name: 'session/start' }) + expect(newEntries[0].meta_json).not.toBe(oldBootstrap.meta_json) + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([]) + expect(searchProjection.getSessionMeta('s1')).toBeNull() + } finally { + db.close() + } + }) + + itIfSqlite('rolls back partial search projection deletion', () => { + const db = new DatabaseCtor(':memory:') + try { + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + searchProjection.createTable() + searchProjection.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'event', + name: 'old/generation', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'old generation', + summaryText: 'old generation', + refs: { generation: 'old' }, + createdAt: 100 + } + ], + 1 + ) + db.exec(` + CREATE TRIGGER fail_projection_meta_delete + BEFORE DELETE ON deepchat_tape_search_projection_meta + WHEN old.session_id = 's1' + BEGIN + SELECT RAISE(ABORT, 'injected projection cleanup failure'); + END; + `) + + expect(() => searchProjection.deleteBySession('s1')).toThrow( + 'injected projection cleanup failure' + ) + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([1]) + expect(searchProjection.isCurrent('s1', 1)).toBe(true) + } finally { + db.close() + } + }) + + itIfSqlite('rolls back SQLite reset deletion when the replacement bootstrap fails', () => { + const db = new DatabaseCtor(':memory:') + try { + const entryStore = new DeepChatTapeEntriesTable(db) + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + entryStore.createTable() + searchProjection.createTable() + entryStore.ensureBootstrapAnchor('s1') + entryStore.appendEvent({ + sessionId: 's1', + name: 'old/generation', + data: { marker: 'old generation' } + }) + searchProjection.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'event', + name: 'old/generation', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'old generation', + summaryText: 'old generation', + refs: {}, + createdAt: 100 + } + ], + 2 + ) + const bootstrapSpy = vi + .spyOn(entryStore, 'ensureBootstrapAnchor') + .mockImplementationOnce(() => { + throw new Error('bootstrap failed') + }) + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: searchProjection + } as any) + + expect(() => tape.resetSessionTape('s1')).toThrow('bootstrap failed') + expect(entryStore.getBySession('s1').map((entry) => entry.name)).toEqual([ + 'session/start', + 'old/generation' + ]) + expect(searchProjection.isCurrent('s1', 2)).toBe(true) + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([2]) + bootstrapSpy.mockRestore() + } finally { + db.close() + } + }) + + itIfSqlite( + 'rolls transcript cleanup back with a failed Tape reset on one connection', + async () => { + const db = new DatabaseCtor(':memory:') + try { + db.exec(` + CREATE TABLE clear_pending (session_id TEXT NOT NULL); + CREATE TABLE clear_transcript (session_id TEXT NOT NULL); + INSERT INTO clear_pending (session_id) VALUES ('s1'); + INSERT INTO clear_transcript (session_id) VALUES ('s1'); + `) + const entryStore = new DeepChatTapeEntriesTable(db) + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + entryStore.createTable() + searchProjection.createTable() + entryStore.ensureBootstrapAnchor('s1') + entryStore.appendEvent({ + sessionId: 's1', + name: 'old/generation', + data: { marker: 'old generation' } + }) + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: searchProjection + } as any) + vi.spyOn(searchProjection, 'deleteBySession').mockImplementationOnce(() => { + throw new Error('search cleanup failed') + }) + const runtime = { + prepareClearMessages: vi.fn().mockResolvedValue(undefined), + finishClearMessages: vi.fn() + } + const mutations = new SessionTranscriptMutations({ + pendingInputs: { + deleteBySession: () => + db.prepare('DELETE FROM clear_pending WHERE session_id = ?').run('s1') + }, + transcript: { + deleteBySession: () => + db.prepare('DELETE FROM clear_transcript WHERE session_id = ?').run('s1') + }, + settings: { resetTape: () => tape.resetSessionTape('s1') }, + runtime, + runInTransaction: (operation) => db.transaction(operation)() + } as any) + + await expect(mutations.clearMessages('s1')).rejects.toThrow('search cleanup failed') + + expect(db.prepare('SELECT COUNT(*) AS count FROM clear_pending').get()).toEqual({ + count: 1 + }) + expect(db.prepare('SELECT COUNT(*) AS count FROM clear_transcript').get()).toEqual({ + count: 1 + }) + expect(entryStore.getBySession('s1').map((entry) => entry.name)).toEqual([ + 'session/start', + 'old/generation' + ]) + expect(runtime.finishClearMessages).not.toHaveBeenCalled() + } finally { + vi.restoreAllMocks() + db.close() + } + } + ) + + itIfSqlite('drops corrupt FTS state instead of blocking a Tape generation reset', () => { + const db = new DatabaseCtor(':memory:') + try { + const entryStore = new DeepChatTapeEntriesTable(db) + const searchProjection = new DeepChatTapeSearchProjectionTable(db) + entryStore.createTable() + searchProjection.createTable() + if (!searchProjection.hasFtsReadyForTesting()) return + entryStore.ensureBootstrapAnchor('s1') + entryStore.appendEvent({ + sessionId: 's1', + name: 'old/generation', + data: { marker: 'private old generation' } + }) + searchProjection.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'event', + name: 'old/generation', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'private old generation', + summaryText: 'private old generation', + refs: {}, + createdAt: 100 + } + ], + 2 + ) + const tape = new SessionTape({ + deepchatTapeEntriesTable: entryStore, + tapeLifecycle: new SqliteTapeLifecycleAdapter(db), + deepchatTapeSearchProjectionTable: searchProjection + } as any) + const prepare = db.prepare.bind(db) + const prepareSpy = vi.spyOn(db, 'prepare').mockImplementation(((sql: string) => { + if ( + sql.trim().replace(/\s+/g, ' ') === + 'DELETE FROM deepchat_tape_search_fts WHERE session_id = ?' + ) { + throw new Error('injected corrupt FTS delete') + } + return prepare(sql) + }) as typeof db.prepare) + const exec = db.exec.bind(db) + const execSpy = vi.spyOn(db, 'exec').mockImplementation(((sql: string) => { + if (sql.trim() === 'DROP TABLE IF EXISTS deepchat_tape_search_fts') { + throw new Error('injected corrupt FTS drop') + } + return exec(sql) + }) as typeof db.exec) + + expect(() => tape.resetSessionTape('s1')).toThrow('injected corrupt FTS drop') + expect(entryStore.getBySession('s1').map((entry) => entry.name)).toEqual([ + 'session/start', + 'old/generation' + ]) + expect(searchProjection.isCurrent('s1', 2)).toBe(true) + execSpy.mockRestore() + + expect(() => tape.resetSessionTape('s1')).not.toThrow() + prepareSpy.mockRestore() + + expect(entryStore.getBySession('s1')).toMatchObject([{ entry_id: 1, name: 'session/start' }]) + expect(searchProjection.getSessionMeta('s1')).toBeNull() + expect(searchProjection.getProjectedEntryIds('s1')).toEqual([]) + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'deepchat_tape_search_fts'" + ) + .get() + ).toBeUndefined() + expect(tape.search('s1', 'private old generation')).toEqual([]) + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'deepchat_tape_search_fts'" + ) + .get() + ).toEqual({ name: 'deepchat_tape_search_fts' }) + } finally { + vi.restoreAllMocks() + db.close() + } + }) +}) diff --git a/test/main/session/data/tapeLineage.test.ts b/test/main/session/data/tapeLineage.test.ts new file mode 100644 index 0000000000..da26bf7726 --- /dev/null +++ b/test/main/session/data/tapeLineage.test.ts @@ -0,0 +1,759 @@ +import { + describe, + expect, + it, + vi, + SessionTape, + DeepChatTapeEntriesTable, + DatabaseCtor, + itIfSqlite, + createTapeTableMock, + createLinkedTapeService, + createSubagentLinkInput, + SqliteTapeLifecycleAdapter +} from './tapeTestHarness' + +describe('SessionTape lineage', () => { + it('links a frozen subagent Tape without copying child entries and retries idempotently', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'child-2', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ sessionId: 'child', name: 'child/result', data: { text: 'done' } }) + const input = { + parentSessionId: 'parent', + childSessionId: 'child', + runId: 'run-1', + taskId: 'task-1', + slotId: 'reviewer', + taskTitle: 'Review', + outcome: 'completed' as const, + resultSummary: 'Done' + } + const first = service.linkSubagentTape(input) + table.appendEvent({ sessionId: 'child', name: 'child/late', data: { text: 'late' } }) + const retry = service.linkSubagentTape(input) + const normalizedRetry = service.linkSubagentTape({ + ...input, + slotId: ' reviewer ', + taskTitle: ' Review ', + resultSummary: ' Done ' + }) + + expect(first).toEqual({ + linkEntry: { sessionId: 'parent', entryId: 2 }, + childSessionId: 'child', + childHeadEntryId: 2, + childEntryCount: 2, + outcome: 'completed' + }) + expect(retry).toEqual(first) + expect(normalizedRetry).toEqual(first) + const links = entries.filter( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + ) + expect(links).toHaveLength(1) + expect(links[0]).toMatchObject({ + source_type: 'subagent', + source_id: 'child', + source_seq: 2 + }) + expect(JSON.parse(links[0].payload_json).data).toMatchObject({ + linkVersion: 2, + childTapeIdentity: expect.stringMatching(/^[a-f0-9]{64}$/) + }) + expect( + entries.some((entry) => entry.session_id === 'parent' && entry.name === 'child/result') + ).toBe(false) + expect(() => service.linkSubagentTape({ ...input, outcome: 'error' })).toThrow( + 'Subagent Tape link conflicts with finalized task run-1/task-1.' + ) + expect(() => service.linkSubagentTape({ ...input, slotId: 'writer' })).toThrow( + 'Subagent Tape link conflicts with finalized task run-1/task-1.' + ) + expect(() => service.linkSubagentTape({ ...input, taskTitle: 'Write' })).toThrow( + 'Subagent Tape link conflicts with finalized task run-1/task-1.' + ) + expect(() => service.linkSubagentTape({ ...input, resultSummary: 'Changed' })).toThrow( + 'Subagent Tape link conflicts with finalized task run-1/task-1.' + ) + + table.ensureBootstrapAnchor('child-2') + expect( + service.linkSubagentTape({ + ...input, + childSessionId: 'child-2', + outcome: 'error' + }) + ).toMatchObject({ + childSessionId: 'child-2', + outcome: 'error' + }) + expect( + entries.filter( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + ) + ).toHaveLength(2) + }) + + it('rejects a stored subagent link whose task identity no longer matches its provenance', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + const input = createSubagentLinkInput('parent', 'child') + service.linkSubagentTape(input) + + const link = entries.find( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + ) + if (!link) { + throw new Error('Expected subagent Tape link fixture.') + } + const payload = JSON.parse(link.payload_json) as { + data?: Record + } + link.payload_json = JSON.stringify({ + ...payload, + data: { ...payload.data, runId: 'different-run' } + }) + + expect(() => service.linkSubagentTape(input)).toThrow( + /Stored subagent Tape link receipt is malformed/ + ) + expect(() => service.getContext('parent', [1], { sourceSessionId: 'child' })).toThrowError( + /not an authorized direct child/ + ) + }) + + it('keeps a version-one link readable while its original unmarked Tape remains present', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/result', + data: { text: 'version one compatibility marker' } + }) + const input = createSubagentLinkInput('parent', 'child') + const receipt = service.linkSubagentTape(input) + const link = entries.find( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + )! + const payload = JSON.parse(link.payload_json) + payload.data.linkVersion = 1 + delete payload.data.childTapeIdentity + link.payload_json = JSON.stringify(payload) + entries.find((entry) => entry.session_id === 'child' && entry.entry_id === 1)!.meta_json = '{}' + + expect(service.linkSubagentTape(input)).toEqual(receipt) + expect( + service.search('parent', 'version one compatibility marker', { + scope: 'linked_subagents' + }) + ).toMatchObject([{ sessionId: 'child', entryId: 2 }]) + }) + + it('fails closed when a version-one link points at malformed incarnation metadata', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/result', + data: { text: 'malformed incarnation metadata marker' } + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + const link = entries.find( + (entry) => entry.session_id === 'parent' && entry.name === 'subagent/tape_linked' + )! + const payload = JSON.parse(link.payload_json) + payload.data.linkVersion = 1 + delete payload.data.childTapeIdentity + link.payload_json = JSON.stringify(payload) + entries.find((entry) => entry.session_id === 'child' && entry.entry_id === 1)!.meta_json = '{' + + expect(() => + service.search('parent', 'malformed incarnation metadata marker', { + scope: 'linked_subagents' + }) + ).toThrowError(/Linked Tape child is unavailable/) + }) + + it('searches direct linked children at frozen heads with one global limit', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child-a', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'child-b', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child-a') + table.ensureBootstrapAnchor('child-b') + table.appendEvent({ + sessionId: 'parent', + name: 'parent/note', + data: { text: 'shared needle from parent' }, + createdAt: 150 + }) + table.appendEvent({ + sessionId: 'child-a', + name: 'child/result', + data: { text: 'shared needle from A' }, + createdAt: 100 + }) + table.appendEvent({ + sessionId: 'child-b', + name: 'child/result', + data: { text: 'shared needle from B' }, + createdAt: 200 + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child-a')) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child-b')) + table.appendEvent({ + sessionId: 'child-a', + name: 'child/late', + data: { text: 'shared needle after cutoff' }, + createdAt: 300 + }) + + table.ensureBootstrapAnchor.mockClear() + table.append.mockClear() + const limited = service.search('parent', 'shared needle', { + scope: 'linked_subagents', + limit: 1 + }) + const all = service.search('parent', 'shared needle', { + scope: 'linked_subagents', + limit: 10 + }) + const combined = service.search('parent', 'shared needle', { + scope: 'current_and_linked', + limit: 10 + }) + + expect(limited).toMatchObject([{ sessionId: 'child-b', entryId: 2 }]) + expect(all.map((result) => [result.sessionId, result.entryId])).toEqual([ + ['child-b', 2], + ['child-a', 2] + ]) + expect(combined.map((result) => [result.sessionId, result.entryId])).toEqual([ + ['child-b', 2], + ['parent', 2], + ['child-a', 2] + ]) + expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( + [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 2 } + ], + 'shared needle', + expect.objectContaining({ limit: 1 }) + ) + expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() + expect(table.append).not.toHaveBeenCalled() + }) + + it('falls back all linked sources together when projection coverage is partial', () => { + const { table } = createTapeTableMock() + const projectionTable = { + searchSourcesReadOnly: vi.fn(() => ({ + coveredSources: [{ sessionId: 'child-a', maxEntryId: 2 }], + rows: [ + { + session_id: 'child-a', + entry_id: 2, + kind: 'event', + name: 'child/result', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'shared partial needle', + summary_text: 'shared partial needle from A', + refs_json: '{}', + created_at: 100, + score: -100 + } + ] + })) + } + const { service } = createLinkedTapeService( + table, + [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child-a', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'child-b', session_kind: 'subagent', parent_session_id: 'parent' } + ], + projectionTable + ) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child-a') + table.ensureBootstrapAnchor('child-b') + table.appendEvent({ + sessionId: 'child-a', + name: 'child/result', + data: { text: 'shared partial needle from A' }, + createdAt: 100 + }) + table.appendEvent({ + sessionId: 'child-b', + name: 'child/result', + data: { text: 'shared partial needle from B' }, + createdAt: 200 + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child-a')) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child-b')) + table.searchEffectiveSourcesAtHeads.mockClear() + + const hits = service.search('parent', 'shared partial needle', { + scope: 'linked_subagents', + limit: 1 + }) + + expect(hits).toMatchObject([{ sessionId: 'child-b', entryId: 2 }]) + expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( + [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 2 } + ], + 'shared partial needle', + expect.objectContaining({ limit: 1 }) + ) + }) + + it('uses an exact frozen projection read-only after the child appends a late tail', () => { + const { table } = createTapeTableMock() + const projectionTable = { + searchSourcesReadOnly: vi.fn((sources: any[]) => ({ + coveredSources: sources, + rows: [ + { + session_id: 'child', + entry_id: 2, + kind: 'event', + name: 'child/result', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'frozen projection needle', + summary_text: 'frozen projection needle', + refs_json: '{}', + created_at: 100, + score: -1 + } + ] + })), + replaceSession: vi.fn(), + appendSession: vi.fn(), + getByEntryIds: vi.fn().mockReturnValue([]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) + } + const { service } = createLinkedTapeService( + table, + [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ], + projectionTable + ) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/result', + data: { text: 'frozen projection needle' }, + createdAt: 100 + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + table.appendEvent({ + sessionId: 'child', + name: 'child/late', + data: { text: 'frozen projection needle late' }, + createdAt: 200 + }) + table.searchEffectiveSourcesAtHeads.mockClear() + + const hits = service.search('parent', 'frozen projection needle', { + scope: 'linked_subagents' + }) + + expect(projectionTable.searchSourcesReadOnly).toHaveBeenCalledWith( + [{ sessionId: 'child', maxEntryId: 2 }], + 'frozen projection needle', + expect.any(Object) + ) + expect(hits).toMatchObject([{ sessionId: 'child', entryId: 2 }]) + expect(table.searchEffectiveSourcesAtHeads).not.toHaveBeenCalled() + expect(projectionTable.replaceSession).not.toHaveBeenCalled() + expect(projectionTable.appendSession).not.toHaveBeenCalled() + }) + + it('deduplicates repeated child links at the newest finalized snapshot', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/first', + data: { text: 'first snapshot marker' } + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + table.appendEvent({ + sessionId: 'child', + name: 'child/second', + data: { text: 'second snapshot marker' } + }) + service.linkSubagentTape({ + ...createSubagentLinkInput('parent', 'child'), + runId: 'run-child-second', + taskId: 'task-child-second' + }) + + expect( + service.search('parent', 'second snapshot marker', { scope: 'linked_subagents' }) + ).toMatchObject([{ sessionId: 'child', entryId: 3 }]) + expect(table.searchEffectiveSourcesAtHeads).toHaveBeenCalledWith( + [{ sessionId: 'child', maxEntryId: 3 }], + 'second snapshot marker', + expect.any(Object) + ) + }) + + it('expands linked context within one source and never crosses the frozen head', () => { + const { table } = createTapeTableMock() + const projectionTable = { + getByEntryIdsIfCurrent: vi.fn(() => [ + { + session_id: 'child', + entry_id: 2, + kind: 'event', + name: 'child/target', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'stale projection text', + summary_text: 'stale projection summary', + refs_json: '{"stale":true}', + created_at: 100 + } + ]) + } + const { service } = createLinkedTapeService( + table, + [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ], + projectionTable + ) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/target', + data: { text: 'target evidence' }, + createdAt: 100 + }) + table.appendEvent({ + sessionId: 'child', + name: 'child/neighbor', + data: { text: 'neighbor evidence' }, + createdAt: 110 + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + table.appendEvent({ + sessionId: 'child', + name: 'child/late', + data: { text: 'late evidence' }, + createdAt: 120 + }) + table.append.mockClear() + + const context = service.getContext('parent', [2], { + sourceSessionId: 'child', + before: 0, + after: 5 + }) + + expect(context).toMatchObject({ + sessionId: 'parent', + sourceSessionId: 'child', + requestedEntryIds: [2], + matchedEntryIds: [2] + }) + expect(context.entries.map((entry) => entry.entryId)).toEqual([2, 3]) + expect(context.entries[0].summary).toContain('target evidence') + expect(context.entries[0].summary).not.toContain('stale projection') + expect(table.getEffectiveContextRowsAtHead).toHaveBeenCalledWith( + { sessionId: 'child', maxEntryId: 3 }, + [2], + expect.objectContaining({ before: 0, after: 5 }) + ) + expect(projectionTable.getByEntryIdsIfCurrent).not.toHaveBeenCalled() + expect(table.append).not.toHaveBeenCalled() + }) + + it('rejects non-direct children at write time and after reparenting', () => { + const { table } = createTapeTableMock() + const { service, sessionById } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'other-parent', session_kind: 'regular', parent_session_id: null }, + { id: 'sibling', session_kind: 'subagent', parent_session_id: 'other-parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('sibling') + + expect(() => service.linkSubagentTape(createSubagentLinkInput('parent', 'sibling'))).toThrow( + 'Session sibling is not a direct subagent child of parent.' + ) + expect( + table + .getBySession('parent') + .some((entry: { name: string | null }) => entry.name === 'subagent/tape_linked') + ).toBe(false) + + sessionById.set('sibling', { + id: 'sibling', + session_kind: 'subagent', + parent_session_id: 'parent' + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'sibling')) + sessionById.set('sibling', { + id: 'sibling', + session_kind: 'subagent', + parent_session_id: 'other-parent' + }) + + let error: unknown + try { + service.getContext('parent', [1], { sourceSessionId: 'sibling' }) + } catch (caught) { + error = caught + } + expect(error).toMatchObject({ + code: 'linked_tape_unauthorized', + parentSessionId: 'parent', + sourceSessionId: 'sibling' + }) + }) + + it('rejects missing parents and non-subagent children before persisting a link', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'regular-child', session_kind: 'regular', parent_session_id: 'parent' }, + { id: 'orphan', session_kind: 'subagent', parent_session_id: 'missing-parent' } + ]) + table.ensureBootstrapAnchor('regular-child') + table.ensureBootstrapAnchor('orphan') + + expect(() => + service.linkSubagentTape(createSubagentLinkInput('parent', 'regular-child')) + ).toThrow('Session regular-child is not a direct subagent child of parent.') + expect(() => + service.linkSubagentTape(createSubagentLinkInput('missing-parent', 'orphan')) + ).toThrow('Session orphan is not a direct subagent child of missing-parent.') + expect( + ['parent', 'missing-parent'].some((sessionId) => + table + .getBySession(sessionId) + .some((entry: { name: string | null }) => entry.name === 'subagent/tape_linked') + ) + ).toBe(false) + }) + + it('reports a finalized linked Tape as unavailable after its durable session is deleted', () => { + const { table } = createTapeTableMock() + const { service, sessionById } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + sessionById.delete('child') + + let error: unknown + try { + service.search('parent', 'anything', { scope: 'linked_subagents' }) + } catch (caught) { + error = caught + } + expect(error).toMatchObject({ + code: 'linked_tape_unavailable', + parentSessionId: 'parent', + sourceSessionId: 'child' + }) + }) + + it('rejects a rebuilt child Tape until a new incarnation is explicitly linked', () => { + const { table } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/original', + data: { text: 'original incarnation marker' } + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + + table.deleteBySession('child') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/rebuilt', + data: { text: 'rebuilt incarnation marker' } + }) + expect( + service.search('child', 'rebuilt incarnation marker', { scope: 'current' }) + ).toHaveLength(1) + expect(() => + service.search('parent', 'rebuilt incarnation marker', { scope: 'linked_subagents' }) + ).toThrowError(/Linked Tape child is unavailable/) + + service.linkSubagentTape({ + ...createSubagentLinkInput('parent', 'child'), + runId: 'run-rebuilt-child', + taskId: 'task-rebuilt-child' + }) + expect( + service.search('parent', 'rebuilt incarnation marker', { scope: 'linked_subagents' }) + ).toMatchObject([{ sessionId: 'child', entryId: 2 }]) + }) + + itIfSqlite('detects linked Tape replacement after entry ids are reused in SQLite', () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const lifecycle = new SqliteTapeLifecycleAdapter(db) + table.createTable() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/original', + data: { text: 'native original incarnation' } + }) + service.linkSubagentTape(createSubagentLinkInput('parent', 'child')) + + lifecycle.deleteBySession('child') + table.ensureBootstrapAnchor('child') + table.appendEvent({ + sessionId: 'child', + name: 'child/rebuilt', + data: { text: 'native rebuilt incarnation' } + }) + + expect(() => + service.search('parent', 'native rebuilt incarnation', { scope: 'linked_subagents' }) + ).toThrowError(/Linked Tape child is unavailable/) + } finally { + db.close() + } + }) + + it('reads legacy external merge links and keeps legacy discard audit-only', () => { + const { table, entries } = createTapeTableMock() + const { service } = createLinkedTapeService(table, [ + { id: 'parent', session_kind: 'regular', parent_session_id: null }, + { id: 'legacy-child', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'malformed-child', session_kind: 'subagent', parent_session_id: 'parent' }, + { id: 'discarded-child', session_kind: 'subagent', parent_session_id: 'parent' } + ]) + table.ensureBootstrapAnchor('parent') + table.ensureBootstrapAnchor('legacy-child') + table.ensureBootstrapAnchor('malformed-child') + table.ensureBootstrapAnchor('discarded-child') + const legacyBootstrap = entries.find( + (entry) => entry.session_id === 'legacy-child' && entry.entry_id === 1 + )! + legacyBootstrap.meta_json = '{}' + table.appendEvent({ + sessionId: 'legacy-child', + name: 'child/result', + data: { text: 'legacy link needle' } + }) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/merge', + source: { type: 'fork', id: 'legacy-child', seq: 0 }, + provenanceKey: 'fork:parent:legacy-child:external-merge:event', + data: { + forkId: 'legacy-child', + forkSessionId: 'legacy-child', + referencedEntryCount: 2, + status: 'completed' + } + }) + table.appendEvent({ + sessionId: 'malformed-child', + name: 'child/result', + data: { text: 'malformed legacy link needle' } + }) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/merge', + source: { type: 'fork', id: 'malformed-child', seq: 1 }, + provenanceKey: 'fork:parent:malformed-child:external-merge:event', + data: { + forkId: 'malformed-child', + forkSessionId: 'malformed-child', + referencedEntryCount: 2, + status: 'completed' + } + }) + table.appendEvent({ + sessionId: 'parent', + name: 'fork/discard', + source: { type: 'fork', id: 'discarded-child', seq: 0 }, + provenanceKey: 'fork:parent:discarded-child:external-discard:event', + data: { + forkId: 'discarded-child', + forkSessionId: 'discarded-child', + status: 'cancelled' + } + }) + + expect( + service.search('parent', 'legacy link needle', { scope: 'linked_subagents' }) + ).toMatchObject([{ sessionId: 'legacy-child', entryId: 2 }]) + expect( + service.search('parent', 'malformed legacy link needle', { scope: 'linked_subagents' }) + ).toEqual([]) + let error: unknown + try { + service.getContext('parent', [1], { sourceSessionId: 'discarded-child' }) + } catch (caught) { + error = caught + } + expect(error).toMatchObject({ code: 'linked_tape_unauthorized' }) + }) +}) diff --git a/test/main/session/data/tapeRecall.test.ts b/test/main/session/data/tapeRecall.test.ts new file mode 100644 index 0000000000..52840d8c2f --- /dev/null +++ b/test/main/session/data/tapeRecall.test.ts @@ -0,0 +1,2491 @@ +import { + performance, + describe, + expect, + it, + vi, + SessionTape, + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + DeepChatTapeEntriesTable, + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, + DeepChatTapeSearchProjectionTable, + NewSessionsTable, + DatabaseCtor, + sqliteAvailable, + sqliteSkipReason, + itIfSqlite, + createTapeTableMock, + createRecord, + createTapeService +} from './tapeTestHarness' + +describe('SessionTape recall', () => { + it('invalidates projections written before atomic generation transitions', () => { + expect(DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION).toBeGreaterThan(2) + }) + + itIfSqlite('prunes legacy and metadata-orphaned search projections during initialization', () => { + const db = new DatabaseCtor(':memory:') + try { + const projection = new DeepChatTapeSearchProjectionTable(db) + projection.createTable() + const row = (sessionId: string, searchText: string) => ({ + sessionId, + entryId: 1, + kind: 'event' as const, + name: 'projection/test', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText, + summaryText: searchText, + refs: {}, + createdAt: 100 + }) + projection.replaceSession('legacy', [row('legacy', 'private legacy text')], 1, 2) + projection.replaceSession('current', [row('current', 'current text')], 1) + db.prepare( + `INSERT INTO deepchat_tape_search_projection ( + session_id, + entry_id, + kind, + name, + source_type, + source_id, + source_seq, + search_text, + summary_text, + refs_json, + created_at + ) + VALUES ('orphan', 1, 'event', 'projection/test', NULL, NULL, NULL, ?, ?, '{}', 100)` + ).run('private orphan text', 'private orphan text') + + const restarted = new DeepChatTapeSearchProjectionTable(db) + restarted.createTable() + + expect(restarted.getProjectedEntryIds('legacy')).toEqual([]) + expect(restarted.getSessionMeta('legacy')).toBeNull() + expect(restarted.getProjectedEntryIds('orphan')).toEqual([]) + expect(restarted.getProjectedEntryIds('current')).toEqual([1]) + expect(restarted.isCurrent('current', 1)).toBe(true) + const ftsExists = db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'deepchat_tape_search_fts'" + ) + .get() + if (ftsExists) { + expect( + db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_search_fts + WHERE session_id IN ('legacy', 'orphan')` + ) + .get() + ).toEqual({ count: 0 }) + expect( + db + .prepare( + `SELECT COUNT(*) AS count + FROM deepchat_tape_search_fts + WHERE session_id = 'current'` + ) + .get() + ).toEqual({ count: 1 }) + } + } finally { + db.close() + } + }) + + itIfSqlite('caches FTS capability detection per SQLite connection', () => { + const db = new DatabaseCtor(':memory:') + try { + const exec = db.exec.bind(db) + let probeStatementCount = 0 + const execSpy = vi.spyOn(db, 'exec').mockImplementation(((sql: string) => { + if (sql.includes('tape_search_fts_probe_')) probeStatementCount += 1 + return exec(sql) + }) as typeof db.exec) + + new DeepChatTapeSearchProjectionTable(db).createTable() + const firstDetectionCount = probeStatementCount + new DeepChatTapeSearchProjectionTable(db).createTable() + + expect(firstDetectionCount).toBeGreaterThan(0) + expect(probeStatementCount).toBe(firstDetectionCount) + execSpy.mockRestore() + } finally { + vi.restoreAllMocks() + db.close() + } + }) + + it('reports info, search, and handoff within one session scope', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([ + createRecord({ id: 'u1' }), + createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { type: 'content', content: 'answer', status: 'success', timestamp: 101 } + ]), + metadata: JSON.stringify({ totalTokens: 9 }), + createdAt: 101, + updatedAt: 101 + }) + ]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + service.handoff('s1', 'phase_done', { summary: ' done ' }) + const handoffAnchor = entries.find((entry) => entry.name === 'handoff/phase_done') + + expect(service.info('s1')).toMatchObject({ + sessionId: 's1', + anchors: 2, + lastAnchor: 'handoff/phase_done', + lastTokenUsage: 9, + migrationState: 'ready' + }) + expect(JSON.parse(handoffAnchor.payload_json).state).toMatchObject({ + summary: 'done', + cursorOrderSeq: 3, + range: { + fromOrderSeq: 1, + toOrderSeq: 2 + }, + sourceMessageIds: ['u1', 'a1'] + }) + expect(service.search('s1', 'hello')).toHaveLength(1) + expect( + service.search('s1', 'hello', { kinds: ['message'], start: '1970-01-01T00:00:00.000Z' }) + ).toHaveLength(1) + expect(service.search('s1', 'hello', { kinds: ['anchor'] })).toHaveLength(0) + expect(service.search('s1', 'hello', { end: '99' })).toHaveLength(0) + expect(() => service.search('s1', 'hello', { start: 'not-a-date' })).toThrow( + 'start must be an ISO date/time or millisecond timestamp.' + ) + expect(service.anchors('s1')).toMatchObject([ + { sessionId: 's1', name: 'session/start' }, + { sessionId: 's1', name: 'handoff/phase_done' } + ]) + expect(service.anchors('s1', { limit: 1 })).toMatchObject([ + { sessionId: 's1', name: 'handoff/phase_done' } + ]) + expect(service.search('s2', 'hello')).toHaveLength(0) + }) + + it('returns only effective message DTOs for a requested source span', () => { + const { table } = createTapeTableMock() + const first = table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { record: createRecord({ id: 'm1', orderSeq: 1 }) }, + meta: { source: 'live', orderSeq: 1, role: 'user' } + }) + const second = table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm2', seq: 0 }, + payload: { record: createRecord({ id: 'm2', orderSeq: 2 }) }, + meta: { source: 'live', orderSeq: 2, role: 'user' } + }) + table.appendEvent({ + sessionId: 's1', + name: 'message/retracted', + source: { type: 'message', id: 'm1', seq: 1 }, + data: { messageId: 'm1', reason: 'deleted' } + }) + const service = new SessionTape({ deepchatTapeEntriesTable: table } as any) + + const span = service.getEffectiveMessageSourceSpan('s1', [first.entry_id, second.entry_id]) + + expect(span).toHaveLength(1) + expect(span[0]).toEqual({ + entryId: second.entry_id, + record: { + role: 'user', + content: createRecord({ id: 'm2', orderSeq: 2 }).content, + orderSeq: 2 + } + }) + expect(span[0]).not.toHaveProperty('payload_json') + }) + + it('projects fallback tape search into compact results and bounded context', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'Run the dev server', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.append({ + sessionId: 's1', + kind: 'tool_result', + name: 'shell', + source: { type: 'tool_result', id: 'm1:tc1', seq: 0 }, + payload: { + messageId: 'm1', + orderSeq: 2, + toolCallId: 'tc1', + command: 'pnpm run dev', + exitStatus: 1, + response: 'Command failed with EADDRINUSE in /tmp/deepchat.log' + }, + meta: { source: 'live', status: 'error' }, + createdAt: 110 + }) + + const hits = service.search('s1', 'pnpm run dev', { limit: 5 }) + expect(hits).toHaveLength(1) + expect(hits[0]).toMatchObject({ + kind: 'tool_result', + summary: expect.stringContaining('EADDRINUSE'), + refs: { + toolCallId: 'tc1', + commands: expect.arrayContaining(['pnpm run dev']), + filePaths: expect.arrayContaining(['/tmp/deepchat.log']), + errorCodes: expect.arrayContaining(['EADDRINUSE']), + exitStatus: 1 + } + }) + expect(hits[0]).not.toHaveProperty('payload') + expect(hits[0]).not.toHaveProperty('meta') + + const context = service.getContext('s1', [hits[0].entryId], { + before: 0, + after: 0, + maxBytesPerEntry: 12, + maxTotalBytes: 12 + }) + expect(context.matchedEntryIds).toEqual([hits[0].entryId]) + expect(context.entries[0]).toMatchObject({ + entryId: hits[0].entryId, + evidence: { truncated: true } + }) + expect(context.entries[0].evidence.bytes).toBeLessThanOrEqual(12) + expect(context.entries[0]).not.toHaveProperty('payload') + expect(context.entries[0]).not.toHaveProperty('meta') + + const exhaustedContext = service.getContext('s1', [hits[0].entryId], { + before: 0, + after: 0, + maxTotalBytes: 0 + }) + expect(exhaustedContext.entries).toEqual([]) + expect(exhaustedContext.matchedEntryIds).toEqual([]) + }) + + it('projects user message attachment metadata into search text and refs', () => { + const { table } = createTapeTableMock() + const projectionTable = { + isCurrent: vi.fn().mockReturnValue(false), + getSessionMeta: vi.fn().mockReturnValue(null), + getProjectedEntryIds: vi.fn().mockReturnValue([]), + appendSession: vi.fn(), + replaceSession: vi.fn(), + search: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm-file', seq: 0 }, + payload: { + record: createRecord({ + id: 'm-file', + content: JSON.stringify({ + text: 'Please review the attachment', + files: [ + { + name: 'a.md', + path: '/tmp/a.md', + content: 'raw attachment body should not be projected', + metadata: { fileName: 'workspace-a.md' } + } + ], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + + service.search('s1', '/tmp/a.md', { limit: 5 }) + + expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) + const projectedRows = projectionTable.replaceSession.mock.calls[0][1] + expect(projectedRows[0]).toMatchObject({ + entryId: 1, + refs: { + filePaths: expect.arrayContaining(['/tmp/a.md']), + fileNames: expect.arrayContaining(['a.md', 'workspace-a.md']) + } + }) + expect(projectedRows[0].searchText).toContain('/tmp/a.md') + expect(projectedRows[0].searchText).toContain('a.md') + expect(projectedRows[0].searchText).toContain('workspace-a.md') + expect(projectedRows[0].searchText).not.toContain('raw attachment body should not be projected') + }) + + it('preserves relative file paths in projected refs', () => { + const { table } = createTapeTableMock() + let projectedRows: any[] = [] + const projectionTable = { + isCurrent: vi.fn().mockReturnValue(false), + getSessionMeta: vi.fn().mockReturnValue(null), + getProjectedEntryIds: vi.fn().mockReturnValue([]), + appendSession: vi.fn(), + replaceSession: vi.fn((_sessionId: string, rows: any[]) => { + projectedRows = rows + }), + search: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm-relative', seq: 0 }, + payload: { + record: createRecord({ + id: 'm-relative', + content: JSON.stringify({ + text: 'Touched src/main/index.ts, ./lib/util.ts, ../shared/types.ts, test/main/foo/bar.ts, /usr/local/bin/deploy, and https://example.com/not-a-file', + files: [], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + + service.search('s1', 'src/main/index.ts', { limit: 5 }) + + expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) + const filePaths = projectedRows[0].refs.filePaths + expect(filePaths).toEqual( + expect.arrayContaining([ + 'src/main/index.ts', + './lib/util.ts', + '../shared/types.ts', + 'test/main/foo/bar.ts', + '/usr/local/bin/deploy' + ]) + ) + expect(filePaths).not.toContain('/main/index.ts') + expect(filePaths).not.toContain('/lib/util.ts') + expect(filePaths).not.toContain('/main/foo/bar.ts') + expect(filePaths).not.toContain('example.com/not-a-file') + }) + + it('rebuilds old tape projection versions before attachment path search', () => { + const { table } = createTapeTableMock() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm-file', seq: 0 }, + payload: { + record: createRecord({ + id: 'm-file', + content: JSON.stringify({ + text: 'Please review the migrated attachment', + files: [ + { + name: 'legacy.md', + path: '/tmp/legacy.md', + content: 'raw migrated body must stay out of projection', + metadata: { fileName: 'legacy-workspace.md' } + } + ], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + let storedVersion = 1 + let storedMaxEntryId = 1 + let projectedRows: any[] = [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm-file', + sourceSeq: 0, + searchText: 'message/user Please review the migrated attachment', + summaryText: 'user: Please review the migrated attachment', + refs: { messageId: 'm-file' }, + createdAt: 100 + } + ] + const projectionTable = { + isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => { + return ( + storedVersion === DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION && + storedMaxEntryId === maxEntryId + ) + }), + getSessionMeta: vi.fn(() => ({ + projectionVersion: storedVersion, + maxEntryId: storedMaxEntryId + })), + getProjectedEntryIds: vi.fn().mockReturnValue([1]), + appendSession: vi.fn(), + replaceSession: vi.fn((_sessionId: string, rows: any[], maxEntryId: number) => { + projectedRows = rows + storedVersion = DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION + storedMaxEntryId = maxEntryId + }), + search: vi.fn((_sessionId: string, query: string) => { + return projectedRows + .filter((row) => row.searchText.includes(query)) + .map((row) => ({ + session_id: row.sessionId, + entry_id: row.entryId, + kind: row.kind, + name: row.name, + source_type: row.sourceType, + source_id: row.sourceId, + source_seq: row.sourceSeq, + search_text: row.searchText, + summary_text: row.summaryText, + refs_json: JSON.stringify(row.refs), + created_at: row.createdAt, + score: 1 + })) + }) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const hits = service.search('s1', '/tmp/legacy.md', { limit: 5 }) + + expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) + expect(projectionTable.appendSession).not.toHaveBeenCalled() + expect(hits).toHaveLength(1) + expect(hits[0].refs).toMatchObject({ + filePaths: expect.arrayContaining(['/tmp/legacy.md']), + fileNames: expect.arrayContaining(['legacy.md', 'legacy-workspace.md']) + }) + expect(projectedRows[0].searchText).not.toContain( + 'raw migrated body must stay out of projection' + ) + }) + + it('prioritizes requested tape context entries before window entries under byte caps', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'before-1', seq: 0 }, + payload: { + record: createRecord({ + id: 'before-1', + content: JSON.stringify({ + text: 'before entry consumes the tiny byte budget first', + files: [], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'before-2', seq: 0 }, + payload: { + record: createRecord({ + id: 'before-2', + content: JSON.stringify({ + text: 'second before entry also appears earlier', + files: [], + links: [] + }), + createdAt: 110, + updatedAt: 110 + }) + }, + meta: { source: 'live', orderSeq: 2, role: 'user' }, + createdAt: 110 + }) + const target = table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'target', seq: 0 }, + payload: { + record: createRecord({ + id: 'target', + content: JSON.stringify({ + text: 'target-ok consumes the available budget', + files: [], + links: [] + }), + createdAt: 120, + updatedAt: 120 + }) + }, + meta: { source: 'live', orderSeq: 3, role: 'user' }, + createdAt: 120 + }) + + const context = service.getContext('s1', [target.entry_id], { + before: 2, + after: 0, + limit: 3, + maxBytesPerEntry: 18, + maxTotalBytes: 18 + }) + + expect(context.matchedEntryIds).toEqual([target.entry_id]) + expect(context.entries.map((entry) => entry.entryId)).toEqual([target.entry_id]) + expect(context.entries[0].evidence.text).toContain('target-ok') + }) + + it('falls back to the current Tape when the context projection is not current', () => { + const { table } = createTapeTableMock() + const entry = table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'current-message', seq: 0 }, + payload: { + record: createRecord({ + id: 'current-message', + content: JSON.stringify({ text: 'current generation context', files: [], links: [] }) + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' } + }) + const projectionTable = { + getByEntryIds: vi.fn(() => [ + { + session_id: 's1', + entry_id: entry.entry_id, + summary_text: 'old private context', + refs_json: '{"messageId":"old-message"}' + } + ]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const context = service.getContext('s1', [entry.entry_id], { before: 0, after: 0 }) + + expect(projectionTable.getByEntryIdsIfCurrent).toHaveBeenCalledWith('s1', entry.entry_id, [ + entry.entry_id + ]) + expect(projectionTable.getByEntryIds).not.toHaveBeenCalled() + expect(context.entries[0]).toMatchObject({ + summary: expect.stringContaining('current generation context'), + refs: { messageId: 'current-message' } + }) + expect(context.entries[0].summary).not.toContain('old private context') + }) + + itIfSqlite('ignores a pre-transition projection version at the same Tape head', () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + table.createTable() + projectionTable.createTable() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'current-message', seq: 0 }, + payload: { + record: createRecord({ + id: 'current-message', + content: JSON.stringify({ text: 'current generation context', files: [], links: [] }) + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' } + }) + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'old-message', + sourceSeq: 0, + searchText: 'old private context', + summaryText: 'old private context', + refs: { messageId: 'old-message' }, + createdAt: 100 + } + ], + 1, + 2 + ) + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const context = service.getContext('s1', [1], { before: 0, after: 0 }) + + expect(context.entries[0]).toMatchObject({ + entryId: 1, + summary: expect.stringContaining('current generation context'), + refs: { messageId: 'current-message' } + }) + expect(context.entries[0].summary).not.toContain('old private context') + } finally { + db.close() + } + }) + + itIfSqlite('ignores stale same-entry projection context when the Tape head has advanced', () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + table.createTable() + projectionTable.createTable() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'current-message', seq: 0 }, + payload: { + record: createRecord({ + id: 'current-message', + content: JSON.stringify({ + text: 'current generation context', + files: [], + links: [] + }) + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' } + }) + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'old-message', + sourceSeq: 0, + searchText: 'old private context', + summaryText: 'old private context', + refs: { messageId: 'old-message' }, + createdAt: 100 + } + ], + 0 + ) + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const context = service.getContext('s1', [1], { before: 0, after: 0 }) + + expect(context.entries[0]).toMatchObject({ + entryId: 1, + summary: expect.stringContaining('current generation context'), + refs: { messageId: 'current-message' } + }) + expect(context.entries[0].summary).not.toContain('old private context') + } finally { + db.close() + } + }) + + it('binds tape projection LIKE fallback params for single and multi-term queries', () => { + const all = vi.fn().mockReturnValue([]) + const db = { + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => all(sql, params), + get: vi.fn().mockReturnValue(undefined), + run: vi.fn() + })) + } + const projectionTable = new DeepChatTapeSearchProjectionTable(db as any) + ;(projectionTable as any).recoverSessionFts = vi.fn() + + projectionTable.search('s1', 'Redis', { limit: 5 }) + expect(all).toHaveBeenLastCalledWith( + expect.stringContaining('FROM deepchat_tape_search_projection'), + ['s1', '%Redis%', '%Redis%', '%Redis%', 5] + ) + + projectionTable.search('s1', 'Redis TTL', { limit: 5 }) + expect(all).toHaveBeenLastCalledWith( + expect.stringContaining('FROM deepchat_tape_search_projection'), + [ + 's1', + '%Redis TTL%', + '%Redis TTL%', + '%Redis TTL%', + '%Redis%', + '%Redis%', + '%Redis%', + '%TTL%', + '%TTL%', + '%TTL%', + 5 + ] + ) + + const oversizedQuery = Array.from({ length: 257 }, (_, index) => `term-${index}`).join(' ') + projectionTable.search('s1', oversizedQuery, { limit: 5 }) + expect(all).toHaveBeenLastCalledWith( + expect.stringContaining('FROM deepchat_tape_search_projection'), + ['s1', `%${oversizedQuery}%`, `%${oversizedQuery}%`, `%${oversizedQuery}%`, 5] + ) + }) + + it('checks projection version and head in the same context row query', () => { + const reads: Array<{ sql: string; params: unknown[] }> = [] + const projectionTable = new DeepChatTapeSearchProjectionTable({ + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => { + reads.push({ sql, params }) + return [] + } + })) + } as any) + + projectionTable.getByEntryIdsIfCurrent('s1', 7, [3, 3, 0], 42) + + expect(reads).toHaveLength(1) + expect(reads[0].sql).toContain('INNER JOIN deepchat_tape_search_projection_meta AS meta') + expect(reads[0].sql).toContain('meta.projection_version = ?') + expect(reads[0].sql).toContain('meta.max_entry_id = ?') + expect(reads[0].params).toEqual([42, 7, 's1', 3]) + }) + + it('uses current tape projection without loading full session rows', () => { + const { table } = createTapeTableMock() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'Redis compact marker', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.getBySession.mockClear() + const projectionTable = { + isCurrent: vi.fn().mockReturnValue(true), + getSessionMeta: vi.fn(), + getProjectedEntryIds: vi.fn(), + appendSession: vi.fn(), + replaceSession: vi.fn(), + search: vi.fn().mockReturnValue([ + { + session_id: 's1', + entry_id: 1, + kind: 'message', + name: 'message/user', + source_type: 'message', + source_id: 'm1', + source_seq: 0, + search_text: 'Redis compact marker', + summary_text: 'Redis compact marker', + refs_json: '{"messageId":"m1"}', + created_at: 100, + score: -2 + } + ]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const hits = service.search('s1', 'Redis compact', { limit: 5 }) + + expect(table.getMaxEntryId).toHaveBeenCalledWith('s1') + expect(table.getBySession).not.toHaveBeenCalled() + expect(projectionTable.search).toHaveBeenCalledWith( + 's1', + 'Redis compact', + expect.objectContaining({ limit: 5 }) + ) + expect(projectionTable.appendSession).not.toHaveBeenCalled() + expect(projectionTable.replaceSession).not.toHaveBeenCalled() + expect(hits[0]).toMatchObject({ + entryId: 1, + kind: 'message', + summary: 'Redis compact marker', + refs: { messageId: 'm1' }, + score: -2 + }) + expect(hits[0]).not.toHaveProperty('payload') + expect(hits[0]).not.toHaveProperty('meta') + }) + + it('falls back to effective tape search when projection search throws', () => { + const { table } = createTapeTableMock() + const projectionTable = { + isCurrent: vi.fn().mockReturnValue(true), + search: vi.fn(() => { + throw new Error('projection failed') + }) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'Redis fallback marker', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + + const hits = service.search('s1', 'Redis fallback', { limit: 5 }) + expect(hits).toHaveLength(1) + expect(hits[0]).toMatchObject({ + kind: 'message', + summary: expect.stringContaining('Redis fallback') + }) + expect(hits[0]).not.toHaveProperty('payload') + expect(hits[0]).not.toHaveProperty('meta') + }) + + it('appends tape projection rows when the previous projection is an effective prefix', () => { + const { table } = createTapeTableMock() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'first redis', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm2', seq: 0 }, + payload: { + record: createRecord({ + id: 'm2', + content: JSON.stringify({ text: 'second vue', files: [], links: [] }), + createdAt: 110, + updatedAt: 110 + }) + }, + meta: { source: 'live', orderSeq: 2, role: 'user' }, + createdAt: 110 + }) + const projectionTable = { + isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => maxEntryId === 1), + getSessionMeta: vi.fn().mockReturnValue({ projectionVersion: 1, maxEntryId: 1 }), + getProjectedEntryIds: vi.fn().mockReturnValue([1]), + appendSession: vi.fn(), + replaceSession: vi.fn(), + search: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.search('s1', 'vue', { limit: 5 }) + + expect(projectionTable.appendSession).toHaveBeenCalledTimes(1) + expect(projectionTable.appendSession.mock.calls[0][1].map((row: any) => row.entryId)).toEqual([ + 2 + ]) + expect(projectionTable.replaceSession).not.toHaveBeenCalled() + }) + + it('rebuilds tape projection when projected entry ids are not an effective prefix', () => { + const { table } = createTapeTableMock() + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'm1', seq: 0 }, + payload: { + record: createRecord({ + id: 'm1', + content: JSON.stringify({ text: 'redis', files: [], links: [] }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + const projectionTable = { + isCurrent: vi.fn((_sessionId: string, maxEntryId: number) => maxEntryId === 0), + getSessionMeta: vi.fn().mockReturnValue({ projectionVersion: 1, maxEntryId: 0 }), + getProjectedEntryIds: vi.fn().mockReturnValue([99]), + appendSession: vi.fn(), + replaceSession: vi.fn(), + search: vi.fn().mockReturnValue([]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.search('s1', 'redis', { limit: 5 }) + + expect(projectionTable.replaceSession).toHaveBeenCalledTimes(1) + expect(projectionTable.appendSession).not.toHaveBeenCalled() + }) + + it('does not run LIKE fallback when FTS fills the tape projection search limit', () => { + const all = vi.fn((sql: string, _params: unknown[]) => + sql.includes('deepchat_tape_search_fts') + ? [ + { + session_id: 's1', + entry_id: 1, + kind: 'message', + name: 'message/user', + source_type: 'message', + source_id: 'm1', + source_seq: 0, + search_text: 'Redis TTL', + summary_text: 'Redis TTL', + refs_json: '{}', + created_at: 100, + score: -1 + } + ] + : [] + ) + const db = { + exec: vi.fn(() => { + throw new Error('unexpected FTS ensure') + }), + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => all(sql, params), + get: (..._params: unknown[]) => { + if ( + sql.includes('deepchat_tape_search_projection_meta') || + sql.includes('deepchat_tape_search_fts_meta') + ) { + return { projection_version: 1, max_entry_id: 1 } + } + return undefined + }, + run: vi.fn() + })), + transaction: vi.fn((callback: () => void) => callback) + } + const projectionTable = new DeepChatTapeSearchProjectionTable(db as any) + ;(projectionTable as any).ftsReady = true + + const hits = projectionTable.search('s1', 'Redis', { limit: 1 }) + + expect(hits).toHaveLength(1) + expect(db.exec).not.toHaveBeenCalled() + const ftsCall = all.mock.calls.find(([sql]) => String(sql).includes('deepchat_tape_search_fts')) + expect(String(ftsCall?.[0])).toContain('deepchat_tape_search_fts.session_id = ?') + expect((ftsCall?.[1] as unknown[]).filter((param) => param === 's1')).toHaveLength(2) + expect( + vi.mocked(db.prepare).mock.calls.some(([sql]) => String(sql).includes('NULL AS score')) + ).toBe(false) + }) + + it('invalidates a failed FTS query and rebuilds the derivative before the next search', () => { + const projectionRow = { + session_id: 's1', + entry_id: 1, + kind: 'message', + name: 'message/user', + source_type: 'message', + source_id: 'm1', + source_seq: 0, + search_text: 'Redis TTL', + summary_text: 'Redis TTL', + refs_json: '{}', + created_at: 100 + } + let ftsMetaPresent = true + let shouldFailFtsQuery = true + let likeQueryCount = 0 + let ftsQueryCount = 0 + const exec = vi.fn() + const prepare = vi.fn((sql: string) => ({ + all: (..._params: unknown[]) => { + if (sql.includes('bm25(deepchat_tape_search_fts)')) { + ftsQueryCount += 1 + if (shouldFailFtsQuery) { + shouldFailFtsQuery = false + throw new Error('injected corrupt FTS query') + } + return [{ ...projectionRow, score: -1 }] + } + if (sql.includes('NULL AS score')) { + likeQueryCount += 1 + return [{ ...projectionRow, score: null }] + } + if (sql.includes('SELECT *') && sql.includes('deepchat_tape_search_projection')) { + return [projectionRow] + } + return [] + }, + get: (..._params: unknown[]) => { + if (sql.includes('sqlite_master') && sql.includes('deepchat_tape_search_fts_meta')) { + return { name: 'deepchat_tape_search_fts_meta' } + } + if (sql.includes('FROM deepchat_tape_search_projection_meta')) { + return { projection_version: 1, max_entry_id: 1 } + } + if (sql.includes('FROM deepchat_tape_search_fts_meta')) { + return ftsMetaPresent ? { projection_version: 1, max_entry_id: 1 } : undefined + } + if (sql.includes('SELECT rowid')) { + return { rowid: 1 } + } + return undefined + }, + run: (..._params: unknown[]) => { + if (sql.includes('DELETE FROM deepchat_tape_search_fts_meta')) { + ftsMetaPresent = false + } + if (sql.includes('INSERT INTO deepchat_tape_search_fts_meta')) { + ftsMetaPresent = true + } + } + })) + const projectionTable = new DeepChatTapeSearchProjectionTable({ + exec, + prepare, + transaction: (callback: () => void) => callback + } as any) + ;(projectionTable as any).ftsReady = true + + expect(projectionTable.search('s1', 'Redis', { limit: 1 })).toMatchObject([ + { entry_id: 1, score: null } + ]) + expect(projectionTable.hasFtsReadyForTesting()).toBe(false) + expect(exec).toHaveBeenCalledWith('DROP TABLE IF EXISTS deepchat_tape_search_fts') + + expect(projectionTable.search('s1', 'Redis', { limit: 1 })).toMatchObject([ + { entry_id: 1, score: -1 } + ]) + expect(projectionTable.hasFtsReadyForTesting()).toBe(true) + expect(ftsQueryCount).toBe(2) + expect(likeQueryCount).toBe(1) + expect( + exec.mock.calls.some(([sql]) => String(sql).includes('CREATE VIRTUAL TABLE IF NOT EXISTS')) + ).toBe(true) + }) + + it('queries memory view manifests by agent without expanding session ids', () => { + const all = vi.fn().mockReturnValue([]) + const db = { + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => all(sql, params) + })) + } + const table = new DeepChatTapeEntriesTable(db as any) + + table.listMemoryViewManifestAnchorsByAgent('agent-a', { + sessionId: 's-1', + messageId: 'msg-1', + limit: 7 + }) + + expect(all).toHaveBeenCalledWith( + expect.stringContaining('INNER JOIN new_sessions AS sessions'), + ['agent-a', 's-1', 'msg-1', 7] + ) + const sql = String(all.mock.calls[0][0]) + expect(sql).not.toContain(' IN (') + expect(sql).toContain('sessions.agent_id = ?') + expect(sql).toContain('tape.session_id = ?') + expect(sql).toContain("json_extract(tape.meta_json, '$.messageId') = ?") + }) + + it('keeps linked raw search candidate-bounded instead of materializing complete Tapes', () => { + const all = vi.fn().mockReturnValue([]) + const db = { + prepare: vi.fn((sql: string) => ({ + all: (...params: unknown[]) => all(sql, params) + })) + } + const table = new DeepChatTapeEntriesTable(db as any) + + table.searchEffectiveSourcesAtHeads( + [ + { sessionId: 'child-b', maxEntryId: 20 }, + { sessionId: 'child-a', maxEntryId: 10 } + ], + 'needle', + { limit: 5 } + ) + + const sql = String(all.mock.calls[0][0]) + const params = all.mock.calls[0][1] + expect(sql).toContain('FROM deepchat_tape_entries AS candidate') + expect(sql).toContain('candidate.entry_id <= source.max_entry_id') + expect(sql).toContain('FROM deepchat_tape_entries AS later_message') + expect(sql).toContain( + "typeof(json_extract(later_message.payload_json, '$.record.content')) = 'text'" + ) + expect(sql).not.toContain('bounded_rows AS') + expect(params[0]).toBe( + JSON.stringify([ + { sessionId: 'child-a', maxEntryId: 10 }, + { sessionId: 'child-b', maxEntryId: 20 } + ]) + ) + expect(params.at(-1)).toBe(5) + }) + + it('reads exact linked projections without invoking FTS recovery or writes', () => { + const run = vi.fn() + const exec = vi.fn() + const reads: Array<{ sql: string; params: unknown[] }> = [] + const prepare = vi.fn((sql: string) => ({ + all: (...params: unknown[]) => { + reads.push({ sql, params }) + if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { + return [{ session_id: 'child', max_entry_id: 2 }] + } + if (sql.includes('SELECT projection.*, NULL AS score')) { + return [ + { + session_id: 'child', + entry_id: 2, + kind: 'event', + name: 'child/result', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'projection needle', + summary_text: 'projection needle', + refs_json: '{}', + created_at: 100, + score: null + } + ] + } + return [] + }, + run + })) + const projectionTable = new DeepChatTapeSearchProjectionTable({ prepare, exec } as any) + + const result = projectionTable.searchSourcesReadOnly( + [{ sessionId: 'child', maxEntryId: 2 }], + 'projection needle', + { limit: 5 } + ) + + expect(result).toMatchObject({ + coveredSources: [{ sessionId: 'child', maxEntryId: 2 }], + rows: [{ session_id: 'child', entry_id: 2 }] + }) + expect(exec).not.toHaveBeenCalled() + expect(run).not.toHaveBeenCalled() + expect(prepare.mock.calls.map(([sql]) => String(sql)).join('\n')).not.toContain( + 'deepchat_tape_search_fts_meta' + ) + expect( + reads.find((read) => read.sql.includes('SELECT projection.*, NULL AS score'))?.params.at(-1) + ).toBe(5) + }) + + it('returns no ranked rows when linked projection coverage is only partial', () => { + const reads: string[] = [] + const prepare = vi.fn((sql: string) => ({ + all: () => { + reads.push(sql) + if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { + return [{ session_id: 'child-a', max_entry_id: 2 }] + } + return [] + } + })) + const projectionTable = new DeepChatTapeSearchProjectionTable({ + prepare, + exec: vi.fn() + } as any) + + const result = projectionTable.searchSourcesReadOnly( + [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 3 } + ], + 'projection needle', + { limit: 5 } + ) + + expect(result).toEqual({ + coveredSources: [{ sessionId: 'child-a', maxEntryId: 2 }], + rows: [] + }) + expect( + reads.some((sql) => sql.includes('FROM deepchat_tape_search_projection AS projection')) + ).toBe(false) + expect(reads.some((sql) => sql.includes('FROM deepchat_tape_search_fts'))).toBe(false) + }) + + it('uses one LIKE ranking when linked FTS freshness is only partial', () => { + const reads: Array<{ sql: string; params: unknown[] }> = [] + const prepare = vi.fn((sql: string) => ({ + all: (...params: unknown[]) => { + reads.push({ sql, params }) + if (sql.includes('deepchat_tape_search_projection_meta AS meta')) { + return [ + { session_id: 'child-a', max_entry_id: 2 }, + { session_id: 'child-b', max_entry_id: 3 } + ] + } + if (sql.includes('deepchat_tape_search_fts_meta AS meta')) { + return [{ session_id: 'child-a', max_entry_id: 2 }] + } + if (sql.includes('SELECT projection.*, NULL AS score')) { + return [ + { + session_id: 'child-b', + entry_id: 3, + kind: 'event', + name: 'child/result', + source_type: null, + source_id: null, + source_seq: null, + search_text: 'projection needle', + summary_text: 'projection needle', + refs_json: '{}', + created_at: 200, + score: null + } + ] + } + return [] + } + })) + const projectionTable = new DeepChatTapeSearchProjectionTable({ + prepare, + exec: vi.fn() + } as any) + ;(projectionTable as any).ftsReady = true + + const sources = [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 3 } + ] + const result = projectionTable.searchSourcesReadOnly(sources, 'projection needle', { limit: 5 }) + + expect(result.rows).toMatchObject([{ session_id: 'child-b', entry_id: 3, score: null }]) + expect(reads.some(({ sql }) => sql.includes('bm25(deepchat_tape_search_fts)'))).toBe(false) + expect( + reads.find(({ sql }) => sql.includes('SELECT projection.*, NULL AS score'))?.params[0] + ).toBe(JSON.stringify(sources)) + }) + + itIfSqlite( + `keeps projected and raw linked multi-term search aligned${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + table.createTable() + projectionTable.createTable() + table.ensureBootstrapAnchor('child') + const entry = table.appendEvent({ + sessionId: 'child', + name: 'child/result', + data: { text: 'alpha separated by several words before beta' }, + createdAt: 100 + }) + const sources = [{ sessionId: 'child', maxEntryId: entry.entry_id }] + projectionTable.replaceSession( + 'child', + [ + { + sessionId: 'child', + entryId: entry.entry_id, + kind: 'event', + name: 'child/result', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'alpha separated by several words before beta', + summaryText: 'alpha separated by several words before beta', + refs: {}, + createdAt: 100 + } + ], + entry.entry_id + ) + + const projected = projectionTable.searchSourcesReadOnly(sources, 'alpha beta', { limit: 5 }) + const raw = table.searchEffectiveSourcesAtHeads(sources, 'alpha beta', { limit: 5 }) + + expect(projected.coveredSources).toEqual(sources) + expect(raw.map((row) => [row.session_id, row.entry_id])).toEqual( + projected.rows.map((row) => [row.session_id, row.entry_id]) + ) + expect(raw).toHaveLength(1) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `queries effective linked sources and context at frozen heads${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + table.ensureBootstrapAnchor('child-a') + table.ensureBootstrapAnchor('child-b') + table.appendEvent({ + sessionId: 'child-a', + name: 'child/result', + data: { text: 'native linked needle A' }, + createdAt: 100 + }) + table.appendEvent({ + sessionId: 'child-b', + name: 'child/result', + data: { text: 'native linked needle B' }, + createdAt: 200 + }) + table.appendEvent({ + sessionId: 'child-a', + name: 'child/late', + data: { text: 'native linked needle late' }, + createdAt: 300 + }) + + const sources = [ + { sessionId: 'child-a', maxEntryId: 2 }, + { sessionId: 'child-b', maxEntryId: 2 } + ] + expect( + table.searchEffectiveSourcesAtHeads(sources, 'native linked needle', { limit: 1 }) + ).toMatchObject([{ session_id: 'child-b', entry_id: 2 }]) + expect( + table + .searchEffectiveSourcesAtHeads(sources, 'native linked needle', { limit: 10 }) + .map((row) => [row.session_id, row.entry_id]) + ).toEqual([ + ['child-b', 2], + ['child-a', 2] + ]) + expect( + table + .getEffectiveContextRowsAtHead({ sessionId: 'child-a', maxEntryId: 2 }, [2], { + before: 1, + after: 5, + limit: 10 + }) + .map((row) => row.entry_id) + ).toEqual([2, 1]) + + table.ensureBootstrapAnchor('child-message') + const original = createRecord({ + id: 'linked-message', + sessionId: 'child-message', + content: JSON.stringify({ text: 'old linked marker', files: [], links: [] }) + }) + const replacement = { + ...original, + content: JSON.stringify({ text: 'new linked marker', files: [], links: [] }), + updatedAt: 200 + } + appendMessageRecordToTape(table, original, 'live') + appendMessageReplacementToTape(table, replacement, 'native_edit') + const replacementHead = table.getMaxEntryId('child-message') + expect( + table.searchEffectiveSourcesAtHeads( + [{ sessionId: 'child-message', maxEntryId: replacementHead }], + 'old linked marker' + ) + ).toEqual([]) + const replacementHits = table.searchEffectiveSourcesAtHeads( + [{ sessionId: 'child-message', maxEntryId: replacementHead }], + 'new linked marker' + ) + expect(replacementHits).toHaveLength(1) + expect( + table + .getEffectiveContextRowsAtHead( + { sessionId: 'child-message', maxEntryId: replacementHead }, + [replacementHits[0].entry_id], + { before: 0, after: 0, limit: 5 } + ) + .map((row) => row.entry_id) + ).toEqual([replacementHits[0].entry_id]) + + table.append({ + sessionId: 'child-message', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: original.id, seq: 0 }, + provenanceKey: null, + payload: { + record: { + id: original.id, + sessionId: original.sessionId, + orderSeq: original.orderSeq, + role: original.role, + status: 'sent' + } + }, + meta: { source: 'malformed_import' } + }) + expect( + table.searchEffectiveSourcesAtHeads( + [ + { + sessionId: 'child-message', + maxEntryId: table.getMaxEntryId('child-message') + } + ], + 'new linked marker' + ) + ).toMatchObject([{ entry_id: replacementHits[0].entry_id }]) + + appendMessageRetractionToTape(table, replacement, 'native_delete') + expect( + table.searchEffectiveSourcesAtHeads( + [ + { + sessionId: 'child-message', + maxEntryId: table.getMaxEntryId('child-message') + } + ], + 'new linked marker' + ) + ).toEqual([]) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `searches exact frozen projections without repairing a later child tail${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + projectionTable.replaceSession( + 'child', + [ + { + sessionId: 'child', + entryId: 2, + kind: 'event', + name: 'child/result', + sourceType: null, + sourceId: null, + sourceSeq: null, + searchText: 'native frozen projection needle', + summaryText: 'native frozen projection needle', + refs: {}, + createdAt: 100 + } + ], + 2 + ) + const before = db + .prepare( + `SELECT projection_version, max_entry_id, updated_at + FROM deepchat_tape_search_projection_meta + WHERE session_id = ?` + ) + .get('child') + + const result = projectionTable.searchSourcesReadOnly( + [{ sessionId: 'child', maxEntryId: 2 }], + 'native frozen projection needle', + { limit: 5 } + ) + + expect(result.coveredSources).toEqual([{ sessionId: 'child', maxEntryId: 2 }]) + expect(result.rows).toMatchObject([{ session_id: 'child', entry_id: 2 }]) + expect( + projectionTable.searchSourcesReadOnly( + [{ sessionId: 'child', maxEntryId: 3 }], + 'native frozen projection needle', + { limit: 5 } + ) + ).toEqual({ rows: [], coveredSources: [] }) + expect( + db + .prepare( + `SELECT projection_version, max_entry_id, updated_at + FROM deepchat_tape_search_projection_meta + WHERE session_id = ?` + ) + .get('child') + ).toEqual(before) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `filters stale FTS rows through the base projection after restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'current', + sourceSeq: 0, + searchText: 'current Redis marker', + summaryText: 'current Redis marker', + refs: { messageId: 'current' }, + createdAt: 200 + } + ], + 2 + ) + db.prepare( + `INSERT INTO deepchat_tape_search_fts ( + search_text, + name, + session_id, + entry_id, + kind, + source_type, + source_id, + source_seq, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'stale removed marker', + 'message/user', + 's1', + 1, + 'message', + 'message', + 'old', + 0, + 'stale removed marker', + '{"messageId":"old"}', + 100 + ) + + const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + restartedProjectionTable.createTable() + + expect(restartedProjectionTable.isCurrent('s1', 2)).toBe(true) + expect(restartedProjectionTable.search('s1', 'stale removed marker', { limit: 5 })).toEqual( + [] + ) + expect( + restartedProjectionTable.search('s1', 'current Redis marker', { limit: 5 })[0] + ).toMatchObject({ + entry_id: 2, + refs_json: '{"messageId":"current"}' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `recovers same-entry stale FTS after a base-only projection write and restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm1', + sourceSeq: 0, + searchText: 'old durable marker', + summaryText: 'old durable marker', + refs: { messageId: 'm1' }, + createdAt: 100 + } + ], + 1 + ) + + projectionTable.disableFtsForTesting() + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm1', + sourceSeq: 0, + searchText: 'new durable marker', + summaryText: 'new durable marker', + refs: { messageId: 'm1' }, + createdAt: 100 + } + ], + 1 + ) + db.prepare( + `INSERT INTO deepchat_tape_search_fts ( + search_text, + name, + session_id, + entry_id, + kind, + source_type, + source_id, + source_seq, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'old durable marker', + 'message/user', + 's1', + 1, + 'message', + 'message', + 'm1', + 0, + 'old durable marker', + '{"messageId":"m1"}', + 100 + ) + + const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + restartedProjectionTable.createTable() + + expect(restartedProjectionTable.isCurrent('s1', 1)).toBe(true) + expect(restartedProjectionTable.search('s1', 'old durable marker', { limit: 5 })).toEqual( + [] + ) + expect( + restartedProjectionTable.search('s1', 'new durable marker', { limit: 5 })[0] + ).toMatchObject({ + entry_id: 1, + search_text: 'new durable marker' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `rebuilds FTS during append when previous FTS meta is missing after restart${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.disableFtsForTesting() + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'old', + sourceSeq: 0, + searchText: 'old append marker', + summaryText: 'old append marker', + refs: { messageId: 'old' }, + createdAt: 100 + } + ], + 1 + ) + + const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + restartedProjectionTable.createTable() + restartedProjectionTable.appendSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'new', + sourceSeq: 0, + searchText: 'new append marker', + summaryText: 'new append marker', + refs: { messageId: 'new' }, + createdAt: 200 + } + ], + 2 + ) + + expect(restartedProjectionTable.isCurrent('s1', 2)).toBe(true) + expect( + restartedProjectionTable.search('s1', 'old append marker', { limit: 1 })[0] + ).toMatchObject({ + entry_id: 1, + refs_json: '{"messageId":"old"}' + }) + expect( + restartedProjectionTable.search('s1', 'new append marker', { limit: 1 })[0] + ).toMatchObject({ + entry_id: 2, + refs_json: '{"messageId":"new"}' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `rebuilds migrated tape FTS when freshness meta is excluded${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'old', + sourceSeq: 0, + searchText: 'old migrated marker', + summaryText: 'old migrated marker', + refs: { messageId: 'old' }, + createdAt: 100 + } + ], + 1 + ) + db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run('s1') + db.prepare('DELETE FROM deepchat_tape_search_fts_meta WHERE session_id = ?').run('s1') + + const migratedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + migratedProjectionTable.createTable() + migratedProjectionTable.appendSession( + 's1', + [ + { + sessionId: 's1', + entryId: 2, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'new', + sourceSeq: 0, + searchText: 'new migrated marker', + summaryText: 'new migrated marker', + refs: { messageId: 'new' }, + createdAt: 200 + } + ], + 2 + ) + + expect(migratedProjectionTable.isCurrent('s1', 2)).toBe(true) + expect( + migratedProjectionTable.search('s1', 'old migrated marker', { limit: 1 })[0] + ).toMatchObject({ + entry_id: 1, + refs_json: '{"messageId":"old"}' + }) + expect( + migratedProjectionTable.search('s1', 'new migrated marker', { limit: 1 })[0] + ).toMatchObject({ + entry_id: 2, + refs_json: '{"messageId":"new"}' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `keeps common-term FTS searches scoped and bounded on large session sets${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + + for (let index = 0; index < 180; index += 1) { + const sessionId = `s-${index}` + const rows = Array.from({ length: 8 }, (_, offset) => ({ + sessionId, + entryId: offset + 1, + kind: 'message' as const, + name: 'message/user', + sourceType: 'message' as const, + sourceId: `m-${index}-${offset}`, + sourceSeq: offset, + searchText: `sharedcommon marker session-${index} row-${offset}`, + summaryText: `sharedcommon marker session-${index} row-${offset}`, + refs: { messageId: `m-${index}-${offset}` }, + createdAt: index * 10 + offset + })) + projectionTable.replaceSession(sessionId, rows, rows.length) + } + + const planRows = db + .prepare( + `EXPLAIN QUERY PLAN + SELECT projection.session_id, + projection.entry_id, + projection.kind, + projection.name, + projection.source_type, + projection.source_id, + projection.source_seq, + projection.search_text, + projection.summary_text, + projection.refs_json, + projection.created_at, + bm25(deepchat_tape_search_fts) AS score + FROM deepchat_tape_search_fts + INNER JOIN deepchat_tape_search_projection AS projection + ON projection.session_id = deepchat_tape_search_fts.session_id + AND projection.entry_id = CAST(deepchat_tape_search_fts.entry_id AS INTEGER) + AND projection.search_text = deepchat_tape_search_fts.search_text + WHERE deepchat_tape_search_fts MATCH ? + AND deepchat_tape_search_fts.session_id = ? + AND projection.session_id = ? + ORDER BY score ASC, projection.entry_id DESC + LIMIT ?` + ) + .all('"sharedcommon"', 's-42', 's-42', 5) as Array<{ detail: string }> + const plan = planRows.map((row) => row.detail).join('\n') + + expect(plan).toMatch(/VIRTUAL TABLE INDEX/i) + expect(plan).toMatch(/SEARCH projection USING (?:COVERING )?INDEX/i) + expect(plan).not.toMatch(/\bSCAN projection\b/i) + + const startedAt = performance.now() + const hits = projectionTable.search('s-42', 'sharedcommon', { limit: 5 }) + const elapsedMs = performance.now() - startedAt + + expect(hits).toHaveLength(5) + expect(hits.every((hit) => hit.session_id === 's-42')).toBe(true) + expect(elapsedMs).toBeLessThan(1500) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `does not trust same-entry stale FTS text even when stale FTS meta is current${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm1', + sourceSeq: 0, + searchText: 'new guarded marker', + summaryText: 'new guarded marker', + refs: { messageId: 'm1' }, + createdAt: 100 + } + ], + 1 + ) + db.prepare('DELETE FROM deepchat_tape_search_fts WHERE session_id = ?').run('s1') + db.prepare( + `INSERT INTO deepchat_tape_search_fts ( + search_text, + name, + session_id, + entry_id, + kind, + source_type, + source_id, + source_seq, + summary_text, + refs_json, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'old guarded marker', + 'message/user', + 's1', + 1, + 'message', + 'message', + 'm1', + 0, + 'old guarded marker', + '{"messageId":"m1"}', + 100 + ) + + const restartedProjectionTable = new DeepChatTapeSearchProjectionTable(db) + restartedProjectionTable.createTable() + + expect(restartedProjectionTable.isCurrent('s1', 1)).toBe(true) + expect(restartedProjectionTable.search('s1', 'old guarded marker', { limit: 5 })).toEqual( + [] + ) + expect( + restartedProjectionTable.search('s1', 'new guarded marker', { limit: 5 })[0] + ).toMatchObject({ + entry_id: 1, + search_text: 'new guarded marker' + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `does not mark a tape projection current when FTS DML fails${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + projectionTable.createTable() + if (!projectionTable.hasFtsReadyForTesting()) { + return + } + projectionTable.dropFtsForTesting() + ;(projectionTable as any).ftsReady = true + + expect(() => + projectionTable.replaceSession( + 's1', + [ + { + sessionId: 's1', + entryId: 1, + kind: 'message', + name: 'message/user', + sourceType: 'message', + sourceId: 'm1', + sourceSeq: 0, + searchText: 'Redis TTL', + summaryText: 'Redis TTL', + refs: { messageId: 'm1' }, + createdAt: 100 + } + ], + 1 + ) + ).toThrow() + expect(projectionTable.isCurrent('s1', 1)).toBe(false) + expect(projectionTable.getProjectedEntryIds('s1')).toEqual([]) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `filters memory view manifests by message in SQLite before limit${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + table.createTable() + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + data: { ignored: true }, + meta: { messageId: 'msg-old' }, + createdAt: 999 + }) + for (let index = 0; index < 505; index += 1) { + table.appendAnchor({ + sessionId: 's1', + name: 'memory/view_assembled', + state: { + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: index, + selected: [`m-${index}`], + dropped: [], + queryHash: `hash-${index}` + }, + meta: { messageId: `msg-${index}` }, + createdAt: index + }) + } + + const rows = table.listMemoryViewManifestAnchorsBySessions(['s1'], { + limit: 1, + messageId: 'msg-0' + }) + + expect(rows).toHaveLength(1) + expect(rows[0]).toMatchObject({ + kind: 'anchor', + name: 'memory/view_assembled', + created_at: 0 + }) + expect(JSON.parse(rows[0].meta_json)).toEqual({ messageId: 'msg-0' }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `queries memory view manifests for large agents without expanding session parameters${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const sessionTable = new NewSessionsTable(db) + const tapeTable = new DeepChatTapeEntriesTable(db) + sessionTable.createTable() + tapeTable.createTable() + for (let index = 0; index < 1200; index += 1) { + const sessionId = `s-${index}` + db.prepare( + `INSERT INTO new_sessions ( + id, + agent_id, + title, + project_dir, + is_pinned, + is_draft, + active_skills, + disabled_agent_tools, + subagent_enabled, + session_kind, + parent_session_id, + subagent_meta_json, + created_at, + updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + sessionId, + 'agent-a', + `Session ${index}`, + null, + 0, + 0, + '[]', + '[]', + index % 2 === 0 ? 0 : 1, + index % 2 === 0 ? 'regular' : 'subagent', + null, + null, + index, + index + ) + tapeTable.appendAnchor({ + sessionId, + name: 'memory/view_assembled', + state: { + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: index, + selected: [`m-${index}`], + dropped: [], + queryHash: `hash-${index}` + }, + meta: { messageId: `msg-${index}` }, + createdAt: index + }) + } + db.prepare( + `INSERT INTO new_sessions ( + id, + agent_id, + title, + project_dir, + is_pinned, + is_draft, + active_skills, + disabled_agent_tools, + subagent_enabled, + session_kind, + parent_session_id, + subagent_meta_json, + created_at, + updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'other-session', + 'other-agent', + 'Other', + null, + 0, + 0, + '[]', + '[]', + 0, + 'regular', + null, + null, + 9999, + 9999 + ) + tapeTable.appendAnchor({ + sessionId: 'other-session', + name: 'memory/view_assembled', + state: { + policyVersion: 1, + tokenBudget: 1000, + estimatedTokens: 9999, + selected: ['other'], + dropped: [], + queryHash: 'other' + }, + meta: { messageId: 'msg-0' }, + createdAt: 9999 + }) + + const rows = tapeTable.listMemoryViewManifestAnchorsByAgent('agent-a', { + messageId: 'msg-0', + limit: 1 + }) + + expect(rows).toHaveLength(1) + expect(rows[0]).toMatchObject({ + session_id: 's-0', + kind: 'anchor', + name: 'memory/view_assembled', + created_at: 0 + }) + } finally { + db.close() + } + } + ) + + itIfSqlite( + `searches a SQLite tape projection and expands compact context without raw payloads${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const table = new DeepChatTapeEntriesTable(db) + const projectionTable = new DeepChatTapeSearchProjectionTable(db) + table.createTable() + projectionTable.createTable() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatTapeSearchProjectionTable: projectionTable, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'u1', seq: 0 }, + payload: { + record: createRecord({ + id: 'u1', + content: JSON.stringify({ + text: 'Check Redis TTL with /usr/local/bin/deploy --flag and error 42.', + files: [], + links: [] + }), + createdAt: 100, + updatedAt: 100 + }) + }, + meta: { source: 'live', orderSeq: 1, role: 'user' }, + createdAt: 100 + }) + table.append({ + sessionId: 's1', + kind: 'tool_result', + name: 'shell', + source: { type: 'tool_result', id: 'u1:tc1', seq: 0 }, + payload: { + messageId: 'u1', + orderSeq: 2, + toolCallId: 'tc1', + exitStatus: 42, + response: 'Exit code 42 in /tmp/deploy.log' + }, + meta: { source: 'live', status: 'error' }, + createdAt: 110 + }) + + const pathHits = service.search('s1', '/usr/local/bin/deploy', { limit: 5 }) + expect(pathHits).toHaveLength(1) + expect(pathHits[0]).toMatchObject({ + kind: 'message', + summary: expect.stringContaining('Redis TTL'), + refs: { + messageId: 'u1', + role: 'user', + filePaths: expect.arrayContaining(['/usr/local/bin/deploy']) + } + }) + expect(pathHits[0]).not.toHaveProperty('payload') + expect(pathHits[0]).not.toHaveProperty('meta') + expect(service.search('s1', 'Redis TTL', { limit: 5 }).map((hit) => hit.entryId)).toContain( + pathHits[0].entryId + ) + const errorHits = service.search('s1', '42', { kinds: ['tool_result'], limit: 5 }) + expect(errorHits[0]).toMatchObject({ + refs: { + toolCallId: 'tc1', + exitStatus: 42 + } + }) + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) + + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'u2', seq: 0 }, + payload: { + record: createRecord({ + id: 'u2', + orderSeq: 3, + content: JSON.stringify({ text: 'zoxide marker 简洁', files: [], links: [] }), + createdAt: 120, + updatedAt: 120 + }) + }, + meta: { source: 'live', orderSeq: 3, role: 'user' }, + createdAt: 120 + }) + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(false) + const rebuiltHits = service.search('s1', '简洁', { limit: 5 }) + expect(rebuiltHits.map((hit) => hit.refs?.messageId)).toContain('u2') + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) + if (projectionTable.hasFtsReadyForTesting()) { + projectionTable.dropFtsForTesting() + ;(projectionTable as any).ftsReady = true + table.append({ + sessionId: 's1', + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'u3', seq: 0 }, + payload: { + record: createRecord({ + id: 'u3', + orderSeq: 4, + content: JSON.stringify({ + text: 'fts recovery marker', + files: [], + links: [] + }), + createdAt: 130, + updatedAt: 130 + }) + }, + meta: { source: 'live', orderSeq: 4, role: 'user' }, + createdAt: 130 + }) + const recoveryHits = service.search('s1', 'fts recovery marker', { limit: 5 }) + expect(recoveryHits.map((hit) => hit.refs?.messageId)).toContain('u3') + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(false) + expect(projectionTable.hasFtsReadyForTesting()).toBe(false) + + const restoredHits = service.search('s1', 'fts recovery marker', { limit: 5 }) + expect(restoredHits.map((hit) => hit.refs?.messageId)).toContain('u3') + expect(projectionTable.isCurrent('s1', table.getMaxEntryId('s1'))).toBe(true) + expect(projectionTable.hasFtsReadyForTesting()).toBe(true) + } + + const context = service.getContext('s1', [pathHits[0].entryId], { + before: 0, + after: 1, + limit: 2, + maxBytesPerEntry: 24, + maxTotalBytes: 24 + }) + expect(context.matchedEntryIds).toEqual([pathHits[0].entryId]) + expect(context.entries[0]).toMatchObject({ + entryId: pathHits[0].entryId, + summary: expect.stringContaining('Redis TTL'), + evidence: { + truncated: true + } + }) + expect(context.entries[0].evidence.bytes).toBeLessThanOrEqual(24) + expect(context.entries[0]).not.toHaveProperty('payload') + expect(context.entries[0]).not.toHaveProperty('meta') + const limitedContext = service.getContext( + 's1', + [pathHits[0].entryId, errorHits[0].entryId], + { + before: 0, + after: 0, + limit: 1 + } + ) + expect(limitedContext.entries.map((entry) => entry.entryId)).toEqual([pathHits[0].entryId]) + expect(limitedContext.matchedEntryIds).toEqual([pathHits[0].entryId]) + } finally { + db.close() + } + } + ) +}) diff --git a/test/main/session/data/tapeReconciler.test.ts b/test/main/session/data/tapeReconciler.test.ts new file mode 100644 index 0000000000..28bf1077fd --- /dev/null +++ b/test/main/session/data/tapeReconciler.test.ts @@ -0,0 +1,399 @@ +import { + describe, + expect, + it, + vi, + buildContext, + toAppSessionId, + SessionTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + createTapeTableMock, + createRecord, + createTapeService +} from './tapeTestHarness' + +describe('SessionTape reconciliation and facts', () => { + it('keeps unkeyed idempotent harness appends distinct like the SQLite store', () => { + const { table, entries } = createTapeTableMock() + const input = { + sessionId: 's1', + kind: 'event', + name: 'unkeyed', + payload: { value: 1 }, + idempotent: true + } + + table.append(input) + table.append(input) + + expect(entries).toHaveLength(2) + expect(entries.map((entry) => entry.entry_id)).toEqual([1, 2]) + }) + + it('backfills message and tool facts idempotently before returning tape records', () => { + const { table, entries } = createTapeTableMock() + const assistantBlocks = [ + { + type: 'tool_call', + status: 'success', + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } + } + ] + const records = [ + createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify(assistantBlocks), + createdAt: 120, + updatedAt: 120 + }), + createRecord({ id: 'u1', orderSeq: 1 }) + ] + const messageStore = { + getMessages: vi.fn().mockReturnValue(records) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const first = service.ensureSessionTapeReady('s1', messageStore as any) + const second = service.ensureSessionTapeReady('s1', messageStore as any) + + expect(first.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) + expect(second.historyRecords.map((record) => record.id)).toEqual(['u1', 'a1']) + expect(records.map((record) => record.id)).toEqual(['a1', 'u1']) + expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) + expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) + expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) + expect(entries.filter((entry) => entry.name === 'migration/backfill')).toHaveLength(1) + }) + + it('appends live tool facts through the stable recorder port idempotently', async () => { + const { table, entries } = createTapeTableMock() + const service = createTapeService(table) + const input = { + sessionId: toAppSessionId('s1'), + messageId: 'a1', + orderSeq: 2, + blockIndex: 0, + block: { + type: 'tool_call' as const, + status: 'success' as const, + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } + }, + provenance: { source: 'tool_call' as const, sourceId: 'a1:tc1', sequence: 0 } + } + + const first = await service.appendToolFact(input) + const second = await service.appendToolFact(input) + + expect(second).toEqual(first) + expect(entries.filter((entry) => entry.kind === 'tool_call')).toHaveLength(1) + expect(JSON.parse(entries.find((entry) => entry.kind === 'tool_call').meta_json)).toEqual({ + source: 'live', + role: 'assistant', + status: 'success', + reason: 'tool_loop' + }) + }) + + it('keeps legacy context builder output stable after tape backfill projection', () => { + const { table } = createTapeTableMock() + const records = [ + createRecord({ id: 'u1', orderSeq: 1 }), + createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { type: 'content', content: 'Tool finished', status: 'success', timestamp: 120 }, + { + type: 'tool_call', + status: 'success', + timestamp: 121, + tool_call: { + id: 'tc1', + name: 'example_tool', + params: '{"foo":"bar"}', + response: 'All good' + } + } + ]), + createdAt: 120, + updatedAt: 121 + }) + ] + const legacyMessageStore = { + getMessages: vi.fn().mockReturnValue(records) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + const legacyContext = buildContext( + 's1', + { text: 'next', files: [] }, + 'System', + 10000, + 4096, + legacyMessageStore as any + ) + const tapeReady = service.ensureSessionTapeReady('s1', legacyMessageStore as any) + const tapeOnlyStore = { + getMessages: vi.fn(() => { + throw new Error('buildContext must use provided tape history records') + }) + } + const tapeContext = buildContext( + 's1', + { text: 'next', files: [] }, + 'System', + 10000, + 4096, + tapeOnlyStore as any, + false, + { + historyRecords: tapeReady.historyRecords + } + ) + + expect(tapeContext).toEqual(legacyContext) + expect(tapeOnlyStore.getMessages).not.toHaveBeenCalled() + }) + + it('rejects handoff anchors without a non-empty summary before writing Tape state', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + expect(() => service.handoff('s1', 'phase_done', { summary: ' ' })).toThrow( + 'Tape handoff requires a non-empty summary.' + ) + expect(() => service.handoff('s1', 'phase_done', { reason: 'phase complete' } as any)).toThrow( + 'Tape handoff requires a non-empty summary.' + ) + + expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() + expect(table.appendAnchor).not.toHaveBeenCalled() + expect(entries).toEqual([]) + }) + + it('migrates legacy session summary into a tape anchor during backfill', () => { + const { table, entries } = createTapeTableMock() + const messageStore = { + getMessages: vi.fn().mockReturnValue([ + createRecord({ id: 'u1', orderSeq: 1 }), + createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([{ type: 'content', content: 'answer', status: 'success' }]) + }) + ]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { + getSummaryState: vi.fn().mockReturnValue({ + summary_text: 'legacy compacted state', + summary_cursor_order_seq: 3, + summary_updated_at: 200 + }) + } + } as any) + + service.ensureSessionTapeReady('s1', messageStore as any) + + const summaryAnchor = entries.find((entry) => entry.name === 'compaction/migrated_summary') + expect(summaryAnchor).toMatchObject({ + kind: 'anchor', + source_type: 'summary', + source_id: 'legacy-summary', + created_at: 200 + }) + expect(JSON.parse(summaryAnchor.payload_json).state).toMatchObject({ + summary: 'legacy compacted state', + cursorOrderSeq: 3, + sourceMessageIds: ['u1', 'a1'] + }) + }) + + it('keeps pending message records for resume but hides pending tool facts from search', () => { + const { table } = createTapeTableMock() + const pendingBlocks = [ + { + type: 'tool_call', + status: 'pending', + timestamp: 100, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'pending result' + } + } + ] + const messageStore = { + getMessages: vi.fn().mockReturnValue([ + createRecord({ + id: 'a1', + orderSeq: 1, + role: 'assistant', + status: 'pending', + content: JSON.stringify(pendingBlocks), + updatedAt: 100 + }) + ]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.ensureSessionTapeReady('s1', messageStore as any) + + expect(service.getMessageRecords('s1')).toMatchObject([{ id: 'a1', status: 'pending' }]) + expect(service.search('s1', 'pending result', { kinds: ['tool_result'] })).toEqual([]) + }) + + it('lets final assistant facts supersede earlier pending tape facts', () => { + const { table, entries } = createTapeTableMock() + const pendingBlocks = [ + { + type: 'tool_call', + status: 'pending', + timestamp: 100, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'pending result' + } + } + ] + const finalBlocks = [ + { + type: 'tool_call', + status: 'success', + timestamp: 200, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'final result' + } + } + ] + const messageStore = { + getMessages: vi + .fn() + .mockReturnValueOnce([ + createRecord({ + id: 'a1', + orderSeq: 1, + role: 'assistant', + status: 'pending', + content: JSON.stringify(pendingBlocks), + metadata: JSON.stringify({ totalTokens: 1 }), + updatedAt: 100 + }) + ]) + .mockReturnValue([ + createRecord({ + id: 'a1', + orderSeq: 1, + role: 'assistant', + status: 'sent', + content: JSON.stringify(finalBlocks), + metadata: JSON.stringify({ totalTokens: 7 }), + updatedAt: 200 + }) + ]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.ensureSessionTapeReady('s1', messageStore as any) + service.ensureSessionTapeReady('s1', messageStore as any) + + expect(service.getMessageRecords('s1')).toMatchObject([ + { + id: 'a1', + status: 'sent' + } + ]) + const effectiveRecord = service.getMessageRecords('s1')[0]! + expect(JSON.parse(effectiveRecord.content)[0].tool_call.response).toBe('final result') + expect( + entries.filter((entry) => entry.kind === 'message' && entry.name === 'message/assistant') + ).toHaveLength(2) + expect(entries.filter((entry) => entry.kind === 'tool_result')).toHaveLength(1) + const finalToolResult = entries.filter((entry) => entry.kind === 'tool_result').at(-1)! + expect(JSON.parse(finalToolResult.payload_json).response).toBe('final result') + expect(service.info('s1').lastTokenUsage).toBe(7) + expect(service.search('s1', 'pending result', { kinds: ['tool_result'] })).toEqual([]) + expect(service.search('s1', 'final result', { kinds: ['tool_result'] })).toHaveLength(1) + }) + + it('uses effective message facts after replacement and retraction events', () => { + const { table, entries } = createTapeTableMock() + const original = createRecord({ id: 'u1', orderSeq: 1 }) + const messageStore = { + getMessages: vi.fn().mockReturnValue([original]) + } + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + service.ensureSessionTapeReady('s1', messageStore as any) + appendMessageReplacementToTape( + table as any, + createRecord({ + id: 'u1', + orderSeq: 1, + content: JSON.stringify({ + text: 'edited', + files: [], + links: [], + search: false, + think: false + }), + updatedAt: 300 + }), + 'test_edit' + ) + + expect(JSON.parse(service.getMessageRecords('s1')[0].content).text).toBe('edited') + expect(service.search('s1', 'hello', { kinds: ['message'] })).toEqual([]) + expect(service.search('s1', 'edited', { kinds: ['message'] })).toHaveLength(1) + expect(entries.filter((entry) => entry.kind === 'message')).toHaveLength(2) + + appendMessageRetractionToTape(table as any, service.getMessageRecords('s1')[0], 'test_delete') + + expect(service.getMessageRecords('s1')).toEqual([]) + expect(service.search('s1', 'edited', { kinds: ['message'] })).toEqual([]) + }) + + it('appends non-idempotent retractions without generated provenance keys', () => { + const { table, entries } = createTapeTableMock() + const record = createRecord({ id: 'u1' }) + + appendMessageRetractionToTape(table as any, record, 'first_delete') + appendMessageRetractionToTape(table as any, record, 'second_delete') + + const retractions = entries.filter((entry) => entry.name === 'message/retracted') + expect(retractions).toHaveLength(2) + expect(retractions.map((entry) => entry.provenance_key)).toEqual([null, null]) + }) +}) diff --git a/test/main/session/data/tapeTestHarness.ts b/test/main/session/data/tapeTestHarness.ts new file mode 100644 index 0000000000..5853059a3b --- /dev/null +++ b/test/main/session/data/tapeTestHarness.ts @@ -0,0 +1,654 @@ +import { performance } from 'node:perf_hooks' +import { describe, expect, it, vi } from 'vitest' +import { buildContext } from '@/agent/deepchat/runtime/contextBuilder' +import { toAppSessionId } from '@/agent/shared/agentSessionIds' +import { SessionTape } from '@/session/data/tape' +import { buildEffectiveTapeView, searchEffectiveTapeRows } from '@/session/data/tapeEffectiveView' +import { + createTapeViewManifest, + type TapeViewManifestBuildInput +} from '@/session/data/tapeViewManifest' +import { + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + appendToolFactsToTape +} from '@/session/data/tapeFacts' +import { buildRequestRefs } from '@/session/data/tapeViewManifest' +import { DeepChatTapeEntriesTable } from '@/session/data/tables/deepchatTapeEntries' +import { SqliteTapeLifecycleAdapter } from '@/tape/infrastructure/sqlite/tapeLifecycleAdapter' +import { + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, + DeepChatTapeSearchProjectionTable +} from '@/session/data/tables/deepchatTapeSearchProjection' +import { DeepChatMemoryIngestionProjectionTable } from '@/memory/data/tables/deepchatMemoryIngestionProjection' +import { DeepChatMessagesTable } from '@/session/data/tables/deepchatMessages' +import { DeepChatMessageTracesTable } from '@/session/data/tables/deepchatMessageTraces' +import { DeepChatSessionsTable } from '@/session/data/tables/deepchatSessions' +import { NewSessionsTable } from '@/session/data/tables/newSessions' +import type { ChatMessageRecord } from '@shared/types/agent-interface' + +const sqliteModule = await import('better-sqlite3-multiple-ciphers').catch(() => null) +const Database = sqliteModule?.default +const DatabaseCtor = Database! +const sqliteSkipReason = 'skipped: better-sqlite3-multiple-ciphers is unavailable' +const requireNativeSqlite = process.env.DEEPCHAT_REQUIRE_NATIVE_SQLITE === '1' + +let sqliteAvailable = false +if (Database) { + try { + const smokeDb = new Database(':memory:') + smokeDb.close() + sqliteAvailable = true + } catch { + sqliteAvailable = false + } +} + +const itIfSqlite = sqliteAvailable + ? it + : requireNativeSqlite + ? (name: string, _test: () => unknown, timeout?: number) => + it( + name, + () => { + throw new Error(sqliteSkipReason) + }, + timeout + ) + : it.skip + +function createTapeTableMock() { + const entries: any[] = [] + let tapeIncarnationSequence = 0 + const table = { + ensureBootstrapAnchor: vi.fn((sessionId: string) => { + if ( + entries.some((entry) => entry.session_id === sessionId && entry.name === 'session/start') + ) { + return + } + table.appendAnchor({ + sessionId, + name: 'session/start', + source: { type: 'session', id: sessionId, seq: 0 }, + state: { owner: 'human' }, + meta: { tapeIncarnationId: `test-tape-${++tapeIncarnationSequence}` }, + idempotent: true + }) + }), + append: vi.fn((input: any) => { + const provenanceKey = + input.provenanceKey !== undefined + ? input.provenanceKey + : input.source + ? [ + input.source.type, + input.source.id, + input.source.seq ?? 0, + input.kind, + input.name ?? '' + ].join(':') + : null + const existing = + input.idempotent && provenanceKey + ? entries.find( + (entry) => + entry.session_id === input.sessionId && entry.provenance_key === provenanceKey + ) + : null + if (existing) { + return existing + } + const row = { + session_id: input.sessionId, + entry_id: + Math.max( + 0, + ...entries + .filter((entry) => entry.session_id === input.sessionId) + .map((entry) => entry.entry_id) + ) + 1, + kind: input.kind, + name: input.name ?? null, + source_type: input.source?.type ?? null, + source_id: input.source?.id ?? null, + source_seq: input.source?.seq ?? null, + provenance_key: provenanceKey, + payload_json: JSON.stringify(input.payload ?? {}), + meta_json: JSON.stringify(input.meta ?? {}), + created_at: input.createdAt ?? Date.now() + } + entries.push(row) + return row + }), + appendAnchor: vi.fn((input: any) => + table.append({ + ...input, + kind: 'anchor', + payload: { name: input.name, state: input.state } + }) + ), + appendEvent: vi.fn((input: any) => + table.append({ + ...input, + kind: 'event', + payload: { name: input.name, data: input.data } + }) + ), + runInTransaction: vi.fn((operation: () => unknown) => { + const snapshot = entries.map((entry) => ({ ...entry })) + try { + return operation() + } catch (error) { + entries.splice(0, entries.length, ...snapshot) + throw error + } + }), + getBySession: vi.fn((sessionId: string) => + entries.filter((entry) => entry.session_id === sessionId) + ), + listMemoryViewManifestAnchorsByAgent: vi.fn( + ( + _agentId: string, + _options?: { sessionId?: string; limit?: number; messageId?: string } + ): any[] => { + throw new Error('configure listMemoryViewManifestAnchorsByAgent for this test') + } + ), + getSubagentLineageEvents: vi.fn((sessionId: string) => + entries.filter( + (entry) => + entry.session_id === sessionId && + entry.kind === 'event' && + (entry.name === 'subagent/tape_linked' || entry.name === 'fork/merge') + ) + ), + getFirstEntriesBySessions: vi.fn((sessionIds: string[]) => + [...new Set(sessionIds)] + .flatMap((sessionId) => { + const first = entries + .filter((entry) => entry.session_id === sessionId) + .sort((left, right) => left.entry_id - right.entry_id)[0] + return first ? [first] : [] + }) + .sort((left, right) => left.session_id.localeCompare(right.session_id)) + ), + getBySessionUpToEntryId: vi.fn((sessionId: string, maxEntryId: number) => + entries.filter((entry) => entry.session_id === sessionId && entry.entry_id <= maxEntryId) + ), + getMaxEntryId: vi.fn((sessionId: string) => + Math.max( + 0, + ...entries.filter((entry) => entry.session_id === sessionId).map((entry) => entry.entry_id) + ) + ), + getMaxEntryIdsBySessions: vi.fn( + (sessionIds: string[]) => + new Map( + sessionIds.map((sessionId) => [ + sessionId, + Math.max( + 0, + ...entries + .filter((entry) => entry.session_id === sessionId) + .map((entry) => entry.entry_id) + ) + ]) + ) + ), + getLatestAnchor: vi.fn( + (sessionId: string) => + entries + .filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor') + .sort((left, right) => right.entry_id - left.entry_id)[0] + ), + getAnchors: vi.fn((sessionId: string, limit: number = 20) => + entries + .filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor') + .sort((left, right) => right.entry_id - left.entry_id) + .slice(0, Math.min(Math.max(Math.floor(limit), 1), 100)) + .reverse() + ), + getLatestSummaryAnchor: vi.fn( + (sessionId: string) => + entries + .filter( + (entry) => + entry.session_id === sessionId && + entry.kind === 'anchor' && + ['compaction/migrated_summary', 'compaction/manual', 'summary/reset'].includes( + entry.name + ) + ) + .sort((left, right) => right.entry_id - left.entry_id)[0] + ), + getByProvenanceKey: vi.fn((sessionId: string, provenanceKey: string) => + entries.find( + (entry) => entry.session_id === sessionId && entry.provenance_key === provenanceKey + ) + ), + countBySession: vi.fn( + (sessionId: string) => entries.filter((entry) => entry.session_id === sessionId).length + ), + countAnchorsBySession: vi.fn( + (sessionId: string) => + entries.filter((entry) => entry.session_id === sessionId && entry.kind === 'anchor').length + ), + countEntriesAfter: vi.fn( + (sessionId: string, entryId: number) => + entries.filter((entry) => entry.session_id === sessionId && entry.entry_id > entryId).length + ), + search: vi.fn((sessionId: string, query: string, options: any = {}) => { + const normalizedQuery = query.trim() + if (!normalizedQuery) { + return [] + } + const limit = Number.isFinite(options.limit) ? Math.floor(options.limit) : 20 + return entries + .filter((entry) => entry.session_id === sessionId) + .filter( + (entry) => + entry.payload_json.includes(normalizedQuery) || + entry.meta_json.includes(normalizedQuery) || + entry.name?.includes(normalizedQuery) + ) + .filter((entry) => !options.kinds?.length || options.kinds.includes(entry.kind)) + .filter( + (entry) => + !Number.isFinite(options.startCreatedAt) || entry.created_at >= options.startCreatedAt + ) + .filter( + (entry) => + !Number.isFinite(options.endCreatedAt) || entry.created_at <= options.endCreatedAt + ) + .sort((left, right) => right.entry_id - left.entry_id) + .slice(0, Math.min(Math.max(limit, 1), 100)) + }), + searchEffectiveSourcesAtHeads: vi.fn((sources: any[], query: string, options: any = {}) => + sources + .flatMap((source) => + searchEffectiveTapeRows( + entries.filter( + (entry) => + entry.session_id === source.sessionId && entry.entry_id <= source.maxEntryId + ), + query, + { ...options, limit: 100 } + ) + ) + .sort( + (left, right) => + right.created_at - left.created_at || + left.session_id.localeCompare(right.session_id) || + right.entry_id - left.entry_id + ) + .slice(0, Math.min(Math.max(Math.floor(options.limit ?? 20), 1), 100)) + ), + getEffectiveContextRowsAtHead: vi.fn( + ( + source: any, + entryIds: number[], + options: { before: number; after: number; limit: number } + ) => { + const effectiveRows = buildEffectiveTapeView( + entries.filter( + (entry) => entry.session_id === source.sessionId && entry.entry_id <= source.maxEntryId + ), + { includePending: false } + ).rows + const indexesByEntryId = new Map( + effectiveRows.map((entry, index) => [entry.entry_id, index]) + ) + const indexes: number[] = [] + for (const entryId of entryIds) { + const index = indexesByEntryId.get(entryId) + if (index !== undefined) indexes.push(index) + } + for (const entryId of entryIds) { + const index = indexesByEntryId.get(entryId) + if (index === undefined) continue + for ( + let cursor = Math.max(0, index - options.before); + cursor <= Math.min(effectiveRows.length - 1, index + options.after); + cursor += 1 + ) { + if (cursor !== index) indexes.push(cursor) + } + } + return [...new Set(indexes)].slice(0, options.limit).map((index) => effectiveRows[index]) + } + ), + deleteBySession: vi.fn((sessionId: string) => { + for (let index = entries.length - 1; index >= 0; index -= 1) { + if (entries[index].session_id === sessionId) { + entries.splice(index, 1) + } + } + }) + } + return { table, entries } +} + +function createRecord(overrides: Partial): ChatMessageRecord { + return { + id: 'm1', + sessionId: 's1', + orderSeq: 1, + role: 'user', + content: JSON.stringify({ text: 'hello', files: [], links: [], search: false, think: false }), + status: 'sent', + isContextEdge: 0, + metadata: '{}', + traceCount: 0, + createdAt: 100, + updatedAt: 100, + ...overrides + } +} + +function createTraceRow(overrides: Record = {}) { + return { + id: 'trace-1', + message_id: 'a1', + session_id: 's1', + provider_id: 'openai', + model_id: 'gpt-4o', + request_seq: 1, + endpoint: 'https://api.openai.test/v1/chat/completions', + headers_json: '{"authorization":"[redacted]"}', + body_json: '{"messages":[{"role":"user","content":"hello"}]}', + truncated: 0, + created_at: 300, + ...overrides + } +} + +function createMessageRow(overrides: Record = {}) { + return { + id: 'a1', + session_id: 's1', + order_seq: 2, + role: 'assistant', + content: '[{"type":"content","content":"done","status":"success"}]', + status: 'sent', + is_context_edge: 0, + metadata: '{"totalTokens":10}', + created_at: 200, + updated_at: 300, + ...overrides + } +} + +function createObservationManifest( + overrides: Partial[0]> = {} +) { + return createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user', content: 'hello' }], + tools: [], + latestEntryId: 0, + anchorEntryIds: [], + included: [], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: true, + assembledAt: 200, + ...overrides + }) +} + +function createTapeService( + table: unknown, + traceRows: Array> = [], + messageRows: Array> = [] +) { + return new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: { + deleteBySession: vi.fn(), + isCurrent: vi.fn(() => { + throw new Error('projection unavailable') + }), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) + }, + deepchatMessageTracesTable: { + listByMessageId: vi.fn((messageId: string) => + traceRows.filter((row) => row.message_id === messageId) + ) + }, + deepchatMessagesTable: { + get: vi.fn((messageId: string) => messageRows.find((row) => row.id === messageId)) + }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) +} + +function createLinkedTapeService( + table: unknown, + sessions: Array<{ + id: string + session_kind: 'regular' | 'subagent' + parent_session_id: string | null + }>, + projectionTable?: unknown +) { + const sessionById = new Map(sessions.map((session) => [session.id, session])) + return { + service: new SessionTape({ + deepchatTapeEntriesTable: table, + tapeLifecycle: table, + deepchatTapeSearchProjectionTable: projectionTable, + newSessionsTable: { + get: vi.fn((sessionId: string) => sessionById.get(sessionId)), + getMany: vi.fn((sessionIds: string[]) => + sessionIds.flatMap((sessionId) => { + const session = sessionById.get(sessionId) + return session ? [session] : [] + }) + ) + }, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any), + sessionById + } +} + +function createSubagentLinkInput(parentSessionId: string, childSessionId: string) { + return { + parentSessionId, + childSessionId, + runId: `run-${childSessionId}`, + taskId: `task-${childSessionId}`, + slotId: 'reviewer', + taskTitle: `Review ${childSessionId}`, + outcome: 'completed' as const, + resultSummary: 'Done' + } +} + +function appendObservationIsolationFacts(table: unknown) { + const original = createRecord({ id: 'u1', orderSeq: 1, createdAt: 100, updatedAt: 100 }) + const edited = createRecord({ + id: 'u1', + orderSeq: 1, + content: JSON.stringify({ + text: 'edited', + files: [], + links: [], + search: false, + think: false + }), + createdAt: 100, + updatedAt: 150 + }) + const retracted = createRecord({ id: 'u2', orderSeq: 2, createdAt: 160, updatedAt: 160 }) + const pending = createRecord({ + id: 'a1', + orderSeq: 3, + role: 'assistant', + status: 'pending', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'pending', + timestamp: 200, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}' } + } + ]), + createdAt: 200, + updatedAt: 200 + }) + const final = createRecord({ + id: 'a1', + orderSeq: 3, + role: 'assistant', + status: 'sent', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 300, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'tape-result-secret' + } + } + ]), + metadata: '{"totalTokens":12}', + createdAt: 200, + updatedAt: 300 + }) + + appendMessageRecordToTape(table as any, original, 'live') + appendMessageReplacementToTape(table as any, edited, 'test_edit') + appendMessageRecordToTape(table as any, retracted, 'live') + appendMessageRetractionToTape(table as any, retracted, 'test_delete') + appendMessageRecordToTape(table as any, pending, 'live') + appendMessageRecordToTape(table as any, final, 'live') + + return { edited, final } +} + +function stripObservationPayloadOptIns(value: T): T { + const copy = structuredClone(value) as any + const stripEntryPayloads = (entries: any[] | undefined) => { + for (const entry of entries ?? []) { + delete entry.payload + delete entry.meta + } + } + + if (copy.request?.state === 'manifest_bound') { + stripEntryPayloads(copy.request.replay.entries) + delete copy.request.replay.hashes.sliceHash + if (copy.request.replay.trace) { + delete copy.request.replay.trace.headersJson + delete copy.request.replay.trace.bodyJson + } + } else if (copy.request?.trace) { + delete copy.request.trace.headersJson + delete copy.request.trace.bodyJson + } + stripEntryPayloads(copy.output?.entries) + return copy +} + +function createSpies(names: string[]) { + return Object.fromEntries(names.map((name) => [name, vi.fn()])) as Record< + string, + ReturnType + > +} + +function trackMemoryPropertyAccess(target: T) { + const memoryPropertyAccess = vi.fn() + return { + memoryPropertyAccess, + presenter: new Proxy(target, { + get(value, property, receiver) { + if (typeof property === 'string' && /memory/i.test(property)) { + memoryPropertyAccess(property) + } + return Reflect.get(value, property, receiver) + } + }) + } +} + +function readObservationMatrix(service: SessionTape) { + return { + defaultObservation: service.readCausalObservationSlice('s1', 'a1'), + repeatedObservation: service.readCausalObservationSlice('s1', 'a1'), + explicitObservation: service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }), + optInObservation: service.readCausalObservationSlice('s1', 'a1', { + includeTapePayloads: true, + includeTracePayload: true + }), + traceOnlyObservation: service.readCausalObservationSlice('s1', 'a-trace') + } +} + +export { + performance, + describe, + expect, + it, + vi, + buildContext, + toAppSessionId, + SessionTape, + buildEffectiveTapeView, + searchEffectiveTapeRows, + createTapeViewManifest, + appendMessageRecordToTape, + appendMessageReplacementToTape, + appendMessageRetractionToTape, + appendToolFactsToTape, + buildRequestRefs, + DeepChatTapeEntriesTable, + SqliteTapeLifecycleAdapter, + DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION, + DeepChatTapeSearchProjectionTable, + DeepChatMemoryIngestionProjectionTable, + DeepChatMessagesTable, + DeepChatMessageTracesTable, + DeepChatSessionsTable, + NewSessionsTable, + DatabaseCtor, + sqliteAvailable, + sqliteSkipReason, + itIfSqlite, + createTapeTableMock, + createRecord, + createTraceRow, + createMessageRow, + createObservationManifest, + createTapeService, + createLinkedTapeService, + createSubagentLinkInput, + appendObservationIsolationFacts, + stripObservationPayloadOptIns, + createSpies, + trackMemoryPropertyAccess, + readObservationMatrix +} diff --git a/test/main/session/data/tapeViewReplay.test.ts b/test/main/session/data/tapeViewReplay.test.ts new file mode 100644 index 0000000000..1af5777a55 --- /dev/null +++ b/test/main/session/data/tapeViewReplay.test.ts @@ -0,0 +1,1578 @@ +import type { TapeViewManifestBuildInput } from '@/tape/domain/viewManifest' +import type { ChatMessageRecord } from '@shared/types/agent-interface' +import { + describe, + expect, + it, + vi, + SessionTape, + createTapeViewManifest, + appendMessageRecordToTape, + appendToolFactsToTape, + buildRequestRefs, + DeepChatTapeEntriesTable, + DeepChatMemoryIngestionProjectionTable, + DeepChatMessagesTable, + DeepChatMessageTracesTable, + DeepChatSessionsTable, + DatabaseCtor, + sqliteAvailable, + sqliteSkipReason, + itIfSqlite, + createTapeTableMock, + createRecord, + createTraceRow, + createMessageRow, + createObservationManifest, + createTapeService, + appendObservationIsolationFacts, + stripObservationPayloadOptIns, + createSpies, + trackMemoryPropertyAccess, + readObservationMatrix +} from './tapeTestHarness' + +describe('SessionTape view and replay', () => { + it('stores and lists view manifests as idempotent tape events', () => { + const { table, entries } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user', + source: 'tape', + reason: 'selected_history' + } + ], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + + const first = service.appendViewManifest(manifest) + const second = service.appendViewManifest(manifest) + + expect(second.entry_id).toBe(first.entry_id) + expect(entries.filter((entry) => entry.name === 'view/assembled')).toHaveLength(1) + expect(JSON.parse(first.meta_json)).toMatchObject({ + policy: 'legacy_context_v1', + policyVersion: 1 + }) + expect(service.listViewManifestsByMessage('s1', 'a1')).toMatchObject([ + { + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + entryId: first.entry_id, + manifest: { + hashes: { + manifestHash: manifest.hashes.manifestHash + }, + policy: 'legacy_context_v1', + policyVersion: 1, + included: [ + { + messageId: 'u1', + entryId: sourceMaps.entryIdByMessageId.get('u1') + } + ] + } + } + ]) + }) + + it('projects memory view anchors into inspection DTOs', () => { + const { table, entries } = createTapeTableMock() + table.listMemoryViewManifestAnchorsByAgent = vi.fn( + (_agentId: string, options?: { messageId?: string }) => + entries.filter( + (entry) => + entry.kind === 'anchor' && + entry.name === 'memory/view_assembled' && + (!options?.messageId || JSON.parse(entry.meta_json).messageId === options.messageId) + ) + ) + const service = new SessionTape({ deepchatTapeEntriesTable: table } as any) + table.appendAnchor({ + sessionId: 's1', + name: 'memory/view_assembled', + state: { + policyVersion: 2, + tokenBudget: 1000, + estimatedTokens: 15, + selected: [ + 'm-string', + { id: 'm-object' }, + 'm-string', + { id: 'm-object' }, + { ignored: true }, + 3 + ], + dropped: ['m-dropped'], + queryHash: 'query-hash' + }, + meta: { messageId: 'msg-1' }, + createdAt: 300 + }) + + const manifests = service.listMemoryViewManifestsByAgent('agent-1', { + sessionId: 's1', + messageId: 'msg-1', + limit: 1 + }) + + expect(table.listMemoryViewManifestAnchorsByAgent).toHaveBeenCalledWith('agent-1', { + sessionId: 's1', + messageId: 'msg-1', + limit: 1 + }) + expect(manifests).toEqual([ + { + sessionId: 's1', + messageId: 'msg-1', + entryId: 1, + policyVersion: 2, + tokenBudget: 1000, + estimatedTokens: 15, + selectedCount: 6, + selectedIds: ['m-string', 'm-object'], + droppedCount: 1, + queryHash: 'query-hash', + createdAt: 300 + } + ]) + expect(manifests[0]).not.toHaveProperty('payload_json') + expect(manifests[0]).not.toHaveProperty('meta_json') + }) + + it('indexes effective tool facts so tool-loop manifests reference real entries', () => { + const { table } = createTapeTableMock() + const assistantRecord = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } + } + ]) + }) + appendToolFactsToTape(table as any, assistantRecord, 'live', 'tool_loop') + + const service = createTapeService(table) + const sourceMaps = service.getViewManifestSourceMaps('s1') + expect(sourceMaps.toolCallEntryIdByToolId.get('tc1')).toBeGreaterThan(0) + expect(sourceMaps.toolResultEntryIdByToolId.get('tc1')).toBeGreaterThan(0) + + const refs = buildRequestRefs( + [ + { role: 'system', content: 'system' }, + { + role: 'assistant', + content: '', + tool_calls: [ + { id: 'tc1', type: 'function', function: { name: 'search', arguments: '{"q":"x"}' } } + ] + }, + { role: 'tool', content: 'result', tool_call_id: 'tc1' } + ], + sourceMaps + ) + expect(refs).toMatchObject([ + { role: 'system', source: 'synthetic' }, + { + role: 'assistant', + source: 'tape', + reason: 'tool_loop_message', + entryId: sourceMaps.toolCallEntryIdByToolId.get('tc1') + }, + { + role: 'tool', + source: 'tape', + reason: 'tool_loop_message', + entryId: sourceMaps.toolResultEntryIdByToolId.get('tc1') + } + ]) + }) + + it('scopes tool source maps to the in-flight message so reused tool ids do not collide', () => { + const { table } = createTapeTableMock() + const blocks = (response: string) => + JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response } + } + ]) + appendToolFactsToTape( + table as any, + createRecord({ id: 'a1', orderSeq: 2, role: 'assistant', content: blocks('first') }), + 'live', + 'tool_loop' + ) + appendToolFactsToTape( + table as any, + createRecord({ id: 'a2', orderSeq: 4, role: 'assistant', content: blocks('second') }), + 'live', + 'tool_loop' + ) + + const service = createTapeService(table) + const scopedToA1 = service.getViewManifestSourceMaps('s1', 'a1') + const scopedToA2 = service.getViewManifestSourceMaps('s1', 'a2') + + expect(scopedToA1.toolCallEntryIdByToolId.get('tc1')).toBeLessThan( + scopedToA2.toolCallEntryIdByToolId.get('tc1')! + ) + expect(scopedToA1.toolResultEntryIdByToolId.get('tc1')).not.toBe( + scopedToA2.toolResultEntryIdByToolId.get('tc1') + ) + }) + + it('exports tool_call and tool_result entries in a tool-loop replay slice', () => { + const { table } = createTapeTableMock() + const assistantRecord = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 120, + tool_call: { id: 'tc1', name: 'search', params: '{"q":"x"}', response: 'result' } + } + ]) + }) + appendToolFactsToTape(table as any, assistantRecord, 'live', 'tool_loop') + + const service = createTapeService(table) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const messages = [ + { role: 'system' as const, content: 'system' }, + { + role: 'assistant' as const, + content: '', + tool_calls: [ + { id: 'tc1', type: 'function' as const, function: { name: 'search', arguments: '{}' } } + ] + }, + { role: 'tool' as const, content: 'result', tool_call_id: 'tc1' } + ] + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: 1, + messages, + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: buildRequestRefs(messages, sourceMaps), + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + service.appendViewManifest(manifest) + + const slice = service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }) + const kinds = slice?.entries.map((entry) => entry.kind) ?? [] + expect(kinds).toContain('tool_call') + expect(kinds).toContain('tool_result') + }) + + it('filters malformed view manifest rows when listing by message', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { + type: 'runtime_event', + id: 'a1', + seq: 1 + }, + data: { + manifest: { + schemaVersion: 1, + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + included: 'not-an-array' + } + } + }) + + expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) + }) + + it('rejects view manifests that disagree with their persisted source envelope', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 1 }, + data: { manifest: createObservationManifest({ messageId: 'other', requestSeq: 1 }) } + }) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 2 }, + data: { manifest: createObservationManifest({ messageId: 'a1', requestSeq: 1 }) } + }) + + expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) + }) + + it('normalizes legacy manifests without hashVersion to hashVersion 1', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + const legacyManifest: Record = { ...manifest } + delete legacyManifest.hashVersion + + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 1 }, + data: { manifest: legacyManifest } + }) + + const [record] = service.listViewManifestsByMessage('s1', 'a1') + expect(record.manifest.hashVersion).toBe(1) + }) + + it('filters manifests whose hashVersion is not a number', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 99 }, + data: { manifest: { ...manifest, hashVersion: '2' } } + }) + + expect(service.listViewManifestsByMessage('s1', 'a1')).toEqual([]) + }) + + it('annotates read records with hash integrity without dropping tampered manifests', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const baseInput = { + sessionId: 's1', + taskType: 'chat' as const, + policy: 'legacy_context_v1' as const, + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + } + const validManifest = createTapeViewManifest({ ...baseInput, messageId: 'a1', requestSeq: 1 }) + service.appendViewManifest(validManifest) + + const tamperedManifest = createTapeViewManifest({ + ...baseInput, + messageId: 'a2', + requestSeq: 1 + }) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a2', seq: 1 }, + data: { manifest: { ...tamperedManifest, latestEntryId: tamperedManifest.latestEntryId + 1 } } + }) + + const [validRecord] = service.listViewManifestsByMessage('s1', 'a1') + const [tamperedRecord] = service.listViewManifestsByMessage('s1', 'a2') + expect(validRecord.integrity).toBe('valid') + expect(tamperedRecord).toBeDefined() + expect(tamperedRecord.integrity).toBe('invalid') + }) + + it('binds reconstruction lineage to the latest reconstruction anchor including handoffs', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('s1') + table.appendAnchor({ + sessionId: 's1', + name: 'compaction/manual', + source: { type: 'summary', id: 's1', seq: 1 }, + state: {} + }) + table.appendAnchor({ + sessionId: 's1', + name: 'handoff/phase_done', + source: { type: 'handoff', id: 's1', seq: 2 }, + state: {} + }) + table.appendAnchor({ + sessionId: 's1', + name: 'fork/merge', + source: { type: 'fork', id: 'child', seq: 3 }, + state: {} + }) + + const sourceMaps = service.getViewManifestSourceMaps('s1') + const entryIdByName = (name: string) => + table.getBySession('s1').find((entry: any) => entry.name === name)?.entry_id + + expect(sourceMaps.anchorEntryIds).toHaveLength(4) + expect(sourceMaps.reconstructionAnchorEntryId).toBe(entryIdByName('handoff/phase_done')) + expect(sourceMaps.reconstructionAnchorEntryIds).toEqual([ + sourceMaps.reconstructionAnchorEntryId + ]) + expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain( + entryIdByName('compaction/manual') + ) + expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('fork/merge')) + }) + + it('keeps memory anchors off the reconstruction lineage', () => { + const { table } = createTapeTableMock() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatSessionsTable: { getSummaryState: vi.fn().mockReturnValue(null) } + } as any) + + table.ensureBootstrapAnchor('s1') + table.appendAnchor({ + sessionId: 's1', + name: 'compaction/manual', + source: { type: 'summary', id: 's1', seq: 1 }, + state: {} + }) + table.appendAnchor({ + sessionId: 's1', + name: 'memory/extract', + source: { type: 'runtime_event', id: 's1', seq: 2 }, + state: { memoryIds: ['m1'], count: 1, reason: 'episodic' } + }) + table.appendAnchor({ + sessionId: 's1', + name: 'memory/reflect', + source: { type: 'runtime_event', id: 's1', seq: 3 }, + state: { reflectionIds: ['r1'], sourceMemoryIds: ['m1'], count: 1 } + }) + + const sourceMaps = service.getViewManifestSourceMaps('s1') + const entryIdByName = (name: string) => + table.getBySession('s1').find((entry: any) => entry.name === name)?.entry_id + + // Memory anchors are recorded on the tape for observability... + expect(sourceMaps.anchorEntryIds).toContain(entryIdByName('memory/extract')) + expect(sourceMaps.anchorEntryIds).toContain(entryIdByName('memory/reflect')) + // ...but never own the reconstruction cursor; only the summary anchor does. + expect(sourceMaps.reconstructionAnchorEntryId).toBe(entryIdByName('compaction/manual')) + expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('memory/extract')) + expect(sourceMaps.reconstructionAnchorEntryIds).not.toContain(entryIdByName('memory/reflect')) + }) + + it('bounds replay slices to the selected view instead of pre-cursor history', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.reconstructionAnchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user', + source: 'tape', + reason: 'selected_history' + } + ], + excluded: [], + summaryCursor: { + summaryCursorOrderSeq: 100, + preCursorOrderSeqMin: 1, + preCursorOrderSeqMax: 99, + preCursorCount: 99 + }, + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 100, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: false, + assembledAt: 200 + }) + service.appendViewManifest(manifest) + + const slice = service.exportReplaySlice('s1', 'a1') + + expect(slice?.refs.excludedEntryIds).toEqual([]) + expect(slice?.refs.anchorEntryIds).toEqual(sourceMaps.reconstructionAnchorEntryIds) + expect(slice?.refs.anchorEntryIds).toHaveLength(1) + expect(slice?.manifestRecord.manifest.excludedRanges).toEqual([ + { fromOrderSeq: 1, toOrderSeq: 99, count: 99, reason: 'before_summary_cursor' } + ]) + expect(slice?.entries).toHaveLength(3) + }) + + it('exports replay slices with metadata-only payloads by default', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [createTraceRow()]) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const manifest = createTapeViewManifest({ + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat', + policy: 'legacy_context_v1', + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user', + source: 'tape', + reason: 'selected_history' + } + ], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: true, + assembledAt: 200 + }) + const manifestEntry = service.appendViewManifest(manifest) + + const nowSpy = vi.spyOn(Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000) + const slice = service.exportReplaySlice('s1', 'a1') + const secondSlice = service.exportReplaySlice('s1', 'a1') + nowSpy.mockRestore() + + expect(slice).toMatchObject({ + schemaVersion: 1, + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + mode: 'trace_bound', + refs: { + manifestEntryId: manifestEntry.entry_id, + includedEntryIds: [sourceMaps.entryIdByMessageId.get('u1')], + anchorEntryIds: sourceMaps.anchorEntryIds + }, + hashes: { + manifestHash: manifest.hashes.manifestHash + } + }) + expect(slice?.hashes.sliceHash).toHaveLength(64) + expect(secondSlice?.hashes.sliceHash).toBe(slice?.hashes.sliceHash) + expect(secondSlice?.createdAt).toBe(2000) + expect(slice?.trace?.bodyHash).toHaveLength(64) + expect(slice?.trace?.bodyJson).toBeUndefined() + expect(slice?.entries.some((entry) => entry.entryId === manifestEntry.entry_id)).toBe(true) + expect( + slice?.entries.every((entry) => entry.payload === undefined && entry.meta === undefined) + ).toBe(true) + }) + + it('exports explicit replay request sequences with opt-in payloads', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [ + createTraceRow({ id: 'trace-1', request_seq: 1 }), + createTraceRow({ + id: 'trace-2', + request_seq: 2, + body_json: '{"messages":[{"role":"tool","content":"done"}]}' + }) + ]) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const baseManifestInput: TapeViewManifestBuildInput = { + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat' as const, + policy: 'legacy_context_v1' as const, + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user', + source: 'tape', + reason: 'selected_history' + } + ], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: true, + assembledAt: 200 + } + const firstManifest = createTapeViewManifest(baseManifestInput) + const secondManifest = createTapeViewManifest({ + ...baseManifestInput, + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 250 + }) + service.appendViewManifest(firstManifest) + service.appendViewManifest(secondManifest) + + const latest = service.exportReplaySlice('s1', 'a1') + const first = service.exportReplaySlice('s1', 'a1', { + requestSeq: 1, + includeTapePayloads: true, + includeTracePayload: true + }) + + expect(latest?.requestSeq).toBe(2) + expect(first?.requestSeq).toBe(1) + expect(first?.trace?.bodyJson).toContain('"hello"') + expect(first?.entries.some((entry) => entry.payload?.record)).toBe(true) + expect(first?.entries.some((entry) => entry.meta?.source === 'backfill')).toBe(true) + }) + + it('binds each replay slice to its own request seq, ignoring sentinel gap traces', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [ + createTraceRow({ + id: 'trace-req-1', + request_seq: 1, + body_json: '{"messages":[{"role":"user","content":"first-request"}]}' + }), + createTraceRow({ + id: 'trace-gap', + request_seq: 0, + endpoint: 'deepchat://interleaved-reasoning-gap', + body_json: '{"providerId":"openai"}' + }), + createTraceRow({ + id: 'trace-req-2', + request_seq: 2, + body_json: '{"messages":[{"role":"tool","content":"second-request"}]}' + }) + ]) + const messageStore = { + getMessages: vi.fn().mockReturnValue([createRecord({ id: 'u1', orderSeq: 1 })]) + } + + service.ensureSessionTapeReady('s1', messageStore as any) + const sourceMaps = service.getViewManifestSourceMaps('s1') + const baseManifestInput = { + sessionId: 's1', + messageId: 'a1', + requestSeq: 1, + taskType: 'chat' as const, + policy: 'legacy_context_v1' as const, + policyVersion: 1, + messages: [{ role: 'user' as const, content: 'hello' }], + tools: [], + latestEntryId: sourceMaps.latestEntryId, + anchorEntryIds: sourceMaps.anchorEntryIds, + included: [ + { + entryId: sourceMaps.entryIdByMessageId.get('u1') ?? null, + messageId: 'u1', + orderSeq: 1, + role: 'user' as const, + source: 'tape' as const, + reason: 'selected_history' as const + } + ], + excluded: [], + tokenBudget: { + contextLength: 1000, + requestedMaxTokens: 100, + effectiveMaxTokens: 100, + reserveTokens: 100, + toolReserveTokens: 0 + }, + providerId: 'openai', + modelId: 'gpt-4o', + summaryCursorOrderSeq: 1, + supportsVision: true, + supportsAudioInput: false, + traceDebugEnabled: true, + assembledAt: 200 + } + service.appendViewManifest(createTapeViewManifest(baseManifestInput)) + service.appendViewManifest( + createTapeViewManifest({ + ...baseManifestInput, + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 250 + }) + ) + + const first = service.exportReplaySlice('s1', 'a1', { + requestSeq: 1, + includeTracePayload: true + }) + const second = service.exportReplaySlice('s1', 'a1', { + requestSeq: 2, + includeTracePayload: true + }) + + expect(first?.trace?.bodyJson).toContain('first-request') + expect(second?.trace?.bodyJson).toContain('second-request') + }) + + it('returns null when exporting a replay slice without a manifest', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [createTraceRow()]) + + expect(service.exportReplaySlice('s1', 'a1')).toBeNull() + }) + + it('rejects non-positive replay request sequences', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [createTraceRow()]) + + expect(() => service.exportReplaySlice('s1', 'a1', { requestSeq: 0 })).toThrow( + 'requestSeq must be a positive integer.' + ) + }) + + it('joins an exact manifest and trace into a causal observation slice', () => { + const { table } = createTapeTableMock() + const assistant = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: '[{"type":"content","content":"done","status":"success"}]', + status: 'sent', + metadata: '{"totalTokens":10}', + createdAt: 200, + updatedAt: 300 + }) + appendMessageRecordToTape(table as any, assistant, 'live') + const service = createTapeService( + table, + [createTraceRow(), createTraceRow({ id: 'other-session', session_id: 's2', request_seq: 9 })], + [createMessageRow()] + ) + service.appendViewManifest(createObservationManifest()) + + const slice = service.readCausalObservationSlice('s1', 'a1', { + currentRuntimeStatus: 'idle' + }) + + expect(slice).toMatchObject({ + schemaVersion: 1, + sessionId: 's1', + messageId: 'a1', + request: { + state: 'manifest_bound', + requestSeq: 1, + replay: { + requestSeq: 1, + mode: 'trace_bound', + trace: { id: 'trace-1', requestSeq: 1 } + } + }, + output: { + correlation: 'message_only', + terminalMessage: { + status: 'sent', + orderSeq: 2, + createdAt: 200, + updatedAt: 300 + } + }, + runtime: { scope: 'current_only', status: 'idle', eventHistory: 'not_persisted' } + }) + expect(slice.output.entries.map((entry) => entry.kind)).toContain('message') + expect(slice.output.terminalMessage?.contentHash).toHaveLength(64) + expect(slice.output.terminalMessage?.metadataHash).toHaveLength(64) + }) + + it('prefers an explicit causal request sequence and otherwise selects the latest raw sequence', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 100 })) + service.appendViewManifest( + createObservationManifest({ + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 200 + }) + ) + + expect(service.readCausalObservationSlice('s1', 'a1').request).toMatchObject({ + state: 'manifest_bound', + requestSeq: 2 + }) + expect(service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }).request).toMatchObject( + { state: 'manifest_bound', requestSeq: 1 } + ) + }) + + it('does not fall back to an older manifest when a later traced request has no manifest', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [ + createTraceRow({ id: 'trace-1', request_seq: 1 }), + createTraceRow({ id: 'trace-2', request_seq: 2 }) + ]) + service.appendViewManifest(createObservationManifest({ requestSeq: 1 })) + + expect(service.readCausalObservationSlice('s1', 'a1').request).toMatchObject({ + state: 'manifest_missing', + requestSeq: 2, + trace: { id: 'trace-2', requestSeq: 2 } + }) + }) + + it('returns a trace-only request without manufacturing a view manifest', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table, [createTraceRow({ request_seq: 4 })]) + + const request = service.readCausalObservationSlice('s1', 'a1').request + + expect(request).toMatchObject({ + state: 'manifest_missing', + requestSeq: 4, + trace: { requestSeq: 4 } + }) + expect( + request.state === 'manifest_missing' ? request.trace?.bodyJson : undefined + ).toBeUndefined() + }) + + it('distinguishes malformed manifests from hash-invalid but readable manifests', () => { + const { table } = createTapeTableMock() + const service = createTapeService(table) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'bad', seq: 2 }, + data: { manifest: { schemaVersion: 99, requestSeq: 2 } } + }) + const tamperedManifest = createObservationManifest({ messageId: 'a1', requestSeq: 1 }) + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 1 }, + data: { manifest: { ...tamperedManifest, latestEntryId: 1 } } + }) + + expect(service.readCausalObservationSlice('s1', 'bad').request).toEqual({ + state: 'manifest_malformed', + requestSeq: 2, + trace: null + }) + const invalid = service.readCausalObservationSlice('s1', 'a1').request + expect(invalid.state).toBe('manifest_bound') + expect(invalid.state === 'manifest_bound' ? invalid.replay.integrity : undefined).toBe( + 'invalid' + ) + }) + + it('ignores request sequence zero interleaved-reasoning sentinels', () => { + const { table } = createTapeTableMock() + table.appendEvent({ + sessionId: 's1', + name: 'view/assembled', + source: { type: 'runtime_event', id: 'a1', seq: 0 }, + data: { manifest: { schemaVersion: 99, requestSeq: 0 } } + }) + const service = createTapeService(table, [ + createTraceRow({ + id: 'trace-gap', + request_seq: 0, + endpoint: 'deepchat://interleaved-reasoning-gap' + }) + ]) + + expect(service.readCausalObservationSlice('s1', 'a1').request).toEqual({ + state: 'request_unavailable', + requestSeq: null, + trace: null + }) + }) + + it('keeps multi-round assistant and tool output correlated at message scope only', () => { + const { table } = createTapeTableMock() + const assistant = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: JSON.stringify([ + { + type: 'tool_call', + status: 'success', + timestamp: 220, + tool_call: { + id: 'tc1', + name: 'search', + params: '{"q":"x"}', + response: 'result' + } + } + ]), + status: 'sent', + createdAt: 200, + updatedAt: 300 + }) + appendMessageRecordToTape(table as any, assistant, 'live') + const service = createTapeService(table, [], [createMessageRow({ content: assistant.content })]) + service.appendViewManifest(createObservationManifest({ requestSeq: 1 })) + service.appendViewManifest( + createObservationManifest({ + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null + }) + ) + + const firstOutput = service.readCausalObservationSlice('s1', 'a1', { requestSeq: 1 }).output + const latestOutput = service.readCausalObservationSlice('s1', 'a1').output + + expect(latestOutput.correlation).toBe('message_only') + expect(latestOutput.entries.map((entry) => entry.kind)).toEqual([ + 'message', + 'tool_call', + 'tool_result' + ]) + expect(latestOutput.entries.map((entry) => entry.entryId)).toEqual( + firstOutput.entries.map((entry) => entry.entryId) + ) + }) + + it('does not infer a completed event from a paused pending assistant message', () => { + const { table } = createTapeTableMock() + const service = createTapeService( + table, + [], + [createMessageRow({ status: 'pending', content: '[{"type":"action","status":"pending"}]' })] + ) + + const slice = service.readCausalObservationSlice('s1', 'a1', { + currentRuntimeStatus: 'generating' + }) + + expect(slice.output.entries).toEqual([]) + expect(slice.output.terminalMessage).toBeNull() + expect(slice.runtime).toEqual({ + scope: 'current_only', + status: 'generating', + eventHistory: 'not_persisted' + }) + expect(JSON.stringify(slice)).not.toContain('completed') + }) + + it('reads an old message without bootstrapping or backfilling Tape', () => { + const { table, entries } = createTapeTableMock() + const messageInsert = vi.fn() + const traceInsert = vi.fn() + const projectionApply = vi.fn() + const projectionReplace = vi.fn() + const cursorWrite = vi.fn() + const service = new SessionTape({ + deepchatTapeEntriesTable: table, + deepchatMessageTracesTable: { + listByMessageId: vi.fn().mockReturnValue([]), + insert: traceInsert + }, + deepchatMessagesTable: { + get: vi.fn().mockReturnValue(createMessageRow()), + insert: messageInsert + }, + deepchatMemoryIngestionProjectionTable: { + applyAppendedEntry: projectionApply, + replaceSession: projectionReplace + }, + deepchatSessionsTable: { + getSummaryState: vi.fn().mockReturnValue(null), + updateMemoryCursorOrderSeq: cursorWrite + } + } as any) + + const slice = service.readCausalObservationSlice('s1', 'a1') + + expect(slice.request).toEqual({ + state: 'request_unavailable', + requestSeq: null, + trace: null + }) + expect(slice.output.terminalMessage?.status).toBe('sent') + expect(slice.runtime).toEqual({ + scope: 'unavailable', + status: null, + eventHistory: 'not_persisted' + }) + expect(entries).toEqual([]) + expect(table.ensureBootstrapAnchor).not.toHaveBeenCalled() + expect(table.append).not.toHaveBeenCalled() + expect(table.appendEvent).not.toHaveBeenCalled() + expect(messageInsert).not.toHaveBeenCalled() + expect(traceInsert).not.toHaveBeenCalled() + expect(projectionApply).not.toHaveBeenCalled() + expect(projectionReplace).not.toHaveBeenCalled() + expect(cursorWrite).not.toHaveBeenCalled() + }) + + it('keeps causal observations metadata-only by default and reuses replay payload opt-ins', () => { + const { table } = createTapeTableMock() + const assistant = createRecord({ + id: 'a1', + orderSeq: 2, + role: 'assistant', + content: 'tape-secret-output', + status: 'sent', + metadata: '{"secret":"tape-metadata"}', + createdAt: 200, + updatedAt: 300 + }) + appendMessageRecordToTape(table as any, assistant, 'live') + const service = createTapeService( + table, + [ + createTraceRow({ + headers_json: '{"authorization":"secret-header"}', + body_json: '{"prompt":"secret-request"}' + }) + ], + [ + createMessageRow({ + content: 'projection-only-content-secret', + metadata: '{"secret":"projection-only-metadata"}', + blocks_json: '["projection-only-blocks"]', + error: 'projection-only-error' + }) + ] + ) + service.appendViewManifest(createObservationManifest()) + + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) + const metadataOnly = service.readCausalObservationSlice('s1', 'a1') + const withPayloads = service.readCausalObservationSlice('s1', 'a1', { + includeTapePayloads: true, + includeTracePayload: true + }) + now.mockRestore() + + expect(JSON.stringify(metadataOnly)).not.toContain('tape-secret-output') + expect(JSON.stringify(metadataOnly)).not.toContain('tape-metadata') + expect(JSON.stringify(metadataOnly)).not.toContain('secret-header') + expect(JSON.stringify(metadataOnly)).not.toContain('secret-request') + expect(JSON.stringify(metadataOnly)).not.toContain('projection-only') + expect(withPayloads.output.entries.some((entry) => entry.payload?.record)).toBe(true) + expect(withPayloads.request.state).toBe('manifest_bound') + expect( + withPayloads.request.state === 'manifest_bound' + ? withPayloads.request.replay.trace?.headersJson + : undefined + ).toContain('secret-header') + expect(JSON.stringify(withPayloads)).toContain('tape-secret-output') + expect(JSON.stringify(withPayloads)).not.toContain('projection-only') + expect(metadataOnly.output.terminalMessage).not.toHaveProperty('content') + expect(metadataOnly.output.terminalMessage).not.toHaveProperty('metadata') + expect(metadataOnly.output.terminalMessage).not.toHaveProperty('blocks') + expect(metadataOnly.output.terminalMessage).not.toHaveProperty('error') + expect(stripObservationPayloadOptIns(withPayloads)).toEqual( + stripObservationPayloadOptIns(metadataOnly) + ) + }) + + it('keeps storage and Memory seams unchanged across every causal observation read mode', () => { + const { table, entries } = createTapeTableMock() + const { final } = appendObservationIsolationFacts(table) + const traceRows = [ + createTraceRow({ id: 'trace-1', request_seq: 1 }), + createTraceRow({ id: 'trace-2', request_seq: 2 }), + createTraceRow({ id: 'trace-only', message_id: 'a-trace', request_seq: 7 }) + ] + const messageRows = [createMessageRow({ order_seq: 3 })] + const projectionState = { + meta: { session_id: 's1', projection_version: 1, max_entry_id: entries.length }, + range: [{ session_id: 's1', message_id: 'a1', order_seq: 3, entry_id: entries.length }] + } + const memoryCursorOrderSeq = 3 + const memoryProjectionWrites = createSpies([ + 'applyAppendedEntry', + 'replaceSession', + 'invalidateSession' + ]) + const memoryProjectionTable = { + ...memoryProjectionWrites, + readCurrentRange: vi.fn(() => structuredClone(projectionState.range)), + getSessionMeta: vi.fn(() => structuredClone(projectionState.meta)) + } + const cursorWrites = createSpies(['updateMemoryCursorOrderSeq', 'rewindMemoryCursorOrderSeq']) + const sessionTable = { + ...cursorWrites, + getSummaryState: vi.fn().mockReturnValue(null), + getMemoryCursorOrderSeq: vi.fn(() => memoryCursorOrderSeq) + } + const messageWrites = createSpies([ + 'insert', + 'updateContent', + 'updateStatus', + 'updateContentAndStatus' + ]) + const messageTable = { + ...messageWrites, + get: vi.fn((messageId: string) => messageRows.find((row) => row.id === messageId)) + } + const traceWrites = createSpies(['insert', 'deleteByMessageId', 'deleteBySessionId']) + const traceTable = { + ...traceWrites, + listByMessageId: vi.fn((messageId: string) => + traceRows.filter((row) => row.message_id === messageId) + ) + } + const { presenter, memoryPropertyAccess } = trackMemoryPropertyAccess({ + deepchatTapeEntriesTable: table, + deepchatMessageTracesTable: traceTable, + deepchatMessagesTable: messageTable, + deepchatSessionsTable: sessionTable, + deepchatMemoryIngestionProjectionTable: memoryProjectionTable + }) + const service = new SessionTape(presenter as any) + service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 210 })) + service.appendViewManifest( + createObservationManifest({ + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 310 + }) + ) + + for (const write of [ + table.ensureBootstrapAnchor, + table.append, + table.appendAnchor, + table.appendEvent, + table.deleteBySession + ]) { + write.mockClear() + } + const captureState = () => ({ + tapeRows: structuredClone(entries), + maxEntryId: table.getMaxEntryId('s1'), + entryOrder: entries.map((entry) => entry.entry_id), + maxMessageOrder: Math.max(0, ...service.getMessageRecords('s1').map((row) => row.orderSeq)), + messageRows: structuredClone(messageRows), + traceRows: structuredClone(traceRows), + viewManifestRows: structuredClone( + entries.filter((entry) => entry.kind === 'event' && entry.name === 'view/assembled') + ), + effectiveView: service.getMessageRecords('s1'), + replay: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }), + replayHash: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 })?.hashes.sliceHash, + projectionMeta: memoryProjectionTable.getSessionMeta(), + projectionRange: memoryProjectionTable.readCurrentRange(), + memoryCursorOrderSeq: sessionTable.getMemoryCursorOrderSeq() + }) + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) + + try { + const before = captureState() + const { defaultObservation, repeatedObservation, explicitObservation, traceOnlyObservation } = + readObservationMatrix(service) + const after = captureState() + + expect(after).toEqual(before) + expect(repeatedObservation).toEqual(defaultObservation) + expect(defaultObservation.request).toMatchObject({ + state: 'manifest_bound', + requestSeq: 2 + }) + expect(explicitObservation.request).toMatchObject({ + state: 'manifest_bound', + requestSeq: 1 + }) + expect(traceOnlyObservation.request).toMatchObject({ + state: 'manifest_missing', + requestSeq: 7, + trace: { id: 'trace-only' } + }) + expect(defaultObservation.output).toMatchObject({ + correlation: 'message_only', + entries: [{ kind: 'message' }, { kind: 'tool_call' }, { kind: 'tool_result' }] + }) + expect(before.effectiveView.map((record) => record.id)).toEqual(['u1', 'a1']) + expect(JSON.parse(before.effectiveView[0].content).text).toBe('edited') + expect(before.effectiveView[1]).toMatchObject({ status: 'sent', content: final.content }) + } finally { + now.mockRestore() + } + + for (const write of [ + table.ensureBootstrapAnchor, + table.append, + table.appendAnchor, + table.appendEvent, + table.deleteBySession, + ...Object.values(messageWrites), + ...Object.values(traceWrites), + ...Object.values(memoryProjectionWrites), + ...Object.values(cursorWrites), + memoryPropertyAccess + ]) { + expect(write).not.toHaveBeenCalled() + } + }) + + itIfSqlite( + `preserves real Tape, message, trace, Memory projection and schema state while observing${sqliteAvailable ? '' : ` (${sqliteSkipReason})`}`, + () => { + const db = new DatabaseCtor(':memory:') + try { + const memoryProjectionTable = new DeepChatMemoryIngestionProjectionTable(db) + const tapeTable = new DeepChatTapeEntriesTable(db, memoryProjectionTable) + const messageTable = new DeepChatMessagesTable(db) + const traceTable = new DeepChatMessageTracesTable(db) + const sessionTable = new DeepChatSessionsTable(db) + memoryProjectionTable.createTable() + tapeTable.createTable() + messageTable.createTable() + traceTable.createTable() + sessionTable.createTable() + sessionTable.create('s1', 'openai', 'gpt-4o', 'full_access') + sessionTable.updateMemoryCursorOrderSeq('s1', 3) + + appendObservationIsolationFacts(tapeTable) + messageTable.insert({ + id: 'a1', + sessionId: 's1', + orderSeq: 3, + role: 'assistant', + content: 'projection-only-content-secret', + status: 'sent', + metadata: '{"secret":"projection-only-metadata"}', + createdAt: 200, + updatedAt: 300 + }) + for (const trace of [ + { id: 'trace-1', messageId: 'a1', requestSeq: 1 }, + { id: 'trace-2', messageId: 'a1', requestSeq: 2 }, + { id: 'trace-only', messageId: 'a-trace', requestSeq: 7 } + ]) { + traceTable.insert({ + ...trace, + sessionId: 's1', + providerId: 'openai', + modelId: 'gpt-4o', + endpoint: 'https://api.openai.test/v1/chat/completions', + headersJson: `{"authorization":"${trace.id}-header"}`, + bodyJson: `{"prompt":"${trace.id}-body"}`, + truncated: false, + createdAt: 300 + trace.requestSeq + }) + } + + const service = new SessionTape({ + deepchatTapeEntriesTable: tapeTable, + deepchatMessagesTable: messageTable, + deepchatMessageTracesTable: traceTable, + deepchatSessionsTable: sessionTable, + deepchatMemoryIngestionProjectionTable: memoryProjectionTable + } as any) + service.appendViewManifest(createObservationManifest({ requestSeq: 1, assembledAt: 210 })) + service.appendViewManifest( + createObservationManifest({ + requestSeq: 2, + taskType: 'tool_loop', + policy: 'tool_loop_shadow', + policyVersion: null, + assembledAt: 310 + }) + ) + const effectiveRecords = service.getMessageRecords('s1') + const sourceMaps = service.getViewManifestSourceMaps('s1') + memoryProjectionTable.replaceSession( + 's1', + effectiveRecords + .filter( + (record): record is ChatMessageRecord & { status: 'sent' | 'error' } => + record.status === 'sent' || record.status === 'error' + ) + .map((record) => ({ + sessionId: 's1', + messageId: record.id, + orderSeq: record.orderSeq, + entryId: sourceMaps.entryIdByMessageId.get(record.id)!, + role: record.role, + content: record.content, + status: record.status, + hadToolUse: record.id === 'a1' + })), + tapeTable.getMaxEntryId('s1') + ) + + const captureState = () => { + const tapeRows = tapeTable.getBySession('s1') + return { + tapeRows, + maxEntryId: tapeTable.getMaxEntryId('s1'), + entryOrder: tapeRows.map((row) => row.entry_id), + messageRow: messageTable.get('a1'), + traceRows: db + .prepare( + `SELECT * + FROM deepchat_message_traces + ORDER BY session_id, message_id, request_seq, id` + ) + .all(), + viewManifestRows: tapeRows.filter( + (row) => row.kind === 'event' && row.name === 'view/assembled' + ), + effectiveView: service.getMessageRecords('s1'), + replay: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 }), + replayHash: service.exportReplaySlice('s1', 'a1', { requestSeq: 2 })?.hashes.sliceHash, + projectionMeta: memoryProjectionTable.getSessionMeta('s1'), + projectionRange: memoryProjectionTable.readCurrentRange('s1', 0, 10), + memoryCursorOrderSeq: sessionTable.getMemoryCursorOrderSeq('s1'), + schema: db + .prepare( + `SELECT type, name, tbl_name, sql + FROM sqlite_master + WHERE name NOT LIKE 'sqlite_%' + ORDER BY type, name, tbl_name` + ) + .all() + } + } + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000) + + try { + const before = captureState() + readObservationMatrix(service) + const after = captureState() + + expect(after).toEqual(before) + } finally { + now.mockRestore() + } + } finally { + db.close() + } + } + ) +}) diff --git a/test/main/session/data/transcript.test.ts b/test/main/session/data/transcript.test.ts index f530befe1b..bf6252cfc9 100644 --- a/test/main/session/data/transcript.test.ts +++ b/test/main/session/data/transcript.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { SessionTranscript } from '@/session/data/transcript' +import { SessionTape } from '@/tape/application/sessionTape' import { cloneBlocksForRenderer } from '@/agent/deepchat/runtime/echo' import logger from '@shared/logger' @@ -142,7 +143,7 @@ describe('SessionTranscript', () => { beforeEach(() => { sqlitePresenter = createMockSqlitePresenter() - store = new SessionTranscript(sqlitePresenter) + store = new SessionTranscript(sqlitePresenter, new SessionTape(sqlitePresenter)) }) describe('createUserMessage', () => { @@ -832,6 +833,34 @@ describe('SessionTranscript', () => { expect(sqlitePresenter.deepchatMessageTracesTable.deleteByMessageIds).not.toHaveBeenCalled() expect(sqlitePresenter.deepchatMessagesTable.deleteFromOrderSeq).toHaveBeenCalledWith('s1', 2) }) + + it('defers projection deletion until every tape retraction has been appended', () => { + const appendEvent = vi + .fn() + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => { + throw new Error('second retraction failed') + }) + const transaction = vi.fn((operation: () => unknown) => () => operation()) + sqlitePresenter.getDatabase = vi.fn().mockReturnValue({ transaction }) + sqlitePresenter.deepchatTapeEntriesTable = { + ensureBootstrapAnchor: vi.fn(), + appendEvent + } + sqlitePresenter.deepchatMessagesTable.getBySession.mockReturnValue([ + createMessageRow({ id: 'm1', order_seq: 1 }), + createMessageRow({ id: 'm2', order_seq: 2 }), + createMessageRow({ id: 'm3', order_seq: 3 }) + ]) + + expect(() => store.deleteFromOrderSeq('s1', 2)).toThrow('second retraction failed') + + expect(transaction).toHaveBeenCalledOnce() + expect(appendEvent).toHaveBeenCalledTimes(2) + expect(sqlitePresenter.deepchatMessagesTable.deleteFromOrderSeq).not.toHaveBeenCalled() + expect(sqlitePresenter.deepchatSearchDocumentsTable.deleteByMessageIds).not.toHaveBeenCalled() + expect(sqlitePresenter.deepchatMessageTracesTable.deleteByMessageIds).not.toHaveBeenCalled() + }) }) describe('trace operations', () => { diff --git a/test/main/session/runtimeIntegration.test.ts b/test/main/session/runtimeIntegration.test.ts index 960fa5d2d5..6e723ed0c2 100644 --- a/test/main/session/runtimeIntegration.test.ts +++ b/test/main/session/runtimeIntegration.test.ts @@ -74,6 +74,7 @@ function createMockSqlitePresenter() { let messagesList: any[] = [] const tapeTable = { + runInTransaction: vi.fn((operation: () => unknown) => operation()), ensureBootstrapAnchor: vi.fn(), append: vi.fn((input: any) => { const row = { @@ -638,10 +639,12 @@ function createMockSqlitePresenter() { }) }, deepchatTapeEntriesTable: tapeTable, + tapeLifecycle: tapeTable, deepchatTapeSearchProjectionTable: { deleteBySession: vi.fn(), isCurrent: vi.fn().mockReturnValue(false), - getByEntryIds: vi.fn().mockReturnValue([]) + getByEntryIds: vi.fn().mockReturnValue([]), + getByEntryIdsIfCurrent: vi.fn().mockReturnValue([]) }, // Expose internal stores for assertion _sessionsStore: sessionsStore, @@ -789,7 +792,8 @@ function createTranscriptMutations( transcript: sessionData.transcript, settings: sessionData.settings, pendingInputs: sessionData.pendingInputs, - runtime + runtime, + runInTransaction: (operation) => sessionData.database.getDatabase().transaction(operation)() }) } @@ -1740,8 +1744,19 @@ describe('Integration: Session Tape boundary', () => { expect(search).toHaveBeenCalledTimes(2) expect(getContext).toHaveBeenCalledOnce() + ensureTapeReady.mockClear() + search.mockClear() + getContext.mockClear() + await sessionData.tape.searchTape('s1', 'needle') + expect(ensureTapeReady.mock.invocationCallOrder[0]).toBeLessThan( + search.mock.invocationCallOrder[0] + ) + await sessionData.tape.getTapeContext('s1', [2]) + expect(ensureTapeReady.mock.invocationCallOrder[1]).toBeLessThan( + getContext.mock.invocationCallOrder[0] + ) expect(ensureTapeReady).toHaveBeenCalledTimes(2) }) diff --git a/test/main/session/transcriptMutations.test.ts b/test/main/session/transcriptMutations.test.ts new file mode 100644 index 0000000000..b0e076f20d --- /dev/null +++ b/test/main/session/transcriptMutations.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it, vi } from 'vitest' +import { SessionTranscriptMutations } from '@/session/transcriptMutations' + +describe('SessionTranscriptMutations', () => { + it('rolls pending inputs and transcript deletion back when Tape reset fails', async () => { + const state = { pendingInputs: 1, transcriptMessages: 2, tapeEntries: 3 } + const runtime = { + prepareClearMessages: vi.fn().mockResolvedValue(undefined), + finishClearMessages: vi.fn() + } + const mutations = new SessionTranscriptMutations({ + pendingInputs: { + deleteBySession: vi.fn(() => { + state.pendingInputs = 0 + }) + }, + transcript: { + deleteBySession: vi.fn(() => { + state.transcriptMessages = 0 + }) + }, + settings: { + resetTape: vi.fn(() => { + state.tapeEntries = 0 + throw new Error('Tape reset failed') + }) + }, + runtime, + runInTransaction: (operation) => { + const snapshot = { ...state } + try { + return operation() + } catch (error) { + Object.assign(state, snapshot) + throw error + } + } + } as any) + + await expect(mutations.clearMessages('s1')).rejects.toThrow('Tape reset failed') + + expect(state).toEqual({ pendingInputs: 1, transcriptMessages: 2, tapeEntries: 3 }) + expect(runtime.prepareClearMessages).toHaveBeenCalledWith('s1') + expect(runtime.finishClearMessages).not.toHaveBeenCalled() + }) + + it('finishes runtime cleanup only after the shared transaction commits', async () => { + const calls: string[] = [] + const mutations = new SessionTranscriptMutations({ + pendingInputs: { deleteBySession: vi.fn(() => calls.push('pending')) }, + transcript: { deleteBySession: vi.fn(() => calls.push('transcript')) }, + settings: { resetTape: vi.fn(() => calls.push('tape')) }, + runtime: { + prepareClearMessages: vi.fn(async () => { + calls.push('prepare') + }), + finishClearMessages: vi.fn(() => calls.push('finish')) + }, + runInTransaction: (operation) => { + calls.push('transaction:start') + const result = operation() + calls.push('transaction:commit') + return result + } + } as any) + + await mutations.clearMessages('s1') + + expect(calls).toEqual([ + 'prepare', + 'transaction:start', + 'pending', + 'transcript', + 'tape', + 'transaction:commit', + 'finish' + ]) + }) +}) diff --git a/test/main/session/usageStatsService.test.ts b/test/main/session/usageStatsService.test.ts index ff5a1c7698..a1ba721c41 100644 --- a/test/main/session/usageStatsService.test.ts +++ b/test/main/session/usageStatsService.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { SessionTranscript } from '@/session/data/transcript' +import { SessionTape } from '@/tape/application/sessionTape' import { DASHBOARD_STATS_BACKFILL_KEY, type UsageStatsRecordInput } from '@/session/usageStats' import { UsageStatsService } from '@/session/usageStatsService' import type { PermissionMode } from '@shared/types/agent-interface' @@ -556,7 +557,7 @@ describe('UsageStatsService', () => { it('keeps a single stats row when live finalize updates a previously backfilled message', async () => { const { service, sqlitePresenter } = createService() - const messageStore = new SessionTranscript(sqlitePresenter) + const messageStore = new SessionTranscript(sqlitePresenter, new SessionTape(sqlitePresenter)) sqlitePresenter.deepchatSessionsTable.create('session-1', 'openai', 'gpt-4o', 'full_access') sqlitePresenter.deepchatMessagesTable.insert({ diff --git a/test/main/tape/layerBoundaries.test.ts b/test/main/tape/layerBoundaries.test.ts new file mode 100644 index 0000000000..5a9487188d --- /dev/null +++ b/test/main/tape/layerBoundaries.test.ts @@ -0,0 +1,643 @@ +import path from 'node:path' +import ts from 'typescript' +import { describe, expect, it, vi } from 'vitest' + +const MAIN_SOURCE_ROOT = path.resolve(process.cwd(), 'src/main') +const TAPE_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape') +const TAPE_DOMAIN_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape/domain') +const TAPE_SQLITE_ROOT = path.join(MAIN_SOURCE_ROOT, 'tape/infrastructure/sqlite') +const TAPE_CAPABILITIES_MODULE = path.join(MAIN_SOURCE_ROOT, 'tape/ports/capabilities') +const TAPE_SESSION_FACADE_MODULE = path.join(MAIN_SOURCE_ROOT, 'tape/application/sessionTape') +const MEMORY_ROUTES_FILE = path.join(MAIN_SOURCE_ROOT, 'memory/routes.ts') +const TAPE_SQLITE_RELATIVE_ROOT = 'tape/infrastructure/sqlite/' +const TYPESCRIPT_SOURCE_EXTENSION = /\.[cm]?tsx?$/ + +interface LegacyTapeCompatibilityContract { + target: string + valueExports: readonly string[] + typeExports: readonly string[] + typeAliases?: Readonly> +} + +const LEGACY_TAPE_COMPATIBILITY_MODULES = new Map([ + [ + 'session/data/tape', + { + target: '@/tape/application/sessionTape', + valueExports: [ + 'AgentTapeViewError', + 'normalizeSubagentTapeLinkInput', + 'normalizeTapeHandoffState', + 'SessionTape' + ], + typeExports: [ + 'AgentTapeViewErrorCode', + 'TapeAnchorResult', + 'TapeBackfillResult', + 'TapeForkHandle', + 'TapeInfo', + 'TapeMigrationState', + 'TapeSearchResult' + ], + typeAliases: { TapeViewManifestSourceMaps: 'TapeViewManifestAssemblySources' } + } + ], + [ + 'session/data/tapeEffectiveView', + { + target: '@/tape/domain/effectiveView', + valueExports: [ + 'buildEffectiveTapeView', + 'getLastEffectiveTokenUsage', + 'searchEffectiveTapeRows' + ], + typeExports: ['EffectiveMessageEntry', 'EffectiveTapeView'] + } + ], + [ + 'session/data/tapeFacts', + { + target: '@/tape/application/factPersistence', + valueExports: [ + 'appendMessageRecordToTape', + 'appendMessageReplacementToTape', + 'appendMessageRetractionToTape', + 'appendTapeToolFact', + 'appendToolFactsToTape', + 'buildTapeToolFactInputs', + 'tapeEntriesToEffectiveMessageRecords', + 'tapeEntryToMessageRecord' + ], + typeExports: ['TapeFactSource'] + } + ], + [ + 'session/data/tapeViewManifest', + { + target: '@/tape/domain/viewManifest', + valueExports: [ + 'buildExcludedRefs', + 'buildIncludedRefs', + 'buildRequestRefs', + 'createTapeViewManifest', + 'hashJson', + 'isCompactionRecord', + 'resolveTapeViewManifestPolicy', + 'stableJsonStringify', + 'TAPE_VIEW_CONTEXT_BUILDER_VERSION', + 'TAPE_VIEW_MANIFEST_EVENT_NAME', + 'TAPE_VIEW_MANIFEST_HASH_VERSION', + 'verifyTapeViewManifestHash' + ], + typeExports: [ + 'ContextSummaryCursorMetadata', + 'TapeViewContextSelection', + 'TapeViewManifestBuildInput', + 'TapeViewManifestPolicyInput', + 'TapeViewManifestPolicyResult' + ], + typeAliases: { TapeViewManifestSourceMaps: 'TapeViewManifestLookupMaps' } + } + ], + [ + 'session/data/tables/deepchatTapeEffectiveSemantics', + { + target: '@/tape/domain/effectiveSemantics', + valueExports: [ + 'messageRecordHasFinalToolUse', + 'parseAssistantBlocks', + 'parseNestedTapeJsonObject', + 'parseTapeJsonObject', + 'readTapeMessageRetractionId', + 'readTapeToolIdentity', + 'readTapeToolStatus', + 'tapeEntryToMessageRecord', + 'tapeMessageRank', + 'tapeToolRank' + ], + typeExports: ['DeepChatTapeToolIdentity'] + } + ], + [ + 'session/data/tables/deepchatTapeEntries', + { + target: '@/tape/infrastructure/sqlite/tapeEntryStore', + valueExports: [ + 'buildDeepChatTapeFtsMatch', + 'buildDeepChatTapeLikeSearchPredicate', + 'DeepChatTapeEntriesTable', + 'normalizeDeepChatTapeReadSources', + 'serializeDeepChatTapeReadSources', + 'SUMMARY_ANCHOR_NAMES', + 'TAPE_INCARNATION_META_KEY' + ], + typeExports: [ + 'DeepChatTapeAppendInput', + 'DeepChatTapeEntryKind', + 'DeepChatTapeEntryRow', + 'DeepChatTapeMutationProjection', + 'DeepChatTapeReadSource', + 'DeepChatTapeSearchInput', + 'DeepChatTapeSourceInput', + 'DeepChatTapeSourceType' + ] + } + ], + [ + 'session/data/tables/deepchatTapeSearchProjection', + { + target: '@/tape/infrastructure/sqlite/tapeSearchProjectionStore', + valueExports: [ + 'DEEPCHAT_TAPE_SEARCH_PROJECTION_VERSION', + 'DeepChatTapeSearchProjectionTable' + ], + typeExports: [ + 'DeepChatTapeSearchProjectionInput', + 'DeepChatTapeSearchProjectionMeta', + 'DeepChatTapeSearchProjectionReadResult', + 'DeepChatTapeSearchProjectionResultRow', + 'DeepChatTapeSearchProjectionRow' + ] + } + ] +]) + +const CAPABILITY_SCOPED_CONSUMER_FILES = [ + 'agent/acp/compatibility/adapters.ts', + 'agent/acp/compatibility/dependencies.ts', + 'agent/deepchat/memory/memoryRuntimeCoordinator.ts', + 'agent/deepchat/runtime/deepChatLoopRunner.ts', + 'agent/deepchat/runtime/turnCoordinator.ts', + 'app/startupMigrations/legacyChatImportService.ts', + 'memory/routes.ts', + 'session/data/settings.ts', + 'session/data/transcript.ts' +].map((file) => path.join(MAIN_SOURCE_ROOT, file)) + +const FORBIDDEN_DOMAIN_SQLITE_IMPORTS = new Set([ + 'better-sqlite3', + 'better-sqlite3-multiple-ciphers', + 'bun:sqlite', + 'node:sqlite', + 'sql.js', + 'sqlite3' +]) +const FORBIDDEN_DOMAIN_LOGGING_IMPORTS = new Set([ + '@shared/logger', + 'electron-log', + 'loglevel', + 'pino', + 'winston' +]) + +const PHYSICAL_TAPE_STORAGE_PATTERN = + /\b(?:deepchat_tape_(?:entries|search_(?:projection(?:_meta)?|fts(?:_meta)?))|DeepChatTape(?:Entries|SearchProjection)Table|deepchatTape(?:Entries|SearchProjection)(?:Table)?)\b/ + +interface StorageBoundaryException { + physicalName?: string + sqliteImport?: string +} + +const ALLOWED_STORAGE_EXCEPTIONS = new Map([ + ['app/databaseSecurity.ts', { physicalName: 'database table-name security allowlist' }], + [ + 'app/startupMigrations/legacyChatImportService.ts', + { physicalName: 'migration-only full-table replacement and projection cleanup' } + ], + [ + 'data/schemaCatalog.ts', + { + physicalName: 'schema creation and migration registry', + sqliteImport: 'schema adapter construction' + } + ], + [ + 'data/sqliteCopyExclusions.ts', + { physicalName: 'SQLite virtual-table copy exclusion metadata' } + ], + [ + 'memory/data/tables/deepchatMemoryIngestionProjection.ts', + { physicalName: 'read-only single-statement Tape-head consistency check' } + ], + [ + 'session/data/database.ts', + { + physicalName: 'SQLite adapter compatibility getters', + sqliteImport: 'SQLite adapter composition' + } + ], + [ + 'session/data/tables/deepchatTapeEntries.ts', + { + physicalName: 'frozen legacy compatibility export', + sqliteImport: 'legacy import-path compatibility re-export' + } + ], + [ + 'session/data/tables/deepchatTapeSearchProjection.ts', + { + physicalName: 'frozen legacy compatibility export', + sqliteImport: 'legacy import-path compatibility re-export' + } + ], + ['tape/ports/application.ts', { physicalName: 'legacy database-shape compatibility adapter' }] +]) + +function listTypeScriptSources(root: string, fs: typeof import('node:fs')): string[] { + return fs + .readdirSync(root, { withFileTypes: true }) + .flatMap((entry) => { + const entryPath = path.join(root, entry.name) + if (entry.isDirectory()) return listTypeScriptSources(entryPath, fs) + if (entry.isFile() && TYPESCRIPT_SOURCE_EXTENSION.test(entry.name)) return [entryPath] + return [] + }) + .sort() +} + +function relativeToMain(file: string): string { + return path.relative(MAIN_SOURCE_ROOT, file).split(path.sep).join('/') +} + +function isInside(root: string, target: string): boolean { + const relativePath = path.relative(root, target) + return relativePath === '' || (!relativePath.startsWith('..') && !path.isAbsolute(relativePath)) +} + +function resolveMainImport(importingFile: string, specifier: string): string | null { + if (specifier.startsWith('@/')) { + return path.resolve(MAIN_SOURCE_ROOT, specifier.slice(2)) + } + if (specifier.startsWith('.')) { + return path.resolve(path.dirname(importingFile), specifier) + } + return null +} + +function withoutTypeScriptExtension(file: string): string { + return file.replace(TYPESCRIPT_SOURCE_EXTENSION, '') +} + +function matchesPackageOrSubpath(specifier: string, packages: ReadonlySet): boolean { + return [...packages].some( + (packageName) => specifier === packageName || specifier.startsWith(`${packageName}/`) + ) +} + +function getForbiddenDomainPackageCategory(specifier: string): string | null { + if ( + specifier === 'electron' || + specifier.startsWith('electron/') || + specifier.startsWith('@electron/') + ) { + return 'Electron runtime' + } + if ( + matchesPackageOrSubpath(specifier, FORBIDDEN_DOMAIN_SQLITE_IMPORTS) || + specifier.startsWith('@libsql/') + ) { + return 'SQLite runtime' + } + if (matchesPackageOrSubpath(specifier, FORBIDDEN_DOMAIN_LOGGING_IMPORTS)) { + return 'logging runtime' + } + return null +} + +function getDomainImportViolation(importingFile: string, specifier: string): string | null { + const forbiddenPackageCategory = getForbiddenDomainPackageCategory(specifier) + if (forbiddenPackageCategory) { + return `${forbiddenPackageCategory} import ${specifier}` + } + + const target = resolveMainImport(importingFile, specifier) + if (target && !isInside(TAPE_DOMAIN_ROOT, target)) { + return `main-process dependency ${specifier}` + } + return null +} + +function isLegacyTapeCompatibilityImport(importingFile: string, specifier: string): boolean { + const target = resolveMainImport(importingFile, specifier) + if (!target) return false + const relativeTarget = relativeToMain(withoutTypeScriptExtension(target)) + return LEGACY_TAPE_COMPATIBILITY_MODULES.has(relativeTarget) +} + +function isTapeModuleImport(importingFile: string, specifier: string): boolean { + const target = resolveMainImport(importingFile, specifier) + return Boolean(target && isInside(TAPE_ROOT, target)) +} + +function findConcreteTapeFacadeImportViolations(source: string, file: string): string[] { + return ts.preProcessFile(source, true, true).importedFiles.flatMap(({ fileName: specifier }) => { + const target = resolveMainImport(file, specifier) + return target && withoutTypeScriptExtension(target) === TAPE_SESSION_FACADE_MODULE + ? [`Concrete Tape facade import: ${specifier}`] + : [] + }) +} + +function findMemoryRouteTapeImportViolations(source: string, file: string): string[] { + const sourceFile = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true) + const tapeReferences = ts + .preProcessFile(source, true, true) + .importedFiles.filter(({ fileName }) => isTapeModuleImport(file, fileName)) + const staticTapeImports = sourceFile.statements.filter( + (statement): statement is ts.ImportDeclaration => + ts.isImportDeclaration(statement) && + ts.isStringLiteral(statement.moduleSpecifier) && + isTapeModuleImport(file, statement.moduleSpecifier.text) + ) + const violations = tapeReferences.flatMap(({ fileName: specifier }) => { + const target = resolveMainImport(file, specifier) + return !target || withoutTypeScriptExtension(target) !== TAPE_CAPABILITIES_MODULE + ? [`Tape import must use the inspection port: ${specifier}`] + : [] + }) + + if (tapeReferences.length !== staticTapeImports.length) { + violations.push('Tape references must use a static type-only import declaration') + } + + violations.push( + ...staticTapeImports.flatMap((statement) => { + const specifier = statement.moduleSpecifier.text + const importClause = statement.importClause + const namedBindings = importClause?.namedBindings + if ( + !importClause || + importClause.name || + !namedBindings || + !ts.isNamedImports(namedBindings) || + namedBindings.elements.length !== 1 + ) { + return [`Tape capabilities import must name only TapeInspectionReader: ${specifier}`] + } + + const [element] = namedBindings.elements + const importedName = element.propertyName?.text ?? element.name.text + const isTypeOnly = importClause.isTypeOnly || element.isTypeOnly + return importedName === 'TapeInspectionReader' && isTypeOnly + ? [] + : [`Memory routes may import only the TapeInspectionReader type: ${importedName}`] + }) + ) + + return [...new Set(violations)] +} + +function compatibilityExportDescriptors(contract: LegacyTapeCompatibilityContract): string[] { + return [ + ...contract.valueExports.map((name) => `value:${name}:${name}`), + ...contract.typeExports.map((name) => `type:${name}:${name}`), + ...Object.entries(contract.typeAliases ?? {}).map( + ([exportedName, importedName]) => `type:${importedName}:${exportedName}` + ) + ].sort() +} + +function isFrozenCompatibilityReexport( + source: string, + contract: LegacyTapeCompatibilityContract +): boolean { + if (!source.includes('@deprecated')) return false + const sourceFile = ts.createSourceFile('compatibility.ts', source, ts.ScriptTarget.Latest, true) + const descriptors = sourceFile.statements.flatMap((statement) => { + if ( + !ts.isExportDeclaration(statement) || + !statement.exportClause || + !ts.isNamedExports(statement.exportClause) || + !statement.moduleSpecifier || + !ts.isStringLiteral(statement.moduleSpecifier) || + statement.moduleSpecifier.text !== contract.target + ) { + return ['invalid'] + } + return statement.exportClause.elements.map((element) => { + const importedName = element.propertyName?.text ?? element.name.text + const exportedName = element.name.text + const kind = statement.isTypeOnly || element.isTypeOnly ? 'type' : 'value' + return `${kind}:${importedName}:${exportedName}` + }) + }) + return ( + sourceFile.statements.length > 0 && + JSON.stringify(descriptors.sort()) === JSON.stringify(compatibilityExportDescriptors(contract)) + ) +} + +describe('Tape layer boundaries', () => { + it('keeps the Tape domain independent from other main-process layers', async () => { + const fs = await vi.importActual('node:fs') + const violations = listTypeScriptSources(TAPE_DOMAIN_ROOT, fs).flatMap((file) => { + const source = fs.readFileSync(file, 'utf8') + const imports = ts.preProcessFile(source, true, true).importedFiles + + return imports.flatMap(({ fileName: specifier }) => { + const violation = getDomainImportViolation(file, specifier) + return violation ? [`${relativeToMain(file)}: ${violation}`] : [] + }) + }) + + expect(violations).toEqual([]) + }) + + it('keeps production code off legacy Tape compatibility imports', async () => { + const fs = await vi.importActual('node:fs') + const violations = listTypeScriptSources(MAIN_SOURCE_ROOT, fs).flatMap((file) => { + const source = fs.readFileSync(file, 'utf8') + return ts + .preProcessFile(source, true, true) + .importedFiles.flatMap(({ fileName: specifier }) => + isLegacyTapeCompatibilityImport(file, specifier) + ? [`${relativeToMain(file)} -> ${specifier}`] + : [] + ) + }) + + expect(violations).toEqual([]) + }) + + it('keeps legacy Tape compatibility modules on frozen deprecated export contracts', async () => { + const fs = await vi.importActual('node:fs') + const violations = [...LEGACY_TAPE_COMPATIBILITY_MODULES.entries()].flatMap( + ([relativeModule, contract]) => { + const file = path.join(MAIN_SOURCE_ROOT, `${relativeModule}.ts`) + const source = fs.readFileSync(file, 'utf8') + return isFrozenCompatibilityReexport(source, contract) + ? [] + : [`${relativeModule}.ts must match its frozen ${contract.target} export contract`] + } + ) + + expect(violations).toEqual([]) + }) + + it('keeps Memory routes on the Tape inspection DTO port', async () => { + const fs = await vi.importActual('node:fs') + const source = fs.readFileSync(MEMORY_ROUTES_FILE, 'utf8') + expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).toEqual([]) + }) + + it('keeps capability-scoped consumers off the concrete Tape facade', async () => { + const fs = await vi.importActual('node:fs') + const violations = CAPABILITY_SCOPED_CONSUMER_FILES.flatMap((file) => + findConcreteTapeFacadeImportViolations(fs.readFileSync(file, 'utf8'), file).map( + (violation) => `${relativeToMain(file)}: ${violation}` + ) + ) + + expect(violations).toEqual([]) + }) + + it.each([ + ['Session', '@/session/data/transcript'], + ['Agent', '@/agent/deepchat/runtime/process'], + ['Memory', '@/memory/routes'], + ['App', '@/app/composition'], + ['Tape ports', '@/tape/ports/capabilities'], + ['Tape SQLite infrastructure', '@/tape/infrastructure/sqlite/tapeEntryStore'], + ['bare SQLite', 'better-sqlite3'], + ['project SQLite driver', 'better-sqlite3-multiple-ciphers'], + ['Node SQLite', 'node:sqlite'], + ['Electron', 'electron'], + ['Electron subpath', 'electron/main'], + ['shared logging', '@shared/logger'], + ['Electron logging', 'electron-log'] + ])('detects forbidden %s imports in the Tape domain', (_category, specifier) => { + const importingFile = path.join(TAPE_DOMAIN_ROOT, 'negative-case.ts') + expect(getDomainImportViolation(importingFile, specifier)).not.toBeNull() + }) + + it.each([ + ['domain sibling', './entry'], + ['domain alias', '@/tape/domain/effectiveView'], + ['shared type', '@shared/types/tape-replay'], + ['Node crypto', 'node:crypto'] + ])('allows pure %s imports in the Tape domain', (_category, specifier) => { + const importingFile = path.join(TAPE_DOMAIN_ROOT, 'allowed-case.ts') + expect(getDomainImportViolation(importingFile, specifier)).toBeNull() + }) + + it.each([ + [path.join(MAIN_SOURCE_ROOT, 'agent/example.ts'), '@/session/data/tape'], + [path.join(MAIN_SOURCE_ROOT, 'session/data/index.ts'), './tapeFacts'], + [ + path.join(MAIN_SOURCE_ROOT, 'memory/example.ts'), + '@/session/data/tables/deepchatTapeEffectiveSemantics' + ], + [path.join(MAIN_SOURCE_ROOT, 'memory/example.ts'), '@/session/data/tables/deepchatTapeEntries'], + [ + path.join(MAIN_SOURCE_ROOT, 'app/example.ts'), + '@/session/data/tables/deepchatTapeSearchProjection' + ] + ])('detects legacy Tape compatibility import %s -> %s', (importingFile, specifier) => { + expect(isLegacyTapeCompatibilityImport(importingFile, specifier)).toBe(true) + }) + + it.each([ + [ + 'raw reader capability', + "import type { TapeRawEntryReader } from '@/tape/ports/capabilities'" + ], + [ + 'effective-view helper', + "import { buildEffectiveTapeView } from '@/tape/domain/effectiveView'" + ], + ['application facade', "import { SessionTape } from '@/tape/application/sessionTape'"], + ['inspection value import', "import { TapeInspectionReader } from '@/tape/ports/capabilities'"], + ['dynamic import', "void import('@/tape/application/sessionTape')"], + ['CommonJS require', "const tape = require('@/tape/domain/effectiveView')"], + ['type re-export', "export type { TapeInspectionReader } from '@/tape/ports/capabilities'"] + ])('detects Memory route Tape bypass through %s', (_category, source) => { + expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).not.toEqual([]) + }) + + it('allows Memory routes to import only the inspection reader type', () => { + const source = "import type { TapeInspectionReader } from '@/tape/ports/capabilities'" + expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).toEqual([]) + }) + + it('allows inline type syntax for the Memory inspection reader', () => { + const source = "import { type TapeInspectionReader } from '@/tape/ports/capabilities'" + expect(findMemoryRouteTapeImportViolations(source, MEMORY_ROUTES_FILE)).toEqual([]) + }) + + it.each([ + ['star export', "/** @deprecated */\nexport * from '@/tape/domain/effectiveView'"], + [ + 'extra export', + "/** @deprecated */\nexport { buildEffectiveTapeView, getLastEffectiveTokenUsage, searchEffectiveTapeRows, unexpected } from '@/tape/domain/effectiveView'\nexport type { EffectiveMessageEntry, EffectiveTapeView } from '@/tape/domain/effectiveView'" + ], + [ + 'missing export', + "/** @deprecated */\nexport { buildEffectiveTapeView } from '@/tape/domain/effectiveView'\nexport type { EffectiveMessageEntry, EffectiveTapeView } from '@/tape/domain/effectiveView'" + ], + [ + 'missing deprecation marker', + "export { buildEffectiveTapeView, getLastEffectiveTokenUsage, searchEffectiveTapeRows } from '@/tape/domain/effectiveView'\nexport type { EffectiveMessageEntry, EffectiveTapeView } from '@/tape/domain/effectiveView'" + ] + ])('rejects a legacy compatibility contract with a %s', (_case, source) => { + const contract = LEGACY_TAPE_COMPATIBILITY_MODULES.get('session/data/tapeEffectiveView')! + expect(isFrozenCompatibilityReexport(source, contract)).toBe(false) + }) + + it.each(['@/tape/application/sessionTape', '../../tape/application/sessionTape'])( + 'detects concrete Tape facade import %s in a capability-scoped consumer', + (specifier) => { + const file = path.join(MAIN_SOURCE_ROOT, 'session/data/consumer.ts') + const source = `import { SessionTape } from '${specifier}'` + expect(findConcreteTapeFacadeImportViolations(source, file)).not.toEqual([]) + } + ) + + it('allows physical Tape storage access only at explicit infrastructure boundaries', async () => { + const fs = await vi.importActual('node:fs') + const matchedExceptionCapabilities = new Set() + const violations = listTypeScriptSources(MAIN_SOURCE_ROOT, fs).flatMap((file) => { + const relativeFile = relativeToMain(file) + if (relativeFile.startsWith(TAPE_SQLITE_RELATIVE_ROOT)) return [] + + const source = fs.readFileSync(file, 'utf8') + const physicalName = source.match(PHYSICAL_TAPE_STORAGE_PATTERN)?.[0] + const sqliteImport = ts + .preProcessFile(source, true, true) + .importedFiles.map(({ fileName }) => ({ + fileName, + target: resolveMainImport(file, fileName) + })) + .find(({ target }) => target && isInside(TAPE_SQLITE_ROOT, target))?.fileName + const exception = ALLOWED_STORAGE_EXCEPTIONS.get(relativeFile) + const fileViolations: string[] = [] + + if (physicalName) { + if (exception?.physicalName) { + matchedExceptionCapabilities.add(`${relativeFile}:physicalName`) + } else { + fileViolations.push(`${relativeFile}: physical name ${physicalName}`) + } + } + if (sqliteImport) { + if (exception?.sqliteImport) { + matchedExceptionCapabilities.add(`${relativeFile}:sqliteImport`) + } else { + fileViolations.push(`${relativeFile}: SQLite import ${sqliteImport}`) + } + } + return fileViolations + }) + + const staleExceptions = [...ALLOWED_STORAGE_EXCEPTIONS.entries()].flatMap(([file, exception]) => + (Object.entries(exception) as Array<[keyof StorageBoundaryException, string]>).flatMap( + ([capability, reason]) => + matchedExceptionCapabilities.has(`${file}:${capability}`) + ? [] + : [`${file} (${capability}): ${reason}`] + ) + ) + + expect(violations).toEqual([]) + expect(staleExceptions).toEqual([]) + }) +}) diff --git a/test/memory-test-scope.json b/test/memory-test-scope.json index f0a41a447f..25630eaf77 100644 --- a/test/memory-test-scope.json +++ b/test/memory-test-scope.json @@ -51,7 +51,12 @@ "test/main/memory/agentMemoryAuditRetention.test.ts", "test/main/memory/agentMemoryPaginationNative.test.ts", "test/main/memory/agentMemoryTable.test.ts", - "test/main/session/data/tape.test.ts", + "test/main/session/data/tables/deepchatTapeEntriesTable.test.ts", + "test/main/session/data/tapeFork.test.ts", + "test/main/session/data/tapeLifecycle.test.ts", + "test/main/session/data/tapeLineage.test.ts", + "test/main/session/data/tapeRecall.test.ts", + "test/main/session/data/tapeViewReplay.test.ts", "test/main/memory/memoryNativeMigration.test.ts", "test/main/memory/memoryVectorStoreV2Native.test.ts", "test/main/memory/memoryUpdateNative.test.ts", From 296f34077bca2487695c58ccfbe429673810167e Mon Sep 17 00:00:00 2001 From: duskzhen Date: Sat, 18 Jul 2026 17:24:56 +0800 Subject: [PATCH 4/6] feat(browser): import sessions and add agent PiP (#1993) * docs(browser): specify import and pip * feat(browser): add import and agent pip * fix(browser): make agent pip read-only * fix(browser): refine pip hover controls * fix(browser): address review feedback --- docs/features/agent-browser-pip/plan.md | 421 +++++ docs/features/agent-browser-pip/spec.md | 641 +++++++ docs/features/agent-browser-pip/tasks.md | 121 ++ docs/features/browser-profile-import/plan.md | 362 ++++ docs/features/browser-profile-import/spec.md | 495 ++++++ docs/features/browser-profile-import/tasks.md | 98 ++ resources/acp-registry/registry.json | 123 +- resources/model-db/providers.json | 1564 ++++++++++++----- src/main/agent/deepchat/runtime/dispatch.ts | 1 + .../browser/BrowserProfileImportService.ts | 484 +++++ .../desktop/browser/YoBrowserPresenter.ts | 408 ++++- .../desktop/browser/YoBrowserToolHandler.ts | 18 +- src/main/desktop/browser/yoBrowserSession.ts | 70 +- src/main/desktop/routes.ts | 43 + src/main/tool/agentTools/agentToolManager.ts | 4 +- src/main/tool/index.ts | 1 + src/main/tool/runtimePorts.ts | 3 +- src/renderer/api/BrowserClient.ts | 46 +- .../components/BrowserDataImportDialog.vue | 250 +++ .../settings/components/DataSettings.vue | 93 +- .../components/browser/AgentBrowserPiP.vue | 514 ++++++ .../src/components/sidepanel/BrowserPanel.vue | 51 +- .../components/sidepanel/ChatSidePanel.vue | 57 +- src/renderer/src/i18n/da-DK/settings.json | 28 +- src/renderer/src/i18n/de-DE/settings.json | 28 +- src/renderer/src/i18n/en-US/settings.json | 26 + src/renderer/src/i18n/es-ES/settings.json | 28 +- src/renderer/src/i18n/fa-IR/settings.json | 28 +- src/renderer/src/i18n/fr-FR/settings.json | 28 +- src/renderer/src/i18n/he-IL/settings.json | 28 +- src/renderer/src/i18n/id-ID/settings.json | 28 +- src/renderer/src/i18n/it-IT/settings.json | 28 +- src/renderer/src/i18n/ja-JP/settings.json | 28 +- src/renderer/src/i18n/ko-KR/settings.json | 28 +- src/renderer/src/i18n/ms-MY/settings.json | 28 +- src/renderer/src/i18n/pl-PL/settings.json | 28 +- src/renderer/src/i18n/pt-BR/settings.json | 28 +- src/renderer/src/i18n/ru-RU/settings.json | 28 +- src/renderer/src/i18n/tr-TR/settings.json | 28 +- src/renderer/src/i18n/vi-VN/settings.json | 28 +- src/renderer/src/i18n/zh-CN/settings.json | 28 +- src/renderer/src/i18n/zh-HK/settings.json | 28 +- src/renderer/src/i18n/zh-TW/settings.json | 28 +- .../lib/icons/icon-collections.generated.ts | 16 +- .../src/lib/icons/icon-whitelist.generated.ts | 4 +- src/renderer/src/stores/ui/sidepanel.ts | 2 +- src/renderer/src/views/ChatTabView.vue | 4 + src/shared/contracts/domainSchemas.ts | 4 +- src/shared/contracts/events.ts | 2 + src/shared/contracts/events/browser.events.ts | 18 + src/shared/contracts/routes.ts | 8 + src/shared/contracts/routes/browser.routes.ts | 64 + src/shared/types/browser.ts | 37 + src/shared/types/desktop.ts | 22 +- src/shared/types/tool.d.ts | 1 + .../BrowserProfileImportService.test.ts | 195 ++ .../browser/YoBrowserPresenter.test.ts | 110 +- test/main/routes/contracts.test.ts | 33 + test/renderer/api/clients.test.ts | 63 + .../components/AgentBrowserPiP.test.ts | 285 +++ .../renderer/components/ChatSidePanel.test.ts | 36 + test/renderer/components/ChatTabView.test.ts | 7 + test/setup.ts | 1 + 63 files changed, 6717 insertions(+), 621 deletions(-) create mode 100644 docs/features/agent-browser-pip/plan.md create mode 100644 docs/features/agent-browser-pip/spec.md create mode 100644 docs/features/agent-browser-pip/tasks.md create mode 100644 docs/features/browser-profile-import/plan.md create mode 100644 docs/features/browser-profile-import/spec.md create mode 100644 docs/features/browser-profile-import/tasks.md create mode 100644 src/main/desktop/browser/BrowserProfileImportService.ts create mode 100644 src/renderer/settings/components/BrowserDataImportDialog.vue create mode 100644 src/renderer/src/components/browser/AgentBrowserPiP.vue create mode 100644 test/main/desktop/browser/BrowserProfileImportService.test.ts create mode 100644 test/renderer/components/AgentBrowserPiP.test.ts diff --git a/docs/features/agent-browser-pip/plan.md b/docs/features/agent-browser-pip/plan.md new file mode 100644 index 0000000000..571039204d --- /dev/null +++ b/docs/features/agent-browser-pip/plan.md @@ -0,0 +1,421 @@ +# Agent Browser Surfaces and Picture-in-Picture Implementation Plan + +## Status + +The read-only mirror revision is implemented. The workspace/tab and Fit-desktop sections below +remain future architecture work if visible multi-tab automation becomes necessary. Packaged +Windows and Linux validation remains open. + +## Delivery Strategy + +Implement lifecycle identity and the single-parent placement controller before rendering PiP or +adaptive page modes. A floating UI built on the current one-page/session model would encode the +wrong ownership and would race the existing panel attach/detach logic. + +```text +LoopRun.runId + | + v +Tool execution context -> YoBrowserToolHandler + | | + | begin/end run | touches Agent tab + v v +Browser workspace ------> placement controller + | | | + v v v + panel PiP detached +``` + +## Architecture Changes + +### 1. Propagate the Agent run identity + +Extend the internal `ToolCallOptions`/execution context with `runId` and pass it from each +`LoopRun` through the tool batch/deferred tool paths, `ToolService`, `AgentToolManager`, and +`YoBrowserToolHandler`. + +Add an explicit browser lifecycle port with idempotent methods equivalent to: + +```ts +beginAgentBrowserActivity(sessionId, runId, toolCallId) +finishAgentRun(sessionId, runId, outcome) +``` + +The first method marks/touches the Agent tab immediately before a browser command. The second is +called from the central terminal path for completed, failed, aborted, and superseded runs. It must +also run during session teardown. + +Do not subscribe to coarse renderer session status as a substitute. The runtime already owns the +exact identity and terminal boundary. + +### 2. Migrate one-page state to a browser workspace + +Refactor `YoBrowserPresenter` from `Map` to a workspace that owns: + +- a tab map and order; +- selected panel tab ID; +- primary Agent tab ID; +- active Agent run ID and last touched Agent tab ID; +- current host window/session activation state; +- panel and conversation bounds; +- Browser-surface visibility; +- PiP container/chrome state and current position; +- dismissed PiP run ID; +- a serialized placement operation queue. + +Keep `BrowserTab` as the page/CDP abstraction. Add owner/run/placement metadata around it rather +than teaching it UI policy. + +Preserve existing callers with an internal compatibility step: session-level navigation methods +resolve the relevant user or Agent active tab, while new typed tab methods are added for panel UI. +Remove the compatibility layer after all call sites and tests migrate; do not maintain duplicate +long-term APIs. + +### 3. Create the single placement controller + +Add one main-process function that computes desired placement from workspace state and performs all +native parent changes. No renderer component may call low-level attach/detach independently after +migration. + +Responsibilities: + +1. Reject stale layout/activation versions. +2. Evaluate the eligibility predicate. +3. Select the latest current-run Agent tab for PiP. +4. Detach its page from the old parent. +5. Attach it to the panel root or PiP container, or leave it detached. +6. Set the page and chrome bounds. +7. Move/rebound the activity overlay. +8. Publish final placement/status. + +Use a per-workspace promise queue or equivalent narrow serialization. Avoid a general state-machine +library. + +### 4. Build the read-only mirror pipeline + +Create one focusless render-host `BaseWindow` lazily for each active Agent page that needs +background rendering. Reparent the existing page `WebContentsView` into it at 1280 x 800. The host +must be transparent, offscreen, non-focusable, excluded from taskbar/Mission Control, and destroyed +after the page returns to panel or no longer needs background rendering. + +Use `webContents.capturePage` because the Electron 40.10.5 macOS feasibility spike showed that +`beginFrameSubscription` produced no frames for this WebContentsView while capture succeeded from a +technically visible transparent host. Limit capture to one in-flight request, resize in main to the +PiP output size, encode in memory, and publish a typed binary frame event. Never write frames to +disk. + +The existing Vue PiP component owns a Canvas, read-only activity halo, title, and controls. Decode a +new frame completely before drawing it so the previous Canvas pixels remain visible during capture, +handoff, and loading. The remote page never receives PiP input. + +### 5. Centralize renderer layout reporting + +Move browser placement facts out of the current panel-only attach/detach watcher. A renderer +controller near `ChatTabView.vue` reports: + +- active session ID and sender webContents identity; +- current conversation content rectangle; +- panel rectangle; +- side-panel open state and active surface; +- monotonically increasing layout version. + +Use VueUse `useResizeObserver` and `useEventListener` for mechanical observation. Keep the existing +stable-rectangle wait for panel transitions, but make it a layout readiness signal rather than +direct native attachment ownership. + +Validate every report in main against the sender's BrowserWindow and current activated session. + +### 6. Add panel tabs + +Migrate `YoBrowserStatus` from a single `page` to `tabs`, `activeTabId`, and placement data. Add +typed operations to create/select/close tabs and navigate the selected user tab. + +Refactor `BrowserPanel.vue` into: + +- a compact tab strip using existing shadcn-vue Button/Tooltip patterns; +- navigation controls bound to the selected tab; +- the native page placeholder/bounds container; +- no direct destroy-on-session-switch policy. + +User link navigation explicitly creates/selects a user tab and may keep opening the Browser panel. +Agent tools resolve the primary Agent tab. A popup inherits its opener's ownership. + +Retain inactive workspace pages after loop completion for inspection. Add no speculative persistent +tab database in V1; state remains process-lifetime only. If memory becomes a measured problem, add a +separate idle eviction policy after profiling. + +### 7. Make Browser chrome container-responsive + +Refactor the Browser panel's fixed toolbar into width-aware chrome owned by `BrowserPanel.vue` or a +focused child component: + +- wide: full tab strip and navigation controls; +- compact: scrollable tabs, icon controls, flexible address field, overflow menu; +- narrow: active title/host, essential navigation, and overflow commands; +- expanded: full controls with a restore-chat action. + +Prefer CSS/container queries and normal flex/grid layout. Keep one `ResizeObserver` at the surface +owner only for native bounds/reporting; do not create breakpoint watchers per tab. Reuse shadcn-vue +Button, Tooltip, DropdownMenu, and Tabs-compatible semantics where they fit. + +Generalize the existing Workspace fullscreen shell state into a narrow side-panel surface mode that +can be `normal` or `expanded` for either Workspace or Browser. Keep this local to the +`ChatSidePanel`/route shell rather than adding global persisted state. + +Fix target-session compatibility by deriving the user-agent Chromium version from the runtime and +keeping desktop semantics at every surface width. + +### 8. Add page presentation policy and feasibility proof + +Add per-tab `presentationMode: 'responsive' | 'fit-desktop'` to main-process workspace state. +Responsive mode uses real content bounds and 100% zoom. + +Before implementing Fit desktop, build a throwaway proof comparing: + +- `webContents.enableDeviceEmulation` with `viewSize` and `scale`; +- `webContents.setZoomFactor`/zoom level with its documented same-origin propagation. + +Test a fixture matrix containing responsive layouts, fixed 1024/1280 px layouts, sticky/fixed +elements, iframes, canvas, editors, forms, context menus, and long pages. Verify renderer mouse/wheel +input, CDP input, screenshot dimensions, DOM coordinates, selection, navigation, panel/PiP movement, +and reset behavior. + +Select no fit API until the proof produces one coordinate model for both the user and Agent. If +neither path is reliable, ship Responsive plus Expand and defer Fit desktop. + +### 9. Implement PiP commands + +#### Dismiss + +Validate that the command's workspace/tab/run still matches, set `dismissedPipRunId`, and recompute. +Do not navigate, close, or interrupt tools. + +#### Open in panel + +Publish a renderer intent to open/activate Browser and select the Agent tab. Keep PiP visible until +the renderer returns stable visible panel bounds, then reparent atomically. If readiness times out, +leave PiP in place. + +#### Drag + +Use pointer capture on the entire PiP surface except buttons. Treat movement beyond a small +threshold as drag; otherwise toggle the controls. Keep drag and clamping renderer-local because the +PiP is now ordinary DOM inside the already-clipped conversation region. Avoid writing settings in +V1. + +### 10. Integrate foreground/session lifecycle + +Combine existing BrowserWindow focus/show/hide/resize/close listeners with typed session activation +events. Also handle minimize/restore explicitly. + +On deactivation or loss of foreground: + +- detach PiP synchronously; +- hide activity overlay; +- retain run/tab state. + +On valid reactivation, recompute from current facts. On terminal run, clear active run state and +detach. On session destruction, destroy every page and the local chrome. + +### 11. Adapt the activity overlay + +First, make `YoBrowserOverlayWindow.updateBounds` accept the page-content bounds produced by the +placement controller for both panel and PiP. Cross-platform test z-order, pointer pass-through, +focus, movement, and rounded clipping. + +If it cannot reliably follow nested PiP, replace only the activity drawing surface with a trusted +local child view in the PiP container. Do not replace it preemptively and do not create a second +remote page. + +## Implementation Phases + +### Phase 0: interaction decisions + +- Confirm PiP close suppresses only the current run. +- Confirm one retained Agent tab per session. +- Confirm behavior while a run waits for permission/question. +- Confirm Responsive + explicit Fit desktop + Expand as the adaptation model. +- Obtain the reference screenshot or approve a visual baseline. + +Resolved requirements: + +- when any right-side panel surface is open, each new Agent browser action activates Browser and + selects the Agent tab; +- if the user returns to Workspace, do not force another switch until the next Agent browser action; +- use a compact activity strip when the conversation cannot fit a usable page card. + +Exit criterion: no unresolved product branch changes placement semantics. + +### Phase 1: lifecycle and workspace foundation + +- Thread `runId` through every immediate and deferred tool path. +- Add explicit browser run finalization. +- Introduce workspace/tab ownership and status contracts. +- Add unit tests for loop/tab ownership and terminal cleanup. + +Exit criterion: tests can distinguish user and Agent tabs and hide current-run presentation exactly +at terminal state without any PiP UI. + +### Phase 2: placement controller, panel tabs, and responsive chrome + +- Centralize layout reports and remove renderer-owned native attachment races. +- Add the single-parent placement controller. +- Add panel tab UI and migrate navigation commands. +- Add container-responsive navigation/tab chrome and Browser expanded mode. +- Replace the stale hard-coded Chromium user-agent version with runtime-derived data. +- Preserve the same page across detach/panel transitions. + +Exit criterion: multiple tabs work in the Browser panel and every view satisfies the placement +invariant under stress tests. + +### Phase 3: page-presentation proof and PiP surface + +- Prove input/screenshot coordinate fidelity for Fit desktop candidates. +- Ship Fit desktop only if one candidate passes; otherwise keep Responsive + Expand. +- Add trusted local chrome and in-window View container. +- Add eligibility, show/hide, drag, dismiss, and open-panel handoff. +- Add size/clamp behavior, accessibility, i18n, and activity indicator. +- Add the compact activity-strip fallback for undersized chat regions. +- Integrate the existing activity overlay. + +Exit criterion: the complete acceptance matrix passes in development builds on macOS, Windows, and +Linux. + +### Phase 4: resilience and packaged QA + +- Test focus, minimize, multiple windows, session switches, route changes, panel animation, window + resize, display scale, tab/page/chrome crashes, and app shutdown. +- Test simultaneous Agent tool completion and user PiP/panel commands. +- Validate packaged local chrome routing and security settings. +- Profile page retention and native view churn. + +Exit criterion: no cross-session frame leak, duplicate attachment, unexpected panel opening, or +tool interruption is observed in packaged builds. + +## Expected File Areas + +Likely implementation areas, subject to final naming: + +- `src/main/agent/deepchat/**` for `runId` propagation and terminal lifecycle calls; +- `src/main/tool/**` and `src/shared/types/tool.d.ts` for internal execution context; +- `src/main/desktop/browser/YoBrowserPresenter.ts` for workspace orchestration; +- focused new files under `src/main/desktop/browser/` for workspace placement and PiP chrome; +- `src/shared/types/browser.ts`, browser routes, and browser events for tab/layout/placement state; +- `src/preload/` for typed commands and a narrow PiP chrome bridge; +- `src/renderer/src/components/sidepanel/BrowserPanel.vue` for tab UI; +- `src/renderer/src/stores/ui/sidepanel.ts` and `ChatSidePanel.vue` for generalized expanded mode; +- a renderer controller near `ChatTabView.vue` for layout facts; +- `src/renderer/src/components/sidepanel/ChatSidePanel.vue` to remove unconditional Agent open; +- `src/renderer/src/i18n/` for visible strings; +- mirrored main and renderer tests. + +Avoid coupling this to `src/main/desktop/tab.ts`; standalone browser windows have a different +window/tab lifecycle and do not need Agent PiP policy. + +## Contract Migration + +Current session-level routes can be migrated in two steps: + +1. Expand status with tabs while preserving derived `page` fields temporarily for existing + renderer callers. +2. Move panel/client callers to explicit `tabId`, then remove the derived single-page fields and old + attach/detach routes in the same feature branch. + +The final architecture must not retain two attachment owners. Route schemas should reject stale tab +IDs and return stable `tab_not_found`, `tab_closed`, `stale_layout`, or `placement_failed` results. + +## Test Strategy + +### Pure state tests + +Use a table-driven reducer/decision function for desired placement. Cover every boolean in the +predicate and every transition in the spec, including dismiss-current-run versus next-run behavior. + +### Main-process tests + +- single parent across panel/PiP/detached transitions; +- exact page WebContents identity across moves; +- serialized races between bounds, focus, loop finish, and tab close; +- stale sender/layout rejection; +- multiple sessions and windows without cross-session display; +- popup ownership inheritance; +- Agent tab recreation after explicit close; +- chrome/page/host destruction cleanup; +- overlay bounds for both placements. + +Wrap native `View` operations behind a narrow testable adapter only where Electron objects cannot be +constructed in Vitest. Do not build a general UI framework. + +### Runtime tests + +- browser tool receives the exact `LoopRun.runId`; +- immediate and deferred tool execution use the same identity; +- completed, failed, aborted, and superseded runs all finalize once; +- a later run does not inherit dismissal or activity from the prior run; +- permission/question pauses do not appear terminal. + +### Renderer tests + +- Agent open events do not open a closed panel; +- user navigation still opens the panel intentionally; +- panel tab selection, close, Agent marker, and navigation state; +- layout versioning and stable bounds readiness; +- open-panel handoff timeout keeps PiP rather than losing the page; +- all visible controls have i18n labels and keyboard focus; +- container-width breakpoints preserve the address bar and all commands at minimum panel width; +- expanded/restore focus behavior and page identity; + +### Packaged cross-platform tests + +- click/focus/input routing between chat, PiP chrome, and remote page; +- responsive, fit-desktop, and expanded rendering across representative site fixtures; +- CDP input and screenshot coordinates at every approved presentation scale; +- drag and clamping at multiple DPI/display scales; +- z-order during panel animation and other overlays; +- focus/blur, minimize/restore, hide/show, and full-screen behavior; +- multiple app windows with different active sessions; +- no PiP outside chat bounds and no flash from an inactive session. + +## Performance and Retention + +- One retained Agent page per active session is the initial memory budget. +- PiP chrome is lazy and destroyed with its host/workspace. +- Reparent and bound updates do not recreate CDP sessions or reload pages. +- Drag updates are coalesced to one per animation frame. +- Layout reports use equality checks and versions to avoid redundant native calls. +- Browser chrome uses CSS/container layout rather than reactive resize fan-out. +- Presentation scaling is applied only after stable bounds, never on every resize sample. +- Measure retained page memory before adding an eviction abstraction. If needed, prefer a simple + idle timeout tied to session state over a general cache. + +## Rejected Approaches + +- **Transparent child BrowserWindow as the user-facing PiP**: platform-dependent movement/focus and + poor in-chat clipping. +- **Live native WebContentsView inside PiP**: cannot be made reliably read-only and creates focus, + z-order, and renderer blur/attach feedback loops. +- **Offscreen shared texture**: requires a native graphics module and complicates moving the same + WebContents back into the visible panel; bounded capture is sufficient for a low-frame-rate + preview. +- **Second WebContentsView pointing at the same URL**: duplicates navigation, page state, CDP, and + potentially side effects. +- **Inject controls into the remote page**: unsafe, spoofable, and breaks arbitrary sites. +- **Renderer CSS iframe/webview**: changes the security model and cannot adopt the existing page + WebContents. +- **Use session `working` status as loop identity**: cannot distinguish runs or terminal races. +- **Always scale narrow pages automatically**: horizontal overflow is not a reliable signal of site + intent and silent scaling can break readability and Agent coordinates. +- **Use only `setZoomFactor` for per-tab fit**: Chromium zoom is same-origin and may propagate beyond + the intended tab. +- **Switch to a mobile user-agent in narrow surfaces**: changes site behavior and session/device + assumptions instead of solving layout. +- **Create one tab per Agent URL load**: unbounded tab growth without a user need. +- **Persist full browser workspace/PiP geometry in V1**: adds restore/migration work before the + lifecycle is proven. + +## Complexity Budget + +No new runtime dependency is expected. Use Electron View/WebContentsView/webContents APIs, existing +VueUse helpers, shadcn-vue controls, typed routes/events, and a small placement decision function. +The feature needs one workspace state owner, one surface layout owner, and one placement owner; +additional window managers or state-machine libraries would make correctness harder, not easier. diff --git a/docs/features/agent-browser-pip/spec.md b/docs/features/agent-browser-pip/spec.md new file mode 100644 index 0000000000..601e6494e3 --- /dev/null +++ b/docs/features/agent-browser-pip/spec.md @@ -0,0 +1,641 @@ +# Agent Browser Surfaces and Picture-in-Picture Spec + +## Status + +V1 mirror revision implemented on 2026-07-17. The first implementation placed the live +`WebContentsView` inside the PiP, which made the preview interactive and introduced a renderer +focus/blur feedback loop. The revised contract keeps the Agent page at a fixed 1280 x 800 render +size and shows a low-frame-rate, read-only Canvas mirror in PiP. A visible multi-tab strip and +Fit-desktop emulation remain deferred. Packaged Windows and Linux validation remains open. + +The referenced screenshot was not available in the current task context, so the interaction and +layout are specified here while exact visual styling remains provisional. + +## Problem + +Today, any YoBrowser `loadUrl` call publishes `browser.open.requested`, and +`ChatSidePanel.vue` responds by opening and activating the right-side Browser panel. This lets a +background Agent tool unexpectedly take over the user's layout. + +YoBrowser also has only one page per chat session. It cannot distinguish a user-opened page from an +Agent automation page or expose multiple browser tabs in the panel. + +The current Browser panel is physically constrained to a default width of 520 px and a minimum of +420 px. `BrowserPanel.vue` keeps all navigation controls in one fixed row, while the remote page +receives the exact narrow native view bounds with no presentation-mode choice. Responsive sites may +adapt, but desktop-oriented sites can become horizontally cramped or unusable. + +The target session also hard-codes a Chrome/142 user-agent string while the pinned Electron 40.10.5 +runtime embeds Chromium 144.0.7559.236. That mismatch is not the main cause of narrow layout, but it +is a compatibility defect and must not become part of a new adaptive-surface design. + +The requested behavior is: + +- never open a closed right-side panel merely because an Agent uses YoBrowser; +- show the Agent-operated page in the existing Browser panel when that surface is already visible; +- otherwise show it as a draggable in-chat floating preview; +- make the floating preview read-only while the Agent operates the same live page in a normal-size + background render host; +- move the same live page between the background render host and panel without reload or state + loss; +- never float user-opened browser pages; +- hide the floating preview when its Agent loop ends, the chat is no longer active/foreground, the + page is closed, or the Browser panel becomes visible. +- make the Browser panel chrome responsive at its current supported widths; +- give desktop-oriented pages a deliberate wide/fit escape hatch instead of leaving them trapped in + an unusable narrow viewport. + +## Critical Architecture Correction + +The user-facing PiP is not another operating-system window and never contains the remote page's +native View. Electron `View` does not expose a reliable view-level ignore-mouse-input contract, and +native child views sit outside normal Vue DOM hit testing. The trusted chat renderer therefore owns +the complete PiP surface and draws captured frames into a Canvas. + +The live Agent `WebContentsView` is temporarily reparented into a focusless render-host +`BaseWindow`. The render host is technically visible so Chromium owns a valid display surface, but +it is transparent, moved offscreen, excluded from taskbar/Mission Control, and never accepts input. +Its page bounds remain 1280 x 800. Opening Browser reparents that exact View into the visible panel. + +```text +focusless render-host BaseWindow ++-- Agent page WebContentsView at 1280 x 800 + +-- capturePage -> resize/encode -> typed frame event + +host BrowserWindow.contentView ++-- chat renderer WebContentsView + +-- trusted PiP Canvas mirror and controls ++-- panel placement: the same Agent page WebContentsView when Browser is visible +``` + +There is still one page `WebContents`, one page `WebContentsView`, and one session. The mirror is +only a transient compressed frame, never a second browser. Moving the page changes only its native +parent and bounds. It does not navigate, clone cookies, recreate CDP, or copy page state. + +## Goals + +- Preserve the user's panel open/closed choice when an Agent opens a URL. +- Add browser tabs so user and Agent pages have explicit identity and lifecycle. +- Reuse one Agent automation tab per chat session in the first release. +- Track which Agent loop owns the current automation activity by `runId`. +- Put each page view in exactly one native placement: Browser panel, render host, or detached. +- Show at most one PiP for the foreground chat window. +- Keep PiP fully inside the active conversation region and let the user drag it. +- Keep the background Agent viewport at 1280 x 800 while PiP scales only its visual mirror. +- Keep responsive page rendering as the safe default and offer explicit desktop-fit and expanded + modes for pages that do not work well at narrow widths. +- Let the user dismiss PiP for the current loop without closing or disrupting the automation tab. +- Let the user open the Browser panel and transfer the same page into it. +- Hide PiP deterministically on every loop/session/window/tab lifecycle edge. +- Preserve the existing Agent activity visualization over whichever placement contains the page. + +## Non-Goals + +- Native video picture-in-picture or `documentPictureInPicture`. +- A free-floating OS window outside the DeepChat window. +- Multiple simultaneous PiP cards. +- Floating pages created and navigated only by the user. +- Persisting PiP position or size across application restarts in V1. +- Guessing that every horizontally overflowing page should be silently scaled down. +- Pretending to be a mobile browser or changing touch/user-agent semantics merely because the + surface is narrow. +- Letting the renderer own or directly manipulate remote-page `webContents`. +- Creating a new Agent tab for every `load_url` call. +- A general browser-window/tab architecture shared with standalone DeepChat browser windows. +- Closing the underlying Agent tab when the PiP close button is pressed. +- Forwarding PiP pointer, wheel, keyboard, selection, or focus events to the remote page. +- Full-frame-rate video streaming or a GPU shared-texture/native-module pipeline. + +## Current Repository Contract + +- `YoBrowserPresenter` owns `Map` with exactly one + `WebContentsView`, one `BrowserTab`, and one `YoBrowserOverlayWindow` per chat session. +- `YoBrowserToolHandler` can identify the chat session but does not receive the active Agent + `runId`. +- `BrowserPanel.vue` attaches the single view to bounds reported by the Browser panel and destroys + inactive-session pages once work stops. +- `sidepanel.ts` clamps the panel to 420-960 px, defaults to 520 px, and caps it at 62% of the + renderer width. +- `BrowserPanel.vue` uses one fixed navigation row and has no compact toolbar, expanded-browser + mode, or page-presentation mode. +- `yoBrowserSession.ts` hard-codes Chrome/142 rather than deriving the user-agent version from the + bundled Chromium runtime. +- `ChatSidePanel.vue` automatically calls `sidepanelStore.openBrowser()` for every + `browser.open.requested` event. +- `useMarkdownLinkNavigation.ts` intentionally opens the Browser panel for user navigation. +- `LoopRun` and the DeepChat runtime already own a stable `runId`; it is not currently threaded + through `ToolCallOptions` to YoBrowser. +- `YoBrowserOverlayWindow` is a transparent transient activity effect, not a PiP surface. + +## Terminology + +- **Browser workspace**: all YoBrowser tabs for one chat session. +- **User tab**: a page explicitly created from user UI. It is never PiP-eligible. +- **Agent tab**: the session's automation page, created or reused by an Agent browser tool. +- **Touched run**: the Agent `runId` that most recently operated an Agent tab. +- **Browser surface visible**: the right-side panel is open and its Browser surface is the active + panel content. +- **Native placement**: `panel`, `render-host`, or `detached`. +- **Mirror state**: `capturing`, `rendering`, or `stopped`; only `capturing` publishes PiP frames. +- **Presentation mode**: `responsive` or `fit-desktop` for how a page uses the available content + rectangle. +- **Active session**: the chat route currently activated in the host renderer/webContents. +- **Foreground host**: the containing DeepChat window is shown, not minimized, and focused. + +## Browser Tab Model + +The current `SessionBrowserState` becomes a per-session workspace: + +```ts +type BrowserTabOwner = 'user' | 'agent' +type BrowserPlacement = 'panel' | 'pip' | 'detached' + +type YoBrowserTabState = { + id: string + owner: BrowserTabOwner + page: BrowserPage + view: WebContentsView + placement: BrowserPlacement + lastAgentRunId: string | null + closed: boolean + createdAt: number + updatedAt: number +} +``` + +The public status becomes a tab collection plus `activeTabId`. User commands act on the selected +user-facing tab. Agent tools act on the dedicated Agent tab unless a future tool explicitly accepts +a tab ID. + +V1 rules: + +- create at most one primary Agent tab per session; +- reuse it across URL loads and across loops until the user explicitly closes it or the workspace + is destroyed; +- set `lastAgentRunId` before executing every Agent browser tool, not only `load_url`; +- pages opened by an Agent-owned popup inherit Agent ownership and the same touched run, but the + panel may list them as additional tabs; +- never change a user tab to Agent ownership implicitly; +- if a closed Agent tab is needed by a later browser tool, create a new Agent tab; +- choose only one current Agent tab for PiP, preferring the page touched by the latest tool. + +## Responsive Surface Design + +There are two different adaptation problems and they must not be conflated: + +1. **Browser chrome adaptation** controls DeepChat-owned tabs, buttons, address bar, and PiP header. +2. **Page presentation** controls how the untrusted website interprets and fills its native content + rectangle. + +### Browser chrome breakpoints + +Use container width, not the application viewport, because the Browser panel and PiP can have +different widths in the same window. + +| Content width | DeepChat chrome behavior | +| --- | --- | +| `>= 640 px` | Full tab strip, back, forward, reload, address bar, presentation, and expand controls | +| `480-639 px` | Scrollable compact tabs, icon navigation, flexible address bar, presentation/expand in overflow | +| `< 480 px` | Active-tab title/host, back, reload, address bar, and one overflow menu; hide nonessential labels | + +The address bar remains the flexible element with `min-width: 0`. Tab titles truncate, the tab strip +scrolls horizontally instead of squeezing every tab, and all hidden commands remain keyboard +reachable through the overflow menu. + +```text +Wide panel ++----------------------------------------------------------------+ +| [User tab] [Agent tab *] [+] | +| [<-] [->] [reload] [ URL ] [Fit] [Expand]| ++----------------------------------------------------------------+ + +Compact panel ++----------------------------------------------+ +| [User] [Agent *] ... | +| [<-] [reload] [ URL ] [...]| ++----------------------------------------------+ + +Narrow PiP ++--------------------------------------+ +| agent · site.example [Panel] [...] [x]| ++--------------------------------------+ +``` + +Use CSS/container-query layout for renderer-owned chrome where possible. Native page bounds still +come from the measured content rectangle. + +### Page presentation modes + +#### Responsive, default + +- The website viewport equals the actual native content bounds. +- Page zoom is 100%. +- CSS media queries, viewport units, and normal responsive behavior work as designed by the site. +- PiP/panel movement changes only bounds after the destination layout is stable. + +This is the only mode that can be selected automatically without guessing site intent. + +#### Fit desktop, explicit per tab + +For desktop-oriented sites, the user can choose **Fit desktop**. YoBrowser presents a wider virtual +desktop viewport and scales it into the available surface. A target virtual width near 1024 CSS px +is a starting hypothesis, not a committed constant. + +Implementation must choose between Electron `webContents.enableDeviceEmulation` and a controlled +zoom strategy only after a feasibility proof verifies: + +- layout/media-query behavior in panel and PiP; +- text readability and minimum scale; +- mouse, wheel, keyboard, focus, selection, and context-menu coordinates; +- CDP `Input.dispatchMouseEvent` coordinates; +- visible and full-page screenshot dimensions used by Agent tools; +- no per-origin zoom leakage into another tab; +- deterministic reset to Responsive mode after navigation, tab reuse, and view reparenting. + +Electron's Chromium zoom policy is same-origin, so `setZoomFactor` must not be assumed to provide +isolated per-tab fit behavior. CDP/device emulation changes CSS viewport coordinates and may affect +Agent screenshots/input. Until the proof passes, the product contract is Responsive plus **Expand**, +not unverified automatic scaling. + +Presentation mode belongs to the tab. It survives panel/PiP movement and the current process +lifetime, but is not persisted across app restarts in V1. + +### Expanded Browser mode + +Add an explicit **Expand** action that temporarily lets the Browser surface occupy the full +`ChatTabView` content area, reusing the existing Workspace fullscreen shell behavior. This is the +reliable escape hatch for complex desktop sites, wide tables, editors, and login flows where a +scaled page would become unreadable. + +Expanded mode does not create a new window or page. It changes the side-panel shell geometry, keeps +the same selected tab/view, provides a clear **Restore chat** action, and returns focus to the +originating control. + +### Safe adaptation policy + +- DeepChat chrome adapts automatically. +- Website viewport stays Responsive automatically. +- DeepChat may detect persistent document-level horizontal overflow and offer **Fit desktop** or + **Expand**, but it does not silently switch modes based on `scrollWidth` heuristics. +- Recompute native bounds during drag/resize at animation-frame cadence, but apply presentation-mode + changes only after bounds stabilize or resizing ends. +- Derive the desktop user-agent Chromium version from the runtime; do not switch to a mobile + user-agent for narrow surfaces. + +## Visibility Predicate + +PiP is visible only when all of these are true: + +```text +tab.owner == agent +AND tab.lastAgentRunId == workspace.activeAgentRunId +AND active Agent run is not terminal +AND tab is not closed +AND right-side panel is closed +AND workspace session is the active chat in its host renderer +AND host window is foreground +AND workspace.dismissedPipRunId != workspace.activeAgentRunId +``` + +Every input is main-process-authoritative or validated against the sender's host window. The +renderer reports layout facts; it does not decide whether an arbitrary tab may float. + +### Confirmed panel-open behavior + +If the right-side panel is already open, an Agent browser action activates the Browser surface, +selects the current Agent tab, waits for stable Browser bounds, and places the page in the panel. It +does this even when Workspace was the active surface. No PiP is shown during that handoff. + +The Browser surface is not pinned. The user may switch back to Workspace afterward. That switch +detaches the Agent page without showing PiP because the panel remains open; the next Agent browser +tool action activates Browser again. This prevents a continuous tab-switch fight while still making +each new Agent operation visible. + +## Loop Lifecycle + +An Agent run becomes relevant to YoBrowser only after its first browser tool starts. There is no +empty PiP at run start. + +```text +run starts + -> no browser action: no PiP state + -> first browser tool: mark Agent tab with runId + -> right-side panel open: activate Browser, then place in panel + -> panel closed: place page in render host + -> session/host foreground and not dismissed: capture mirror into PiP + -> more browser tools: reuse tab and recompute placement + -> run completes / fails / aborts: stop capture and hide PiP immediately +``` + +A permission or question pause is not a terminal loop state. PiP remains eligible unless the user +dismisses it or another visibility condition becomes false. Terminal means completed, failed, +cancelled, superseded, or session teardown. + +Loop completion hides PiP but does not close the tab. This preserves the result for later inspection +in the Browser panel. A separate retention policy handles eventual destruction. + +## Placement Invariant and Transitions + +For each page view: + +```text +number of native parents <= 1 +placement == panel => parent is host.contentView +placement == render-host => parent is focusless renderHost.contentView at 1280 x 800 +placement == detached => no native parent +``` + +All transitions go through one main-process placement controller. It removes the old parent before +adding the new parent, applies bounds, and starts or stops frame capture. The PiP Canvas is not a +native parent and never owns the page. Concurrent renderer mode requests, panel bounds, captures, +and Agent tool events are serialized per workspace. + +| Event | From | Result | +| --- | --- | --- | +| First Agent browser action, panel open | Detached | Activate Browser, then Panel | +| First Agent browser action, panel closed | Detached | Render host; mirror if eligible | +| User clicks **Open in panel** | Render host | Open/activate Browser, then Panel | +| Browser surface becomes visible | Render host | Panel | +| User switches Browser to Workspace while panel stays open | Panel | Detached until next Agent browser action | +| User closes panel during active eligible run | Panel | Render host and mirror | +| User clicks PiP **Close** | Render host | Keep rendering; stop frames and suppress mirror for this run | +| Chat session deactivates | Render host or Panel | Stop frames; retain render host only while Agent work requires it | +| Host blurs/hides/minimizes | Render host | Stop frames; keep Agent rendering | +| Same host returns foreground with eligible run | Render host | Resume mirror capture | +| Agent run becomes terminal | Render host | Stop capture, release host, retain page detached | +| Agent tab closes | Any | Destroy and no PiP | + +The panel transition waits for a stable content rectangle before reparenting. During the short +handoff, show the existing Browser placeholder; never display the remote page twice. + +## User Experience + +### Before: Agent navigation forces the panel open + +```text ++--------------------------------------+-----------------------+ +| Active conversation | Browser | +| | [Back] [ URL ] | +| Agent is working... | | +| | agent-operated page | +| | | ++--------------------------------------+-----------------------+ +``` + +### After: closed panel stays closed and Agent page floats inside the chat + +```text ++----------------------------------------------------------------+ +| Active conversation | +| | +| Agent is working... +-------------------------------+ | +| | | | +| | read-only Agent page mirror | | +| | | | +| | [Open panel] [x] | | +| +-------------------------------+ | +| | ++----------------------------------------------------------------+ +``` + +### After: Browser panel was already visible + +```text ++--------------------------------------+-------------------------+ +| Active conversation | Browser | +| | [User tab] [Agent tab] | +| Agent is working... | | +| | same live Agent page | +| | | ++--------------------------------------+-------------------------+ +``` + +### PiP controls + +- Controls are hidden initially. A click without drag on any non-button point toggles a toolbar with + a truncated title, **Open in panel**, and **Close**. +- Every point in the PiP except buttons is a drag handle. Movement beyond the drag threshold must not + also toggle the toolbar. +- The Canvas and decorative overlays use no remote-page input forwarding. Pointer, wheel, keyboard, + selection, and focus stay in the trusted chat renderer. +- **Close** sets `dismissedPipRunId` to the current run and stops mirror capture. It does not close + the tab, stop the 1280 x 800 render host, cancel CDP, or abort the Agent loop. +- **Open in panel** opens and activates the Browser panel, selects the Agent tab, and reparents the + same page after the panel bounds stabilize. +- A keyboard user can focus both buttons. Dragging is pointer-based; keyboard movement is a later + enhancement unless accessibility review requires it for V1. + +### Geometry + +- Default to the bottom-right of the measured conversation content with a 12 px inset. +- Prefer a 400 x 250 px 16:10 mirror when the conversation has room. Keep the captured frame at + 480 x 300 so the smaller display remains legible without changing the background page's + 1280 x 800 CSS viewport. +- Reveal the top actions and a centered drag affordance on hover or keyboard focus. A non-drag tap + keeps those controls visible for pointer devices without hover. +- Clamp height to at most 58% of the conversation region and keep the full header reachable. +- On smaller regions, shrink to the available bounds. If usable page content would fall below + 360 x 240, replace the page card with a compact Agent-browser activity strip containing + **Open in panel** and **Dismiss** rather than obscuring the entire conversation with an unusable + webpage. +- Drag updates are renderer-local, throttled to animation-frame cadence, and clamped against the + measured conversation bounds. +- Remember the last position only for the current host window lifetime. Re-clamp on resize, panel + animation, route changes, and display scale changes. + +Exact radius, shadow, header height, and icon treatment should be matched to the missing reference +screenshot during implementation review. + +## Panel Tab Interaction + +The Browser panel adds a compact tab strip below its existing Workspace/Browser selector and above +navigation controls. This is browser-tab state, not a second copy of the global side-panel tabs. + +```text ++--------------------------------------------------+ +| [Workspace] [Browser] [x] | +| [User page] [Agent page *] [+] | +| [<-] [->] [reload] [ URL ] | +| | +| active page | ++--------------------------------------------------+ +``` + +- User navigation from a markdown link may keep its current explicit behavior of opening the + Browser panel and creates/selects a user tab. +- Agent navigation never selects a user tab and never creates one tab per URL. +- An Agent marker distinguishes automation tabs. +- Closing an Agent tab destroys that page and removes PiP eligibility. The Agent receives a stable, + recoverable page-closed result if a concurrent command loses the tab. +- Closing the panel detaches user pages. During an eligible Agent run, its active Agent page moves + to PiP; otherwise it also detaches. +- **Expand** is available for user and Agent tabs. Presentation mode is stored per tab and follows + the same live page into panel, PiP, and expanded placement. + +## Active Session and Foreground Semantics + +- Bind workspace visibility to the renderer `webContentsId` that reports the session as activated. +- Reject layout reports whose sender does not own the target host window/session route. +- A session visible in another window wins only when it is the currently activated copy; do not + mirror one page view into two windows. +- Window blur, hide, minimize, close, route replacement, or session deactivation hides the Canvas + mirror and stops frame delivery. These events are derived from BrowserWindow state, not + `document.hasFocus()`, because focus can legitimately move between WebContents in one window. +- Window refocus may restore PiP only if the same non-terminal Agent run remains active and was not + dismissed. +- Switching to another chat cannot expose the previous chat's page even for one frame. + +## Activity Overlay + +The existing `YoBrowserOverlayWindow` visualizes pointer, keyboard, navigation, and vision activity. +It should remain an activity layer, not become the PiP implementation. + +Its native bounds follow the page only in the visible Browser panel. PiP reuses the typed activity +event and draws a trusted renderer halo over the Canvas, avoiding another native overlay window, +z-order interaction, or focus change. + +## Events and Public State + +Replace the ambiguous open request with intent-rich state. Proposed public concepts: + +- workspace/tabs status: IDs, owner, URL/title/favicon, page status, active tab, placement; +- per-tab presentation mode and current effective page viewport/scale metadata; +- Agent browser activity: `sessionId`, `runId`, `tabId`, tool activity phase; +- layout report: host window, active session, conversation bounds, panel bounds, Browser surface + visibility; +- placement status: `panel`, `pip`, `detached`, plus a redacted reason; +- preview-mode command: `capturing`, `rendering`, or `stopped`; +- typed preview-frame event: session/run/sequence, dimensions, MIME type, and bounded binary data; +- PiP commands: dismiss current run and open the current page in panel. Drag position remains local. + +`runId` must be added to the internal tool execution context and passed through the Agent tool +manager into `YoBrowserToolHandler`. Do not derive loop ownership from session `working` status or +from a timeout: a session can wait for permission, queue another turn, or be superseded. + +## Security and Isolation + +- Remote page execution remains in its existing sandbox/session. PiP receives only an inert, + downscaled image frame and never remote DOM or script. +- PiP chrome loads only a packaged local route with no remote navigation, no Node integration, and a + narrow validated IPC surface. +- Remote pages cannot receive PiP input, drag the PiP, close it, activate the panel, spoof Agent + ownership, or send PiP commands. +- Preview frames remain memory-only, are bounded in dimensions and rate, and are never logged, + cached, or written to disk. +- Device-emulation or zoom state is applied only by main-process tab policy; remote pages cannot + change the stored presentation mode. +- Bounds and IDs received from renderers are validated and clamped in main. +- The page's DevTools/CDP attachment remains unchanged during reparenting. +- No page URL, title, or favicon from an inactive session is shown after a session switch. +- The trusted header truncates untrusted title/host text and never interprets it as HTML. + +## Failure Semantics + +- A failed render-host creation or reparent falls back to `detached`, publishes `placement_failed`, + and never leaves the page registered under two parents. +- A capture or decode failure retains the last Canvas frame and retries on the next bounded tick; + it never clears the PiP to a blank frame. +- At most one capture is in flight per page. New ticks are dropped rather than queued. +- A destroyed host window clears placement synchronously and destroys its PiP chrome. +- A stale bounds or session-activation update is ignored by monotonically increasing layout + versions. +- If panel bounds never stabilize, keep the PiP visible and report the handoff failure; do not reload + the page. +- If PiP chrome crashes, detach PiP and keep the Agent page alive for tools. +- If the remote page crashes, show the existing browser error state in the panel/placeholder and let + tool calls return the current page error. +- Loop-finalization cleanup is idempotent and safe when the page or window has already closed. + +## Feasibility Assessment + +| Requirement | Feasibility | Evidence / risk | +| --- | --- | --- | +| Same page in render host and panel by movement | High | Electron View supports child-view reparenting; serialize parent changes | +| Read-only in-chat mirror and dragging | High | Canvas stays in the trusted renderer and cannot target remote content | +| Preserve page/CDP/session state | High | Reuse the same `WebContentsView`; no navigation or recreation | +| macOS transparent render host capture | High | Proven locally against Electron 40.10.5 with 1280 x 800 animated content | +| Windows/Linux transparent render host capture | Medium | Requires packaged compositor/window-manager validation | +| Bounded frame cost | Medium-high | 480 x 300 output, adaptive 1-4 FPS, one in-flight capture, stop when ineligible | +| Agent-only eligibility | High | Requires `runId` propagation and explicit tab ownership | +| Hide exactly at loop terminal state | High | Runtime owns `LoopRun.runId`; add an explicit finalization port | +| Multiple browser tabs | Medium-high | Current single-page status/contracts and panel must be migrated | +| Responsive DeepChat chrome | High | Container-width layout can reuse current panel ownership and existing UI primitives | +| Responsive website rendering | High for native Responsive mode | Current native bounds already drive the page viewport | +| Fit desktop mode | Medium | Must prove emulation/zoom input and Agent screenshot coordinate fidelity | +| Expanded Browser surface | High | Reuse the existing side-panel fullscreen shell behavior | +| Existing activity overlay in nested PiP | Medium | Separate transparent window may need a local-view fallback after cross-platform QA | +| Identical cross-platform window behavior | High with in-window View | Avoids child `BrowserWindow` platform differences | + +The largest implementation risk is not rendering the card. It is lifecycle correctness across +Agent finalization, session switching, panel animation, window focus, tab closing, and concurrent +bounds updates. The placement invariant and run ID are mandatory, not optional polish. + +## Acceptance Criteria + +- An Agent browser tool never opens a closed right-side panel. +- A user-opened page never appears in PiP. +- The first browser tool in an active loop creates/reuses an Agent tab and associates the exact + `runId`. +- If the Browser surface is visible, the Agent tab is shown there with no PiP. +- If any right-side panel surface is open when an Agent browser action starts, DeepChat activates + Browser and shows the Agent tab there instead of opening PiP. +- Switching back to Workspace is respected until the next Agent browser action; DeepChat does not + continuously force Browser active. +- If all PiP predicate conditions hold, exactly one PiP appears inside the active conversation. +- PiP is read-only: clicking, dragging, scrolling, typing, or focusing it never affects the page. +- The live remote page exists in only one native placement; the Canvas contains only its last + completed image frame. +- While mirrored, page layout reports a stable 1280 x 800 CSS viewport independent of PiP size. +- Moving PiP to the panel preserves URL, DOM state, scroll, focusable page state, cookies, and CDP + target identity without reload. +- Closing PiP suppresses it for the rest of that run without closing the page or interrupting the + Agent. +- A later Agent run may show the retained Agent tab again after its first browser action. +- Completing, failing, cancelling, or superseding the loop hides PiP immediately. +- Session deactivation and real BrowserWindow blur/hide/minimize hide PiP; internal focus movement + between chat and page WebContents does not hide or flash it. +- New frames replace Canvas pixels only after decode, so capture and handoff never create a blank + intermediate frame. +- Dragging stays inside current chat bounds through resize and scale changes. +- Browser tabs and controls remain usable at 420 px panel width without clipping the address bar or + hiding commands from keyboard users. +- Responsive mode exposes the real content bounds at 100% page zoom. +- Fit desktop is never selected silently and passes user-input/CDP/screenshot coordinate tests + before release. +- Expand/restore preserves the exact tab and page state without reload. +- The YoBrowser user-agent Chromium version matches the bundled runtime rather than a stale literal. +- Panel tabs distinguish user and Agent pages and closing a tab destroys only that page. +- Tool operations remain correct while the page moves between placements. +- Cross-platform focus, input, z-order, overlay, and accessibility tests pass. +- Format, i18n, lint, typecheck, and focused tests pass after implementation. + +## Decision Gates + +1. **What does PiP close mean?** Recommendation: dismiss presentation for the current run only; it + must not close the tab or disrupt automation. +2. **Agent tab retention**: recommendation is one retained Agent tab per session, destroyed when the + session/browser workspace is explicitly destroyed or by a later memory-pressure policy. +3. **Run pause behavior**: recommendation is to keep PiP eligible while waiting for permission or a + question, because the loop has not ended; the user can dismiss it. +4. **Page adaptation**: recommendation is automatic responsive chrome plus Responsive page mode by + default, with explicit Fit desktop and Expand actions. Silent overflow-based scaling is too + unpredictable for arbitrary sites and Agent coordinates. +5. **Fit implementation**: choose device emulation versus controlled zoom only after the coordinate + fidelity proof; no API is approved by this spec yet. +6. **Exact styling**: requires the referenced screenshot or a new visual decision before UI polish. + +Confirmed product decisions: + +- an already open right-side panel automatically switches from Workspace to Browser for each new + Agent browser action; +- an undersized conversation shows a compact Agent-browser activity strip instead of forcing an + unusably small webpage. + +## Verified References + +- [Electron View API](https://www.electronjs.org/docs/latest/api/view) +- [Electron WebContentsView API](https://www.electronjs.org/docs/latest/api/web-contents-view) +- [Electron BaseWindow child-window behavior](https://www.electronjs.org/docs/latest/api/base-window) +- [Electron webContents zoom and device-emulation APIs](https://www.electronjs.org/docs/latest/api/web-contents) +- [Electron 40.10.5 runtime versions](https://releases.electronjs.org/release/v40.10.5) +- [Chrome DevTools Protocol Input coordinates](https://chromedevtools.github.io/devtools-protocol/tot/Input/) diff --git a/docs/features/agent-browser-pip/tasks.md b/docs/features/agent-browser-pip/tasks.md new file mode 100644 index 0000000000..29fa302343 --- /dev/null +++ b/docs/features/agent-browser-pip/tasks.md @@ -0,0 +1,121 @@ +# Agent Browser Surfaces and Picture-in-Picture Tasks + +## Status + +V1 read-only mirror revision implemented. Visible browser multi-tabs, Fit-desktop emulation, and +packaged cross-platform validation remain open. + +## Completed Discovery + +- [x] Trace current YoBrowser page, native view, CDP, overlay, panel, and session lifecycles. +- [x] Identify the unconditional Agent `browser.open.requested` panel-opening behavior. +- [x] Confirm the current model is one page per session with no user/Agent ownership. +- [x] Locate stable `LoopRun.runId` ownership and the missing tool-context propagation. +- [x] Verify Electron View/WebContentsView reparenting and child-window platform constraints. +- [x] Define the single-parent invariant, eligibility predicate, interaction matrix, and failure + behavior. +- [x] Diagnose the current 420-520 px panel constraints, fixed Browser toolbar, native viewport + behavior, and stale hard-coded Chromium user-agent version. +- [x] Write the proposal, implementation plan, feasibility assessment, and acceptance criteria. + +## Product Decisions Required Before Implementation + +- [x] Automatically switch an already open Workspace surface to Browser for each new Agent browser + action, without pinning Browser afterward. +- [x] Confirm PiP **Close** dismisses only the current run and never closes the Agent page. +- [x] Confirm one primary Agent page is retained per session and reused across loops. +- [x] Confirm PiP remains eligible while the renderer session remains in the working state. +- [x] Use a compact Agent-browser activity strip instead of a webpage below the usable PiP size. +- [ ] Confirm Responsive default, explicit Fit desktop, and Browser Expand as the page-adaptation + model. +- [x] Confirm undersized chats use a compact Agent activity strip instead of an unusable page card. +- [x] Confirm PiP is a low-frame-rate read-only mirror with a 1280 x 800 background viewport. +- [ ] Supply the referenced screenshot or approve a visual baseline for radius, shadow, toolbar, + sizing, and placement. + +## Phase 1: Run Identity and Workspace Foundation + +- [x] Add `runId` to internal tool execution options. +- [ ] Thread `runId` through immediate, batch, resumed, and deferred tool paths. +- [x] Pass `runId` through `AgentToolManager` into `YoBrowserToolHandler`. +- [ ] Add an idempotent browser run-finalization port for every terminal outcome. +- [ ] Replace per-session single-page state with workspace/tab state. +- [x] Add explicit user/Agent page ownership and last-touched run identity. +- [ ] Reuse one primary Agent tab per session and define popup inheritance. +- [x] Expand shared browser status/events for ownership and run identity. +- [x] Add runtime tests for ownership and run-sensitive placement. + +## Phase 2: Placement Ownership and Panel Tabs + +- [ ] Add a versioned renderer layout report for conversation and panel bounds. +- [ ] Validate layout sender, host window, and active session in main. +- [ ] Add one serialized placement controller per workspace. +- [ ] Enforce panel/PiP/detached single-parent invariants. +- [ ] Remove direct competing attach/detach ownership from `BrowserPanel.vue`. +- [ ] Add browser tab create/select/close operations. +- [ ] Add a compact panel tab strip and Agent marker using existing UI primitives. +- [ ] Add container-responsive wide, compact, and narrow Browser chrome. +- [ ] Keep the address bar usable and every hidden command keyboard-accessible at 420 px. +- [x] Generalize the existing Workspace fullscreen shell behavior into Browser Expand/Restore. +- [x] Derive the YoBrowser desktop user-agent Chromium version from the runtime. +- [x] Route user navigation and Agent tools through explicit page ownership. +- [x] Stop destroying a completed Agent page merely because the session becomes inactive. +- [ ] Remove the compatibility single-page status/routes after all callers migrate. +- [ ] Add main/renderer tests for tab and placement races. + +## Phase 3: Page Presentation, PiP Surface, and Interaction + +- [ ] Build representative responsive/fixed-width website fixtures. +- [ ] Compare device emulation and controlled zoom for per-tab Fit desktop. +- [ ] Verify user input, CDP input, DOM, screenshot, iframe, fixed/sticky, and reset coordinates. +- [ ] Ship Fit desktop only if one path passes the complete coordinate proof. +- [ ] Add per-tab Responsive/Fit presentation state without restart persistence. +- [x] Replace the native-page PiP with a lazy Canvas mirror. +- [x] Add the focusless render host and explicit `capturing`/`rendering`/`stopped` modes. +- [x] Add bounded capture, resize, encoding, and typed in-memory frame delivery. +- [x] Keep preview chrome in the trusted chat renderer and expose only typed mode/frame contracts. +- [x] Render sanitized title, Agent activity, **Open in panel**, and **Close**. +- [x] Implement the V1 eligibility predicate. +- [x] Remove the unconditional Agent-triggered `sidepanelStore.openBrowser()` behavior. +- [x] Implement default responsive bounds and small-region fallback. +- [x] Implement compact Agent activity-strip fallback below usable page size. +- [x] Make every non-button PiP point draggable and distinguish click from drag by threshold. +- [x] Toggle the close/expand toolbar on a non-drag click without forwarding page input. +- [x] Implement current-run dismissal without page/tool interruption. +- [x] Implement panel handoff with no reload or duplicate display. +- [x] Move the Agent page to the 1280 x 800 render host when Browser hides during an active run. +- [x] Keep panel activity overlay behavior and render the PiP activity halo in trusted Canvas chrome. +- [x] Add accessible labels, focus behavior, and i18n strings. +- [x] Add focused tests for mirror eligibility, frame retention, input isolation, dismissal, drag, + focus, and panel handoff. + +## Phase 4: Lifecycle and Cross-Platform Validation + +- [ ] Test loop fail, abort, supersede, and teardown cleanup. +- [x] Test loop completion cleanup. +- [ ] Test permission/question pause behavior. +- [ ] Test session switch, route change, multi-window activation, and stale events. +- [x] Test focus/blur mode transitions and host-focus isolation. +- [ ] Test show/hide, minimize/restore, resize, maximize, and display scale. +- [ ] Test tab close and page/chrome/host crash recovery. +- [ ] Test concurrent user commands and Agent tool activity. +- [ ] Test page input, chat focus, native z-order, and activity overlay on macOS. +- [ ] Test Browser chrome at every container breakpoint and at non-integer display scales. +- [ ] Test Responsive, Fit desktop if approved, Expand, and Restore without page reload/state loss. +- [ ] Repeat packaged focus/z-order/drag tests on Windows and Linux. +- [ ] Verify the exact page WebContents/CDP identity survives every placement move. +- [ ] Verify no inactive-session content appears, including transient frames. +- [ ] Profile retained Agent-tab memory before considering eviction. +- [x] Run `pnpm run format`. +- [x] Run `pnpm run i18n`. +- [x] Run `pnpm run lint`. +- [x] Run typecheck, build, focused suites, and the full main/renderer test suite. +- [ ] Capture BEFORE/AFTER screenshots or GIFs and include the ASCII layouts in the PR. + +## Deferred Work + +- [ ] Persist PiP position/size only if users demonstrate a need. +- [ ] Add a measured idle Agent-tab eviction policy only if memory profiling requires it. +- [ ] Add keyboard PiP movement if accessibility review requires it. +- [ ] Generalize Agent tools to explicit tab IDs only when multi-tab automation is requested. +- [ ] Share architecture with standalone browser windows only after a separate convergence design. diff --git a/docs/features/browser-profile-import/plan.md b/docs/features/browser-profile-import/plan.md new file mode 100644 index 0000000000..7a83753990 --- /dev/null +++ b/docs/features/browser-profile-import/plan.md @@ -0,0 +1,362 @@ +# Browser Website Data and Session Import Implementation Plan + +## Status + +V1 implemented. This document remains the design record; unchecked tasks are validation or later +data-category work rather than claims about the shipped cookie importer. + +The smallest credible first release is explicit, one-way Chrome website-session import backed by +cookies on one platform whose supported decryption path passes packaged fixtures. Passwords, +tab-bound `sessionStorage`, Windows App-Bound data, and generic profile cloning are separate from +that milestone. + +## Recommended Delivery Shape + +```text +Settings renderer + | + | typed routes: scan -> preview -> apply -> status + v +BrowserProfileImportPresenter (main process) + | + +-- source discovery + +-- Chromium-profile cookie reader + +-- platform key access + +-- staging and validation + +-- target mutation lock + +-- CDP target writer and verifier + +-- redacted progress/events + | + v +persist:yo-browser Electron session +``` + +Keep this in the main process. Source paths, encryption keys, decrypted values, rollback data, and +CDP cookie parameters must never cross into a renderer. + +## Architecture + +### 1. One presenter, explicit source readers + +Add a `BrowserProfileImportPresenter` owned by the desktop presenter layer. It coordinates a single +import operation and exposes only discovery, preview, apply, cancel-before-mutation, and last-result +operations. + +Do not start with a plugin framework. Use a small internal reader contract and concrete readers for +only the combinations that have passed fixtures, for example: + +```ts +type BrowserProfileSourceReader = { + discover(): Promise + previewCookies(profile: DetectedBrowserProfile): Promise +} +``` + +The reader returns normalized in-memory records, not source-specific database rows. Chrome and Arc +must not share a reader merely because both are Chromium-derived; shared decoding helpers are fine +after their fixtures prove identical behavior. + +### 2. Typed renderer boundary + +Add browser-data import routes and events under the existing shared contract structure. Proposed +operations: + +- `browserDataImport.scan` +- `browserDataImport.preview` +- `browserDataImport.apply` +- `browserDataImport.cancel` +- `browserDataImport.getStatus` + +Renderer-visible payloads contain IDs, display names, capabilities, counts, progress, and stable +redacted error codes. They never contain profile secrets, cookie names, cookie domains beyond an +explicit aggregate preview decision, values, encryption metadata, keychain labels, or raw paths. + +Use an opaque scan token to bind preview/apply to the exact discovered profile. Resolve the source +path again in main and reject a token after the source metadata changes. + +### 3. Source discovery + +Implement deterministic known-location discovery per browser and OS. For every candidate: + +1. Canonicalize the user-data and profile paths. +2. Require the expected metadata and category files. +3. Read profile names from `Local State` when valid. +4. Fall back to validated `Default` / `Profile N` directories. +5. Detect a running source through lock files and process-independent file-open behavior; never kill + the browser. +6. Compute category capability from browser, version, platform, key access, and source state. + +No recursive home-directory scan is needed. + +### 4. Consistent snapshots + +Use a private temporary directory with mode `0700`. + +- For SQLite, use the existing `better-sqlite3-multiple-ciphers` backup support where the source can + be opened consistently, or copy the database plus WAL/SHM only after proving consistency. +- Validate `PRAGMA integrity_check`, schema version, required columns, and row decoding on the + snapshot. +- For a future LevelDB category, require the source browser to be closed and copy the complete + storage directory except transient lock files before opening the snapshot. +- Register startup cleanup for abandoned DeepChat import directories. + +The operation must not mutate YoBrowser until the entire selected source category is staged and +validated. + +### 5. Platform key access + +Build only the platform path selected for V1. + +For a macOS Chrome proof, trigger the system-owned Keychain authorization surface and retrieve the +exact source browser's Safe Storage secret through a main-process-only OS adapter. Derive the +Chromium key according to the source version and decrypt the staged `v10` records. The proof may use +the system `security` executable to avoid a new native dependency, provided arguments and output are +never logged, DeepChat never receives the password typed into the OS prompt, and denial is handled +normally. A production implementation should keep that choice only if authorization behavior, +signing, and sandboxed packaging tests pass. + +Do not implement Windows Chrome/Arc App-Bound decryption in V1. User account or UAC authorization +does not give DeepChat Chrome's application identity. Do not add a browser extension, companion +service, `SYSTEM` process, process injection, memory scan, reverse-engineered +`elevation_service.exe` key, or remote-debugging workaround. Report the source/category as +unsupported before preview mutates anything. + +Use Chromium as a reference implementation, not as a library dependency. The minimal reader needs +only deterministic profile discovery, a consistent SQLite snapshot, explicit encrypted-record +version dispatch, the supported platform key adapter, authenticated record decryption, and cookie +normalization. Do not embed Chromium's `ProfileManager` or copy its profile lifecycle machinery into +DeepChat. + +### 6. Cookie normalization and CDP application + +Normalize source rows into a target-independent cookie record that retains: + +- name and value; +- domain and host-only semantics; +- path; +- expiry or session-only state; +- Secure, HttpOnly, SameSite, and prefix constraints; +- priority, source scheme, and source port; +- partition key and cross-site ancestor state where available. + +Version the mapper by the source schema and target Electron Chromium/CDP major versions. Unknown +enum values or unsupported partition metadata block preview rather than being silently discarded. + +The target operation is transactional at the application level: + +```text +acquire target lock + -> snapshot current target cookies + -> clear target cookies + -> apply staged source cookies in bounded batches + -> read back and normalize + -> compare identities and values + -> success: flush + release lock + reload tabs + -> failure: clear + restore target snapshot + verify + release lock +``` + +Use CDP `Storage`/`Network` cookie commands against `persist:yo-browser`; do not use Electron's +public Cookies API as the sole writer because it cannot express the full partitioned-cookie shape. + +### 7. Mutation ownership + +Add one target-data mutation coordinator shared by import and the existing clear-sandbox action. +While it is held: + +- a second import or clear request is rejected as `target_busy`; +- source preview may continue because it is read-only; +- YoBrowser navigation may remain visible, but the final clear/apply window should be short; +- affected tabs reload only after verified success; +- application shutdown waits for rollback/apply completion within a bounded deadline, then records + recovery-required state if safe completion is impossible. + +### 8. Settings UX + +Extend the existing YoBrowser sandbox card in `DataSettings.vue`. Reuse shadcn-vue Dialog, Button, +Checkbox/Field, Alert, Badge, Progress/Spinner, and existing settings patterns. Use i18n keys for all +visible copy. + +The renderer drives a strict state machine from main-process status. It does not infer readiness +from counts and cannot enable apply before a successful preview token is returned. + +The confirmation screen must say that selected target categories are replaced, that the source is +not modified, and that an OS/source-browser authorization prompt may appear. The result screen +reports imported, skipped, expired, unsupported, and failed counts separately. It reports data +transfer success, not a guarantee that every site server will accept the session. + +## Feasibility Proofs Before Product Work + +### Proof A: macOS Chrome cookie round trip + +Create a sanitized Chrome fixture containing persistent, session, HttpOnly, SameSite variants, +prefix-constrained, and partitioned cookies. Prove: + +- snapshot succeeds with Chrome open and closed; +- system-owned key authorization is understandable and denial is recoverable; +- every supported row decrypts; +- CDP writes and reads back normalized values; +- session cookies remain non-persistent; +- target rollback restores its original cookies byte-for-value where public fields permit. + +This is high feasibility based on the current local Chrome schema, but it is not complete until a +packaged, signed application passes the test. + +### Proof B: Arc source compatibility + +Collect sanitized Arc fixtures on macOS. Verify user-data paths, profile metadata, +cookie schema, keychain identifiers, encryption versions, and running-browser snapshot behavior. +Until this passes, Arc remains experimental rather than inheriting Chrome support. Do not spend +fixture effort on Windows while that platform is outside the release scope. + +### Proof C: partitioned-cookie fidelity + +Test the exact Electron-bundled CDP version with partitioned cookies. If `partitionKey` and +cross-site ancestor state cannot round-trip, block that profile preview or explicitly exclude those +records with user-visible counts; never flatten them into unpartitioned cookies. + +### Proof D: `localStorage` inactive-origin write + +Only if `localStorage` is still desired for the same milestone, prove CDP can clear, write, and read +back multiple inactive origins without navigating real remote pages. Failure moves the category to +a later release. + +## Implementation Phases + +### Phase 0: decisions and fixtures + +- Choose the first directly supported V1 platform and its OS-owned authorization surface. +- Freeze V1 categories. +- Collect legally safe, sanitized fixtures for every supported browser/OS/version combination. +- Record source and target Chromium version support ranges. + +Exit criterion: the selected path has passing feasibility proofs and no unsupported category is +presented as importable. + +### Phase 1: main-process cookie core + +- Add discovery, source snapshot, schema decoder, platform key access, cookie normalization, and + redacted errors. +- Add target lock, CDP snapshot/apply/readback, rollback, and recovery cleanup. +- Add focused unit and fixture tests without renderer work. + +Exit criterion: a headless integration test can repeat source-authoritative cookie sync and can +recover from injected apply failures. + +### Phase 2: typed contracts and settings UI + +- Add routes, events, preload/browser client calls, and status types. +- Add scan, preview, confirmation, progress, result, and clear-conflict UX. +- Add accessibility, keyboard, i18n, and settings tests. + +Exit criterion: a user can understand platform/category limitations before any target mutation. + +### Phase 3: packaged platform validation + +- Validate notarized macOS builds and Windows/Linux packages selected for support. +- Test OS key prompts, source-browser open/closed states, multi-profile discovery, rollback on crash + injection, and app restart cleanup. +- Document the support matrix and known non-transferable login systems. + +Exit criterion: release checks pass on every advertised platform. + +### Later phases, only if separately approved + +- `localStorage` import after Proof D. +- Live-tab `sessionStorage` handoff. +- A separate credential-vault SDD for passwords. + +## Expected File Areas + +Exact names may change after the feasibility proof, but implementation should remain localized to: + +- `src/main/desktop/browser/import/` for discovery, readers, crypto, normalization, and coordinator; +- `src/main/desktop/browser/yoBrowserSession.ts` for target session access/flush hooks; +- `src/main/desktop/browser/YoBrowserPresenter.ts` for tab reload coordination only; +- `src/shared/contracts/routes/` and `src/shared/contracts/events/` for typed boundaries; +- `src/shared/types/` for redacted public state; +- `src/preload/` for the existing safe route exposure pattern; +- `src/renderer/settings/components/DataSettings.vue` and a focused import dialog component; +- mirrored `test/main/**` and `test/renderer/**` suites plus sanitized fixtures. + +Do not place source-profile parsing in the renderer or generalize the standalone browser tab system +to solve this feature. + +## Test Strategy + +### Unit + +- path discovery and canonicalization; +- schema version/column validation; +- Chromium timestamp and enum mapping; +- cookie identity normalization, prefix rules, and expiry filtering; +- platform decrypt success, denial, corrupt ciphertext, and unknown versions; +- state-machine transitions and redacted error serialization; +- target comparison and rollback planning. + +### Main-process integration + +- repeat import with changed and deleted source cookies; +- source-authoritative clearing of target-only cookies; +- session and persistent cookie behavior across restart; +- CHIPS round trip; +- apply failure at every batch boundary; +- rollback failure and recovery-required state; +- import versus clear lock contention; +- source browser open/closed fixtures; +- no secret values in captured logs/events; +- no authorization password enters a DeepChat renderer, IPC payload, log, or process-owned field. + +### Renderer + +- per-profile/per-category capability rendering; +- preview before confirmation; +- apply disabled for stale/failed previews; +- cancellation only before mutation; +- result and error accessibility; +- no accidental source-path or secret rendering. + +### Packaged smoke tests + +- OS key prompt behavior; +- signing/notarization permissions; +- multiple browser channels and profiles; +- unsupported browser/version messaging; +- removal of temporary snapshots after success, failure, crash, and restart. + +## Rollout and Compatibility + +- Hide the entry point when no supported source exists, but keep an explanation in the settings + card rather than an empty dialog. +- Gate experimental Arc and `localStorage` support independently from Chrome cookies. +- Store only non-secret last-sync metadata: browser ID, opaque profile ID, categories, timestamp, + counts, source version, and importer version. +- Do not automatically re-run old imports after an app upgrade. +- If the target Electron Chromium major changes, rerun CDP fidelity fixtures before keeping support + enabled. + +## Rejected Approaches + +- **Point Electron at Chrome's user-data directory**: risks concurrent profile corruption and mixes + incompatible browser-owned state. +- **Copy the whole profile directory**: brings locks, caches, extensions, keys, versioned formats, + and non-transferable credentials without a safe rollback boundary. +- **Attach remote debugging to the user's normal Chrome profile**: current Chrome restricts this and + it is not a stable import contract. +- **Run a privileged App-Bound decryption proof of concept**: the referenced Chrome 130 script + creates a Windows service, executes under `SYSTEM`, and uses a hard-coded key recovered from + `elevation_service.exe`. This bypasses the protection, depends on browser internals, and is outside + the product's authorization and maintenance contract. +- **Install a companion browser extension**: its implementation, distribution, broad cookie + permissions, and ongoing browser-store policy cost are not justified for a platform fallback. +- **Use only Electron `cookies.set`**: loses modern cookie fields. +- **Make password decryption part of cookie import**: still leaves no target password manager and + creates a much larger secret-handling surface. +- **Add a general background sync daemon in V1**: creates consent, locking, freshness, and failure + semantics before the manual import is proven. + +## Complexity Budget + +V1 should add no new runtime dependency unless the packaged macOS key-access proof shows the system +API cannot be used safely. It should support one browser/platform/category combination completely +before adding adapters. A generic migration framework, scheduler, diff viewer, and bidirectional +sync engine are explicitly out of scope. diff --git a/docs/features/browser-profile-import/spec.md b/docs/features/browser-profile-import/spec.md new file mode 100644 index 0000000000..33c0f1750b --- /dev/null +++ b/docs/features/browser-profile-import/spec.md @@ -0,0 +1,495 @@ +# Browser Website Data and Session Import Spec + +## Status + +V1 implemented on 2026-07-17. The shipped scope is explicit, one-way, source-authoritative +non-partitioned cookie import from macOS Chrome, with Arc exposed as experimental. Windows and +Linux report unsupported. Passwords, `localStorage`, `sessionStorage`, and partitioned cookies +remain deferred. + +Last verified against the repository and upstream documentation on 2026-07-17. + +## Problem + +YoBrowser currently uses one independent persistent Electron session, +`persist:yo-browser`. The right-side browser panel and standalone DeepChat browser windows share +that session, but it does not share authentication state with Chrome, Arc, or another installed +Chromium browser. + +Users who are already signed in elsewhere therefore have to sign in again. Clearing the YoBrowser +sandbox is supported, but importing or refreshing source-browser state is not. + +The requested product behavior is a repeatable, user-initiated, one-way import where a selected +Chromium profile is the source of truth and supported website data replaces the corresponding +YoBrowser data. The intended outcome is practical session portability: after import, YoBrowser +should open supported sites with the same transferable signed-in state and site preferences as the +selected source profile. + +## Product Contract + +This is not a whole-profile clone. It is a website-data import whose success is measured by useful +session continuity. + +- Cookies can be imported with useful fidelity, subject to operating-system encryption and newer + device-bound session protections. +- `localStorage` can potentially be imported, but Chromium's on-disk format is internal and must + pass a compatibility proof before it becomes a supported category. +- `sessionStorage` belongs to a specific top-level browsing context. Copying a profile directory + does not attach a Chrome tab's namespace to an Electron tab. +- Saved passwords are not required to carry an already authenticated website session. Electron + exposes encrypted application storage but no public API for importing records into a Chromium + password manager. Making imported passwords useful would require a DeepChat credential vault and + explicit autofill UX. +- Passkeys, Device Bound Session Credentials, client certificates, and hardware-bound tokens are + intentionally non-transferable or may remain unusable after a cookie copy. + +The product may say "website data imported" or "session data synced" when the selected categories +were verified. It must not claim that every site is signed in: expired server sessions, +device-bound credentials, and site-side risk checks can still request authentication. + +## Goals + +- Discover supported installed browsers and their local profiles without modifying the source. +- Let the user select one source browser/profile and preview what can actually be imported. +- Support repeated, explicit, one-way synchronization. +- Make the source authoritative for each selected and supported category. +- Optimize for the user-visible outcome that supported sites retain their transferable signed-in + session after the target tabs reload. +- Preserve modern cookie attributes where the target Chromium/CDP version supports them. +- Stage and validate all source data before mutating YoBrowser. +- Roll back the target category when an apply step fails. +- Report imported, skipped, expired, unsupported, and failed records without exposing secret values. +- Keep all processing local to the device. + +## Non-Goals + +- Real-time or bidirectional synchronization. +- Writing anything back to Chrome, Arc, or another source browser. +- Copying an entire live Chromium profile directory into Electron. +- Circumventing Chrome App-Bound Encryption, operating-system authorization, enterprise policy, or + device-bound authentication. Authorization must use an OS-owned or source-browser-owned prompt. +- Asking the user to type an operating-system, Chrome, Arc, or keychain password into a DeepChat + renderer form. +- Claiming every Chromium-derived browser is supported by one generic adapter. +- Importing browsing history, bookmarks, downloads, the user's installed extensions, payment cards, + passkeys, client certificates, or browser settings in the first release. +- Giving an Agent access to raw saved passwords. +- Installing a browser extension or companion service to extract source data. +- Creating a Windows service, running code as `SYSTEM`, injecting into a source-browser process, + scraping process memory, or using reverse-engineered browser keys to defeat App-Bound Encryption. + +## Terminology + +- **Source browser**: An installed Chrome, Arc, or later explicitly supported Chromium browser. +- **Source profile**: One profile directory selected by the user. +- **Target session**: DeepChat's shared `persist:yo-browser` Electron session. +- **Quick sync**: A cookie-only import that may be possible while the source browser is running. +- **Full storage sync**: Any import that includes Chromium storage directories or LevelDB data and + may require the source browser to be closed. +- **Source authoritative**: Existing target data in the selected category and scope is removed + before validated source data is applied. +- **Session portability**: Transferable client-side website state is present in YoBrowser after + import; it does not imply that the website's server accepts every copied session indefinitely. +- **Direct importer**: A small main-process reader that discovers a supported Chromium profile, + snapshots only the selected data stores, uses the supported OS/browser cryptography path, and + normalizes records before applying them to YoBrowser. It never loads the source profile as a + DeepChat browser profile. + +## Current Repository Contract + +- `src/main/desktop/browser/yoBrowserSession.ts` owns the persistent target session. +- `src/main/desktop/browser/YoBrowserPresenter.ts` creates Agent browser views with that session. +- `src/main/desktop/tab.ts` also assigns that session to standalone DeepChat browser windows. +- `src/renderer/settings/components/DataSettings.vue` currently exposes only a destructive clear + action for the YoBrowser sandbox. +- `better-sqlite3-multiple-ciphers` already provides read-only SQLite access and an online backup + API. +- `level` is already installed and is used by the existing provider-import code for LevelDB + snapshots. + +The import target is therefore global to YoBrowser, not per chat session. + +## Source Discovery + +Discovery must be adapter-driven and read-only. + +Each detected source must include: + +- browser ID, display name, channel, and executable presence; +- user-data directory and profile directory; +- profile display name when safely available; +- last modified time; +- whether the source appears to be running; +- per-category capability and the reason for any unsupported state. + +Initial discovery scope: + +| Browser | macOS | Windows | Linux | Notes | +| --- | --- | --- | --- | --- | +| Google Chrome stable | Required | Unsupported in V1 | Feasibility gate | Direct cookie support varies by OS | +| Chromium | Optional after Chrome | Unsupported in V1 | Optional after Chrome | Separate key-store identity | +| Arc | Experimental in V1 | Unsupported in V1 | Not available | Uses an explicit Arc source identity | +| Edge / Brave | Future | Future | Future | Do not imply support from Chromium ancestry alone | + +Chromium documents that profiles are subdirectories of the user-data directory, commonly +`Default` or `Profile N`. Profile metadata may be read from `Local State`, but discovery must fall +back to validated directory inspection when metadata is missing. + +[GUESS] Arc's storage and keychain identifiers remain compatible enough with Chromium for a shared +reader. This must be proven with sanitized Arc fixtures on every supported OS before Arc is labeled +supported. Arc's public documentation confirms that profiles contain cookies, logins, passwords, +history, and extensions, but it does not define a stable on-disk contract. + +## Capability Matrix + +### Data categories + +| Category | Direct profile read | Target apply | Proposed release status | +| --- | --- | --- | --- | +| Cookies | Platform-dependent | High via CDP | V1 candidate | +| Partitioned cookies (CHIPS) | Platform-dependent | Medium-high via CDP | V1 only after protocol tests | +| `localStorage` | Medium, internal LevelDB schema | Medium | Feasibility gate | +| `sessionStorage` | Low without source tab mapping | Medium per target tab | Later live-tab handoff research | +| IndexedDB | Low-medium, internal schema and blobs | Low-medium | Not V1 | +| Service workers / Cache Storage | Low value and high compatibility risk | Medium | Not V1 | +| Saved passwords | Platform-dependent decryption | No Electron password-manager import API | Separate product decision | +| Passkeys / DBSC / client keys | Intentionally restricted | Not transferable | Unsupported | + +### Operating-system constraints + +| Platform | Direct cookie feasibility | Main constraint | Recommended path | +| --- | --- | --- | --- | +| macOS | High for Chrome after proof | Source Keychain item and system-owned user authorization | Direct reader first | +| Windows | Unsupported in V1 | Chrome App-Bound Encryption binds cookies to Chrome identity | No extension and no bypass; show unsupported state | +| Linux | Medium | Secret Service/KWallet availability and browser-specific key identity | Direct reader only after desktop-matrix proof | + +Chrome introduced App-Bound Encryption for Windows cookies in Chrome 127. Entering the Windows +account password does not grant an unrelated application Chrome's app identity. DeepChat must not +use process injection, memory scraping, `SYSTEM` elevation, a reverse-engineered browser key, or +another bypass. The product decision is to omit Windows support instead of building a companion +extension or privileged helper. Chrome 136 also rejects remote-debugging switches against the +default user-data directory, so "attach CDP to the user's current Chrome profile" is not an +acceptable general solution. + +The direct importer should reuse Chromium's documented profile layout, source schemas, timestamp +conventions, and OS cryptography behavior where those form a supportable contract. Chromium's +`ProfileManager` is useful as a reference for discovering and selecting profiles owned by Chromium, +but it is not a cross-application migration or decryption API. + +The linked `thewh1teagle` Windows proof of concept is not an acceptable implementation path. It was +tested against Chrome 130, creates a local Windows service through `pypsexec`, executes part of the +unwrap under `SYSTEM`, and uses a hard-coded AES key recovered from `elevation_service.exe`. That is +an App-Bound Encryption bypass built from version-specific internals, not an OS-owned authorization +flow. Its useful input is limited to the neutral importer mechanics: locate `Local State`, recognize +encrypted-record versions, snapshot the SQLite cookie store, authenticate AES-GCM records, and +handle the 32-byte cookie plaintext prefix for schema versions where fixtures prove it. + +## Authorization Contract + +Import is always initiated by the user. Authorization is explicit and may include Touch ID, +Windows Hello, an operating-system account password, or a keychain/keyring prompt. + +The authorization surface must be owned by the operating system or source browser whenever it +protects source secrets. DeepChat may trigger that surface and explain why it appears, but it must +not render a look-alike password prompt, receive the password, store it, or pass it through IPC. + +Authorization does not change the support matrix: + +- on macOS/Linux, successful OS key access may unlock a supported direct reader; +- on Windows, current Chrome App-Bound Encryption remains unsupported in V1 even if the user can + approve a UAC or account prompt, because authorization does not grant Chrome's application + identity; +- denial or cancellation returns to preview without changing YoBrowser; +- the user authorizes each import operation in V1; no reusable background authorization token is + retained. + +## Cookie Import Contract + +Cookies are the recommended first category because they produce most of the requested +signed-in-session benefit without creating a password manager. + +Before target mutation, the importer must: + +1. Create a consistent source SQLite snapshot. +2. Validate the `meta.version` and required columns. +3. Decrypt every non-expired source cookie selected for import. +4. Map Chromium time values and enum fields to CDP cookie parameters. +5. Validate domain, path, prefix, SameSite, Secure, HttpOnly, source scheme, source port, priority, + and partition-key data. +6. Abort before target mutation if any required encrypted record cannot be handled. + +V1 applies only non-partitioned cookies through Electron's public `Session.cookies` API, then reads +them back and verifies normalized identity/value pairs. Partitioned cookies are counted and skipped +because the public shape does not expose the required partition key. A future CHIPS-capable release +must move that category to CDP `Storage.getCookies`, `Storage.clearCookies`, and +`Storage.setCookies` with protocol fixtures. + +Source-authoritative cookie sync means: + +- snapshot all target cookies for rollback; +- clear the target cookie store; +- set the complete validated source set in bounded batches; +- read back the target set and compare normalized cookie identities and values; +- restore the target snapshot if apply or verification fails; +- flush the cookie store; +- reload open YoBrowser tabs after success. + +Cookie creation and last-access timestamps cannot be set through the public CDP cookie parameter +shape. The UI must not call this a byte-identical copy. + +Session cookies remain session cookies. DeepChat must not invent an expiration date to persist them. +They may need to be imported again after DeepChat restarts. + +## Local Storage Contract + +`localStorage` is origin-scoped persistent data, but Chromium stores it in an internal LevelDB +format. It can become supported only if a feasibility proof demonstrates all of the following: + +- consistent source snapshots while the source browser is closed; +- correct decoding of Chromium storage keys and string encodings; +- correct handling of storage keys and partitioning in the target Electron Chromium version; +- authoritative target clearing; +- target writes without loading arbitrary remote pages or triggering site side effects; +- round-trip verification on fixtures generated by the exact Chrome and Electron major versions in + the support matrix. + +The preferred live target path is CDP `DOMStorage.clear` plus `DOMStorage.setDOMStorageItem` against +an importer target. If CDP cannot write an inactive origin reliably, V1 must defer `localStorage` +rather than copy target LevelDB files while Electron owns them. + +## Session Storage Contract + +`sessionStorage` is partitioned by origin and top-level browsing context. A Chrome profile's +`Session Storage` LevelDB contains namespaces that refer to Chrome tabs. A DeepChat tab does not +inherit those namespace identities merely because files are copied. + +Therefore: + +- offline profile-level `sessionStorage` import is not a V1 commitment; +- a later user-initiated open-tab handoff may be considered only if a supported source-browser API + can export the current tab's URL and `sessionStorage` without an extension or encryption bypass; +- the UI must describe this as "open tab handoff", not profile synchronization; +- closing either source or target tab ends the relevant page session as normal. + +## Password Contract + +Saved passwords require a separate implementation track, but they can remain part of the broader +"make YoBrowser useful with existing browser data" roadmap. + +Reading Chrome's `Login Data` does not make Electron autofill those credentials. Electron provides +`safeStorage` for application-owned encrypted strings, but its documented public session APIs do +not provide a password-manager import surface. This is an inference from the Electron 40.10.5 +public API and must be rechecked before implementation. + +The acceptable password direction is an explicit DeepChat credential-vault feature with: + +- user-triggered Chrome/Arc CSV export or another user-mediated export; +- source-browser or operating-system authorization rather than a DeepChat-owned system-password + prompt; +- immediate encrypted ingestion and a prominent plaintext-file warning; +- origin-bound, user-triggered fill; +- no automatic submit; +- no Agent, tool, renderer, log, export, or telemetry access to decrypted credentials; +- OS-backed encryption with a hard failure on insecure Linux `basic_text` storage; +- independent threat modeling, tests, and settings UX. + +That work is not required for session portability and should not be hidden inside cookie-sync V1. + +## User Experience + +### Before + +```text ++--------------------------------------------------------------+ +| YoBrowser Sandbox | +| Independent cookies and local storage. | +| [Clear YoBrowser data] | ++--------------------------------------------------------------+ +``` + +### Proposed settings card + +```text ++--------------------------------------------------------------+ +| YoBrowser data | +| Source: Google Chrome / Personal | +| Last sync: 2026-07-17 14:32 · Cookies: 1,284 | +| [Import or sync] [Clear YoBrowser] | ++--------------------------------------------------------------+ +``` + +### Proposed import dialog + +```text ++------------------------------------------------------------------+ +| Import browser data [x] | +| | +| Browser [Google Chrome v] Profile [Personal v] | +| | +| [x] Cookies Ready | +| [ ] Local storage Experimental · Chrome must be closed | +| [ ] Session storage Not available for profile import | +| [ ] Passwords Requires a separate credential vault | +| | +| Source data replaces the selected YoBrowser categories. | +| Chrome is never modified. | +| System or Chrome authorization may be requested after Preview. | +| [Cancel] [Preview] | ++------------------------------------------------------------------+ +``` + +### Unsupported Windows source + +This state appears when a Windows Chrome or Arc profile uses App-Bound Encryption. + +```text ++------------------------------------------------------------------+ +| Import browser data [x] | +| | +| Windows Chrome protects cookies with Chrome's app identity. | +| This Chrome profile cannot be imported safely by DeepChat. | +| | +| DeepChat will not install an extension, elevate a helper, or | +| bypass Chrome's encryption. | +| | +| [Close] | ++------------------------------------------------------------------+ +``` + +### Interaction flow + +1. The user opens Data & Privacy and selects **Import or sync**. +2. DeepChat scans known locations and shows only validated source profiles. +3. The user selects a source profile. +4. DeepChat displays per-category capability, browser-running requirements, and platform limits. +5. **Preview** performs source snapshot, authorization when required, decryption, validation, and + count calculation without changing YoBrowser. Protected authorization UI is owned by the OS or + source browser, not DeepChat. +6. The confirmation view states exactly which target categories will be replaced. +7. DeepChat pauses conflicting YoBrowser mutations, applies the staged data, verifies, and reloads + affected tabs. +8. The result shows counts and stable error codes, never secret values or source URLs beyond what + the user selected. +9. Running the flow again repeats the same source-authoritative operation. + +There is no schedule or background watcher in V1. + +## Import States + +The renderer may display these stable states: + +- `idle` +- `scanning` +- `previewing` +- `ready` +- `applying` +- `verifying` +- `rolling_back` +- `succeeded` +- `failed` +- `cancelled` + +Cancellation is allowed before target mutation. After mutation starts, the operation must finish or +roll back rather than stop halfway. + +## Security and Privacy Requirements + +- Source paths must be canonicalized and restricted to discovered/explicitly selected profiles. +- Symlinks and path traversal must not escape the selected profile root. +- Source databases and LevelDB files are opened read-only. +- Temporary snapshots use a mode-`0700` directory and are always removed after success, failure, or + startup recovery. +- Decrypted cookie/password values are never logged, serialized to renderer state, included in + exceptions, copied to the clipboard, or written to plaintext staging files. +- Renderer contracts expose counts, capability, progress, and redacted failures only. +- A source Keychain/keyring denial is a normal unsupported/denied result, not a retry loop. +- DeepChat never receives or stores the password entered into an OS/source-browser authorization + surface. +- The source profile is never opened as an Electron `Session`. +- Import is blocked while another import or clear operation owns the target mutation lock. +- Rollback data receives the same secret handling as source data. +- Windows App-Bound sources fail capability detection before any key or cookie value is staged. + +## Failure Semantics + +Stable categories should include: + +- `browser_not_found` +- `profile_not_found` +- `source_browser_busy` +- `source_schema_unsupported` +- `source_snapshot_failed` +- `key_access_denied` +- `encryption_unsupported` +- `record_decryption_failed` +- `target_busy` +- `target_apply_failed` +- `target_verification_failed` +- `target_rollback_failed` +- `category_unsupported` + +If rollback fails, DeepChat must mark the target as requiring an explicit clear or re-import and +must not report success. + +## Acceptance Criteria + +- The settings UI discovers and distinguishes multiple supported source profiles. +- Every source/profile/category combination reports `ready`, `requires_action`, `experimental`, or + `unsupported` with a concrete reason. +- Preview does not mutate the source or target. +- A supported cookie sync can be repeated and produces source-authoritative target cookies. +- After successful import and tab reload, transferable source sessions are available to YoBrowser; + sites with expired or device-bound sessions may still request login without invalidating the + import result. +- Partitioned cookie fields are preserved when the target CDP version supports them; otherwise the + preview blocks rather than silently flattening them. +- No target mutation occurs when source decryption or validation is incomplete. +- A failed target apply restores the previous target cookie set and verifies the rollback. +- Open YoBrowser tabs reload only after a successful apply. +- Chrome/Arc files are never written. +- Windows App-Bound cookies are not decrypted through a bypass. +- Protected source access uses OS/source-browser authorization; DeepChat never collects the + authorization password itself. +- `sessionStorage` and saved-password limitations are visible before confirmation. +- Secret values never cross the main-process contract boundary. +- Focused unit, integration, and fixture tests cover supported schema and platform combinations. +- Format, i18n, lint, typecheck, and relevant tests pass after implementation. + +## Decision Gates + +Implementation should not begin until these product decisions are made: + +1. **Windows Chrome support (decided)**: current Chrome/Arc App-Bound data is unsupported. Do not + build a companion extension, privileged service, process-injection path, or reverse-engineered + decryption path. Revisit only if Windows or the source browser exposes a supported export API. +2. **Delivery sequence**: recommendation is cookie-backed session portability as the first shipping + slice. Add `localStorage` after the CDP arbitrary-origin proof, then current-tab + `sessionStorage` handoff; these remain roadmap capabilities rather than discarded requirements. +3. **Password sequencing**: passwords remain in the roadmap, but recommendation is a separate + credential-vault SDD after transferable session data because they do not contribute to an already + active site session. +4. **Arc support label**: recommendation is experimental until sanitized macOS Arc fixtures prove + profile discovery and key access. Windows Arc remains outside V1 with Windows Chrome. + +Confirmed product requirements: import is user-initiated, source-authoritative, repeatable, and does +not use an encryption bypass. V1 therefore has no startup/background synchronization. + +## Verified References + +- [Chromium user-data directory and profile locations](https://chromium.googlesource.com/chromium/src/+/main/docs/user_data_dir.md) +- [Chromium persistent cookie-store schema and encrypted values](https://chromium.googlesource.com/chromium/src/+/main/net/extras/sqlite/sqlite_persistent_cookie_store.cc) +- [Chromium OS Crypt contract](https://chromium.googlesource.com/chromium/src/+/HEAD/components/os_crypt/sync/README.md) +- [Chromium ProfileManager lifecycle contract](https://chromium.googlesource.com/chromium/src/+/refs/heads/main/chrome/browser/profiles/profile_manager.h) +- [Chrome App-Bound Encryption on Windows](https://security.googleblog.com/2024/07/improving-security-of-chrome-cookies-on.html) +- [Chromium App-Bound Encryption path validation](https://chromium.googlesource.com/chromium/src/+/main/chrome/browser/os_crypt/README.md) +- [Chrome 136 remote-debugging restrictions](https://developer.chrome.com/blog/remote-debugging-port) +- [thewh1teagle Chrome 130 App-Bound proof of concept](https://gist.github.com/thewh1teagle/d0bbc6bc678812e39cba74e1d407e5c7) +- [Chrome DevTools Protocol Storage domain](https://chromedevtools.github.io/devtools-protocol/tot/Storage/) +- [Chrome DevTools Protocol Network cookie parameters](https://chromedevtools.github.io/devtools-protocol/tot/Network/) +- [Chrome DevTools Protocol DOMStorage domain](https://chromedevtools.github.io/devtools-protocol/tot/DOMStorage/) +- [Electron Session API](https://www.electronjs.org/docs/latest/api/session) +- [Electron Cookies API](https://www.electronjs.org/docs/latest/api/cookies) +- [Electron safeStorage API](https://www.electronjs.org/docs/latest/api/safe-storage) +- [HTML Web Storage standard](https://html.spec.whatwg.org/multipage/webstorage.html) +- [Arc profile behavior](https://resources.arc.net/hc/en-us/articles/19227964556183-Profiles-Separate-Work-Personal-Browsing) +- [Chrome password export](https://support.google.com/chrome/answer/95606) diff --git a/docs/features/browser-profile-import/tasks.md b/docs/features/browser-profile-import/tasks.md new file mode 100644 index 0000000000..ea2a19d5ac --- /dev/null +++ b/docs/features/browser-profile-import/tasks.md @@ -0,0 +1,98 @@ +# Browser Website Data and Session Import Tasks + +## Status + +V1 cookie-backed session import implemented. Packaged macOS/Arc fixture validation and deferred +data categories remain open. + +## Completed Discovery + +- [x] Trace the current `persist:yo-browser` session ownership and clear-data flow. +- [x] Confirm that Agent and standalone DeepChat browser views share the target session. +- [x] Inspect the current local Chrome Cookies schema without reading or recording secret values. +- [x] Verify current Electron public Session, Cookies, `safeStorage`, View, and CDP constraints. +- [x] Record Chrome App-Bound Encryption and remote-debugging restrictions. +- [x] Evaluate the referenced Chrome 130 `SYSTEM`/hard-coded-key proof of concept and reject its + App-Bound bypass while retaining only neutral schema and snapshot lessons. +- [x] Separate cookies, `localStorage`, `sessionStorage`, passwords, and non-transferable credentials + into honest capability categories. +- [x] Write the proposal, implementation plan, risks, acceptance criteria, and decision gates. + +## Product Decisions Required Before Implementation + +- [x] Keep current Windows Chrome/Arc App-Bound data unsupported; do not build a companion extension + or privileged/reverse-engineered decryption path. +- [x] Confirm V1 session portability is cookie-backed, subject to feasibility proofs. +- [x] Keep passwords in the roadmap through a separate credential-vault proposal. +- [x] Confirm Arc is labeled experimental until fixture validation passes. +- [x] Keep synchronization explicitly user-triggered, not scheduled/background. +- [x] Require OS/source-browser authorization and prohibit encryption bypasses. +- [x] Decide whether target scope is always the entire global YoBrowser session or may be limited to + selected domains in a later release. + +## Feasibility Gates + +- [ ] Build sanitized Chrome cookie fixtures for each supported source schema. +- [ ] Prove consistent SQLite snapshot behavior with Chrome open and closed. +- [ ] Prove packaged macOS Chrome system-owned authorization, key access, decryption, and denial + handling without DeepChat receiving the authorization password. +- [ ] Prove CDP target set/readback for all supported cookie attributes. +- [ ] Prove partitioned-cookie round trip for the bundled Electron Chromium version. +- [ ] Prove full target-cookie rollback after injected failures. +- [ ] Collect and validate sanitized Arc fixtures on every proposed Arc platform. +- [ ] If `localStorage` remains proposed, prove inactive-origin clear/write/readback before adding it + to the release scope. + +## Phase 1: Main-Process Cookie Core + +- [x] Add deterministic known-location browser/profile discovery. +- [x] Add canonical path and source-profile validation. +- [ ] Add a private snapshot workspace with crash/startup cleanup. +- [x] Add the selected Chrome source schema reader. +- [x] Add the selected OS key-access/decryption adapter. +- [x] Ensure protected authorization is rendered by the OS/source browser, never a DeepChat + password form. +- [x] Add versioned cookie normalization and validation. +- [x] Add stable, redacted error codes. +- [x] Add one target mutation coordinator shared with clear-sandbox data. +- [x] Add target snapshot, clear, batch apply, readback, and normalized comparison for + non-partitioned cookies. +- [x] Add rollback handling. +- [x] Flush the target cookie store and reload open YoBrowser tabs only after verified success. +- [x] Add unit and main-process integration tests for decryption, replacement, verification, and + rollback. + +## Phase 2: Contracts and Settings UX + +- [x] Add shared schemas for scan, preview, and apply. +- [x] Add main routes and client exposure through existing patterns. +- [ ] Extend the YoBrowser settings card with last-sync metadata. +- [x] Add the source/profile/category selection dialog using existing shadcn-vue primitives. +- [x] Add preview counts and capability reasons. +- [x] Add source-authoritative replacement confirmation. +- [ ] Add apply/verify/rollback progress and final result UX. +- [ ] Prevent cancellation after target mutation starts. +- [x] Add accessible labels, keyboard behavior, and i18n strings. +- [ ] Add renderer tests for state, stale tokens, failures, and secret redaction. + +## Phase 3: Platform and Release Validation + +- [ ] Test signed/notarized application builds on every advertised platform. +- [ ] Test multiple source profiles and browser-running states. +- [ ] Test source schema/version rejection with actionable UI. +- [ ] Test import/clear contention and app shutdown during apply. +- [ ] Test temporary-data removal after success, failure, crash, and restart. +- [ ] Audit logs, telemetry, renderer state, crash reports, and errors for secret leakage. +- [ ] Verify authorization passwords never enter DeepChat IPC, memory fields, logs, or telemetry. +- [x] Run `pnpm run format`. +- [x] Run `pnpm run i18n`. +- [x] Run `pnpm run lint`. +- [x] Run typecheck, build, focused suites, and the full main/renderer test suite. +- [ ] Update user documentation with the exact support matrix and limitations. + +## Deferred, Separately Approved Work + +- [ ] Add `localStorage` only after its feasibility gate passes. +- [ ] Design live-tab `sessionStorage` handoff as a separate flow. +- [ ] Write a separate SDD for an origin-bound credential vault if password autofill is approved. +- [ ] Evaluate additional browsers only with explicit discovery, crypto, schema, and fixture support. diff --git a/resources/acp-registry/registry.json b/resources/acp-registry/registry.json index 3ccb406df5..ac60011f61 100644 --- a/resources/acp-registry/registry.json +++ b/resources/acp-registry/registry.json @@ -122,7 +122,7 @@ { "id": "cline", "name": "Cline", - "version": "3.0.42", + "version": "3.0.44", "description": "Autonomous coding agent CLI - capable of creating/editing files, running commands, using the browser, and more", "repository": "https://github.com/cline/cline", "website": "https://cline.bot/cli", @@ -133,7 +133,7 @@ "icon": "https://cdn.agentclientprotocol.com/registry/v1/latest/cline.svg", "distribution": { "npx": { - "package": "cline@3.0.42", + "package": "cline@3.0.44", "args": [ "--acp" ] @@ -508,7 +508,7 @@ { "id": "factory-droid", "name": "Factory Droid", - "version": "0.173.0", + "version": "0.174.0", "description": "Factory Droid - AI coding agent powered by Factory AI", "website": "https://factory.ai/product/cli", "authors": [ @@ -517,7 +517,7 @@ "license": "proprietary", "distribution": { "npx": { - "package": "droid@0.173.0", + "package": "droid@0.174.0", "args": [ "exec", "--output-format", @@ -534,7 +534,7 @@ { "id": "fast-agent", "name": "fast-agent", - "version": "0.9.10", + "version": "0.9.14", "description": "Code and build agents with comprehensive multi-provider support", "repository": "https://github.com/evalstate/fast-agent", "website": "https://fast-agent.ai", @@ -544,7 +544,7 @@ "license": "Apache 2.0", "distribution": { "uvx": { - "package": "fast-agent-acp==0.9.10", + "package": "fast-agent-acp==0.9.14", "args": [ "-x" ] @@ -555,7 +555,7 @@ { "id": "gemini", "name": "Gemini CLI", - "version": "0.50.0", + "version": "0.51.0", "description": "Google's official CLI for Gemini", "repository": "https://github.com/google-gemini/gemini-cli", "website": "https://geminicli.com", @@ -565,7 +565,7 @@ "license": "Apache-2.0", "distribution": { "npx": { - "package": "@google/gemini-cli@0.50.0", + "package": "@google/gemini-cli@0.51.0", "args": [ "--acp" ] @@ -671,7 +671,7 @@ { "id": "grok-build", "name": "Grok Build", - "version": "0.2.101", + "version": "0.2.102", "description": "xAI's coding agent and CLI", "website": "https://x.ai/cli", "authors": [ @@ -680,7 +680,7 @@ "license": "proprietary", "distribution": { "npx": { - "package": "@xai-official/grok@0.2.101", + "package": "@xai-official/grok@0.2.102", "args": [ "agent", "stdio" @@ -692,7 +692,7 @@ { "id": "harn", "name": "Harn", - "version": "0.10.19", + "version": "0.10.22", "description": "Harn runs .harn agent pipelines as a native ACP coding agent over stdio.", "repository": "https://github.com/burin-labs/harn", "website": "https://harnlang.com", @@ -703,49 +703,49 @@ "distribution": { "binary": { "darwin-aarch64": { - "archive": "https://github.com/burin-labs/harn/releases/download/v0.10.19/harn-aarch64-apple-darwin.tar.gz", + "archive": "https://github.com/burin-labs/harn/releases/download/v0.10.22/harn-aarch64-apple-darwin.tar.gz", "cmd": "./harn", "args": [ "serve", "acp" ], - "sha256": "05ba15c0cc05a9afa8a05182c0893b02a50242f8cd804005850a331e6155394a" + "sha256": "ee2d2fb2172d893c656848b8c25c4b527b162c03cac9931667f6da4d7bc8da07" }, "darwin-x86_64": { - "archive": "https://github.com/burin-labs/harn/releases/download/v0.10.19/harn-x86_64-apple-darwin.tar.gz", + "archive": "https://github.com/burin-labs/harn/releases/download/v0.10.22/harn-x86_64-apple-darwin.tar.gz", "cmd": "./harn", "args": [ "serve", "acp" ], - "sha256": "9804e91ce510c51a32e8c801a2865575c3091e577ff8c43aa5e2a67f16484959" + "sha256": "6a984d21eb6102bc9993ab89169a2acdc207e453ed88ef09157f1c6d4ed593b5" }, "linux-aarch64": { - "archive": "https://github.com/burin-labs/harn/releases/download/v0.10.19/harn-aarch64-unknown-linux-gnu.tar.gz", + "archive": "https://github.com/burin-labs/harn/releases/download/v0.10.22/harn-aarch64-unknown-linux-gnu.tar.gz", "cmd": "./harn", "args": [ "serve", "acp" ], - "sha256": "718ba0a9d42c9f1155bd27d183a6e314900c5829e202d41195545c6071f47033" + "sha256": "5d1d16000915e5d20cba00c794581890a44dccf5365a7ee729bc7248a37fddcf" }, "linux-x86_64": { - "archive": "https://github.com/burin-labs/harn/releases/download/v0.10.19/harn-x86_64-unknown-linux-gnu.tar.gz", + "archive": "https://github.com/burin-labs/harn/releases/download/v0.10.22/harn-x86_64-unknown-linux-gnu.tar.gz", "cmd": "./harn", "args": [ "serve", "acp" ], - "sha256": "5e1114e04105541a7d456ce6c20c031929b27471c271da34aa8dd9357c0b6122" + "sha256": "ac36baa6ddf61e64ee5d499be8c35b461527756f5006dbab333bc518d7ce6fca" }, "windows-x86_64": { - "archive": "https://github.com/burin-labs/harn/releases/download/v0.10.19/harn-x86_64-pc-windows-msvc.zip", + "archive": "https://github.com/burin-labs/harn/releases/download/v0.10.22/harn-x86_64-pc-windows-msvc.zip", "cmd": "harn.exe", "args": [ "serve", "acp" ], - "sha256": "ef8bb6b5643071cb482fa44bb56729df9af1a49d9212dcd9852ea817d051fe14" + "sha256": "352727aa4be51be711a70838df0f30856b5e6ef79014737d3ebeb867d4b128c4" } } }, @@ -806,7 +806,7 @@ { "id": "kilo", "name": "Kilo", - "version": "7.4.9", + "version": "7.4.11", "description": "The open source coding agent", "repository": "https://github.com/Kilo-Org/kilocode", "website": "https://kilo.ai/", @@ -818,48 +818,48 @@ "distribution": { "binary": { "darwin-aarch64": { - "archive": "https://github.com/Kilo-Org/kilocode/releases/download/v7.4.9/kilo-darwin-arm64.zip", + "archive": "https://github.com/Kilo-Org/kilocode/releases/download/v7.4.11/kilo-darwin-arm64.zip", "cmd": "./kilo", "args": [ "acp" ], - "sha256": "a3fab28d5b97f6d0b297f2d12302e2664344b7fcee3ccc9adc3a84db0be15c96" + "sha256": "14a030a354f3b51f0241662627702e7b06cddf3fcb6e0f1415279e9d3a3b8998" }, "darwin-x86_64": { - "archive": "https://github.com/Kilo-Org/kilocode/releases/download/v7.4.9/kilo-darwin-x64.zip", + "archive": "https://github.com/Kilo-Org/kilocode/releases/download/v7.4.11/kilo-darwin-x64.zip", "cmd": "./kilo", "args": [ "acp" ], - "sha256": "cf6fe6bc77e63bbdf387e34fc31d1aa86a603b6de48011a86ffdc28708bc1453" + "sha256": "66e302e09b96fd9794012bdb608622b589e4810ba31eca35918d8acd1a32a438" }, "linux-aarch64": { - "archive": "https://github.com/Kilo-Org/kilocode/releases/download/v7.4.9/kilo-linux-arm64.tar.gz", + "archive": "https://github.com/Kilo-Org/kilocode/releases/download/v7.4.11/kilo-linux-arm64.tar.gz", "cmd": "./kilo", "args": [ "acp" ], - "sha256": "e0c4ff2ec2f65689406c43e6aff06cea30be8d12ed7193e1ed46a0e0e28d86fd" + "sha256": "48c5405eb05efb3558d120b521d1a2b097cd8e99d36d7caf015e7c467922793c" }, "linux-x86_64": { - "archive": "https://github.com/Kilo-Org/kilocode/releases/download/v7.4.9/kilo-linux-x64.tar.gz", + "archive": "https://github.com/Kilo-Org/kilocode/releases/download/v7.4.11/kilo-linux-x64.tar.gz", "cmd": "./kilo", "args": [ "acp" ], - "sha256": "0d3fc944dbf2b987ce9953d5c4701dbdc950b0905a186c814a24e8915fd7012c" + "sha256": "b060dd4e094b0f9966de03d8a0d0d5cc8e6bb23ab04f1d04b7e3951d13079770" }, "windows-x86_64": { - "archive": "https://github.com/Kilo-Org/kilocode/releases/download/v7.4.9/kilo-windows-x64.zip", + "archive": "https://github.com/Kilo-Org/kilocode/releases/download/v7.4.11/kilo-windows-x64.zip", "cmd": "./kilo.exe", "args": [ "acp" ], - "sha256": "2452f063e6149bffca64e862ae5b2063fe2be6a14200012f92743a8d1a3b4c0c" + "sha256": "1c04d25b1484526b1eb8abe44bbcfc179fb71b1efa88b0164cd5709ca8bb00e7" } }, "npx": { - "package": "@kilocode/cli@7.4.9", + "package": "@kilocode/cli@7.4.11", "args": [ "acp" ] @@ -869,7 +869,7 @@ { "id": "kimi", "name": "Kimi CLI", - "version": "1.48.0", + "version": "1.49.0", "description": "Moonshot AI's coding assistant", "repository": "https://github.com/MoonshotAI/kimi-cli", "website": "https://moonshotai.github.io/kimi-cli/", @@ -880,39 +880,44 @@ "distribution": { "binary": { "darwin-aarch64": { - "archive": "https://github.com/MoonshotAI/kimi-cli/releases/download/1.48.0/kimi-1.48.0-aarch64-apple-darwin.tar.gz", + "archive": "https://github.com/MoonshotAI/kimi-cli/releases/download/1.49.0/kimi-1.49.0-aarch64-apple-darwin.tar.gz", "cmd": "./kimi", "args": [ "acp" - ] + ], + "sha256": "15018b20b203aee09658fdc64840c4846fc17c108d8dba1a19a95581d3ce2921" }, "linux-aarch64": { - "archive": "https://github.com/MoonshotAI/kimi-cli/releases/download/1.48.0/kimi-1.48.0-aarch64-unknown-linux-gnu.tar.gz", + "archive": "https://github.com/MoonshotAI/kimi-cli/releases/download/1.49.0/kimi-1.49.0-aarch64-unknown-linux-gnu.tar.gz", "cmd": "./kimi", "args": [ "acp" - ] + ], + "sha256": "5ac54cabce16ede27b9d2069b9b88edee25528646e7bb5befa9980a1ca71febb" }, "linux-x86_64": { - "archive": "https://github.com/MoonshotAI/kimi-cli/releases/download/1.48.0/kimi-1.48.0-x86_64-unknown-linux-gnu.tar.gz", + "archive": "https://github.com/MoonshotAI/kimi-cli/releases/download/1.49.0/kimi-1.49.0-x86_64-unknown-linux-gnu.tar.gz", "cmd": "./kimi", "args": [ "acp" - ] + ], + "sha256": "6ce0b83f583c45a64cc9f51ffe7e1a8e03ee79acda69945fcf8c23341b9d892f" }, "windows-aarch64": { - "archive": "https://github.com/MoonshotAI/kimi-cli/releases/download/1.48.0/kimi-1.48.0-aarch64-pc-windows-msvc.zip", + "archive": "https://github.com/MoonshotAI/kimi-cli/releases/download/1.49.0/kimi-1.49.0-aarch64-pc-windows-msvc.zip", "cmd": "./kimi.exe", "args": [ "acp" - ] + ], + "sha256": "3ac8f05c7bd18d902a324c6c03a71084cfbe785b9669bbd556c071ee1d8f2f26" }, "windows-x86_64": { - "archive": "https://github.com/MoonshotAI/kimi-cli/releases/download/1.48.0/kimi-1.48.0-x86_64-pc-windows-msvc.zip", + "archive": "https://github.com/MoonshotAI/kimi-cli/releases/download/1.49.0/kimi-1.49.0-x86_64-pc-windows-msvc.zip", "cmd": "./kimi.exe", "args": [ "acp" - ] + ], + "sha256": "2acbbc7ca8c8ac4b03dab1d970f53a292bd226168151b423499feab9fc203ddd" } } }, @@ -1004,7 +1009,7 @@ { "id": "opencode", "name": "OpenCode", - "version": "1.18.2", + "version": "1.18.3", "description": "The open source coding agent", "repository": "https://github.com/anomalyco/opencode", "website": "https://opencode.ai", @@ -1016,52 +1021,52 @@ "distribution": { "binary": { "darwin-aarch64": { - "archive": "https://github.com/anomalyco/opencode/releases/download/v1.18.2/opencode-darwin-arm64.zip", + "archive": "https://github.com/anomalyco/opencode/releases/download/v1.18.3/opencode-darwin-arm64.zip", "cmd": "./opencode", "args": [ "acp" ], - "sha256": "2cb1eb3301a73598890364dfeb45c535155a3855b37ed1d190172821e74e462c" + "sha256": "946f62b155638b911144b7bef520ee4a6442f696297907873463bca3524e40ef" }, "darwin-x86_64": { - "archive": "https://github.com/anomalyco/opencode/releases/download/v1.18.2/opencode-darwin-x64.zip", + "archive": "https://github.com/anomalyco/opencode/releases/download/v1.18.3/opencode-darwin-x64.zip", "cmd": "./opencode", "args": [ "acp" ], - "sha256": "3672491ae6383468be40e54ee5c6fcbac354eb5cfb7c6afbd0ee3ae0c6af4b2e" + "sha256": "4ea147867ba19e4ec03559df557811f1674f40788aea4d10326dc563b7667c6d" }, "linux-aarch64": { - "archive": "https://github.com/anomalyco/opencode/releases/download/v1.18.2/opencode-linux-arm64.tar.gz", + "archive": "https://github.com/anomalyco/opencode/releases/download/v1.18.3/opencode-linux-arm64.tar.gz", "cmd": "./opencode", "args": [ "acp" ], - "sha256": "93352b30d37d8da2e5c226085f1afbf37cf57cfcecedc813520ff2d0f8581540" + "sha256": "da0a631174eba380b2a1d51f9d364fa3812da433e72743c72471d4b5da59c69d" }, "linux-x86_64": { - "archive": "https://github.com/anomalyco/opencode/releases/download/v1.18.2/opencode-linux-x64.tar.gz", + "archive": "https://github.com/anomalyco/opencode/releases/download/v1.18.3/opencode-linux-x64.tar.gz", "cmd": "./opencode", "args": [ "acp" ], - "sha256": "97c95e004bb73d2039f957ea33be0635ea4e22b8dceaedf8f0983765950cf1b6" + "sha256": "60f27b2679f00a511b6539f97e02448afaf58d9c66e2448285ea0c517ca84583" }, "windows-aarch64": { - "archive": "https://github.com/anomalyco/opencode/releases/download/v1.18.2/opencode-windows-arm64.zip", + "archive": "https://github.com/anomalyco/opencode/releases/download/v1.18.3/opencode-windows-arm64.zip", "cmd": "./opencode", "args": [ "acp" ], - "sha256": "7eb407240101a0ebca9acb99ed819a09a4211fdafa381161af6ca49f85736e4b" + "sha256": "a549fb2e9041db9438bcd9b77bfa0a4b2476caf2d550f37479aabfec1b079bfb" }, "windows-x86_64": { - "archive": "https://github.com/anomalyco/opencode/releases/download/v1.18.2/opencode-windows-x64.zip", + "archive": "https://github.com/anomalyco/opencode/releases/download/v1.18.3/opencode-windows-x64.zip", "cmd": "./opencode.exe", "args": [ "acp" ], - "sha256": "aca23b18d03639c6bd576a2dd23cebfa6dab80d31358f17cfa03fd1f2a5c41fd" + "sha256": "68bc62930f6cb5755e0409aa9de0bb270a66ed2b8c9cf0c029e9f2287ed5486e" } } } @@ -1165,7 +1170,7 @@ { "id": "qwen-code", "name": "Qwen Code", - "version": "0.19.10", + "version": "0.19.11", "description": "Alibaba's Qwen coding assistant", "repository": "https://github.com/QwenLM/qwen-code", "website": "https://qwenlm.github.io/qwen-code-docs/en/users/overview", @@ -1175,7 +1180,7 @@ "license": "Apache-2.0", "distribution": { "npx": { - "package": "@qwen-code/qwen-code@0.19.10", + "package": "@qwen-code/qwen-code@0.19.11", "args": [ "--acp", "--experimental-skills" diff --git a/resources/model-db/providers.json b/resources/model-db/providers.json index 29764373e1..0370d123a8 100644 --- a/resources/model-db/providers.json +++ b/resources/model-db/providers.json @@ -493,9 +493,9 @@ "release_date": "2026-05-29", "last_updated": "2026-05-29", "cost": { - "input": 0.2, - "output": 1.15, - "cache_read": 0.04, + "input": 0.19, + "output": 1.14, + "cache_read": 0.03, "cache_write": 0 }, "type": "chat" @@ -614,9 +614,9 @@ "release_date": "2026-06-12", "last_updated": "2026-06-12", "cost": { - "input": 0.719, - "output": 3.49, - "cache_read": 0.149, + "input": 0.75, + "output": 3.5, + "cache_read": 0.16, "cache_write": 0 }, "type": "chat" @@ -37554,6 +37554,42 @@ }, "type": "chat" }, + { + "id": "k3", + "name": "Kimi K3", + "display_name": "Kimi K3", + "modalities": { + "input": [ + "text", + "image", + "video" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 1048576, + "output": 131072 + }, + "temperature": true, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "attachment": false, + "open_weights": true, + "release_date": "2026-07-16", + "last_updated": "2026-07-16", + "cost": { + "input": 0, + "output": 0, + "cache_read": 0, + "cache_write": 0 + }, + "type": "chat" + }, { "id": "k2p6", "name": "Kimi K2.6", @@ -40434,6 +40470,43 @@ }, "type": "chat" }, + { + "id": "zai-org/GLM-5.2", + "name": "GLM-5.2", + "display_name": "GLM-5.2", + "modalities": { + "input": [ + "text" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 1048576, + "output": 131072 + }, + "temperature": true, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true + } + }, + "attachment": false, + "open_weights": true, + "release_date": "2026-06-13", + "last_updated": "2026-06-13", + "cost": { + "input": 1.4375, + "output": 5.75 + }, + "type": "chat" + }, { "id": "mistralai/Mistral-Medium-3.5-128B", "name": "Mistral Medium 3.5", @@ -75663,6 +75736,47 @@ }, "type": "chat" }, + { + "id": "kimi-k3", + "name": "Kimi K3", + "display_name": "Kimi K3", + "modalities": { + "input": [ + "text", + "image", + "video" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 1048576, + "output": 131072 + }, + "temperature": true, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true, + "interleaved": true, + "summaries": true, + "visibility": "summary", + "continuation": [ + "thinking_blocks" + ] + } + }, + "attachment": true, + "open_weights": true, + "release_date": "2026-07-16", + "last_updated": "2026-07-16", + "type": "chat" + }, { "id": "kimi-k2.7-code-highspeed", "name": "Kimi K2.7 Code Highspeed", @@ -120641,6 +120755,62 @@ } ] }, + "thinkingmachines": { + "id": "thinkingmachines", + "name": "Thinking Machines", + "display_name": "Thinking Machines", + "api": "https://tinker.thinkingmachines.dev/services/tinker-prod/oai/api/v1", + "doc": "https://tinker-docs.thinkingmachines.ai/tinker/compatible-apis/openai/", + "models": [ + { + "id": "inkling", + "name": "Inkling", + "display_name": "Inkling", + "modalities": { + "input": [ + "text", + "image", + "audio", + "pdf" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 256000, + "output": 256000 + }, + "temperature": true, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true, + "interleaved": true, + "summaries": true, + "visibility": "summary", + "continuation": [ + "thinking_blocks" + ] + } + }, + "attachment": true, + "open_weights": true, + "release_date": "2026-07-15", + "last_updated": "2026-07-15", + "cost": { + "input": 3.74, + "output": 9.36, + "cache_read": 0.748 + }, + "type": "chat" + } + ] + }, "nebius": { "id": "nebius", "name": "Nebius Token Factory", @@ -129318,54 +129488,9 @@ "type": "chat" }, { - "id": "deepseek-v4-flash", - "name": "DeepSeek V4 Flash", - "display_name": "DeepSeek V4 Flash", - "modalities": { - "input": [ - "text" - ], - "output": [ - "text" - ] - }, - "limit": { - "context": 1000000, - "output": 384000 - }, - "temperature": true, - "tool_call": true, - "reasoning": { - "supported": true, - "default": true - }, - "extra_capabilities": { - "reasoning": { - "supported": true, - "interleaved": true, - "summaries": true, - "visibility": "summary", - "continuation": [ - "thinking_blocks" - ] - } - }, - "attachment": false, - "open_weights": true, - "knowledge": "2025-05", - "release_date": "2026-04-24", - "last_updated": "2026-04-24", - "cost": { - "input": 0.14, - "output": 0.28, - "cache_read": 0.0028 - }, - "type": "chat" - }, - { - "id": "qwen3.5-plus", - "name": "Qwen3.5 Plus", - "display_name": "Qwen3.5 Plus", + "id": "kimi-k3", + "name": "Kimi K3", + "display_name": "Kimi K3", "modalities": { "input": [ "text", @@ -129377,8 +129502,8 @@ ] }, "limit": { - "context": 262144, - "output": 65536 + "context": 1048576, + "output": 131072 }, "temperature": true, "tool_call": true, @@ -129398,22 +129523,20 @@ } }, "attachment": true, - "open_weights": false, - "knowledge": "2025-04", - "release_date": "2026-02-16", - "last_updated": "2026-02-16", + "open_weights": true, + "release_date": "2026-07-16", + "last_updated": "2026-07-16", "cost": { - "input": 0.2, - "output": 1.2, - "cache_read": 0.02, - "cache_write": 0.25 + "input": 3, + "output": 15, + "cache_read": 0.3 }, "type": "chat" }, { - "id": "mimo-v2-pro", - "name": "MiMo V2 Pro", - "display_name": "MiMo V2 Pro", + "id": "deepseek-v4-flash", + "name": "DeepSeek V4 Flash", + "display_name": "DeepSeek V4 Flash", "modalities": { "input": [ "text" @@ -129423,8 +129546,8 @@ ] }, "limit": { - "context": 1048576, - "output": 128000 + "context": 1000000, + "output": 384000 }, "temperature": true, "tool_call": true, @@ -129443,38 +129566,186 @@ ] } }, - "attachment": true, + "attachment": false, "open_weights": true, - "knowledge": "2024-12", - "release_date": "2026-03-18", - "last_updated": "2026-03-18", + "knowledge": "2025-05", + "release_date": "2026-04-24", + "last_updated": "2026-04-24", "cost": { - "input": 1, - "output": 3, - "cache_read": 0.2, - "tiers": [ - { - "input": 2, - "output": 6, - "cache_read": 0.4, - "tier": { - "type": "context", - "size": 256000 - } - } - ], - "context_over_200k": { - "input": 2, - "output": 6, - "cache_read": 0.4 - } + "input": 0.14, + "output": 0.28, + "cache_read": 0.0028 }, "type": "chat" }, { - "id": "kimi-k2.6", - "name": "Kimi K2.6", - "display_name": "Kimi K2.6", + "id": "qwen3.5-plus", + "name": "Qwen3.5 Plus", + "display_name": "Qwen3.5 Plus", + "modalities": { + "input": [ + "text", + "image", + "video" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 262144, + "output": 65536 + }, + "temperature": true, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true, + "interleaved": true, + "summaries": true, + "visibility": "summary", + "continuation": [ + "thinking_blocks" + ] + } + }, + "attachment": true, + "open_weights": false, + "knowledge": "2025-04", + "release_date": "2026-02-16", + "last_updated": "2026-02-16", + "cost": { + "input": 0.2, + "output": 1.2, + "cache_read": 0.02, + "cache_write": 0.25 + }, + "type": "chat" + }, + { + "id": "grok-4.5", + "name": "Grok 4.5", + "display_name": "Grok 4.5", + "modalities": { + "input": [ + "text", + "image" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 500000, + "output": 500000 + }, + "temperature": true, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true + } + }, + "attachment": true, + "open_weights": false, + "release_date": "2026-07-08", + "last_updated": "2026-07-08", + "cost": { + "input": 2, + "output": 6, + "cache_read": 0.5, + "tiers": [ + { + "input": 4, + "output": 12, + "cache_read": 1, + "tier": { + "type": "context", + "size": 200000 + } + } + ], + "context_over_200k": { + "input": 4, + "output": 12, + "cache_read": 1 + } + }, + "type": "chat" + }, + { + "id": "mimo-v2-pro", + "name": "MiMo V2 Pro", + "display_name": "MiMo V2 Pro", + "modalities": { + "input": [ + "text" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 1048576, + "output": 128000 + }, + "temperature": true, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true, + "interleaved": true, + "summaries": true, + "visibility": "summary", + "continuation": [ + "thinking_blocks" + ] + } + }, + "attachment": true, + "open_weights": true, + "knowledge": "2024-12", + "release_date": "2026-03-18", + "last_updated": "2026-03-18", + "cost": { + "input": 1, + "output": 3, + "cache_read": 0.2, + "tiers": [ + { + "input": 2, + "output": 6, + "cache_read": 0.4, + "tier": { + "type": "context", + "size": 256000 + } + } + ], + "context_over_200k": { + "input": 2, + "output": 6, + "cache_read": 0.4 + } + }, + "type": "chat" + }, + { + "id": "kimi-k2.6", + "name": "Kimi K2.6", + "display_name": "Kimi K2.6", "modalities": { "input": [ "text", @@ -156277,6 +156548,47 @@ }, "type": "chat" }, + { + "id": "kimi-k3", + "name": "Kimi K3", + "display_name": "Kimi K3", + "modalities": { + "input": [ + "text", + "image", + "video" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 1048576, + "output": 131072 + }, + "temperature": true, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true, + "interleaved": true, + "summaries": true, + "visibility": "summary", + "continuation": [ + "thinking_blocks" + ] + } + }, + "attachment": true, + "open_weights": true, + "release_date": "2026-07-16", + "last_updated": "2026-07-16", + "type": "chat" + }, { "id": "kimi-k2-thinking", "name": "Kimi K2 Thinking", @@ -158404,8 +158716,8 @@ ] }, "limit": { - "context": 1048576, - "output": 1048576 + "context": 256000, + "output": 256000 }, "temperature": true, "tool_call": true, @@ -175414,8 +175726,7 @@ "display_name": "MiMo V2.5 Pro", "modalities": { "input": [ - "text", - "pdf" + "text" ], "output": [ "text" @@ -176649,36 +176960,44 @@ "type": "chat" }, { - "id": "recraft/recraft-v4.1-utility", - "name": "Recraft V4.1 Utility", - "display_name": "Recraft V4.1 Utility", + "id": "thinkingmachines/inkling", + "name": "Inkling", + "display_name": "Inkling", "modalities": { "input": [ - "text" + "text", + "image", + "pdf" ], "output": [ - "image" + "text" ] }, "limit": { - "context": 8192, - "output": 8192 + "context": 256000, + "output": 256000 }, "temperature": true, - "tool_call": false, + "tool_call": true, "reasoning": { - "supported": false + "supported": true, + "default": true + }, + "attachment": true, + "open_weights": true, + "release_date": "2026-07-15", + "last_updated": "2026-07-15", + "cost": { + "input": 1, + "output": 4.05, + "cache_read": 0.17 }, - "attachment": false, - "open_weights": false, - "release_date": "2026-05-14", - "last_updated": "2026-05-14", "type": "chat" }, { - "id": "recraft/recraft-v4", - "name": "Recraft V4", - "display_name": "Recraft V4", + "id": "recraft/recraft-v4.1-utility", + "name": "Recraft V4.1 Utility", + "display_name": "Recraft V4.1 Utility", "modalities": { "input": [ "text" @@ -176698,14 +177017,14 @@ }, "attachment": false, "open_weights": false, - "release_date": "2026-02-17", - "last_updated": "2026-02-17", + "release_date": "2026-05-14", + "last_updated": "2026-05-14", "type": "chat" }, { - "id": "recraft/recraft-v4.1-utility-pro", - "name": "Recraft V4.1 Utility Pro", - "display_name": "Recraft V4.1 Utility Pro", + "id": "recraft/recraft-v4", + "name": "Recraft V4", + "display_name": "Recraft V4", "modalities": { "input": [ "text" @@ -176725,14 +177044,41 @@ }, "attachment": false, "open_weights": false, - "release_date": "2026-05-14", - "last_updated": "2026-05-14", + "release_date": "2026-02-17", + "last_updated": "2026-02-17", "type": "chat" }, { - "id": "recraft/recraft-v4.1-pro", - "name": "Recraft V4.1 Pro", - "display_name": "Recraft V4.1 Pro", + "id": "recraft/recraft-v4.1-utility-pro", + "name": "Recraft V4.1 Utility Pro", + "display_name": "Recraft V4.1 Utility Pro", + "modalities": { + "input": [ + "text" + ], + "output": [ + "image" + ] + }, + "limit": { + "context": 8192, + "output": 8192 + }, + "temperature": true, + "tool_call": false, + "reasoning": { + "supported": false + }, + "attachment": false, + "open_weights": false, + "release_date": "2026-05-14", + "last_updated": "2026-05-14", + "type": "chat" + }, + { + "id": "recraft/recraft-v4.1-pro", + "name": "Recraft V4.1 Pro", + "display_name": "Recraft V4.1 Pro", "modalities": { "input": [ "text" @@ -177673,6 +178019,69 @@ }, "type": "chat" }, + { + "id": "anthropic/claude-opus-4.8-fast", + "name": "Claude Opus 4.8 (Fast)", + "display_name": "Claude Opus 4.8 (Fast)", + "modalities": { + "input": [ + "text", + "image", + "pdf" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "temperature": true, + "tool_call": true, + "reasoning": { + "supported": true, + "default": false + }, + "extra_capabilities": { + "reasoning": { + "supported": true, + "default_enabled": false, + "mode": "effort", + "effort": "high", + "effort_options": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "interleaved": true, + "summaries": true, + "visibility": "omitted", + "continuation": [ + "thinking_blocks" + ], + "notes": [ + "Claude Opus 4.7 and newer Opus models require thinking.type = \"adaptive\" to enable thinking explicitly.", + "Manual budget_tokens requests return 400 on Claude Opus 4.7 and newer adaptive-only Opus models.", + "task_budget is separate from thinking control and should not be treated as a thinking budget." + ] + } + }, + "attachment": true, + "open_weights": false, + "knowledge": "2026-01", + "release_date": "2026-05-28", + "last_updated": "2026-05-28", + "cost": { + "input": 10, + "output": 50, + "cache_read": 1, + "cache_write": 12.5 + }, + "type": "chat" + }, { "id": "anthropic/claude-opus-4.8", "name": "Claude Opus 4.8", @@ -178202,6 +178611,69 @@ }, "type": "chat" }, + { + "id": "anthropic/claude-opus-4.7-fast", + "name": "Claude Opus 4.7 (Fast)", + "display_name": "Claude Opus 4.7 (Fast)", + "modalities": { + "input": [ + "text", + "image", + "pdf" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "temperature": true, + "tool_call": true, + "reasoning": { + "supported": true, + "default": false + }, + "extra_capabilities": { + "reasoning": { + "supported": true, + "default_enabled": false, + "mode": "effort", + "effort": "high", + "effort_options": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "interleaved": true, + "summaries": true, + "visibility": "omitted", + "continuation": [ + "thinking_blocks" + ], + "notes": [ + "Claude Opus 4.7 and newer Opus models require thinking.type = \"adaptive\" to enable thinking explicitly.", + "Manual budget_tokens requests return 400 on Claude Opus 4.7 and newer adaptive-only Opus models.", + "task_budget is separate from thinking control and should not be treated as a thinking budget." + ] + } + }, + "attachment": true, + "open_weights": false, + "knowledge": "2026-01-31", + "release_date": "2026-04-16", + "last_updated": "2026-04-16", + "cost": { + "input": 30, + "output": 150, + "cache_read": 3, + "cache_write": 37.5 + }, + "type": "chat" + }, { "id": "anthropic/claude-opus-4.1", "name": "Claude Opus 4.1", @@ -179021,6 +179493,46 @@ }, "type": "chat" }, + { + "id": "moonshotai/kimi-k3", + "name": "Kimi K3", + "display_name": "Kimi K3", + "modalities": { + "input": [ + "text", + "image", + "pdf" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 1000000, + "output": 131072 + }, + "temperature": true, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true + } + }, + "attachment": true, + "open_weights": false, + "release_date": "2026-07-16", + "last_updated": "2026-07-16", + "cost": { + "input": 3, + "output": 15, + "cache_read": 0.3 + }, + "type": "chat" + }, { "id": "moonshotai/kimi-k2-thinking", "name": "Kimi K2 Thinking", @@ -183597,6 +184109,33 @@ "last_updated": "2024-01-25", "type": "embedding" }, + { + "id": "openai/gpt-realtime-whisper", + "name": "gpt-realtime-whisper", + "display_name": "gpt-realtime-whisper", + "modalities": { + "input": [ + "audio" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 8192, + "output": 8192 + }, + "temperature": true, + "tool_call": false, + "reasoning": { + "supported": false + }, + "attachment": false, + "open_weights": false, + "release_date": "2026-05-07", + "last_updated": "2026-05-07", + "type": "chat" + }, { "id": "openai/gpt-5.1-thinking", "name": "GPT 5.1 Thinking", @@ -226933,6 +227472,25 @@ "name": "PPInfra", "display_name": "PPInfra", "models": [ + { + "id": "moonshotai/kimi-k3", + "name": "Kimi K3", + "display_name": "Kimi K3", + "limit": { + "context": 1048576 + }, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true + } + }, + "type": "chat" + }, { "id": "zai-org/glm-5.2", "name": "GLM 5.2", @@ -234348,6 +234906,38 @@ }, "type": "chat" }, + { + "id": "kimi-k3", + "name": "kimi-k3", + "display_name": "kimi-k3", + "modalities": { + "input": [ + "text", + "image", + "video" + ] + }, + "limit": { + "context": 1048576, + "output": 1048576 + }, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true + } + }, + "cost": { + "input": 3, + "output": 15, + "cache_read": 0.3 + }, + "type": "chat" + }, { "id": "claude-sonnet-5", "name": "claude-sonnet-5", @@ -234835,6 +235425,38 @@ }, "type": "chat" }, + { + "id": "coding-kimi-k3", + "name": "coding-kimi-k3", + "display_name": "coding-kimi-k3", + "modalities": { + "input": [ + "text", + "image", + "video" + ] + }, + "limit": { + "context": 1048576, + "output": 1048576 + }, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true + } + }, + "cost": { + "input": 0.12, + "output": 0.439999, + "cache_read": 0.012 + }, + "type": "chat" + }, { "id": "coding-glm-5.2-free", "name": "coding-glm-5.2-free", @@ -234866,20 +235488,21 @@ "type": "chat" }, { - "id": "gemini-3.1-flash-image", - "name": "gemini-3.1-flash-image", - "display_name": "gemini-3.1-flash-image", + "id": "coding-kimi-k3-free", + "name": "coding-kimi-k3-free", + "display_name": "coding-kimi-k3-free", "modalities": { "input": [ "text", - "image" + "image", + "video" ] }, "limit": { - "context": 8192, - "output": 8192 + "context": 1048576, + "output": 1048576 }, - "tool_call": false, + "tool_call": true, "reasoning": { "supported": true, "default": true @@ -234890,11 +235513,11 @@ } }, "cost": { - "input": 0.5, - "output": 3, - "cache_read": 0.5 + "input": 0, + "output": 0, + "cache_read": 0 }, - "type": "imageGeneration" + "type": "chat" }, { "id": "kimi-k2.7-code", @@ -234961,34 +235584,9 @@ "type": "chat" }, { - "id": "kling-v3-omni", - "name": "kling-v3-omni", - "display_name": "kling-v3-omni", - "modalities": { - "input": [ - "text", - "image" - ] - }, - "limit": { - "context": 8192, - "output": 8192 - }, - "tool_call": false, - "reasoning": { - "supported": false - }, - "cost": { - "input": 2, - "output": 0, - "cache_read": 0 - }, - "type": "chat" - }, - { - "id": "kling-video-o1", - "name": "kling-video-o1", - "display_name": "kling-video-o1", + "id": "gemini-3.1-flash-image", + "name": "gemini-3.1-flash-image", + "display_name": "gemini-3.1-flash-image", "modalities": { "input": [ "text", @@ -235000,30 +235598,6 @@ "output": 8192 }, "tool_call": false, - "reasoning": { - "supported": false - }, - "cost": { - "input": 2, - "output": 0, - "cache_read": 0 - }, - "type": "chat" - }, - { - "id": "longcat-2.0", - "name": "longcat-2.0", - "display_name": "longcat-2.0", - "modalities": { - "input": [ - "text" - ] - }, - "limit": { - "context": 8192, - "output": 8192 - }, - "tool_call": true, "reasoning": { "supported": true, "default": true @@ -235034,11 +235608,11 @@ } }, "cost": { - "input": 0.7746, - "output": 3.0984, - "cache_read": 0.015492 + "input": 0.5, + "output": 3, + "cache_read": 0.5 }, - "type": "chat" + "type": "imageGeneration" }, { "id": "gemini-3-pro-image", @@ -235145,6 +235719,86 @@ }, "type": "chat" }, + { + "id": "kling-v3-omni", + "name": "kling-v3-omni", + "display_name": "kling-v3-omni", + "modalities": { + "input": [ + "text", + "image" + ] + }, + "limit": { + "context": 8192, + "output": 8192 + }, + "tool_call": false, + "reasoning": { + "supported": false + }, + "cost": { + "input": 2, + "output": 0, + "cache_read": 0 + }, + "type": "chat" + }, + { + "id": "kling-video-o1", + "name": "kling-video-o1", + "display_name": "kling-video-o1", + "modalities": { + "input": [ + "text", + "image" + ] + }, + "limit": { + "context": 8192, + "output": 8192 + }, + "tool_call": false, + "reasoning": { + "supported": false + }, + "cost": { + "input": 2, + "output": 0, + "cache_read": 0 + }, + "type": "chat" + }, + { + "id": "longcat-2.0", + "name": "longcat-2.0", + "display_name": "longcat-2.0", + "modalities": { + "input": [ + "text" + ] + }, + "limit": { + "context": 8192, + "output": 8192 + }, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true + } + }, + "cost": { + "input": 0.7746, + "output": 3.0984, + "cache_read": 0.015492 + }, + "type": "chat" + }, { "id": "hy3-preview", "name": "hy3-preview", @@ -235463,18 +236117,17 @@ "type": "chat" }, { - "id": "grok-4.3", - "name": "grok-4.3", - "display_name": "grok-4.3", + "id": "coding-glm-5.2", + "name": "coding-glm-5.2", + "display_name": "coding-glm-5.2", "modalities": { "input": [ - "text", - "image" + "text" ] }, "limit": { - "context": 1000000, - "output": 1000000 + "context": 8192, + "output": 8192 }, "tool_call": true, "reasoning": { @@ -235487,24 +236140,24 @@ } }, "cost": { - "input": 1.25, - "output": 2.5, - "cache_read": 0.2 + "input": 0.06, + "output": 0.22 }, "type": "chat" }, { - "id": "coding-glm-5.2", - "name": "coding-glm-5.2", - "display_name": "coding-glm-5.2", + "id": "grok-4.3", + "name": "grok-4.3", + "display_name": "grok-4.3", "modalities": { "input": [ - "text" + "text", + "image" ] }, "limit": { - "context": 8192, - "output": 8192 + "context": 1000000, + "output": 1000000 }, "tool_call": true, "reasoning": { @@ -235517,8 +236170,9 @@ } }, "cost": { - "input": 0.06, - "output": 0.22 + "input": 1.25, + "output": 2.5, + "cache_read": 0.2 }, "type": "chat" }, @@ -241877,13 +242531,12 @@ "type": "chat" }, { - "id": "wan2.2-i2v-plus", - "name": "wan2.2-i2v-plus", - "display_name": "wan2.2-i2v-plus", + "id": "wan2.5-t2v-preview", + "name": "wan2.5-t2v-preview", + "display_name": "wan2.5-t2v-preview", "modalities": { "input": [ - "text", - "image" + "text" ] }, "limit": { @@ -241901,12 +242554,13 @@ "type": "chat" }, { - "id": "wan2.5-t2v-preview", - "name": "wan2.5-t2v-preview", - "display_name": "wan2.5-t2v-preview", + "id": "wan2.5-i2v-preview", + "name": "wan2.5-i2v-preview", + "display_name": "wan2.5-i2v-preview", "modalities": { "input": [ - "text" + "text", + "image" ] }, "limit": { @@ -241924,9 +242578,9 @@ "type": "chat" }, { - "id": "wan2.5-i2v-preview", - "name": "wan2.5-i2v-preview", - "display_name": "wan2.5-i2v-preview", + "id": "wan2.2-i2v-plus", + "name": "wan2.2-i2v-plus", + "display_name": "wan2.2-i2v-plus", "modalities": { "input": [ "text", @@ -247299,24 +247953,6 @@ }, "type": "chat" }, - { - "id": "tencent/Hunyuan-A13B-Instruct", - "name": "tencent/Hunyuan-A13B-Instruct", - "display_name": "tencent/Hunyuan-A13B-Instruct", - "limit": { - "context": 8192, - "output": 8192 - }, - "tool_call": false, - "reasoning": { - "supported": false - }, - "cost": { - "input": 0.14, - "output": 0.56 - }, - "type": "chat" - }, { "id": "MiniMaxAI/MiniMax-M1-80k", "name": "MiniMaxAI/MiniMax-M1-80k", @@ -247451,6 +248087,24 @@ }, "type": "chat" }, + { + "id": "tencent/Hunyuan-A13B-Instruct", + "name": "tencent/Hunyuan-A13B-Instruct", + "display_name": "tencent/Hunyuan-A13B-Instruct", + "limit": { + "context": 8192, + "output": 8192 + }, + "tool_call": false, + "reasoning": { + "supported": false + }, + "cost": { + "input": 0.14, + "output": 0.56 + }, + "type": "chat" + }, { "id": "unsloth/gemma-3-27b-it", "name": "unsloth/gemma-3-27b-it", @@ -247644,24 +248298,6 @@ }, "type": "chat" }, - { - "id": "tencent/Hunyuan-MT-7B", - "name": "tencent/Hunyuan-MT-7B", - "display_name": "tencent/Hunyuan-MT-7B", - "limit": { - "context": 8192, - "output": 8192 - }, - "tool_call": false, - "reasoning": { - "supported": false - }, - "cost": { - "input": 0.2, - "output": 0.2 - }, - "type": "chat" - }, { "id": "BAAI/bge-large-en-v1.5", "name": "BAAI/bge-large-en-v1.5", @@ -247734,6 +248370,24 @@ }, "type": "rerank" }, + { + "id": "tencent/Hunyuan-MT-7B", + "name": "tencent/Hunyuan-MT-7B", + "display_name": "tencent/Hunyuan-MT-7B", + "limit": { + "context": 8192, + "output": 8192 + }, + "tool_call": false, + "reasoning": { + "supported": false + }, + "cost": { + "input": 0.2, + "output": 0.2 + }, + "type": "chat" + }, { "id": "V3", "name": "V3", @@ -253994,9 +254648,14 @@ "type": "chat" }, { - "id": "text-ada-001", - "name": "text-ada-001", - "display_name": "text-ada-001", + "id": "tts-1-hd-1106", + "name": "tts-1-hd-1106", + "display_name": "tts-1-hd-1106", + "modalities": { + "input": [ + "audio" + ] + }, "limit": { "context": 8192, "output": 8192 @@ -254006,15 +254665,14 @@ "supported": false }, "cost": { - "input": 0.4, - "output": 0.4 - }, - "type": "chat" + "input": 30, + "output": 30 + } }, { - "id": "tts-1", - "name": "tts-1", - "display_name": "tts-1", + "id": "tts-1-hd", + "name": "tts-1-hd", + "display_name": "tts-1-hd", "modalities": { "input": [ "audio" @@ -254029,14 +254687,19 @@ "supported": false }, "cost": { - "input": 15, - "output": 15 + "input": 30, + "output": 30 } }, { - "id": "text-search-ada-doc-001", - "name": "text-search-ada-doc-001", - "display_name": "text-search-ada-doc-001", + "id": "tts-1-1106", + "name": "tts-1-1106", + "display_name": "tts-1-1106", + "modalities": { + "input": [ + "audio" + ] + }, "limit": { "context": 8192, "output": 8192 @@ -254046,15 +254709,14 @@ "supported": false }, "cost": { - "input": 20, - "output": 20 - }, - "type": "chat" + "input": 15, + "output": 15 + } }, { - "id": "tts-1-hd-1106", - "name": "tts-1-hd-1106", - "display_name": "tts-1-hd-1106", + "id": "tts-1", + "name": "tts-1", + "display_name": "tts-1", "modalities": { "input": [ "audio" @@ -254069,14 +254731,14 @@ "supported": false }, "cost": { - "input": 30, - "output": 30 + "input": 15, + "output": 15 } }, { - "id": "text-moderation-stable", - "name": "text-moderation-stable", - "display_name": "text-moderation-stable", + "id": "text-search-ada-doc-001", + "name": "text-search-ada-doc-001", + "display_name": "text-search-ada-doc-001", "limit": { "context": 8192, "output": 8192 @@ -254086,15 +254748,15 @@ "supported": false }, "cost": { - "input": 0.2, - "output": 0.2 + "input": 20, + "output": 20 }, "type": "chat" }, { - "id": "text-moderation-latest", - "name": "text-moderation-latest", - "display_name": "text-moderation-latest", + "id": "text-moderation-007", + "name": "text-moderation-007", + "display_name": "text-moderation-007", "limit": { "context": 8192, "output": 8192 @@ -254110,9 +254772,14 @@ "type": "chat" }, { - "id": "text-moderation-007", - "name": "text-moderation-007", - "display_name": "text-moderation-007", + "id": "text-embedding-ada-002", + "name": "text-embedding-ada-002", + "display_name": "text-embedding-ada-002", + "modalities": { + "input": [ + "text" + ] + }, "limit": { "context": 8192, "output": 8192 @@ -254122,18 +254789,18 @@ "supported": false }, "cost": { - "input": 0.2, - "output": 0.2 + "input": 0.1, + "output": 0.1 }, - "type": "chat" + "type": "embedding" }, { - "id": "tts-1-hd", - "name": "tts-1-hd", - "display_name": "tts-1-hd", + "id": "text-embedding-3-small", + "name": "text-embedding-3-small", + "display_name": "text-embedding-3-small", "modalities": { "input": [ - "audio" + "text" ] }, "limit": { @@ -254145,14 +254812,15 @@ "supported": false }, "cost": { - "input": 30, - "output": 30 - } + "input": 0.02, + "output": 0.02 + }, + "type": "embedding" }, { - "id": "text-embedding-v1", - "name": "text-embedding-v1", - "display_name": "text-embedding-v1", + "id": "text-embedding-3-large", + "name": "text-embedding-3-large", + "display_name": "text-embedding-3-large", "modalities": { "input": [ "text" @@ -254167,20 +254835,15 @@ "supported": false }, "cost": { - "input": 0.1, - "output": 0.1 + "input": 0.13, + "output": 0.13 }, "type": "embedding" }, { - "id": "text-embedding-ada-002", - "name": "text-embedding-ada-002", - "display_name": "text-embedding-ada-002", - "modalities": { - "input": [ - "text" - ] - }, + "id": "text-moderation-stable", + "name": "text-moderation-stable", + "display_name": "text-moderation-stable", "limit": { "context": 8192, "output": 8192 @@ -254190,20 +254853,15 @@ "supported": false }, "cost": { - "input": 0.1, - "output": 0.1 + "input": 0.2, + "output": 0.2 }, - "type": "embedding" + "type": "chat" }, { - "id": "text-embedding-3-small", - "name": "text-embedding-3-small", - "display_name": "text-embedding-3-small", - "modalities": { - "input": [ - "text" - ] - }, + "id": "text-davinci-edit-001", + "name": "text-davinci-edit-001", + "display_name": "text-davinci-edit-001", "limit": { "context": 8192, "output": 8192 @@ -254213,10 +254871,10 @@ "supported": false }, "cost": { - "input": 0.02, - "output": 0.02 + "input": 20, + "output": 20 }, - "type": "embedding" + "type": "chat" }, { "id": "whisper-1", @@ -254288,14 +254946,9 @@ "type": "chat" }, { - "id": "text-embedding-3-large", - "name": "text-embedding-3-large", - "display_name": "text-embedding-3-large", - "modalities": { - "input": [ - "text" - ] - }, + "id": "text-davinci-003", + "name": "text-davinci-003", + "display_name": "text-davinci-003", "limit": { "context": 8192, "output": 8192 @@ -254305,20 +254958,15 @@ "supported": false }, "cost": { - "input": 0.13, - "output": 0.13 + "input": 20, + "output": 20 }, - "type": "embedding" + "type": "chat" }, { - "id": "tts-1-1106", - "name": "tts-1-1106", - "display_name": "tts-1-1106", - "modalities": { - "input": [ - "audio" - ] - }, + "id": "text-davinci-002", + "name": "text-davinci-002", + "display_name": "text-davinci-002", "limit": { "context": 8192, "output": 8192 @@ -254328,14 +254976,15 @@ "supported": false }, "cost": { - "input": 15, - "output": 15 - } + "input": 20, + "output": 20 + }, + "type": "chat" }, { - "id": "text-davinci-edit-001", - "name": "text-davinci-edit-001", - "display_name": "text-davinci-edit-001", + "id": "text-curie-001", + "name": "text-curie-001", + "display_name": "text-curie-001", "limit": { "context": 8192, "output": 8192 @@ -254345,15 +254994,15 @@ "supported": false }, "cost": { - "input": 20, - "output": 20 + "input": 2, + "output": 2 }, "type": "chat" }, { - "id": "text-davinci-003", - "name": "text-davinci-003", - "display_name": "text-davinci-003", + "id": "text-babbage-001", + "name": "text-babbage-001", + "display_name": "text-babbage-001", "limit": { "context": 8192, "output": 8192 @@ -254363,15 +255012,15 @@ "supported": false }, "cost": { - "input": 20, - "output": 20 + "input": 0.5, + "output": 0.5 }, "type": "chat" }, { - "id": "text-davinci-002", - "name": "text-davinci-002", - "display_name": "text-davinci-002", + "id": "text-ada-001", + "name": "text-ada-001", + "display_name": "text-ada-001", "limit": { "context": 8192, "output": 8192 @@ -254381,15 +255030,15 @@ "supported": false }, "cost": { - "input": 20, - "output": 20 + "input": 0.4, + "output": 0.4 }, "type": "chat" }, { - "id": "text-curie-001", - "name": "text-curie-001", - "display_name": "text-curie-001", + "id": "text-moderation-latest", + "name": "text-moderation-latest", + "display_name": "text-moderation-latest", "limit": { "context": 8192, "output": 8192 @@ -254399,8 +255048,8 @@ "supported": false }, "cost": { - "input": 2, - "output": 2 + "input": 0.2, + "output": 0.2 }, "type": "chat" }, @@ -254513,9 +255162,14 @@ "type": "chat" }, { - "id": "text-babbage-001", - "name": "text-babbage-001", - "display_name": "text-babbage-001", + "id": "text-embedding-v1", + "name": "text-embedding-v1", + "display_name": "text-embedding-v1", + "modalities": { + "input": [ + "text" + ] + }, "limit": { "context": 8192, "output": 8192 @@ -254525,10 +255179,10 @@ "supported": false }, "cost": { - "input": 0.5, - "output": 0.5 + "input": 0.1, + "output": 0.1 }, - "type": "chat" + "type": "embedding" }, { "id": "meta-llama-3-70b", @@ -257185,8 +257839,8 @@ ] }, "limit": { - "context": 131072, - "output": 131072 + "context": 110000, + "output": 110000 }, "tool_call": true, "reasoning": { @@ -257782,8 +258436,8 @@ ] }, "limit": { - "context": 131072, - "output": 131072 + "context": 80000, + "output": 80000 }, "tool_call": false, "reasoning": { @@ -257926,6 +258580,33 @@ }, "type": "imageGeneration" }, + { + "id": "meta/muse-spark-1.1", + "name": "Meta: Muse Spark 1.1", + "display_name": "Meta: Muse Spark 1.1", + "modalities": { + "input": [ + "text", + "image", + "video", + "audio" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 1048576, + "output": 1048576 + }, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "attachment": true, + "type": "imageGeneration" + }, { "id": "microsoft/phi-4", "name": "Microsoft: Phi 4", @@ -258139,7 +258820,7 @@ ] }, "limit": { - "context": 204800, + "context": 196608, "output": 131072 }, "temperature": true, @@ -258821,6 +259502,35 @@ }, "type": "imageGeneration" }, + { + "id": "moonshotai/kimi-k3", + "name": "MoonshotAI: Kimi K3", + "display_name": "MoonshotAI: Kimi K3", + "modalities": { + "input": [ + "text", + "image" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 1048576, + "output": 1048576 + }, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true + } + }, + "type": "imageGeneration" + }, { "id": "morph/morph-v3-fast", "name": "Morph: Morph V3 Fast", @@ -264548,6 +265258,36 @@ }, "type": "chat" }, + { + "id": "moonshotai/kimi-k3", + "name": "Kimi K3", + "display_name": "Kimi K3", + "modalities": { + "input": [ + "text", + "image", + "video" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 1048576, + "output": 1048576 + }, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true + } + }, + "type": "chat" + }, { "id": "gpt-5.6-terra", "name": "gpt-5.6-terra", @@ -272599,9 +273339,9 @@ "release_date": "2026-06-12", "last_updated": "2026-06-12", "cost": { - "input": 0.719, - "output": 3.49, - "cache_read": 0.149, + "input": 0.75, + "output": 3.5, + "cache_read": 0.16, "cache_write": 0 }, "type": "chat" @@ -272652,6 +273392,46 @@ }, "type": "chat" }, + { + "id": "moonshotai/kimi-k3", + "name": "MoonshotAI: Kimi K3", + "display_name": "MoonshotAI: Kimi K3", + "modalities": { + "input": [ + "text", + "image", + "pdf" + ], + "output": [ + "text" + ] + }, + "limit": { + "context": 1000000, + "output": 131072 + }, + "temperature": true, + "tool_call": true, + "reasoning": { + "supported": true, + "default": true + }, + "extra_capabilities": { + "reasoning": { + "supported": true + } + }, + "attachment": true, + "open_weights": false, + "release_date": "2026-07-16", + "last_updated": "2026-07-16", + "cost": { + "input": 3, + "output": 15, + "cache_read": 0.3 + }, + "type": "chat" + }, { "id": "openai/chat-latest", "name": "OpenAI: Chat Latest (GPT-5.5 Instant)", @@ -274976,9 +275756,9 @@ "release_date": "2026-05-29", "last_updated": "2026-05-29", "cost": { - "input": 0.2, - "output": 1.15, - "cache_read": 0.04, + "input": 0.19, + "output": 1.14, + "cache_read": 0.03, "cache_write": 0 }, "type": "chat" @@ -275013,9 +275793,9 @@ "release_date": "2026-05-29", "last_updated": "2026-05-29", "cost": { - "input": 0.2, - "output": 1.15, - "cache_read": 0.04, + "input": 0.19, + "output": 1.14, + "cache_read": 0.03, "cache_write": 0 }, "type": "chat" diff --git a/src/main/agent/deepchat/runtime/dispatch.ts b/src/main/agent/deepchat/runtime/dispatch.ts index 9d41f7cf85..efc2d715ee 100644 --- a/src/main/agent/deepchat/runtime/dispatch.ts +++ b/src/main/agent/deepchat/runtime/dispatch.ts @@ -1345,6 +1345,7 @@ async function runToolCall(params: { } const enabledMcpServerIds = controls?.getEnabledMcpServerIds?.() const result = await toolExecution.execute(toolCall, { + runId: io.requestId, onProgress: applyProgressUpdate, signal: io.abortSignal, permissionMode: toolPermissionMode, diff --git a/src/main/desktop/browser/BrowserProfileImportService.ts b/src/main/desktop/browser/BrowserProfileImportService.ts new file mode 100644 index 0000000000..f17a4ec2ee --- /dev/null +++ b/src/main/desktop/browser/BrowserProfileImportService.ts @@ -0,0 +1,484 @@ +import { createDecipheriv, createHash, pbkdf2Sync } from 'crypto' +import { execFile } from 'child_process' +import { chmod, mkdtemp, readFile, readdir, rm, stat } from 'fs/promises' +import { homedir, tmpdir } from 'os' +import { join } from 'path' +import { promisify } from 'util' +import type { CookiesSetDetails, Session } from 'electron' +import Database from 'better-sqlite3-multiple-ciphers' +import { nanoid } from 'nanoid' +import type { + BrowserImportApplyResult, + BrowserImportPreview, + BrowserImportProfile, + BrowserImportScanResult +} from '@shared/types/browser' + +const execFileAsync = promisify(execFile) +const STAGE_TTL_MS = 5 * 60 * 1000 +const CHROME_EPOCH_OFFSET_SECONDS = 11_644_473_600 + +type BrowserSource = { + id: BrowserImportProfile['browser'] + name: string + userDataDirectory: string + keychainService: string +} + +const BROWSER_SOURCES: BrowserSource[] = [ + { + id: 'chrome', + name: 'Google Chrome', + userDataDirectory: join(homedir(), 'Library', 'Application Support', 'Google', 'Chrome'), + keychainService: 'Chrome Safe Storage' + }, + { + id: 'arc', + name: 'Arc', + userDataDirectory: join(homedir(), 'Library', 'Application Support', 'Arc', 'User Data'), + keychainService: 'Arc Safe Storage' + } +] + +type ChromeCookieRow = { + host_key: string + top_frame_site_key: string + name: string + value: string + encrypted_value: Buffer + path: string + expires_utc: number + is_secure: number + is_httponly: number + samesite: number +} + +type DiscoveredProfile = BrowserImportProfile & { + source: BrowserSource + directoryName: string + cookiePath: string +} + +type StagedImport = { + createdAt: number + profile: DiscoveredProfile + sourceSize: number + sourceMtimeMs: number + cookies: CookiesSetDetails[] + skippedExpired: number + skippedPartitioned: number +} + +export class BrowserProfileImportService { + private readonly stagedImports = new Map() + private readonly stageExpiryTimers = new Map() + private applying = false + + constructor( + private readonly getTargetSession: () => Session, + private readonly getTargetUnpartitionedCookies: () => Promise, + private readonly platform: NodeJS.Platform = process.platform + ) {} + + async scan(): Promise { + if (this.platform !== 'darwin') { + return { + platformSupported: false, + profiles: [], + reason: 'platform_unsupported' + } + } + + const profiles = await this.discoverChromeProfiles() + return { + platformSupported: true, + profiles: profiles.map(this.toPublicProfile), + ...(profiles.length === 0 ? { reason: 'browser_not_found' as const } : {}) + } + } + + async preview(profileId: string): Promise { + if (this.platform !== 'darwin') { + throw new Error('browser_import_platform_unsupported') + } + this.removeExpiredStages() + const profile = (await this.discoverChromeProfiles()).find((item) => item.id === profileId) + if (!profile || !profile.supported) { + throw new Error('browser_import_profile_unavailable') + } + + const sourceStat = await stat(profile.cookiePath) + const password = await this.readSafeStoragePassword(profile.source.keychainService) + const temporaryDirectory = await mkdtemp(join(tmpdir(), 'deepchat-browser-import-')) + await chmod(temporaryDirectory, 0o700) + const snapshotPath = join(temporaryDirectory, 'Cookies') + + try { + const sourceDatabase = new Database(profile.cookiePath, { + readonly: true, + fileMustExist: true + }) + try { + await sourceDatabase.backup(snapshotPath) + } finally { + sourceDatabase.close() + } + + const decoded = this.readCookieSnapshot(snapshotPath, password) + const token = nanoid(24) + this.stagedImports.set(token, { + createdAt: Date.now(), + profile, + sourceSize: sourceStat.size, + sourceMtimeMs: sourceStat.mtimeMs, + cookies: decoded.cookies, + skippedExpired: decoded.skippedExpired, + skippedPartitioned: decoded.skippedPartitioned + }) + const expiryTimer = setTimeout(() => this.deleteStage(token), STAGE_TTL_MS) + expiryTimer.unref() + this.stageExpiryTimers.set(token, expiryTimer) + + return { + token, + profile: this.toPublicProfile(profile), + cookieCount: decoded.cookies.length, + skippedExpired: decoded.skippedExpired, + skippedPartitioned: decoded.skippedPartitioned + } + } finally { + password.fill(0) + await rm(temporaryDirectory, { recursive: true, force: true }) + } + } + + async apply(token: string): Promise { + if (this.applying) { + throw new Error('browser_import_already_running') + } + + this.removeExpiredStages() + const staged = this.stagedImports.get(token) + if (!staged) { + throw new Error('browser_import_preview_expired') + } + + const sourceStat = await stat(staged.profile.cookiePath) + if (sourceStat.size !== staged.sourceSize || sourceStat.mtimeMs !== staged.sourceMtimeMs) { + this.deleteStage(token) + throw new Error('browser_import_source_changed') + } + + this.applying = true + try { + const target = this.getTargetSession() + const rollbackCookies = await this.getTargetUnpartitionedCookies() + + try { + await this.removeCookies(target, rollbackCookies) + await this.setCookies(target, staged.cookies) + await target.cookies.flushStore() + await this.verifyCookies(staged.cookies) + } catch (error) { + await this.removeCookies(target, await this.getTargetUnpartitionedCookies()) + await this.setCookies(target, rollbackCookies) + await target.cookies.flushStore() + throw error + } + } finally { + this.deleteStage(token) + this.applying = false + } + + return { + importedCookies: staged.cookies.length, + skippedExpired: staged.skippedExpired, + skippedPartitioned: staged.skippedPartitioned, + syncedAt: Date.now() + } + } + + private async discoverChromeProfiles(): Promise { + const profiles: DiscoveredProfile[] = [] + for (const source of BROWSER_SOURCES) { + let entries: Array<{ name: string; isDirectory(): boolean }> + try { + entries = await readdir(source.userDataDirectory, { withFileTypes: true }) + } catch { + continue + } + + const profileNames = await this.readProfileNames(source) + for (const entry of entries) { + if (!entry.isDirectory() || !/^(Default|Profile \d+)$/.test(entry.name)) { + continue + } + + const profileDirectory = join(source.userDataDirectory, entry.name) + const networkCookiePath = join(profileDirectory, 'Network', 'Cookies') + const legacyCookiePath = join(profileDirectory, 'Cookies') + const cookiePath = await this.firstExistingPath([networkCookiePath, legacyCookiePath]) + if (!cookiePath) { + continue + } + + profiles.push({ + id: `${source.id}:${entry.name}`, + browser: source.id, + browserName: source.name, + profileName: profileNames.get(entry.name) || entry.name, + supported: true, + source, + directoryName: entry.name, + cookiePath + }) + } + } + + return profiles.sort((left, right) => left.id.localeCompare(right.id)) + } + + private async readProfileNames(source: BrowserSource): Promise> { + const result = new Map() + try { + const contents = await readFile(join(source.userDataDirectory, 'Local State'), 'utf8') + const parsed = JSON.parse(contents) as { + profile?: { info_cache?: Record } + } + for (const [directoryName, value] of Object.entries(parsed.profile?.info_cache ?? {})) { + if (typeof value.name === 'string' && value.name.trim()) { + result.set(directoryName, value.name.trim()) + } + } + } catch { + // Directory names remain usable when Local State is absent or malformed. + } + return result + } + + private async firstExistingPath(paths: string[]): Promise { + for (const path of paths) { + try { + if ((await stat(path)).isFile()) { + return path + } + } catch { + // Continue to the next known path. + } + } + return null + } + + private async readSafeStoragePassword(keychainService: string): Promise { + try { + const { stdout } = await execFileAsync('/usr/bin/security', [ + 'find-generic-password', + '-w', + '-s', + keychainService + ]) + const password = stdout.trim() + if (!password) { + throw new Error('empty keychain result') + } + return Buffer.from(password, 'utf8') + } catch { + throw new Error('browser_import_key_access_denied') + } + } + + private readCookieSnapshot( + snapshotPath: string, + password: Buffer + ): { + cookies: CookiesSetDetails[] + skippedExpired: number + skippedPartitioned: number + } { + const database = new Database(snapshotPath, { readonly: true, fileMustExist: true }) + try { + const integrity = database.pragma('integrity_check', { simple: true }) + if (integrity !== 'ok') { + throw new Error('browser_import_source_snapshot_invalid') + } + + const schemaVersion = Number( + ( + database.prepare("SELECT value FROM meta WHERE key = 'version'").get() as + | { value?: unknown } + | undefined + )?.value ?? 0 + ) + const cookieColumns = database.prepare('PRAGMA table_info(cookies)').all() as Array<{ + name: string + }> + const hasTopFrameSiteKey = cookieColumns.some( + (column) => column.name === 'top_frame_site_key' + ) + const rows = database + .prepare( + `SELECT host_key, ${hasTopFrameSiteKey ? 'top_frame_site_key' : "'' AS top_frame_site_key"}, name, value, + CAST(encrypted_value AS BLOB) AS encrypted_value, + path, expires_utc, is_secure, is_httponly, samesite + FROM cookies` + ) + .all() as ChromeCookieRow[] + + const key = pbkdf2Sync(password, 'saltysalt', 1003, 16, 'sha1') + const cookies: CookiesSetDetails[] = [] + let skippedExpired = 0 + let skippedPartitioned = 0 + const nowSeconds = Date.now() / 1000 + + try { + for (const row of rows) { + const expirationDate = this.chromeTimeToUnixSeconds(row.expires_utc) + if (expirationDate !== undefined && expirationDate <= nowSeconds) { + skippedExpired += 1 + continue + } + if (row.top_frame_site_key) { + skippedPartitioned += 1 + continue + } + + const value = row.value || this.decryptChromeCookie(row, key, schemaVersion) + cookies.push(this.rowToCookieDetails(row, value, expirationDate)) + } + } finally { + key.fill(0) + } + + return { cookies, skippedExpired, skippedPartitioned } + } finally { + database.close() + } + } + + private decryptChromeCookie(row: ChromeCookieRow, key: Buffer, schemaVersion: number): string { + if (!Buffer.isBuffer(row.encrypted_value) || row.encrypted_value.length <= 3) { + return '' + } + if (row.encrypted_value.subarray(0, 3).toString('ascii') !== 'v10') { + throw new Error('browser_import_encryption_unsupported') + } + + const decipher = createDecipheriv('aes-128-cbc', key, Buffer.alloc(16, 0x20)) + const plaintext = Buffer.concat([ + decipher.update(row.encrypted_value.subarray(3)), + decipher.final() + ]) + const value = schemaVersion >= 24 ? this.removeHostDigest(row.host_key, plaintext) : plaintext + return value.toString('utf8') + } + + private removeHostDigest(host: string, plaintext: Buffer): Buffer { + if (plaintext.length < 32) { + throw new Error('browser_import_cookie_digest_missing') + } + const expected = createHash('sha256').update(host).digest() + if (!plaintext.subarray(0, 32).equals(expected)) { + throw new Error('browser_import_cookie_digest_invalid') + } + return plaintext.subarray(32) + } + + private rowToCookieDetails( + row: ChromeCookieRow, + value: string, + expirationDate?: number + ): CookiesSetDetails { + const hostname = row.host_key.replace(/^\./, '') + const secure = Boolean(row.is_secure) + return { + url: `${secure ? 'https' : 'http'}://${hostname}${row.path || '/'}`, + name: row.name, + value, + path: row.path || '/', + secure, + httpOnly: Boolean(row.is_httponly), + sameSite: this.toElectronSameSite(row.samesite), + ...(row.host_key.startsWith('.') ? { domain: row.host_key } : {}), + ...(expirationDate === undefined ? {} : { expirationDate }) + } + } + + private chromeTimeToUnixSeconds(value: number): number | undefined { + if (!Number.isFinite(value) || value <= 0) { + return undefined + } + return value / 1_000_000 - CHROME_EPOCH_OFFSET_SECONDS + } + + private toElectronSameSite(value: number): CookiesSetDetails['sameSite'] { + if (value === 0) return 'no_restriction' + if (value === 1) return 'lax' + if (value === 2) return 'strict' + return 'unspecified' + } + + private async setCookies(target: Session, cookies: CookiesSetDetails[]): Promise { + const batchSize = 50 + for (let index = 0; index < cookies.length; index += batchSize) { + await Promise.all( + cookies.slice(index, index + batchSize).map((cookie) => target.cookies.set(cookie)) + ) + } + } + + private async removeCookies(target: Session, cookies: CookiesSetDetails[]): Promise { + await this.setCookies( + target, + cookies.map((cookie) => ({ + ...cookie, + value: '', + expirationDate: 1 + })) + ) + } + + private async verifyCookies(expected: CookiesSetDetails[]): Promise { + const actual = await this.getTargetUnpartitionedCookies() + const actualValues = new Map( + actual.map((cookie) => [ + `${cookie.domain ?? new URL(cookie.url).hostname}\u0000${cookie.path ?? '/'}\u0000${cookie.name}`, + cookie.value + ]) + ) + + for (const cookie of expected) { + const domain = cookie.domain ?? new URL(cookie.url).hostname + const identity = `${domain}\u0000${cookie.path ?? '/'}\u0000${cookie.name}` + if (actualValues.get(identity) !== cookie.value) { + throw new Error('browser_import_verification_failed') + } + } + } + + private removeExpiredStages(): void { + const now = Date.now() + for (const [token, staged] of this.stagedImports) { + if (now - staged.createdAt > STAGE_TTL_MS) { + this.deleteStage(token) + } + } + } + + private deleteStage(token: string): void { + this.stagedImports.delete(token) + const timer = this.stageExpiryTimers.get(token) + if (timer) clearTimeout(timer) + this.stageExpiryTimers.delete(token) + } + + private toPublicProfile(profile: DiscoveredProfile): BrowserImportProfile { + return { + id: profile.id, + browser: profile.browser, + browserName: profile.browserName, + profileName: profile.profileName, + supported: profile.supported, + ...(profile.reason ? { reason: profile.reason } : {}) + } + } +} diff --git a/src/main/desktop/browser/YoBrowserPresenter.ts b/src/main/desktop/browser/YoBrowserPresenter.ts index fb2eff808c..e11f50beac 100644 --- a/src/main/desktop/browser/YoBrowserPresenter.ts +++ b/src/main/desktop/browser/YoBrowserPresenter.ts @@ -1,8 +1,9 @@ -import { BrowserWindow, WebContents, WebContentsView } from 'electron' +import { BaseWindow, BrowserWindow, WebContents, WebContentsView } from 'electron' import type { Rectangle } from 'electron' import { is } from '@electron-toolkit/utils' import { nanoid } from 'nanoid' -import type { DeepchatEventPublisher } from '@shared/contracts/events' +import { DEEPCHAT_EVENT_CHANNEL } from '@shared/contracts/channels' +import { createDeepchatEventEnvelope, type DeepchatEventPublisher } from '@shared/contracts/events' import logger from '@shared/logger' import { BrowserPageStatus, @@ -19,10 +20,15 @@ import { import type { DownloadInfo } from '@shared/types/browser' import type { IWindowPresenter, IYoBrowserPresenter } from '@shared/types/desktop' import { BrowserTab as BrowserPage } from './BrowserTab' +import { BrowserProfileImportService } from './BrowserProfileImportService' import { CDPManager } from './CDPManager' import { DownloadManager } from './DownloadManager' import { ScreenshotManager } from './ScreenshotManager' -import { clearYoBrowserSessionData, getYoBrowserSession } from './yoBrowserSession' +import { + clearYoBrowserSessionData, + getYoBrowserSession, + getYoBrowserUnpartitionedCookies +} from './yoBrowserSession' import { YoBrowserOverlayWindow } from './YoBrowserOverlayWindow' import { YoBrowserToolHandler } from './YoBrowserToolHandler' @@ -46,6 +52,16 @@ type SessionBrowserState = { visible: boolean attachedWindowId: number | null lastBounds: Rectangle | null + owner: 'agent' | 'user' + agentRunId?: string + previewHost: BaseWindow | null + previewMode: 'capturing' | 'rendering' | 'stopped' + previewTargetWindowId: number | null + previewTimer: ReturnType | null + previewCapture: Promise | null + previewEpoch: number + previewSequence: number + previewBurstUntil: number } type HostWindowListeners = { @@ -58,12 +74,23 @@ type HostWindowListeners = { closed: () => void } +const PREVIEW_VIEWPORT = { width: 1280, height: 800 } +const PREVIEW_FRAME = { width: 480, height: 300 } +const PREVIEW_ACTIVE_INTERVAL_MS = 250 +const PREVIEW_IDLE_INTERVAL_MS = 1000 +const PREVIEW_MAX_BYTES = 512 * 1024 + export class YoBrowserPresenter implements IYoBrowserPresenter { private readonly sessionBrowsers = new Map() private readonly hostWindowListeners = new Map() private readonly cdpManager = new CDPManager() private readonly screenshotManager = new ScreenshotManager(this.cdpManager) private readonly downloadManager = new DownloadManager() + private readonly profileImportService = new BrowserProfileImportService( + getYoBrowserSession, + getYoBrowserUnpartitionedCookies + ) + private browserDataMutationActive = false private readonly windowPresenter: IWindowPresenter readonly toolHandler: YoBrowserToolHandler @@ -88,7 +115,8 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { url: string, timeoutMs?: number, hostWindowId?: number, - activitySource?: YoBrowserActivitySource + activitySource?: YoBrowserActivitySource, + agentRunId?: string ): Promise { const normalizedSessionId = sessionId.trim() if (!normalizedSessionId) { @@ -104,13 +132,25 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { } const state = this.ensureSessionBrowserState(normalizedSessionId) + this.updateOwner(state, activitySource, agentRunId) + if (activitySource === 'agent' && !state.visible) { + this.ensurePreviewHost(state) + } else if (activitySource !== 'agent' && state.previewHost) { + await this.releasePreviewHost(state) + } this.logLifecycle('open requested', { sessionId: normalizedSessionId, windowId: resolvedHostWindowId, url }) - this.emitOpenRequested(normalizedSessionId, resolvedHostWindowId, url) + this.emitOpenRequested( + normalizedSessionId, + resolvedHostWindowId, + url, + activitySource ?? 'user', + state.agentRunId + ) const navigate = () => state.page.navigateUntilDomReady(url, timeoutMs ?? 30000) if (activitySource === 'agent') { @@ -139,6 +179,8 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return false } + await this.releasePreviewHost(state) + this.detachOtherSessionBrowsers(hostWindowId, sessionId) if (state.attachedWindowId != null && state.attachedWindowId !== hostWindowId) { @@ -161,7 +203,6 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { this.attachHostWindowListeners(hostWindowId) state.attachedWindowId = hostWindowId state.updatedAt = Date.now() - this.preserveHostWebContentsFocus(hostWindow) this.emitWindowUpdated(sessionId) return true } @@ -177,11 +218,13 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return } - const normalizedBounds = this.normalizeBounds(bounds) + const hostWindow = BrowserWindow.fromId(hostWindowId) + const normalizedBounds = this.normalizeBounds(bounds, hostWindow) state.lastBounds = normalizedBounds state.updatedAt = Date.now() if (!visible || normalizedBounds.width <= 0 || normalizedBounds.height <= 0) { + state.view.setVisible(false) this.setSessionVisibility(state, false) return } @@ -194,10 +237,9 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { } state.view.setBounds(normalizedBounds) - const hostWindow = BrowserWindow.fromId(hostWindowId) + state.view.setVisible(true) if (hostWindow && !hostWindow.isDestroyed() && hostWindow.isFocused()) { await state.overlay.updateBounds(hostWindow, normalizedBounds, true) - this.preserveHostWebContentsFocus(hostWindow) } this.setSessionVisibility(state, true) } @@ -213,12 +255,62 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { this.setSessionVisibility(state, false) } + async setPreviewMode( + sessionId: string, + mode: 'capturing' | 'rendering' | 'stopped', + hostWindowId?: number, + runId?: string + ): Promise { + const state = this.sessionBrowsers.get(sessionId) + if (!state) { + return false + } + + if (mode === 'stopped') { + if (runId != null && state.agentRunId != null && runId !== state.agentRunId) { + return false + } + await this.releasePreviewHost(state) + return true + } + + if ( + state.owner !== 'agent' || + !state.agentRunId || + (runId != null && runId !== state.agentRunId) || + state.visible + ) { + return false + } + + if (mode === 'capturing') { + const targetWindow = hostWindowId == null ? null : BrowserWindow.fromId(hostWindowId) + if (!targetWindow || targetWindow.isDestroyed()) { + return false + } + } + + await this.stopPreviewCapture(state) + if (!this.ensurePreviewHost(state)) { + return false + } + state.previewMode = mode + state.previewTargetWindowId = hostWindowId ?? null + state.previewEpoch += 1 + + if (mode === 'capturing') { + this.schedulePreviewCapture(state, 0, state.previewEpoch) + } + return true + } + async destroySessionBrowser(sessionId: string): Promise { const state = this.sessionBrowsers.get(sessionId) if (!state) { return } + await this.releasePreviewHost(state) await this.detachSessionBrowser(sessionId) state.page.destroy() state.overlay.destroy() @@ -312,13 +404,26 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { sessionId: string, method: string, params?: Record, - activitySource?: YoBrowserActivitySource + activitySource?: YoBrowserActivitySource, + agentRunId?: string ): Promise { const state = this.sessionBrowsers.get(sessionId) if (!state) { throw new Error(`Session browser ${sessionId} is not initialized`) } + if (activitySource === 'agent') { + this.updateOwner(state, activitySource, agentRunId) + if (!state.visible) { + this.ensurePreviewHost(state) + } + const windowId = state.attachedWindowId ?? this.resolveHostWindowId() + if (windowId != null) { + this.emitOpenRequested(sessionId, windowId, state.page.url, 'agent', state.agentRunId) + } + this.emitWindowUpdated(sessionId) + } + const descriptor = this.describeCdpActivity(method, params) if (activitySource === 'agent' && descriptor) { return await this.runAgentActivity(sessionId, descriptor, () => @@ -338,12 +443,34 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { } async clearSandboxData(): Promise { - await clearYoBrowserSessionData() - for (const state of this.sessionBrowsers.values()) { - if (!state.page.contents.isDestroyed()) { - state.page.contents.reloadIgnoringCache() + await this.runBrowserDataMutation(async () => { + await clearYoBrowserSessionData() + for (const state of this.sessionBrowsers.values()) { + if (!state.page.contents.isDestroyed()) { + state.page.contents.reloadIgnoringCache() + } } - } + }) + } + + async scanImportSources() { + return await this.profileImportService.scan() + } + + async previewImport(profileId: string) { + return await this.profileImportService.preview(profileId) + } + + async applyImport(token: string) { + return await this.runBrowserDataMutation(async () => { + const result = await this.profileImportService.apply(token) + for (const state of this.sessionBrowsers.values()) { + if (!state.page.contents.isDestroyed()) { + state.page.contents.reloadIgnoringCache() + } + } + return result + }) } async shutdown(): Promise { @@ -380,7 +507,16 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { updatedAt: now, visible: false, attachedWindowId: null, - lastBounds: null + lastBounds: null, + owner: 'user', + previewHost: null, + previewMode: 'stopped', + previewTargetWindowId: null, + previewTimer: null, + previewCapture: null, + previewEpoch: 0, + previewSequence: 0, + previewBurstUntil: 0 } this.sessionBrowsers.set(sessionId, state) @@ -389,6 +525,18 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return state } + private async runBrowserDataMutation(mutation: () => Promise): Promise { + if (this.browserDataMutationActive) { + throw new Error('browser_data_mutation_in_progress') + } + this.browserDataMutationActive = true + try { + return await mutation() + } finally { + this.browserDataMutationActive = false + } + } + private setupPageListeners(state: SessionBrowserState, contents: WebContents): void { const sessionId = state.sessionId const getState = () => this.sessionBrowsers.get(sessionId) @@ -485,6 +633,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return } + this.disposePreviewHost(state) state.page.destroy() state.overlay.destroy() state.attachedWindowId = null @@ -606,6 +755,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { // Ignore already detached view. } } + state.view.setVisible(false) state.attachedWindowId = null state.overlay.hide() } @@ -668,7 +818,9 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { canGoBack: state.page.contents.navigationHistory.canGoBack(), canGoForward: state.page.contents.navigationHistory.canGoForward(), visible: state.visible, - loading: state.page.contents.isLoading() || state.page.status === BrowserPageStatus.Loading + loading: state.page.contents.isLoading() || state.page.status === BrowserPageStatus.Loading, + owner: state.owner, + ...(state.agentRunId ? { agentRunId: state.agentRunId } : {}) } } @@ -702,12 +854,22 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { void state.overlay.updateBounds(hostWindow, state.lastBounds, true) } - private normalizeBounds(bounds: Rectangle): Rectangle { + private normalizeBounds(bounds: Rectangle, hostWindow?: BrowserWindow | null): Rectangle { + const contentBounds = + hostWindow && !hostWindow.isDestroyed() && typeof hostWindow.getContentBounds === 'function' + ? hostWindow.getContentBounds() + : null + const x = Math.max(0, Math.round(bounds.x)) + const y = Math.max(0, Math.round(bounds.y)) + const maxWidth = contentBounds ? Math.max(0, contentBounds.width - x) : Number.MAX_SAFE_INTEGER + const maxHeight = contentBounds + ? Math.max(0, contentBounds.height - y) + : Number.MAX_SAFE_INTEGER return { - x: Math.max(0, Math.round(bounds.x)), - y: Math.max(0, Math.round(bounds.y)), - width: Math.max(0, Math.round(bounds.width)), - height: Math.max(0, Math.round(bounds.height)) + x, + y, + width: Math.min(maxWidth, Math.max(0, Math.round(bounds.width))), + height: Math.min(maxHeight, Math.max(0, Math.round(bounds.height))) } } @@ -732,11 +894,19 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { }) } - private emitOpenRequested(sessionId: string, windowId: number, url: string): void { + private emitOpenRequested( + sessionId: string, + windowId: number, + url: string, + source: 'agent' | 'user', + runId?: string + ): void { const payload = { sessionId, windowId, - url + url, + source, + ...(runId ? { runId } : {}) } this.publishEvent('browser.open.requested', { @@ -745,6 +915,21 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { }) } + private updateOwner( + state: SessionBrowserState, + activitySource?: YoBrowserActivitySource, + agentRunId?: string + ): void { + if (activitySource === 'agent') { + state.owner = 'agent' + state.agentRunId = agentRunId?.trim() || state.agentRunId || nanoid(12) + return + } + + state.owner = 'user' + state.agentRunId = undefined + } + private emitWindowUpdated(sessionId: string): void { const status = this.toStatus(this.sessionBrowsers.get(sessionId) ?? null) this.publishEvent('browser.status.changed', { @@ -812,6 +997,9 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { phase: YoBrowserActivityPayload['phase'] ): void { const state = this.sessionBrowsers.get(sessionId) ?? null + if (state) { + state.previewBurstUntil = Date.now() + 1500 + } const windowId = state?.attachedWindowId ?? this.resolveHostWindowId() ?? null const payload: YoBrowserActivityPayload = { id: activityId, @@ -1057,20 +1245,178 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return undefined } - private preserveHostWebContentsFocus(hostWindow: BrowserWindow): void { - if (hostWindow.isDestroyed() || hostWindow.webContents.isDestroyed()) { + private ensurePreviewHost(state: SessionBrowserState): boolean { + if (state.previewHost && !state.previewHost.isDestroyed()) { + return true + } + + let host: BaseWindow | null = null + try { + host = new BaseWindow({ + x: -10000, + y: -10000, + width: PREVIEW_VIEWPORT.width, + height: PREVIEW_VIEWPORT.height, + show: true, + opacity: 0, + focusable: false, + skipTaskbar: true, + hiddenInMissionControl: true, + frame: false, + transparent: true + }) + host.setIgnoreMouseEvents(true) + if (state.attachedWindowId != null) { + this.detachFromWindow(state, state.attachedWindowId) + this.setSessionVisibility(state, false) + } + host.contentView.addChildView(state.view) + state.view.setBounds({ x: 0, y: 0, ...PREVIEW_VIEWPORT }) + state.view.setVisible(true) + state.page.contents.setBackgroundThrottling(false) + state.previewHost = host + host.once('closed', () => { + if (state.previewHost === host) { + state.previewHost = null + state.previewMode = 'stopped' + state.previewTargetWindowId = null + state.previewEpoch += 1 + state.view.setVisible(false) + if (!state.page.contents.isDestroyed()) { + state.page.contents.setBackgroundThrottling(true) + } + } + }) + return true + } catch (error) { + if (host && !host.isDestroyed()) { + host.destroy() + } + logger.warn('[YoBrowser] Preview render host unavailable', { + sessionId: state.sessionId, + error: error instanceof Error ? error.message : String(error) + }) + return false + } + } + + private async stopPreviewCapture(state: SessionBrowserState): Promise { + state.previewEpoch += 1 + if (state.previewTimer) { + clearTimeout(state.previewTimer) + state.previewTimer = null + } + const capture = state.previewCapture + if (capture) { + let timeout: ReturnType | null = null + await Promise.race([ + capture.catch(() => undefined), + new Promise((resolve) => { + timeout = setTimeout(resolve, 500) + timeout.unref?.() + }) + ]) + if (timeout) { + clearTimeout(timeout) + } + } + } + + private async releasePreviewHost(state: SessionBrowserState): Promise { + await this.stopPreviewCapture(state) + this.disposePreviewHost(state) + } + + private disposePreviewHost(state: SessionBrowserState): void { + state.previewEpoch += 1 + if (state.previewTimer) { + clearTimeout(state.previewTimer) + state.previewTimer = null + } + state.previewMode = 'stopped' + state.previewTargetWindowId = null + const host = state.previewHost + state.previewHost = null + if (!host || host.isDestroyed()) { return } + try { + host.contentView.removeChildView(state.view) + } catch { + // Ignore already detached views during shutdown. + } + state.view.setVisible(false) + if (!state.page.contents.isDestroyed()) { + state.page.contents.setBackgroundThrottling(true) + } + host.destroy() + } - if (!hostWindow.isFocused()) { + private schedulePreviewCapture(state: SessionBrowserState, delayMs: number, epoch: number): void { + if (state.previewMode !== 'capturing' || state.previewEpoch !== epoch) { return } + state.previewTimer = setTimeout(() => { + state.previewTimer = null + const capture = this.capturePreviewFrame(state, epoch) + state.previewCapture = capture + void capture.finally(() => { + if (state.previewCapture === capture) { + state.previewCapture = null + } + }) + }, delayMs) + state.previewTimer.unref?.() + } - queueMicrotask(() => { - if (hostWindow.isDestroyed() || hostWindow.webContents.isDestroyed()) { + private async capturePreviewFrame(state: SessionBrowserState, epoch: number): Promise { + if ( + state.previewMode !== 'capturing' || + state.previewEpoch !== epoch || + state.previewTargetWindowId == null || + !state.agentRunId || + state.page.contents.isDestroyed() + ) { + return + } + + try { + const image = await state.page.contents.capturePage( + { x: 0, y: 0, ...PREVIEW_VIEWPORT }, + { stayHidden: true } + ) + if (state.previewMode !== 'capturing' || state.previewEpoch !== epoch) { return } - hostWindow.webContents.focus() - }) + const data = image.resize(PREVIEW_FRAME).toJPEG(72) + if (data.byteLength <= PREVIEW_MAX_BYTES) { + state.previewSequence += 1 + this.windowPresenter.sendToWindow( + state.previewTargetWindowId, + DEEPCHAT_EVENT_CHANNEL, + createDeepchatEventEnvelope('browser.preview.frame', { + sessionId: state.sessionId, + runId: state.agentRunId, + sequence: state.previewSequence, + ...PREVIEW_FRAME, + mimeType: 'image/jpeg', + data, + timestamp: Date.now() + }) + ) + } + } catch (error) { + logger.warn('[YoBrowser] Preview capture failed', { + sessionId: state.sessionId, + error: error instanceof Error ? error.message : String(error) + }) + } + + const active = state.page.contents.isLoading() || Date.now() < state.previewBurstUntil + this.schedulePreviewCapture( + state, + active ? PREVIEW_ACTIVE_INTERVAL_MS : PREVIEW_IDLE_INTERVAL_MS, + epoch + ) } } diff --git a/src/main/desktop/browser/YoBrowserToolHandler.ts b/src/main/desktop/browser/YoBrowserToolHandler.ts index ec241265b1..26133d3e37 100644 --- a/src/main/desktop/browser/YoBrowserToolHandler.ts +++ b/src/main/desktop/browser/YoBrowserToolHandler.ts @@ -22,7 +22,8 @@ export class YoBrowserToolHandler { async callTool( toolName: string, args: Record, - conversationId?: string + conversationId?: string, + runId?: string ): Promise { try { const sessionId = conversationId?.trim() @@ -39,7 +40,9 @@ export class YoBrowserToolHandler { throw new Error('url is required') } return JSON.stringify( - await this.presenter.loadUrl(sessionId, url, undefined, undefined, 'agent') + runId + ? await this.presenter.loadUrl(sessionId, url, undefined, undefined, 'agent', runId) + : await this.presenter.loadUrl(sessionId, url, undefined, undefined, 'agent') ) } case 'cdp_send': { @@ -50,18 +53,15 @@ export class YoBrowserToolHandler { const status = await this.presenter.getBrowserStatus(sessionId) const page = status.page - if ( - !status.initialized || - !status.visible || - !page || - page.status === BrowserPageStatus.Closed - ) { + if (!status.initialized || !page || page.status === BrowserPageStatus.Closed) { throw await this.createUnavailableError(sessionId, method, status) } try { const params = this.normalizeCdpParams(args.params) - const response = await this.presenter.sendCdpCommand(sessionId, method, params, 'agent') + const response = runId + ? await this.presenter.sendCdpCommand(sessionId, method, params, 'agent', runId) + : await this.presenter.sendCdpCommand(sessionId, method, params, 'agent') return JSON.stringify(response ?? {}) } catch (error) { if (error instanceof Error && error.name === 'YoBrowserNotReadyError') { diff --git a/src/main/desktop/browser/yoBrowserSession.ts b/src/main/desktop/browser/yoBrowserSession.ts index 472893b41f..a73674ac6a 100644 --- a/src/main/desktop/browser/yoBrowserSession.ts +++ b/src/main/desktop/browser/yoBrowserSession.ts @@ -1,17 +1,18 @@ -import { session, type Session } from 'electron' +import { session, WebContentsView, type CookiesSetDetails, type Session } from 'electron' export const YO_BROWSER_PARTITION = 'persist:yo-browser' let cachedSession: Session | null = null function buildYoBrowserUserAgent(): string { + const chromeVersion = process.versions.chrome ?? '0.0.0.0' if (process.platform === 'darwin') { - return 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36' + return `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${chromeVersion} Safari/537.36` } if (process.platform === 'win32') { - return 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36' + return `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${chromeVersion} Safari/537.36` } - return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36' + return `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${chromeVersion} Safari/537.36` } function configureYoBrowserSession(target: Session): void { @@ -42,6 +43,67 @@ export function getYoBrowserSession(): Session { return cachedSession } +type DevToolsCookie = { + name: string + value: string + domain: string + path: string + expires: number | null + httpOnly: boolean + secure: boolean + session: boolean + sameSite?: 'Strict' | 'Lax' | 'None' + partitionKey?: unknown + partitionKeyOpaque?: boolean +} + +function toElectronCookie(cookie: DevToolsCookie): CookiesSetDetails { + const path = cookie.path || '/' + return { + url: `${cookie.secure ? 'https' : 'http'}://${cookie.domain.replace(/^\./, '')}${path}`, + name: cookie.name, + value: cookie.value, + ...(cookie.domain.startsWith('.') ? { domain: cookie.domain } : {}), + path, + secure: cookie.secure, + httpOnly: cookie.httpOnly, + sameSite: + cookie.sameSite === 'Strict' + ? 'strict' + : cookie.sameSite === 'Lax' + ? 'lax' + : cookie.sameSite === 'None' + ? 'no_restriction' + : 'unspecified', + ...(cookie.session || cookie.expires === null || cookie.expires < 0 + ? {} + : { expirationDate: cookie.expires }) + } +} + +export async function getYoBrowserUnpartitionedCookies(): Promise { + const view = new WebContentsView({ + webPreferences: { + sandbox: true, + session: getYoBrowserSession() + } + }) + const debugSession = view.webContents.debugger + + try { + debugSession.attach('1.3') + const response = (await debugSession.sendCommand('Storage.getCookies')) as { + cookies?: DevToolsCookie[] + } + return (response.cookies ?? []) + .filter((cookie) => !cookie.partitionKey && !cookie.partitionKeyOpaque) + .map(toElectronCookie) + } finally { + if (debugSession.isAttached()) debugSession.detach() + view.webContents.close() + } +} + export async function clearYoBrowserSessionData(): Promise { const targetSession = getYoBrowserSession() await Promise.all([ diff --git a/src/main/desktop/routes.ts b/src/main/desktop/routes.ts index e8da48a212..c4c42b4aed 100644 --- a/src/main/desktop/routes.ts +++ b/src/main/desktop/routes.ts @@ -9,6 +9,7 @@ import type { DialogServicePort } from '@shared/types/dialog' import type { DesktopSettings } from './settings' import { browserAttachCurrentWindowRoute, + browserApplyImportRoute, browserClearSandboxDataRoute, browserDestroyRoute, browserDetachRoute, @@ -16,7 +17,10 @@ import { browserGoBackRoute, browserGoForwardRoute, browserLoadUrlRoute, + browserPreviewImportRoute, browserReloadRoute, + browserScanImportSourcesRoute, + browserSetPreviewModeRoute, browserUpdateCurrentWindowBoundsRoute, configGetFloatingButtonRoute, configGetLanguageRoute, @@ -384,6 +388,20 @@ export function createDesktopRoutes(deps: { return browserDetachRoute.output.parse({ detached: true }) } ], + [ + browserSetPreviewModeRoute.name, + async (rawInput, context) => { + const input = browserSetPreviewModeRoute.input.parse(rawInput) + return browserSetPreviewModeRoute.output.parse({ + updated: await browserPresenter.setPreviewMode( + input.sessionId, + input.mode, + context.windowId ?? undefined, + input.runId + ) + }) + } + ], [ browserDestroyRoute.name, async (rawInput) => { @@ -426,6 +444,31 @@ export function createDesktopRoutes(deps: { return browserClearSandboxDataRoute.output.parse({ cleared: true }) } ], + [ + browserScanImportSourcesRoute.name, + async (rawInput) => { + browserScanImportSourcesRoute.input.parse(rawInput) + return browserScanImportSourcesRoute.output.parse( + await browserPresenter.scanImportSources() + ) + } + ], + [ + browserPreviewImportRoute.name, + async (rawInput) => { + const input = browserPreviewImportRoute.input.parse(rawInput) + return browserPreviewImportRoute.output.parse( + await browserPresenter.previewImport(input.profileId) + ) + } + ], + [ + browserApplyImportRoute.name, + async (rawInput) => { + const input = browserApplyImportRoute.input.parse(rawInput) + return browserApplyImportRoute.output.parse(await browserPresenter.applyImport(input.token)) + } + ], [ tabCaptureCurrentAreaRoute.name, async (rawInput, context) => { diff --git a/src/main/tool/agentTools/agentToolManager.ts b/src/main/tool/agentTools/agentToolManager.ts index 26fed6ad8f..069d318486 100644 --- a/src/main/tool/agentTools/agentToolManager.ts +++ b/src/main/tool/agentTools/agentToolManager.ts @@ -115,6 +115,7 @@ interface AgentToolManagerOptions { interface AgentToolExecutionOptions { toolCallId?: string + runId?: string onProgress?: (update: AgentToolProgressUpdate) => void signal?: AbortSignal allowExternalFileAccess?: boolean @@ -636,7 +637,8 @@ export class AgentToolManager { const response = await this.getYoBrowserToolHandler().callTool( toolName, args, - conversationId + conversationId, + options?.runId ) return { content: response diff --git a/src/main/tool/index.ts b/src/main/tool/index.ts index 8eea8cc31c..1b90adf2b9 100644 --- a/src/main/tool/index.ts +++ b/src/main/tool/index.ts @@ -313,6 +313,7 @@ export class ToolService implements ToolServicePort { request.conversationId, { toolCallId: request.id, + runId: options?.runId, onProgress: options?.onProgress, signal: options?.signal, allowExternalFileAccess: allowsExternalFileAccess(options?.permissionMode), diff --git a/src/main/tool/runtimePorts.ts b/src/main/tool/runtimePorts.ts index 13e9bc49ac..3bda002d1c 100644 --- a/src/main/tool/runtimePorts.ts +++ b/src/main/tool/runtimePorts.ts @@ -130,7 +130,8 @@ export interface AgentBrowserToolPort { callTool( toolName: string, args: Record, - conversationId?: string + conversationId?: string, + runId?: string ): Promise } diff --git a/src/renderer/api/BrowserClient.ts b/src/renderer/api/BrowserClient.ts index e2249d708b..38325cd905 100644 --- a/src/renderer/api/BrowserClient.ts +++ b/src/renderer/api/BrowserClient.ts @@ -2,10 +2,13 @@ import type { DeepchatBridge } from '@shared/contracts/bridge' import { browserActivityChangedEvent, browserOpenRequestedEvent, - browserStatusChangedEvent + browserPreviewFrameEvent, + browserStatusChangedEvent, + type DeepchatEventPayload } from '@shared/contracts/events' import { browserAttachCurrentWindowRoute, + browserApplyImportRoute, browserClearSandboxDataRoute, browserDestroyRoute, browserDetachRoute, @@ -13,7 +16,10 @@ import { browserGoBackRoute, browserGoForwardRoute, browserLoadUrlRoute, + browserPreviewImportRoute, browserReloadRoute, + browserScanImportSourcesRoute, + browserSetPreviewModeRoute, browserUpdateCurrentWindowBoundsRoute } from '@shared/contracts/routes' import type { YoBrowserStatus } from '@shared/types/browser' @@ -73,6 +79,15 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() return result.detached } + async function setPreviewMode( + sessionId: string, + mode: 'capturing' | 'rendering' | 'stopped', + runId?: string + ) { + const result = await bridge.invoke(browserSetPreviewModeRoute.name, { sessionId, mode, runId }) + return result.updated + } + async function destroy(sessionId: string) { const result = await bridge.invoke(browserDestroyRoute.name, { sessionId }) return result.destroyed @@ -98,6 +113,18 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() return result.cleared } + async function scanImportSources() { + return await bridge.invoke(browserScanImportSourcesRoute.name, {}) + } + + async function previewImport(profileId: string) { + return await bridge.invoke(browserPreviewImportRoute.name, { profileId }) + } + + async function applyImport(token: string) { + return await bridge.invoke(browserApplyImportRoute.name, { token }) + } + async function openExternal(url: string) { await openRuntimeExternal(url) } @@ -107,6 +134,8 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() sessionId: string windowId: number url: string + source: 'agent' | 'user' + runId?: string version: number }) => void ) { @@ -118,6 +147,8 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() sessionId: string windowId: number url: string + source: 'agent' | 'user' + runId?: string version: number }) => void ) { @@ -168,22 +199,33 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() return bridge.on(browserActivityChangedEvent.name, listener) } + function onPreviewFrame( + listener: (payload: DeepchatEventPayload) => void + ) { + return bridge.on(browserPreviewFrameEvent.name, listener) + } + return { getStatus, loadUrl, attachCurrentWindow, updateCurrentWindowBounds, detach, + setPreviewMode, destroy, goBack, goForward, reload, clearSandboxData, + scanImportSources, + previewImport, + applyImport, openExternal, onOpenRequested, onOpenRequestedForCurrentWindow, onStatusChanged, - onActivityChanged + onActivityChanged, + onPreviewFrame } } diff --git a/src/renderer/settings/components/BrowserDataImportDialog.vue b/src/renderer/settings/components/BrowserDataImportDialog.vue new file mode 100644 index 0000000000..12e0ab76e7 --- /dev/null +++ b/src/renderer/settings/components/BrowserDataImportDialog.vue @@ -0,0 +1,250 @@ + + + diff --git a/src/renderer/settings/components/DataSettings.vue b/src/renderer/settings/components/DataSettings.vue index a1d66730bb..e1d64cc103 100644 --- a/src/renderer/settings/components/DataSettings.vue +++ b/src/renderer/settings/components/DataSettings.vue @@ -937,49 +937,55 @@

- - - - - - - {{ - t('settings.data.yoBrowser.confirmTitle') - }} - - {{ t('settings.data.yoBrowser.confirmDescription') }} - - - - - {{ t('dialog.cancel') }} - - - {{ - isClearingSandbox - ? t('settings.data.yoBrowser.clearing') - : t('settings.data.yoBrowser.confirmAction') - }} - - - - +
+ + + + + + + + {{ + t('settings.data.yoBrowser.confirmTitle') + }} + + {{ t('settings.data.yoBrowser.confirmDescription') }} + + + + + {{ t('dialog.cancel') }} + + + {{ + isClearingSandbox + ? t('settings.data.yoBrowser.clearing') + : t('settings.data.yoBrowser.confirmAction') + }} + + + + +
@@ -1074,6 +1080,7 @@ import { useToast } from '@/components/use-toast' import PrivacySettingsSection from './common/PrivacySettingsSection.vue' import SettingsPageShell from './control-center/SettingsPageShell.vue' import ProviderConfigImportDialog from './ProviderConfigImportDialog.vue' +import BrowserDataImportDialog from './BrowserDataImportDialog.vue' const PROVIDER_IMPORT_SECTION = 'provider-import' const DATABASE_REPAIR_SECTION = 'database-repair' diff --git a/src/renderer/src/components/browser/AgentBrowserPiP.vue b/src/renderer/src/components/browser/AgentBrowserPiP.vue new file mode 100644 index 0000000000..e1d7140248 --- /dev/null +++ b/src/renderer/src/components/browser/AgentBrowserPiP.vue @@ -0,0 +1,514 @@ + + + + + diff --git a/src/renderer/src/components/sidepanel/BrowserPanel.vue b/src/renderer/src/components/sidepanel/BrowserPanel.vue index f40e253b0c..4e4bf57520 100644 --- a/src/renderer/src/components/sidepanel/BrowserPanel.vue +++ b/src/renderer/src/components/sidepanel/BrowserPanel.vue @@ -41,6 +41,18 @@ spellcheck="false" /> +
@@ -61,15 +73,15 @@ import { createBrowserClient } from '@api/BrowserClient' import BrowserPlaceholder from './BrowserPlaceholder.vue' import type { YoBrowserStatus } from '@shared/types/browser' import { useSidepanelStore } from '@/stores/ui/sidepanel' -import { useSessionStore } from '@/stores/ui/session' const props = defineProps<{ sessionId: string | null + isFullscreen?: boolean }>() +const emit = defineEmits<{ (event: 'toggle-fullscreen'): void }>() const { t } = useI18n() const sidepanelStore = useSidepanelStore() -const sessionStore = useSessionStore() const browserClient = createBrowserClient() const containerRef = ref(null) @@ -86,7 +98,6 @@ const urlInput = ref('') const canGoBack = ref(false) const canGoForward = ref(false) let lastSyncedBounds: Rectangle | null = null -const pendingBrowserDestroySessionIds = new Set() let visibilityRunId = 0 let stopOpenRequestedListener: (() => void) | null = null let stopStatusChangedListener: (() => void) | null = null @@ -103,10 +114,6 @@ const isBrowserPanelVisible = computed( () => sidepanelStore.open && sidepanelStore.activeTab === 'browser' ) -const getSessionUiStatus = (sessionId: string) => { - return sessionStore.sessions.find((session) => session.id === sessionId)?.status ?? null -} - const callBrowserAction = async (action: string, run: () => Promise): Promise => { try { return await run() @@ -337,6 +344,8 @@ const handleOpenRequested = async (payload: { sessionId: string windowId: number url: string + source: 'agent' | 'user' + runId?: string version: number }) => { if (payload.sessionId !== currentSessionId.value) { @@ -442,24 +451,6 @@ const cleanupInactiveSession = async (sessionId: string) => { } await hideEmbedded(sessionId) - if (getSessionUiStatus(sessionId) === 'working') { - pendingBrowserDestroySessionIds.add(sessionId) - return - } - - pendingBrowserDestroySessionIds.delete(sessionId) - await callBrowserAction('destroy', () => browserClient.destroy(sessionId)) -} - -const flushPendingSessionDestroys = async () => { - for (const sessionId of Array.from(pendingBrowserDestroySessionIds)) { - if (getSessionUiStatus(sessionId) === 'working') { - continue - } - - pendingBrowserDestroySessionIds.delete(sessionId) - await callBrowserAction('destroy', () => browserClient.destroy(sessionId)) - } } useResizeObserver(containerRef, () => { @@ -496,16 +487,6 @@ watch( { immediate: true } ) -watch( - () => sessionStore.sessions.map((session) => `${session.id}:${session.status}`).join('|'), - () => { - void flushPendingSessionDestroys() - if (currentSessionId.value) { - void loadState(currentSessionId.value) - } - } -) - onMounted(async () => { window.addEventListener('resize', scheduleVisibleBoundsSync) stopOpenRequestedListener = browserClient.onOpenRequestedForCurrentWindow(handleOpenRequested) diff --git a/src/renderer/src/components/sidepanel/ChatSidePanel.vue b/src/renderer/src/components/sidepanel/ChatSidePanel.vue index 20d5335c7b..253a639102 100644 --- a/src/renderer/src/components/sidepanel/ChatSidePanel.vue +++ b/src/renderer/src/components/sidepanel/ChatSidePanel.vue @@ -3,17 +3,18 @@ data-testid="chat-side-panel-shell" class="chat-side-panel-shell h-full min-h-0 overflow-hidden" :class="[ - isWorkspaceFullscreenActive ? 'absolute inset-0 w-full' : 'relative shrink-0', + isSidepanelFullscreenActive ? 'absolute inset-0 w-full' : 'relative shrink-0', { 'chat-side-panel-shell--resizing': isResizing } ]" :style="shellStyle" :data-workspace-fullscreen="String(isWorkspaceFullscreenActive)" + :data-browser-fullscreen="String(isBrowserFullscreenActive)" >
@@ -111,16 +117,23 @@ const layoutWidth = ref(shouldShow.value ? sidepanelStore.width : 0) const panelVisible = ref(shouldShow.value) const isResizing = ref(false) const isWorkspaceFullscreen = ref(false) +const isBrowserFullscreen = ref(false) const fullscreenMotionState = ref<'expanding' | 'collapsing' | null>(null) const isWorkspaceFullscreenActive = computed(() => { return isWorkspaceFullscreen.value && shouldShow.value && sidepanelStore.activeTab === 'workspace' }) +const isBrowserFullscreenActive = computed(() => { + return isBrowserFullscreen.value && shouldShow.value && sidepanelStore.activeTab === 'browser' +}) +const isSidepanelFullscreenActive = computed( + () => isWorkspaceFullscreenActive.value || isBrowserFullscreenActive.value +) const shellStyle = computed(() => { return { - width: isWorkspaceFullscreenActive.value ? '100%' : `${layoutWidth.value}px`, - ...(isWorkspaceFullscreenActive.value ? { zIndex: 'var(--dc-z-sidepanel)' } : {}) + width: isSidepanelFullscreenActive.value ? '100%' : `${layoutWidth.value}px`, + ...(isSidepanelFullscreenActive.value ? { zIndex: 'var(--dc-z-sidepanel)' } : {}) } }) @@ -128,13 +141,17 @@ const handleBrowserOpenRequested = (payload: { sessionId: string windowId: number url: string + source: 'agent' | 'user' + runId?: string version: number }) => { if (!props.sessionId || payload.sessionId !== props.sessionId) { return } - sidepanelStore.openBrowser() + if (payload.source !== 'agent' || sidepanelStore.open) { + sidepanelStore.openBrowser() + } } const clearPanelMotionHandles = () => { @@ -188,6 +205,11 @@ const resetWorkspaceFullscreen = () => { clearFullscreenMotionHandle() } +const resetBrowserFullscreen = () => { + isBrowserFullscreen.value = false + clearFullscreenMotionHandle() +} + const toggleWorkspaceFullscreen = () => { if (!shouldShow.value || sidepanelStore.activeTab !== 'workspace') { return @@ -202,6 +224,20 @@ const toggleWorkspaceFullscreen = () => { isWorkspaceFullscreen.value = !isWorkspaceFullscreen.value } +const toggleBrowserFullscreen = () => { + if (!shouldShow.value || sidepanelStore.activeTab !== 'browser') { + return + } + + clearFullscreenMotionHandle() + fullscreenMotionState.value = isBrowserFullscreen.value ? 'collapsing' : 'expanding' + fullscreenMotionTimer = window.setTimeout(() => { + fullscreenMotionTimer = null + fullscreenMotionState.value = null + }, FULLSCREEN_MOTION_MS) + isBrowserFullscreen.value = !isBrowserFullscreen.value +} + const handleWorkspaceInsertFileReference = (filePath: string) => { const sessionId = props.sessionId?.trim() const targetPath = filePath.trim() @@ -222,7 +258,7 @@ const handleWorkspaceInsertFileReference = (filePath: string) => { const startResize = (event: MouseEvent) => { event.preventDefault() - if (isWorkspaceFullscreenActive.value) { + if (isSidepanelFullscreenActive.value) { return } @@ -260,6 +296,7 @@ watch(shouldShow, (visible) => { if (!visible) { resetWorkspaceFullscreen() + resetBrowserFullscreen() } if (visible) { @@ -286,6 +323,9 @@ watch( if (activeTab !== 'workspace') { resetWorkspaceFullscreen() } + if (activeTab !== 'browser') { + resetBrowserFullscreen() + } } ) @@ -294,6 +334,7 @@ watch( (sessionId, previousSessionId) => { if (!sessionId || sessionId !== previousSessionId) { resetWorkspaceFullscreen() + resetBrowserFullscreen() } } ) diff --git a/src/renderer/src/i18n/da-DK/settings.json b/src/renderer/src/i18n/da-DK/settings.json index 9276f9d6a4..f687dbb36e 100644 --- a/src/renderer/src/i18n/da-DK/settings.json +++ b/src/renderer/src/i18n/da-DK/settings.json @@ -152,7 +152,33 @@ "confirmDescription": "YoBrowser-sessionens cookies, cache, LocalStorage, IndexedDB og andre data vil blive slettet, og du skal muligvis logge ind på hjemmesiden igen.", "confirmTitle": "Bekræft at rydde YoBrowser-data?", "description": "En uafhængig browsersandbox, der ikke deler cookies og lokal lagring med hovedapplikationen. YoBrowser-data kan ryddes med et enkelt klik her.", - "title": "YoBrowser Sandbox" + "title": "YoBrowser Sandbox", + "import": { + "button": "Import website data", + "title": "Import browser session data", + "description": "Choose a Chrome or Arc profile. macOS may ask you to authorize Keychain access. The source is never modified.", + "scanning": "Looking for browser profiles...", + "unsupportedTitle": "Not supported on this platform", + "unsupportedDescription": "Direct browser data import is currently available only on macOS. Windows remains unsupported because current Chrome data is app-bound.", + "profileLabel": "Source profile", + "profilePlaceholder": "Select a profile", + "noProfiles": "No supported Chrome or Arc profiles with cookie data were found.", + "cookies": "Cookies and sessions", + "supported": "Included in this import", + "otherData": "Passwords and web storage", + "notIncluded": "Not included in this version", + "preview": "Authorize and preview", + "previewTitle": "Ready to replace YoBrowser cookies", + "previewDescription": "{count} cookies can be imported. Existing YoBrowser cookies will be replaced.", + "skipped": "Skipped: {expired} expired, {partitioned} partitioned.", + "confirm": "Replace and import", + "doneTitle": "Import complete", + "doneDescription": "Imported {count} cookies. Open browser pages were reloaded.", + "keyDenied": "Keychain access was denied or the browser key could not be read.", + "previewExpired": "The source changed or the preview expired. Create a new preview.", + "encryptionUnsupported": "This profile uses an encryption format that this importer does not support.", + "failed": "Import failed before completion. Existing YoBrowser cookies were restored when mutation had started." + } }, "databaseRepair": { "title": "Databasereparation", diff --git a/src/renderer/src/i18n/de-DE/settings.json b/src/renderer/src/i18n/de-DE/settings.json index 6772c94cb6..0264821f34 100644 --- a/src/renderer/src/i18n/de-DE/settings.json +++ b/src/renderer/src/i18n/de-DE/settings.json @@ -547,7 +547,33 @@ "clearedTitle": "YoBrowser-Daten gelöscht", "clearedDescription": "Die Sandbox-Sitzung wurde zurückgesetzt; zugehörige Tabs werden neu geladen.", "clearFailedTitle": "Löschen fehlgeschlagen", - "clearFailedDescription": "Bitte erneut versuchen oder Protokolle für weitere Informationen prüfen." + "clearFailedDescription": "Bitte erneut versuchen oder Protokolle für weitere Informationen prüfen.", + "import": { + "button": "Import website data", + "title": "Import browser session data", + "description": "Choose a Chrome or Arc profile. macOS may ask you to authorize Keychain access. The source is never modified.", + "scanning": "Looking for browser profiles...", + "unsupportedTitle": "Not supported on this platform", + "unsupportedDescription": "Direct browser data import is currently available only on macOS. Windows remains unsupported because current Chrome data is app-bound.", + "profileLabel": "Source profile", + "profilePlaceholder": "Select a profile", + "noProfiles": "No supported Chrome or Arc profiles with cookie data were found.", + "cookies": "Cookies and sessions", + "supported": "Included in this import", + "otherData": "Passwords and web storage", + "notIncluded": "Not included in this version", + "preview": "Authorize and preview", + "previewTitle": "Ready to replace YoBrowser cookies", + "previewDescription": "{count} cookies can be imported. Existing YoBrowser cookies will be replaced.", + "skipped": "Skipped: {expired} expired, {partitioned} partitioned.", + "confirm": "Replace and import", + "doneTitle": "Import complete", + "doneDescription": "Imported {count} cookies. Open browser pages were reloaded.", + "keyDenied": "Keychain access was denied or the browser key could not be read.", + "previewExpired": "The source changed or the preview expired. Create a new preview.", + "encryptionUnsupported": "This profile uses an encryption format that this importer does not support.", + "failed": "Import failed before completion. Existing YoBrowser cookies were restored when mutation had started." + } }, "privacyTitle": "Data & Privacy", "privacyDescription": "Backup-Synchronisierung, Privatsphärenmodus, Datenwartung und gefährliche Aktionen verwalten.", diff --git a/src/renderer/src/i18n/en-US/settings.json b/src/renderer/src/i18n/en-US/settings.json index d42a33e68d..64e1b49134 100644 --- a/src/renderer/src/i18n/en-US/settings.json +++ b/src/renderer/src/i18n/en-US/settings.json @@ -589,6 +589,32 @@ "confirmDescription": "The YoBrowser session's cookies, cache, LocalStorage, IndexedDB and other data will be deleted, and you may need to log in to the website again.", "confirmTitle": "Confirm to clear YoBrowser data?", "description": "An independent browser sandbox that does not share cookies and local storage with the main application. \nYoBrowser data can be cleared with one click here.", + "import": { + "button": "Import website data", + "title": "Import browser session data", + "description": "Choose a Chrome or Arc profile. macOS may ask you to authorize Keychain access. The source is never modified.", + "scanning": "Looking for browser profiles...", + "unsupportedTitle": "Not supported on this platform", + "unsupportedDescription": "Direct browser data import is currently available only on macOS. Windows remains unsupported because current Chrome data is app-bound.", + "profileLabel": "Source profile", + "profilePlaceholder": "Select a profile", + "noProfiles": "No supported Chrome or Arc profiles with cookie data were found.", + "cookies": "Cookies and sessions", + "supported": "Included in this import", + "otherData": "Passwords and web storage", + "notIncluded": "Not included in this version", + "preview": "Authorize and preview", + "previewTitle": "Ready to replace YoBrowser cookies", + "previewDescription": "{count} cookies can be imported. Existing YoBrowser cookies will be replaced.", + "skipped": "Skipped: {expired} expired, {partitioned} partitioned.", + "confirm": "Replace and import", + "doneTitle": "Import complete", + "doneDescription": "Imported {count} cookies. Open browser pages were reloaded.", + "keyDenied": "Keychain access was denied or the browser key could not be read.", + "previewExpired": "The source changed or the preview expired. Create a new preview.", + "encryptionUnsupported": "This profile uses an encryption format that this importer does not support.", + "failed": "Import failed before completion. Existing YoBrowser cookies were restored when mutation had started." + }, "title": "YoBrowser Sandbox" }, "privacyTitle": "Data & Privacy", diff --git a/src/renderer/src/i18n/es-ES/settings.json b/src/renderer/src/i18n/es-ES/settings.json index d3a44c6bcb..51bab5b0cf 100644 --- a/src/renderer/src/i18n/es-ES/settings.json +++ b/src/renderer/src/i18n/es-ES/settings.json @@ -547,7 +547,33 @@ "clearedTitle": "YoBrowser datos borrados", "clearedDescription": "La sesión de sandbox se restableció y se recargará la pestaña asociada.", "clearFailedTitle": "Error al borrar", - "clearFailedDescription": "Inténtelo de nuevo o consulte los registros para obtener más información." + "clearFailedDescription": "Inténtelo de nuevo o consulte los registros para obtener más información.", + "import": { + "button": "Import website data", + "title": "Import browser session data", + "description": "Choose a Chrome or Arc profile. macOS may ask you to authorize Keychain access. The source is never modified.", + "scanning": "Looking for browser profiles...", + "unsupportedTitle": "Not supported on this platform", + "unsupportedDescription": "Direct browser data import is currently available only on macOS. Windows remains unsupported because current Chrome data is app-bound.", + "profileLabel": "Source profile", + "profilePlaceholder": "Select a profile", + "noProfiles": "No supported Chrome or Arc profiles with cookie data were found.", + "cookies": "Cookies and sessions", + "supported": "Included in this import", + "otherData": "Passwords and web storage", + "notIncluded": "Not included in this version", + "preview": "Authorize and preview", + "previewTitle": "Ready to replace YoBrowser cookies", + "previewDescription": "{count} cookies can be imported. Existing YoBrowser cookies will be replaced.", + "skipped": "Skipped: {expired} expired, {partitioned} partitioned.", + "confirm": "Replace and import", + "doneTitle": "Import complete", + "doneDescription": "Imported {count} cookies. Open browser pages were reloaded.", + "keyDenied": "Keychain access was denied or the browser key could not be read.", + "previewExpired": "The source changed or the preview expired. Create a new preview.", + "encryptionUnsupported": "This profile uses an encryption format that this importer does not support.", + "failed": "Import failed before completion. Existing YoBrowser cookies were restored when mutation had started." + } }, "privacyTitle": "Datos y privacidad", "privacyDescription": "Administre copias de seguridad, sincronización, modo de privacidad, mantenimiento y operaciones peligrosas.", diff --git a/src/renderer/src/i18n/fa-IR/settings.json b/src/renderer/src/i18n/fa-IR/settings.json index 0cd027462a..17f15204d5 100644 --- a/src/renderer/src/i18n/fa-IR/settings.json +++ b/src/renderer/src/i18n/fa-IR/settings.json @@ -219,7 +219,33 @@ "confirmDescription": "کوکی‌ها، حافظه پنهان، LocalStorage، IndexedDB و سایر داده‌های جلسه YoBrowser حذف خواهند شد و ممکن است لازم باشد دوباره وارد وب‌سایت شوید.", "confirmTitle": "برای پاک کردن داده‌های YoBrowser تأیید می‌کنید؟", "description": "یک سندباکس مرورگر مستقل که کوکی‌ها و فضای ذخیره‌سازی محلی را با برنامه اصلی به اشتراک نمی‌گذارد. داده های YoBrowser را می توان با یک کلیک در اینجا پاک کرد.", - "title": "YoBrowser Sandbox" + "title": "YoBrowser Sandbox", + "import": { + "button": "Import website data", + "title": "Import browser session data", + "description": "Choose a Chrome or Arc profile. macOS may ask you to authorize Keychain access. The source is never modified.", + "scanning": "Looking for browser profiles...", + "unsupportedTitle": "Not supported on this platform", + "unsupportedDescription": "Direct browser data import is currently available only on macOS. Windows remains unsupported because current Chrome data is app-bound.", + "profileLabel": "Source profile", + "profilePlaceholder": "Select a profile", + "noProfiles": "No supported Chrome or Arc profiles with cookie data were found.", + "cookies": "Cookies and sessions", + "supported": "Included in this import", + "otherData": "Passwords and web storage", + "notIncluded": "Not included in this version", + "preview": "Authorize and preview", + "previewTitle": "Ready to replace YoBrowser cookies", + "previewDescription": "{count} cookies can be imported. Existing YoBrowser cookies will be replaced.", + "skipped": "Skipped: {expired} expired, {partitioned} partitioned.", + "confirm": "Replace and import", + "doneTitle": "Import complete", + "doneDescription": "Imported {count} cookies. Open browser pages were reloaded.", + "keyDenied": "Keychain access was denied or the browser key could not be read.", + "previewExpired": "The source changed or the preview expired. Create a new preview.", + "encryptionUnsupported": "This profile uses an encryption format that this importer does not support.", + "failed": "Import failed before completion. Existing YoBrowser cookies were restored when mutation had started." + } }, "databaseRepair": { "title": "ترمیم پایگاه داده", diff --git a/src/renderer/src/i18n/fr-FR/settings.json b/src/renderer/src/i18n/fr-FR/settings.json index 207ed1a511..f62c9a3e97 100644 --- a/src/renderer/src/i18n/fr-FR/settings.json +++ b/src/renderer/src/i18n/fr-FR/settings.json @@ -219,7 +219,33 @@ "confirmDescription": "Les cookies, le cache, LocalStorage, IndexedDB et autres données de la session YoBrowser seront supprimés et vous devrez peut-être vous reconnecter au site Web.", "confirmTitle": "Confirmer pour effacer les données de YoBrowser ?", "description": "Un bac à sable de navigateur indépendant qui ne partage pas de cookies ni de stockage local avec l'application principale. Les données YoBrowser peuvent être effacées en un seul clic ici.", - "title": "Bac à sable YoBrowser" + "title": "Bac à sable YoBrowser", + "import": { + "button": "Import website data", + "title": "Import browser session data", + "description": "Choose a Chrome or Arc profile. macOS may ask you to authorize Keychain access. The source is never modified.", + "scanning": "Looking for browser profiles...", + "unsupportedTitle": "Not supported on this platform", + "unsupportedDescription": "Direct browser data import is currently available only on macOS. Windows remains unsupported because current Chrome data is app-bound.", + "profileLabel": "Source profile", + "profilePlaceholder": "Select a profile", + "noProfiles": "No supported Chrome or Arc profiles with cookie data were found.", + "cookies": "Cookies and sessions", + "supported": "Included in this import", + "otherData": "Passwords and web storage", + "notIncluded": "Not included in this version", + "preview": "Authorize and preview", + "previewTitle": "Ready to replace YoBrowser cookies", + "previewDescription": "{count} cookies can be imported. Existing YoBrowser cookies will be replaced.", + "skipped": "Skipped: {expired} expired, {partitioned} partitioned.", + "confirm": "Replace and import", + "doneTitle": "Import complete", + "doneDescription": "Imported {count} cookies. Open browser pages were reloaded.", + "keyDenied": "Keychain access was denied or the browser key could not be read.", + "previewExpired": "The source changed or the preview expired. Create a new preview.", + "encryptionUnsupported": "This profile uses an encryption format that this importer does not support.", + "failed": "Import failed before completion. Existing YoBrowser cookies were restored when mutation had started." + } }, "databaseRepair": { "title": "Réparation de la base de données", diff --git a/src/renderer/src/i18n/he-IL/settings.json b/src/renderer/src/i18n/he-IL/settings.json index 1eb66b5460..673df72268 100644 --- a/src/renderer/src/i18n/he-IL/settings.json +++ b/src/renderer/src/i18n/he-IL/settings.json @@ -219,7 +219,33 @@ "confirmDescription": "העוגיות, המטמון, LocalStorage, IndexedDB ונתונים אחרים של הפעלת YoBrowser יימחקו, וייתכן שתצטרך להיכנס שוב לאתר.", "confirmTitle": "לאשר לנקות נתוני YoBrowser?", "description": "ארגז חול עצמאי של דפדפן שאינו חולק עוגיות ואחסון מקומי עם האפליקציה הראשית. ניתן לנקות את נתוני YoBrowser בלחיצה אחת כאן.", - "title": "ארגז חול של YoBrowser" + "title": "ארגז חול של YoBrowser", + "import": { + "button": "Import website data", + "title": "Import browser session data", + "description": "Choose a Chrome or Arc profile. macOS may ask you to authorize Keychain access. The source is never modified.", + "scanning": "Looking for browser profiles...", + "unsupportedTitle": "Not supported on this platform", + "unsupportedDescription": "Direct browser data import is currently available only on macOS. Windows remains unsupported because current Chrome data is app-bound.", + "profileLabel": "Source profile", + "profilePlaceholder": "Select a profile", + "noProfiles": "No supported Chrome or Arc profiles with cookie data were found.", + "cookies": "Cookies and sessions", + "supported": "Included in this import", + "otherData": "Passwords and web storage", + "notIncluded": "Not included in this version", + "preview": "Authorize and preview", + "previewTitle": "Ready to replace YoBrowser cookies", + "previewDescription": "{count} cookies can be imported. Existing YoBrowser cookies will be replaced.", + "skipped": "Skipped: {expired} expired, {partitioned} partitioned.", + "confirm": "Replace and import", + "doneTitle": "Import complete", + "doneDescription": "Imported {count} cookies. Open browser pages were reloaded.", + "keyDenied": "Keychain access was denied or the browser key could not be read.", + "previewExpired": "The source changed or the preview expired. Create a new preview.", + "encryptionUnsupported": "This profile uses an encryption format that this importer does not support.", + "failed": "Import failed before completion. Existing YoBrowser cookies were restored when mutation had started." + } }, "databaseRepair": { "title": "תיקון מסד הנתונים", diff --git a/src/renderer/src/i18n/id-ID/settings.json b/src/renderer/src/i18n/id-ID/settings.json index 3561aa0603..7a67002ae7 100644 --- a/src/renderer/src/i18n/id-ID/settings.json +++ b/src/renderer/src/i18n/id-ID/settings.json @@ -547,7 +547,33 @@ "clearedTitle": "Menghapus data YoBrowser", "clearedDescription": "Sesi sandbox telah disetel ulang dan tab terkait akan dimuat ulang.", "clearFailedTitle": "Pembersihan gagal", - "clearFailedDescription": "Silakan coba lagi atau periksa log untuk informasi lebih lanjut." + "clearFailedDescription": "Silakan coba lagi atau periksa log untuk informasi lebih lanjut.", + "import": { + "button": "Impor data situs web", + "title": "Impor data sesi browser", + "description": "Pilih profil Chrome atau Arc. macOS mungkin meminta Anda mengizinkan akses Rantai Kunci. Sumber tidak akan diubah.", + "scanning": "Mencari profil browser...", + "unsupportedTitle": "Tidak didukung di platform ini", + "unsupportedDescription": "Impor data browser secara langsung saat ini hanya tersedia di macOS. Windows belum didukung karena data Chrome saat ini terikat pada aplikasi.", + "profileLabel": "Profil sumber", + "profilePlaceholder": "Pilih profil", + "noProfiles": "Tidak ditemukan profil Chrome atau Arc yang didukung dengan data cookie.", + "cookies": "Cookie dan sesi", + "supported": "Disertakan dalam impor ini", + "otherData": "Kata sandi dan penyimpanan web", + "notIncluded": "Tidak disertakan dalam versi ini", + "preview": "Izinkan dan pratinjau", + "previewTitle": "Siap mengganti cookie YoBrowser", + "previewDescription": "{count} cookie dapat diimpor. Cookie YoBrowser yang ada akan diganti.", + "skipped": "Dilewati: {expired} kedaluwarsa, {partitioned} terpartisi.", + "confirm": "Ganti dan impor", + "doneTitle": "Impor selesai", + "doneDescription": "{count} cookie telah diimpor. Halaman browser yang terbuka telah dimuat ulang.", + "keyDenied": "Akses Rantai Kunci ditolak atau kunci browser tidak dapat dibaca.", + "previewExpired": "Sumber berubah atau pratinjau kedaluwarsa. Buat pratinjau baru.", + "encryptionUnsupported": "Profil ini menggunakan format enkripsi yang tidak didukung oleh pengimpor ini.", + "failed": "Impor gagal sebelum selesai. Cookie YoBrowser yang ada telah dipulihkan jika perubahan sempat dimulai." + } }, "privacyTitle": "Data & Privasi", "privacyDescription": "Kelola sinkronisasi cadangan, mode privasi, pemeliharaan data, dan operasi berbahaya.", diff --git a/src/renderer/src/i18n/it-IT/settings.json b/src/renderer/src/i18n/it-IT/settings.json index 3b6b64357b..6d354ac986 100644 --- a/src/renderer/src/i18n/it-IT/settings.json +++ b/src/renderer/src/i18n/it-IT/settings.json @@ -547,7 +547,33 @@ "clearedTitle": "Dati YoBrowser cancellati", "clearedDescription": "La sessione sandbox è stata ripristinata e le schede correlate verranno ricaricate.", "clearFailedTitle": "Cancellazione non riuscita", - "clearFailedDescription": "Riprova o controlla i log per altre informazioni." + "clearFailedDescription": "Riprova o controlla i log per altre informazioni.", + "import": { + "button": "Importa dati dei siti web", + "title": "Importa dati della sessione del browser", + "description": "Scegli un profilo Chrome o Arc. macOS potrebbe chiederti di autorizzare l'accesso al Portachiavi. La sorgente non viene mai modificata.", + "scanning": "Ricerca dei profili del browser...", + "unsupportedTitle": "Non supportato su questa piattaforma", + "unsupportedDescription": "L'importazione diretta dei dati del browser è attualmente disponibile solo su macOS. Windows non è supportato perché i dati correnti di Chrome sono vincolati all'applicazione.", + "profileLabel": "Profilo sorgente", + "profilePlaceholder": "Seleziona un profilo", + "noProfiles": "Non sono stati trovati profili Chrome o Arc supportati con dati dei cookie.", + "cookies": "Cookie e sessioni", + "supported": "Inclusi in questa importazione", + "otherData": "Password e archiviazione web", + "notIncluded": "Non inclusi in questa versione", + "preview": "Autorizza e visualizza l'anteprima", + "previewTitle": "Pronto a sostituire i cookie di YoBrowser", + "previewDescription": "È possibile importare {count} cookie. I cookie esistenti di YoBrowser verranno sostituiti.", + "skipped": "Ignorati: {expired} scaduti, {partitioned} partizionati.", + "confirm": "Sostituisci e importa", + "doneTitle": "Importazione completata", + "doneDescription": "Importati {count} cookie. Le pagine del browser aperte sono state ricaricate.", + "keyDenied": "L'accesso al Portachiavi è stato negato oppure non è stato possibile leggere la chiave del browser.", + "previewExpired": "La sorgente è cambiata o l'anteprima è scaduta. Crea una nuova anteprima.", + "encryptionUnsupported": "Questo profilo usa un formato di crittografia non supportato dallo strumento di importazione.", + "failed": "L'importazione non è stata completata. I cookie esistenti di YoBrowser sono stati ripristinati se la modifica era già iniziata." + } }, "privacyTitle": "Dati e privacy", "privacyDescription": "Gestisci backup, sincronizzazione, modalità privacy, manutenzione dati e operazioni rischiose.", diff --git a/src/renderer/src/i18n/ja-JP/settings.json b/src/renderer/src/i18n/ja-JP/settings.json index 15b3e044e2..2ae8c84f02 100644 --- a/src/renderer/src/i18n/ja-JP/settings.json +++ b/src/renderer/src/i18n/ja-JP/settings.json @@ -219,7 +219,33 @@ "confirmDescription": "YoBrowser セッションの Cookie、キャッシュ、LocalStorage、IndexedDB およびその他のデータが削除されるため、Web サイトへの再ログインが必要になる場合があります。", "confirmTitle": "YoBrowser データをクリアしますか?", "description": "Cookie やローカル ストレージをメイン アプリケーションと共有しない独立したブラウザ サンドボックス。ここで、YoBrowser のデータをワンクリックでクリアできます。", - "title": "YoBrowser サンドボックス" + "title": "YoBrowser サンドボックス", + "import": { + "button": "Web サイトデータをインポート", + "title": "ブラウザーのセッションデータをインポート", + "description": "Chrome または Arc のプロファイルを選択してください。macOS からキーチェーンへのアクセス許可を求められる場合があります。インポート元は変更されません。", + "scanning": "ブラウザーのプロファイルを検索しています...", + "unsupportedTitle": "このプラットフォームではサポートされていません", + "unsupportedDescription": "ブラウザーデータの直接インポートは現在 macOS でのみ利用できます。現在の Chrome データはアプリに関連付けられているため、Windows はサポートされていません。", + "profileLabel": "インポート元のプロファイル", + "profilePlaceholder": "プロファイルを選択", + "noProfiles": "Cookie データを含む、サポート対象の Chrome または Arc プロファイルが見つかりませんでした。", + "cookies": "Cookie とセッション", + "supported": "今回のインポートに含まれます", + "otherData": "パスワードと Web ストレージ", + "notIncluded": "このバージョンには含まれません", + "preview": "許可してプレビュー", + "previewTitle": "YoBrowser の Cookie を置き換える準備ができました", + "previewDescription": "{count} 件の Cookie をインポートできます。既存の YoBrowser の Cookie は置き換えられます。", + "skipped": "スキップ: 期限切れ {expired} 件、パーティション化 {partitioned} 件。", + "confirm": "置き換えてインポート", + "doneTitle": "インポートが完了しました", + "doneDescription": "{count} 件の Cookie をインポートしました。開いているブラウザーページを再読み込みしました。", + "keyDenied": "キーチェーンへのアクセスが拒否されたか、ブラウザーのキーを読み取れませんでした。", + "previewExpired": "インポート元が変更されたか、プレビューの有効期限が切れました。新しいプレビューを作成してください。", + "encryptionUnsupported": "このプロファイルでは、インポーターがサポートしていない暗号化形式が使用されています。", + "failed": "インポートが完了前に失敗しました。変更が開始されていた場合、既存の YoBrowser の Cookie は復元されました。" + } }, "databaseRepair": { "title": "データベース修復", diff --git a/src/renderer/src/i18n/ko-KR/settings.json b/src/renderer/src/i18n/ko-KR/settings.json index 7491d1b580..837e426e4d 100644 --- a/src/renderer/src/i18n/ko-KR/settings.json +++ b/src/renderer/src/i18n/ko-KR/settings.json @@ -219,7 +219,33 @@ "confirmDescription": "YoBrowser 세션의 쿠키, 캐시, LocalStorage, IndexedDB 및 기타 데이터가 삭제되며 웹사이트에 다시 로그인해야 할 수도 있습니다.", "confirmTitle": "YoBrowser 데이터를 삭제하시겠습니까?", "description": "기본 애플리케이션과 쿠키 및 로컬 저장소를 공유하지 않는 독립적인 브라우저 샌드박스입니다. 여기를 클릭하면 YoBrowser 데이터를 삭제할 수 있습니다.", - "title": "요브라우저 샌드박스" + "title": "요브라우저 샌드박스", + "import": { + "button": "웹사이트 데이터 가져오기", + "title": "브라우저 세션 데이터 가져오기", + "description": "Chrome 또는 Arc 프로필을 선택하세요. macOS에서 키체인 접근 권한을 요청할 수 있습니다. 원본 데이터는 변경되지 않습니다.", + "scanning": "브라우저 프로필을 찾는 중...", + "unsupportedTitle": "이 플랫폼에서는 지원되지 않음", + "unsupportedDescription": "브라우저 데이터 직접 가져오기는 현재 macOS에서만 사용할 수 있습니다. 현재 Chrome 데이터는 앱에 바인딩되어 있으므로 Windows는 지원되지 않습니다.", + "profileLabel": "원본 프로필", + "profilePlaceholder": "프로필 선택", + "noProfiles": "쿠키 데이터가 있는 지원 대상 Chrome 또는 Arc 프로필을 찾지 못했습니다.", + "cookies": "쿠키 및 세션", + "supported": "이번 가져오기에 포함됨", + "otherData": "비밀번호 및 웹 저장소", + "notIncluded": "이 버전에는 포함되지 않음", + "preview": "권한 부여 및 미리보기", + "previewTitle": "YoBrowser 쿠키를 교체할 준비가 되었습니다", + "previewDescription": "쿠키 {count}개를 가져올 수 있습니다. 기존 YoBrowser 쿠키가 교체됩니다.", + "skipped": "건너뜀: 만료됨 {expired}개, 파티션됨 {partitioned}개.", + "confirm": "교체하고 가져오기", + "doneTitle": "가져오기 완료", + "doneDescription": "쿠키 {count}개를 가져왔습니다. 열려 있는 브라우저 페이지를 다시 불러왔습니다.", + "keyDenied": "키체인 접근이 거부되었거나 브라우저 키를 읽을 수 없습니다.", + "previewExpired": "원본이 변경되었거나 미리보기가 만료되었습니다. 새 미리보기를 만드세요.", + "encryptionUnsupported": "이 프로필은 가져오기 도구에서 지원하지 않는 암호화 형식을 사용합니다.", + "failed": "가져오기가 완료되기 전에 실패했습니다. 변경이 시작된 경우 기존 YoBrowser 쿠키가 복원되었습니다." + } }, "databaseRepair": { "title": "데이터베이스 복구", diff --git a/src/renderer/src/i18n/ms-MY/settings.json b/src/renderer/src/i18n/ms-MY/settings.json index c3b687e02b..25a424ddae 100644 --- a/src/renderer/src/i18n/ms-MY/settings.json +++ b/src/renderer/src/i18n/ms-MY/settings.json @@ -547,7 +547,33 @@ "clearedTitle": "Data YoBrowser dikosongkan", "clearedDescription": "Sesi kotak pasir telah ditetapkan semula dan tab yang berkaitan akan dimuatkan semula.", "clearFailedTitle": "Pembersihan gagal", - "clearFailedDescription": "Sila cuba lagi atau semak log untuk mendapatkan maklumat lanjut." + "clearFailedDescription": "Sila cuba lagi atau semak log untuk mendapatkan maklumat lanjut.", + "import": { + "button": "Import data laman web", + "title": "Import data sesi pelayar", + "description": "Pilih profil Chrome atau Arc. macOS mungkin meminta anda membenarkan akses Rantai Kunci. Sumber tidak akan diubah.", + "scanning": "Mencari profil pelayar...", + "unsupportedTitle": "Tidak disokong pada platform ini", + "unsupportedDescription": "Import terus data pelayar kini hanya tersedia pada macOS. Windows belum disokong kerana data Chrome semasa terikat pada aplikasi.", + "profileLabel": "Profil sumber", + "profilePlaceholder": "Pilih profil", + "noProfiles": "Tiada profil Chrome atau Arc yang disokong dengan data kuki ditemui.", + "cookies": "Kuki dan sesi", + "supported": "Disertakan dalam import ini", + "otherData": "Kata laluan dan storan web", + "notIncluded": "Tidak disertakan dalam versi ini", + "preview": "Benarkan dan pratonton", + "previewTitle": "Sedia untuk menggantikan kuki YoBrowser", + "previewDescription": "{count} kuki boleh diimport. Kuki YoBrowser sedia ada akan digantikan.", + "skipped": "Dilangkau: {expired} tamat tempoh, {partitioned} berpartisi.", + "confirm": "Gantikan dan import", + "doneTitle": "Import selesai", + "doneDescription": "{count} kuki telah diimport. Halaman pelayar yang terbuka telah dimuatkan semula.", + "keyDenied": "Akses Rantai Kunci ditolak atau kunci pelayar tidak dapat dibaca.", + "previewExpired": "Sumber berubah atau pratonton tamat tempoh. Cipta pratonton baharu.", + "encryptionUnsupported": "Profil ini menggunakan format penyulitan yang tidak disokong oleh pengimport ini.", + "failed": "Import gagal sebelum selesai. Kuki YoBrowser sedia ada telah dipulihkan jika perubahan telah bermula." + } }, "privacyTitle": "Data & Privacy", "privacyDescription": "Urus penyegerakan sandaran, mod privasi, penyelenggaraan data dan operasi berbahaya.", diff --git a/src/renderer/src/i18n/pl-PL/settings.json b/src/renderer/src/i18n/pl-PL/settings.json index 13b92ab843..07f9fb6181 100644 --- a/src/renderer/src/i18n/pl-PL/settings.json +++ b/src/renderer/src/i18n/pl-PL/settings.json @@ -547,7 +547,33 @@ "confirmDescription": "Pliki cookie, pamięć podręczna, LocalStorage, IndexedDB i inne dane sesji YoBrowser zostaną usunięte, a konieczne może być ponowne zalogowanie się na stronie.", "confirmTitle": "Potwierdzić, aby wyczyścić dane YoBrowser?", "description": "Niezależna piaskownica przeglądarki, która nie udostępnia plików cookie i pamięci lokalnej z główną aplikacją. \nTutaj możesz wyczyścić dane YoBrowser jednym kliknięciem.", - "title": "Piaskownica YoBrowser" + "title": "Piaskownica YoBrowser", + "import": { + "button": "Importuj dane witryn", + "title": "Importuj dane sesji przeglądarki", + "description": "Wybierz profil Chrome lub Arc. macOS może poprosić o zezwolenie na dostęp do pęku kluczy. Źródło nigdy nie jest modyfikowane.", + "scanning": "Wyszukiwanie profili przeglądarki...", + "unsupportedTitle": "Brak obsługi na tej platformie", + "unsupportedDescription": "Bezpośredni import danych przeglądarki jest obecnie dostępny tylko w systemie macOS. System Windows nie jest obsługiwany, ponieważ bieżące dane Chrome są powiązane z aplikacją.", + "profileLabel": "Profil źródłowy", + "profilePlaceholder": "Wybierz profil", + "noProfiles": "Nie znaleziono obsługiwanych profili Chrome ani Arc z danymi plików cookie.", + "cookies": "Pliki cookie i sesje", + "supported": "Uwzględnione w tym imporcie", + "otherData": "Hasła i pamięć internetowa", + "notIncluded": "Nieuwzględnione w tej wersji", + "preview": "Zezwól i wyświetl podgląd", + "previewTitle": "Gotowe do zastąpienia plików cookie YoBrowser", + "previewDescription": "Można zaimportować {count} plików cookie. Istniejące pliki cookie YoBrowser zostaną zastąpione.", + "skipped": "Pominięto: {expired} wygasłych, {partitioned} partycjonowanych.", + "confirm": "Zastąp i importuj", + "doneTitle": "Import zakończony", + "doneDescription": "Zaimportowano {count} plików cookie. Otwarte strony przeglądarki zostały ponownie wczytane.", + "keyDenied": "Odmówiono dostępu do pęku kluczy lub nie można było odczytać klucza przeglądarki.", + "previewExpired": "Źródło uległo zmianie lub podgląd wygasł. Utwórz nowy podgląd.", + "encryptionUnsupported": "Ten profil używa formatu szyfrowania, którego importer nie obsługuje.", + "failed": "Import nie został ukończony. Jeśli modyfikacja została rozpoczęta, istniejące pliki cookie YoBrowser zostały przywrócone." + } }, "privacyTitle": "Dane i prywatność", "privacyDescription": "Zarządzaj kopiami zapasowymi, synchronizacją, trybem prywatności, konserwacją i niebezpiecznymi operacjami.", diff --git a/src/renderer/src/i18n/pt-BR/settings.json b/src/renderer/src/i18n/pt-BR/settings.json index be4c9bff3b..9d2a5190ca 100644 --- a/src/renderer/src/i18n/pt-BR/settings.json +++ b/src/renderer/src/i18n/pt-BR/settings.json @@ -219,7 +219,33 @@ "confirmDescription": "Os cookies, cache, LocalStorage, IndexedDB e outros dados da sessão YoBrowser serão excluídos e pode ser necessário fazer login no site novamente.", "confirmTitle": "Confirmar para limpar os dados do YoBrowser?", "description": "Uma sandbox de navegador independente que não compartilha cookies e armazenamento local com o aplicativo principal. Os dados do YoBrowser podem ser apagados com um clique aqui.", - "title": "Caixa de areia YoBrowser" + "title": "Caixa de areia YoBrowser", + "import": { + "button": "Importar dados de sites", + "title": "Importar dados da sessão do navegador", + "description": "Escolha um perfil do Chrome ou Arc. O macOS pode solicitar autorização para acessar as Chaves. A fonte nunca é modificada.", + "scanning": "Procurando perfis do navegador...", + "unsupportedTitle": "Não compatível com esta plataforma", + "unsupportedDescription": "A importação direta de dados do navegador está disponível atualmente apenas no macOS. O Windows não é compatível porque os dados atuais do Chrome são vinculados ao aplicativo.", + "profileLabel": "Perfil de origem", + "profilePlaceholder": "Selecione um perfil", + "noProfiles": "Nenhum perfil compatível do Chrome ou Arc com dados de cookies foi encontrado.", + "cookies": "Cookies e sessões", + "supported": "Incluídos nesta importação", + "otherData": "Senhas e armazenamento da web", + "notIncluded": "Não incluídos nesta versão", + "preview": "Autorizar e visualizar", + "previewTitle": "Pronto para substituir os cookies do YoBrowser", + "previewDescription": "É possível importar {count} cookies. Os cookies existentes do YoBrowser serão substituídos.", + "skipped": "Ignorados: {expired} expirados, {partitioned} particionados.", + "confirm": "Substituir e importar", + "doneTitle": "Importação concluída", + "doneDescription": "{count} cookies importados. As páginas abertas no navegador foram recarregadas.", + "keyDenied": "O acesso às Chaves foi negado ou não foi possível ler a chave do navegador.", + "previewExpired": "A fonte mudou ou a visualização expirou. Crie uma nova visualização.", + "encryptionUnsupported": "Este perfil usa um formato de criptografia que não é compatível com este importador.", + "failed": "A importação falhou antes da conclusão. Os cookies existentes do YoBrowser foram restaurados caso a alteração já tivesse começado." + } }, "databaseRepair": { "title": "Reparo do banco de dados", diff --git a/src/renderer/src/i18n/ru-RU/settings.json b/src/renderer/src/i18n/ru-RU/settings.json index 5b378d2685..0fa0fb2434 100644 --- a/src/renderer/src/i18n/ru-RU/settings.json +++ b/src/renderer/src/i18n/ru-RU/settings.json @@ -219,7 +219,33 @@ "confirmDescription": "Файлы cookie, кеш, LocalStorage, IndexedDB и другие данные сеанса YoBrowser будут удалены, и вам, возможно, придется снова войти на веб-сайт.", "confirmTitle": "Подтвердить очистку данных YoBrowser?", "description": "Независимая песочница браузера, которая не использует файлы cookie и локальное хранилище совместно с основным приложением. Данные YoBrowser можно очистить одним щелчком мыши здесь.", - "title": "Песочница YoBrowser" + "title": "Песочница YoBrowser", + "import": { + "button": "Импортировать данные сайтов", + "title": "Импортировать данные сеанса браузера", + "description": "Выберите профиль Chrome или Arc. macOS может запросить разрешение на доступ к Связке ключей. Исходные данные не изменяются.", + "scanning": "Поиск профилей браузера...", + "unsupportedTitle": "Не поддерживается на этой платформе", + "unsupportedDescription": "Прямой импорт данных браузера пока доступен только в macOS. Windows не поддерживается, поскольку текущие данные Chrome привязаны к приложению.", + "profileLabel": "Исходный профиль", + "profilePlaceholder": "Выберите профиль", + "noProfiles": "Поддерживаемые профили Chrome или Arc с файлами cookie не найдены.", + "cookies": "Файлы cookie и сеансы", + "supported": "Включено в этот импорт", + "otherData": "Пароли и веб-хранилище", + "notIncluded": "Не включено в эту версию", + "preview": "Разрешить и просмотреть", + "previewTitle": "Всё готово к замене файлов cookie YoBrowser", + "previewDescription": "Можно импортировать файлов cookie: {count}. Существующие файлы cookie YoBrowser будут заменены.", + "skipped": "Пропущено: просроченных — {expired}, секционированных — {partitioned}.", + "confirm": "Заменить и импортировать", + "doneTitle": "Импорт завершён", + "doneDescription": "Импортировано файлов cookie: {count}. Открытые страницы браузера перезагружены.", + "keyDenied": "В доступе к Связке ключей отказано или не удалось прочитать ключ браузера.", + "previewExpired": "Источник изменился или срок действия предпросмотра истёк. Создайте новый предпросмотр.", + "encryptionUnsupported": "Этот профиль использует формат шифрования, который не поддерживается средством импорта.", + "failed": "Не удалось завершить импорт. Если изменение уже началось, существующие файлы cookie YoBrowser были восстановлены." + } }, "databaseRepair": { "title": "Восстановление базы данных", diff --git a/src/renderer/src/i18n/tr-TR/settings.json b/src/renderer/src/i18n/tr-TR/settings.json index 8f068f930a..6ae663f2ac 100644 --- a/src/renderer/src/i18n/tr-TR/settings.json +++ b/src/renderer/src/i18n/tr-TR/settings.json @@ -547,7 +547,33 @@ "clearedTitle": "YoBrowser verileri temizlendi", "clearedDescription": "Korumalı alan oturumu sıfırlandı ve ilgili sekme yeniden yüklenecek.", "clearFailedTitle": "Temizleme başarısız oldu", - "clearFailedDescription": "Lütfen tekrar deneyin veya daha fazla bilgi için günlükleri kontrol edin." + "clearFailedDescription": "Lütfen tekrar deneyin veya daha fazla bilgi için günlükleri kontrol edin.", + "import": { + "button": "Import website data", + "title": "Import browser session data", + "description": "Choose a Chrome or Arc profile. macOS may ask you to authorize Keychain access. The source is never modified.", + "scanning": "Looking for browser profiles...", + "unsupportedTitle": "Not supported on this platform", + "unsupportedDescription": "Direct browser data import is currently available only on macOS. Windows remains unsupported because current Chrome data is app-bound.", + "profileLabel": "Source profile", + "profilePlaceholder": "Select a profile", + "noProfiles": "No supported Chrome or Arc profiles with cookie data were found.", + "cookies": "Cookies and sessions", + "supported": "Included in this import", + "otherData": "Passwords and web storage", + "notIncluded": "Not included in this version", + "preview": "Authorize and preview", + "previewTitle": "Ready to replace YoBrowser cookies", + "previewDescription": "{count} cookies can be imported. Existing YoBrowser cookies will be replaced.", + "skipped": "Skipped: {expired} expired, {partitioned} partitioned.", + "confirm": "Replace and import", + "doneTitle": "Import complete", + "doneDescription": "Imported {count} cookies. Open browser pages were reloaded.", + "keyDenied": "Keychain access was denied or the browser key could not be read.", + "previewExpired": "The source changed or the preview expired. Create a new preview.", + "encryptionUnsupported": "This profile uses an encryption format that this importer does not support.", + "failed": "Import failed before completion. Existing YoBrowser cookies were restored when mutation had started." + } }, "privacyTitle": "Veri ve Gizlilik", "privacyDescription": "Yedeklemeyi, senkronizasyonu, gizlilik modunu, bakımı ve tehlikeli işlemleri yönetin.", diff --git a/src/renderer/src/i18n/vi-VN/settings.json b/src/renderer/src/i18n/vi-VN/settings.json index e41945e3b6..f30c90f986 100644 --- a/src/renderer/src/i18n/vi-VN/settings.json +++ b/src/renderer/src/i18n/vi-VN/settings.json @@ -547,7 +547,33 @@ "confirmDescription": "Cookie, bộ nhớ đệm, LocalStorage, IndexedDB và các dữ liệu khác của phiên YoBrowser sẽ bị xóa và bạn có thể cần phải đăng nhập lại vào trang web.", "confirmTitle": "Xác nhận xóa dữ liệu YoBrowser?", "description": "Hộp cát trình duyệt độc lập không chia sẻ cookie và bộ nhớ cục bộ với ứng dụng chính. \nDữ liệu YoBrowser có thể bị xóa chỉ bằng một cú nhấp chuột vào đây.", - "title": "Hộp cát YoBrowser" + "title": "Hộp cát YoBrowser", + "import": { + "button": "Import website data", + "title": "Import browser session data", + "description": "Choose a Chrome or Arc profile. macOS may ask you to authorize Keychain access. The source is never modified.", + "scanning": "Looking for browser profiles...", + "unsupportedTitle": "Not supported on this platform", + "unsupportedDescription": "Direct browser data import is currently available only on macOS. Windows remains unsupported because current Chrome data is app-bound.", + "profileLabel": "Source profile", + "profilePlaceholder": "Select a profile", + "noProfiles": "No supported Chrome or Arc profiles with cookie data were found.", + "cookies": "Cookies and sessions", + "supported": "Included in this import", + "otherData": "Passwords and web storage", + "notIncluded": "Not included in this version", + "preview": "Authorize and preview", + "previewTitle": "Ready to replace YoBrowser cookies", + "previewDescription": "{count} cookies can be imported. Existing YoBrowser cookies will be replaced.", + "skipped": "Skipped: {expired} expired, {partitioned} partitioned.", + "confirm": "Replace and import", + "doneTitle": "Import complete", + "doneDescription": "Imported {count} cookies. Open browser pages were reloaded.", + "keyDenied": "Keychain access was denied or the browser key could not be read.", + "previewExpired": "The source changed or the preview expired. Create a new preview.", + "encryptionUnsupported": "This profile uses an encryption format that this importer does not support.", + "failed": "Import failed before completion. Existing YoBrowser cookies were restored when mutation had started." + } }, "privacyTitle": "Dữ liệu & Quyền riêng tư", "privacyDescription": "Quản lý sao lưu, đồng bộ hóa, chế độ riêng tư, bảo trì và các hoạt động nguy hiểm.", diff --git a/src/renderer/src/i18n/zh-CN/settings.json b/src/renderer/src/i18n/zh-CN/settings.json index 8e806d61a4..0b9c247615 100644 --- a/src/renderer/src/i18n/zh-CN/settings.json +++ b/src/renderer/src/i18n/zh-CN/settings.json @@ -588,7 +588,33 @@ "clearedTitle": "已清空 YoBrowser 数据", "clearedDescription": "沙盒会话已重置，相关标签页将重新加载。", "clearFailedTitle": "清空失败", - "clearFailedDescription": "请重试或查看日志获取更多信息。" + "clearFailedDescription": "请重试或查看日志获取更多信息。", + "import": { + "button": "导入网站数据", + "title": "导入浏览器会话数据", + "description": "选择 Chrome 或 Arc 配置。macOS 可能要求授权访问钥匙串，源浏览器数据不会被修改。", + "scanning": "正在查找浏览器配置…", + "unsupportedTitle": "当前平台暂不支持", + "unsupportedDescription": "直接导入目前仅支持 macOS。Windows 的新版 Chrome 数据受应用绑定保护，因此不提供导入。", + "profileLabel": "来源配置", + "profilePlaceholder": "选择一个配置", + "noProfiles": "没有找到包含 Cookie 数据的 Chrome 或 Arc 配置。", + "cookies": "Cookie 与登录会话", + "supported": "本次导入包含", + "otherData": "密码与网页存储", + "notIncluded": "当前版本暂不包含", + "preview": "授权并预览", + "previewTitle": "可以替换 YoBrowser Cookie", + "previewDescription": "可导入 {count} 个 Cookie。现有 YoBrowser Cookie 将被覆盖。", + "skipped": "已跳过：过期 {expired} 个，分区 Cookie {partitioned} 个。", + "confirm": "覆盖并导入", + "doneTitle": "导入完成", + "doneDescription": "已导入 {count} 个 Cookie，并重新加载已打开的浏览器页面。", + "keyDenied": "钥匙串授权被拒绝，或无法读取浏览器密钥。", + "previewExpired": "来源数据已变化或预览已过期，请重新预览。", + "encryptionUnsupported": "该配置使用了当前导入器不支持的加密格式。", + "failed": "导入未能完成；如果已开始覆盖，原有 YoBrowser Cookie 已恢复。" + } }, "privacyTitle": "Data & Privacy", "privacyDescription": "管理备份同步、隐私模式、数据维护与危险操作。", diff --git a/src/renderer/src/i18n/zh-HK/settings.json b/src/renderer/src/i18n/zh-HK/settings.json index 47d9a15143..71bb4cb630 100644 --- a/src/renderer/src/i18n/zh-HK/settings.json +++ b/src/renderer/src/i18n/zh-HK/settings.json @@ -219,7 +219,33 @@ "confirmDescription": "將刪除 YoBrowser 會話的 Cookie、緩存、LocalStorage、IndexedDB 等數據，可能需要重新登錄網站。", "confirmTitle": "確認清空 YoBrowser 數據？", "description": "獨立的瀏覽器沙盒，不與主應用共享 Cookie 和本地存儲。\n可在此一鍵清空 YoBrowser 數據。", - "title": "YoBrowser 沙盒" + "title": "YoBrowser 沙盒", + "import": { + "button": "Import website data", + "title": "Import browser session data", + "description": "Choose a Chrome or Arc profile. macOS may ask you to authorize Keychain access. The source is never modified.", + "scanning": "Looking for browser profiles...", + "unsupportedTitle": "Not supported on this platform", + "unsupportedDescription": "Direct browser data import is currently available only on macOS. Windows remains unsupported because current Chrome data is app-bound.", + "profileLabel": "Source profile", + "profilePlaceholder": "Select a profile", + "noProfiles": "No supported Chrome or Arc profiles with cookie data were found.", + "cookies": "Cookies and sessions", + "supported": "Included in this import", + "otherData": "Passwords and web storage", + "notIncluded": "Not included in this version", + "preview": "Authorize and preview", + "previewTitle": "Ready to replace YoBrowser cookies", + "previewDescription": "{count} cookies can be imported. Existing YoBrowser cookies will be replaced.", + "skipped": "Skipped: {expired} expired, {partitioned} partitioned.", + "confirm": "Replace and import", + "doneTitle": "Import complete", + "doneDescription": "Imported {count} cookies. Open browser pages were reloaded.", + "keyDenied": "Keychain access was denied or the browser key could not be read.", + "previewExpired": "The source changed or the preview expired. Create a new preview.", + "encryptionUnsupported": "This profile uses an encryption format that this importer does not support.", + "failed": "Import failed before completion. Existing YoBrowser cookies were restored when mutation had started." + } }, "databaseRepair": { "title": "資料庫修復", diff --git a/src/renderer/src/i18n/zh-TW/settings.json b/src/renderer/src/i18n/zh-TW/settings.json index 1ea922ae5d..09630a1a92 100644 --- a/src/renderer/src/i18n/zh-TW/settings.json +++ b/src/renderer/src/i18n/zh-TW/settings.json @@ -219,7 +219,33 @@ "confirmDescription": "將刪除 YoBrowser 會話的 Cookie、緩存、LocalStorage、IndexedDB 等數據，可能需要重新登錄網站。", "confirmTitle": "確認清空 YoBrowser 數據？", "description": "獨立的瀏覽器沙盒，不與主應用共享 Cookie 和本地存儲。\n可在此一鍵清空 YoBrowser 數據。", - "title": "YoBrowser 沙盒" + "title": "YoBrowser 沙盒", + "import": { + "button": "Import website data", + "title": "Import browser session data", + "description": "Choose a Chrome or Arc profile. macOS may ask you to authorize Keychain access. The source is never modified.", + "scanning": "Looking for browser profiles...", + "unsupportedTitle": "Not supported on this platform", + "unsupportedDescription": "Direct browser data import is currently available only on macOS. Windows remains unsupported because current Chrome data is app-bound.", + "profileLabel": "Source profile", + "profilePlaceholder": "Select a profile", + "noProfiles": "No supported Chrome or Arc profiles with cookie data were found.", + "cookies": "Cookies and sessions", + "supported": "Included in this import", + "otherData": "Passwords and web storage", + "notIncluded": "Not included in this version", + "preview": "Authorize and preview", + "previewTitle": "Ready to replace YoBrowser cookies", + "previewDescription": "{count} cookies can be imported. Existing YoBrowser cookies will be replaced.", + "skipped": "Skipped: {expired} expired, {partitioned} partitioned.", + "confirm": "Replace and import", + "doneTitle": "Import complete", + "doneDescription": "Imported {count} cookies. Open browser pages were reloaded.", + "keyDenied": "Keychain access was denied or the browser key could not be read.", + "previewExpired": "The source changed or the preview expired. Create a new preview.", + "encryptionUnsupported": "This profile uses an encryption format that this importer does not support.", + "failed": "Import failed before completion. Existing YoBrowser cookies were restored when mutation had started." + } }, "databaseRepair": { "title": "資料庫修復", diff --git a/src/renderer/src/lib/icons/icon-collections.generated.ts b/src/renderer/src/lib/icons/icon-collections.generated.ts index 5e88a411ca..e036858e96 100644 --- a/src/renderer/src/lib/icons/icon-collections.generated.ts +++ b/src/renderer/src/lib/icons/icon-collections.generated.ts @@ -353,6 +353,9 @@ export const lucideIconCollection = { images: { body: '' }, + import: { + body: '' + }, inbox: { body: '' }, @@ -395,9 +398,6 @@ export const lucideIconCollection = { 'list-tree': { body: '' }, - loader: { - body: '' - }, 'loader-circle': { body: '' }, @@ -479,6 +479,9 @@ export const lucideIconCollection = { 'panel-right-close': { body: '' }, + 'panel-right-open': { + body: '' + }, paperclip: { body: '' }, @@ -724,9 +727,6 @@ export const lucideIconCollection = { 'help-circle': { parent: 'circle-question-mark' }, - 'loader-2': { - parent: 'loader-circle' - }, 'message-circle-question': { parent: 'message-circle-question-mark' }, @@ -746,7 +746,7 @@ export const lucideIconCollection = { parent: 'circle-x' } }, - lastModified: 1782727192, + lastModified: 1778908382, width: 24, height: 24 } as const @@ -807,7 +807,7 @@ export const vscodeIconCollection = { } }, aliases: {}, - lastModified: 1782727390, + lastModified: 1779168064, width: 32, height: 32 } as const diff --git a/src/renderer/src/lib/icons/icon-whitelist.generated.ts b/src/renderer/src/lib/icons/icon-whitelist.generated.ts index e6e57a70ea..8ae68d8513 100644 --- a/src/renderer/src/lib/icons/icon-whitelist.generated.ts +++ b/src/renderer/src/lib/icons/icon-whitelist.generated.ts @@ -124,6 +124,7 @@ export const GENERATED_ICON_WHITELIST: Record { const clampWidth = (nextWidth: number) => { const maxWidth = resolveMaxWidth() - const minWidth = Math.min(420, maxWidth) + const minWidth = Math.min(360, maxWidth) const widthValue = Number(nextWidth) if (!Number.isFinite(widthValue)) { return Math.min(maxWidth, Math.max(minWidth, 520)) diff --git a/src/renderer/src/views/ChatTabView.vue b/src/renderer/src/views/ChatTabView.vue index ace6dc4c06..fb5f2245eb 100644 --- a/src/renderer/src/views/ChatTabView.vue +++ b/src/renderer/src/views/ChatTabView.vue @@ -29,6 +29,9 @@ + ( + (value) => value instanceof Uint8Array && value.byteLength <= 512 * 1024 + ), + timestamp: TimestampMsSchema + }) +}) diff --git a/src/shared/contracts/routes.ts b/src/shared/contracts/routes.ts index ff7c690a24..0c847ceea2 100644 --- a/src/shared/contracts/routes.ts +++ b/src/shared/contracts/routes.ts @@ -3,6 +3,7 @@ import type { RouteContract } from './common' import { acpTerminalInputRoute, acpTerminalKillRoute } from './routes/acp-terminal.routes' import { browserAttachCurrentWindowRoute, + browserApplyImportRoute, browserClearSandboxDataRoute, browserDestroyRoute, browserDetachRoute, @@ -10,7 +11,10 @@ import { browserGoBackRoute, browserGoForwardRoute, browserLoadUrlRoute, + browserPreviewImportRoute, browserReloadRoute, + browserScanImportSourcesRoute, + browserSetPreviewModeRoute, browserUpdateCurrentWindowBoundsRoute } from './routes/browser.routes' import { @@ -679,6 +683,10 @@ const DEEPCHAT_ROUTE_CATALOG_PART_2 = { [browserGoForwardRoute.name]: browserGoForwardRoute, [browserReloadRoute.name]: browserReloadRoute, [browserClearSandboxDataRoute.name]: browserClearSandboxDataRoute, + [browserScanImportSourcesRoute.name]: browserScanImportSourcesRoute, + [browserSetPreviewModeRoute.name]: browserSetPreviewModeRoute, + [browserPreviewImportRoute.name]: browserPreviewImportRoute, + [browserApplyImportRoute.name]: browserApplyImportRoute, [tabCaptureCurrentAreaRoute.name]: tabCaptureCurrentAreaRoute, [tabStitchImagesWithWatermarkRoute.name]: tabStitchImagesWithWatermarkRoute } satisfies Record diff --git a/src/shared/contracts/routes/browser.routes.ts b/src/shared/contracts/routes/browser.routes.ts index b7a577f983..c3a81c16d1 100644 --- a/src/shared/contracts/routes/browser.routes.ts +++ b/src/shared/contracts/routes/browser.routes.ts @@ -56,6 +56,18 @@ export const browserDetachRoute = defineRouteContract({ }) }) +export const browserSetPreviewModeRoute = defineRouteContract({ + name: 'browser.setPreviewMode', + input: z.object({ + sessionId: z.string().min(1), + mode: z.enum(['capturing', 'rendering', 'stopped']), + runId: z.string().min(1).optional() + }), + output: z.object({ + updated: z.boolean() + }) +}) + export const browserDestroyRoute = defineRouteContract({ name: 'browser.destroy', input: z.object({ @@ -103,3 +115,55 @@ export const browserClearSandboxDataRoute = defineRouteContract({ cleared: z.boolean() }) }) + +const BrowserImportCapabilityReasonSchema = z.enum([ + 'platform_unsupported', + 'browser_not_found', + 'profile_data_missing' +]) + +const BrowserImportProfileSchema = z.object({ + id: z.string().min(1), + browser: z.enum(['chrome', 'arc']), + browserName: z.string().min(1), + profileName: z.string().min(1), + supported: z.boolean(), + reason: BrowserImportCapabilityReasonSchema.optional() +}) + +export const browserScanImportSourcesRoute = defineRouteContract({ + name: 'browser.import.scan', + input: z.object({}).default({}), + output: z.object({ + platformSupported: z.boolean(), + profiles: z.array(BrowserImportProfileSchema), + reason: BrowserImportCapabilityReasonSchema.optional() + }) +}) + +export const browserPreviewImportRoute = defineRouteContract({ + name: 'browser.import.preview', + input: z.object({ + profileId: z.string().min(1) + }), + output: z.object({ + token: z.string().min(1), + profile: BrowserImportProfileSchema, + cookieCount: z.number().int().nonnegative(), + skippedExpired: z.number().int().nonnegative(), + skippedPartitioned: z.number().int().nonnegative() + }) +}) + +export const browserApplyImportRoute = defineRouteContract({ + name: 'browser.import.apply', + input: z.object({ + token: z.string().min(1) + }), + output: z.object({ + importedCookies: z.number().int().nonnegative(), + skippedExpired: z.number().int().nonnegative(), + skippedPartitioned: z.number().int().nonnegative(), + syncedAt: z.number().int().nonnegative() + }) +}) diff --git a/src/shared/types/browser.ts b/src/shared/types/browser.ts index dbd5bfa99b..2f8e7ac66b 100644 --- a/src/shared/types/browser.ts +++ b/src/shared/types/browser.ts @@ -23,6 +23,43 @@ export interface YoBrowserStatus { canGoForward: boolean visible: boolean loading: boolean + owner?: 'agent' | 'user' + agentRunId?: string +} + +export type BrowserImportCapabilityReason = + | 'platform_unsupported' + | 'browser_not_found' + | 'profile_data_missing' + +export interface BrowserImportProfile { + id: string + browser: 'chrome' | 'arc' + browserName: string + profileName: string + supported: boolean + reason?: BrowserImportCapabilityReason +} + +export interface BrowserImportScanResult { + platformSupported: boolean + profiles: BrowserImportProfile[] + reason?: BrowserImportCapabilityReason +} + +export interface BrowserImportPreview { + token: string + profile: BrowserImportProfile + cookieCount: number + skippedExpired: number + skippedPartitioned: number +} + +export interface BrowserImportApplyResult { + importedCookies: number + skippedExpired: number + skippedPartitioned: number + syncedAt: number } export interface ScreenshotOptions { diff --git a/src/shared/types/desktop.ts b/src/shared/types/desktop.ts index 3db2b60068..c1f59953a2 100644 --- a/src/shared/types/desktop.ts +++ b/src/shared/types/desktop.ts @@ -1,5 +1,13 @@ import type { BrowserWindow, WebContents, WebContentsView } from 'electron' -import type { BrowserPageInfo, DownloadInfo, ScreenshotOptions, YoBrowserStatus } from './browser' +import type { + BrowserImportApplyResult, + BrowserImportPreview, + BrowserImportScanResult, + BrowserPageInfo, + DownloadInfo, + ScreenshotOptions, + YoBrowserStatus +} from './browser' import type { MCPToolDefinition } from './mcp' import type { ProviderInstallPreview } from '@shared/providerDeeplink' import type { SettingsNavigationPayload } from '@shared/settingsNavigation' @@ -113,6 +121,12 @@ export interface IYoBrowserPresenter { visible: boolean ): Promise detachSessionBrowser(sessionId: string): Promise + setPreviewMode( + sessionId: string, + mode: 'capturing' | 'rendering' | 'stopped', + hostWindowId?: number, + runId?: string + ): Promise destroySessionBrowser(sessionId: string): Promise goBack(sessionId: string): Promise goForward(sessionId: string): Promise @@ -125,13 +139,17 @@ export interface IYoBrowserPresenter { getBrowserPage(sessionId: string): Promise startDownload(url: string, savePath?: string): Promise clearSandboxData(): Promise + scanImportSources(): Promise + previewImport(profileId: string): Promise + applyImport(token: string): Promise shutdown(): Promise readonly toolHandler: { getToolDefinitions(): MCPToolDefinition[] callTool( toolName: string, args: Record, - conversationId?: string + conversationId?: string, + runId?: string ): Promise } } diff --git a/src/shared/types/tool.d.ts b/src/shared/types/tool.d.ts index f42256b627..0ffef8eeb8 100644 --- a/src/shared/types/tool.d.ts +++ b/src/shared/types/tool.d.ts @@ -33,6 +33,7 @@ export interface ToolDefinitionContext { } export interface ToolCallOptions { + runId?: string onProgress?: (update: AgentToolProgressUpdate) => void signal?: AbortSignal permissionMode?: PermissionMode diff --git a/test/main/desktop/browser/BrowserProfileImportService.test.ts b/test/main/desktop/browser/BrowserProfileImportService.test.ts new file mode 100644 index 0000000000..9d2805d050 --- /dev/null +++ b/test/main/desktop/browser/BrowserProfileImportService.test.ts @@ -0,0 +1,195 @@ +import { mkdtemp, rm, stat, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { createCipheriv, createHash, pbkdf2Sync } from 'node:crypto' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { Cookie, CookiesSetDetails, Session } from 'electron' +import { BrowserProfileImportService } from '@/desktop/browser/BrowserProfileImportService' + +const temporaryDirectories: string[] = [] + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true, force: true })) + ) +}) + +const createCookie = (details: CookiesSetDetails): Cookie => ({ + name: details.name, + value: details.value, + domain: details.domain ?? new URL(details.url).hostname, + hostOnly: details.domain === undefined, + path: details.path ?? '/', + secure: details.secure ?? false, + httpOnly: details.httpOnly ?? false, + session: details.expirationDate === undefined, + sameSite: details.sameSite ?? 'unspecified' +}) + +const createSession = (options?: { corruptImportedValue?: boolean }) => { + let cookies: Cookie[] = [ + createCookie({ + url: 'https://old.example/', + name: 'old', + value: 'old-value' + }) + ] + const set = vi.fn(async (details: CookiesSetDetails) => { + const nextCookie = createCookie({ + ...details, + value: options?.corruptImportedValue && details.name === 'session' ? 'corrupt' : details.value + }) + const identity = (cookie: Cookie) => `${cookie.domain}\0${cookie.path}\0${cookie.name}` + cookies = cookies.filter((cookie) => identity(cookie) !== identity(nextCookie)) + if ((details.expirationDate ?? Number.POSITIVE_INFINITY) > Date.now() / 1000) { + cookies.push(nextCookie) + } + }) + const target = { + clearStorageData: vi.fn(), + cookies: { + get: vi.fn(async () => [...cookies]), + set, + flushStore: vi.fn(async () => undefined) + } + } as unknown as Session + const readCookies = () => cookies + const readUnpartitionedCookies = async () => + readCookies().map((cookie) => ({ + url: `${cookie.secure ? 'https' : 'http'}://${cookie.domain?.replace(/^\./, '')}${cookie.path}`, + name: cookie.name, + value: cookie.value, + path: cookie.path, + secure: cookie.secure, + httpOnly: cookie.httpOnly, + sameSite: cookie.sameSite, + ...(cookie.hostOnly ? {} : { domain: cookie.domain }), + ...(cookie.session ? {} : { expirationDate: cookie.expirationDate }) + })) + return { target, readCookies, readUnpartitionedCookies } +} + +const stageImport = async (service: BrowserProfileImportService) => { + const directory = await mkdtemp(join(tmpdir(), 'deepchat-import-test-')) + temporaryDirectories.push(directory) + const cookiePath = join(directory, 'Cookies') + await writeFile(cookiePath, 'snapshot') + const sourceStat = await stat(cookiePath) + + const internals = service as unknown as { + stagedImports: Map + } + internals.stagedImports.set('preview-token', { + createdAt: Date.now(), + profile: { + id: 'chrome:Default', + browser: 'chrome', + browserName: 'Google Chrome', + profileName: 'Default', + supported: true, + source: {}, + directoryName: 'Default', + cookiePath + }, + sourceSize: sourceStat.size, + sourceMtimeMs: sourceStat.mtimeMs, + cookies: [ + { + url: 'https://example.com/', + name: 'session', + value: 'session-value', + path: '/', + secure: true, + httpOnly: true, + sameSite: 'lax' + } + ], + skippedExpired: 2, + skippedPartitioned: 1 + }) +} + +describe('BrowserProfileImportService', () => { + it('reports direct import as unsupported outside macOS', async () => { + const service = new BrowserProfileImportService( + () => ({}) as Session, + async () => [], + 'win32' + ) + + await expect(service.scan()).resolves.toEqual({ + platformSupported: false, + profiles: [], + reason: 'platform_unsupported' + }) + }) + + it('decrypts a macOS Chromium v10 cookie with the schema host digest', () => { + const service = new BrowserProfileImportService( + () => ({}) as Session, + async () => [], + 'darwin' + ) + const host = '.example.com' + const key = pbkdf2Sync('keychain-password', 'saltysalt', 1003, 16, 'sha1') + const cipher = createCipheriv('aes-128-cbc', key, Buffer.alloc(16, 0x20)) + const plaintext = Buffer.concat([ + createHash('sha256').update(host).digest(), + Buffer.from('session-value') + ]) + const encryptedValue = Buffer.concat([ + Buffer.from('v10'), + cipher.update(plaintext), + cipher.final() + ]) + + const internals = service as unknown as { + decryptChromeCookie( + row: { host_key: string; encrypted_value: Buffer }, + key: Buffer, + schemaVersion: number + ): string + } + expect( + internals.decryptChromeCookie({ host_key: host, encrypted_value: encryptedValue }, key, 24) + ).toBe('session-value') + }) + + it('replaces target cookies and verifies the imported values', async () => { + const { target, readCookies, readUnpartitionedCookies } = createSession() + const service = new BrowserProfileImportService( + () => target, + readUnpartitionedCookies, + 'darwin' + ) + await stageImport(service) + + await expect(service.apply('preview-token')).resolves.toMatchObject({ + importedCookies: 1, + skippedExpired: 2, + skippedPartitioned: 1 + }) + expect(readCookies()).toHaveLength(1) + expect(readCookies()[0]).toMatchObject({ name: 'session', value: 'session-value' }) + expect(target.clearStorageData).not.toHaveBeenCalled() + }) + + it('restores target cookies when readback verification fails', async () => { + const { target, readCookies, readUnpartitionedCookies } = createSession({ + corruptImportedValue: true + }) + const service = new BrowserProfileImportService( + () => target, + readUnpartitionedCookies, + 'darwin' + ) + await stageImport(service) + + await expect(service.apply('preview-token')).rejects.toThrow( + 'browser_import_verification_failed' + ) + expect(readCookies()).toHaveLength(1) + expect(readCookies()[0]).toMatchObject({ name: 'old', value: 'old-value' }) + expect(target.clearStorageData).not.toHaveBeenCalled() + }) +}) diff --git a/test/main/desktop/browser/YoBrowserPresenter.test.ts b/test/main/desktop/browser/YoBrowserPresenter.test.ts index 4eb32f1ec2..cfaf9eb95c 100644 --- a/test/main/desktop/browser/YoBrowserPresenter.test.ts +++ b/test/main/desktop/browser/YoBrowserPresenter.test.ts @@ -51,6 +51,12 @@ class MockWebContents extends EventEmitter { this.emit('destroyed') }) sendInputEvent = vi.fn() + setBackgroundThrottling = vi.fn() + capturePage = vi.fn(async () => ({ + resize: vi.fn(() => ({ + toJPEG: vi.fn(() => Buffer.from('preview-frame')) + })) + })) constructor(id: number) { super() @@ -126,6 +132,21 @@ class MockBrowserWindow extends EventEmitter { } } +class MockBaseWindow extends EventEmitter { + contentView = new MockContentView() + destroyed = false + setIgnoreMouseEvents = vi.fn() + + isDestroyed() { + return this.destroyed + } + + destroy() { + this.destroyed = true + this.emit('closed') + } +} + describe('YoBrowserPresenter', () => { beforeEach(() => { vi.resetModules() @@ -145,6 +166,7 @@ describe('YoBrowserPresenter', () => { let nextWebContentsId = 100 const windows = new Map() const viewConfigs: Array> = [] + const previewHosts: MockBaseWindow[] = [] vi.doMock('electron', () => { class MockWebContentsView { @@ -152,6 +174,7 @@ describe('YoBrowserPresenter', () => { setBorderRadius = vi.fn() setBackgroundColor = vi.fn() setBounds = vi.fn() + setVisible = vi.fn() constructor(options: Record) { viewConfigs.push(options) @@ -163,6 +186,12 @@ describe('YoBrowserPresenter', () => { app: { getPath: vi.fn(() => 'C:/mock-user-data') }, + BaseWindow: class extends MockBaseWindow { + constructor() { + super() + previewHosts.push(this) + } + }, BrowserWindow: { fromId: (id: number) => windows.get(id) ?? null }, @@ -186,6 +215,7 @@ describe('YoBrowserPresenter', () => { vi.doMock('@/desktop/browser/yoBrowserSession', () => ({ getYoBrowserSession: () => ({}), + getYoBrowserUnpartitionedCookies: vi.fn(async () => []), clearYoBrowserSessionData: vi.fn() })) @@ -215,7 +245,11 @@ describe('YoBrowserPresenter', () => { }), closeWindow: vi.fn(async () => undefined), getFocusedWindow: vi.fn(() => windows.get(1) ?? null), - getAllWindows: vi.fn(() => Array.from(windows.values())) + getAllWindows: vi.fn(() => Array.from(windows.values())), + sendToWindow: vi.fn((windowId: number, channel: string, envelope: unknown) => { + sendToAllWindowsMock(channel, envelope, windowId) + return true + }) } const presenter = new YoBrowserPresenter(windowPresenter as any, (name, payload) => { @@ -231,6 +265,7 @@ describe('YoBrowserPresenter', () => { presenter, windows, viewConfigs, + previewHosts, getSessionWebContents } } @@ -425,6 +460,75 @@ describe('YoBrowserPresenter', () => { expect(JSON.stringify(activityPayloads)).not.toContain('example.com') }) + it('captures a bounded preview frame without creating a second page', async () => { + const { presenter, windows, previewHosts, getSessionWebContents } = await setupPresenter() + windows.set(1, new MockBrowserWindow(1)) + + const loadPromise = presenter.loadUrl( + 'session-a', + 'https://example.com', + undefined, + 1, + 'agent', + 'run-1' + ) + await Promise.resolve() + const webContents = getSessionWebContents('session-a') + webContents?.emitDomReady() + await loadPromise + sendToAllWindowsMock.mockClear() + + expect(await presenter.setPreviewMode('session-a', 'capturing', 1, 'run-1')).toBe(true) + await vi.advanceTimersByTimeAsync(1) + + expect(previewHosts).toHaveLength(1) + expect(webContents?.capturePage).toHaveBeenCalledWith( + { x: 0, y: 0, width: 1280, height: 800 }, + { stayHidden: true } + ) + expect(sendToAllWindowsMock).toHaveBeenCalledWith( + 'deepchat:event', + expect.objectContaining({ + name: 'browser.preview.frame', + payload: expect.objectContaining({ + sessionId: 'session-a', + runId: 'run-1', + width: 480, + height: 300, + mimeType: 'image/jpeg' + }) + }), + 1 + ) + + const captureCount = webContents?.capturePage.mock.calls.length + expect(await presenter.setPreviewMode('session-a', 'rendering', 1, 'run-1')).toBe(true) + await vi.advanceTimersByTimeAsync(1100) + expect(webContents?.capturePage).toHaveBeenCalledTimes(captureCount ?? 0) + + expect(await presenter.setPreviewMode('session-a', 'stopped', 1, 'run-1')).toBe(true) + expect(previewHosts[0].destroyed).toBe(true) + }) + + it('rejects preview capture for a stale Agent run', async () => { + const { presenter, windows, getSessionWebContents } = await setupPresenter() + windows.set(1, new MockBrowserWindow(1)) + + const loadPromise = presenter.loadUrl( + 'session-a', + 'https://example.com', + undefined, + 1, + 'agent', + 'run-1' + ) + await Promise.resolve() + getSessionWebContents('session-a')?.emitDomReady() + await loadPromise + + expect(await presenter.setPreviewMode('session-a', 'capturing', 1, 'run-old')).toBe(false) + }) + it('maps agent CDP mouse and screenshot commands to overlay activity', async () => { const { presenter, windows, getSessionWebContents } = await setupPresenter() windows.set(1, new MockBrowserWindow(1)) @@ -555,7 +659,7 @@ describe('YoBrowserPresenter', () => { expect(extractPoint('document.querySelector("button")?.click()')).toBeUndefined() }) - it('returns focus to the host renderer after attaching the browser view', async () => { + it('does not steal focus from the attached browser view', async () => { const { presenter, windows, getSessionWebContents } = await setupPresenter() const hostWindow = new MockBrowserWindow(1) windows.set(1, hostWindow) @@ -568,7 +672,7 @@ describe('YoBrowserPresenter', () => { await presenter.attachSessionBrowser('session-a', 1) await Promise.resolve() - expect(hostWindow.webContents.focus).toHaveBeenCalled() + expect(hostWindow.webContents.focus).not.toHaveBeenCalled() }) it('does not show overlay activity when the host window is backgrounded', async () => { diff --git a/test/main/routes/contracts.test.ts b/test/main/routes/contracts.test.ts index be96045f22..ecc2147c37 100644 --- a/test/main/routes/contracts.test.ts +++ b/test/main/routes/contracts.test.ts @@ -72,6 +72,7 @@ describe('main kernel contracts', () => { 'acpTerminal.kill', 'browser.attachCurrentWindow', 'browser.clearSandboxData', + 'browser.setPreviewMode', 'databaseSecurity.repairSchema', 'debug.createMockChatSession', 'config.addManualAcpAgent', @@ -1700,6 +1701,7 @@ describe('main kernel contracts', () => { 'appRuntime.windowFocused', 'browser.activity.changed', 'browser.open.requested', + 'browser.preview.frame', 'browser.status.changed', 'chat.plan.updated', 'chat.stream.completed', @@ -1782,6 +1784,37 @@ describe('main kernel contracts', () => { expect(new Set(eventKeys).size).toBe(eventKeys.length) }) + it('accepts only byte arrays for browser preview frames', () => { + const payload = { + sessionId: 'session-1', + runId: 'run-1', + sequence: 0, + width: 480, + height: 300, + mimeType: 'image/jpeg', + timestamp: Date.now() + } as const + + expect( + DEEPCHAT_EVENT_CATALOG['browser.preview.frame'].payload.safeParse({ + ...payload, + data: new Uint8Array([1, 2, 3]) + }).success + ).toBe(true) + expect( + DEEPCHAT_EVENT_CATALOG['browser.preview.frame'].payload.safeParse({ + ...payload, + data: new Uint16Array([1, 2, 3]) + }).success + ).toBe(false) + expect( + DEEPCHAT_EVENT_CATALOG['browser.preview.frame'].payload.safeParse({ + ...payload, + data: new DataView(new ArrayBuffer(3)) + }).success + ).toBe(false) + }) + it('validates typed chat stream payloads', () => { expect(() => chatStreamUpdatedEvent.payload.parse({ diff --git a/test/renderer/api/clients.test.ts b/test/renderer/api/clients.test.ts index 09d317e099..099ec8d512 100644 --- a/test/renderer/api/clients.test.ts +++ b/test/renderer/api/clients.test.ts @@ -943,6 +943,29 @@ describe('renderer api clients', () => { return { updated: true } case 'browser.clearSandboxData': return { cleared: true } + case 'browser.import.scan': + return { platformSupported: true, profiles: [] } + case 'browser.import.preview': + return { + token: 'preview-token', + profile: { + id: payload?.profileId, + browser: 'chrome', + browserName: 'Google Chrome', + profileName: 'Default', + supported: true + }, + cookieCount: 3, + skippedExpired: 1, + skippedPartitioned: 0 + } + case 'browser.import.apply': + return { + importedCookies: 3, + skippedExpired: 1, + skippedPartitioned: 0, + syncedAt: 123 + } case 'window.closeSettings': return { closed: true } case 'window.focusMain': @@ -2089,6 +2112,36 @@ describe('renderer api clients', () => { expect(bridge.invoke).toHaveBeenCalledWith('browser.clearSandboxData', {}) }) + it('routes browser preview mode through the shared registry name', async () => { + const bridge = createBridge() + const browserClient = createBrowserClient(bridge) + + await browserClient.setPreviewMode('session-1', 'capturing', 'run-1') + + expect(bridge.invoke).toHaveBeenCalledWith('browser.setPreviewMode', { + sessionId: 'session-1', + mode: 'capturing', + runId: 'run-1' + }) + }) + + it('routes browser website-data import through typed registry names', async () => { + const bridge = createBridge() + const browserClient = createBrowserClient(bridge) + + await browserClient.scanImportSources() + await browserClient.previewImport('chrome:Default') + await browserClient.applyImport('preview-token') + + expect(bridge.invoke).toHaveBeenNthCalledWith(1, 'browser.import.scan', {}) + expect(bridge.invoke).toHaveBeenNthCalledWith(2, 'browser.import.preview', { + profileId: 'chrome:Default' + }) + expect(bridge.invoke).toHaveBeenNthCalledWith(3, 'browser.import.apply', { + token: 'preview-token' + }) + }) + it('routes database security operations through the shared registry names', async () => { const bridge = createBridge() const databaseSecurityClient = createDatabaseSecurityClient(bridge) @@ -2749,4 +2802,14 @@ describe('renderer api clients', () => { expect(bridge.on).toHaveBeenCalledWith('browser.activity.changed', listener) }) + + it('subscribes to browser preview frames', () => { + const bridge = createBridge() + const browserClient = createBrowserClient(bridge) + const listener = vi.fn() + + browserClient.onPreviewFrame(listener) + + expect(bridge.on).toHaveBeenCalledWith('browser.preview.frame', listener) + }) }) diff --git a/test/renderer/components/AgentBrowserPiP.test.ts b/test/renderer/components/AgentBrowserPiP.test.ts new file mode 100644 index 0000000000..676fa8740d --- /dev/null +++ b/test/renderer/components/AgentBrowserPiP.test.ts @@ -0,0 +1,285 @@ +import { defineComponent, nextTick, reactive } from 'vue' +import { flushPromises, mount } from '@vue/test-utils' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { YoBrowserStatus } from '@shared/types/browser' + +const mountedWrappers: Array<{ unmount: () => void }> = [] + +const createStatus = (runId = 'run-1'): YoBrowserStatus => ({ + initialized: true, + page: { + id: 'page-1', + url: 'https://example.com', + title: 'Example', + status: 'ready' as never, + createdAt: 1, + updatedAt: 1 + }, + canGoBack: false, + canGoForward: false, + visible: false, + loading: false, + owner: 'agent', + agentRunId: runId +}) + +const setup = async (options: { wide?: boolean } = {}) => { + vi.resetModules() + if (options.wide) { + vi.spyOn(HTMLElement.prototype, 'getBoundingClientRect').mockReturnValue({ + x: 0, + y: 0, + top: 0, + left: 0, + right: 800, + bottom: 600, + width: 800, + height: 600, + toJSON: () => ({}) + }) + } + let statusChangedHandler: + | ((payload: { sessionId: string; status: YoBrowserStatus | null }) => void) + | null = null + let previewFrameHandler: + | ((payload: { + sessionId: string + runId: string + sequence: number + width: number + height: number + mimeType: 'image/jpeg' + data: Uint8Array + timestamp: number + }) => void) + | null = null + let windowStateHandler: ((payload: { exists: boolean; isFocused: boolean }) => void) | null = null + const status = createStatus() + const browserClient = { + getStatus: vi.fn(async () => status), + setPreviewMode: vi.fn(async () => true), + onOpenRequestedForCurrentWindow: vi.fn(() => vi.fn()), + onStatusChanged: vi.fn( + (handler: (payload: { sessionId: string; status: YoBrowserStatus | null }) => void) => { + statusChangedHandler = handler + return vi.fn() + } + ), + onActivityChanged: vi.fn(() => vi.fn()), + onPreviewFrame: vi.fn((handler: NonNullable) => { + previewFrameHandler = handler + return vi.fn() + }) + } + const windowClient = { + getCurrentState: vi.fn(async () => ({ exists: true, isFocused: true })), + onCurrentStateChanged: vi.fn((handler: NonNullable) => { + windowStateHandler = handler + return vi.fn() + }) + } + const sidepanelStore = reactive({ + open: false, + activeTab: 'workspace', + openBrowser: vi.fn(() => { + sidepanelStore.open = true + sidepanelStore.activeTab = 'browser' + }) + }) + const sessionStore = reactive({ + sessions: [{ id: 'session-1', status: 'working' }] + }) + + vi.doMock('vue-i18n', () => ({ useI18n: () => ({ t: (key: string) => key }) })) + vi.doMock('@iconify/vue', () => ({ + Icon: defineComponent({ name: 'Icon', template: '' }) + })) + vi.doMock('@api/BrowserClient', () => ({ createBrowserClient: () => browserClient })) + vi.doMock('@api/WindowClient', () => ({ createWindowClient: () => windowClient })) + vi.doMock('@/stores/ui/sidepanel', () => ({ useSidepanelStore: () => sidepanelStore })) + vi.doMock('@/stores/ui/session', () => ({ useSessionStore: () => sessionStore })) + + const AgentBrowserPiP = (await import('@/components/browser/AgentBrowserPiP.vue')).default + const wrapper = mount(AgentBrowserPiP, { + props: { sessionId: 'session-1' }, + global: { + stubs: { + Button: defineComponent({ + name: 'Button', + emits: ['click'], + template: '' + }) + } + } + }) + mountedWrappers.push(wrapper) + await flushPromises() + + return { + wrapper, + browserClient, + sidepanelStore, + sessionStore, + emitStatus: statusChangedHandler!, + emitPreviewFrame: previewFrameHandler!, + emitWindowState: windowStateHandler! + } +} + +afterEach(async () => { + mountedWrappers.splice(0).forEach((wrapper) => wrapper.unmount()) + await flushPromises() + vi.restoreAllMocks() + vi.unstubAllGlobals() +}) + +describe('AgentBrowserPiP', () => { + it('shows a compact activity bar for an active Agent run and hides when the loop ends', async () => { + const { wrapper, browserClient, sessionStore } = await setup() + + expect(wrapper.find('[data-testid="agent-browser-pip"]').exists()).toBe(true) + + sessionStore.sessions[0].status = 'completed' + await nextTick() + await flushPromises() + + expect(wrapper.find('[data-testid="agent-browser-pip"]').exists()).toBe(false) + expect(browserClient.setPreviewMode).toHaveBeenCalledWith('session-1', 'stopped', 'run-1') + }) + + it('moves the active Agent browser into the sidepanel on request', async () => { + const { wrapper, browserClient, sidepanelStore } = await setup() + + await wrapper.get('[aria-label="common.open"]').trigger('click') + await flushPromises() + + expect(browserClient.setPreviewMode).toHaveBeenCalledWith('session-1', 'stopped', 'run-1') + expect(sidepanelStore.openBrowser).toHaveBeenCalledTimes(1) + expect(wrapper.find('[data-testid="agent-browser-pip"]').exists()).toBe(false) + }) + + it('stays dismissed for the current run', async () => { + const { wrapper } = await setup() + + await wrapper.get('[aria-label="common.close"]').trigger('click') + await nextTick() + + expect(wrapper.find('[data-testid="agent-browser-pip"]').exists()).toBe(false) + }) + + it('reveals controls on click and drags from the mirror surface', async () => { + const { wrapper, browserClient } = await setup({ wide: true }) + const pip = wrapper.get('[data-testid="agent-browser-pip"]') + + const toolbar = wrapper.get('[data-testid="agent-browser-pip-toolbar"]') + expect(toolbar.classes()).toContain('group-hover:opacity-100') + expect(wrapper.find('[data-testid="agent-browser-pip-drag-hint"]').exists()).toBe(true) + expect(pip.attributes('style')).toContain('width: 400px') + expect(pip.attributes('style')).toContain('height: 250px') + await pip.trigger('pointerdown', { button: 0, pointerId: 1, clientX: 100, clientY: 100 }) + await pip.trigger('pointerup', { pointerId: 1, clientX: 100, clientY: 100 }) + expect(toolbar.classes()).toContain('opacity-100') + + const leftBefore = pip.attributes('style') + await pip.trigger('pointerdown', { button: 0, pointerId: 2, clientX: 100, clientY: 100 }) + await pip.trigger('pointermove', { pointerId: 2, clientX: 150, clientY: 130 }) + await pip.trigger('pointerup', { pointerId: 2, clientX: 150, clientY: 130 }) + + expect(pip.attributes('style')).not.toBe(leftBefore) + expect(browserClient.setPreviewMode).toHaveBeenCalledWith('session-1', 'capturing', 'run-1') + }) + + it('draws a decoded preview frame into the local Canvas', async () => { + const drawImage = vi.fn() + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ drawImage } as never) + const close = vi.fn() + vi.stubGlobal( + 'createImageBitmap', + vi.fn().mockResolvedValueOnce({ close }).mockRejectedValueOnce(new Error('decode failed')) + ) + const { wrapper, emitPreviewFrame } = await setup({ wide: true }) + + emitPreviewFrame({ + sessionId: 'session-1', + runId: 'run-1', + sequence: 1, + width: 480, + height: 300, + mimeType: 'image/jpeg', + data: new Uint8Array([1, 2, 3]), + timestamp: Date.now() + }) + await flushPromises() + + expect(drawImage).toHaveBeenCalledWith(expect.anything(), 0, 0, 480, 300) + expect(close).toHaveBeenCalled() + expect(wrapper.find('[data-testid="agent-browser-pip-placeholder"]').exists()).toBe(false) + + emitPreviewFrame({ + sessionId: 'session-1', + runId: 'run-1', + sequence: 2, + width: 480, + height: 300, + mimeType: 'image/jpeg', + data: new Uint8Array([4, 5, 6]), + timestamp: Date.now() + }) + await flushPromises() + + expect(drawImage).toHaveBeenCalledTimes(1) + expect(wrapper.find('[data-testid="agent-browser-pip-placeholder"]').exists()).toBe(false) + }) + + it('accepts a reset frame sequence when the Agent run changes', async () => { + const drawImage = vi.fn() + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ drawImage } as never) + vi.stubGlobal( + 'createImageBitmap', + vi.fn(async () => ({ close: vi.fn() })) + ) + const { wrapper, emitStatus, emitPreviewFrame } = await setup({ wide: true }) + + emitPreviewFrame({ + sessionId: 'session-1', + runId: 'run-1', + sequence: 10, + width: 480, + height: 300, + mimeType: 'image/jpeg', + data: new Uint8Array([1]), + timestamp: Date.now() + }) + await flushPromises() + + emitStatus({ sessionId: 'session-1', status: createStatus('run-2') }) + await nextTick() + expect(wrapper.find('[data-testid="agent-browser-pip-placeholder"]').exists()).toBe(true) + + emitPreviewFrame({ + sessionId: 'session-1', + runId: 'run-2', + sequence: 0, + width: 480, + height: 300, + mimeType: 'image/jpeg', + data: new Uint8Array([2]), + timestamp: Date.now() + }) + await flushPromises() + + expect(drawImage).toHaveBeenCalledTimes(2) + expect(wrapper.find('[data-testid="agent-browser-pip-placeholder"]').exists()).toBe(false) + }) + + it('keeps rendering in the background without publishing frames when the window blurs', async () => { + const { wrapper, browserClient, emitWindowState } = await setup({ wide: true }) + + emitWindowState({ exists: true, isFocused: false }) + await nextTick() + await flushPromises() + + expect(wrapper.find('[data-testid="agent-browser-pip"]').exists()).toBe(false) + expect(browserClient.setPreviewMode).toHaveBeenCalledWith('session-1', 'rendering', 'run-1') + }) +}) diff --git a/test/renderer/components/ChatSidePanel.test.ts b/test/renderer/components/ChatSidePanel.test.ts index 6dbab952e4..a309682a22 100644 --- a/test/renderer/components/ChatSidePanel.test.ts +++ b/test/renderer/components/ChatSidePanel.test.ts @@ -129,6 +129,42 @@ describe('ChatSidePanel', () => { expect(sidepanelStore.openBrowser).toHaveBeenCalledTimes(1) }) + it('keeps a closed sidepanel closed for Agent browser activity', async () => { + const { sidepanelStore, emitOpenRequested } = await setup({ + open: false, + activeTab: 'workspace' + }) + + emitOpenRequested({ + windowId: 7, + sessionId: 'session-1', + url: 'https://example.com', + source: 'agent', + runId: 'run-1', + version: Date.now() + }) + + expect(sidepanelStore.openBrowser).not.toHaveBeenCalled() + }) + + it('switches an open workspace sidepanel to Browser for Agent activity', async () => { + const { sidepanelStore, emitOpenRequested } = await setup({ + open: true, + activeTab: 'workspace' + }) + + emitOpenRequested({ + windowId: 7, + sessionId: 'session-1', + url: 'https://example.com', + source: 'agent', + runId: 'run-1', + version: Date.now() + }) + + expect(sidepanelStore.openBrowser).toHaveBeenCalledTimes(1) + }) + it('dispatches session-scoped workspace insertion requests from the workspace panel', async () => { const insertionListener = vi.fn() window.addEventListener(WORKSPACE_EVENTS.INSERT_REFERENCE_REQUESTED, insertionListener) diff --git a/test/renderer/components/ChatTabView.test.ts b/test/renderer/components/ChatTabView.test.ts index 4733f35cef..8bd45e7de0 100644 --- a/test/renderer/components/ChatTabView.test.ts +++ b/test/renderer/components/ChatTabView.test.ts @@ -153,6 +153,13 @@ const setup = async (options: SetupOptions = {}) => { template: '
' }) })) + + vi.doMock('@/components/browser/AgentBrowserPiP.vue', () => ({ + default: defineComponent({ + name: 'AgentBrowserPiP', + template: '
' + }) + })) vi.doMock('@/pages/AgentWelcomePage.vue', () => ({ default: defineComponent({ name: 'AgentWelcomePage', diff --git a/test/setup.ts b/test/setup.ts index ddb7b02cca..2782b2d396 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -21,6 +21,7 @@ function getDefaultDeepchatInvokeResult( case 'browser.attachCurrentWindow': return { attached: true } case 'browser.updateCurrentWindowBounds': + case 'browser.setPreviewMode': return { updated: true } case 'browser.detach': return { detached: true } From a9ea0e70328206dfb6e4778aae4b28013baf5a2e Mon Sep 17 00:00:00 2001 From: xiaomo Date: Sun, 19 Jul 2026 01:13:29 +0800 Subject: [PATCH 5/6] refactor(renderer): restructure chat main and diagnostics (#1994) * refactor(renderer): establish app boundaries * fix(build): handle removed browser source root * fix(renderer): address startup review feedback * refactor(renderer): localize chat page feature * chore(architecture): refresh baseline * refactor(chat): split page actions * refactor(chat): localize display model * test(renderer): cover app runtime lifecycle * chore(resources): refresh provider metadata * feat(renderer): add performance diagnostics * docs(architecture): complete diagnostics tasks * fix(renderer): address review findings --- .github/workflows/prcheck.yml | 3 + AGENTS.md | 2 +- ...derer-application-boundaries-baseline.json | 727 ++++++++++++++++++ .../chat-display-model-boundary/plan.md | 34 + .../chat-display-model-boundary/spec.md | 35 + .../chat-display-model-boundary/tasks.md | 7 + .../chatpage-decomposition/spec.md | 25 +- .../chatpage-decomposition/tasks.md | 13 +- .../renderer-application-boundaries/plan.md | 84 ++ .../renderer-application-boundaries/spec.md | 96 +++ .../renderer-application-boundaries/tasks.md | 32 + .../renderer-performance-diagnostics/plan.md | 88 +++ .../renderer-performance-diagnostics/spec.md | 46 ++ .../renderer-performance-diagnostics/tasks.md | 11 + docs/guides/getting-started.md | 2 +- electron.vite.config.ts | 1 - package.json | 2 + scripts/agent-cleanup-guard.mjs | 4 +- scripts/generate-architecture-baseline.mjs | 2 +- scripts/generate-icon-collections.mjs | 1 - ...enerate-renderer-architecture-baseline.mjs | 132 ++++ src/main/app/composition.ts | 7 + src/main/app/rendererPerformanceLogService.ts | 95 +++ src/main/app/routes.ts | 17 +- src/renderer/api/PerformanceClient.ts | 17 + src/renderer/browser/assets/ChromeClose.svg | 3 - .../browser/assets/ChromeMaximize-1.svg | 3 - .../browser/assets/ChromeMaximize.svg | 3 - .../browser/assets/ChromeMinimize.svg | 3 - src/renderer/src/App.vue | 620 +-------------- .../src/apps/chat-main/ChatMainApp.vue | 646 ++++++++++++++++ .../chat/ChatToolInteractionOverlay.vue | 2 +- .../src/components/chat/MessageList.vue | 2 +- .../src/components/chat/MessageListRow.vue | 2 +- .../components/message/MessageBlockAction.vue | 2 +- .../message/MessageBlockActivityGroup.vue | 2 +- .../components/message/MessageBlockAudio.vue | 2 +- .../message/MessageBlockContent.vue | 2 +- .../components/message/MessageBlockError.vue | 2 +- .../components/message/MessageBlockImage.vue | 2 +- .../message/MessageBlockQuestionRequest.vue | 2 +- .../components/message/MessageBlockThink.vue | 2 +- .../message/MessageBlockToolCall.vue | 2 +- .../components/message/MessageBlockVideo.vue | 2 +- .../src/components/message/MessageContent.vue | 2 +- .../message/MessageItemAssistant.vue | 2 +- .../components/message/MessageItemUser.vue | 2 +- .../message/messageActivityGroups.ts | 2 +- .../composables/message/useMessageWindow.ts | 2 +- src/renderer/src/composables/useArtifacts.ts | 2 +- .../chat-page}/ChatPage.vue | 570 ++++---------- .../composables/useChatPageEventBridge.ts | 95 +++ .../chat-page/composables}/useChatSearch.ts | 2 +- .../composables}/useComposerSubmit.ts | 0 .../composables}/useDisplayMessages.ts | 4 +- .../chat-page/composables}/useListGestures.ts | 7 +- .../composables/useMessageActions.ts | 141 ++++ .../composables}/useMessageVirtualization.ts | 2 +- .../composables/usePendingInputActions.ts | 85 ++ .../composables}/usePlanFloatLifecycle.ts | 0 .../composables}/useSessionRestore.ts | 0 .../composables/useToolInteraction.ts | 172 +++++ .../chat-page/composables/useVoiceInput.ts | 174 +++++ .../chat-page/model/displayMessage.ts} | 0 .../performance/rendererPerformance.ts | 245 ++++++ src/renderer/src/stores/ui/message.ts | 2 +- src/renderer/src/views/ChatTabView.vue | 65 +- src/shared/contracts/routes.ts | 5 +- .../contracts/routes/performance.routes.ts | 70 ++ test/fixtures/mockMessages.ts | 2 +- .../app/rendererPerformanceLogService.test.ts | 130 ++++ test/main/app/routes.test.ts | 81 ++ test/renderer/components/App.startup.test.ts | 56 +- test/renderer/components/ChatPage.test.ts | 33 +- test/renderer/components/ChatTabView.test.ts | 87 ++- test/renderer/components/MessageList.test.ts | 2 +- .../message/MessageBlockActivityGroup.test.ts | 2 +- .../message/MessageBlockBasics.test.ts | 2 +- .../message/MessageBlockContent.test.ts | 2 +- .../message/MessageBlockMedia.test.ts | 2 +- .../message/MessageBlockToolCall.test.ts | 2 +- .../message/MessageItemAssistant.test.ts | 2 +- .../message/MessageItemUser.test.ts | 2 +- .../message/messageActivityGroups.test.ts | 2 +- .../composables/useMessageWindow.test.ts | 5 +- .../useChatPageEventBridge.test.ts | 108 +++ .../composables/useMessageActions.test.ts | 151 ++++ .../usePendingInputActions.test.ts | 154 ++++ .../composables/useToolInteraction.test.ts | 209 +++++ .../composables/useVoiceInput.test.ts | 194 +++++ test/renderer/lib/rendererPerformance.test.ts | 123 +++ .../performance/chatRendering.perf.test.ts | 2 +- test/renderer/stores/sessionStore.test.ts | 17 + tsconfig.app.json | 10 +- tsconfig.app.tsgo.json | 10 +- vitest.config.renderer.ts | 1 - vitest.config.ts | 1 - 97 files changed, 4668 insertions(+), 1162 deletions(-) create mode 100644 docs/architecture/baselines/renderer-application-boundaries-baseline.json create mode 100644 docs/architecture/chat-display-model-boundary/plan.md create mode 100644 docs/architecture/chat-display-model-boundary/spec.md create mode 100644 docs/architecture/chat-display-model-boundary/tasks.md create mode 100644 docs/architecture/renderer-application-boundaries/plan.md create mode 100644 docs/architecture/renderer-application-boundaries/spec.md create mode 100644 docs/architecture/renderer-application-boundaries/tasks.md create mode 100644 docs/architecture/renderer-performance-diagnostics/plan.md create mode 100644 docs/architecture/renderer-performance-diagnostics/spec.md create mode 100644 docs/architecture/renderer-performance-diagnostics/tasks.md create mode 100644 scripts/generate-renderer-architecture-baseline.mjs create mode 100644 src/main/app/rendererPerformanceLogService.ts create mode 100644 src/renderer/api/PerformanceClient.ts delete mode 100644 src/renderer/browser/assets/ChromeClose.svg delete mode 100644 src/renderer/browser/assets/ChromeMaximize-1.svg delete mode 100644 src/renderer/browser/assets/ChromeMaximize.svg delete mode 100644 src/renderer/browser/assets/ChromeMinimize.svg create mode 100644 src/renderer/src/apps/chat-main/ChatMainApp.vue rename src/renderer/src/{pages => features/chat-page}/ChatPage.vue (75%) create mode 100644 src/renderer/src/features/chat-page/composables/useChatPageEventBridge.ts rename src/renderer/src/{pages/chat-page => features/chat-page/composables}/useChatSearch.ts (98%) rename src/renderer/src/{pages/chat-page => features/chat-page/composables}/useComposerSubmit.ts (100%) rename src/renderer/src/{pages/chat-page => features/chat-page/composables}/useDisplayMessages.ts (99%) rename src/renderer/src/{pages/chat-page => features/chat-page/composables}/useListGestures.ts (97%) create mode 100644 src/renderer/src/features/chat-page/composables/useMessageActions.ts rename src/renderer/src/{pages/chat-page => features/chat-page/composables}/useMessageVirtualization.ts (99%) create mode 100644 src/renderer/src/features/chat-page/composables/usePendingInputActions.ts rename src/renderer/src/{pages/chat-page => features/chat-page/composables}/usePlanFloatLifecycle.ts (100%) rename src/renderer/src/{pages/chat-page => features/chat-page/composables}/useSessionRestore.ts (100%) create mode 100644 src/renderer/src/features/chat-page/composables/useToolInteraction.ts create mode 100644 src/renderer/src/features/chat-page/composables/useVoiceInput.ts rename src/renderer/src/{components/chat/messageListItems.ts => features/chat-page/model/displayMessage.ts} (100%) create mode 100644 src/renderer/src/platform/performance/rendererPerformance.ts create mode 100644 src/shared/contracts/routes/performance.routes.ts create mode 100644 test/main/app/rendererPerformanceLogService.test.ts create mode 100644 test/main/app/routes.test.ts create mode 100644 test/renderer/features/chat-page/composables/useChatPageEventBridge.test.ts create mode 100644 test/renderer/features/chat-page/composables/useMessageActions.test.ts create mode 100644 test/renderer/features/chat-page/composables/usePendingInputActions.test.ts create mode 100644 test/renderer/features/chat-page/composables/useToolInteraction.test.ts create mode 100644 test/renderer/features/chat-page/composables/useVoiceInput.test.ts create mode 100644 test/renderer/lib/rendererPerformance.test.ts diff --git a/.github/workflows/prcheck.yml b/.github/workflows/prcheck.yml index 6f88f91c86..b77d37ad52 100644 --- a/.github/workflows/prcheck.yml +++ b/.github/workflows/prcheck.yml @@ -74,6 +74,9 @@ jobs: - name: format:check run: pnpm run format:check + - name: Check renderer architecture baseline + run: pnpm run architecture:renderer-baseline:check + - name: Configure pnpm workspace for Linux ${{ matrix.arch }} run: pnpm run install:sharp env: diff --git a/AGENTS.md b/AGENTS.md index 391710c8e6..035031b1ee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ ## Project Structure & Module Organization - `src/main/`: Electron main process; presenters in `presenter/` (Window/Tab/Thread/Mcp/Config/LLMProvider), `eventbus.ts` for app events. - `src/preload/`: Secure IPC bridge (contextIsolation on). -- `src/renderer/`: Vue 3 app. App code in `src/renderer/src` (`components/`, `stores`, `views`, `i18n`, `lib`). Secondary renderers live in `src/renderer/browser`, `src/renderer/settings`, `src/renderer/floating`, and `src/renderer/splash`. +- `src/renderer/`: Vue 3 app. Main-window composition lives in `src/renderer/src/apps/chat-main`; shared main renderer code remains in `components/`, `stores`, `views`, `i18n`, and `lib`. Secondary renderers live in `src/renderer/browser-overlay`, `src/renderer/settings`, `src/renderer/floating`, and `src/renderer/splash`. - `src/shared/`: Shared TS types/utilities. - `test/`: Vitest suites (`test/main`, `test/renderer`) with setup files. - `scripts/`: Build/signing/runtime installers, commit checks. diff --git a/docs/architecture/baselines/renderer-application-boundaries-baseline.json b/docs/architecture/baselines/renderer-application-boundaries-baseline.json new file mode 100644 index 0000000000..3e5cfcd4df --- /dev/null +++ b/docs/architecture/baselines/renderer-application-boundaries-baseline.json @@ -0,0 +1,727 @@ +{ + "schemaVersion": 1, + "apps": [ + { + "id": "chat-main", + "html": "src/renderer/index.html", + "entry": "src/renderer/src/main.ts", + "htmlExists": true, + "entryExists": true + }, + { + "id": "browser-overlay", + "html": "src/renderer/browser-overlay/index.html", + "entry": "src/renderer/browser-overlay/main.ts", + "htmlExists": true, + "entryExists": true + }, + { + "id": "floating", + "html": "src/renderer/floating/index.html", + "entry": "src/renderer/floating/main.ts", + "htmlExists": true, + "entryExists": true + }, + { + "id": "splash", + "html": "src/renderer/splash/index.html", + "entry": "src/renderer/splash/main.ts", + "htmlExists": true, + "entryExists": true + }, + { + "id": "settings", + "html": "src/renderer/settings/index.html", + "entry": "src/renderer/settings/main.ts", + "htmlExists": true, + "entryExists": true + } + ], + "browser": { + "legacyDirectoryExists": false, + "activeOverlayDirectory": "src/renderer/browser-overlay" + }, + "settingsToChatAppImports": [ + { + "file": "src/renderer/settings/App.vue", + "specifier": "../src/composables/useDeviceVersion" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "../src/composables/useFontManager" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "../src/lib/iconLoader" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "../src/lib/startupDeferred" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "../src/lib/storeInitializer" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "../src/stores/language" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "../src/stores/modelCheck" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "../src/stores/uiSettingsStore" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "@/components/settings/ModelCheckDialog.vue" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "@/stores/modelStore" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "@/stores/ollamaStore" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "@/stores/providerDeeplinkImport" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "@/stores/providerStore" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "@/stores/startupWorkloadStore" + }, + { + "file": "src/renderer/settings/App.vue", + "specifier": "@/stores/theme" + }, + { + "file": "src/renderer/settings/components/AboutUsSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/AboutUsSettings.vue", + "specifier": "@/stores/language" + }, + { + "file": "src/renderer/settings/components/AboutUsSettings.vue", + "specifier": "@/stores/theme" + }, + { + "file": "src/renderer/settings/components/AboutUsSettings.vue", + "specifier": "@/stores/upgrade" + }, + { + "file": "src/renderer/settings/components/AcpDebugDialog.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/AcpDebugDialog.vue", + "specifier": "@/stores/uiSettingsStore" + }, + { + "file": "src/renderer/settings/components/AcpDependencyDialog.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/AcpProfileDialog.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/AcpSettings.vue", + "specifier": "@/components/agent/AgentTransferDialog.vue" + }, + { + "file": "src/renderer/settings/components/AcpSettings.vue", + "specifier": "@/components/icons/AcpAgentIcon.vue" + }, + { + "file": "src/renderer/settings/components/AcpSettings.vue", + "specifier": "@/components/mcp-config/AgentMcpSelector.vue" + }, + { + "file": "src/renderer/settings/components/AcpSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/AcpTerminalDialog.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/AddCustomModelButton.vue", + "specifier": "@/components/settings/ModelConfigDialog.vue" + }, + { + "file": "src/renderer/settings/components/AddCustomProviderDialog.vue", + "specifier": "@/stores/providerStore" + }, + { + "file": "src/renderer/settings/components/BedrockProviderSettingsDetail.vue", + "specifier": "@/stores/modelStore" + }, + { + "file": "src/renderer/settings/components/BedrockProviderSettingsDetail.vue", + "specifier": "@/stores/providerStore" + }, + { + "file": "src/renderer/settings/components/BuiltinKnowledgeSettings.vue", + "specifier": "@/components/icons/ModelIcon.vue" + }, + { + "file": "src/renderer/settings/components/BuiltinKnowledgeSettings.vue", + "specifier": "@/components/ModelSelect.vue" + }, + { + "file": "src/renderer/settings/components/BuiltinKnowledgeSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/BuiltinKnowledgeSettings.vue", + "specifier": "@/stores/mcp" + }, + { + "file": "src/renderer/settings/components/BuiltinKnowledgeSettings.vue", + "specifier": "@/stores/modelStore" + }, + { + "file": "src/renderer/settings/components/BuiltinKnowledgeSettings.vue", + "specifier": "@/stores/theme" + }, + { + "file": "src/renderer/settings/components/common/AutoCompactionSettingsSection.vue", + "specifier": "@/stores/uiSettingsStore" + }, + { + "file": "src/renderer/settings/components/common/DefaultModelSettingsSection.vue", + "specifier": "@/components/icons/ModelIcon.vue" + }, + { + "file": "src/renderer/settings/components/common/DefaultModelSettingsSection.vue", + "specifier": "@/components/ModelSelect.vue" + }, + { + "file": "src/renderer/settings/components/common/DefaultModelSettingsSection.vue", + "specifier": "@/stores/modelStore" + }, + { + "file": "src/renderer/settings/components/common/DefaultModelSettingsSection.vue", + "specifier": "@/stores/theme" + }, + { + "file": "src/renderer/settings/components/common/LoggingSettingsSection.vue", + "specifier": "@/stores/language" + }, + { + "file": "src/renderer/settings/components/common/LoggingSettingsSection.vue", + "specifier": "@/stores/uiSettingsStore" + }, + { + "file": "src/renderer/settings/components/common/PrivacySettingsSection.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/common/PrivacySettingsSection.vue", + "specifier": "@/stores/uiSettingsStore" + }, + { + "file": "src/renderer/settings/components/common/ProxySettingsSection.vue", + "specifier": "@/stores/language" + }, + { + "file": "src/renderer/settings/components/common/SettingToggleRow.vue", + "specifier": "@/stores/language" + }, + { + "file": "src/renderer/settings/components/CommonSettings.vue", + "specifier": "@/stores/uiSettingsStore" + }, + { + "file": "src/renderer/settings/components/CronJobsSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/DataSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/DataSettings.vue", + "specifier": "@/lib/cloudSyncForm" + }, + { + "file": "src/renderer/settings/components/DataSettings.vue", + "specifier": "@/lib/utils" + }, + { + "file": "src/renderer/settings/components/DataSettings.vue", + "specifier": "@/stores/language" + }, + { + "file": "src/renderer/settings/components/DataSettings.vue", + "specifier": "@/stores/sync" + }, + { + "file": "src/renderer/settings/components/DeepChatAgentsSettings.vue", + "specifier": "@/components/agent/AgentTransferDialog.vue" + }, + { + "file": "src/renderer/settings/components/DeepChatAgentsSettings.vue", + "specifier": "@/components/icons/AgentAvatar.vue" + }, + { + "file": "src/renderer/settings/components/DeepChatAgentsSettings.vue", + "specifier": "@/components/icons/ModelIcon.vue" + }, + { + "file": "src/renderer/settings/components/DeepChatAgentsSettings.vue", + "specifier": "@/components/ModelSelect.vue" + }, + { + "file": "src/renderer/settings/components/DeepChatAgentsSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/DeepChatAgentsSettings.vue", + "specifier": "@/stores/modelStore" + }, + { + "file": "src/renderer/settings/components/DifyKnowledgeSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/DifyKnowledgeSettings.vue", + "specifier": "@/stores/mcp" + }, + { + "file": "src/renderer/settings/components/display/FontSettingsSection.vue", + "specifier": "@/stores/language" + }, + { + "file": "src/renderer/settings/components/display/FontSettingsSection.vue", + "specifier": "@/stores/uiSettingsStore" + }, + { + "file": "src/renderer/settings/components/DisplaySettings.vue", + "specifier": "@/stores/floatingButton" + }, + { + "file": "src/renderer/settings/components/DisplaySettings.vue", + "specifier": "@/stores/language" + }, + { + "file": "src/renderer/settings/components/DisplaySettings.vue", + "specifier": "@/stores/theme" + }, + { + "file": "src/renderer/settings/components/DisplaySettings.vue", + "specifier": "@/stores/uiSettingsStore" + }, + { + "file": "src/renderer/settings/components/EnvironmentsSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/EnvironmentsSettings.vue", + "specifier": "@/stores/ui/project" + }, + { + "file": "src/renderer/settings/components/FastGptKnowledgeSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/FastGptKnowledgeSettings.vue", + "specifier": "@/stores/mcp" + }, + { + "file": "src/renderer/settings/components/GeminiSafetyConfig.vue", + "specifier": "@/lib/gemini" + }, + { + "file": "src/renderer/settings/components/GitHubCopilotOAuth.vue", + "specifier": "@/stores/modelCheck" + }, + { + "file": "src/renderer/settings/components/GitHubCopilotOAuth.vue", + "specifier": "@/stores/providerStore" + }, + { + "file": "src/renderer/settings/components/GrokOAuth.vue", + "specifier": "@/stores/modelCheck" + }, + { + "file": "src/renderer/settings/components/KnowledgeFile.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/KnowledgeFileItem.vue", + "specifier": "@/lib/utils" + }, + { + "file": "src/renderer/settings/components/McpBuiltinMarket.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/McpSettings.vue", + "specifier": "@/components/mcp-config/components/McpServers.vue" + }, + { + "file": "src/renderer/settings/components/McpSettings.vue", + "specifier": "@/components/onboarding/GuidedOnboardingOverlay.vue" + }, + { + "file": "src/renderer/settings/components/McpSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/McpSettings.vue", + "specifier": "@/composables/useGuidedOnboardingStep" + }, + { + "file": "src/renderer/settings/components/McpSettings.vue", + "specifier": "@/stores/language" + }, + { + "file": "src/renderer/settings/components/McpSettings.vue", + "specifier": "@/stores/mcp" + }, + { + "file": "src/renderer/settings/components/McpSettings.vue", + "specifier": "@/stores/ui/agent" + }, + { + "file": "src/renderer/settings/components/McpSettings.vue", + "specifier": "@/stores/ui/session" + }, + { + "file": "src/renderer/settings/components/MemoryConfigInlinePanel.vue", + "specifier": "@/components/icons/ModelIcon.vue" + }, + { + "file": "src/renderer/settings/components/MemoryConfigInlinePanel.vue", + "specifier": "@/components/ModelSelect.vue" + }, + { + "file": "src/renderer/settings/components/MemoryConfigInlinePanel.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/MemoryConfigInlinePanel.vue", + "specifier": "@/stores/modelStore" + }, + { + "file": "src/renderer/settings/components/MemoryDiagnosticsPanel.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/MemoryInboxBar.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/MemoryInlinePanel.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/MemoryListView.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/MemoryPersonaPanel.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettings.vue", + "specifier": "@/components/icons/ModelIcon.vue" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettings.vue", + "specifier": "@/components/onboarding/GuidedOnboardingOverlay.vue" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettings.vue", + "specifier": "@/composables/useGuidedOnboardingStep" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettings.vue", + "specifier": "@/stores/language" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettings.vue", + "specifier": "@/stores/modelStore" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettings.vue", + "specifier": "@/stores/providerStore" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettings.vue", + "specifier": "@/stores/startupWorkloadStore" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettings.vue", + "specifier": "@/stores/theme" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettingsDetail.vue", + "specifier": "@/lib/gemini" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettingsDetail.vue", + "specifier": "@/lib/gemini" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettingsDetail.vue", + "specifier": "@/stores/modelCheck" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettingsDetail.vue", + "specifier": "@/stores/modelStore" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettingsDetail.vue", + "specifier": "@/stores/providerStore" + }, + { + "file": "src/renderer/settings/components/ModelProviderSettingsDetail.vue", + "specifier": "@/stores/uiSettingsStore" + }, + { + "file": "src/renderer/settings/components/NotificationsHooksSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/NowledgeMemSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/OllamaProviderSettingsDetail.vue", + "specifier": "@/components/settings/ModelConfigItem.vue" + }, + { + "file": "src/renderer/settings/components/OllamaProviderSettingsDetail.vue", + "specifier": "@/stores/modelCheck" + }, + { + "file": "src/renderer/settings/components/OllamaProviderSettingsDetail.vue", + "specifier": "@/stores/modelStore" + }, + { + "file": "src/renderer/settings/components/OllamaProviderSettingsDetail.vue", + "specifier": "@/stores/ollamaStore" + }, + { + "file": "src/renderer/settings/components/OllamaProviderSettingsDetail.vue", + "specifier": "@/stores/providerStore" + }, + { + "file": "src/renderer/settings/components/OpenAICodexOAuth.vue", + "specifier": "@/stores/modelCheck" + }, + { + "file": "src/renderer/settings/components/prompt/CustomPromptSettingsSection.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/prompt/CustomPromptSettingsSection.vue", + "specifier": "@/lib/download" + }, + { + "file": "src/renderer/settings/components/prompt/CustomPromptSettingsSection.vue", + "specifier": "@/stores/prompts" + }, + { + "file": "src/renderer/settings/components/prompt/PromptEditorSheet.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/prompt/PromptEditorSheet.vue", + "specifier": "@/lib/utils" + }, + { + "file": "src/renderer/settings/components/prompt/SystemPromptSettingsSection.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/prompt/SystemPromptSettingsSection.vue", + "specifier": "@/stores/systemPromptStore" + }, + { + "file": "src/renderer/settings/components/ProviderApiConfig.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/ProviderApiConfig.vue", + "specifier": "@/stores/modelCheck" + }, + { + "file": "src/renderer/settings/components/ProviderConfigImportDialog.vue", + "specifier": "@/lib/utils" + }, + { + "file": "src/renderer/settings/components/ProviderDeeplinkImportDialog.vue", + "specifier": "@/components/icons/ModelIcon.vue" + }, + { + "file": "src/renderer/settings/components/ProviderDeeplinkImportDialog.vue", + "specifier": "@/stores/theme" + }, + { + "file": "src/renderer/settings/components/ProviderModelList.vue", + "specifier": "@/components/settings/ModelConfigItem.vue" + }, + { + "file": "src/renderer/settings/components/ProviderModelList.vue", + "specifier": "@/stores/modelStore" + }, + { + "file": "src/renderer/settings/components/ProviderModelList.vue", + "specifier": "@/stores/uiSettingsStore" + }, + { + "file": "src/renderer/settings/components/ProviderRateLimitConfig.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/RagflowKnowledgeSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/RagflowKnowledgeSettings.vue", + "specifier": "@/stores/mcp" + }, + { + "file": "src/renderer/settings/components/RemoteSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/SettingsOverview.vue", + "specifier": "@/stores/modelStore" + }, + { + "file": "src/renderer/settings/components/SettingsOverview.vue", + "specifier": "@/stores/providerStore" + }, + { + "file": "src/renderer/settings/components/SettingsOverview.vue", + "specifier": "@/stores/sync" + }, + { + "file": "src/renderer/settings/components/SettingsOverview.vue", + "specifier": "@/stores/ui/agent" + }, + { + "file": "src/renderer/settings/components/ShortcutSettings.vue", + "specifier": "@/stores/language" + }, + { + "file": "src/renderer/settings/components/ShortcutSettings.vue", + "specifier": "@/stores/shortcutKey" + }, + { + "file": "src/renderer/settings/components/skills/InstallFromGitDialog.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/skills/InstallSkillToAgentDialog.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/skills/SkillAgentsTab.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/skills/SkillAgentsTab.vue", + "specifier": "@/stores/skillsStore" + }, + { + "file": "src/renderer/settings/components/skills/SkillDetailDialog.vue", + "specifier": "@/components/markdown/MarkdownRenderer.vue" + }, + { + "file": "src/renderer/settings/components/skills/SkillFolderTree.vue", + "specifier": "@/stores/skillsStore" + }, + { + "file": "src/renderer/settings/components/skills/SkillImportExportTab.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/skills/SkillInstallDialog.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/skills/SkillInstallDialog.vue", + "specifier": "@/stores/skillsStore" + }, + { + "file": "src/renderer/settings/components/skills/SkillsHeader.vue", + "specifier": "@/stores/language" + }, + { + "file": "src/renderer/settings/components/skills/SkillsSettings.vue", + "specifier": "@/components/onboarding/GuidedOnboardingOverlay.vue" + }, + { + "file": "src/renderer/settings/components/skills/SkillsSettings.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/skills/SkillsSettings.vue", + "specifier": "@/composables/useGuidedOnboardingStep" + }, + { + "file": "src/renderer/settings/components/skills/SkillsSettings.vue", + "specifier": "@/stores/skillsStore" + }, + { + "file": "src/renderer/settings/components/skills/SkillsSettings.vue", + "specifier": "@/stores/ui/agent" + }, + { + "file": "src/renderer/settings/components/skills/SkillsSettings.vue", + "specifier": "@/stores/ui/session" + }, + { + "file": "src/renderer/settings/components/skills/SkillSyncDialog/ExportWizard.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/skills/SkillSyncDialog/ExportWizard.vue", + "specifier": "@/stores/skillsStore" + }, + { + "file": "src/renderer/settings/components/skills/SkillSyncDialog/ImportWizard.vue", + "specifier": "@/components/use-toast" + }, + { + "file": "src/renderer/settings/components/VertexProviderSettingsDetail.vue", + "specifier": "@/stores/providerStore" + }, + { + "file": "src/renderer/settings/components/VoiceAIProviderConfig.vue", + "specifier": "@/stores/providerStore" + }, + { + "file": "src/renderer/settings/lib/guidedOnboardingSettings.ts", + "specifier": "@/lib/onboardingResume" + }, + { + "file": "src/renderer/settings/main.ts", + "specifier": "../src/lib/iconLoader" + }, + { + "file": "src/renderer/settings/main.ts", + "specifier": "@/i18n" + } + ], + "settingsToChatAppImportCount": 170 +} diff --git a/docs/architecture/chat-display-model-boundary/plan.md b/docs/architecture/chat-display-model-boundary/plan.md new file mode 100644 index 0000000000..bce0141698 --- /dev/null +++ b/docs/architecture/chat-display-model-boundary/plan.md @@ -0,0 +1,34 @@ +# 实施计划 + +## 1. 建立 feature model + +在 `src/renderer/src/features/chat-page/model/displayMessage.ts` 原样迁移 +`messageListItems.ts` 的所有 type 和 pure helper： + +- `DisplayMessage` 家族与 `MessageListItem`； +- assistant block 的 internal tool、filter、presence 策略； +- compaction 判定。 + +新模块只保留对 `@shared/types/*` 的 type/value 依赖，因此可以被 feature、component、store 和 +纯 UI helper 使用而不引入运行时分层循环。 + +## 2. 原子切换消费者 + +把 renderer source、renderer tests 和 fixtures 的 import 改为 +`@/features/chat-page/model/displayMessage`。特别保留 component 对 feature model 的单向读取： +组件不会导入 ChatPage、feature composable 或 store。 + +删除旧的 `components/chat/messageListItems.ts`，不建立过渡 re-export；这样新增引用无法继续落在 +错误的 component 目录。 + +## 3. 验证 + +1. 搜索确认旧模块 import 为零。 +2. 执行 formatter；随后执行 i18n、lint 与 renderer type check。 +3. 运行直接覆盖消息 list/window、message block 和 chat feature composable 的 targeted suites；如 + repository runner 支持，再执行完整 renderer test suite。 + +## 兼容性与回滚 + +模块导出名和实现不变，调用方数据格式与运行时行为不变。若出现解析或架构问题，单次提交即可将 +imports 和文件位置恢复，无需迁移数据或 IPC contract。 diff --git a/docs/architecture/chat-display-model-boundary/spec.md b/docs/architecture/chat-display-model-boundary/spec.md new file mode 100644 index 0000000000..91fb983ab0 --- /dev/null +++ b/docs/architecture/chat-display-model-boundary/spec.md @@ -0,0 +1,35 @@ +# Chat 展示模型边界收敛 + +## 背景 + +`DisplayMessage`、其 block/usage 契约，以及 assistant block 的可渲染性策略目前定义在 +`src/renderer/src/components/chat/messageListItems.ts`。但记录到展示消息的转换、流式尾部拼装、 +是否显示空 assistant 消息的决策都由 `features/chat-page` 持有。feature 因而反向依赖 UI 目录以 +取得自己的领域展示模型，模型所有权与实际业务所有权不一致。 + +这些契约同时被 chat list、message block 组件、message store、chat-only composable 与测试使用。 +它们不是可复用通用 UI 的模型，而是 chat-page feature 的输入/输出契约。 + +## 目标 + +1. 将 `DisplayMessage` 家族、`MessageListItem` 与 assistant block 可渲染性策略迁入 + `src/renderer/src/features/chat-page/model/`。 +2. 所有消费者直接引用 feature model，不保留 `components/chat/messageListItems.ts` 作为第二个 + 定义或兼容 re-export。 +3. 保持类型形状、过滤条件与 compaction 判定完全不变；这是纯模块边界调整。 +4. 记录 chat 组件可以依赖该 feature 的**纯 model contract**：model 只依赖 shared types， + 不依赖 Vue、Pinia、IPC、component 或 composable，且不得反向导入消费者。 + +## 非目标 + +- 不调整消息转换、流式缓存、虚拟化或 Markdown 渲染逻辑。 +- 不变更 `MessageList` / `MessageListRow` 的 props、events 或 template。 +- 不迁移通用 renderer foundation，也不改变 IPC/store 数据格式。 +- 不在本切片拆分 `ChatStatusBar` 或继续提取 ChatPage viewport 生命周期。 + +## 验收标准 + +1. `components/chat/messageListItems.ts` 不再存在，且 renderer 与测试中没有对其的 import。 +2. 所有原有导出的 type 和 helper 在 feature model 有同名导出，运行行为保持一致。 +3. feature model 不导入 Vue、Pinia、renderer store、renderer API 或 component。 +4. `pnpm run format`、`pnpm run i18n`、`pnpm run lint` 和相关 renderer type/test 校验通过。 diff --git a/docs/architecture/chat-display-model-boundary/tasks.md b/docs/architecture/chat-display-model-boundary/tasks.md new file mode 100644 index 0000000000..4c91772991 --- /dev/null +++ b/docs/architecture/chat-display-model-boundary/tasks.md @@ -0,0 +1,7 @@ +# 任务清单 + +- [x] 审计展示模型、renderability policy 与全部 source/test 消费者。 +- [x] 在 `features/chat-page/model/` 建立纯展示模型模块。 +- [x] 原子更新 renderer source、tests 与 fixtures 的 import。 +- [x] 删除旧 component-local 模块并确认无残留引用。 +- [x] 执行 format、i18n、lint、typecheck 与 targeted renderer tests。 diff --git a/docs/architecture/chatpage-decomposition/spec.md b/docs/architecture/chatpage-decomposition/spec.md index b4d9f444e1..59fe8759aa 100644 --- a/docs/architecture/chatpage-decomposition/spec.md +++ b/docs/architecture/chatpage-decomposition/spec.md @@ -2,7 +2,8 @@ ## 背景 -`src/renderer/src/pages/ChatPage.vue` 已膨胀到 3050 行，单个 ` diff --git a/src/renderer/src/apps/chat-main/ChatMainApp.vue b/src/renderer/src/apps/chat-main/ChatMainApp.vue new file mode 100644 index 0000000000..eee16723d3 --- /dev/null +++ b/src/renderer/src/apps/chat-main/ChatMainApp.vue @@ -0,0 +1,646 @@ + + + diff --git a/src/renderer/src/components/chat/ChatToolInteractionOverlay.vue b/src/renderer/src/components/chat/ChatToolInteractionOverlay.vue index 2548204813..0b0d3b4f2a 100644 --- a/src/renderer/src/components/chat/ChatToolInteractionOverlay.vue +++ b/src/renderer/src/components/chat/ChatToolInteractionOverlay.vue @@ -98,7 +98,7 @@ import { useI18n } from 'vue-i18n' import { Button } from '@shadcn/components/ui/button' import { Icon } from '@iconify/vue' import type { ToolInteractionResponse } from '@shared/types/agent-interface' -import type { DisplayAssistantMessageBlock } from '@/components/chat/messageListItems' +import type { DisplayAssistantMessageBlock } from '@/features/chat-page/model/displayMessage' type PendingInteractionView = { messageId: string diff --git a/src/renderer/src/components/chat/MessageList.vue b/src/renderer/src/components/chat/MessageList.vue index 77fc5939ed..87067c4358 100644 --- a/src/renderer/src/components/chat/MessageList.vue +++ b/src/renderer/src/components/chat/MessageList.vue @@ -51,7 +51,7 @@ import { type DisplayAssistantMessageBlock, type DisplayMessage, type MessageListItem -} from './messageListItems' +} from '@/features/chat-page/model/displayMessage' import MessageListRow from './MessageListRow.vue' const props = withDefaults( diff --git a/src/renderer/src/components/chat/MessageListRow.vue b/src/renderer/src/components/chat/MessageListRow.vue index b1b5aedcf8..49b854fa40 100644 --- a/src/renderer/src/components/chat/MessageListRow.vue +++ b/src/renderer/src/components/chat/MessageListRow.vue @@ -59,7 +59,7 @@ import { isCompactionMessageItem, type DisplayUserMessage, type MessageListItem -} from './messageListItems' +} from '@/features/chat-page/model/displayMessage' const props = withDefaults( defineProps<{ diff --git a/src/renderer/src/components/message/MessageBlockAction.vue b/src/renderer/src/components/message/MessageBlockAction.vue index 9fd3705b98..365ba02cea 100644 --- a/src/renderer/src/components/message/MessageBlockAction.vue +++ b/src/renderer/src/components/message/MessageBlockAction.vue @@ -48,7 +48,7 @@ import { useI18n } from 'vue-i18n' import { Icon } from '@iconify/vue' import { Button } from '@shadcn/components/ui/button' import { computed, ref, onMounted, onUnmounted } from 'vue' -import type { DisplayAssistantMessageBlock } from '@/components/chat/messageListItems' +import type { DisplayAssistantMessageBlock } from '@/features/chat-page/model/displayMessage' const { t } = useI18n() diff --git a/src/renderer/src/components/message/MessageBlockActivityGroup.vue b/src/renderer/src/components/message/MessageBlockActivityGroup.vue index 1754396209..bb9d4f8604 100644 --- a/src/renderer/src/components/message/MessageBlockActivityGroup.vue +++ b/src/renderer/src/components/message/MessageBlockActivityGroup.vue @@ -63,7 +63,7 @@ import { useI18n } from 'vue-i18n' import type { DisplayAssistantMessageBlock, DisplayMessageUsage -} from '@/components/chat/messageListItems' +} from '@/features/chat-page/model/displayMessage' import { formatActivityDuration } from './messageActivityGroups' import MessageBlockThink from './MessageBlockThink.vue' import MessageBlockToolCall from './MessageBlockToolCall.vue' diff --git a/src/renderer/src/components/message/MessageBlockAudio.vue b/src/renderer/src/components/message/MessageBlockAudio.vue index fa6405fcb0..19c9886751 100644 --- a/src/renderer/src/components/message/MessageBlockAudio.vue +++ b/src/renderer/src/components/message/MessageBlockAudio.vue @@ -35,7 +35,7 @@ import { computed, ref } from 'vue' import { Icon } from '@iconify/vue' import { useI18n } from 'vue-i18n' import { Spinner } from '@shadcn/components/ui/spinner' -import type { DisplayAssistantMessageBlock } from '@/components/chat/messageListItems' +import type { DisplayAssistantMessageBlock } from '@/features/chat-page/model/displayMessage' const keyMap = { 'mcp.sampling.contentType.audio': 'Audio', diff --git a/src/renderer/src/components/message/MessageBlockContent.vue b/src/renderer/src/components/message/MessageBlockContent.vue index f68920164c..4180c3761b 100644 --- a/src/renderer/src/components/message/MessageBlockContent.vue +++ b/src/renderer/src/components/message/MessageBlockContent.vue @@ -47,7 +47,7 @@ import ToolCallPreview from '../artifacts/ToolCallPreview.vue' import { useBlockContent, type ProcessedPart } from '@/composables/useArtifacts' import { useArtifactStore } from '@/stores/artifact' import MarkdownRenderer from '@/components/markdown/MarkdownRenderer.vue' -import type { DisplayAssistantMessageBlock } from '@/components/chat/messageListItems' +import type { DisplayAssistantMessageBlock } from '@/features/chat-page/model/displayMessage' const artifactStore = useArtifactStore() const props = defineProps<{ diff --git a/src/renderer/src/components/message/MessageBlockError.vue b/src/renderer/src/components/message/MessageBlockError.vue index 6131759034..66ae84cd9f 100644 --- a/src/renderer/src/components/message/MessageBlockError.vue +++ b/src/renderer/src/components/message/MessageBlockError.vue @@ -47,7 +47,7 @@ import { useI18n } from 'vue-i18n' import { Icon } from '@iconify/vue' import { computed, ref, useId } from 'vue' -import type { DisplayAssistantMessageBlock } from '@/components/chat/messageListItems' +import type { DisplayAssistantMessageBlock } from '@/features/chat-page/model/displayMessage' const { t } = useI18n() const props = defineProps<{ diff --git a/src/renderer/src/components/message/MessageBlockImage.vue b/src/renderer/src/components/message/MessageBlockImage.vue index fbdebad37b..b02a6517be 100644 --- a/src/renderer/src/components/message/MessageBlockImage.vue +++ b/src/renderer/src/components/message/MessageBlockImage.vue @@ -76,7 +76,7 @@ import { Button } from '@shadcn/components/ui/button' import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@shadcn/components/ui/dialog' import { Spinner } from '@shadcn/components/ui/spinner' import { Tooltip, TooltipContent, TooltipTrigger } from '@shadcn/components/ui/tooltip' -import type { DisplayAssistantMessageBlock } from '@/components/chat/messageListItems' +import type { DisplayAssistantMessageBlock } from '@/features/chat-page/model/displayMessage' import ImageActionContextMenu from './ImageActionContextMenu.vue' import { useImageActions } from '@/composables/useImageActions' diff --git a/src/renderer/src/components/message/MessageBlockQuestionRequest.vue b/src/renderer/src/components/message/MessageBlockQuestionRequest.vue index 6468dd3a53..2b2d988faa 100644 --- a/src/renderer/src/components/message/MessageBlockQuestionRequest.vue +++ b/src/renderer/src/components/message/MessageBlockQuestionRequest.vue @@ -28,7 +28,7 @@