diff --git a/src/daily-cache.ts b/src/daily-cache.ts index c5439c34..a598bdc8 100644 --- a/src/daily-cache.ts +++ b/src/daily-cache.ts @@ -671,6 +671,22 @@ export async function ensureCacheHydrated( c = { ...c, days: freshDays, lastComputedDate: latestFresh } } + // A cache can claim `complete` while its watermark points PAST its newest + // populated day — what a run finalizing off a degraded (read-only) parse + // leaves behind: it advanced lastComputedDate over days the parse never + // covered. Since gapStart is lastComputedDate + 1, that hole is invisible + // to the gap logic forever. Trust the DATA over the marker: pull the + // watermark back to the newest day actually present so the ordinary gap + // parse re-derives the tail. Nothing is dropped — the cached days all stay, + // and a genuinely idle tail simply re-derives as empty. A cache with NO + // days is exempt: it has no newest day to trust, and a machine with no + // history at all must still be able to finalize (line below) rather than + // re-backfill the whole window on every launch. + const newestCachedDate = c.days.reduce((max, d) => (max === null || d.date > max ? d.date : max), null) + if (newestCachedDate !== null && c.lastComputedDate !== null && c.lastComputedDate > newestCachedDate) { + c = { ...c, lastComputedDate: newestCachedDate } + } + // Three reasons to re-derive the whole retention window: // 1. Savings config changed — cached `savingsUSD` totals are stale. // 2. The cache was never finalized against a COMPLETE session parse (an old @@ -691,6 +707,7 @@ export async function ensureCacheHydrated( const tzChanged = c.tzKey !== undefined && c.tzKey !== tzKey if (c.savingsConfigHash !== savingsConfigHash || c.complete !== true || tzChanged) { const baseline = c.days + const priorWatermark = c.lastComputedDate const backfillStart = new Date(now.getFullYear(), now.getMonth(), now.getDate() - BACKFILL_DAYS) let freshDays: DailyEntry[] = [] if (backfillStart.getTime() <= yesterdayEnd.getTime()) { @@ -708,7 +725,13 @@ export async function ensureCacheHydrated( version: DAILY_CACHE_VERSION, savingsConfigHash, tzKey, - lastComputedDate: yesterdayStr, + // The watermark records how far history has actually been derived, so + // only a COMPLETE parse may advance it. A partial one produced no data + // for whatever it could not read; moving the watermark to yesterday + // anyway would place those days behind the next run's gapStart and + // freeze the hole in (retention still anchors on yesterdayStr — the + // real calendar edge — so holding the watermark can't evict anything). + lastComputedDate: parseWasComplete ? yesterdayStr : priorWatermark, days: applyRetention(merged, yesterdayStr), complete: parseWasComplete, } @@ -733,12 +756,17 @@ export async function ensureCacheHydrated( const gapRange: DateRange = { start: gapStart, end: yesterdayEnd } const gapProjects = await parseSessions(gapRange) const gapDays = aggregateDays(gapProjects) + const parseWasComplete = sessionComplete() + const priorWatermark = c.lastComputedDate c = addNewDays(c, gapDays, yesterdayStr) // Finalize as complete ONLY when the session parse that produced these days // was itself complete. If it was partial, leave `complete: false` so the // next launch (once the session cache is whole) re-backfills instead of - // freezing the partial history. - c = { ...c, complete: sessionComplete() } + // freezing the partial history — and hold the watermark where it was, for + // the same reason as the re-derive path above: a partial parse cannot + // vouch for the days it never read, and gapStart is the only thing that + // will ever bring them back. + c = { ...c, lastComputedDate: parseWasComplete ? c.lastComputedDate : priorWatermark, complete: parseWasComplete } await saveDailyCache(c) } else if (c.complete !== true && sessionComplete()) { // No gap to fill (already current through yesterday) but not yet marked — diff --git a/src/parser.ts b/src/parser.ts index ee165e3a..4e654beb 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1931,6 +1931,7 @@ async function scanProjectDirs( const cached = section.files[filePath] const action = reconcileFile(fp, cached) if (cached && (readOnly || action.action === 'unchanged')) { + if (readOnly && action.action !== 'unchanged') readOnlyServedStale = true unchangedFiles.push({ filePath, dirName, source, cached: section.files[filePath]! }) } else if (!readOnly) { if (action.action === 'appended') { @@ -1942,6 +1943,10 @@ async function scanProjectDirs( continue } changedFiles.push({ filePath, info: { dirName, fp, source } }) + } else { + // Read-only with no cache entry at all: this file is dropped from what + // we serve, so the snapshot under-reports whatever days it covers. + readOnlyServedStale = true } } dirsDone++ @@ -2826,9 +2831,13 @@ async function parseProviderSources( // re-read a file that already threw and hasn't changed. It re-parses only // when the file changes (then `reconcileFile` reports non-'unchanged'). if (cached && (readOnly || (action.action === 'unchanged' && (cached.failed || !cachedFileNeedsProviderReparse(providerName, source.path, cached))))) { + if (readOnly && action.action !== 'unchanged') readOnlyServedStale = true unchangedSources.push({ source, cached }) } else if (!readOnly) { changedSources.push({ source, fp }) + } else { + // Read-only with no cache entry at all — see scanProjectDirs. + readOnlyServedStale = true } } @@ -3525,6 +3534,15 @@ export function isSessionHydrationComplete(): boolean { return sessionHydrationComplete } +// Set by the read-only serving paths when the snapshot they served did NOT +// match what is on disk: in read-only mode a changed file is served at its +// stale fingerprint and a file with no cache entry is skipped entirely. A +// read-only run under which nothing changed is equivalent to a full parse and +// stays trustworthy; one that skipped real data is a PARTIAL hydration, and +// finalizing daily history off it freezes the days it never saw out of the +// chart (gapStart = lastComputedDate + 1 never looks back at them). +let readOnlyServedStale = false + export async function parseAllSessions(dateRange?: DateRange, providerFilter?: string): Promise { const key = cacheKey(dateRange, providerFilter) const cached = sessionCache.get(key) @@ -3594,6 +3612,7 @@ async function runParse( options: RunParseOptions = {}, ): Promise { const { isCold = false, readOnly = false, refreshLock } = options + readOnlyServedStale = false const seenMsgIds = new Set() const seenKeys = new Set() const allSources = await discoverAllSessions(providerFilter) @@ -3698,7 +3717,10 @@ async function runParse( if (refreshLock) throw new RefreshPublicationUnavailableError() } } - sessionHydrationComplete = true + // Assigned, not forced true: a read-only run that had to skip or stale real + // files reached the end of the scan without hydrating everything, and the + // daily backfill must not finalize history off it. + sessionHydrationComplete = !readOnly || !readOnlyServedStale // Merge across providers by normalised project path so the same repository // is not double-counted when it was worked on with more than one tool diff --git a/tests/daily-cache-degraded-completeness.test.ts b/tests/daily-cache-degraded-completeness.test.ts new file mode 100644 index 00000000..76e3abb6 --- /dev/null +++ b/tests/daily-cache-degraded-completeness.test.ts @@ -0,0 +1,167 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { mkdir, rm } from 'fs/promises' +import { existsSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' + +import type { DateRange, ProjectSummary } from '../src/types.js' + +import { + DAILY_CACHE_VERSION, + type DailyCache, + type DailyEntry, + type ProviderDaySlice, + currentTzKey, + ensureCacheHydrated, + saveDailyCache, +} from '../src/daily-cache.js' + +const TMP_CACHE_ROOT = join(tmpdir(), `codeburn-degraded-test-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`) + +beforeEach(async () => { + process.env['CODEBURN_CACHE_DIR'] = TMP_CACHE_ROOT + await mkdir(TMP_CACHE_ROOT, { recursive: true }) +}) + +afterEach(async () => { + if (existsSync(TMP_CACHE_ROOT)) { + await rm(TMP_CACHE_ROOT, { recursive: true, force: true }) + } +}) + +function slice(cost: number, calls: number, extra: Partial = {}): ProviderDaySlice { + return { cost, calls, savingsUSD: 0, ...extra } +} + +function day(date: string, providers: Record, overrides: Partial = {}): DailyEntry { + const cost = Object.values(providers).reduce((s, p) => s + p.cost, 0) + const calls = Object.values(providers).reduce((s, p) => s + p.calls, 0) + return { + date, + cost, + savingsUSD: 0, + calls, + sessions: 0, + inputTokens: 0, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + editTurns: 0, + oneShotTurns: 0, + models: {}, + categories: {}, + providers, + ...overrides, + } +} + +function daysAgoStr(n: number): string { + const d = new Date() + d.setDate(d.getDate() - n) + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}` +} + +const noSessions = async (): Promise => [] + +/// The day whose session files are long gone: it exists in the daily cache and +/// nowhere else, so every path below must still hand it back untouched. +const VANISHED = day(daysAgoStr(40), { claude: slice(399.70, 1572) }, { carried: true }) + +async function seed(overrides: Partial = {}): Promise { + await saveDailyCache({ + version: DAILY_CACHE_VERSION, + savingsConfigHash: 'cfg-A', + tzKey: currentTzKey(), + lastComputedDate: daysAgoStr(4), + days: [VANISHED, day(daysAgoStr(4), { claude: slice(120, 900) })], + complete: true, + ...overrides, + }) +} + +/** The vanished-sources day is still there, with its original accounting. */ +function expectPreserved(cache: DailyCache): void { + const kept = cache.days.find(d => d.date === VANISHED.date) + expect(kept).toMatchObject({ cost: 399.70, calls: 1572 }) + expect(kept!.providers['claude']!.cost).toBe(399.70) +} + +describe('daily cache: a degraded session parse never finalizes history', () => { + it('does not publish complete, and does not advance the watermark past what it covered', async () => { + await seed() + const out = await ensureCacheHydrated(noSessions, () => [], 'cfg-A', () => false) + // The parse covered nothing it can vouch for, so the watermark stays put: + // advancing it to yesterday would put the missed days behind gapStart + // (lastComputedDate + 1) forever. + expect(out.lastComputedDate).toBe(daysAgoStr(4)) + expect(out.complete).toBe(false) + expectPreserved(out) + }) + + it('does not advance the watermark on the full re-derive path either', async () => { + await seed({ complete: false }) + const out = await ensureCacheHydrated(noSessions, () => [], 'cfg-A', () => false) + expect(out.lastComputedDate).toBe(daysAgoStr(4)) + expect(out.complete).toBe(false) + expectPreserved(out) + }) + + it('a later healthy run rebuilds the days the degraded run missed', async () => { + await seed() + await ensureCacheHydrated(noSessions, () => [], 'cfg-A', () => false) + const missed = [1, 2, 3].map(n => day(daysAgoStr(n), { claude: slice(n * 10, n * 100) })) + const healed = await ensureCacheHydrated(noSessions, () => missed, 'cfg-A', () => true) + expect(healed.days.map(d => d.date)).toEqual([ + daysAgoStr(40), daysAgoStr(4), daysAgoStr(3), daysAgoStr(2), daysAgoStr(1), + ]) + expect(healed.lastComputedDate).toBe(daysAgoStr(1)) + expect(healed.complete).toBe(true) + expectPreserved(healed) + }) +}) + +describe('daily cache: a complete cache that outruns its own data is not trusted', () => { + it('re-derives the days between the newest entry and the watermark', async () => { + // The field artifact: complete: true, lastComputedDate yesterday, entries + // stopping four days earlier — written by a run that finalized off a parse + // which never covered those days. + await seed({ lastComputedDate: daysAgoStr(1) }) + const ranges: DateRange[] = [] + const missed = [1, 2, 3].map(n => day(daysAgoStr(n), { claude: slice(n * 10, n * 100) })) + const out = await ensureCacheHydrated( + async (range) => { ranges.push(range); return [] }, + () => missed, + 'cfg-A', + () => true, + ) + expect(ranges).toHaveLength(1) + expect(out.days.map(d => d.date)).toEqual([ + daysAgoStr(40), daysAgoStr(4), daysAgoStr(3), daysAgoStr(2), daysAgoStr(1), + ]) + expect(out.days.find(d => d.date === daysAgoStr(2))!.cost).toBe(20) + expect(out.complete).toBe(true) + expectPreserved(out) + }) + + it('a degraded re-derivation of those days still keeps every carried day', async () => { + await seed({ lastComputedDate: daysAgoStr(1) }) + const out = await ensureCacheHydrated(noSessions, () => [], 'cfg-A', () => false) + expect(out.complete).toBe(false) + expect(out.days.map(d => d.date)).toEqual([daysAgoStr(40), daysAgoStr(4)]) + expectPreserved(out) + }) + + it('an empty cache still finalizes — no re-parse treadmill on a machine with no history', async () => { + await seed({ days: [], lastComputedDate: daysAgoStr(1) }) + let parses = 0 + const out = await ensureCacheHydrated( + async () => { parses += 1; return [] }, + () => [], + 'cfg-A', + () => true, + ) + expect(parses).toBe(0) + expect(out.lastComputedDate).toBe(daysAgoStr(1)) + expect(out.complete).toBe(true) + }) +}) diff --git a/tests/parser-cache-refresh-timeout.test.ts b/tests/parser-cache-refresh-timeout.test.ts index 92415775..8883c5a7 100644 --- a/tests/parser-cache-refresh-timeout.test.ts +++ b/tests/parser-cache-refresh-timeout.test.ts @@ -7,7 +7,7 @@ vi.mock('../src/cache-refresh-lock.js', () => ({ acquireCacheRefreshLock: async () => ({ outcome: 'timed-out' as const }), })) -import { clearSessionCache, parseAllSessions } from '../src/parser.js' +import { clearSessionCache, isSessionHydrationComplete, parseAllSessions } from '../src/parser.js' import { sessionCachePath } from '../src/session-cache.js' let root: string @@ -59,4 +59,46 @@ describe('parseAllSessions warm refresh timeout', () => { expect(output(await parseAllSessions(undefined, 'claude'))).toBe(50) expect(await readFile(sessionCachePath(), 'utf-8')).toBe(before) }) + + // The snapshot a timed-out refresh serves is only as good as what has changed + // under it. Anything the daily backfill finalizes off a snapshot that skipped + // real files freezes those days out of history for good, so the completeness + // signal has to distinguish the two cases. + it('does not report a complete hydration when the served snapshot is stale', async () => { + await writeSession(50) + await parseAllSessions(undefined, 'claude') + expect(isSessionHydrationComplete()).toBe(true) + + await writeSession(5000) + clearSessionCache() + await parseAllSessions(undefined, 'claude') + expect(isSessionHydrationComplete()).toBe(false) + }) + + it('does not report a complete hydration when a session file is missing from the snapshot', async () => { + await writeSession(50) + await parseAllSessions(undefined, 'claude') + + await writeFile(join(sessionPath, '..', 'other.jsonl'), JSON.stringify({ + type: 'assistant', + sessionId: 'sess-2', + timestamp: '2026-05-16T10:00:00Z', + cwd: '/tmp/proj', + message: { + id: 'msg-other', type: 'message', role: 'assistant', model: 'claude-sonnet-4-5', + content: [], usage: { input_tokens: 100, output_tokens: 7 }, + }, + }) + '\n') + clearSessionCache() + await parseAllSessions(undefined, 'claude') + expect(isSessionHydrationComplete()).toBe(false) + }) + + it('still reports a complete hydration when nothing changed under the snapshot', async () => { + await writeSession(50) + await parseAllSessions(undefined, 'claude') + clearSessionCache() + await parseAllSessions(undefined, 'claude') + expect(isSessionHydrationComplete()).toBe(true) + }) })