From 77b41ce63bcd2ed915e11bb7c30103919ede0ec6 Mon Sep 17 00:00:00 2001 From: AVSR Pavan Kumar Date: Wed, 22 Jul 2026 23:50:51 +0530 Subject: [PATCH] feat(act): measure realized savings for defer-* actions in act report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The defer-enable / defer-alwaysload / defer-threshold plan kinds (part 2 of the deferral-coverage work) applied and undid correctly but were invisible to `act report` — it returned "not measurable: no baseline captured at apply time", because report.ts never captured a baseline for them or knew how to compute their realized delta. Wire them in, mirroring the mcp-remove path since deferral has the same effect (MCP tool-def schema leaves the upfront prefix): - needsConfigBaseline + captureBaseline now cover the defer-* kinds. Servers are the named set for defer-alwaysload, or the observed MCP surface for defer-enable / defer-threshold (which re-enable deferral across everything). Per-session tokens use observed tool counts, or the 5-tools x 400 fallback, exactly like mcp-remove. - A new deferRow computes realized savings as per-session prefix tokens times the post-apply sessions where deferral actually became active — detected by the same deferred-tools-inventory signal the mcp-deferral-off detector uses. Sessions begun before the client restarted still run deferral-off and are excluded; if none benefited, the row reports "not yet in effect" with zero realized rather than claiming a saving that has not taken hold. Records applied before this change (no baseline) keep the existing "no baseline captured at apply time" note, so nothing regresses. 11 tests cover the measured, partial, not-yet-in-effect, no-sessions, empty-baseline, and missing-baseline paths, plus baseline capture for each kind. Verified live end to end: apply captures the baseline, and act report reports realized savings scaled to the sessions that adopted deferral. --- src/act/report.ts | 80 +++++++++++++++++++++- tests/act-report.test.ts | 140 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 1 deletion(-) diff --git a/src/act/report.ts b/src/act/report.ts index 95ec6e8c..359f4cba 100644 --- a/src/act/report.ts +++ b/src/act/report.ts @@ -42,6 +42,13 @@ const HONEST_FOOTER = + 'Guard rows are correlation, not attribution. Realized numbers are rounded down.' const MCP_KINDS = new Set(['mcp-remove', 'mcp-project-scope']) +// defer-* re-enable native MCP tool deferral (part 2 of #614): the same +// prefix schema tokens mcp-remove eliminates, deferral moves out of the +// upfront prefix. Realized the same way — per-session schema tokens times the +// post-apply sessions that benefited — but "benefited" flips: instead of a +// server no longer loading, it is deferral having become active (the session +// now carries a deferred-tools inventory, the detector's own signal). +const DEFER_KINDS = new Set(['defer-enable', 'defer-alwaysload', 'defer-threshold']) const ARCHIVE_DEF_TOKENS: Partial> = { 'archive-skill': TOKENS_PER_SKILL_DEF, 'archive-agent': TOKENS_PER_AGENT_DEF, @@ -178,6 +185,28 @@ function countSessionsLoading(projects: ProjectSummary[], servers: string[]): nu return allSessions(projects).filter(s => sessionLoadsAny(s, servers)).length } +// Deferral is active in a session exactly when Claude Code emitted a +// deferred-tools inventory for it — the same signal the mcp-deferral-off +// detector uses (its absence, alongside MCP overhead, is what flags a gap). +function sessionHasDeferralActive(s: SessionSummary): boolean { + return (s.mcpInventory?.length ?? 0) > 0 +} + +// MCP servers observed loading in the window (via inventory or invocation). +// defer-enable / defer-threshold re-enable deferral for the whole MCP surface +// rather than a named set, so the affected servers are derived here. +function observedMcpServers(projects: ProjectSummary[]): string[] { + const servers = new Set() + for (const s of allSessions(projects)) { + for (const fqn of s.mcpInventory ?? []) { + const seg = fqn.split('__')[1] + if (seg) servers.add(seg) + } + for (const server of Object.keys(s.mcpBreakdown)) servers.add(server) + } + return [...servers] +} + // A kind whose realized effect is a token saving (everything except guard, // which is a dollars/yield correlation, and out-of-scope kinds). function isTokenKind(kind: ActionKind): boolean { @@ -226,6 +255,33 @@ function mcpRow( return { ...base, estimatedForWindow, status: 'measured', realizedTokens: Math.floor(perSessionTokens * savedSessions), confidence } } +function deferRow( + base: ActReportRow, sessions: SessionSummary[], + baseline: ActionBaseline, afterStart: Date, now: Date, +): ActReportRow { + const perSessionTokens = Object.values(baseline.metrics).reduce((a, b) => a + b, 0) + if (perSessionTokens === 0) return { ...base, note: 'not measurable: empty baseline' } + if (sessions.length === 0) return { ...base, note: 'not measurable: no sessions in the window yet' } + const estimatedForWindow = Math.floor(perSessionTokens * sessions.length) + // A post-apply session realized the saving only if deferral actually became + // active in it. ENABLE_TOOL_SEARCH is read at process start, so sessions + // begun before the user restarted still run deferral-off — those aren't + // counted, and if none benefited we report it plainly rather than claim a + // saving that hasn't taken effect. + const deferredSessions = sessions.filter(sessionHasDeferralActive).length + const confidence = confidenceFor(sessions.length, baseline, afterStart, now) + if (deferredSessions === 0) { + return { + ...base, + estimatedForWindow, + status: 'reverted', + confidence, + note: `not yet in effect: deferral is still inactive in ${sessions.length} post-apply session${sessions.length === 1 ? '' : 's'} (takes effect on the next session; the client may not have restarted, or the change was reverted)`, + } + } + return { ...base, estimatedForWindow, status: 'measured', realizedTokens: Math.floor(perSessionTokens * deferredSessions), confidence } +} + function archiveRow( base: ActReportRow, rec: ActionRecord, sessions: SessionSummary[], baseline: ActionBaseline, afterStart: Date, now: Date, @@ -378,6 +434,7 @@ async function computeRow( if (!baseline) return { ...base, note: 'not measurable: no baseline captured at apply time' } if (MCP_KINDS.has(rec.kind)) return mcpRow(base, rec, sessions, baseline, afterStart, now) + if (DEFER_KINDS.has(rec.kind)) return deferRow(base, sessions, baseline, afterStart, now) if (rec.kind in ARCHIVE_DEF_TOKENS) return archiveRow(base, rec, sessions, baseline, afterStart, now) if (rec.kind === 'claude-md-rule') return readEditRow(base, sessions, baseline, afterStart, now) if (rec.kind === 'shell-config') return { ...base, note: 'not measurable: bash result token sizes are not retained in the summary' } @@ -585,7 +642,15 @@ function mcpServersFromApply(finding: WasteFinding): string[] { } function needsConfigBaseline(kind: ActionKind): boolean { - return MCP_KINDS.has(kind) || kind in ARCHIVE_DEF_TOKENS || kind === 'claude-md-rule' || kind === 'shell-config' + return MCP_KINDS.has(kind) || DEFER_KINDS.has(kind) || kind in ARCHIVE_DEF_TOKENS || kind === 'claude-md-rule' || kind === 'shell-config' +} + +// Servers whose upfront schema deferral removes from the prefix. defer-alwaysload +// names them; defer-enable / defer-threshold re-enable deferral across the whole +// observed MCP surface. +function deferServers(finding: WasteFinding, ctx: CaptureCtx): string[] { + if (finding.apply?.kind === 'defer-alwaysload') return finding.apply.servers.map(s => s.server) + return observedMcpServers(ctx.projects) } export function captureBaseline(finding: WasteFinding, kind: ActionKind, ctx: CaptureCtx): ActionBaseline | undefined { @@ -608,6 +673,19 @@ export function captureBaseline(finding: WasteFinding, kind: ActionKind, ctx: Ca return { ...common, sessions: countSessionsLoading(ctx.projects, servers), metrics } } + if (DEFER_KINDS.has(kind)) { + const servers = deferServers(finding, ctx) + if (servers.length === 0) return undefined + const covByServer = new Map(ctx.coverage.map(c => [c.server, c])) + const metrics: Record = {} + for (const server of servers) { + const cov = covByServer.get(server) + const tools = cov && cov.toolsAvailable > 0 ? cov.toolsAvailable : TOOLS_PER_MCP_SERVER + metrics[server] = tools * TOKENS_PER_MCP_TOOL + } + return { ...common, sessions: countSessionsLoading(ctx.projects, servers), metrics } + } + const defTokens = ARCHIVE_DEF_TOKENS[kind] if (defTokens !== undefined) { const names = finding.apply?.kind === 'archive' ? finding.apply.names : [] diff --git a/tests/act-report.test.ts b/tests/act-report.test.ts index d690f138..af919a33 100644 --- a/tests/act-report.test.ts +++ b/tests/act-report.test.ts @@ -7,10 +7,12 @@ import { journalPath } from '../src/act/journal.js' import { buildActReportJson, buildOptimizeAppliedHeader, + captureBaseline, computeActReport, renderActReport, } from '../src/act/report.js' import type { ActionRecord } from '../src/act/types.js' +import type { WasteFinding } from '../src/optimize.js' import type { ClassifiedTurn, ProjectSummary } from '../src/types.js' type Session = ProjectSummary['sessions'][number] @@ -584,3 +586,141 @@ describe('json + render shape', () => { expect(out).toMatch(/scaled to the measured window/) }) }) + +// --------------------------------------------------------------------------- +// defer-* realized deltas (part 2 of #614) +// --------------------------------------------------------------------------- + +function deferRecord(over: Partial = {}): ActionRecord { + const at = daysAgo(10) + return { + id: 'd1', + at, + kind: 'defer-enable', + findingId: 'mcp-deferral-off', + description: 'Remove the ENABLE_TOOL_SEARCH=false override from settings.json', + changes: [], + status: 'applied', + // 2 servers x 2000 tokens/session = 4000 prefix tokens/session. + baseline: { windowDays: 14, capturedAt: at, estimatedTokens: 40_000, sessions: 5, metrics: { everything: 2000, 'fs-tools': 2000 } }, + ...over, + } +} + +// NOTE: an mcpInventory on a post-apply session means the OPPOSITE for defer-* +// vs mcp-remove. For mcp-remove it is "the server loaded again" (reverted); +// for defer-* it is "deferral became active" (saved). Same fixture, inverse +// meaning — exactly the design. +const DEFERRED = { mcpInventory: ['mcp__everything__get-sum'] } + +describe('defer realized delta', () => { + it('measures savings across post-apply sessions where deferral became active', async () => { + const actionsDir = await writeJournal([deferRecord()]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), DEFERRED))]) }) + + const row = report.rows[0]! + expect(row.status).toBe('measured') + expect(row.realizedTokens).toBe(20_000) // 4000/session * 5 deferred sessions + expect(row.estimatedForWindow).toBe(20_000) + expect(report.totalRealizedTokens).toBe(20_000) + }) + + it('reports "not yet in effect" with zero savings when no post-apply session shows deferral', async () => { + const actionsDir = await writeJournal([deferRecord()]) + // sessions with MCP activity but NO inventory = deferral still off + const off = sessionsAt(4, daysAgo(5), { mcpBreakdown: { everything: { calls: 2, savingsUSD: 0, costUSD: 0 } } }) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(off)]) }) + + const row = report.rows[0]! + expect(row.status).toBe('reverted') + expect(row.realizedTokens ?? 0).toBe(0) + expect(row.note).toMatch(/not yet in effect/) + expect(report.totalRealizedTokens).toBe(0) + }) + + it('counts only the sessions where deferral actually became active (partial)', async () => { + const actionsDir = await writeJournal([deferRecord()]) + const active = sessionsAt(3, daysAgo(5), DEFERRED) + const stillOff = sessionsAt(2, daysAgo(4)) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf([...active, ...stillOff])]) }) + + const row = report.rows[0]! + expect(row.status).toBe('measured') + expect(row.realizedTokens).toBe(12_000) // 4000 * 3 active (2 still-off excluded) + expect(row.estimatedForWindow).toBe(20_000) // 4000 * all 5 window sessions + }) + + it('is not measurable when no post-apply sessions exist yet', async () => { + const actionsDir = await writeJournal([deferRecord()]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf([])]) }) + expect(report.rows[0]!.note).toMatch(/no sessions in the window yet/) + }) + + it('is not measurable with an empty baseline (zero prefix tokens)', async () => { + const rec = deferRecord({ baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 0, sessions: 5, metrics: { everything: 0 } } }) + const actionsDir = await writeJournal([rec]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), DEFERRED))]) }) + expect(report.rows[0]!.note).toMatch(/empty baseline/) + }) + + it('falls back to the no-baseline note for records applied before baselines existed', async () => { + const rec = deferRecord({ baseline: undefined }) + const actionsDir = await writeJournal([rec]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), DEFERRED))]) }) + expect(report.rows[0]!.note).toMatch(/no baseline captured at apply time/) + }) + + it('measures defer-alwaysload against its named servers', async () => { + const rec = deferRecord({ + kind: 'defer-alwaysload', + findingId: 'mcp-alwaysload-hygiene', + description: 'Unpin an alwaysLoad MCP server', + baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 30_000, sessions: 6, metrics: { 'heavy-server': 5000 } }, + }) + const actionsDir = await writeJournal([rec]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(6, daysAgo(5), DEFERRED))]) }) + const row = report.rows[0]! + expect(row.status).toBe('measured') + expect(row.realizedTokens).toBe(30_000) // 5000 * 6 + }) + + it('measures defer-threshold like the other defer kinds', async () => { + const rec = deferRecord({ kind: 'defer-threshold', findingId: 'mcp-defer-threshold', description: 'Tighten the auto threshold' }) + const actionsDir = await writeJournal([rec]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(5, daysAgo(5), DEFERRED))]) }) + expect(report.rows[0]!.status).toBe('measured') + expect(report.rows[0]!.realizedTokens).toBe(20_000) + }) +}) + +describe('defer baseline capture', () => { + const finding = (apply: WasteFinding['apply']): WasteFinding => ({ + id: 'mcp-deferral-off', + title: 't', explanation: 'e', impact: 'medium', tokensSaved: 40_000, + fix: { type: 'command', label: 'l', text: 'x' }, + apply, + }) + const ctx = (projects: ProjectSummary[]) => ({ projects, coverage: [], windowDays: 14, now: NOW }) + + it('derives servers from observed MCP usage for defer-enable', () => { + const projects = [projectOf(sessionsAt(3, daysAgo(5), { mcpBreakdown: { everything: { calls: 2, savingsUSD: 0, costUSD: 0 }, 'fs-tools': { calls: 1, savingsUSD: 0, costUSD: 0 } } }))] + const b = captureBaseline(finding({ kind: 'defer-enable', cause: 'env-false', settingPath: '/x', settingScope: 'project settings', value: 'false' }), 'defer-enable', ctx(projects)) + expect(b).toBeDefined() + // no coverage -> 5 tools x 400 fallback per server + expect(b!.metrics.everything).toBe(2000) + expect(b!.metrics['fs-tools']).toBe(2000) + }) + + it('uses the named servers for defer-alwaysload', () => { + const projects = [projectOf(sessionsAt(2, daysAgo(5)))] + const b = captureBaseline(finding({ kind: 'defer-alwaysload', servers: [{ server: 'pinned', paths: ['/a/.mcp.json'] }] }), 'defer-alwaysload', ctx(projects)) + expect(b).toBeDefined() + expect(Object.keys(b!.metrics)).toEqual(['pinned']) + }) + + it('returns undefined when there is no observed MCP surface to defer', () => { + const projects = [projectOf(sessionsAt(3, daysAgo(5)))] // no mcpBreakdown, no inventory + const b = captureBaseline(finding({ kind: 'defer-enable', cause: 'env-false', settingPath: '/x', settingScope: 'project settings', value: 'false' }), 'defer-enable', ctx(projects)) + expect(b).toBeUndefined() + }) +})