From c304bc20cd06db5ddd162514cbadf8ef9bc626ca Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 7 Aug 2026 16:30:49 -0500 Subject: [PATCH 01/12] fix(inference): merge agentic spec-decode points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Treat speculative decoding as point-level metadata for agentic scenarios while retaining fixed-sequence curve identity. Keep availability and run selection aligned, preserve overlapping point identity, and expose the method in bilingual tooltips. 中文：修复（推理）：合并智能体场景中的投机解码点。智能体曲线不再按投机解码方式拆分，同时保持定长场景的现有曲线标识；同步可用性与运行筛选，避免重叠点丢失，并在中英文 tooltip 中展示每个点的投机解码方式。 --- docs/data-transforms.md | 2 +- .../components/inference/InferenceContext.tsx | 16 ++++++- .../components/inference/ui/ScatterGraph.tsx | 11 +---- .../inference/utils/point-identity.test.ts | 48 +++++++++++++++++++ .../inference/utils/point-identity.ts | 16 +++++++ .../inference/utils/runEnumeration.test.ts | 10 ++++ .../inference/utils/runEnumeration.ts | 7 +-- .../inference/utils/tooltip-utils.test.ts | 17 +++++++ .../inference/utils/tooltipUtils.ts | 24 ++++++++-- packages/app/src/lib/api.ts | 1 + .../app/src/lib/benchmark-transform.test.ts | 22 +++++++++ packages/app/src/lib/chart-utils.test.ts | 16 +++++++ packages/app/src/lib/chart-utils.ts | 24 +++++++--- packages/db/src/queries/workflow-info.ts | 4 +- 14 files changed, 191 insertions(+), 27 deletions(-) create mode 100644 packages/app/src/components/inference/utils/point-identity.test.ts create mode 100644 packages/app/src/components/inference/utils/point-identity.ts diff --git a/docs/data-transforms.md b/docs/data-transforms.md index 1dfaf33cd..b04ae973e 100644 --- a/docs/data-transforms.md +++ b/docs/data-transforms.md @@ -90,7 +90,7 @@ This is the most complex and bug-prone part of the pipeline. A bad hardware key 1. Base GPU: `entry.hw.split('-')[0]` strips any `-DP` / `-MN` variant suffix from the hardware field (e.g. `"h100-8"` → `"h100"`). 2. Framework suffix: appends `_${entry.framework}`. The direct key (`h100_trt`) is tested via `isKnownGpu()` (checks whether the base GPU exists in `HW_REGISTRY`). If the direct key's base is unknown and `entry.disagg` is true, a `-disagg` variant is tried. -3. Spec decoding suffix: if `entry.mtp === 'on'` or `entry.spec_decoding === 'mtp'`, appends `_mtp`. Otherwise, any non-`'none'` `spec_decoding` value is appended as-is (e.g. `_eagle`). +3. Spec decoding suffix: for fixed-sequence rows, if `entry.mtp === 'on'` or `entry.spec_decoding === 'mtp'`, appends `_mtp`. Otherwise, any non-`'none'` `spec_decoding` value is appended as-is (e.g. `_eagle`). Agentic rows deliberately omit this suffix because one production curve may combine speculative and standard-decoding points; `spec_decoding` remains on each point for filtering, tooltip metadata, and point-level identity. The resulting key's base GPU must exist in `HW_REGISTRY`. Display fields (label, suffix, gpu tooltip) are derived dynamically by `getHardwareConfig()`. Unrecognised base GPUs fall back to the `unknown` hardware config. diff --git a/packages/app/src/components/inference/InferenceContext.tsx b/packages/app/src/components/inference/InferenceContext.tsx index ca38ae70d..4b43c7e07 100644 --- a/packages/app/src/components/inference/InferenceContext.tsx +++ b/packages/app/src/components/inference/InferenceContext.tsx @@ -505,7 +505,13 @@ export function InferenceProvider({ if (rowToSequence(r) !== effectiveSequence) return false; if (!effectivePrecisions.includes(r.precision)) return false; if (!r.hardware) return false; - const hwKey = buildAvailabilityHwKey(r.hardware, r.framework, r.spec_method, r.disagg); + const hwKey = buildAvailabilityHwKey( + r.hardware, + r.framework, + r.spec_method, + r.disagg, + r.benchmark_type, + ); return selectedGPUs.includes(hwKey); }); const dates = [...new Set(rows.map((r) => r.date))].toSorted(); @@ -530,7 +536,13 @@ export function InferenceProvider({ if (rowToSequence(r) !== effectiveSequence) continue; if (!effectivePrecisions.includes(r.precision)) continue; if (!r.hardware) continue; - const hwKey = buildAvailabilityHwKey(r.hardware, r.framework, r.spec_method, r.disagg); + const hwKey = buildAvailabilityHwKey( + r.hardware, + r.framework, + r.spec_method, + r.disagg, + r.benchmark_type, + ); if (isKnownGpu(hwKey)) hwKeys.add(hwKey); } return [...hwKeys] diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index e123bde11..89382a779 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -70,6 +70,7 @@ import { generateTooltipContent, getPointLabel, } from '@/components/inference/utils/tooltipUtils'; +import { scatterPointConfigId } from '@/components/inference/utils/point-identity'; import LegendPointsDialog from '@/components/inference/ui/LegendPointsDialog'; import { OFFLOAD_HALO_DASHARRAY, @@ -819,15 +820,7 @@ const ScatterGraph = React.memo( return ids; }, [trackedConfigs]); - const buildPointConfigId = useCallback((point: InferenceData): string => { - let key = `${point.hwKey}|${point.precision}|${point.tp}|${point.conc}|${point.decode_ep ?? 0}|${point.prefill_tp ?? 0}|${point.prefill_ep ?? 0}`; - if (point.disagg) key += `|disagg|${point.num_prefill_gpu ?? 0}|${point.num_decode_gpu ?? 0}`; - // Agentic runs emit two rows per (config, conc) — one offload=on, one off. - // Without this suffix, d3's data join treats them as the same point and - // drops one variant (along with its halo). - if (point.offload_mode) key += `|offload-${point.offload_mode}`; - return key; - }, []); + const buildPointConfigId = useCallback(scatterPointConfigId, []); // filteredData: visible points only (for scale domain calculation) const filteredData = useMemo( diff --git a/packages/app/src/components/inference/utils/point-identity.test.ts b/packages/app/src/components/inference/utils/point-identity.test.ts new file mode 100644 index 000000000..5f3ee14ae --- /dev/null +++ b/packages/app/src/components/inference/utils/point-identity.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; + +import type { InferenceData } from '@/components/inference/types'; + +import { scatterPointConfigId } from './point-identity'; + +const point = (overrides: Partial): InferenceData => + ({ + hwKey: 'h200_vllm', + precision: 'fp8', + tp: 8, + conc: 32, + ...overrides, + }) as InferenceData; + +describe('scatterPointConfigId', () => { + it('keeps overlapping agentic MTP and standard-decoding points distinct', () => { + const standard = scatterPointConfigId( + point({ benchmark_type: 'agentic_traces', spec_decoding: 'none' }), + ); + const mtp = scatterPointConfigId( + point({ benchmark_type: 'agentic_traces', spec_decoding: 'mtp' }), + ); + + expect(standard).not.toBe(mtp); + expect(standard).toContain('|spec-none'); + expect(mtp).toContain('|spec-mtp'); + }); + + it('keeps agentic offload variants distinct alongside spec methods', () => { + const off = scatterPointConfigId( + point({ + benchmark_type: 'agentic_traces', + spec_decoding: 'mtp', + offload_mode: 'off', + }), + ); + const on = scatterPointConfigId( + point({ + benchmark_type: 'agentic_traces', + spec_decoding: 'mtp', + offload_mode: 'on', + }), + ); + + expect(off).not.toBe(on); + }); +}); diff --git a/packages/app/src/components/inference/utils/point-identity.ts b/packages/app/src/components/inference/utils/point-identity.ts new file mode 100644 index 000000000..dd897796c --- /dev/null +++ b/packages/app/src/components/inference/utils/point-identity.ts @@ -0,0 +1,16 @@ +import type { InferenceData } from '@/components/inference/types'; + +/** Stable D3 join key for one scatter point within a chart series. */ +export function scatterPointConfigId(point: InferenceData): string { + let key = `${point.hwKey}|${point.precision}|${point.tp}|${point.conc}|${point.decode_ep ?? 0}|${point.prefill_tp ?? 0}|${point.prefill_ep ?? 0}`; + if (point.disagg) { + key += `|disagg|${point.num_prefill_gpu ?? 0}|${point.num_decode_gpu ?? 0}`; + } + if (point.offload_mode) key += `|offload-${point.offload_mode}`; + // Agentic series omit spec decoding from hwKey so one curve can mix methods. + // It remains point identity to avoid collapsing overlapping MTP/STP results. + if (point.benchmark_type === 'agentic_traces') { + key += `|spec-${point.spec_decoding ?? 'none'}`; + } + return key; +} diff --git a/packages/app/src/components/inference/utils/runEnumeration.test.ts b/packages/app/src/components/inference/utils/runEnumeration.test.ts index 155fae66a..333d2b502 100644 --- a/packages/app/src/components/inference/utils/runEnumeration.test.ts +++ b/packages/app/src/components/inference/utils/runEnumeration.test.ts @@ -16,6 +16,7 @@ function rc(over: Partial): RunConfigRow { framework: 'vllm', spec_method: 'none', disagg: false, + benchmark_type: 'single_turn', ...over, }; } @@ -66,6 +67,15 @@ describe('dataRunsForDate', () => { expect(runs.map((r) => r.runId)).toEqual(['2']); }); + it('maps agentic MTP and non-MTP run coverage to one GPU series', () => { + const rows = [ + rc({ github_run_id: 1, spec_method: 'none', benchmark_type: 'agentic_traces' }), + rc({ github_run_id: 2, spec_method: 'mtp', benchmark_type: 'agentic_traces' }), + ]; + const runs = dataRunsForDate(rows, SCOPE); + expect(runs.map((r) => r.runId)).toEqual(['1', '2']); + }); + it('excludes runs for other models, precisions, and GPUs', () => { const rows = [ rc({ github_run_id: 1 }), // matches diff --git a/packages/app/src/components/inference/utils/runEnumeration.ts b/packages/app/src/components/inference/utils/runEnumeration.ts index fda1ae30f..94758cb78 100644 --- a/packages/app/src/components/inference/utils/runEnumeration.ts +++ b/packages/app/src/components/inference/utils/runEnumeration.ts @@ -10,9 +10,9 @@ * changelog entry, and that newest run is exactly the one the plain-date "latest" * view shows — so enumerating from changelog entries alone would silently drop it. * - * Runs are scoped to the selected GPUs using the canonical {@link getHardwareKey} - * so MTP and disagg variants (separate hw keys) are kept distinct, exactly as the - * chart keys them. + * Runs are scoped to the selected GPUs using the canonical {@link getHardwareKey}. + * Fixed-sequence MTP variants stay distinct; agentic MTP and non-MTP points map + * to the same series, exactly as the chart keys them. */ import type { AggDataEntry } from '@/components/inference/types'; @@ -46,6 +46,7 @@ function runConfigHwKey(rc: RunConfigRow): string { framework: rc.framework, disagg: rc.disagg, spec_decoding: rc.spec_method, + benchmark_type: rc.benchmark_type, } as unknown as AggDataEntry); } diff --git a/packages/app/src/components/inference/utils/tooltip-utils.test.ts b/packages/app/src/components/inference/utils/tooltip-utils.test.ts index 7adac5550..8d47a2e94 100644 --- a/packages/app/src/components/inference/utils/tooltip-utils.test.ts +++ b/packages/app/src/components/inference/utils/tooltip-utils.test.ts @@ -491,6 +491,23 @@ describe('generateOverlayTooltipContent', () => { expect(html).toContain('CPU Cache Hit Rate: 42.0%'); }); + it('shows point-level speculative decoding for mixed agentic overlays', () => { + const mtp = generateOverlayTooltipContent( + overlayConfig({ + data: pt({ benchmark_type: 'agentic_traces', spec_decoding: 'mtp' }), + }), + ); + const standardZh = generateOverlayTooltipContent( + overlayConfig({ + data: pt({ benchmark_type: 'agentic_traces', spec_decoding: 'none' }), + locale: 'zh', + }), + ); + + expect(mtp).toContain('Speculative Decoding: MTP'); + expect(standardZh).toContain('投机解码: 关闭'); + }); + it('hides stale CPU cache hits for unofficial overlays without offload', () => { const html = generateOverlayTooltipContent( overlayConfig({ diff --git a/packages/app/src/components/inference/utils/tooltipUtils.ts b/packages/app/src/components/inference/utils/tooltipUtils.ts index 5660cbdb2..53bc0cea6 100644 --- a/packages/app/src/components/inference/utils/tooltipUtils.ts +++ b/packages/app/src/components/inference/utils/tooltipUtils.ts @@ -1,4 +1,5 @@ import { formatNumber, getDisplayLabel } from '@/lib/utils'; +import { specMethodDisplayLabel } from '@/lib/compare-variant-slug'; import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; import type { Locale } from '@/lib/i18n'; import { isKvOffloadEnabled } from '@/lib/kv-offload'; @@ -172,10 +173,25 @@ const generateCacheMetadataHTML = (d: InferenceData, locale: Locale): string => * Agentic-only request success and token totals. Cache metadata is rendered * separately because fixed-sequence rows can carry it too. */ -const generateAgenticHTML = (d: InferenceData): string => { +const AGENTIC_STRINGS = { + en: { speculativeDecoding: 'Speculative Decoding', off: 'Off' }, + zh: { speculativeDecoding: '投机解码', off: '关闭' }, +} as const; + +const generateAgenticHTML = (d: InferenceData, locale: Locale): string => { if (d.benchmark_type !== 'agentic_traces') return ''; + const t = AGENTIC_STRINGS[locale]; const parts: string[] = []; + const specMethod = d.spec_decoding ?? 'none'; + parts.push( + tooltipLine( + t.speculativeDecoding, + specMethod === 'none' || specMethod === '' + ? t.off + : specMethodDisplayLabel(d.model, specMethod), + ), + ); if (d.num_requests_total !== undefined && d.num_requests_successful !== undefined) { const successPct = @@ -370,7 +386,7 @@ export const generateTooltipContent = (config: TooltipConfig): string => { Precision: ${d.precision.toUpperCase()} ${generateCacheMetadataHTML(d, locale)} - ${generateAgenticHTML(d)} + ${generateAgenticHTML(d, locale)} ${runLinkHTML(runUrl)} ${viewChartsButtonHTML(isPinned, Boolean(hasTrace), d.id)} ${ @@ -430,7 +446,7 @@ export const generateOverlayTooltipContent = (config: OverlayTooltipConfig): str Precision: ${d.precision.toUpperCase()} ${generateCacheMetadataHTML(d, locale)} - ${generateAgenticHTML(d)} + ${generateAgenticHTML(d, locale)} `; }; @@ -501,7 +517,7 @@ export const generateGPUGraphTooltipContent = (config: TooltipConfig): string => Precision: ${d.precision.toUpperCase()} ${generateCacheMetadataHTML(d, locale)} - ${generateAgenticHTML(d)} + ${generateAgenticHTML(d, locale)} ${runLinkHTML(runUrl)} ${viewChartsButtonHTML(isPinned, Boolean(hasTrace), d.id)} diff --git a/packages/app/src/lib/api.ts b/packages/app/src/lib/api.ts index 9b7930e97..c7d9c0016 100644 --- a/packages/app/src/lib/api.ts +++ b/packages/app/src/lib/api.ts @@ -102,6 +102,7 @@ export interface RunConfigRow { framework: string; spec_method: string; disagg: boolean; + benchmark_type: string; } export interface WorkflowInfoResponse { diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index 95ff1958e..0248d7958 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -739,6 +739,28 @@ describe('transformBenchmarkRows — hardware key resolution', () => { expect(hardwareConfig).toHaveProperty('h200_trt_mtp'); }); + it('groups mixed agentic spec methods into one hardware series', () => { + const rows = [ + makeRow({ + benchmark_type: 'agentic_traces', + hardware: 'h200', + framework: 'trt', + spec_method: 'none', + }), + makeRow({ + benchmark_type: 'agentic_traces', + hardware: 'h200', + framework: 'trt', + spec_method: 'mtp', + }), + ]; + + const { chartData, hardwareConfig } = transformBenchmarkRows(rows); + expect(Object.keys(hardwareConfig)).toEqual(['h200_trt']); + expect(chartData[0].map((point) => point.hwKey)).toEqual(['h200_trt', 'h200_trt']); + expect(chartData[0].map((point) => point.spec_decoding)).toEqual(['none', 'mtp']); + }); + it('handles AMD hardware with vllm framework', () => { const rows = [makeRow({ hardware: 'mi300x', framework: 'vllm' })]; const { chartData, hardwareConfig } = transformBenchmarkRows(rows); diff --git a/packages/app/src/lib/chart-utils.test.ts b/packages/app/src/lib/chart-utils.test.ts index cfde6deda..b5d667cba 100644 --- a/packages/app/src/lib/chart-utils.test.ts +++ b/packages/app/src/lib/chart-utils.test.ts @@ -256,6 +256,15 @@ describe('buildAvailabilityHwKey', () => { expect(buildAvailabilityHwKey('h200', undefined, 'mtp')).toBe('h200_mtp'); }); + it('omits speculative decoding from agentic availability series keys', () => { + expect(buildAvailabilityHwKey('h200', 'sglang', 'mtp', false, 'agentic_traces')).toBe( + 'h200_sglang', + ); + expect(buildAvailabilityHwKey('h200', 'sglang', 'eagle', false, 'agentic_traces')).toBe( + 'h200_sglang', + ); + }); + it('handles undefined framework and spec method', () => { expect(buildAvailabilityHwKey('b200', undefined, undefined)).toBe('b200'); }); @@ -921,6 +930,13 @@ describe('getHardwareKey', () => { ); }); + it('omits speculative decoding from agentic series identity', () => { + const base = { hw: 'h100-sxm', framework: 'vllm', benchmark_type: 'agentic_traces' }; + expect(getHardwareKey(entry({ ...base, spec_decoding: 'none' }))).toBe('h100_vllm'); + expect(getHardwareKey(entry({ ...base, spec_decoding: 'mtp' }))).toBe('h100_vllm'); + expect(getHardwareKey(entry({ ...base, spec_decoding: 'eagle' }))).toBe('h100_vllm'); + }); + it('appends spec_decoding suffix when not "none" and not "mtp"', () => { expect(getHardwareKey(entry({ hw: 'b200-sxm', framework: '', spec_decoding: 'eagle' }))).toBe( 'b200_eagle', diff --git a/packages/app/src/lib/chart-utils.ts b/packages/app/src/lib/chart-utils.ts index e9b792348..087b3a114 100644 --- a/packages/app/src/lib/chart-utils.ts +++ b/packages/app/src/lib/chart-utils.ts @@ -196,7 +196,12 @@ export const Y_AXIS_METRICS = [ export type YAxisMetric = (typeof Y_AXIS_METRICS)[number]; /** - * Determines the correct hardware key based on the hardware name and MTP status. + * Determines the chart-series hardware key. + * + * Fixed-sequence curves keep speculative decoding in their identity. Agentic + * curves deliberately do not: one production curve may choose a speculative + * method for some load points and standard decoding for others. The point + * still carries `spec_decoding` for filters, tooltips, and point-level keys. */ export const getHardwareKey = (entry: AggDataEntry): string => { let normalizedHwName = entry.hw.split('-')[0]; @@ -215,10 +220,12 @@ export const getHardwareKey = (entry: AggDataEntry): string => { normalizedHwName = candidateDirect; } } - if (entry.mtp === 'on' || entry['spec_decoding'] === 'mtp') { - normalizedHwName = `${normalizedHwName}_mtp`; - } else if (entry['spec_decoding'] && entry['spec_decoding'] !== 'none') { - normalizedHwName = `${normalizedHwName}_${entry['spec_decoding']}`; + if (entry.benchmark_type !== 'agentic_traces') { + if (entry.mtp === 'on' || entry['spec_decoding'] === 'mtp') { + normalizedHwName = `${normalizedHwName}_mtp`; + } else if (entry['spec_decoding'] && entry['spec_decoding'] !== 'none') { + normalizedHwName = `${normalizedHwName}_${entry['spec_decoding']}`; + } } return normalizedHwName; }; @@ -270,6 +277,7 @@ export function buildAvailabilityHwKey( framework?: string, specMethod?: string, disagg?: boolean, + benchmarkType?: string, ): string { let hwKey = hardware.split('-')[0]; const fw = framework ? resolveFrameworkAlias(framework) : undefined; @@ -285,8 +293,10 @@ export function buildAvailabilityHwKey( hwKey = candidateDirect; } } - if (specMethod === 'mtp') hwKey = `${hwKey}_mtp`; - else if (specMethod && specMethod !== 'none') hwKey = `${hwKey}_${specMethod}`; + if (benchmarkType !== 'agentic_traces') { + if (specMethod === 'mtp') hwKey = `${hwKey}_mtp`; + else if (specMethod && specMethod !== 'none') hwKey = `${hwKey}_${specMethod}`; + } return hwKey; } diff --git a/packages/db/src/queries/workflow-info.ts b/packages/db/src/queries/workflow-info.ts index 01e13dd88..033218105 100644 --- a/packages/db/src/queries/workflow-info.ts +++ b/packages/db/src/queries/workflow-info.ts @@ -76,6 +76,7 @@ export interface RunConfigRow { framework: string; spec_method: string; disagg: boolean; + benchmark_type: string; } /** @@ -96,7 +97,8 @@ export async function getRunConfigsByDate(sql: DbClient, date: string): Promise< c.hardware, c.framework, c.spec_method, - c.disagg + c.disagg, + br.benchmark_type FROM benchmark_results br JOIN configs c ON c.id = br.config_id JOIN latest_workflow_runs wr ON wr.id = br.workflow_run_id From 4018ccfa779dceeec254c5438313986284c7af60 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 7 Aug 2026 16:41:25 -0500 Subject: [PATCH 02/12] fix(inference): distinguish tracked agentic spec modes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep performance-over-time selections and historical matches distinct for agentic MTP and standard-decoding points while preserving fixed-sequence behavior.\n\n中文：区分智能体场景中被跟踪的投机解码模式，避免 Performance Over Time 将相同拓扑的 MTP 与标准解码点合并，同时保持定长场景行为不变。 --- .../components/inference/InferenceContext.tsx | 24 +++--- .../inference/hooks/useTrendData.test.ts | 79 +++++++++++-------- .../inference/hooks/useTrendData.ts | 28 +------ .../app/src/components/inference/types.ts | 3 + .../inference/utils/point-identity.test.ts | 28 ++++++- .../inference/utils/point-identity.ts | 25 +++++- 6 files changed, 114 insertions(+), 73 deletions(-) diff --git a/packages/app/src/components/inference/InferenceContext.tsx b/packages/app/src/components/inference/InferenceContext.tsx index 4b43c7e07..bfad5c20d 100644 --- a/packages/app/src/components/inference/InferenceContext.tsx +++ b/packages/app/src/components/inference/InferenceContext.tsx @@ -73,6 +73,7 @@ import { comparisonExclusion as resolveComparisonExclusion, } from './utils/comparison-exclusion'; import { resolveLabelState, serializeLabelState } from './utils/label-defaults'; +import { trackedConfigIdentity } from './utils/point-identity'; import { EMPTY_QUICK_FILTERS, parseDeploymentModes, @@ -554,27 +555,26 @@ export function InferenceProvider({ }, [availabilityRows, dbModelKeys, effectiveSequence, effectivePrecisions, selectedModel]); // --- Tracked config functions --- - const buildTrackedConfigId = useCallback((point: InferenceData): string => { - let key = `${point.hwKey}|${point.precision}|${point.tp}|${point.conc}`; - if (point.disagg) { - key += `|disagg|${point.num_prefill_gpu ?? 0}|${point.num_decode_gpu ?? 0}`; - } - return key; - }, []); - const addTrackedConfig = useCallback( (point: InferenceData, chartType: string) => { setTrackedConfigs((prev) => { - const id = buildTrackedConfigId(point); + const id = trackedConfigIdentity(point); if (prev.some((c) => c.id === id)) { return prev.filter((c) => c.id !== id); } if (prev.length >= 6) return prev; const hwConfig = hardwareConfig[point.hwKey]; - const label = hwConfig + let label = hwConfig ? `${getDisplayLabel(hwConfig)} — TP${point.tp} conc=${point.conc} ${point.precision.toUpperCase()}` : `${point.hwKey} — TP${point.tp} conc=${point.conc} ${point.precision.toUpperCase()}`; + if (point.benchmark_type === 'agentic_traces') { + const specLabel = + point.spec_decoding && point.spec_decoding !== 'none' + ? point.spec_decoding.toUpperCase() + : 'STP'; + label += ` ${specLabel}`; + } const color = TABLEAU_10[prev.length % TABLEAU_10.length]; return [ @@ -591,11 +591,13 @@ export function InferenceProvider({ disagg: point.disagg, num_prefill_gpu: point.num_prefill_gpu, num_decode_gpu: point.num_decode_gpu, + benchmark_type: point.benchmark_type, + spec_decoding: point.spec_decoding, }, ]; }); }, - [buildTrackedConfigId, hardwareConfig], + [hardwareConfig], ); const removeTrackedConfig = useCallback((id: string) => { diff --git a/packages/app/src/components/inference/hooks/useTrendData.test.ts b/packages/app/src/components/inference/hooks/useTrendData.test.ts index edf926b00..37edf12b1 100644 --- a/packages/app/src/components/inference/hooks/useTrendData.test.ts +++ b/packages/app/src/components/inference/hooks/useTrendData.test.ts @@ -1,25 +1,7 @@ import { describe, it, expect } from 'vitest'; import type { InferenceData, TrackedConfig, TrendDataPoint } from '@/components/inference/types'; - -// ─── Re-implement the pure functions from useTrendData.ts for testing ─── -// These are module-private in the hook, so we replicate them here to verify behavior. - -function buildMatchKey(config: TrackedConfig): string { - let key = `${config.hwKey}|${config.precision}|${config.tp}|${config.conc}`; - if (config.disagg) { - key += `|disagg|${config.num_prefill_gpu ?? 0}|${config.num_decode_gpu ?? 0}`; - } - return key; -} - -function buildPointMatchKey(point: InferenceData): string { - let key = `${point.hwKey}|${point.precision}|${point.tp}|${point.conc}`; - if (point.disagg) { - key += `|disagg|${point.num_prefill_gpu ?? 0}|${point.num_decode_gpu ?? 0}`; - } - return key; -} +import { trackedConfigIdentity } from '@/components/inference/utils/point-identity'; function buildTrendLines( accumulator: Map>, @@ -73,10 +55,10 @@ function makePoint(overrides: Partial = {}): InferenceData { // ─── Tests ─── -describe('buildMatchKey', () => { +describe('trackedConfigIdentity for tracked configs', () => { it('builds a key from config fields', () => { const config = makeConfig(); - expect(buildMatchKey(config)).toBe('h100|fp8|8|64'); + expect(trackedConfigIdentity(config)).toBe('h100|fp8|8|64'); }); it('includes disagg fields when disagg is true', () => { @@ -85,24 +67,24 @@ describe('buildMatchKey', () => { num_prefill_gpu: 2, num_decode_gpu: 6, }); - expect(buildMatchKey(config)).toBe('h100|fp8|8|64|disagg|2|6'); + expect(trackedConfigIdentity(config)).toBe('h100|fp8|8|64|disagg|2|6'); }); it('uses 0 for missing disagg GPU counts', () => { const config = makeConfig({ disagg: true }); - expect(buildMatchKey(config)).toBe('h100|fp8|8|64|disagg|0|0'); + expect(trackedConfigIdentity(config)).toBe('h100|fp8|8|64|disagg|0|0'); }); it('does not include disagg when disagg is false', () => { const config = makeConfig({ disagg: false }); - expect(buildMatchKey(config)).toBe('h100|fp8|8|64'); + expect(trackedConfigIdentity(config)).toBe('h100|fp8|8|64'); }); }); -describe('buildPointMatchKey', () => { +describe('trackedConfigIdentity for chart points', () => { it('builds a key from data point fields', () => { const point = makePoint(); - expect(buildPointMatchKey(point)).toBe('h100|fp8|8|64'); + expect(trackedConfigIdentity(point)).toBe('h100|fp8|8|64'); }); it('includes disagg fields for disaggregated points', () => { @@ -111,7 +93,7 @@ describe('buildPointMatchKey', () => { num_prefill_gpu: 1, num_decode_gpu: 7, }); - expect(buildPointMatchKey(point)).toBe('h100|fp8|8|64|disagg|1|7'); + expect(trackedConfigIdentity(point)).toBe('h100|fp8|8|64|disagg|1|7'); }); it('produces the same key as buildMatchKey for matching config and point', () => { @@ -127,13 +109,13 @@ describe('buildPointMatchKey', () => { tp: 4, conc: 128, }); - expect(buildPointMatchKey(point)).toBe(buildMatchKey(config)); + expect(trackedConfigIdentity(point)).toBe(trackedConfigIdentity(config)); }); it('produces different keys for different configs', () => { const point1 = makePoint({ hwKey: 'h100', tp: 8 }); const point2 = makePoint({ hwKey: 'h200', tp: 8 }); - expect(buildPointMatchKey(point1)).not.toBe(buildPointMatchKey(point2)); + expect(trackedConfigIdentity(point1)).not.toBe(trackedConfigIdentity(point2)); }); }); @@ -305,24 +287,55 @@ describe('match key consistency between config and point', () => { num_prefill_gpu: 4, num_decode_gpu: 68, }); - expect(buildPointMatchKey(point)).toBe(buildMatchKey(config)); + expect(trackedConfigIdentity(point)).toBe(trackedConfigIdentity(config)); }); it('non-disagg config does not match disagg point', () => { const config = makeConfig({ disagg: false }); const point = makePoint({ disagg: true, num_prefill_gpu: 2, num_decode_gpu: 6 }); - expect(buildPointMatchKey(point)).not.toBe(buildMatchKey(config)); + expect(trackedConfigIdentity(point)).not.toBe(trackedConfigIdentity(config)); }); it('different concurrency values produce different keys', () => { const config1 = makeConfig({ conc: 64 }); const config2 = makeConfig({ conc: 128 }); - expect(buildMatchKey(config1)).not.toBe(buildMatchKey(config2)); + expect(trackedConfigIdentity(config1)).not.toBe(trackedConfigIdentity(config2)); }); it('different TP values produce different keys', () => { const config1 = makeConfig({ tp: 4 }); const config2 = makeConfig({ tp: 8 }); - expect(buildMatchKey(config1)).not.toBe(buildMatchKey(config2)); + expect(trackedConfigIdentity(config1)).not.toBe(trackedConfigIdentity(config2)); + }); + + it('keeps agentic MTP and standard-decoding configs distinct', () => { + const standard = makeConfig({ + benchmark_type: 'agentic_traces', + spec_decoding: 'none', + }); + const mtp = makeConfig({ + benchmark_type: 'agentic_traces', + spec_decoding: 'mtp', + }); + + expect(trackedConfigIdentity(standard)).not.toBe(trackedConfigIdentity(mtp)); + }); + + it('matches an agentic tracked config only to the same point-level decode method', () => { + const config = makeConfig({ + benchmark_type: 'agentic_traces', + spec_decoding: 'mtp', + }); + const mtpPoint = makePoint({ + benchmark_type: 'agentic_traces', + spec_decoding: 'mtp', + }); + const standardPoint = makePoint({ + benchmark_type: 'agentic_traces', + spec_decoding: 'none', + }); + + expect(trackedConfigIdentity(mtpPoint)).toBe(trackedConfigIdentity(config)); + expect(trackedConfigIdentity(standardPoint)).not.toBe(trackedConfigIdentity(config)); }); }); diff --git a/packages/app/src/components/inference/hooks/useTrendData.ts b/packages/app/src/components/inference/hooks/useTrendData.ts index 5e89014fa..07488702c 100644 --- a/packages/app/src/components/inference/hooks/useTrendData.ts +++ b/packages/app/src/components/inference/hooks/useTrendData.ts @@ -14,26 +14,12 @@ import { transformBenchmarkRows } from '@/lib/benchmark-transform'; import type { Model, Sequence } from '@/lib/data-mappings'; import { computeInputCostFields, computeOutputCostFields } from '@/lib/utils'; +import { trackedConfigIdentity } from '../utils/point-identity'; + function computeAllCostFields(data: InferenceData[]): InferenceData[] { return computeInputCostFields(computeOutputCostFields(data)); } -function buildMatchKey(config: TrackedConfig): string { - let key = `${config.hwKey}|${config.precision}|${config.tp}|${config.conc}`; - if (config.disagg) { - key += `|disagg|${config.num_prefill_gpu ?? 0}|${config.num_decode_gpu ?? 0}`; - } - return key; -} - -function buildPointMatchKey(point: InferenceData): string { - let key = `${point.hwKey}|${point.precision}|${point.tp}|${point.conc}`; - if (point.disagg) { - key += `|disagg|${point.num_prefill_gpu ?? 0}|${point.num_decode_gpu ?? 0}`; - } - return key; -} - interface UseTrendDataResult { trendLines: Map; loading: boolean; @@ -78,12 +64,6 @@ export function useTrendData( rowsByDate.get(row.date)!.push(row); } - // Build match keys for configs - const configMatchKeys = new Map(); - for (const config of trackedConfigs) { - configMatchKeys.set(buildMatchKey(config), config); - } - // Accumulate trend data per config per date const accumulator = new Map>(); @@ -98,13 +78,13 @@ export function useTrendData( // Build lookup by match key const pointsByKey = new Map(); for (const point of processed) { - const key = buildPointMatchKey(point); + const key = trackedConfigIdentity(point); if (!pointsByKey.has(key)) pointsByKey.set(key, point); } // Match tracked configs for (const config of trackedConfigs.filter((c) => c.chartType === chartType)) { - const matchKey = buildMatchKey(config); + const matchKey = trackedConfigIdentity(config); const point = pointsByKey.get(matchKey); if (!point) continue; diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index 22a208cdb..db9cfed82 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -888,6 +888,9 @@ export interface TrackedConfig { disagg?: boolean; num_prefill_gpu?: number; num_decode_gpu?: number; + /** Scenario and point-level decode method used to match mixed agentic curves exactly. */ + benchmark_type?: string; + spec_decoding?: string; } /** diff --git a/packages/app/src/components/inference/utils/point-identity.test.ts b/packages/app/src/components/inference/utils/point-identity.test.ts index 5f3ee14ae..fac5e159c 100644 --- a/packages/app/src/components/inference/utils/point-identity.test.ts +++ b/packages/app/src/components/inference/utils/point-identity.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import type { InferenceData } from '@/components/inference/types'; -import { scatterPointConfigId } from './point-identity'; +import { scatterPointConfigId, trackedConfigIdentity } from './point-identity'; const point = (overrides: Partial): InferenceData => ({ @@ -46,3 +46,29 @@ describe('scatterPointConfigId', () => { expect(off).not.toBe(on); }); }); + +describe('trackedConfigIdentity', () => { + it('keeps agentic MTP and standard-decoding trend selections distinct', () => { + const standard = trackedConfigIdentity( + point({ benchmark_type: 'agentic_traces', spec_decoding: 'none' }), + ); + const mtp = trackedConfigIdentity( + point({ benchmark_type: 'agentic_traces', spec_decoding: 'mtp' }), + ); + + expect(standard).not.toBe(mtp); + expect(standard).toContain('|spec-none'); + expect(mtp).toContain('|spec-mtp'); + }); + + it('keeps fixed-sequence identity behavior unchanged', () => { + const standard = trackedConfigIdentity( + point({ benchmark_type: 'single_turn', spec_decoding: 'none' }), + ); + const mtp = trackedConfigIdentity( + point({ benchmark_type: 'single_turn', spec_decoding: 'mtp' }), + ); + + expect(standard).toBe(mtp); + }); +}); diff --git a/packages/app/src/components/inference/utils/point-identity.ts b/packages/app/src/components/inference/utils/point-identity.ts index dd897796c..5303bbf8a 100644 --- a/packages/app/src/components/inference/utils/point-identity.ts +++ b/packages/app/src/components/inference/utils/point-identity.ts @@ -1,4 +1,14 @@ -import type { InferenceData } from '@/components/inference/types'; +import type { InferenceData, TrackedConfig } from '@/components/inference/types'; + +interface AgenticSpecIdentity { + benchmark_type?: string; + spec_decoding?: string; +} + +/** Point-level identity suffix for decode methods merged into one agentic curve. */ +export function agenticSpecDecodingKeySuffix(point: AgenticSpecIdentity): string { + return point.benchmark_type === 'agentic_traces' ? `|spec-${point.spec_decoding || 'none'}` : ''; +} /** Stable D3 join key for one scatter point within a chart series. */ export function scatterPointConfigId(point: InferenceData): string { @@ -9,8 +19,15 @@ export function scatterPointConfigId(point: InferenceData): string { if (point.offload_mode) key += `|offload-${point.offload_mode}`; // Agentic series omit spec decoding from hwKey so one curve can mix methods. // It remains point identity to avoid collapsing overlapping MTP/STP results. - if (point.benchmark_type === 'agentic_traces') { - key += `|spec-${point.spec_decoding ?? 'none'}`; - } + key += agenticSpecDecodingKeySuffix(point); return key; } + +/** Stable identity for a point selected in the performance-over-time view. */ +export function trackedConfigIdentity(point: InferenceData | TrackedConfig): string { + let key = `${point.hwKey}|${point.precision}|${point.tp}|${point.conc}`; + if (point.disagg) { + key += `|disagg|${point.num_prefill_gpu ?? 0}|${point.num_decode_gpu ?? 0}`; + } + return key + agenticSpecDecodingKeySuffix(point); +} From 4bf754c4c3d210efa817934f9b4374929118a872 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 7 Aug 2026 17:14:47 -0500 Subject: [PATCH 03/12] fix(inference): complete mixed agentic curves Address independent review findings across run selection, history, overview, tracking, replay, labels, and changelog matching while preserving fixed-sequence behavior. --- .../api/v1/benchmarks/history/route.test.ts | 42 +++++++++- .../app/api/v1/benchmarks/history/route.ts | 31 +++++-- .../app/src/app/api/v1/workflow-info/route.ts | 2 +- .../inference/hooks/useChartData.test.ts | 82 +++++++++++++++++++ .../inference/hooks/useChartData.ts | 51 +++--------- .../hooks/useInterpolatedTrendData.ts | 6 +- .../inference/hooks/useTrendData.ts | 8 +- .../inference/replay/ReplayPanel.tsx | 8 +- .../__tests__/buildReplayTimeline.test.ts | 27 ++++++ .../inference/replay/buildReplayTimeline.ts | 12 +-- .../ui/ScatterGraph.decoration.test.tsx | 22 ++++- .../components/inference/ui/ScatterGraph.tsx | 28 ++++--- .../utils/changelogFormatters.test.ts | 13 ++- .../inference/utils/changelogFormatters.tsx | 3 +- packages/app/src/hooks/api/use-ai-chart.ts | 12 ++- .../src/hooks/api/use-benchmark-history.ts | 13 ++- packages/app/src/lib/api.ts | 5 ++ .../app/src/lib/benchmark-run-selection.ts | 76 +++++++++++++++++ .../app/src/lib/default-precisions.test.ts | 18 ++++ packages/app/src/lib/default-precisions.ts | 9 +- packages/app/src/lib/overview-data.test.ts | 27 ++++++ packages/app/src/lib/overview-data.ts | 41 ++++++++-- packages/app/src/lib/overview-links.test.ts | 11 +++ packages/app/src/lib/overview-links.ts | 3 +- packages/db/src/queries/benchmarks.ts | 23 +++++- 25 files changed, 474 insertions(+), 99 deletions(-) create mode 100644 packages/app/src/lib/benchmark-run-selection.ts diff --git a/packages/app/src/app/api/v1/benchmarks/history/route.test.ts b/packages/app/src/app/api/v1/benchmarks/history/route.test.ts index e16549347..bfb5f2cf8 100644 --- a/packages/app/src/app/api/v1/benchmarks/history/route.test.ts +++ b/packages/app/src/app/api/v1/benchmarks/history/route.test.ts @@ -78,7 +78,39 @@ describe('GET /api/v1/benchmarks/history', () => { expect(res.status).toBe(200); const body = await res.json(); expect(body).toEqual(mockRows); - expect(mockGetAllBenchmarksForHistory).toHaveBeenCalledWith('mock-sql', ['dsr1'], 1024, 1024); + expect(mockGetAllBenchmarksForHistory).toHaveBeenCalledWith( + 'mock-sql', + ['dsr1'], + 1024, + 1024, + undefined, + ); + }); + + it('returns agentic history without numeric sequence lengths', async () => { + const mockRows = [{ date: '2026-03-01', benchmark_type: 'agentic_traces' }]; + mockGetAllBenchmarksForHistory.mockResolvedValueOnce(mockRows); + + const res = await GET( + req('/api/v1/benchmarks/history?model=DeepSeek-R1-0528&benchmarkType=agentic_traces'), + ); + expect(res.status).toBe(200); + expect(await res.json()).toEqual(mockRows); + expect(mockGetAllBenchmarksForHistory).toHaveBeenCalledWith( + 'mock-sql', + ['dsr1'], + null, + null, + 'agentic_traces', + ); + }); + + it('rejects unsupported benchmark types', async () => { + const res = await GET( + req('/api/v1/benchmarks/history?model=DeepSeek-R1-0528&benchmarkType=single_turn'), + ); + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: 'Unsupported benchmarkType' }); }); it('returns 500 when query throws', async () => { @@ -101,6 +133,12 @@ describe('GET /api/v1/benchmarks/history', () => { expect(res.status).toBe(200); const body = await res.json(); expect(body).toEqual([]); - expect(mockGetAllBenchmarksForHistory).toHaveBeenCalledWith('mock-sql', ['dsr1'], 1024, 8192); + expect(mockGetAllBenchmarksForHistory).toHaveBeenCalledWith( + 'mock-sql', + ['dsr1'], + 1024, + 8192, + undefined, + ); }); }); diff --git a/packages/app/src/app/api/v1/benchmarks/history/route.ts b/packages/app/src/app/api/v1/benchmarks/history/route.ts index 29f2442b3..611096c7c 100644 --- a/packages/app/src/app/api/v1/benchmarks/history/route.ts +++ b/packages/app/src/app/api/v1/benchmarks/history/route.ts @@ -11,18 +11,28 @@ import { loadFixture } from '@/lib/test-fixtures'; export const dynamic = 'force-dynamic'; const getCachedBenchmarkHistory = cachedQuery( - (modelKeys: string[], isl: number, osl: number) => - getAllBenchmarksForHistory(getDb(), modelKeys, isl, osl), - 'benchmark-history', + (modelKeys: string[], isl: number | null, osl: number | null, benchmarkType?: string) => + getAllBenchmarksForHistory(getDb(), modelKeys, isl, osl, benchmarkType), + 'benchmark-history-v2', { blobOnly: true }, ); export async function GET(request: NextRequest) { const model = request.nextUrl.searchParams.get('model') ?? ''; - const isl = Number(request.nextUrl.searchParams.get('isl')); - const osl = Number(request.nextUrl.searchParams.get('osl')); - - if (!model || !isl || !osl) { + const rawIsl = request.nextUrl.searchParams.get('isl'); + const rawOsl = request.nextUrl.searchParams.get('osl'); + const benchmarkType = request.nextUrl.searchParams.get('benchmarkType') ?? undefined; + const isl = rawIsl === null ? null : Number(rawIsl); + const osl = rawOsl === null ? null : Number(rawOsl); + const isAgentic = benchmarkType === 'agentic_traces'; + + if (!model) { + return NextResponse.json({ error: 'model, isl, and osl are required' }, { status: 400 }); + } + if (benchmarkType !== undefined && !isAgentic) { + return NextResponse.json({ error: 'Unsupported benchmarkType' }, { status: 400 }); + } + if (!isAgentic && (!isl || !osl)) { return NextResponse.json({ error: 'model, isl, and osl are required' }, { status: 400 }); } if (FIXTURES_MODE) return cachedJson(loadFixture('benchmarks-history')); @@ -32,7 +42,12 @@ export async function GET(request: NextRequest) { if (!modelKeys || modelKeys.length === 0) { return NextResponse.json({ error: 'Unknown model' }, { status: 400 }); } - const rows = await getCachedBenchmarkHistory(modelKeys, isl, osl); + const rows = await getCachedBenchmarkHistory( + modelKeys, + isAgentic ? null : isl, + isAgentic ? null : osl, + benchmarkType, + ); return cachedJson(rows); } catch (error) { console.error('Error fetching benchmark history:', error); diff --git a/packages/app/src/app/api/v1/workflow-info/route.ts b/packages/app/src/app/api/v1/workflow-info/route.ts index 4c5c12bc2..63b330c3b 100644 --- a/packages/app/src/app/api/v1/workflow-info/route.ts +++ b/packages/app/src/app/api/v1/workflow-info/route.ts @@ -23,7 +23,7 @@ const getCachedWorkflowInfo = cachedQuery(async (date: string) => { getRunConfigsByDate(sql, date), ]); return { runs, changelogs, configs, runConfigs }; -}, 'workflow-info'); +}, 'workflow-info-v2'); export async function GET(request: NextRequest) { const date = request.nextUrl.searchParams.get('date') ?? ''; diff --git a/packages/app/src/components/inference/hooks/useChartData.test.ts b/packages/app/src/components/inference/hooks/useChartData.test.ts index 4a20413ea..bc9ead42f 100644 --- a/packages/app/src/components/inference/hooks/useChartData.test.ts +++ b/packages/app/src/components/inference/hooks/useChartData.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from 'vitest'; import chartDefinitions from '@/components/inference/inference-chart-config.json'; +import { dedupeAgenticHistoryRuns } from '@/lib/benchmark-run-selection'; import { applyAgenticPercentileToXLabel, @@ -19,7 +20,10 @@ interface DedupeInput { disagg: boolean; precision: string; offload_mode?: string | null; + benchmark_type?: string; date: string; + workflow_run_id?: number; + run_started_at?: string | null; } const drow = (over: Partial = {}): DedupeInput => ({ @@ -81,6 +85,84 @@ describe('dedupeRowsToLatestPerConfig', () => { ]; expect(dedupeRowsToLatestPerConfig(rows).map((r) => r.id)).toEqual([2]); }); + + it('dedupes mixed agentic spec methods as one curve', () => { + const rows = [ + drow({ id: 1, benchmark_type: 'agentic_traces', spec_method: 'none', date: '2026-06-01' }), + drow({ id: 2, benchmark_type: 'agentic_traces', spec_method: 'mtp', date: '2026-06-03' }), + drow({ id: 3, benchmark_type: 'agentic_traces', spec_method: 'eagle', date: '2026-06-03' }), + ]; + + expect(dedupeRowsToLatestPerConfig(rows).map((r) => r.id)).toEqual([2, 3]); + }); + + it('continues deduping fixed-sequence spec methods independently', () => { + const rows = [ + drow({ id: 1, benchmark_type: 'single_turn', spec_method: 'none', date: '2026-06-01' }), + drow({ id: 2, benchmark_type: 'single_turn', spec_method: 'mtp', date: '2026-06-03' }), + ]; + + expect(dedupeRowsToLatestPerConfig(rows).map((r) => r.id)).toEqual([1, 2]); + }); + + it('keeps mixed agentic points from only the newest same-day workflow run', () => { + const rows = [ + drow({ + id: 1, + benchmark_type: 'agentic_traces', + spec_method: 'none', + workflow_run_id: 10, + run_started_at: '2026-06-03T10:00:00Z', + }), + drow({ + id: 2, + benchmark_type: 'agentic_traces', + spec_method: 'mtp', + workflow_run_id: 10, + run_started_at: '2026-06-03T10:00:00Z', + }), + drow({ + id: 3, + benchmark_type: 'agentic_traces', + spec_method: 'mtp', + workflow_run_id: 11, + run_started_at: '2026-06-03T12:00:00Z', + }), + ]; + + expect(dedupeRowsToLatestPerConfig(rows).map((r) => r.id)).toEqual([3]); + }); +}); + +describe('dedupeAgenticHistoryRuns', () => { + it('keeps the newest agentic workflow per series on each date', () => { + const rows = [ + drow({ + id: 1, + benchmark_type: 'agentic_traces', + spec_method: 'none', + workflow_run_id: 20, + run_started_at: '2026-06-01T10:00:00Z', + }), + drow({ + id: 2, + benchmark_type: 'agentic_traces', + spec_method: 'mtp', + workflow_run_id: 21, + run_started_at: '2026-06-01T12:00:00Z', + }), + drow({ + id: 3, + benchmark_type: 'agentic_traces', + spec_method: 'none', + date: '2026-06-02', + workflow_run_id: 22, + run_started_at: '2026-06-02T10:00:00Z', + }), + ]; + + expect(dedupeAgenticHistoryRuns(rows).map((row) => row.id)).toEqual([2, 3]); + }); }); describe('buildComparisonDates', () => { diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index 18d5b62aa..1ee1d2429 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -24,6 +24,10 @@ import { hardwareKeyMatchesAnyBase, } from '@/lib/constants'; import { mergeRunScopedRows, transformBenchmarkRows } from '@/lib/benchmark-transform'; +import { + dedupeAgenticHistoryRuns, + dedupeRowsToLatestPerConfig, +} from '@/lib/benchmark-run-selection'; import { Sequence, type Model } from '@/lib/data-mappings'; import { calculateCostsForGpus, calculatePowerForGpus } from '@/lib/utils'; import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisField'; @@ -136,41 +140,7 @@ export function applyAgenticPercentileToXLabel(label: string, pctlWord: string): : `${pctlWord} ${label}`; } -/** The dedup key fields a chart series is identified by. */ -interface DedupeRow { - hardware: string; - framework: string; - spec_method: string; - disagg: boolean; - precision: string; - offload_mode?: string | null; - date: string; -} - -// offload_mode normalized `?? 'off'` to match the SQL layer's getBenchmarksForRun -// lineKey — agentic offload=on and offload=off are distinct series. -const dedupeSeriesKey = (r: DedupeRow): string => - `${r.hardware}|${r.framework}|${r.spec_method}|${r.disagg}|${r.precision}|${r.offload_mode ?? 'off'}`; - -/** - * For each series — (hardware, framework, spec_method, disagg, precision, - * offload_mode) — keep only the rows from that series' most recent date. When - * parallelism settings change between runs, old config_ids create stale points - * under the same legend line; dropping all-but-latest removes them. - * - * Without `offload_mode` in the key, an offload=on sweep ingested on a LATER date - * than the offload=off sweep would win the shared group and silently drop the - * (earlier-dated) offload=off variant — a data-loss regression. - */ -export function dedupeRowsToLatestPerConfig(rows: T[]): T[] { - const maxDatePerGroup = new Map(); - for (const r of rows) { - const k = dedupeSeriesKey(r); - const cur = maxDatePerGroup.get(k); - if (!cur || r.date > cur) maxDatePerGroup.set(k, r.date); - } - return rows.filter((r) => r.date === maxDatePerGroup.get(dedupeSeriesKey(r))); -} +export { dedupeRowsToLatestPerConfig }; export function useChartData( selectedModel: Model, @@ -301,11 +271,12 @@ export function useChartData( selectedRunDate ? { ...r, date: selectedRunDate, actualDate: r.date } : r, ); if (comparisonDates.length === 0) return mainRows; - const extraRows = comparisonQueries.flatMap((q, i) => - (q.data ?? []) - .filter(seqFilter) - .map((r) => ({ ...r, date: comparisonDates[i], actualDate: r.date })), - ); + const extraRows = comparisonQueries.flatMap((q, i) => { + const filtered = (q.data ?? []).filter(seqFilter); + const selected = + selectedSequence === Sequence.AgenticTraces ? dedupeAgenticHistoryRuns(filtered) : filtered; + return selected.map((r) => ({ ...r, date: comparisonDates[i], actualDate: r.date })); + }); return [...mainRows, ...extraRows]; }, [allRows, selectedSequence, comparisonDates, comparisonDataKey, selectedRunDate]); diff --git a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts index cde86370d..e71b4e9f9 100644 --- a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts +++ b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts @@ -13,7 +13,8 @@ import { getHardwareKey } from '@/lib/chart-utils'; import { getGpuSpecs, isKnownGpu } from '@/lib/constants'; import { rowToAggDataEntry } from '@/lib/benchmark-transform'; import type { BenchmarkRow } from '@/lib/api'; -import type { Model, Sequence } from '@/lib/data-mappings'; +import { dedupeAgenticHistoryRuns } from '@/lib/benchmark-run-selection'; +import { Sequence, type Model } from '@/lib/data-mappings'; // Trend points never sit on a roofline — they're synthetic per-(date, config) // aggregates, not the per-load Pareto-frontier points the chart marks. Hardcode @@ -195,6 +196,7 @@ export function useInterpolatedTrendData({ enabled ? selectedModel : '', seqIslOsl?.isl ?? 0, seqIslOsl?.osl ?? 0, + selectedSequence === Sequence.AgenticTraces ? 'agentic_traces' : undefined, ); // Build lightweight InferenceData points grouped by date and hwKey. @@ -204,7 +206,7 @@ export function useInterpolatedTrendData({ const result = new Map>(); - for (const row of allRows) { + for (const row of dedupeAgenticHistoryRuns(allRows)) { if (!selectedPrecisions.includes(row.precision)) continue; const point = rowToLightweightPoint(row); diff --git a/packages/app/src/components/inference/hooks/useTrendData.ts b/packages/app/src/components/inference/hooks/useTrendData.ts index 07488702c..089b0af63 100644 --- a/packages/app/src/components/inference/hooks/useTrendData.ts +++ b/packages/app/src/components/inference/hooks/useTrendData.ts @@ -11,7 +11,8 @@ import type { } from '@/components/inference/types'; import { useBenchmarkHistory } from '@/hooks/api/use-benchmark-history'; import { transformBenchmarkRows } from '@/lib/benchmark-transform'; -import type { Model, Sequence } from '@/lib/data-mappings'; +import { dedupeAgenticHistoryRuns } from '@/lib/benchmark-run-selection'; +import { Sequence as SequenceValue, type Model, type Sequence } from '@/lib/data-mappings'; import { computeInputCostFields, computeOutputCostFields } from '@/lib/utils'; import { trackedConfigIdentity } from '../utils/point-identity'; @@ -43,6 +44,7 @@ export function useTrendData( trackedConfigs.length > 0 ? selectedModel : '', seqIslOsl?.isl ?? 0, seqIslOsl?.osl ?? 0, + selectedSequence === SequenceValue.AgenticTraces ? 'agentic_traces' : undefined, ); const trendLines = useMemo(() => { @@ -57,9 +59,11 @@ export function useTrendData( // Chart type to index: interactivity = 0, e2e = 1 const chartTypeIndex: Record = { interactivity: 0, e2e: 1 }; + const selectedRows = dedupeAgenticHistoryRuns(allRows); + // Group rows by date const rowsByDate = new Map(); - for (const row of allRows) { + for (const row of selectedRows) { if (!rowsByDate.has(row.date)) rowsByDate.set(row.date, []); rowsByDate.get(row.date)!.push(row); } diff --git a/packages/app/src/components/inference/replay/ReplayPanel.tsx b/packages/app/src/components/inference/replay/ReplayPanel.tsx index 7273ed24c..3d36bca0f 100644 --- a/packages/app/src/components/inference/replay/ReplayPanel.tsx +++ b/packages/app/src/components/inference/replay/ReplayPanel.tsx @@ -21,6 +21,7 @@ import { } from '@/components/ui/select'; import { useBenchmarkHistory } from '@/hooks/api/use-benchmark-history'; import { track } from '@/lib/analytics'; +import { Sequence } from '@/lib/data-mappings'; import { cn } from '@/lib/utils'; import { buildReplayTimeline, computeFullRunDomain } from './buildReplayTimeline'; @@ -68,7 +69,12 @@ export default function ReplayPanel({ const { selectedModel, selectedSequence, activeHwTypes } = inference; const { isl = 0, osl = 0 } = sequenceToIslOsl(selectedSequence) ?? {}; - const history = useBenchmarkHistory(selectedModel, isl, osl); + const history = useBenchmarkHistory( + selectedModel, + isl, + osl, + selectedSequence === Sequence.AgenticTraces ? 'agentic_traces' : undefined, + ); const effectiveX = chartDefinition.chartType === 'e2e' diff --git a/packages/app/src/components/inference/replay/__tests__/buildReplayTimeline.test.ts b/packages/app/src/components/inference/replay/__tests__/buildReplayTimeline.test.ts index 6e8025e66..e46d15812 100644 --- a/packages/app/src/components/inference/replay/__tests__/buildReplayTimeline.test.ts +++ b/packages/app/src/components/inference/replay/__tests__/buildReplayTimeline.test.ts @@ -216,6 +216,33 @@ describe('buildReplayTimeline', () => { expect(t.configs.length).toBeGreaterThanOrEqual(2); }); + it('keeps overlapping agentic MTP and standard-decoding replay points distinct', () => { + const rows = [ + baseRow({ + benchmark_type: 'agentic_traces', + spec_method: 'none', + isl: null, + osl: null, + metrics: { tput_per_gpu: 1000, median_itl: 0.02 }, + }), + baseRow({ + benchmark_type: 'agentic_traces', + spec_method: 'mtp', + isl: null, + osl: null, + metrics: { tput_per_gpu: 1200, median_itl: 1 / 55 }, + }), + ]; + + const t = buildReplayTimeline(rows, interactivityChartDef, 'y_tpPerGpu', null, ['fp4']); + expect(t.configs).toHaveLength(2); + expect(new Set(t.configs.map((config) => config.hwKey))).toEqual(new Set(['h100_trt'])); + expect(t.configs.map((config) => config.configId).toSorted()).toEqual([ + 'h100_trt|fp4|0|32|0|0|0|spec-mtp', + 'h100_trt|fp4|0|32|0|0|0|spec-none', + ]); + }); + it('computes a global x/y domain spanning all observations', () => { const rows = [ baseRow({ date: '2025-01-01', metrics: { tput_per_gpu: 100, median_intvty: 10 } }), diff --git a/packages/app/src/components/inference/replay/buildReplayTimeline.ts b/packages/app/src/components/inference/replay/buildReplayTimeline.ts index 917616046..258c38fb4 100644 --- a/packages/app/src/components/inference/replay/buildReplayTimeline.ts +++ b/packages/app/src/components/inference/replay/buildReplayTimeline.ts @@ -8,6 +8,8 @@ import type { InferenceData, YAxisMetricKey, } from '@/components/inference/types'; +import { scatterPointConfigId } from '@/components/inference/utils/point-identity'; +import { dedupeAgenticHistoryRuns } from '@/lib/benchmark-run-selection'; import type { PerStepValue } from './interpolateAtTime'; @@ -81,12 +83,6 @@ export function computeFullRunDomain( return { x: safeDomain(xMin, xMax), y: safeDomain(yMin, yMax) }; } -const buildPointConfigId = (point: InferenceData): string => { - let key = `${point.hwKey}|${point.precision}|${point.tp}|${point.conc}|${point.decode_ep ?? 0}|${point.prefill_tp ?? 0}|${point.prefill_ep ?? 0}`; - if (point.disagg) key += `|disagg|${point.num_prefill_gpu ?? 0}|${point.num_decode_gpu ?? 0}`; - return key; -}; - const safeDomain = (lo: number, hi: number): [number, number] => { if (!Number.isFinite(lo) || !Number.isFinite(hi)) return [0, 1]; if (lo === hi) { @@ -155,7 +151,7 @@ export function buildReplayTimeline( let yMax = -Infinity; const dateSet = new Set(); - for (const row of rows) { + for (const row of dedupeAgenticHistoryRuns(rows)) { if (!selectedPrecisions.includes(row.precision)) continue; const entry = rowToAggDataEntry(row); @@ -181,7 +177,7 @@ export function buildReplayTimeline( if (xVal <= 0 || yMetric <= 0) continue; const finalPoint: InferenceData = { ...point, x: xVal, y: yMetric }; - const configId = buildPointConfigId(finalPoint); + const configId = scatterPointConfigId(finalPoint); const dateMs = Date.parse(`${row.date}T00:00:00Z`); if (Number.isNaN(dateMs)) continue; diff --git a/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx b/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx index eff46f6b4..2a8b64ca6 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx @@ -41,7 +41,7 @@ vi.mock('@/hooks/api/use-trace-availability', () => ({ useTraceAvailability: () => ({ data: undefined }), })); -import ScatterGraph from './ScatterGraph'; +import ScatterGraph, { pointLabelText } from './ScatterGraph'; // ── Environment stubs ──────────────────────────────────────────────────────── class MockResizeObserver { @@ -73,6 +73,26 @@ const CHART_DEFINITION = { chartType: 'interactivity' } as unknown as ChartDefin const noop = () => {}; +describe('pointLabelText', () => { + it('labels mixed agentic points with their point-level decode mode', () => { + const standard = point('h100', 'fp8', 1, 1, 8); + standard.benchmark_type = 'agentic_traces'; + standard.spec_decoding = 'none'; + const mtp = { ...standard, spec_decoding: 'mtp' }; + + expect(pointLabelText(standard, false)).toBe('8\nC=16\nSTP'); + expect(pointLabelText(mtp, false)).toBe('8\nC=16\nMTP'); + }); + + it('keeps fixed-sequence labels unchanged', () => { + const fixed = point('h100', 'fp8', 1, 1, 8); + fixed.benchmark_type = 'single_turn'; + fixed.spec_decoding = 'mtp'; + + expect(pointLabelText(fixed, false)).toBe('8\nC=16'); + }); +}); + function baseInferenceState() { return { activeHwTypes: new Set(['h100', 'b200']), diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index 89382a779..de84dc13f 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -70,7 +70,10 @@ import { generateTooltipContent, getPointLabel, } from '@/components/inference/utils/tooltipUtils'; -import { scatterPointConfigId } from '@/components/inference/utils/point-identity'; +import { + scatterPointConfigId, + trackedConfigIdentity, +} from '@/components/inference/utils/point-identity'; import LegendPointsDialog from '@/components/inference/ui/LegendPointsDialog'; import { OFFLOAD_HALO_DASHARRAY, @@ -246,9 +249,13 @@ function groupPointsByDate(points: InferenceData[]): Map `${d.hwKey}_${d.precision}_${d.date}-${d.x}-${d.y}`; -/** Point label lines: TP (or full parallelism label) plus the C= concurrency. */ -const pointLabelText = (d: InferenceData, advanced: boolean): string => - advanced ? `${getPointLabel(d)}\nC=${d.conc}` : `${d.tp}\nC=${d.conc}`; +/** Point label lines: TP (or full parallelism label), concurrency, then agentic decode mode. */ +export const pointLabelText = (d: InferenceData, advanced: boolean): string => { + const base = advanced ? `${getPointLabel(d)}\nC=${d.conc}` : `${d.tp}\nC=${d.conc}`; + if (d.benchmark_type !== 'agentic_traces') return base; + const specMethod = d.spec_decoding ?? 'none'; + return `${base}\n${specMethod === 'none' || specMethod === '' ? 'STP' : specMethod.toUpperCase()}`; +}; // Referentially stable "no overlay data" result (see processedOverlayData). const EMPTY_OVERLAY_DATA: InferenceData[] = []; @@ -821,6 +828,7 @@ const ScatterGraph = React.memo( }, [trackedConfigs]); const buildPointConfigId = useCallback(scatterPointConfigId, []); + const buildTrackedConfigId = useCallback(trackedConfigIdentity, []); // filteredData: visible points only (for scale domain calculation) const filteredData = useMemo( @@ -1423,7 +1431,7 @@ const ScatterGraph = React.memo( yLabel, selectedYAxisMetric, hardwareConfig, - isTracked: trackedConfigIdsRef.current.has(buildPointConfigId(d)), + isTracked: trackedConfigIdsRef.current.has(buildTrackedConfigId(d)), runUrl: d.run_url ? updateRepoUrl(d.run_url) : undefined, hasTrace: typeof d.id === 'number' ? traceAvailability?.[d.id] === true : false, locale, @@ -1450,7 +1458,7 @@ const ScatterGraph = React.memo( if (trackBtn) { trackBtn.addEventListener('click', (btnEvent) => { btnEvent.stopPropagation(); - const configId = buildPointConfigId(d); + const configId = buildTrackedConfigId(d); if (trackedConfigIdsRef.current.has(configId)) removeTrackedConfig(configId); else addTrackedConfig(d, chartDefinition.chartType); chartRef.current?.dismissTooltip(); @@ -1484,7 +1492,7 @@ const ScatterGraph = React.memo( yLabel, selectedYAxisMetric, hardwareConfig, - buildPointConfigId, + buildTrackedConfigId, addTrackedConfig, removeTrackedConfig, chartDefinition.chartType, @@ -2895,7 +2903,7 @@ const ScatterGraph = React.memo( // Tracked ring highlights zoomGroup.selectAll('.dot-group').each(function (d) { - const isTracked = trackedConfigIdsRef.current.has(buildPointConfigId(d)); + const isTracked = trackedConfigIdsRef.current.has(buildTrackedConfigId(d)); d3.select(this) .selectAll('.tracked-ring') .data(isTracked ? [true] : []) @@ -2932,7 +2940,7 @@ const ScatterGraph = React.memo( .on('dblclick', function (event, d) { event.stopPropagation(); event.preventDefault(); - const configId = buildPointConfigId(d); + const configId = buildTrackedConfigId(d); const wasTracked = trackedConfigIdsRef.current.has(configId); if (wasTracked) removeTrackedConfig(configId); else addTrackedConfig(d, chartDefinition.chartType); @@ -2977,7 +2985,7 @@ const ScatterGraph = React.memo( } }, [ - buildPointConfigId, + buildTrackedConfigId, hardwareConfig, addTrackedConfig, removeTrackedConfig, diff --git a/packages/app/src/components/inference/utils/changelogFormatters.test.ts b/packages/app/src/components/inference/utils/changelogFormatters.test.ts index b657e97e2..3a9e09a42 100644 --- a/packages/app/src/components/inference/utils/changelogFormatters.test.ts +++ b/packages/app/src/components/inference/utils/changelogFormatters.test.ts @@ -62,9 +62,9 @@ describe('changelogConfigToHwKey', () => { ); }); - it('keeps a trailing MTP spec method while dropping agentic metadata', () => { + it('drops a trailing MTP spec method from agentic legend identity', () => { expect(changelogConfigToHwKey('dsv4-fp4-mi355x-sglang-agentic-hicache-mtp')).toBe( - 'mi355x_sglang_mtp', + 'mi355x_sglang', ); }); }); @@ -98,6 +98,15 @@ describe('configKeyMatchesHwKey', () => { ).toBe(true); }); + it('matches an agentic MTP changelog key to the mixed-spec legend key', () => { + expect( + configKeyMatchesHwKey( + 'dsv4-fp4-mi355x-mori-sglang-agentic-hicache-mtp', + 'mi355x_mori-sglang', + ), + ).toBe(true); + }); + it('matches sglang framework', () => { expect(configKeyMatchesHwKey('gptoss-fp8-mi300x-sglang', 'mi300x_sglang')).toBe(true); }); diff --git a/packages/app/src/components/inference/utils/changelogFormatters.tsx b/packages/app/src/components/inference/utils/changelogFormatters.tsx index 476ed2211..6dd4d9c7b 100644 --- a/packages/app/src/components/inference/utils/changelogFormatters.tsx +++ b/packages/app/src/components/inference/utils/changelogFormatters.tsx @@ -32,7 +32,8 @@ export function changelogConfigToHwKey(configKey: string): string | null { if (!framework) return null; const trailingParts = remainder.slice(framework.length).split('-').filter(Boolean); - const specSuffix = trailingParts.includes('mtp') ? '_mtp' : ''; + const isAgentic = trailingParts.includes('agentic'); + const specSuffix = !isAgentic && trailingParts.includes('mtp') ? '_mtp' : ''; return `${gpu}_${resolveFrameworkAlias(framework)}${specSuffix}`; } diff --git a/packages/app/src/hooks/api/use-ai-chart.ts b/packages/app/src/hooks/api/use-ai-chart.ts index 55d2597ce..4d18fa1de 100644 --- a/packages/app/src/hooks/api/use-ai-chart.ts +++ b/packages/app/src/hooks/api/use-ai-chart.ts @@ -22,6 +22,7 @@ import { type ReliabilityRow, } from '@/lib/api'; import { transformBenchmarkRows } from '@/lib/benchmark-transform'; +import { dedupeAgenticHistoryRuns } from '@/lib/benchmark-run-selection'; import { getNestedYValue, normalizeEvalHardwareKey, @@ -391,11 +392,16 @@ async function resolveSpec(spec: AiChartSpec): Promise { } // Benchmarks or History - const { isl, osl } = sequenceToIslOsl(spec.sequence); - const rows = + const isAgentic = spec.sequence === 'agentic-traces'; + const { isl, osl } = isAgentic ? { isl: 0, osl: 0 } : sequenceToIslOsl(spec.sequence); + const fetchedRows = spec.dataSource === 'history' - ? await fetchBenchmarkHistory(spec.model, isl, osl) + ? await fetchBenchmarkHistory(spec.model, isl, osl, isAgentic ? 'agentic_traces' : undefined) : await fetchBenchmarks(spec.model); + const rows = + spec.dataSource === 'history' && isAgentic + ? dedupeAgenticHistoryRuns(fetchedRows) + : fetchedRows; const { chartData } = transformBenchmarkRows(rows); let points = chartData[0] ?? []; diff --git a/packages/app/src/hooks/api/use-benchmark-history.ts b/packages/app/src/hooks/api/use-benchmark-history.ts index 4d4378172..a65b926f7 100644 --- a/packages/app/src/hooks/api/use-benchmark-history.ts +++ b/packages/app/src/hooks/api/use-benchmark-history.ts @@ -2,10 +2,15 @@ import { useQuery } from '@tanstack/react-query'; import { fetchBenchmarkHistory } from '@/lib/api'; -export function useBenchmarkHistory(model: string, isl: number, osl: number) { +export function useBenchmarkHistory( + model: string, + isl: number, + osl: number, + benchmarkType?: 'agentic_traces', +) { return useQuery({ - queryKey: ['benchmark-history', model, isl, osl], - queryFn: ({ signal }) => fetchBenchmarkHistory(model, isl, osl, signal), - enabled: Boolean(model && isl && osl), + queryKey: ['benchmark-history', model, isl, osl, benchmarkType], + queryFn: ({ signal }) => fetchBenchmarkHistory(model, isl, osl, benchmarkType, signal), + enabled: Boolean(model && (benchmarkType === 'agentic_traces' || (isl && osl))), }); } diff --git a/packages/app/src/lib/api.ts b/packages/app/src/lib/api.ts index c7d9c0016..73a790cbb 100644 --- a/packages/app/src/lib/api.ts +++ b/packages/app/src/lib/api.ts @@ -52,6 +52,9 @@ export interface BenchmarkRow { */ workers?: WorkerPower[]; date: string; + /** Internal workflow identity used to keep merged agentic curves within one run. */ + workflow_run_id?: number; + run_started_at?: string | null; run_url: string | null; } @@ -174,9 +177,11 @@ export function fetchBenchmarkHistory( model: string, isl: number, osl: number, + benchmarkType?: 'agentic_traces', signal?: AbortSignal, ) { const params = new URLSearchParams({ model, isl: String(isl), osl: String(osl) }); + if (benchmarkType) params.set('benchmarkType', benchmarkType); return fetchJson(`/api/v1/benchmarks/history?${params}`, signal); } diff --git a/packages/app/src/lib/benchmark-run-selection.ts b/packages/app/src/lib/benchmark-run-selection.ts new file mode 100644 index 000000000..f7332b96c --- /dev/null +++ b/packages/app/src/lib/benchmark-run-selection.ts @@ -0,0 +1,76 @@ +/** Fields needed to select one workflow run for a rendered benchmark series. */ +export interface BenchmarkSeriesRow { + hardware: string; + framework: string; + spec_method: string; + disagg: boolean; + precision: string; + offload_mode?: string | null; + benchmark_type?: string; + date: string; + workflow_run_id?: number; + run_started_at?: string | null; +} + +const seriesKey = (row: BenchmarkSeriesRow): string => { + const specMethod = row.benchmark_type === 'agentic_traces' ? '' : row.spec_method; + return `${row.hardware}|${row.framework}|${specMethod}|${row.disagg}|${row.precision}|${row.offload_mode ?? 'off'}`; +}; + +function isLaterRun(candidate: BenchmarkSeriesRow, current: BenchmarkSeriesRow): boolean { + const startedAt = candidate.run_started_at ?? ''; + const currentStartedAt = current.run_started_at ?? ''; + return ( + startedAt > currentStartedAt || + (startedAt === currentStartedAt && + (candidate.workflow_run_id ?? Number.NEGATIVE_INFINITY) > + (current.workflow_run_id ?? Number.NEGATIVE_INFINITY)) + ); +} + +function isWinningRun(row: BenchmarkSeriesRow, winner: BenchmarkSeriesRow): boolean { + return ( + row.run_started_at === winner.run_started_at && row.workflow_run_id === winner.workflow_run_id + ); +} + +/** Keep only the newest date for each chart series and, for agentic, one workflow run. */ +export function dedupeRowsToLatestPerConfig(rows: T[]): T[] { + const winnerPerGroup = new Map(); + for (const row of rows) { + const key = seriesKey(row); + const current = winnerPerGroup.get(key); + if (!current || row.date > current.date) { + winnerPerGroup.set(key, row); + continue; + } + if ( + row.date === current.date && + row.benchmark_type === 'agentic_traces' && + isLaterRun(row, current) + ) { + winnerPerGroup.set(key, row); + } + } + return rows.filter((row) => { + const winner = winnerPerGroup.get(seriesKey(row)); + if (!winner || row.date !== winner.date) return false; + return row.benchmark_type !== 'agentic_traces' || isWinningRun(row, winner); + }); +} + +/** For historical views, keep one agentic workflow run per series on each calendar date. */ +export function dedupeAgenticHistoryRuns(rows: T[]): T[] { + const winnerPerDateAndSeries = new Map(); + for (const row of rows) { + if (row.benchmark_type !== 'agentic_traces') continue; + const key = `${row.date}|${seriesKey(row)}`; + const current = winnerPerDateAndSeries.get(key); + if (!current || isLaterRun(row, current)) winnerPerDateAndSeries.set(key, row); + } + return rows.filter((row) => { + if (row.benchmark_type !== 'agentic_traces') return true; + const winner = winnerPerDateAndSeries.get(`${row.date}|${seriesKey(row)}`); + return winner !== undefined && isWinningRun(row, winner); + }); +} diff --git a/packages/app/src/lib/default-precisions.test.ts b/packages/app/src/lib/default-precisions.test.ts index 9d8a72cac..0f1140a41 100644 --- a/packages/app/src/lib/default-precisions.test.ts +++ b/packages/app/src/lib/default-precisions.test.ts @@ -33,6 +33,24 @@ describe('countCurvesByPrecision', () => { it('returns {} for no rows', () => { expect(countCurvesByPrecision([])).toEqual({}); }); + + it('counts mixed agentic spec methods as one rendered curve', () => { + const base = { ...row('fp8', 'b200'), benchmark_type: 'agentic_traces' }; + const counts = countCurvesByPrecision([ + base, + { ...base, spec_method: 'mtp' }, + { ...base, spec_method: 'eagle' }, + ]); + + expect(counts).toEqual({ fp8: 1 }); + }); + + it('continues counting fixed-sequence spec methods as separate curves', () => { + const base = { ...row('fp8', 'b200'), benchmark_type: 'single_turn' }; + const counts = countCurvesByPrecision([base, { ...base, spec_method: 'mtp' }]); + + expect(counts).toEqual({ fp8: 2 }); + }); }); describe('pickDefaultPrecisions', () => { diff --git a/packages/app/src/lib/default-precisions.ts b/packages/app/src/lib/default-precisions.ts index 71c493868..85fd3d2fe 100644 --- a/packages/app/src/lib/default-precisions.ts +++ b/packages/app/src/lib/default-precisions.ts @@ -8,8 +8,9 @@ import { Precision, PRECISION_OPTIONS } from './data-mappings'; * showing a near-empty chart on first load. Instead we default to whichever * precision has the most data, with a guard against committing to a sparse one. * - * "Curves" = distinct (hardware, framework, spec_method, disagg) series that - * would render for a precision — i.e. how many lines the chart draws. + * "Curves" = distinct chart series that would render for a precision. Fixed- + * sequence series include `spec_method`; agentic series do not because one line + * can contain points from multiple speculative-decoding methods. */ /** @@ -26,6 +27,7 @@ interface CurveRow { framework: string; spec_method: string; disagg: boolean; + benchmark_type?: string; } /** Count distinct curves per precision from already-filtered rows (model + sequence). */ @@ -37,7 +39,8 @@ export function countCurvesByPrecision(rows: CurveRow[]): Record curves = new Set(); seen.set(r.precision, curves); } - curves.add(`${r.hardware}|${r.framework}|${r.spec_method}|${r.disagg}`); + const specMethod = r.benchmark_type === 'agentic_traces' ? '' : r.spec_method; + curves.add(`${r.hardware}|${r.framework}|${specMethod}|${r.disagg}`); } const counts: Record = {}; for (const [precision, curves] of seen) counts[precision] = curves.size; diff --git a/packages/app/src/lib/overview-data.test.ts b/packages/app/src/lib/overview-data.test.ts index 8f934ad03..2bcde42dd 100644 --- a/packages/app/src/lib/overview-data.test.ts +++ b/packages/app/src/lib/overview-data.test.ts @@ -1090,6 +1090,33 @@ describe('overview platform selection', () => { ); }); + it('builds one AgentX frontier from mixed standard and MTP points', () => { + const summary = buildOverviewModelSummary(Model.GLM_5_2, [ + agenticRow(40, 30, 12600, 1200, { + hardware: 'b200', + conc: 8, + spec_method: 'none', + }), + agenticRow(50, 25, 10800, 850, { + hardware: 'b200', + conc: 12, + spec_method: 'mtp', + }), + agenticRow(60, 20, 9000, 800, { + hardware: 'b200', + conc: 16, + spec_method: 'none', + }), + ]); + + const config = summary.platforms.find(({ hardware }) => hardware === 'b200')?.read.config; + expect(config).toMatchObject({ + specMethod: 'mixed', + specLabel: 'STP + MTP', + hwKey: 'b200_sglang', + }); + }); + it('restricts AgentX points to the E2E frontier on total throughput', () => { // The slower-E2E point wins on output tokens but loses on total tokens, so // the total-token frontier drops it and the tier read becomes unreachable. diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts index bd789e840..cc074e3be 100644 --- a/packages/app/src/lib/overview-data.ts +++ b/packages/app/src/lib/overview-data.ts @@ -74,7 +74,7 @@ export interface OverviewTierValue { } /** One chart-equivalent serving series. Topology and GPU-count variants may - * contribute points, while release/framework/spec/precision/deployment stay exact. */ + * contribute points; agentic series may also mix point-level decode methods. */ export interface OverviewConfigResult { key: string; dbModel: string; @@ -292,11 +292,12 @@ function overviewScenarioRows( } function overviewServingSeriesKey(row: BenchmarkRow): string { + const specMethod = row.benchmark_type === 'agentic_traces' ? '' : row.spec_method; return JSON.stringify([ row.model, row.hardware, row.framework, - row.spec_method, + specMethod, row.precision, row.disagg, row.is_multinode, @@ -325,7 +326,19 @@ function buildConfigs( (latest, row) => (row.date > latest ? row.date : latest), configRows[0].date, ); - const latestRows = configRows.filter((row) => row.date === latestDate); + let latestRows = configRows.filter((row) => row.date === latestDate); + if (scenario === 'agentx' && latestRows.some((row) => row.workflow_run_id !== undefined)) { + const winningRow = latestRows.reduce((winner, row) => { + const startedAt = row.run_started_at ?? ''; + const winnerStartedAt = winner.run_started_at ?? ''; + if (startedAt !== winnerStartedAt) return startedAt > winnerStartedAt ? row : winner; + return (row.workflow_run_id ?? Number.NEGATIVE_INFINITY) > + (winner.workflow_run_id ?? Number.NEGATIVE_INFINITY) + ? row + : winner; + }); + latestRows = latestRows.filter((row) => row.workflow_run_id === winningRow.workflow_run_id); + } const config = buildConfigResult(model, scenario, latestRows[0].precision, key, latestRows); if (config) configs.push(config); } @@ -593,7 +606,17 @@ function buildConfigResult( if (feed.length === 0) return null; const first = rows[0]; - const { hardware, framework, spec_method: specMethod, disagg, is_multinode: isMultinode } = first; + const { hardware, framework, disagg, is_multinode: isMultinode } = first; + const specMethods = [...new Set(rows.map((row) => row.spec_method))]; + const specMethod = specMethods.length === 1 ? specMethods[0] : 'mixed'; + const specLabel = + specMethod === 'mixed' + ? specMethods + .map((method) => + method === 'none' || method === '' ? 'STP' : resolveFrameworkPartLabel(model, method), + ) + .join(' + ') + : resolveFrameworkPartLabel(model, specMethod); const sourceRunUrls = [ ...new Set(rows.flatMap((row) => (row.run_url === null ? [] : [row.run_url]))), ].toSorted(); @@ -601,11 +624,17 @@ function buildConfigResult( key, dbModel: first.model, hardware, - hwKey: buildAvailabilityHwKey(hardware, framework, specMethod, disagg), + hwKey: buildAvailabilityHwKey( + hardware, + framework, + specMethod, + disagg, + scenario === 'agentx' ? 'agentic_traces' : 'single_turn', + ), framework, frameworkLabel: resolveFrameworkPartLabel(model, framework), specMethod, - specLabel: resolveFrameworkPartLabel(model, specMethod), + specLabel, disagg, isMultinode, precision, diff --git a/packages/app/src/lib/overview-links.test.ts b/packages/app/src/lib/overview-links.test.ts index cd70f5dd5..8639c4f38 100644 --- a/packages/app/src/lib/overview-links.test.ts +++ b/packages/app/src/lib/overview-links.test.ts @@ -133,6 +133,17 @@ describe('buildOverviewDashboardHref', () => { 'i_spec=mtp', ); }); + + it('does not filter an AgentX mixed-spec curve to only one decode method', () => { + const href = buildOverviewDashboardHref( + 'en', + summary({ scenario: 'agentx' }), + config({ specMethod: 'mixed', hwKey: 'b200_sglang' }), + ); + + expect(href).toContain('i_seq=agentic-traces'); + expect(href).not.toContain('i_spec='); + }); }); describe('detailHref', () => { diff --git a/packages/app/src/lib/overview-links.ts b/packages/app/src/lib/overview-links.ts index 959bef622..1175cc7d7 100644 --- a/packages/app/src/lib/overview-links.ts +++ b/packages/app/src/lib/overview-links.ts @@ -62,7 +62,8 @@ function inferenceRoute(locale: 'en' | 'zh'): string { * (mirrors `pointSpecMode` in quickFilters.ts, minus its hwKey suffix check — * overview `specMethod` comes straight from `spec_method`). */ -function dashboardSpecMode(specMethod: string): 'mtp' | 'stp' { +function dashboardSpecMode(specMethod: string): 'mtp' | 'stp' | undefined { + if (specMethod === 'mixed') return undefined; return specMethod === 'none' || specMethod === '' ? 'stp' : 'mtp'; } diff --git a/packages/db/src/queries/benchmarks.ts b/packages/db/src/queries/benchmarks.ts index 87c5efbd2..d7989e4cc 100644 --- a/packages/db/src/queries/benchmarks.ts +++ b/packages/db/src/queries/benchmarks.ts @@ -45,6 +45,9 @@ export interface BenchmarkRow { */ workers?: BenchmarkWorkerRow[]; date: string; + /** Internal workflow identity used to keep merged agentic curves within one run. */ + workflow_run_id?: number; + run_started_at?: string | null; run_url: string | null; } @@ -147,6 +150,8 @@ export async function getLatestBenchmarks( br.metrics, br.workers, br.date::text, + br.workflow_run_id, + wr.run_started_at::text, CASE WHEN wr.html_url IS NOT NULL THEN wr.html_url || '/attempts/' || wr.run_attempt ELSE NULL END AS run_url FROM benchmark_results br JOIN configs c ON c.id = br.config_id @@ -194,6 +199,8 @@ export async function getLatestBenchmarks( lb.metrics, lb.workers, lb.date::text, + lb.workflow_run_id, + wr.run_started_at::text, CASE WHEN wr.html_url IS NOT NULL THEN wr.html_url || '/attempts/' || wr.run_attempt ELSE NULL END AS run_url FROM latest_benchmarks lb JOIN configs c ON c.id = lb.config_id @@ -246,6 +253,8 @@ export async function getBenchmarksForRun( br.metrics, br.workers, br.date::text, + br.workflow_run_id, + wr.run_started_at::text, CASE WHEN wr.html_url IS NOT NULL THEN wr.html_url || '/attempts/' || wr.run_attempt ELSE NULL END AS run_url FROM benchmark_results br JOIN configs c ON c.id = br.config_id @@ -266,10 +275,15 @@ export async function getBenchmarksForRun( export async function getAllBenchmarksForHistory( sql: DbClient, modelKey: string | string[], - isl: number, - osl: number, + isl: number | null, + osl: number | null, + benchmarkType?: string, ): Promise { const modelKeys = Array.isArray(modelKey) ? modelKey : [modelKey]; + const sequenceFilter = + benchmarkType === 'agentic_traces' + ? sql`br.benchmark_type = 'agentic_traces'` + : sql`br.isl = ${isl} AND br.osl = ${osl}`; const rows = await sql` SELECT br.id, @@ -299,11 +313,12 @@ export async function getAllBenchmarksForHistory( br.metrics - '{std_ttft,std_tpot,std_e2el,std_intvty,std_itl,mean_ttft,mean_tpot,mean_e2el,mean_intvty,mean_itl}'::text[] as metrics, br.workers, br.date::text, + br.workflow_run_id, + wr.run_started_at::text, CASE WHEN wr.html_url IS NOT NULL THEN wr.html_url || '/attempts/' || wr.run_attempt ELSE NULL END AS run_url FROM configs c JOIN benchmark_results br ON br.config_id = c.id - AND br.isl = ${isl} - AND br.osl = ${osl} + AND ${sequenceFilter} AND br.error IS NULL JOIN latest_workflow_runs wr ON wr.id = br.workflow_run_id WHERE c.model = ANY(${modelKeys}) From ee605cb1d440a2c0d57db8952260c1897ca8dd41 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 7 Aug 2026 17:20:32 -0500 Subject: [PATCH 04/12] fix(ai-chart): select one agentic workflow run Apply the existing agentic latest-run selector to live AI chart benchmark data so mixed decode curves cannot combine workflows. --- packages/app/src/hooks/api/use-ai-chart.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/app/src/hooks/api/use-ai-chart.ts b/packages/app/src/hooks/api/use-ai-chart.ts index 4d18fa1de..b62fb02b8 100644 --- a/packages/app/src/hooks/api/use-ai-chart.ts +++ b/packages/app/src/hooks/api/use-ai-chart.ts @@ -22,7 +22,10 @@ import { type ReliabilityRow, } from '@/lib/api'; import { transformBenchmarkRows } from '@/lib/benchmark-transform'; -import { dedupeAgenticHistoryRuns } from '@/lib/benchmark-run-selection'; +import { + dedupeAgenticHistoryRuns, + dedupeRowsToLatestPerConfig, +} from '@/lib/benchmark-run-selection'; import { getNestedYValue, normalizeEvalHardwareKey, @@ -401,7 +404,9 @@ async function resolveSpec(spec: AiChartSpec): Promise { const rows = spec.dataSource === 'history' && isAgentic ? dedupeAgenticHistoryRuns(fetchedRows) - : fetchedRows; + : isAgentic + ? dedupeRowsToLatestPerConfig(fetchedRows) + : fetchedRows; const { chartData } = transformBenchmarkRows(rows); let points = chartData[0] ?? []; From 1dd7957e1039e0531a92be008af54f1079622afe Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 12:06:04 -0500 Subject: [PATCH 05/12] fix: isolate mixed agentic decode identity Preserve fixed-sequence chart, replay, changelog, cache-key, and API response semantics while merging speculative decode methods only for agentic scenarios. --- .../api/v1/benchmarks/history/route.test.ts | 27 +++++-------- .../app/api/v1/benchmarks/history/route.ts | 27 ++++++------- .../src/app/api/v1/benchmarks/route.test.ts | 35 +++++++++++++++++ .../app/src/app/api/v1/benchmarks/route.ts | 9 +++-- .../app/api/v1/workflow-info/route.test.ts | 17 +++++++++ .../app/src/app/api/v1/workflow-info/route.ts | 25 ++++++++++-- .../__tests__/buildReplayTimeline.test.ts | 12 ++++++ .../inference/replay/buildReplayTimeline.ts | 12 +++++- .../components/inference/ui/ChartDisplay.tsx | 38 ++++++++++++++----- .../inference/ui/ComparisonChangelog.tsx | 29 +++++++++++--- .../components/inference/ui/ScatterGraph.tsx | 9 ++++- .../utils/changelogFormatters.test.ts | 13 +++++-- .../inference/utils/changelogFormatters.tsx | 21 ++++++++-- .../inference/utils/runEnumeration.test.ts | 8 ++-- .../inference/utils/runEnumeration.ts | 10 +++-- packages/app/src/hooks/api/use-ai-chart.ts | 8 +++- .../src/hooks/api/use-benchmark-history.ts | 6 ++- .../hooks/api/use-comparison-changelogs.ts | 6 ++- .../src/lib/agentic-workflow-metadata.test.ts | 30 +++++++++++++++ .../app/src/lib/agentic-workflow-metadata.ts | 14 +++++++ packages/app/src/lib/api-documentation.ts | 22 ++++++++--- packages/app/src/lib/api-route-catalog.ts | 10 ++--- packages/app/src/lib/api.ts | 16 ++++---- packages/app/src/lib/benchmark-data.server.ts | 4 +- packages/db/src/queries/workflow-info.ts | 12 ++++-- 25 files changed, 318 insertions(+), 102 deletions(-) create mode 100644 packages/app/src/lib/agentic-workflow-metadata.test.ts create mode 100644 packages/app/src/lib/agentic-workflow-metadata.ts diff --git a/packages/app/src/app/api/v1/benchmarks/history/route.test.ts b/packages/app/src/app/api/v1/benchmarks/history/route.test.ts index bfb5f2cf8..cbb8affe5 100644 --- a/packages/app/src/app/api/v1/benchmarks/history/route.test.ts +++ b/packages/app/src/app/api/v1/benchmarks/history/route.test.ts @@ -78,13 +78,7 @@ describe('GET /api/v1/benchmarks/history', () => { expect(res.status).toBe(200); const body = await res.json(); expect(body).toEqual(mockRows); - expect(mockGetAllBenchmarksForHistory).toHaveBeenCalledWith( - 'mock-sql', - ['dsr1'], - 1024, - 1024, - undefined, - ); + expect(mockGetAllBenchmarksForHistory).toHaveBeenCalledWith('mock-sql', ['dsr1'], 1024, 1024); }); it('returns agentic history without numeric sequence lengths', async () => { @@ -105,12 +99,15 @@ describe('GET /api/v1/benchmarks/history', () => { ); }); - it('rejects unsupported benchmark types', async () => { + it('preserves fixed-sequence behavior when benchmarkType=single_turn is supplied', async () => { + mockGetAllBenchmarksForHistory.mockResolvedValueOnce([]); const res = await GET( - req('/api/v1/benchmarks/history?model=DeepSeek-R1-0528&benchmarkType=single_turn'), + req( + '/api/v1/benchmarks/history?model=DeepSeek-R1-0528&isl=1024&osl=1024&benchmarkType=single_turn', + ), ); - expect(res.status).toBe(400); - expect(await res.json()).toEqual({ error: 'Unsupported benchmarkType' }); + expect(res.status).toBe(200); + expect(mockGetAllBenchmarksForHistory).toHaveBeenCalledWith('mock-sql', ['dsr1'], 1024, 1024); }); it('returns 500 when query throws', async () => { @@ -133,12 +130,6 @@ describe('GET /api/v1/benchmarks/history', () => { expect(res.status).toBe(200); const body = await res.json(); expect(body).toEqual([]); - expect(mockGetAllBenchmarksForHistory).toHaveBeenCalledWith( - 'mock-sql', - ['dsr1'], - 1024, - 8192, - undefined, - ); + expect(mockGetAllBenchmarksForHistory).toHaveBeenCalledWith('mock-sql', ['dsr1'], 1024, 8192); }); }); diff --git a/packages/app/src/app/api/v1/benchmarks/history/route.ts b/packages/app/src/app/api/v1/benchmarks/history/route.ts index 611096c7c..113a6ed65 100644 --- a/packages/app/src/app/api/v1/benchmarks/history/route.ts +++ b/packages/app/src/app/api/v1/benchmarks/history/route.ts @@ -7,13 +7,20 @@ import { getAllBenchmarksForHistory } from '@semianalysisai/inferencex-db/querie import { cachedJson, cachedQuery } from '@/lib/api-cache'; import { loadFixture } from '@/lib/test-fixtures'; +import { agenticWorkflowMetadataOnly } from '@/lib/agentic-workflow-metadata'; export const dynamic = 'force-dynamic'; const getCachedBenchmarkHistory = cachedQuery( - (modelKeys: string[], isl: number | null, osl: number | null, benchmarkType?: string) => - getAllBenchmarksForHistory(getDb(), modelKeys, isl, osl, benchmarkType), - 'benchmark-history-v2', + (modelKeys: string[], isl: number, osl: number) => + getAllBenchmarksForHistory(getDb(), modelKeys, isl, osl), + 'benchmark-history', + { blobOnly: true }, +); +const getCachedAgenticBenchmarkHistory = cachedQuery( + (modelKeys: string[]) => + getAllBenchmarksForHistory(getDb(), modelKeys, null, null, 'agentic_traces'), + 'benchmark-history-agentic', { blobOnly: true }, ); @@ -29,9 +36,6 @@ export async function GET(request: NextRequest) { if (!model) { return NextResponse.json({ error: 'model, isl, and osl are required' }, { status: 400 }); } - if (benchmarkType !== undefined && !isAgentic) { - return NextResponse.json({ error: 'Unsupported benchmarkType' }, { status: 400 }); - } if (!isAgentic && (!isl || !osl)) { return NextResponse.json({ error: 'model, isl, and osl are required' }, { status: 400 }); } @@ -42,13 +46,10 @@ export async function GET(request: NextRequest) { if (!modelKeys || modelKeys.length === 0) { return NextResponse.json({ error: 'Unknown model' }, { status: 400 }); } - const rows = await getCachedBenchmarkHistory( - modelKeys, - isAgentic ? null : isl, - isAgentic ? null : osl, - benchmarkType, - ); - return cachedJson(rows); + const rows = isAgentic + ? await getCachedAgenticBenchmarkHistory(modelKeys) + : await getCachedBenchmarkHistory(modelKeys, isl!, osl!); + return cachedJson(agenticWorkflowMetadataOnly(rows)); } catch (error) { console.error('Error fetching benchmark history:', error); return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); diff --git a/packages/app/src/app/api/v1/benchmarks/route.test.ts b/packages/app/src/app/api/v1/benchmarks/route.test.ts index a1809efda..629339016 100644 --- a/packages/app/src/app/api/v1/benchmarks/route.test.ts +++ b/packages/app/src/app/api/v1/benchmarks/route.test.ts @@ -64,6 +64,41 @@ describe('GET /api/v1/benchmarks', () => { ); }); + it('does not expose agentic run-selection metadata on fixed-sequence rows', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce([ + { + id: 1, + benchmark_type: 'single_turn', + workflow_run_id: 42, + run_started_at: '2026-08-12T10:00:00Z', + }, + ]); + + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528')); + expect(await res.json()).toEqual([{ id: 1, benchmark_type: 'single_turn' }]); + }); + + it('keeps run-selection metadata on agentic rows', async () => { + mockGetLatestBenchmarks.mockResolvedValueOnce([ + { + id: 1, + benchmark_type: 'agentic_traces', + workflow_run_id: 42, + run_started_at: '2026-08-12T10:00:00Z', + }, + ]); + + const res = await GET(req('/api/v1/benchmarks?model=DeepSeek-R1-0528')); + expect(await res.json()).toEqual([ + { + id: 1, + benchmark_type: 'agentic_traces', + workflow_run_id: 42, + run_started_at: '2026-08-12T10:00:00Z', + }, + ]); + }); + it('passes date param to query when provided', async () => { mockGetLatestBenchmarks.mockResolvedValueOnce([]); diff --git a/packages/app/src/app/api/v1/benchmarks/route.ts b/packages/app/src/app/api/v1/benchmarks/route.ts index ce151eccf..84ef529f8 100644 --- a/packages/app/src/app/api/v1/benchmarks/route.ts +++ b/packages/app/src/app/api/v1/benchmarks/route.ts @@ -11,6 +11,7 @@ import { import { cachedJson, cachedQuery } from '@/lib/api-cache'; import { toCalculatorBenchmarkRows } from '@/lib/benchmark-api-view'; +import { agenticWorkflowMetadataOnly } from '@/lib/agentic-workflow-metadata'; import { loadFixture } from '@/lib/test-fixtures'; export const dynamic = 'force-dynamic'; @@ -18,7 +19,7 @@ export const dynamic = 'force-dynamic'; const getCachedBenchmarks = cachedQuery( (dbModelKeys: string[], date?: string, exact?: boolean, runId?: string) => getLatestBenchmarks(getDb(), dbModelKeys, date, exact, runId), - 'benchmarks', + 'benchmarks-agentic-run-metadata', { blobOnly: true }, ); @@ -26,14 +27,14 @@ const getCachedBenchmarks = cachedQuery( // under a distinct key prefix so it never collides with the latest/as-of query. const getCachedBenchmarksForRun = cachedQuery( (dbModelKeys: string[], runId: string) => getBenchmarksForRun(getDb(), dbModelKeys, runId), - 'benchmarks-run', + 'benchmarks-run-agentic-run-metadata', { blobOnly: true }, ); const getCachedCalculatorBenchmarks = cachedQuery( async (dbModelKeys: string[], sequence: string, date?: string) => toCalculatorBenchmarkRows(await getLatestBenchmarks(getDb(), dbModelKeys, date), sequence), - 'benchmarks-calculator', + 'benchmarks-calculator-agentic-run-metadata', { blobOnly: true }, ); @@ -70,7 +71,7 @@ export async function GET(request: NextRequest) { : exactRun && runId ? await getCachedBenchmarksForRun(dbModelKeys, runId) : await getCachedBenchmarks(dbModelKeys, date, exact || undefined, runId); - return cachedJson(rows); + return cachedJson(agenticWorkflowMetadataOnly(rows)); } catch (error) { console.error('Error fetching benchmarks:', error); return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); diff --git a/packages/app/src/app/api/v1/workflow-info/route.test.ts b/packages/app/src/app/api/v1/workflow-info/route.test.ts index 32652b01b..e9e07faf0 100644 --- a/packages/app/src/app/api/v1/workflow-info/route.test.ts +++ b/packages/app/src/app/api/v1/workflow-info/route.test.ts @@ -89,6 +89,23 @@ describe('GET /api/v1/workflow-info', () => { expect(mockGetRunConfigsByDate).toHaveBeenCalledWith('mock-sql', '2026-03-01'); }); + it('scopes run coverage only when agentic is explicitly requested', async () => { + mockGetWorkflowRunsByDate.mockResolvedValueOnce([]); + mockGetChangelogByDate.mockResolvedValueOnce([]); + mockGetDateConfigs.mockResolvedValueOnce([]); + mockGetRunConfigsByDate.mockResolvedValueOnce([]); + + const res = await GET( + req('/api/v1/workflow-info?date=2026-03-01&benchmarkType=agentic_traces'), + ); + expect(res.status).toBe(200); + expect(mockGetRunConfigsByDate).toHaveBeenCalledWith( + 'mock-sql', + '2026-03-01', + 'agentic_traces', + ); + }); + it('accepts empty date param (returns all)', async () => { mockGetWorkflowRunsByDate.mockResolvedValueOnce([]); mockGetChangelogByDate.mockResolvedValueOnce([]); diff --git a/packages/app/src/app/api/v1/workflow-info/route.ts b/packages/app/src/app/api/v1/workflow-info/route.ts index 63b330c3b..d8d646e97 100644 --- a/packages/app/src/app/api/v1/workflow-info/route.ts +++ b/packages/app/src/app/api/v1/workflow-info/route.ts @@ -14,19 +14,34 @@ import { loadFixture } from '@/lib/test-fixtures'; export const dynamic = 'force-dynamic'; -const getCachedWorkflowInfo = cachedQuery(async (date: string) => { +async function loadWorkflowInfo(date: string, benchmarkType?: 'agentic_traces') { const sql = getDb(); const [runs, changelogs, configs, runConfigs] = await Promise.all([ getWorkflowRunsByDate(sql, date), getChangelogByDate(sql, date), getDateConfigs(sql, date), - getRunConfigsByDate(sql, date), + benchmarkType ? getRunConfigsByDate(sql, date, benchmarkType) : getRunConfigsByDate(sql, date), ]); return { runs, changelogs, configs, runConfigs }; -}, 'workflow-info-v2'); +} + +// Preserve the established public cache and response for calls without a +// scenario. Scenario-scoped calls use a separate cache namespace. +const getCachedWorkflowInfo = cachedQuery( + (date: string) => loadWorkflowInfo(date), + 'workflow-info', +); +const getCachedScenarioWorkflowInfo = cachedQuery( + (date: string) => loadWorkflowInfo(date, 'agentic_traces'), + 'workflow-info-scenario', +); export async function GET(request: NextRequest) { const date = request.nextUrl.searchParams.get('date') ?? ''; + const benchmarkType = + request.nextUrl.searchParams.get('benchmarkType') === 'agentic_traces' + ? 'agentic_traces' + : undefined; if (date && !/^\d{4}-\d{2}-\d{2}$/u.test(date)) { return NextResponse.json( { error: 'Invalid date format (YYYY-MM-DD required)' }, @@ -36,7 +51,9 @@ export async function GET(request: NextRequest) { if (FIXTURES_MODE) return cachedJson(loadFixture('workflow-info')); try { - const data = await getCachedWorkflowInfo(date); + const data = benchmarkType + ? await getCachedScenarioWorkflowInfo(date) + : await getCachedWorkflowInfo(date); return cachedJson(data); } catch (error) { console.error('Error fetching workflow info:', error); diff --git a/packages/app/src/components/inference/replay/__tests__/buildReplayTimeline.test.ts b/packages/app/src/components/inference/replay/__tests__/buildReplayTimeline.test.ts index e46d15812..6dd9aac94 100644 --- a/packages/app/src/components/inference/replay/__tests__/buildReplayTimeline.test.ts +++ b/packages/app/src/components/inference/replay/__tests__/buildReplayTimeline.test.ts @@ -243,6 +243,18 @@ describe('buildReplayTimeline', () => { ]); }); + it('preserves fixed-sequence replay identity when offload metadata differs', () => { + const rows = [ + baseRow({ benchmark_type: 'single_turn', offload_mode: 'off' }), + baseRow({ benchmark_type: 'single_turn', offload_mode: 'on' }), + ]; + + const t = buildReplayTimeline(rows, interactivityChartDef, 'y_tpPerGpu', null, ['fp4']); + expect(t.configs).toHaveLength(1); + expect(t.configs[0].configId).not.toContain('offload'); + expect(t.configs[0].configId).not.toContain('spec-'); + }); + it('computes a global x/y domain spanning all observations', () => { const rows = [ baseRow({ date: '2025-01-01', metrics: { tput_per_gpu: 100, median_intvty: 10 } }), diff --git a/packages/app/src/components/inference/replay/buildReplayTimeline.ts b/packages/app/src/components/inference/replay/buildReplayTimeline.ts index 258c38fb4..717b12840 100644 --- a/packages/app/src/components/inference/replay/buildReplayTimeline.ts +++ b/packages/app/src/components/inference/replay/buildReplayTimeline.ts @@ -8,7 +8,7 @@ import type { InferenceData, YAxisMetricKey, } from '@/components/inference/types'; -import { scatterPointConfigId } from '@/components/inference/utils/point-identity'; +import { agenticSpecDecodingKeySuffix } from '@/components/inference/utils/point-identity'; import { dedupeAgenticHistoryRuns } from '@/lib/benchmark-run-selection'; import type { PerStepValue } from './interpolateAtTime'; @@ -83,6 +83,14 @@ export function computeFullRunDomain( return { x: safeDomain(xMin, xMax), y: safeDomain(yMin, yMax) }; } +const buildReplayPointConfigId = (point: InferenceData): string => { + let key = `${point.hwKey}|${point.precision}|${point.tp}|${point.conc}|${point.decode_ep ?? 0}|${point.prefill_tp ?? 0}|${point.prefill_ep ?? 0}`; + if (point.disagg) key += `|disagg|${point.num_prefill_gpu ?? 0}|${point.num_decode_gpu ?? 0}`; + // Preserve the pre-existing replay identity for fixed-sequence points. Agentic + // curves need only the decode-method suffix because their hwKey now merges it. + return key + agenticSpecDecodingKeySuffix(point); +}; + const safeDomain = (lo: number, hi: number): [number, number] => { if (!Number.isFinite(lo) || !Number.isFinite(hi)) return [0, 1]; if (lo === hi) { @@ -177,7 +185,7 @@ export function buildReplayTimeline( if (xVal <= 0 || yMetric <= 0) continue; const finalPoint: InferenceData = { ...point, x: xVal, y: yMetric }; - const configId = scatterPointConfigId(finalPoint); + const configId = buildReplayPointConfigId(finalPoint); const dateMs = Date.parse(`${row.date}T00:00:00Z`); if (Number.isNaN(dateMs)) continue; diff --git a/packages/app/src/components/inference/ui/ChartDisplay.tsx b/packages/app/src/components/inference/ui/ChartDisplay.tsx index f80f823bb..85160daf2 100644 --- a/packages/app/src/components/inference/ui/ChartDisplay.tsx +++ b/packages/app/src/components/inference/ui/ChartDisplay.tsx @@ -43,7 +43,7 @@ import { useUnofficialRun } from '@/components/unofficial-run-provider'; import { type Model, type Precision, - type Sequence, + Sequence, getModelLabel, getPrecisionLabel, getSequenceLabel, @@ -218,12 +218,21 @@ export default function ChartDisplay() { setSelectedXAxisMode, quickFilters, } = useInference(); + const selectedBenchmarkType: 'single_turn' | 'agentic_traces' = + selectedSequence === Sequence.AgenticTraces ? 'agentic_traces' : 'single_turn'; + const workflowInfoBenchmarkType = + selectedSequence === Sequence.AgenticTraces ? 'agentic_traces' : undefined; const { changelogs, loading: changelogsLoading, totalDatesQueried, - } = useComparisonChangelogs(selectedGPUs, selectedDateRange, dateRangeAvailableDates); + } = useComparisonChangelogs( + selectedGPUs, + selectedDateRange, + dateRangeAvailableDates, + workflowInfoBenchmarkType, + ); const [mounted, setMounted] = useState(false); useEffect(() => setMounted(true), []); @@ -232,7 +241,6 @@ export default function ChartDisplay() { () => DISPLAY_MODEL_TO_DB[selectedModel] ?? [selectedModel], [selectedModel], ); - // Stable run numbering shared by the changelog and the chart legend: each of a // date's runs gets a fixed 1-based number (by start time) regardless of which // are on the chart, so the two surfaces always show the same #N for a run and a @@ -241,14 +249,17 @@ export default function ChartDisplay() { const runNumbering = useMemo(() => { const map = new Map(); for (const c of changelogs) { - dataRunsForDate(c.runConfigs, { modelDbKeys, selectedGPUs, selectedPrecisions }).forEach( - (run, idx) => { - map.set(makeRunComparisonEntry(c.date, run.runId), idx + 1); - }, - ); + dataRunsForDate(c.runConfigs, { + modelDbKeys, + selectedGPUs, + selectedPrecisions, + benchmarkType: selectedBenchmarkType, + }).forEach((run, idx) => { + map.set(makeRunComparisonEntry(c.date, run.runId), idx + 1); + }); } return map; - }, [changelogs, modelDbKeys, selectedGPUs, selectedPrecisions]); + }, [changelogs, modelDbKeys, selectedGPUs, selectedPrecisions, selectedBenchmarkType]); // Expand a plain-date selection into one entry per run once that date's runs are // known. Picking a date that has multiple runs shows each run as its own series @@ -257,7 +268,12 @@ export default function ChartDisplay() { // in sync. Idempotent: after expansion no expandable plain date remains. useEffect(() => { const runConfigsByDate = new Map(changelogs.map((c) => [c.date, c.runConfigs])); - const scope = { modelDbKeys, selectedGPUs, selectedPrecisions }; + const scope = { + modelDbKeys, + selectedGPUs, + selectedPrecisions, + benchmarkType: selectedBenchmarkType, + }; setSelectedDatesFromRunExpansion((prev) => { let changed = false; const out: string[] = []; @@ -283,6 +299,7 @@ export default function ChartDisplay() { modelDbKeys, selectedGPUs, selectedPrecisions, + selectedBenchmarkType, selectedDates, setSelectedDatesFromRunExpansion, ]); @@ -998,6 +1015,7 @@ export default function ChartDisplay() { selectedGPUs={selectedGPUs} selectedPrecisions={selectedPrecisions} modelDbKeys={modelDbKeys} + selectedSequence={selectedSequence} loading={changelogsLoading} totalDatesQueried={totalDatesQueried} selectedDates={selectedDates} diff --git a/packages/app/src/components/inference/ui/ComparisonChangelog.tsx b/packages/app/src/components/inference/ui/ComparisonChangelog.tsx index ae906b750..f87a8084c 100644 --- a/packages/app/src/components/inference/ui/ComparisonChangelog.tsx +++ b/packages/app/src/components/inference/ui/ComparisonChangelog.tsx @@ -16,6 +16,7 @@ import { import { makeRunComparisonEntry } from '@/components/inference/utils/comparisonEntry'; import { dataRunsForDate } from '@/components/inference/utils/runEnumeration'; import { getHardwareConfig } from '@/lib/constants'; +import { Sequence, type Sequence as SequenceType } from '@/lib/data-mappings'; import { getDisplayLabel, updateRepoUrl } from '@/lib/utils'; /** Git Commit and Workflow Run external links for a run, each shown when known. */ @@ -72,6 +73,7 @@ interface ComparisonChangelogProps { * model-scoped). */ modelDbKeys: string[]; + selectedSequence: SequenceType; loading?: boolean; totalDatesQueried: number; selectedDates: string[]; @@ -88,6 +90,7 @@ export default function ComparisonChangelog({ selectedGPUs, selectedPrecisions, modelDbKeys, + selectedSequence, loading, totalDatesQueried, selectedDates, @@ -98,6 +101,10 @@ export default function ComparisonChangelog({ firstAvailableDate, }: ComparisonChangelogProps) { const [isExpanded, setIsExpanded] = useState(true); + const benchmarkType = + selectedSequence === Sequence.AgenticTraces ? 'agentic_traces' : 'single_turn'; + const changelogBenchmarkType = + selectedSequence === Sequence.AgenticTraces ? 'agentic_traces' : undefined; // Filter changelog entries to only show those matching selected GPUs and precisions. // Always keep range endpoints and first appearance date visible. @@ -120,7 +127,7 @@ export default function ComparisonChangelog({ return ( modelDbKeys.some((m) => key.startsWith(`${m}-`)) && precSet.has(precision) && - selectedGPUs.some((gpu) => configKeyMatchesHwKey(key, gpu)) + selectedGPUs.some((gpu) => configKeyMatchesHwKey(key, gpu, changelogBenchmarkType)) ); }), ), @@ -136,7 +143,14 @@ export default function ComparisonChangelog({ return mapped .filter((item) => item.entries.length > 0 || pinnedDates.has(item.date)) .toSorted((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()); - }, [changelogs, modelDbKeys, selectedGPUs, selectedPrecisions, pinnedDates]); + }, [ + changelogs, + modelDbKeys, + selectedGPUs, + selectedPrecisions, + pinnedDates, + changelogBenchmarkType, + ]); const datesOnChart = useMemo(() => { const set = new Set(selectedDates); @@ -156,10 +170,10 @@ export default function ComparisonChangelog({ return ( modelDbKeys.some((m) => key.startsWith(`${m}-`)) && precSet.has(precision) && - selectedGPUs.some((gpu) => configKeyMatchesHwKey(key, gpu)) + selectedGPUs.some((gpu) => configKeyMatchesHwKey(key, gpu, changelogBenchmarkType)) ); }); - }, [modelDbKeys, selectedPrecisions, selectedGPUs]); + }, [modelDbKeys, selectedPrecisions, selectedGPUs, changelogBenchmarkType]); /** * Every run that produced data for the selected config on a date, earliest @@ -173,6 +187,7 @@ export default function ComparisonChangelog({ modelDbKeys, selectedGPUs, selectedPrecisions, + benchmarkType, }).map((run) => { const cl = clByRun.get(run.runId); return { @@ -183,7 +198,7 @@ export default function ComparisonChangelog({ }; }); }, - [modelDbKeys, selectedGPUs, selectedPrecisions, entryMatchesSelection], + [modelDbKeys, selectedGPUs, selectedPrecisions, benchmarkType, entryMatchesSelection], ); // Entries the "Add all to chart" button would add: every run not yet on the @@ -219,7 +234,9 @@ export default function ComparisonChangelog({ if (selectedGPUs.length <= 1) return ''; return selectedGPUs .filter((gpu) => - entries.some((e) => e.config_keys.some((k) => configKeyMatchesHwKey(k, gpu))), + entries.some((e) => + e.config_keys.some((k) => configKeyMatchesHwKey(k, gpu, changelogBenchmarkType)), + ), ) .map((gpu) => getDisplayLabel(getHardwareConfig(gpu, displayModel))) .join(', '); diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index 3a49da459..a604887eb 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -750,14 +750,19 @@ const ScatterGraph = React.memo( const hwKeys = cl.entries.flatMap((entry: any) => (entry.config_keys ?? entry['config-keys'] ?? []) .filter((key: string) => selectedPrecisions.includes(key.split('-')[1])) - .map(changelogConfigToHwKey) + .map((key: string) => + changelogConfigToHwKey( + key, + selectedSequence === Sequence.AgenticTraces ? 'agentic_traces' : undefined, + ), + ) .filter((key: string | null): key is string => key !== null), ); return new Set(hwKeys); } } return new Set(); - }, [availableRuns, selectedRunId, selectedPrecisions]); + }, [availableRuns, selectedRunId, selectedPrecisions, selectedSequence]); // --- Data Processing --- const groupedData = useMemo( diff --git a/packages/app/src/components/inference/utils/changelogFormatters.test.ts b/packages/app/src/components/inference/utils/changelogFormatters.test.ts index 3a9e09a42..a1e80866c 100644 --- a/packages/app/src/components/inference/utils/changelogFormatters.test.ts +++ b/packages/app/src/components/inference/utils/changelogFormatters.test.ts @@ -63,9 +63,9 @@ describe('changelogConfigToHwKey', () => { }); it('drops a trailing MTP spec method from agentic legend identity', () => { - expect(changelogConfigToHwKey('dsv4-fp4-mi355x-sglang-agentic-hicache-mtp')).toBe( - 'mi355x_sglang', - ); + expect( + changelogConfigToHwKey('dsv4-fp4-mi355x-sglang-agentic-hicache-mtp', 'agentic_traces'), + ).toBe('mi355x_sglang'); }); }); @@ -103,10 +103,17 @@ describe('configKeyMatchesHwKey', () => { configKeyMatchesHwKey( 'dsv4-fp4-mi355x-mori-sglang-agentic-hicache-mtp', 'mi355x_mori-sglang', + 'agentic_traces', ), ).toBe(true); }); + it('keeps fixed-sequence changelogs out of agentic views', () => { + expect( + configKeyMatchesHwKey('dsv4-fp4-mi355x-mori-sglang', 'mi355x_mori-sglang', 'agentic_traces'), + ).toBe(false); + }); + it('matches sglang framework', () => { expect(configKeyMatchesHwKey('gptoss-fp8-mi300x-sglang', 'mi300x_sglang')).toBe(true); }); diff --git a/packages/app/src/components/inference/utils/changelogFormatters.tsx b/packages/app/src/components/inference/utils/changelogFormatters.tsx index 6dd4d9c7b..17a0f2038 100644 --- a/packages/app/src/components/inference/utils/changelogFormatters.tsx +++ b/packages/app/src/components/inference/utils/changelogFormatters.tsx @@ -20,7 +20,10 @@ const CHANGELOG_FRAMEWORK_KEYS = [ * `agentic`, `hicache`, and `pcp` after the serving framework; those are not * framework labels and must not become part of the legend identity. */ -export function changelogConfigToHwKey(configKey: string): string | null { +export function changelogConfigToHwKey( + configKey: string, + benchmarkType?: 'single_turn' | 'agentic_traces', +): string | null { const parts = configKey.toLowerCase().split('-'); const gpu = parts[2]; const remainder = parts.slice(3).join('-'); @@ -33,7 +36,13 @@ export function changelogConfigToHwKey(configKey: string): string | null { const trailingParts = remainder.slice(framework.length).split('-').filter(Boolean); const isAgentic = trailingParts.includes('agentic'); - const specSuffix = !isAgentic && trailingParts.includes('mtp') ? '_mtp' : ''; + if (benchmarkType === 'agentic_traces' && !isAgentic) return null; + const specSuffix = + benchmarkType === 'agentic_traces' && isAgentic + ? '' + : trailingParts.includes('mtp') + ? '_mtp' + : ''; return `${gpu}_${resolveFrameworkAlias(framework)}${specSuffix}`; } @@ -63,8 +72,12 @@ export function formatChangelogDescription(desc: string | string[]) { * Check if a changelog config key matches a hwKey. * Normalizes both to hyphen-separated form for comparison. */ -export function configKeyMatchesHwKey(configKey: string, hwKey: string): boolean { - return changelogConfigToHwKey(configKey) === hwKey; +export function configKeyMatchesHwKey( + configKey: string, + hwKey: string, + benchmarkType?: 'single_turn' | 'agentic_traces', +): boolean { + return changelogConfigToHwKey(configKey, benchmarkType) === hwKey; } export function formatConfigKeys(key: string) { diff --git a/packages/app/src/components/inference/utils/runEnumeration.test.ts b/packages/app/src/components/inference/utils/runEnumeration.test.ts index 333d2b502..3d92f637c 100644 --- a/packages/app/src/components/inference/utils/runEnumeration.test.ts +++ b/packages/app/src/components/inference/utils/runEnumeration.test.ts @@ -16,7 +16,6 @@ function rc(over: Partial): RunConfigRow { framework: 'vllm', spec_method: 'none', disagg: false, - benchmark_type: 'single_turn', ...over, }; } @@ -25,6 +24,7 @@ const SCOPE = { modelDbKeys: ['minimaxm3'], selectedGPUs: ['mi300x_vllm'], selectedPrecisions: ['fp8'], + benchmarkType: 'single_turn' as const, }; describe('dataRunsForDate', () => { @@ -69,10 +69,10 @@ describe('dataRunsForDate', () => { it('maps agentic MTP and non-MTP run coverage to one GPU series', () => { const rows = [ - rc({ github_run_id: 1, spec_method: 'none', benchmark_type: 'agentic_traces' }), - rc({ github_run_id: 2, spec_method: 'mtp', benchmark_type: 'agentic_traces' }), + rc({ github_run_id: 1, spec_method: 'none' }), + rc({ github_run_id: 2, spec_method: 'mtp' }), ]; - const runs = dataRunsForDate(rows, SCOPE); + const runs = dataRunsForDate(rows, { ...SCOPE, benchmarkType: 'agentic_traces' }); expect(runs.map((r) => r.runId)).toEqual(['1', '2']); }); diff --git a/packages/app/src/components/inference/utils/runEnumeration.ts b/packages/app/src/components/inference/utils/runEnumeration.ts index 94758cb78..1c84b6f25 100644 --- a/packages/app/src/components/inference/utils/runEnumeration.ts +++ b/packages/app/src/components/inference/utils/runEnumeration.ts @@ -37,16 +37,18 @@ export interface RunScope { selectedGPUs: string[]; /** Selected DB precisions, e.g. ['fp8']. */ selectedPrecisions: string[]; + /** Scenario currently rendered by the chart. */ + benchmarkType: 'single_turn' | 'agentic_traces'; } /** The hw key a runConfig maps to, built the same way the chart builds series keys. */ -function runConfigHwKey(rc: RunConfigRow): string { +function runConfigHwKey(rc: RunConfigRow, benchmarkType: 'single_turn' | 'agentic_traces'): string { return getHardwareKey({ hw: rc.hardware, framework: rc.framework, disagg: rc.disagg, spec_decoding: rc.spec_method, - benchmark_type: rc.benchmark_type, + benchmark_type: benchmarkType, } as unknown as AggDataEntry); } @@ -56,7 +58,7 @@ function runConfigHwKey(rc: RunConfigRow): string { * assigns read in the order the runs actually happened. */ export function dataRunsForDate(runConfigs: RunConfigRow[], scope: RunScope): DataRun[] { - const { modelDbKeys, selectedGPUs, selectedPrecisions } = scope; + const { modelDbKeys, selectedGPUs, selectedPrecisions, benchmarkType } = scope; const precSet = new Set(selectedPrecisions); const gpuSet = new Set(selectedGPUs); const byRun = new Map(); @@ -64,7 +66,7 @@ export function dataRunsForDate(runConfigs: RunConfigRow[], scope: RunScope): Da for (const rc of runConfigs) { if (!modelDbKeys.includes(rc.model)) continue; if (!precSet.has(rc.precision)) continue; - if (!gpuSet.has(runConfigHwKey(rc))) continue; + if (!gpuSet.has(runConfigHwKey(rc, benchmarkType))) continue; const id = String(rc.github_run_id); if (!byRun.has(id)) { diff --git a/packages/app/src/hooks/api/use-ai-chart.ts b/packages/app/src/hooks/api/use-ai-chart.ts index b62fb02b8..92783463a 100644 --- a/packages/app/src/hooks/api/use-ai-chart.ts +++ b/packages/app/src/hooks/api/use-ai-chart.ts @@ -399,7 +399,13 @@ async function resolveSpec(spec: AiChartSpec): Promise { const { isl, osl } = isAgentic ? { isl: 0, osl: 0 } : sequenceToIslOsl(spec.sequence); const fetchedRows = spec.dataSource === 'history' - ? await fetchBenchmarkHistory(spec.model, isl, osl, isAgentic ? 'agentic_traces' : undefined) + ? await fetchBenchmarkHistory( + spec.model, + isl, + osl, + undefined, + isAgentic ? 'agentic_traces' : undefined, + ) : await fetchBenchmarks(spec.model); const rows = spec.dataSource === 'history' && isAgentic diff --git a/packages/app/src/hooks/api/use-benchmark-history.ts b/packages/app/src/hooks/api/use-benchmark-history.ts index a65b926f7..971919d19 100644 --- a/packages/app/src/hooks/api/use-benchmark-history.ts +++ b/packages/app/src/hooks/api/use-benchmark-history.ts @@ -9,8 +9,10 @@ export function useBenchmarkHistory( benchmarkType?: 'agentic_traces', ) { return useQuery({ - queryKey: ['benchmark-history', model, isl, osl, benchmarkType], - queryFn: ({ signal }) => fetchBenchmarkHistory(model, isl, osl, benchmarkType, signal), + queryKey: benchmarkType + ? ['benchmark-history', model, isl, osl, benchmarkType] + : ['benchmark-history', model, isl, osl], + queryFn: ({ signal }) => fetchBenchmarkHistory(model, isl, osl, signal, benchmarkType), enabled: Boolean(model && (benchmarkType === 'agentic_traces' || (isl && osl))), }); } diff --git a/packages/app/src/hooks/api/use-comparison-changelogs.ts b/packages/app/src/hooks/api/use-comparison-changelogs.ts index 01d60ad3e..b56976140 100644 --- a/packages/app/src/hooks/api/use-comparison-changelogs.ts +++ b/packages/app/src/hooks/api/use-comparison-changelogs.ts @@ -43,6 +43,7 @@ export function useComparisonChangelogs( selectedGPUs: string[], selectedDateRange: { startDate: string; endDate: string }, availableDates: string[], + benchmarkType?: 'agentic_traces', ) { const hasGPUs = selectedGPUs.length > 0; const hasDateRange = Boolean(selectedDateRange.startDate) && Boolean(selectedDateRange.endDate); @@ -64,8 +65,9 @@ export function useComparisonChangelogs( const queries = useQueries({ queries: datesToQuery.map((date) => ({ - queryKey: ['workflow-info', date], - queryFn: ({ signal }: { signal: AbortSignal }) => fetchWorkflowInfo(date, signal), + queryKey: benchmarkType ? ['workflow-info', date, benchmarkType] : ['workflow-info', date], + queryFn: ({ signal }: { signal: AbortSignal }) => + fetchWorkflowInfo(date, signal, benchmarkType), enabled: hasGPUs, })), }); diff --git a/packages/app/src/lib/agentic-workflow-metadata.test.ts b/packages/app/src/lib/agentic-workflow-metadata.test.ts new file mode 100644 index 000000000..d0d677872 --- /dev/null +++ b/packages/app/src/lib/agentic-workflow-metadata.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest'; + +import { agenticWorkflowMetadataOnly } from './agentic-workflow-metadata'; + +describe('agenticWorkflowMetadataOnly', () => { + it('preserves workflow metadata for agentic rows', () => { + const [row] = agenticWorkflowMetadataOnly([ + { + benchmark_type: 'agentic_traces', + workflow_run_id: 42, + run_started_at: '2026-08-12T10:00:00Z', + }, + ]); + expect(row.workflow_run_id).toBe(42); + expect(row.run_started_at).toBe('2026-08-12T10:00:00Z'); + }); + + it('preserves the prior fixed-sequence response shape', () => { + const [row] = agenticWorkflowMetadataOnly([ + { + benchmark_type: 'single_turn', + workflow_run_id: 42, + run_started_at: '2026-08-12T10:00:00Z', + }, + ]); + expect(row).toEqual({ benchmark_type: 'single_turn' }); + expect('workflow_run_id' in row).toBe(false); + expect('run_started_at' in row).toBe(false); + }); +}); diff --git a/packages/app/src/lib/agentic-workflow-metadata.ts b/packages/app/src/lib/agentic-workflow-metadata.ts new file mode 100644 index 000000000..9c605fcbd --- /dev/null +++ b/packages/app/src/lib/agentic-workflow-metadata.ts @@ -0,0 +1,14 @@ +interface WorkflowMetadataRow { + benchmark_type: string; + workflow_run_id?: number; + run_started_at?: string | null; +} + +/** Keep workflow identity internal to agentic rows; fixed API rows retain their prior shape. */ +export function agenticWorkflowMetadataOnly(rows: T[]): T[] { + return rows.map((row) => { + if (row.benchmark_type === 'agentic_traces') return row; + const { workflow_run_id: _workflowRunId, run_started_at: _runStartedAt, ...fixedRow } = row; + return fixedRow as T; + }); +} diff --git a/packages/app/src/lib/api-documentation.ts b/packages/app/src/lib/api-documentation.ts index ef5852f98..5167a85da 100644 --- a/packages/app/src/lib/api-documentation.ts +++ b/packages/app/src/lib/api-documentation.ts @@ -234,9 +234,11 @@ const benchmarkRowSchema = objectSchemaWithOptional( metrics: metricMapSchema, workers: arraySchema(workerPowerSchema), date: { type: 'string', format: 'date' }, + workflow_run_id: integerSchema, + run_started_at: { type: ['string', 'null'], format: 'date-time' }, run_url: nullableStringSchema, }, - ['workers'], + ['workers', 'workflow_run_id', 'run_started_at'], ); const benchmarkRowsSchema = arraySchema(benchmarkRowSchema); const benchmarkExample = [ @@ -590,6 +592,16 @@ export const apiOperations: readonly ApiOperation[] = [ { type: 'string', format: 'date' }, '2026-08-08', ), + parameter( + 'benchmarkType', + 'query', + false, + 'string', + 'Set to agentic_traces to scope per-run configuration coverage to Agentic Traces.', + '设为 agentic_traces 可将每次运行的配置覆盖限定为 Agentic Traces。', + { type: 'string', enum: ['agentic_traces'] }, + 'agentic_traces', + ), parameter( 'exact', 'query', @@ -660,7 +672,7 @@ export const apiOperations: readonly ApiOperation[] = [ parameter( 'model', 'query', - false, + true, 'string', 'Display model name.', '展示模型名称。', @@ -670,7 +682,7 @@ export const apiOperations: readonly ApiOperation[] = [ parameter( 'isl', 'query', - true, + false, 'integer', 'Positive input sequence length in tokens. Required unless benchmarkType=agentic_traces.', '正整数输入序列长度，单位为 token；除非 benchmarkType=agentic_traces，否则必填。', @@ -702,8 +714,8 @@ export const apiOperations: readonly ApiOperation[] = [ success('Historical benchmark rows.', '历史基准行。', benchmarkRowsSchema, benchmarkExample), errorResponse( '400', - 'Required parameters are missing, benchmarkType is unsupported, or the model is unsupported.', - '必填参数缺失、benchmarkType 不受支持，或模型不受支持。', + 'Required parameters are missing or the model is unsupported.', + '必填参数缺失或模型不受支持。', 'Missing required parameters', ), errorResponse('500', 'The history query failed.', '历史查询失败。', 'Internal server error'), diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index 39b040bf8..07dadbf19 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -108,7 +108,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'list-benchmarks', - sourceSha256: 'daf24b2a08ab021782084fc6e5e145013fca518e58d18837355a088e040f257b', + sourceSha256: '37b5a31613a9c5a2e1de35758551dfdbbb8b920fcd6ae6baeedf973c8802bc2c', }, { source: 'src/app/api/v1/benchmarks/history/route.ts', @@ -116,7 +116,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'list-benchmark-history', - sourceSha256: 'a15d1b1202c3b3bcd7abc9026dc308200a5290e4133cf71bd72cff4449e0206b', + sourceSha256: 'd4b3d2ad8ed6e35df70c6b651f71c9d86b6e1dad9c3eaf6eadc2a8591318d1fb', }, { source: 'src/app/api/v1/collectivex/latest/route.ts', @@ -353,7 +353,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'get-workflow-info', - sourceSha256: '466ec7d64baf781172a0ef49e54d4d91b791bbc96f058608008ad857dbbe8df5', + sourceSha256: 'b7b0f215e9bf2c766ce2d4c9a13090504c1704a67f697af3a2fd1f318de1e6d6', }, ] as const satisfies readonly ApiRouteCatalogEntry[]; @@ -380,7 +380,7 @@ export const apiContractSourceDigests = [ }, { source: 'src/lib/api.ts', - sourceSha256: '41512217a86cf60c1abc06992f1ae4e15d08ba1d8d2d2ff303a870342a7bec2b', + sourceSha256: '03809377af6c2ee938169a065e06dccb25d983d7171a54b998ffa08dd970d306', reviewArea: { en: 'Public API client parameter serialization and TypeScript response contracts.', zh: '公开 API 客户端的参数序列化和 TypeScript 响应契约。', @@ -532,7 +532,7 @@ export const apiContractSourceDigests = [ }, { source: '../db/src/queries/workflow-info.ts', - sourceSha256: 'f453406ebd4c8ddfe713478ee72af4859fa166579f915266a5925f9d70d1751e', + sourceSha256: 'b6611604d41ab69c00cb804ad255d4cbc9f70e41e84ad43330538558956f9eb6', reviewArea: { en: 'Availability rows plus workflow runs, changelogs, configurations, and run coverage responses.', zh: '可用配置行以及工作流运行、变更记录、配置和运行覆盖响应。', diff --git a/packages/app/src/lib/api.ts b/packages/app/src/lib/api.ts index 07fd733ff..0b2a03f71 100644 --- a/packages/app/src/lib/api.ts +++ b/packages/app/src/lib/api.ts @@ -105,7 +105,6 @@ export interface RunConfigRow { framework: string; spec_method: string; disagg: boolean; - benchmark_type: string; } export interface WorkflowInfoResponse { @@ -182,19 +181,22 @@ export function fetchBenchmarkHistory( model: string, isl: number, osl: number, - benchmarkType?: 'agentic_traces', signal?: AbortSignal, + benchmarkType?: 'agentic_traces', ) { const params = new URLSearchParams({ model, isl: String(isl), osl: String(osl) }); if (benchmarkType) params.set('benchmarkType', benchmarkType); return fetchJson(`/api/v1/benchmarks/history?${params}`, signal); } -export function fetchWorkflowInfo(date: string, signal?: AbortSignal) { - return fetchJson( - `/api/v1/workflow-info?date=${encodeURIComponent(date)}`, - signal, - ); +export function fetchWorkflowInfo( + date: string, + signal?: AbortSignal, + benchmarkType?: 'agentic_traces', +) { + const params = new URLSearchParams({ date }); + if (benchmarkType) params.set('benchmarkType', benchmarkType); + return fetchJson(`/api/v1/workflow-info?${params}`, signal); } export interface AvailabilityRow { diff --git a/packages/app/src/lib/benchmark-data.server.ts b/packages/app/src/lib/benchmark-data.server.ts index e288596c2..c9dd917b1 100644 --- a/packages/app/src/lib/benchmark-data.server.ts +++ b/packages/app/src/lib/benchmark-data.server.ts @@ -16,7 +16,7 @@ export const getCachedBenchmarks = cachedQuery( return getLatestBenchmarks(getDb(), dbModelKeys); }, - 'benchmarks', + 'benchmarks-agentic-run-metadata', { blobOnly: true }, ); @@ -28,6 +28,6 @@ export const getCachedBenchmarksAsOf = cachedQuery( return getLatestBenchmarks(getDb(), dbModelKeys, date); }, - 'benchmarks-as-of', + 'benchmarks-as-of-agentic-run-metadata', { blobOnly: true }, ); diff --git a/packages/db/src/queries/workflow-info.ts b/packages/db/src/queries/workflow-info.ts index 033218105..e32401395 100644 --- a/packages/db/src/queries/workflow-info.ts +++ b/packages/db/src/queries/workflow-info.ts @@ -76,7 +76,6 @@ export interface RunConfigRow { framework: string; spec_method: string; disagg: boolean; - benchmark_type: string; } /** @@ -85,7 +84,12 @@ export interface RunConfigRow { * shipped data without a changelog entry still surfaces — the comparison UI uses * this to enumerate every run on a date, not just runs with changelog notes. */ -export async function getRunConfigsByDate(sql: DbClient, date: string): Promise { +export async function getRunConfigsByDate( + sql: DbClient, + date: string, + benchmarkType?: 'agentic_traces', +): Promise { + const benchmarkTypeFilter = benchmarkType ? sql`AND br.benchmark_type = ${benchmarkType}` : sql``; const rows = await sql` SELECT DISTINCT wr.github_run_id, @@ -97,13 +101,13 @@ export async function getRunConfigsByDate(sql: DbClient, date: string): Promise< c.hardware, c.framework, c.spec_method, - c.disagg, - br.benchmark_type + c.disagg FROM benchmark_results br JOIN configs c ON c.id = br.config_id JOIN latest_workflow_runs wr ON wr.id = br.workflow_run_id WHERE br.date = ${date}::date AND br.error IS NULL + ${benchmarkTypeFilter} `; return rows as unknown as RunConfigRow[]; } From d7cd9ca36a665762ae17eb28d9dfb25cceace842 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 13:35:44 -0500 Subject: [PATCH 06/12] fix: hide standard decoding point labels Render an agentic decode-method label only for active speculative methods such as MTP or EAGLE. Preserve fixed-sequence server row shape while retaining agentic run metadata. --- .../components/inference/ui/ScatterGraph.decoration.test.tsx | 2 +- packages/app/src/components/inference/ui/ScatterGraph.tsx | 4 ++-- packages/app/src/lib/benchmark-data.server.ts | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx b/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx index 6f8ece5a2..3cbb43a26 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx @@ -80,7 +80,7 @@ describe('pointLabelText', () => { standard.spec_decoding = 'none'; const mtp = { ...standard, spec_decoding: 'mtp' }; - expect(pointLabelText(standard, false)).toBe('8\nC=16\nSTP'); + expect(pointLabelText(standard, false)).toBe('8\nC=16'); expect(pointLabelText(mtp, false)).toBe('8\nC=16\nMTP'); }); diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index a604887eb..13a2b4b5f 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -246,12 +246,12 @@ function groupPointsByDate(points: InferenceData[]): Map `${d.hwKey}_${d.precision}_${d.date}-${d.x}-${d.y}`; -/** Point label lines: TP (or full parallelism label), concurrency, then agentic decode mode. */ +/** Point label lines, with an extra line only when agentic speculative decoding is active. */ export const pointLabelText = (d: InferenceData, advanced: boolean): string => { const base = advanced ? `${getPointLabel(d)}\nC=${d.conc}` : `${d.tp}\nC=${d.conc}`; if (d.benchmark_type !== 'agentic_traces') return base; const specMethod = d.spec_decoding ?? 'none'; - return `${base}\n${specMethod === 'none' || specMethod === '' ? 'STP' : specMethod.toUpperCase()}`; + return specMethod === 'none' || specMethod === '' ? base : `${base}\n${specMethod.toUpperCase()}`; }; // Referentially stable "no overlay data" result (see processedOverlayData). diff --git a/packages/app/src/lib/benchmark-data.server.ts b/packages/app/src/lib/benchmark-data.server.ts index c9dd917b1..c3dc23ca5 100644 --- a/packages/app/src/lib/benchmark-data.server.ts +++ b/packages/app/src/lib/benchmark-data.server.ts @@ -5,6 +5,7 @@ import { } from '@semianalysisai/inferencex-db/queries/benchmarks'; import { cachedQuery } from '@/lib/api-cache'; +import { agenticWorkflowMetadataOnly } from '@/lib/agentic-workflow-metadata'; import { loadFixture } from '@/lib/test-fixtures'; /** Cache slot is keyed on the dbKeys array. Both `/compare/` and @@ -14,7 +15,7 @@ export const getCachedBenchmarks = cachedQuery( (dbModelKeys: string[]) => { if (FIXTURES_MODE) return Promise.resolve(loadFixture('benchmarks')); - return getLatestBenchmarks(getDb(), dbModelKeys); + return getLatestBenchmarks(getDb(), dbModelKeys).then(agenticWorkflowMetadataOnly); }, 'benchmarks-agentic-run-metadata', { blobOnly: true }, @@ -26,7 +27,7 @@ export const getCachedBenchmarksAsOf = cachedQuery( (dbModelKeys: string[], date: string) => { if (FIXTURES_MODE) return Promise.resolve(loadFixture('benchmarks')); - return getLatestBenchmarks(getDb(), dbModelKeys, date); + return getLatestBenchmarks(getDb(), dbModelKeys, date).then(agenticWorkflowMetadataOnly); }, 'benchmarks-as-of-agentic-run-metadata', { blobOnly: true }, From cfaa046f86c118af74465ccd61c5250784b5ce2e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 13:47:17 -0500 Subject: [PATCH 07/12] feat: mark agentic spec decoding with dashed plus --- .../app/cypress/component/gpu-graph.cy.tsx | 53 ++++++++ .../src/components/inference/ui/GPUGraph.tsx | 29 ++++- .../ui/ScatterGraph.decoration.test.tsx | 116 +++++++++++++++++- .../components/inference/ui/ScatterGraph.tsx | 94 ++++++++++---- .../inference/ui/SpecDecodeLegendKey.tsx | 66 ++++++++++ 5 files changed, 332 insertions(+), 26 deletions(-) create mode 100644 packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx diff --git a/packages/app/cypress/component/gpu-graph.cy.tsx b/packages/app/cypress/component/gpu-graph.cy.tsx index 82d214077..2ab69ff44 100644 --- a/packages/app/cypress/component/gpu-graph.cy.tsx +++ b/packages/app/cypress/component/gpu-graph.cy.tsx @@ -132,6 +132,59 @@ describe('GPUGraph', () => { cy.get('[data-testid="gpu-graph"] svg .visible-shape').should('have.length.greaterThan', 0); }); + it('shows agentic spec decoding as a dashed plus and composes it with offload', () => { + const data = [ + createMockInferenceData({ + hwKey: 'h100', + x: 32, + y: 180, + date: '2025-03-01', + precision: Precision.FP4, + benchmark_type: 'agentic_traces', + spec_decoding: 'mtp', + offload_mode: 'on', + }), + createMockInferenceData({ + hwKey: 'h100', + x: 64, + y: 210, + date: '2025-03-01', + precision: Precision.FP4, + benchmark_type: 'agentic_traces', + spec_decoding: 'none', + offload_mode: 'off', + }), + ]; + + mountWithProviders( +
+ +
, + { + inference: { + hardwareConfig: hwConfig, + selectedGPUs: ['h100'], + selectedDates: ['2025-03-01'], + selectedDateRange: { startDate: '', endDate: '' }, + activeDates: new Set(['2025-03-01_h100']), + selectedPrecisions: [Precision.FP4], + }, + }, + ); + + cy.get('#test-gpu-agentic-decorations svg .spec-decode-marker').should('have.length', 1); + cy.get('#test-gpu-agentic-decorations svg .offload-halo').should('have.length', 1); + cy.get('#test-gpu-agentic-decorations [data-testid="spec-decode-marker-key"]').should('exist'); + cy.get('#test-gpu-agentic-decorations [data-testid="offload-halo-key"]').should('exist'); + }); + it('renders date line labels along each roofline when showLineLabels is on', () => { // Two GPUs × two dates with enough points each to form rooflines. const data = [ diff --git a/packages/app/src/components/inference/ui/GPUGraph.tsx b/packages/app/src/components/inference/ui/GPUGraph.tsx index ea820653f..f51218996 100644 --- a/packages/app/src/components/inference/ui/GPUGraph.tsx +++ b/packages/app/src/components/inference/ui/GPUGraph.tsx @@ -62,6 +62,13 @@ import { OFFLOAD_HALO_STROKE_WIDTH, OffloadHaloLegendKey, } from '@/components/inference/ui/OffloadHaloLegendKey'; +import { + SPEC_DECODE_MARKER_DASHARRAY, + SPEC_DECODE_MARKER_PATH, + SPEC_DECODE_MARKER_STROKE_WIDTH, + SpecDecodeLegendKey, + hasAgenticSpecDecoding, +} from '@/components/inference/ui/SpecDecodeLegendKey'; const CHART_MARGIN = { top: 24, right: 10, bottom: 60, left: 60 }; @@ -140,6 +147,7 @@ const GPUGraph = React.memo( const { resolvedTheme } = useTheme(); const chartRef = useRef(null); const hasOffloadHalo = useMemo(() => data.some((point) => point.offload_mode === 'on'), [data]); + const hasSpecDecodeMarker = useMemo(() => data.some(hasAgenticSpecDecoding), [data]); // Shared date+GPU pairs. `dates` holds comparison-series entries (plain dates // and/or specific-run entries); a same-day range endpoint is dropped when that @@ -931,6 +939,18 @@ const GPUGraph = React.memo( .attr('stroke-dasharray', OFFLOAD_HALO_DASHARRAY) .attr('opacity', 0.9) .attr('pointer-events', 'none'); + d3.select(this) + .selectAll('.spec-decode-marker') + .data(hasAgenticSpecDecoding(d) ? [true] : []) + .join('path') + .attr('class', 'spec-decode-marker') + .attr('d', SPEC_DECODE_MARKER_PATH) + .attr('fill', 'none') + .attr('stroke', 'var(--foreground)') + .attr('stroke-width', SPEC_DECODE_MARKER_STROKE_WIDTH) + .attr('stroke-dasharray', SPEC_DECODE_MARKER_DASHARRAY) + .attr('stroke-linecap', 'round') + .attr('pointer-events', 'none'); }); }} legendElement={ @@ -1030,7 +1050,14 @@ const GPUGraph = React.memo( }, ]} precisionIndicators={selectedPrecisions} - keyIndicators={hasOffloadHalo ? : undefined} + keyIndicators={ + hasOffloadHalo || hasSpecDecodeMarker ? ( + <> + {hasOffloadHalo && } + {hasSpecDecodeMarker && } + + ) : undefined + } /> } /> diff --git a/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx b/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx index 3cbb43a26..0a6ba6a5c 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx @@ -22,7 +22,9 @@ vi.mock('@/lib/d3-chart/chart-setup', { spy: true }); vi.mock('@/lib/analytics', () => ({ track: vi.fn() })); vi.mock('next-themes', () => ({ useTheme: () => ({ resolvedTheme: 'dark' }) })); // The legend is React-rendered (covered elsewhere) — keep the tree light. -vi.mock('@/components/ui/chart-legend', () => ({ default: () => null })); +vi.mock('@/components/ui/chart-legend', () => ({ + default: ({ keyIndicators }: { keyIndicators?: React.ReactNode }) => keyIndicators ?? null, +})); const inferenceState = vi.hoisted(() => ({ current: {} as Record })); vi.mock('@/components/inference/InferenceContext', () => ({ @@ -74,14 +76,16 @@ const CHART_DEFINITION = { chartType: 'interactivity' } as unknown as ChartDefin const noop = () => {}; describe('pointLabelText', () => { - it('labels mixed agentic points with their point-level decode mode', () => { + it('keeps decode mode out of mixed agentic point labels', () => { const standard = point('h100', 'fp8', 1, 1, 8); standard.benchmark_type = 'agentic_traces'; standard.spec_decoding = 'none'; const mtp = { ...standard, spec_decoding: 'mtp' }; + const eagle = { ...standard, spec_decoding: 'eagle' }; expect(pointLabelText(standard, false)).toBe('8\nC=16'); - expect(pointLabelText(mtp, false)).toBe('8\nC=16\nMTP'); + expect(pointLabelText(mtp, false)).toBe('8\nC=16'); + expect(pointLabelText(eagle, false)).toBe('8\nC=16'); }); it('keeps fixed-sequence labels unchanged', () => { @@ -232,6 +236,48 @@ describe('ScatterGraph toggle decoration', () => { unmount(); }); + it('composes agentic spec decoding and KV-offload point decorations', () => { + const standard = { + ...point('h100', 'fp8', 1, 1, 1), + benchmark_type: 'agentic_traces', + spec_decoding: 'none', + offload_mode: 'off', + } as InferenceData; + const mtp = { + ...point('h100', 'fp8', 20, 200, 2), + benchmark_type: 'agentic_traces', + spec_decoding: 'mtp', + offload_mode: 'off', + } as InferenceData; + const mtpWithOffload = { + ...point('h100', 'fp8', 40, 400, 4), + benchmark_type: 'agentic_traces', + spec_decoding: 'mtp', + offload_mode: 'on', + } as InferenceData; + const fixedMtp = { + ...point('h100', 'fp8', 100, 1000, 8), + benchmark_type: 'single_turn', + spec_decoding: 'mtp', + offload_mode: 'off', + } as InferenceData; + + const { container, unmount } = mountChart({ + data: [standard, mtp, mtpWithOffload, fixedMtp], + }); + const groups = dotGroups(container); + + expect(groups[0].querySelector('.spec-decode-marker')).toBeNull(); + expect(groups[1].querySelector('.spec-decode-marker')).not.toBeNull(); + expect(groups[1].querySelector('.offload-halo')).toBeNull(); + expect(groups[2].querySelector('.spec-decode-marker')).not.toBeNull(); + expect(groups[2].querySelector('.offload-halo')).not.toBeNull(); + expect(groups[3].querySelector('.spec-decode-marker')).toBeNull(); + expect(container.querySelector('[data-testid="spec-decode-marker-key"]')).not.toBeNull(); + expect(container.querySelector('[data-testid="offload-halo-key"]')).not.toBeNull(); + unmount(); + }); + it('hides a toggled-off hw via opacity without rebuilding the chart', () => { const { container, rerender, unmount } = mountChart(); const buildsAfterMount = rebuildCount(); @@ -305,6 +351,28 @@ describe('ScatterGraph toggle decoration', () => { unmount(); }); + it('keeps the spec marker above a point after a precision-shape swap', () => { + const agenticFp4 = { + ...point('h100', 'fp4', 40, 400, 4), + benchmark_type: 'agentic_traces', + spec_decoding: 'mtp', + } as InferenceData; + const { container, rerender, unmount } = mountChart({ + data: [POINTS[0], agenticFp4, POINTS[1]], + }); + + inferenceState.current = { + ...inferenceState.current, + selectedPrecisions: ['fp8', 'fp4'], + }; + rerender(); + + const fp4Dot = dotGroups(container, 'h100').find((d) => d.dataset.precision === 'fp4')!; + expect(fp4Dot.querySelector('.visible-shape')!.tagName.toLowerCase()).toBe('rect'); + expect(fp4Dot.lastElementChild).toBe(fp4Dot.querySelector('.spec-decode-marker')); + unmount(); + }); + it('rebuilds when the scale domain actually changes (data refresh path intact)', () => { const { rerender, unmount } = mountChart(); const buildsAfterMount = rebuildCount(); @@ -394,6 +462,48 @@ describe('ScatterGraph toggle decoration', () => { unmount(); }); + it('composes spec decoding and offload decorations on unofficial-run points', () => { + const runUrl = 'https://github.com/o/r/actions/runs/123'; + const overlayPoints = [ + { + ...point('h100', 'fp8', 30, 300, 2), + benchmark_type: 'agentic_traces', + spec_decoding: 'mtp', + offload_mode: 'on', + run_url: runUrl, + } as InferenceData, + { + ...point('h100', 'fp8', 35, 350, 4), + benchmark_type: 'agentic_traces', + spec_decoding: 'none', + offload_mode: 'off', + run_url: runUrl, + } as InferenceData, + ]; + overlayState.current = { + ...baseOverlayState(), + isUnofficialRun: true, + activeOverlayHwTypes: new Set(['h100']), + allOverlayHwTypes: new Set(['h100']), + runIndexByUrl: { [runUrl]: 0 }, + unofficialRunInfos: [{ id: '123', branch: 'test-branch', url: runUrl }], + }; + + const { container, unmount } = mountChart({ + overlayData: { + data: overlayPoints, + hardwareConfig: HARDWARE_CONFIG, + } as unknown as Parameters[0]['overlayData'], + }); + const groups = [...container.querySelectorAll('.unofficial-overlay-pt')]; + + expect(groups[0].querySelector('.spec-decode-marker')).not.toBeNull(); + expect(groups[0].querySelector('.offload-halo')).not.toBeNull(); + expect(groups[1].querySelector('.spec-decode-marker')).toBeNull(); + expect(groups[1].querySelector('.offload-halo')).toBeNull(); + unmount(); + }); + it('applies quick filters to unofficial-run overlay markers', () => { const overlayPoints = [point('h100', 'fp8', 30, 300, 2), point('h100', 'fp8', 35, 350, 4)].map( (p) => ({ ...p, run_url: 'https://github.com/o/r/actions/runs/123' }), diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index 13a2b4b5f..3a9e8e963 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -77,6 +77,13 @@ import { OFFLOAD_HALO_STROKE_WIDTH, OffloadHaloLegendKey, } from '@/components/inference/ui/OffloadHaloLegendKey'; +import { + SPEC_DECODE_MARKER_DASHARRAY, + SPEC_DECODE_MARKER_PATH, + SPEC_DECODE_MARKER_STROKE_WIDTH, + SpecDecodeLegendKey, + hasAgenticSpecDecoding, +} from '@/components/inference/ui/SpecDecodeLegendKey'; import { buildLegendPointsRows } from '@/components/inference/utils/legend-points-table'; import { type ParetoPointLabel, @@ -200,6 +207,38 @@ const getXPath = (size: number) => { return `M ${-s} ${-s} L ${s} ${s} M ${s} ${-s} L ${-s} ${s}`; }; +/** Apply the two independent point decorations; either, both, or neither may render. */ +function renderPointDecorations( + group: d3.Selection, + point: InferenceData, + stroke: string, +): void { + group + .selectAll('.offload-halo') + .data(point.offload_mode === 'on' ? [true] : []) + .join('circle') + .attr('class', 'offload-halo') + .attr('r', OFFLOAD_HALO_RADIUS) + .attr('fill', 'none') + .attr('stroke', stroke) + .attr('stroke-width', OFFLOAD_HALO_STROKE_WIDTH) + .attr('stroke-dasharray', OFFLOAD_HALO_DASHARRAY) + .attr('opacity', 0.9) + .attr('pointer-events', 'none'); + group + .selectAll('.spec-decode-marker') + .data(hasAgenticSpecDecoding(point) ? [true] : []) + .join('path') + .attr('class', 'spec-decode-marker') + .attr('d', SPEC_DECODE_MARKER_PATH) + .attr('fill', 'none') + .attr('stroke', stroke) + .attr('stroke-width', SPEC_DECODE_MARKER_STROKE_WIDTH) + .attr('stroke-dasharray', SPEC_DECODE_MARKER_DASHARRAY) + .attr('stroke-linecap', 'round') + .attr('pointer-events', 'none'); +} + const formatChangelogDescription = (desc: string | string[]): React.JSX.Element => { if (typeof desc === 'string') { return ( @@ -246,13 +285,9 @@ function groupPointsByDate(points: InferenceData[]): Map `${d.hwKey}_${d.precision}_${d.date}-${d.x}-${d.y}`; -/** Point label lines, with an extra line only when agentic speculative decoding is active. */ -export const pointLabelText = (d: InferenceData, advanced: boolean): string => { - const base = advanced ? `${getPointLabel(d)}\nC=${d.conc}` : `${d.tp}\nC=${d.conc}`; - if (d.benchmark_type !== 'agentic_traces') return base; - const specMethod = d.spec_decoding ?? 'none'; - return specMethod === 'none' || specMethod === '' ? base : `${base}\n${specMethod.toUpperCase()}`; -}; +/** Point label lines. Decode mode is shown by a point marker instead of text. */ +export const pointLabelText = (d: InferenceData, advanced: boolean): string => + advanced ? `${getPointLabel(d)}\nC=${d.conc}` : `${d.tp}\nC=${d.conc}`; // Referentially stable "no overlay data" result (see processedOverlayData). const EMPTY_OVERLAY_DATA: InferenceData[] = []; @@ -1058,6 +1093,12 @@ const ScatterGraph = React.memo( processedOverlayData.some((point) => point.offload_mode === 'on'), [pointsData, processedOverlayData], ); + const hasSpecDecodeMarker = useMemo( + () => + pointsData.some(hasAgenticSpecDecoding) || + processedOverlayData.some(hasAgenticSpecDecoding), + [pointsData, processedOverlayData], + ); // Bulk presence lookup for agentic points: which ids have a stored // trace_replay blob → controls the "View charts" button in the pinned @@ -2389,6 +2430,17 @@ const ScatterGraph = React.memo( overlayRunColor(overlayRunIndex(d.run_url ?? null, runIndexByUrl)), ); + // Point decorations compose independently: offload adds the + // dashed halo, while agentic speculative decoding adds a dashed + // plus. A point using both displays both markers. + overlayPoints.each(function (d) { + renderPointDecorations( + d3.select(this), + d, + overlayRunColor(overlayRunIndex(d.run_url ?? null, runIndexByUrl)), + ); + }); + // Labels const showLabels = showPointLabels && !showGradientLabels; overlayPoints.each(function (d) { @@ -2864,7 +2916,7 @@ const ScatterGraph = React.memo( const layersRef = useRef(layers); layersRef.current = layers; - // --- onRender: CSS transitions, offload halos, and log tick formatting --- + // --- onRender: CSS transitions, point decorations, and log tick formatting --- const onRender = useCallback( (ctx: RenderContext) => { // Stash the render context for the decoration effect. @@ -2876,19 +2928,7 @@ const ScatterGraph = React.memo( // Offload halo: dashed ring on every point that used KV offload (Pareto or not) zoomGroup.selectAll('.dot-group').each(function (d) { - const showHalo = d.offload_mode === 'on'; - d3.select(this) - .selectAll('.offload-halo') - .data(showHalo ? [true] : []) - .join('circle') - .attr('class', 'offload-halo') - .attr('r', OFFLOAD_HALO_RADIUS) - .attr('fill', 'none') - .attr('stroke', 'var(--foreground)') - .attr('stroke-width', OFFLOAD_HALO_STROKE_WIDTH) - .attr('stroke-dasharray', OFFLOAD_HALO_DASHARRAY) - .attr('opacity', 0.9) - .attr('pointer-events', 'none'); + renderPointDecorations(d3.select(this), d, 'var(--foreground)'); }); avoidLabelCollisions(zoomGroup); @@ -2955,6 +2995,9 @@ const ScatterGraph = React.memo( getShapeKeyForPrecision(d.precision, ir.selectedPrecisions), color, ); + // A precision toggle may replace and append the visible SVG shape. + // Keep decorations above that shape after the swap. + sel.selectAll('.offload-halo, .spec-decode-marker').raise(); }); // Overlay X markers: Optimal Only visibility (mirrors the official dot @@ -3429,7 +3472,14 @@ const ScatterGraph = React.memo( : [] } precisionIndicators={selectedPrecisions} - keyIndicators={hasOffloadHalo ? : undefined} + keyIndicators={ + hasOffloadHalo || hasSpecDecodeMarker ? ( + <> + {hasOffloadHalo && } + {hasSpecDecodeMarker && } + + ) : undefined + } enableTooltips={true} /> } diff --git a/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx b/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx new file mode 100644 index 000000000..054829509 --- /dev/null +++ b/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx @@ -0,0 +1,66 @@ +import { POINT_SIZE } from '@/lib/chart-rendering'; +import { useLocale } from '@/lib/use-locale'; + +export const SPEC_DECODE_MARKER_SIZE = POINT_SIZE + 1.5; +export const SPEC_DECODE_MARKER_STROKE_WIDTH = 1.5; +export const SPEC_DECODE_MARKER_DASHARRAY = '2 1.5'; +export const SPEC_DECODE_MARKER_PATH = `M ${-SPEC_DECODE_MARKER_SIZE} 0 H ${SPEC_DECODE_MARKER_SIZE} M 0 ${-SPEC_DECODE_MARKER_SIZE} V ${SPEC_DECODE_MARKER_SIZE}`; + +interface SpecDecodePoint { + benchmark_type?: string | null; + spec_decoding?: string | null; +} + +/** True only for agentic points that actually use a speculative decode method. */ +export function hasAgenticSpecDecoding(point: SpecDecodePoint): boolean { + if (point.benchmark_type !== 'agentic_traces') return false; + const method = point.spec_decoding?.trim().toLowerCase(); + return Boolean(method && method !== 'none'); +} + +const STRINGS = { + en: { + marker: 'Dashed +:', + meaning: 'Speculative decoding', + }, + zh: { + marker: '虚线 +：', + meaning: '推测解码', + }, +} as const; + +/** Legend key for the dashed plus drawn over agentic speculative-decoding points. */ +export function SpecDecodeLegendKey() { + const locale = useLocale(); + const t = STRINGS[locale]; + + return ( +
+ + + {t.marker} + {t.meaning} + +
+ ); +} From d607d6d9806aea9698df17e21c6fb7858dce19c1 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 13:50:26 -0500 Subject: [PATCH 08/12] style: match spec marker to offload halo --- .../app/src/components/inference/ui/SpecDecodeLegendKey.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx b/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx index 054829509..5f6b25545 100644 --- a/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx +++ b/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx @@ -1,7 +1,10 @@ import { POINT_SIZE } from '@/lib/chart-rendering'; import { useLocale } from '@/lib/use-locale'; +import { OFFLOAD_HALO_RADIUS } from '@/components/inference/ui/OffloadHaloLegendKey'; -export const SPEC_DECODE_MARKER_SIZE = POINT_SIZE + 1.5; +// Match the plus diameter to the KV-offload halo so combined markers share +// one clean outer boundary. +export const SPEC_DECODE_MARKER_SIZE = OFFLOAD_HALO_RADIUS; export const SPEC_DECODE_MARKER_STROKE_WIDTH = 1.5; export const SPEC_DECODE_MARKER_DASHARRAY = '2 1.5'; export const SPEC_DECODE_MARKER_PATH = `M ${-SPEC_DECODE_MARKER_SIZE} 0 H ${SPEC_DECODE_MARKER_SIZE} M 0 ${-SPEC_DECODE_MARKER_SIZE} V ${SPEC_DECODE_MARKER_SIZE}`; From c6a1c7e59f767d784d57e295185ca16bf2468980 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 13:51:19 -0500 Subject: [PATCH 09/12] style: simplify point marker legend labels --- .../app/cypress/component/chart-legend.cy.tsx | 2 +- .../inference/ui/OffloadHaloLegendKey.tsx | 5 +---- .../inference/ui/SpecDecodeLegendKey.tsx | 17 ++++------------- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/packages/app/cypress/component/chart-legend.cy.tsx b/packages/app/cypress/component/chart-legend.cy.tsx index 8aa4e358e..15b9ab100 100644 --- a/packages/app/cypress/component/chart-legend.cy.tsx +++ b/packages/app/cypress/component/chart-legend.cy.tsx @@ -141,8 +141,8 @@ describe('ChartLegend (sidebar variant)', () => { cy.get('[data-testid="offload-halo-key"]') .should('be.visible') - .and('contain.text', 'Dashed halo:') .and('contain.text', 'KV offload ON') + .and('not.contain.text', 'Dashed halo:') .and('not.have.class', 'no-export'); cy.get('[data-testid="offload-halo-key"] circle[stroke-dasharray="3 2"]').should('exist'); cy.get('[data-testid="chart-legend"]').then(($legend) => { diff --git a/packages/app/src/components/inference/ui/OffloadHaloLegendKey.tsx b/packages/app/src/components/inference/ui/OffloadHaloLegendKey.tsx index 82c9f20fb..c7e38e10b 100644 --- a/packages/app/src/components/inference/ui/OffloadHaloLegendKey.tsx +++ b/packages/app/src/components/inference/ui/OffloadHaloLegendKey.tsx @@ -34,10 +34,7 @@ export function OffloadHaloLegendKey() { strokeDasharray={OFFLOAD_HALO_DASHARRAY} /> - - Dashed halo: - KV offload ON - + KV offload ON ); } diff --git a/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx b/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx index 5f6b25545..793eccf31 100644 --- a/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx +++ b/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx @@ -22,20 +22,14 @@ export function hasAgenticSpecDecoding(point: SpecDecodePoint): boolean { } const STRINGS = { - en: { - marker: 'Dashed +:', - meaning: 'Speculative decoding', - }, - zh: { - marker: '虚线 +：', - meaning: '推测解码', - }, + en: 'Speculative decoding', + zh: '推测解码', } as const; /** Legend key for the dashed plus drawn over agentic speculative-decoding points. */ export function SpecDecodeLegendKey() { const locale = useLocale(); - const t = STRINGS[locale]; + const label = STRINGS[locale]; return (
- - {t.marker} - {t.meaning} - + {label}
); } From 85f7db895bb6288a437f179dfbfc242939685736 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 14:02:23 -0500 Subject: [PATCH 10/12] refactor: move agentic optimization details to hover --- .../app/cypress/component/gpu-graph.cy.tsx | 19 ++++-- .../inference/ui/AgenticOptimizationNote.tsx | 53 ++++++++++++++++ .../src/components/inference/ui/GPUGraph.tsx | 28 ++------- .../ui/ScatterGraph.decoration.test.tsx | 40 ++++--------- .../components/inference/ui/ScatterGraph.tsx | 50 ++++------------ .../inference/ui/SpecDecodeLegendKey.tsx | 60 ------------------- 6 files changed, 95 insertions(+), 155 deletions(-) create mode 100644 packages/app/src/components/inference/ui/AgenticOptimizationNote.tsx delete mode 100644 packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx diff --git a/packages/app/cypress/component/gpu-graph.cy.tsx b/packages/app/cypress/component/gpu-graph.cy.tsx index 2ab69ff44..0634ffff6 100644 --- a/packages/app/cypress/component/gpu-graph.cy.tsx +++ b/packages/app/cypress/component/gpu-graph.cy.tsx @@ -5,7 +5,7 @@ import { createMockChartDefinition, createMockHardwareConfig, } from '../support/mock-data'; -import { Precision } from '@/lib/data-mappings'; +import { Precision, Sequence } from '@/lib/data-mappings'; const defaultChartDef = createMockChartDefinition(); const hwConfig = createMockHardwareConfig(); @@ -132,7 +132,7 @@ describe('GPUGraph', () => { cy.get('[data-testid="gpu-graph"] svg .visible-shape').should('have.length.greaterThan', 0); }); - it('shows agentic spec decoding as a dashed plus and composes it with offload', () => { + it('shows spec decoding only on hover while retaining the offload halo', () => { const data = [ createMockInferenceData({ hwKey: 'h100', @@ -175,14 +175,25 @@ describe('GPUGraph', () => { selectedDateRange: { startDate: '', endDate: '' }, activeDates: new Set(['2025-03-01_h100']), selectedPrecisions: [Precision.FP4], + selectedSequence: Sequence.AgenticTraces, }, }, ); - cy.get('#test-gpu-agentic-decorations svg .spec-decode-marker').should('have.length', 1); + cy.get('#test-gpu-agentic-decorations svg .spec-decode-marker').should('not.exist'); cy.get('#test-gpu-agentic-decorations svg .offload-halo').should('have.length', 1); - cy.get('#test-gpu-agentic-decorations [data-testid="spec-decode-marker-key"]').should('exist'); + cy.get('#test-gpu-agentic-decorations [data-testid="spec-decode-marker-key"]').should( + 'not.exist', + ); cy.get('#test-gpu-agentic-decorations [data-testid="offload-halo-key"]').should('exist'); + cy.get('#test-gpu-agentic-decorations [data-testid="agentic-optimization-note"]') + .should('contain.text', 'Inference optimizations enabled') + .find('button') + .focus(); + cy.contains('Each configuration may use inference optimizations').should('be.visible'); + cy.get('#test-gpu-agentic-decorations svg .dot-group').first().trigger('mouseenter'); + cy.get('[data-chart-tooltip]').should('contain.text', 'Speculative Decoding'); + cy.get('[data-chart-tooltip]').should('contain.text', 'MTP'); }); it('renders date line labels along each roofline when showLineLabels is on', () => { diff --git a/packages/app/src/components/inference/ui/AgenticOptimizationNote.tsx b/packages/app/src/components/inference/ui/AgenticOptimizationNote.tsx new file mode 100644 index 000000000..020d0c38b --- /dev/null +++ b/packages/app/src/components/inference/ui/AgenticOptimizationNote.tsx @@ -0,0 +1,53 @@ +import { Info } from 'lucide-react'; + +import { + TooltipContent, + TooltipProvider, + TooltipRoot, + TooltipTrigger, +} from '@/components/ui/tooltip'; +import { useLocale } from '@/lib/use-locale'; + +const STRINGS = { + en: { + label: 'Inference optimizations enabled', + aria: 'About inference optimizations', + details: + 'Each configuration may use inference optimizations such as speculative decoding. Hover over a point to see its exact settings.', + }, + zh: { + label: '已启用推理优化', + aria: '关于推理优化', + details: '每项配置可能使用推测解码等推理优化。将鼠标悬停在数据点上可查看其具体设置。', + }, +} as const; + +/** Agentic-only legend note; point tooltips carry the exact optimization method. */ +export function AgenticOptimizationNote() { + const t = STRINGS[useLocale()]; + + return ( +
+ *{t.label} + + + + + + + {t.details} + + + +
+ ); +} diff --git a/packages/app/src/components/inference/ui/GPUGraph.tsx b/packages/app/src/components/inference/ui/GPUGraph.tsx index f51218996..1171ab66b 100644 --- a/packages/app/src/components/inference/ui/GPUGraph.tsx +++ b/packages/app/src/components/inference/ui/GPUGraph.tsx @@ -8,7 +8,7 @@ import { useTheme } from 'next-themes'; import { useInference } from '@/components/inference/InferenceContext'; import ChartLegend from '@/components/ui/chart-legend'; import { getHardwareConfig, getModelSortIndex } from '@/lib/constants'; -import { getChartWatermark } from '@/lib/data-mappings'; +import { getChartWatermark, Sequence } from '@/lib/data-mappings'; import { generateGpuDateColors } from '@/lib/dynamic-colors'; import { useLocale } from '@/lib/use-locale'; import { formatNumber, getDisplayLabel, updateRepoUrl } from '@/lib/utils'; @@ -62,13 +62,7 @@ import { OFFLOAD_HALO_STROKE_WIDTH, OffloadHaloLegendKey, } from '@/components/inference/ui/OffloadHaloLegendKey'; -import { - SPEC_DECODE_MARKER_DASHARRAY, - SPEC_DECODE_MARKER_PATH, - SPEC_DECODE_MARKER_STROKE_WIDTH, - SpecDecodeLegendKey, - hasAgenticSpecDecoding, -} from '@/components/inference/ui/SpecDecodeLegendKey'; +import { AgenticOptimizationNote } from '@/components/inference/ui/AgenticOptimizationNote'; const CHART_MARGIN = { top: 24, right: 10, bottom: 60, left: 60 }; @@ -122,6 +116,7 @@ const GPUGraph = React.memo( selectedGPUs, selectedDateRange, selectedDates, + selectedSequence, setSelectedDates, toggleActiveDate, removeActiveDate, @@ -147,7 +142,6 @@ const GPUGraph = React.memo( const { resolvedTheme } = useTheme(); const chartRef = useRef(null); const hasOffloadHalo = useMemo(() => data.some((point) => point.offload_mode === 'on'), [data]); - const hasSpecDecodeMarker = useMemo(() => data.some(hasAgenticSpecDecoding), [data]); // Shared date+GPU pairs. `dates` holds comparison-series entries (plain dates // and/or specific-run entries); a same-day range endpoint is dropped when that @@ -939,18 +933,6 @@ const GPUGraph = React.memo( .attr('stroke-dasharray', OFFLOAD_HALO_DASHARRAY) .attr('opacity', 0.9) .attr('pointer-events', 'none'); - d3.select(this) - .selectAll('.spec-decode-marker') - .data(hasAgenticSpecDecoding(d) ? [true] : []) - .join('path') - .attr('class', 'spec-decode-marker') - .attr('d', SPEC_DECODE_MARKER_PATH) - .attr('fill', 'none') - .attr('stroke', 'var(--foreground)') - .attr('stroke-width', SPEC_DECODE_MARKER_STROKE_WIDTH) - .attr('stroke-dasharray', SPEC_DECODE_MARKER_DASHARRAY) - .attr('stroke-linecap', 'round') - .attr('pointer-events', 'none'); }); }} legendElement={ @@ -1051,10 +1033,10 @@ const GPUGraph = React.memo( ]} precisionIndicators={selectedPrecisions} keyIndicators={ - hasOffloadHalo || hasSpecDecodeMarker ? ( + hasOffloadHalo || selectedSequence === Sequence.AgenticTraces ? ( <> {hasOffloadHalo && } - {hasSpecDecodeMarker && } + {selectedSequence === Sequence.AgenticTraces && } ) : undefined } diff --git a/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx b/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx index 0a6ba6a5c..57320c91c 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.decoration.test.tsx @@ -236,7 +236,7 @@ describe('ScatterGraph toggle decoration', () => { unmount(); }); - it('composes agentic spec decoding and KV-offload point decorations', () => { + it('keeps speculative decoding out of point decorations and shows only KV offload', () => { const standard = { ...point('h100', 'fp8', 1, 1, 1), benchmark_type: 'agentic_traces', @@ -262,19 +262,21 @@ describe('ScatterGraph toggle decoration', () => { offload_mode: 'off', } as InferenceData; + inferenceState.current = { + ...baseInferenceState(), + selectedSequence: 'agentic-traces', + }; const { container, unmount } = mountChart({ data: [standard, mtp, mtpWithOffload, fixedMtp], }); const groups = dotGroups(container); - expect(groups[0].querySelector('.spec-decode-marker')).toBeNull(); - expect(groups[1].querySelector('.spec-decode-marker')).not.toBeNull(); + expect(container.querySelector('.spec-decode-marker')).toBeNull(); expect(groups[1].querySelector('.offload-halo')).toBeNull(); - expect(groups[2].querySelector('.spec-decode-marker')).not.toBeNull(); expect(groups[2].querySelector('.offload-halo')).not.toBeNull(); - expect(groups[3].querySelector('.spec-decode-marker')).toBeNull(); - expect(container.querySelector('[data-testid="spec-decode-marker-key"]')).not.toBeNull(); + expect(container.querySelector('[data-testid="spec-decode-marker-key"]')).toBeNull(); expect(container.querySelector('[data-testid="offload-halo-key"]')).not.toBeNull(); + expect(container.querySelector('[data-testid="agentic-optimization-note"]')).not.toBeNull(); unmount(); }); @@ -351,28 +353,6 @@ describe('ScatterGraph toggle decoration', () => { unmount(); }); - it('keeps the spec marker above a point after a precision-shape swap', () => { - const agenticFp4 = { - ...point('h100', 'fp4', 40, 400, 4), - benchmark_type: 'agentic_traces', - spec_decoding: 'mtp', - } as InferenceData; - const { container, rerender, unmount } = mountChart({ - data: [POINTS[0], agenticFp4, POINTS[1]], - }); - - inferenceState.current = { - ...inferenceState.current, - selectedPrecisions: ['fp8', 'fp4'], - }; - rerender(); - - const fp4Dot = dotGroups(container, 'h100').find((d) => d.dataset.precision === 'fp4')!; - expect(fp4Dot.querySelector('.visible-shape')!.tagName.toLowerCase()).toBe('rect'); - expect(fp4Dot.lastElementChild).toBe(fp4Dot.querySelector('.spec-decode-marker')); - unmount(); - }); - it('rebuilds when the scale domain actually changes (data refresh path intact)', () => { const { rerender, unmount } = mountChart(); const buildsAfterMount = rebuildCount(); @@ -462,7 +442,7 @@ describe('ScatterGraph toggle decoration', () => { unmount(); }); - it('composes spec decoding and offload decorations on unofficial-run points', () => { + it('keeps speculative decoding out of unofficial-run point decorations', () => { const runUrl = 'https://github.com/o/r/actions/runs/123'; const overlayPoints = [ { @@ -497,7 +477,7 @@ describe('ScatterGraph toggle decoration', () => { }); const groups = [...container.querySelectorAll('.unofficial-overlay-pt')]; - expect(groups[0].querySelector('.spec-decode-marker')).not.toBeNull(); + expect(groups[0].querySelector('.spec-decode-marker')).toBeNull(); expect(groups[0].querySelector('.offload-halo')).not.toBeNull(); expect(groups[1].querySelector('.spec-decode-marker')).toBeNull(); expect(groups[1].querySelector('.offload-halo')).toBeNull(); diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index 3a9e8e963..063d007c2 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -77,13 +77,7 @@ import { OFFLOAD_HALO_STROKE_WIDTH, OffloadHaloLegendKey, } from '@/components/inference/ui/OffloadHaloLegendKey'; -import { - SPEC_DECODE_MARKER_DASHARRAY, - SPEC_DECODE_MARKER_PATH, - SPEC_DECODE_MARKER_STROKE_WIDTH, - SpecDecodeLegendKey, - hasAgenticSpecDecoding, -} from '@/components/inference/ui/SpecDecodeLegendKey'; +import { AgenticOptimizationNote } from '@/components/inference/ui/AgenticOptimizationNote'; import { buildLegendPointsRows } from '@/components/inference/utils/legend-points-table'; import { type ParetoPointLabel, @@ -207,8 +201,8 @@ const getXPath = (size: number) => { return `M ${-s} ${-s} L ${s} ${s} M ${s} ${-s} L ${-s} ${s}`; }; -/** Apply the two independent point decorations; either, both, or neither may render. */ -function renderPointDecorations( +/** Render the KV-offload halo around a point when applicable. */ +function renderOffloadHalo( group: d3.Selection, point: InferenceData, stroke: string, @@ -225,18 +219,6 @@ function renderPointDecorations( .attr('stroke-dasharray', OFFLOAD_HALO_DASHARRAY) .attr('opacity', 0.9) .attr('pointer-events', 'none'); - group - .selectAll('.spec-decode-marker') - .data(hasAgenticSpecDecoding(point) ? [true] : []) - .join('path') - .attr('class', 'spec-decode-marker') - .attr('d', SPEC_DECODE_MARKER_PATH) - .attr('fill', 'none') - .attr('stroke', stroke) - .attr('stroke-width', SPEC_DECODE_MARKER_STROKE_WIDTH) - .attr('stroke-dasharray', SPEC_DECODE_MARKER_DASHARRAY) - .attr('stroke-linecap', 'round') - .attr('pointer-events', 'none'); } const formatChangelogDescription = (desc: string | string[]): React.JSX.Element => { @@ -1093,13 +1075,6 @@ const ScatterGraph = React.memo( processedOverlayData.some((point) => point.offload_mode === 'on'), [pointsData, processedOverlayData], ); - const hasSpecDecodeMarker = useMemo( - () => - pointsData.some(hasAgenticSpecDecoding) || - processedOverlayData.some(hasAgenticSpecDecoding), - [pointsData, processedOverlayData], - ); - // Bulk presence lookup for agentic points: which ids have a stored // trace_replay blob → controls the "View charts" button in the pinned // tooltip. We deliberately don't fetch the histograms themselves here; @@ -2430,11 +2405,10 @@ const ScatterGraph = React.memo( overlayRunColor(overlayRunIndex(d.run_url ?? null, runIndexByUrl)), ); - // Point decorations compose independently: offload adds the - // dashed halo, while agentic speculative decoding adds a dashed - // plus. A point using both displays both markers. + // Match official points: KV offload is the only persistent + // point decoration. Decode method remains in the tooltip. overlayPoints.each(function (d) { - renderPointDecorations( + renderOffloadHalo( d3.select(this), d, overlayRunColor(overlayRunIndex(d.run_url ?? null, runIndexByUrl)), @@ -2916,7 +2890,7 @@ const ScatterGraph = React.memo( const layersRef = useRef(layers); layersRef.current = layers; - // --- onRender: CSS transitions, point decorations, and log tick formatting --- + // --- onRender: CSS transitions, offload halos, and log tick formatting --- const onRender = useCallback( (ctx: RenderContext) => { // Stash the render context for the decoration effect. @@ -2928,7 +2902,7 @@ const ScatterGraph = React.memo( // Offload halo: dashed ring on every point that used KV offload (Pareto or not) zoomGroup.selectAll('.dot-group').each(function (d) { - renderPointDecorations(d3.select(this), d, 'var(--foreground)'); + renderOffloadHalo(d3.select(this), d, 'var(--foreground)'); }); avoidLabelCollisions(zoomGroup); @@ -2996,8 +2970,8 @@ const ScatterGraph = React.memo( color, ); // A precision toggle may replace and append the visible SVG shape. - // Keep decorations above that shape after the swap. - sel.selectAll('.offload-halo, .spec-decode-marker').raise(); + // Keep the offload halo above that shape after the swap. + sel.selectAll('.offload-halo').raise(); }); // Overlay X markers: Optimal Only visibility (mirrors the official dot @@ -3473,10 +3447,10 @@ const ScatterGraph = React.memo( } precisionIndicators={selectedPrecisions} keyIndicators={ - hasOffloadHalo || hasSpecDecodeMarker ? ( + hasOffloadHalo || selectedSequence === Sequence.AgenticTraces ? ( <> {hasOffloadHalo && } - {hasSpecDecodeMarker && } + {selectedSequence === Sequence.AgenticTraces && } ) : undefined } diff --git a/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx b/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx deleted file mode 100644 index 793eccf31..000000000 --- a/packages/app/src/components/inference/ui/SpecDecodeLegendKey.tsx +++ /dev/null @@ -1,60 +0,0 @@ -import { POINT_SIZE } from '@/lib/chart-rendering'; -import { useLocale } from '@/lib/use-locale'; -import { OFFLOAD_HALO_RADIUS } from '@/components/inference/ui/OffloadHaloLegendKey'; - -// Match the plus diameter to the KV-offload halo so combined markers share -// one clean outer boundary. -export const SPEC_DECODE_MARKER_SIZE = OFFLOAD_HALO_RADIUS; -export const SPEC_DECODE_MARKER_STROKE_WIDTH = 1.5; -export const SPEC_DECODE_MARKER_DASHARRAY = '2 1.5'; -export const SPEC_DECODE_MARKER_PATH = `M ${-SPEC_DECODE_MARKER_SIZE} 0 H ${SPEC_DECODE_MARKER_SIZE} M 0 ${-SPEC_DECODE_MARKER_SIZE} V ${SPEC_DECODE_MARKER_SIZE}`; - -interface SpecDecodePoint { - benchmark_type?: string | null; - spec_decoding?: string | null; -} - -/** True only for agentic points that actually use a speculative decode method. */ -export function hasAgenticSpecDecoding(point: SpecDecodePoint): boolean { - if (point.benchmark_type !== 'agentic_traces') return false; - const method = point.spec_decoding?.trim().toLowerCase(); - return Boolean(method && method !== 'none'); -} - -const STRINGS = { - en: 'Speculative decoding', - zh: '推测解码', -} as const; - -/** Legend key for the dashed plus drawn over agentic speculative-decoding points. */ -export function SpecDecodeLegendKey() { - const locale = useLocale(); - const label = STRINGS[locale]; - - return ( -
- - {label} -
- ); -} From 63708b881d727496ed4278539ec3c9669030e3f3 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 14:16:57 -0500 Subject: [PATCH 11/12] test: lock mixed agentic spec methods to one series --- .../inference/hooks/useChartData.test.ts | 76 +++++++++++++++++++ .../app/src/lib/benchmark-transform.test.ts | 68 ++++++++++++++++- 2 files changed, 141 insertions(+), 3 deletions(-) diff --git a/packages/app/src/components/inference/hooks/useChartData.test.ts b/packages/app/src/components/inference/hooks/useChartData.test.ts index 5a4e1a626..94c9075d3 100644 --- a/packages/app/src/components/inference/hooks/useChartData.test.ts +++ b/packages/app/src/components/inference/hooks/useChartData.test.ts @@ -137,6 +137,38 @@ describe('dedupeRowsToLatestPerConfig', () => { expect(dedupeRowsToLatestPerConfig(rows).map((r) => r.id)).toEqual([3]); }); + + it('keeps every spec method produced by the winning agentic workflow run', () => { + const rows = [ + drow({ + id: 1, + benchmark_type: 'agentic_traces', + spec_method: 'none', + workflow_run_id: 12, + run_started_at: '2026-06-03T12:00:00Z', + }), + drow({ + id: 2, + benchmark_type: 'agentic_traces', + spec_method: 'mtp', + workflow_run_id: 12, + run_started_at: '2026-06-03T12:00:00Z', + }), + drow({ + id: 3, + benchmark_type: 'agentic_traces', + spec_method: 'eagle', + workflow_run_id: 12, + run_started_at: '2026-06-03T12:00:00Z', + }), + ]; + + expect(dedupeRowsToLatestPerConfig(rows).map((r) => r.spec_method)).toEqual([ + 'none', + 'mtp', + 'eagle', + ]); + }); }); describe('dedupeAgenticHistoryRuns', () => { @@ -168,6 +200,50 @@ describe('dedupeAgenticHistoryRuns', () => { expect(dedupeAgenticHistoryRuns(rows).map((row) => row.id)).toEqual([2, 3]); }); + + it('keeps mixed spec points together in the winning workflow on every date', () => { + const rows = [ + drow({ + id: 1, + benchmark_type: 'agentic_traces', + spec_method: 'none', + workflow_run_id: 30, + run_started_at: '2026-06-01T10:00:00Z', + }), + drow({ + id: 2, + benchmark_type: 'agentic_traces', + spec_method: 'mtp', + workflow_run_id: 30, + run_started_at: '2026-06-01T10:00:00Z', + }), + drow({ + id: 3, + benchmark_type: 'agentic_traces', + spec_method: 'eagle', + date: '2026-06-02', + workflow_run_id: 31, + run_started_at: '2026-06-02T10:00:00Z', + }), + drow({ + id: 4, + benchmark_type: 'agentic_traces', + spec_method: 'none', + date: '2026-06-02', + workflow_run_id: 31, + run_started_at: '2026-06-02T10:00:00Z', + }), + ]; + + expect( + dedupeAgenticHistoryRuns(rows).map((row) => [row.date, row.workflow_run_id, row.spec_method]), + ).toEqual([ + ['2026-06-01', 30, 'none'], + ['2026-06-01', 30, 'mtp'], + ['2026-06-02', 31, 'eagle'], + ['2026-06-02', 31, 'none'], + ]); + }); }); describe('buildComparisonDates', () => { diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index 0248d7958..35f1eb7ef 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, vi } from 'vitest'; import { getPointLabel } from '@/components/inference/utils/tooltipUtils'; import type { BenchmarkRow } from '@/lib/api'; +import { dedupeAgenticHistoryRuns } from '@/lib/benchmark-run-selection'; import { mergeRunScopedRows, @@ -739,7 +740,7 @@ describe('transformBenchmarkRows — hardware key resolution', () => { expect(hardwareConfig).toHaveProperty('h200_trt_mtp'); }); - it('groups mixed agentic spec methods into one hardware series', () => { + it('groups none, MTP, and EAGLE agentic points into one hardware series', () => { const rows = [ makeRow({ benchmark_type: 'agentic_traces', @@ -753,12 +754,73 @@ describe('transformBenchmarkRows — hardware key resolution', () => { framework: 'trt', spec_method: 'mtp', }), + makeRow({ + benchmark_type: 'agentic_traces', + hardware: 'h200', + framework: 'trt', + spec_method: 'eagle', + }), ]; const { chartData, hardwareConfig } = transformBenchmarkRows(rows); expect(Object.keys(hardwareConfig)).toEqual(['h200_trt']); - expect(chartData[0].map((point) => point.hwKey)).toEqual(['h200_trt', 'h200_trt']); - expect(chartData[0].map((point) => point.spec_decoding)).toEqual(['none', 'mtp']); + expect(chartData[0].map((point) => point.hwKey)).toEqual(['h200_trt', 'h200_trt', 'h200_trt']); + expect(chartData[0].map((point) => point.spec_decoding)).toEqual(['none', 'mtp', 'eagle']); + }); + + it('keeps one mixed-spec agentic series identity across comparison dates', () => { + const rows = dedupeAgenticHistoryRuns([ + makeRow({ + id: 1, + benchmark_type: 'agentic_traces', + spec_method: 'none', + conc: 8, + date: '2026-06-01', + workflow_run_id: 30, + run_started_at: '2026-06-01T10:00:00Z', + }), + makeRow({ + id: 2, + benchmark_type: 'agentic_traces', + spec_method: 'mtp', + conc: 16, + date: '2026-06-01', + workflow_run_id: 30, + run_started_at: '2026-06-01T10:00:00Z', + }), + makeRow({ + id: 3, + benchmark_type: 'agentic_traces', + spec_method: 'eagle', + conc: 32, + date: '2026-06-02', + workflow_run_id: 31, + run_started_at: '2026-06-02T10:00:00Z', + }), + makeRow({ + id: 4, + benchmark_type: 'agentic_traces', + spec_method: 'none', + conc: 64, + date: '2026-06-02', + workflow_run_id: 31, + run_started_at: '2026-06-02T10:00:00Z', + }), + ]); + + const { chartData, hardwareConfig } = transformBenchmarkRows(rows); + const pointsByDate = Object.groupBy(chartData[0], (point) => point.date); + + expect(Object.keys(hardwareConfig)).toEqual(['h200_trt']); + expect(pointsByDate['2026-06-01']?.map((point) => point.spec_decoding)).toEqual([ + 'none', + 'mtp', + ]); + expect(pointsByDate['2026-06-02']?.map((point) => point.spec_decoding)).toEqual([ + 'eagle', + 'none', + ]); + expect(new Set(chartData[0].map((point) => point.hwKey))).toEqual(new Set(['h200_trt'])); }); it('handles AMD hardware with vllm framework', () => { From 40c9d6721e2d01909a082ae180ab062ac6f27597 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 14:22:17 -0500 Subject: [PATCH 12/12] test: align offload legend assertions --- packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index 9fe0f70ce..63453aa3e 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -174,7 +174,6 @@ describe('X-Axis Mode Toggle (inference chart)', () => { it('explains the offload halo in the legend and distinguishes it from plain points', () => { cy.get('#chart-0 [data-testid="offload-halo-key"]') .should('be.visible') - .and('contain.text', 'Dashed halo:') .and('contain.text', 'KV offload ON'); cy.get('#chart-0 .offload-halo').should('have.length.at.least', 1); cy.get('#chart-0 .dot-group').then(($points) => { @@ -513,7 +512,6 @@ describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () ); cy.get('#chart-0 [data-testid="offload-halo-key"]') .should('be.visible') - .and('contain.text', 'Dashed halo:') .and('contain.text', 'KV offload ON'); });