From 3c5d6c63ff1390bb9555ced47fc9087f6a76e04f Mon Sep 17 00:00:00 2001 From: krisztianfekete Date: Tue, 30 Jun 2026 17:18:17 +0200 Subject: [PATCH 1/3] add initial Runs dashboard --- src/agentevals/api/streaming_routes.py | 95 +++-- src/agentevals/run/result_builder.py | 47 ++- ui/src/App.tsx | 2 + ui/src/api/client.ts | 33 ++ ui/src/components/runs/PassRateTrendChart.tsx | 121 ++++++ .../components/runs/PerMetricTrendChart.tsx | 123 +++++++ ui/src/components/runs/RunsHistoryTable.tsx | 238 ++++++++++++ ui/src/components/runs/RunsView.tsx | 348 ++++++++++++++++++ ui/src/components/runs/runHistory.ts | 135 +++++++ ui/src/components/sidebar/Sidebar.tsx | 13 +- ui/src/config.ts | 1 + ui/src/lib/types.ts | 51 ++- 12 files changed, 1174 insertions(+), 33 deletions(-) create mode 100644 ui/src/components/runs/PassRateTrendChart.tsx create mode 100644 ui/src/components/runs/PerMetricTrendChart.tsx create mode 100644 ui/src/components/runs/RunsHistoryTable.tsx create mode 100644 ui/src/components/runs/RunsView.tsx create mode 100644 ui/src/components/runs/runHistory.ts diff --git a/src/agentevals/api/streaming_routes.py b/src/agentevals/api/streaming_routes.py index b6a3872..d432a83 100644 --- a/src/agentevals/api/streaming_routes.py +++ b/src/agentevals/api/streaming_routes.py @@ -7,14 +7,14 @@ import logging from typing import TYPE_CHECKING -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import FileResponse from pydantic import BaseModel, ConfigDict, Field -from ..config import BuiltinMetricDef, EvalRunConfig, EvaluatorDef +from ..config import BuiltinMetricDef, EvalParams, EvalRunConfig, EvaluatorDef from ..converter import convert_traces from ..loader.otlp import OtlpJsonLoader -from ..runner import run_evaluation +from ..runner import RunResult, run_evaluation from ..trace_attrs import OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_REQUEST_MODEL from ..utils.log_enrichment import enrich_spans_with_logs from .dependencies import require_trace_manager @@ -167,9 +167,47 @@ async def create_eval_set_from_session( return await _do_create_eval_set(request, manager) +async def _persist_sessions_run( + http_request: Request, + *, + evaluators: list[EvaluatorDef], + eval_set_dict: dict, + session_ids: list[str], + run_results: list[RunResult], + session_errors: list[str], +) -> None: + """Best-effort: persist a UI-driven session evaluation as a single Run row + (plus Result rows) when ``app.state.run_service`` is configured (postgres + backend), mirroring the offline ``/api/evaluate`` persistence. + + Every evaluated session's traces are aggregated into one run so the run + history counts one evaluation per "Evaluate" click, the same way one + offline upload of N traces produces one run. No-op (and never raises) on + the memory backend or on persistence failure, so the live eval results are + always returned to the caller regardless.""" + service = getattr(http_request.app.state, "run_service", None) + if service is None: + return + trace_results = [tr for rr in run_results for tr in rr.trace_results] + if not trace_results: + return + combined = RunResult(trace_results=trace_results, errors=session_errors) + try: + await service.record_eval_run( + params=EvalParams(evaluators=evaluators), + eval_set_dict=eval_set_dict, + trace_format="otlp-json", + upload_filenames=session_ids, + run_result=combined, + ) + except Exception: + logger.exception("failed to persist evaluate-sessions run; results still returned to caller") + + @streaming_router.post("/evaluate-sessions", response_model=StandardResponse[EvaluateSessionsData]) async def evaluate_sessions( request: EvaluateSessionsRequest, + http_request: Request, manager: StreamingTraceManager = Depends(require_trace_manager), ): """Evaluate all sessions against a golden session converted to EvalSet.""" @@ -200,7 +238,7 @@ async def evaluate_sessions( sem = asyncio.Semaphore(5) - async def eval_one_session(session_id: str, session) -> SessionEvalResult: + async def eval_one_session(session_id: str, session) -> tuple[SessionEvalResult, RunResult | None]: async with sem: try: trace_file = await manager._save_spans_to_temp_file(session) @@ -216,34 +254,47 @@ async def eval_one_session(session_id: str, session) -> SessionEvalResult: if eval_result.trace_results: trace_result = eval_result.trace_results[0] - return SessionEvalResult( - session_id=session_id, - trace_id=trace_result.trace_id, - num_invocations=trace_result.num_invocations, - metric_results=[ - { - "metricName": mr.metric_name, - "score": mr.score, - "evalStatus": mr.eval_status, - "error": mr.error, - "perInvocationScores": mr.per_invocation_scores, - "details": mr.details, - } - for mr in trace_result.metric_results - ], + return ( + SessionEvalResult( + session_id=session_id, + trace_id=trace_result.trace_id, + num_invocations=trace_result.num_invocations, + metric_results=[ + { + "metricName": mr.metric_name, + "score": mr.score, + "evalStatus": mr.eval_status, + "error": mr.error, + "perInvocationScores": mr.per_invocation_scores, + "details": mr.details, + } + for mr in trace_result.metric_results + ], + ), + eval_result, ) else: logger.warning("No trace results for session %s", session_id) - return SessionEvalResult(session_id=session_id, error="No trace results") + return SessionEvalResult(session_id=session_id, error="No trace results"), None except Exception as exc: logger.error(f"Failed to evaluate session {session_id}: {exc}", exc_info=True) - return SessionEvalResult(session_id=session_id, error=str(exc)) + return SessionEvalResult(session_id=session_id, error=str(exc)), None - results = await asyncio.gather(*[eval_one_session(sid, sess) for sid, sess in sessions_to_evaluate]) + evaluated = await asyncio.gather(*[eval_one_session(sid, sess) for sid, sess in sessions_to_evaluate]) + results = [session_result for session_result, _ in evaluated] logger.info("Evaluation complete. Total results: %d", len(results)) + await _persist_sessions_run( + http_request, + evaluators=request.evaluators, + eval_set_dict=eval_set_response.data.eval_set, + session_ids=[sid for sid, _ in sessions_to_evaluate], + run_results=[run_result for _, run_result in evaluated if run_result is not None], + session_errors=[f"{r.session_id}: {r.error}" for r in results if r.error], + ) + return StandardResponse( data=EvaluateSessionsData( golden_session_id=request.golden_session_id, diff --git a/src/agentevals/run/result_builder.py b/src/agentevals/run/result_builder.py index 8c18a03..e798a84 100644 --- a/src/agentevals/run/result_builder.py +++ b/src/agentevals/run/result_builder.py @@ -99,27 +99,58 @@ def build_results(run_id: UUID, params: EvalParams, run_result: RunResult) -> li return out +def _status_key(mr: MetricResult) -> str: + """Map a :class:`MetricResult` onto a :class:`ResultStatus` string, + matching :func:`result_from_metric_result` so the summary counts and the + persisted ``result`` rows never disagree.""" + if mr.error: + return "errored" + status = (mr.eval_status or "").upper() + if status == "PASSED": + return "passed" + if status == "FAILED": + return "failed" + return "skipped" + + def summarize_run_result(run_result: RunResult) -> dict[str, Any]: """Summary blob persisted alongside the run row. Counts mirror :class:`agentevals.storage.models.ResultStatus` values so a caller polling ``GET /api/runs/{id}`` can compute pass/fail rates without fetching the full result list. + + ``per_metric`` aggregates the same statuses by ``evaluator_name`` plus a + mean ``score``, so the run-history dashboard can chart per-metric trends + across runs from the list response alone, without an N+1 over + ``/api/runs/{id}/results``. ``avg_score`` is ``None`` when no invocation of + that evaluator produced a numeric score (e.g. it only errored). """ counts = {"passed": 0, "failed": 0, "errored": 0, "skipped": 0} + per_metric: dict[str, dict[str, Any]] = {} for tr in run_result.trace_results: for mr in tr.metric_results: - if mr.error: - counts["errored"] += 1 - elif (mr.eval_status or "").upper() == "PASSED": - counts["passed"] += 1 - elif (mr.eval_status or "").upper() == "FAILED": - counts["failed"] += 1 - else: - counts["skipped"] += 1 + key = _status_key(mr) + counts[key] += 1 + + metric = per_metric.setdefault( + mr.metric_name, + {"passed": 0, "failed": 0, "errored": 0, "skipped": 0, "_score_sum": 0.0, "_score_count": 0}, + ) + metric[key] += 1 + if mr.score is not None: + metric["_score_sum"] += mr.score + metric["_score_count"] += 1 + + for metric in per_metric.values(): + score_count = metric.pop("_score_count") + score_sum = metric.pop("_score_sum") + metric["avg_score"] = (score_sum / score_count) if score_count else None + return { "trace_count": len(run_result.trace_results), "result_counts": counts, + "per_metric": per_metric, "errors": list(run_result.errors), "performance_metrics": run_result.performance_metrics, } diff --git a/ui/src/App.tsx b/ui/src/App.tsx index dfffdbe..d7d66bd 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -7,6 +7,7 @@ import { InspectorView } from './components/inspector/InspectorView'; import { BuilderView } from './components/builder/BuilderView'; import { LiveStreamingView } from './components/streaming/LiveStreamingView'; import { AnnotationQueueView } from './components/annotation-queue/AnnotationQueueView'; +import { RunsView } from './components/runs/RunsView'; import { Sidebar } from './components/sidebar/Sidebar'; function AppContent() { @@ -24,6 +25,7 @@ function AppContent() { {state.currentView === 'builder' && } {state.currentView === 'streaming' && } {state.currentView === 'annotation-queue' && } + {state.currentView === 'runs' && } {state.currentView === 'comparison' && (
Comparison view coming soon... diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index feb20f6..549b16f 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -5,9 +5,20 @@ import type { MetricMetadata, StandardResponse, ConvertTracesResponse, + Run, + RunStatus, } from '../lib/types'; import { config } from '../config'; +export class StorageUnavailableError extends Error { + constructor() { + super( + 'Run history requires the durable storage backend. Start the server with AGENTEVALS_STORAGE_BACKEND=postgres and AGENTEVALS_DATABASE_URL set.', + ); + this.name = 'StorageUnavailableError'; + } +} + const API_BASE_URL = `${config.api.baseUrl}/api`; async function unwrap(response: Response): Promise { @@ -186,6 +197,28 @@ export async function evaluateTracesStreaming( } } +export async function listRuns(options?: { + status?: RunStatus[]; + limit?: number; + before?: string; +}): Promise { + const params = new URLSearchParams(); + (options?.status ?? []).forEach(s => params.append('status', s)); + if (options?.limit != null) params.set('limit', String(options.limit)); + if (options?.before) params.set('before', options.before); + const query = params.toString(); + + const response = await fetch(`${config.api.endpoints.runs}${query ? `?${query}` : ''}`); + + if (response.status === 503) { + throw new StorageUnavailableError(); + } + if (!response.ok) { + throw new Error(`Failed to list runs: ${response.statusText}`); + } + return unwrap(response); +} + export async function listMetrics(): Promise { try { const response = await fetch(`${API_BASE_URL}/metrics`); diff --git a/ui/src/components/runs/PassRateTrendChart.tsx b/ui/src/components/runs/PassRateTrendChart.tsx new file mode 100644 index 0000000..4cd93c4 --- /dev/null +++ b/ui/src/components/runs/PassRateTrendChart.tsx @@ -0,0 +1,121 @@ +import React from 'react'; +import { css } from '@emotion/react'; +import { Line } from 'react-chartjs-2'; +import { + Chart as ChartJS, + CategoryScale, + LinearScale, + PointElement, + LineElement, + Filler, + Tooltip, + Legend, +} from 'chart.js'; +import type { Run } from '../../lib/types'; +import { CHART_COLORS, formatTimestamp, passRate } from './runHistory'; + +ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Filler, Tooltip, Legend); + +interface PassRateTrendChartProps { + runs: Run[]; +} + +export const PassRateTrendChart: React.FC = ({ runs }) => { + const points = runs + .map(run => ({ run, rate: passRate(run) })) + .filter((p): p is { run: Run; rate: number } => p.rate !== null); + + if (points.length === 0) { + return ( +
+

Pass rate over time

+

No decided (pass/fail) results yet in this eval.

+
+ ); + } + + const data = { + labels: points.map(p => formatTimestamp(p.run.createdAt)), + datasets: [ + { + label: 'Pass rate', + data: points.map(p => Math.round(p.rate * 1000) / 10), + borderColor: CHART_COLORS.passRate, + backgroundColor: CHART_COLORS.passRateFill, + fill: true, + tension: 0.25, + pointRadius: 3, + pointHoverRadius: 5, + borderWidth: 2, + }, + ], + }; + + const options = { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { display: false }, + tooltip: { + backgroundColor: 'rgba(0, 0, 0, 0.9)', + titleColor: '#fff', + bodyColor: '#fff', + padding: 12, + cornerRadius: 6, + callbacks: { + label: (ctx: { parsed: { y: number } }) => `Pass rate: ${ctx.parsed.y}%`, + }, + }, + }, + scales: { + y: { + min: 0, + max: 100, + ticks: { + color: CHART_COLORS.text, + font: { size: 12 }, + callback: (value: string | number) => `${value}%`, + }, + grid: { color: CHART_COLORS.grid }, + }, + x: { + ticks: { color: CHART_COLORS.text, font: { size: 11 }, maxRotation: 0, autoSkip: true }, + grid: { color: CHART_COLORS.grid }, + }, + }, + }; + + return ( +
+

Pass rate over time

+
+ +
+
+ ); +}; + +const cardStyle = css` + background: var(--bg-surface); + border: 1px solid var(--border-default); + border-radius: 8px; + padding: 20px; + + h3 { + margin: 0 0 16px 0; + font-size: 1rem; + font-weight: 600; + color: var(--text-primary); + } +`; + +const chartWrapperStyle = css` + height: 300px; + position: relative; +`; + +const emptyStyle = css` + color: var(--text-tertiary); + font-size: 0.875rem; + margin: 0; +`; diff --git a/ui/src/components/runs/PerMetricTrendChart.tsx b/ui/src/components/runs/PerMetricTrendChart.tsx new file mode 100644 index 0000000..28e32c0 --- /dev/null +++ b/ui/src/components/runs/PerMetricTrendChart.tsx @@ -0,0 +1,123 @@ +import React from 'react'; +import { css } from '@emotion/react'; +import { Line } from 'react-chartjs-2'; +import { + Chart as ChartJS, + CategoryScale, + LinearScale, + PointElement, + LineElement, + Tooltip, + Legend, +} from 'chart.js'; +import type { Run } from '../../lib/types'; +import { CHART_COLORS, METRIC_PALETTE, formatTimestamp, metricNamesAcross } from './runHistory'; + +ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Tooltip, Legend); + +interface PerMetricTrendChartProps { + runs: Run[]; +} + +export const PerMetricTrendChart: React.FC = ({ runs }) => { + const metricNames = metricNamesAcross(runs); + + if (metricNames.length === 0) { + return ( +
+

Per-metric score trend

+

+ No per-metric scores recorded. Runs created before this feature won't have them. +

+
+ ); + } + + const labels = runs.map(run => formatTimestamp(run.createdAt)); + + const datasets = metricNames.map((name, index) => { + const color = METRIC_PALETTE[index % METRIC_PALETTE.length]; + return { + label: name, + // null where this metric is absent or produced no numeric score in a run, + // so the line breaks instead of implying a real drop to zero. + data: runs.map(run => { + const score = run.summary?.per_metric?.[name]?.avg_score; + return score == null ? null : Math.round(score * 1000) / 1000; + }), + borderColor: color, + backgroundColor: color, + spanGaps: false, + tension: 0.25, + pointRadius: 3, + pointHoverRadius: 5, + borderWidth: 2, + }; + }); + + const options = { + responsive: true, + maintainAspectRatio: false, + interaction: { mode: 'index' as const, intersect: false }, + plugins: { + legend: { + position: 'bottom' as const, + labels: { color: CHART_COLORS.text, font: { size: 12 }, padding: 14, usePointStyle: true }, + }, + tooltip: { + backgroundColor: 'rgba(0, 0, 0, 0.9)', + titleColor: '#fff', + bodyColor: '#fff', + padding: 12, + cornerRadius: 6, + }, + }, + scales: { + y: { + min: 0, + max: 1, + ticks: { color: CHART_COLORS.text, font: { size: 12 } }, + grid: { color: CHART_COLORS.grid }, + title: { display: true, text: 'Avg score', color: CHART_COLORS.text, font: { size: 12 } }, + }, + x: { + ticks: { color: CHART_COLORS.text, font: { size: 11 }, maxRotation: 0, autoSkip: true }, + grid: { color: CHART_COLORS.grid }, + }, + }, + }; + + return ( +
+

Per-metric score trend

+
+ +
+
+ ); +}; + +const cardStyle = css` + background: var(--bg-surface); + border: 1px solid var(--border-default); + border-radius: 8px; + padding: 20px; + + h3 { + margin: 0 0 16px 0; + font-size: 1rem; + font-weight: 600; + color: var(--text-primary); + } +`; + +const chartWrapperStyle = css` + height: 300px; + position: relative; +`; + +const emptyStyle = css` + color: var(--text-tertiary); + font-size: 0.875rem; + margin: 0; +`; diff --git a/ui/src/components/runs/RunsHistoryTable.tsx b/ui/src/components/runs/RunsHistoryTable.tsx new file mode 100644 index 0000000..1af2874 --- /dev/null +++ b/ui/src/components/runs/RunsHistoryTable.tsx @@ -0,0 +1,238 @@ +import React from 'react'; +import { css } from '@emotion/react'; +import { Table } from 'antd'; +import type { ColumnsType } from 'antd/es/table'; +import type { Run } from '../../lib/types'; +import { + STATUS_COLORS, + evalSetGroup, + formatDuration, + formatTimestamp, + passRate, + runDurationMs, +} from './runHistory'; + +interface RunsHistoryTableProps { + runs: Run[]; +} + +export const RunsHistoryTable: React.FC = ({ runs }) => { + const rows = [...runs].sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt)); + + const columns: ColumnsType = [ + { + title: 'When', + dataIndex: 'createdAt', + key: 'when', + render: (value: string) => {formatTimestamp(value)}, + }, + { + title: 'Status', + dataIndex: 'status', + key: 'status', + render: (status: Run['status']) => ( + + + {status} + + ), + }, + { + title: 'Eval set', + key: 'evalSet', + render: (_: unknown, run: Run) => ( + {evalSetGroup(run).label} + ), + }, + { + title: 'Traces', + key: 'traces', + align: 'right', + render: (_: unknown, run: Run) => ( + {run.summary?.trace_count ?? '-'} + ), + }, + { + title: 'Results', + key: 'results', + render: (_: unknown, run: Run) => { + const counts = run.summary?.result_counts; + if (!counts) return -; + return ( + + {counts.passed}P + {counts.failed}F + {counts.errored > 0 && ( + {counts.errored}E + )} + {counts.skipped > 0 && ( + {counts.skipped}S + )} + + ); + }, + }, + { + title: 'Pass rate', + key: 'passRate', + render: (_: unknown, run: Run) => { + const rate = passRate(run); + if (rate === null) return -; + const pct = Math.round(rate * 100); + return ( +
+
+
= 0.5 ? 'var(--status-success)' : 'var(--status-failure)', + }} + /> +
+ {pct}% +
+ ); + }, + }, + { + title: 'Duration', + key: 'duration', + align: 'right', + render: (_: unknown, run: Run) => ( + {formatDuration(runDurationMs(run))} + ), + }, + { + title: 'Models', + key: 'models', + render: (_: unknown, run: Run) => { + const models = run.summary?.performance_metrics?.models; + if (!Array.isArray(models) || models.length === 0) return -; + return {models.join(', ')}; + }, + }, + ]; + + return ( +
+ + rowKey="runId" + columns={columns} + dataSource={rows} + pagination={{ pageSize: 10, hideOnSinglePage: true }} + size="small" + /> +
+ ); +}; + +const tableStyle = css` + .ant-table { + background: var(--bg-surface); + border: 1px solid var(--border-default); + border-radius: 8px; + font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif; + } + + .ant-table-thead > tr > th { + background: var(--bg-elevated); + color: var(--text-primary); + font-weight: 600; + border-bottom: 2px solid var(--border-default); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.5px; + padding: 12px 16px; + } + + .ant-table-tbody > tr > td { + background: var(--bg-surface); + color: var(--text-secondary); + border-bottom: 1px solid var(--border-subtle); + padding: 12px 16px; + } + + .ant-table-tbody > tr:hover > td { + background: var(--bg-elevated); + } + + .ant-pagination .ant-pagination-item a, + .ant-pagination .ant-pagination-item-link { + color: var(--text-secondary); + } + + .ant-pagination .ant-pagination-item-active { + background: var(--bg-elevated); + border-color: var(--accent-primary); + } + + .ant-pagination .ant-pagination-item-active a { + color: var(--accent-primary); + } +`; + +const monoStyle = css` + font-family: var(--font-mono); + font-size: 0.8125rem; + color: var(--text-secondary); +`; + +const statusStyle = css` + display: inline-flex; + align-items: center; + gap: 6px; + text-transform: capitalize; + font-size: 0.8125rem; + color: var(--text-primary); +`; + +const dotStyle = css` + width: 8px; + height: 8px; + border-radius: 50%; + display: inline-block; +`; + +const evalSetCellStyle = css` + color: var(--text-primary); + font-size: 0.8125rem; +`; + +const countsStyle = css` + display: inline-flex; + gap: 8px; + font-family: var(--font-mono); + font-size: 0.8125rem; + font-weight: 600; +`; + +const barCellStyle = css` + display: flex; + align-items: center; + gap: 8px; +`; + +const barTrackStyle = css` + width: 64px; + height: 6px; + border-radius: 3px; + background: var(--bg-elevated); + overflow: hidden; +`; + +const barFillStyle = css` + height: 100%; + border-radius: 3px; + transition: width 0.3s ease; +`; + +const mutedStyle = css` + color: var(--text-tertiary); +`; + +const modelsStyle = css` + font-family: var(--font-mono); + font-size: 0.75rem; + color: var(--text-tertiary); +`; diff --git a/ui/src/components/runs/RunsView.tsx b/ui/src/components/runs/RunsView.tsx new file mode 100644 index 0000000..5dd5f0f --- /dev/null +++ b/ui/src/components/runs/RunsView.tsx @@ -0,0 +1,348 @@ +import React, { useEffect, useMemo, useState } from 'react'; +import { css } from '@emotion/react'; +import { Select } from 'antd'; +import { RefreshCw, History, AlertCircle } from 'lucide-react'; +import type { Run } from '../../lib/types'; +import { listRuns, StorageUnavailableError } from '../../api/client'; +import { + groupRuns, + metricNamesAcross, + passRate, + sortByCreatedAsc, +} from './runHistory'; +import { PassRateTrendChart } from './PassRateTrendChart'; +import { PerMetricTrendChart } from './PerMetricTrendChart'; +import { RunsHistoryTable } from './RunsHistoryTable'; + +const ALL_GROUPS = '__all__'; + +export const RunsView: React.FC = () => { + const [runs, setRuns] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [storageUnavailable, setStorageUnavailable] = useState(false); + const [selectedGroup, setSelectedGroup] = useState(ALL_GROUPS); + + const load = async () => { + setLoading(true); + setError(null); + setStorageUnavailable(false); + try { + const fetched = await listRuns({ limit: 200 }); + setRuns(fetched); + } catch (err) { + if (err instanceof StorageUnavailableError) { + setStorageUnavailable(true); + } else { + setError(err instanceof Error ? err.message : 'Failed to load runs'); + } + } finally { + setLoading(false); + } + }; + + useEffect(() => { + load(); + }, []); + + const groups = useMemo(() => groupRuns(runs), [runs]); + + const selectedRuns = useMemo(() => { + if (selectedGroup === ALL_GROUPS) return runs; + return groups.find(g => g.group.key === selectedGroup)?.runs ?? []; + }, [runs, groups, selectedGroup]); + + const trendRuns = useMemo(() => sortByCreatedAsc(selectedRuns), [selectedRuns]); + + const stats = useMemo(() => { + const rates = trendRuns.map(passRate).filter((r): r is number => r !== null); + const latest = rates.length ? rates[rates.length - 1] : null; + const avg = rates.length ? rates.reduce((a, b) => a + b, 0) / rates.length : null; + return { + total: selectedRuns.length, + latest, + avg, + metrics: metricNamesAcross(selectedRuns).length, + }; + }, [selectedRuns, trendRuns]); + + return ( +
+
+
+ +

Run history

+
+
+ {groups.length > 0 && ( + { css={selectStyle} popupMatchSelectWidth={false} options={[ - { value: ALL_GROUPS, label: `All eval sets (${runs.length})` }, + { + value: ALL_GROUPS, + label: `All ${groupBy === 'agent' ? 'agents' : 'eval sets'} (${runs.length})`, + }, ...groups.map(g => ({ value: g.group.key, label: `${g.group.label} (${g.runs.length})`, @@ -145,7 +173,7 @@ export const RunsView: React.FC = () => {
- + )}
diff --git a/ui/src/components/runs/runHistory.ts b/ui/src/components/runs/runHistory.ts index ac66ed2..dcdaa2e 100644 --- a/ui/src/components/runs/runHistory.ts +++ b/ui/src/components/runs/runHistory.ts @@ -31,16 +31,35 @@ export function evalSetGroup(run: Run): RunGroup { return { key: 'ungrouped', label: 'Ungrouped runs' }; } +// Agent identity for grouping, derived from summary.agents (service.name, with +// the backend's agent_name fallback already applied). Cross-framework, so this +// is the dimension to use when comparing the same agent's runs over time. +export function agentGroup(run: Run): RunGroup { + const agents = run.summary?.agents; + if (Array.isArray(agents) && agents.length) { + return { key: `agent:${[...agents].sort().join('|')}`, label: agents.join(', ') }; + } + return { key: 'agent:unknown', label: 'Unknown agent' }; +} + +export type GroupBy = 'evalSet' | 'agent'; + +const GROUPERS: Record RunGroup> = { + evalSet: evalSetGroup, + agent: agentGroup, +}; + export interface GroupedRuns { group: RunGroup; runs: Run[]; } -// Returns groups sorted by run count desc, so the busiest eval surfaces first. -export function groupRuns(runs: Run[]): GroupedRuns[] { +// Returns groups sorted by run count desc, so the busiest group surfaces first. +export function groupRuns(runs: Run[], groupBy: GroupBy = 'evalSet'): GroupedRuns[] { + const grouper = GROUPERS[groupBy]; const byKey = new Map(); for (const run of runs) { - const group = evalSetGroup(run); + const group = grouper(run); const existing = byKey.get(group.key); if (existing) existing.runs.push(run); else byKey.set(group.key, { group, runs: [run] }); @@ -91,6 +110,23 @@ export function sortByCreatedAsc(runs: Run[]): Run[] { return [...runs].sort((a, b) => Date.parse(a.createdAt) - Date.parse(b.createdAt)); } +export function runAgents(run: Run): string[] { + const agents = run.summary?.agents; + return Array.isArray(agents) ? agents : []; +} + +// Union of agent identities (service.name) across runs, ordered by frequency so +// the most active agent leads the legend. Used to draw one trend line per agent. +export function agentNamesAcross(runs: Run[]): string[] { + const frequency = new Map(); + for (const run of runs) { + for (const agent of runAgents(run)) { + frequency.set(agent, (frequency.get(agent) ?? 0) + 1); + } + } + return [...frequency.entries()].sort((a, b) => b[1] - a[1]).map(([name]) => name); +} + // Union of evaluator names seen across a set of runs, ordered by how often they // appear so the most consistently-tracked metrics lead the legend. export function metricNamesAcross(runs: Run[]): string[] { diff --git a/ui/src/components/streaming/LiveStreamingView.tsx b/ui/src/components/streaming/LiveStreamingView.tsx index 85d229b..a51adac 100644 --- a/ui/src/components/streaming/LiveStreamingView.tsx +++ b/ui/src/components/streaming/LiveStreamingView.tsx @@ -441,7 +441,10 @@ export function LiveStreamingView() { if (import.meta.env.DEV) { console.log('Fetching traces for all sessions'); } - const sessionIds = Array.from(activeSessions.keys()); + // The golden session defines the eval set, so scoring it against itself + // is a trivial pass. Exclude it so only the agents being meaningfully + // evaluated against the golden are scored and surfaced. + const sessionIds = Array.from(activeSessions.keys()).filter(id => id !== selectedGoldenId); const traceFiles = await Promise.all( sessionIds.map(async (sessionId) => { diff --git a/ui/src/lib/types.ts b/ui/src/lib/types.ts index cdc35ee..49cd295 100644 --- a/ui/src/lib/types.ts +++ b/ui/src/lib/types.ts @@ -257,17 +257,61 @@ export interface RunSummary { trace_count: number; result_counts: ResultCounts; per_metric?: Record; + agents?: string[]; errors?: string[]; performance_metrics?: { models?: string[] } & Record; } // spec.evalSet/evalConfig are stored as raw dicts, so their inner keys keep // the original snake_case (e.g. ADK eval_set_id) rather than being camelCased. +export interface RunEvaluatorConfig { + name?: string; + type?: string; + threshold?: number; + judge_model?: string; + trajectory_match_type?: string; +} + export interface RunSpecSummary { approach?: string; evalSet?: { eval_set_id?: string; name?: string; eval_cases?: unknown[] } | null; - evalConfig?: { evaluators?: Array<{ name?: string; type?: string }> } & Record; - target?: { kind?: string } & Record; + evalConfig?: { evaluators?: RunEvaluatorConfig[] } & Record; + target?: { kind?: string; trace_count?: number; trace_files?: string[] } & Record; +} + +// One persisted Result row (GET /api/runs/{id}/results). Top-level fields are +// camelCased by the API; the free-form `details` keeps snake_case inner keys. +export type ResultStatus2 = 'passed' | 'failed' | 'errored' | 'skipped'; + +export interface PersistedComparison { + invocation_id?: string | null; + matched?: boolean; + expected?: ToolCallComparison[]; + actual?: ToolCallComparison[]; +} + +export interface ResultDetails { + comparisons?: PersistedComparison[]; + [key: string]: unknown; +} + +export interface RunResultRow { + resultId: string; + runId: string; + evalSetItemId: string; + evalSetItemName: string; + evaluatorName: string; + evaluatorType: string; + status: ResultStatus2; + score: number | null; + perInvocationScores: (number | null)[]; + traceId?: string | null; + spanId?: string | null; + details?: ResultDetails; + errorText?: string | null; + tokensUsed?: Record | null; + latencyMs?: number | null; + createdAt: string; } export interface Run { From e4307f2da253d141fa9fbf22d5e9f0f080c1ce3a Mon Sep 17 00:00:00 2001 From: krisztianfekete Date: Wed, 1 Jul 2026 11:54:24 +0200 Subject: [PATCH 3/3] make sure golden reference is shown on eval results page --- examples/dice_agent/agent.py | 2 +- .../zero-code-examples/pydantic-ai/run.py | 1 + ui/src/components/dashboard/DashboardView.tsx | 21 +++++- .../dashboard/PerformanceCharts.tsx | 12 ++- ui/src/components/dashboard/TraceTable.tsx | 75 ++++++++++++++----- .../streaming/LiveStreamingView.tsx | 11 ++- ui/src/context/TraceContext.tsx | 4 + ui/src/context/TraceProvider.tsx | 5 ++ 8 files changed, 103 insertions(+), 28 deletions(-) diff --git a/examples/dice_agent/agent.py b/examples/dice_agent/agent.py index c2551e1..38866b4 100644 --- a/examples/dice_agent/agent.py +++ b/examples/dice_agent/agent.py @@ -56,7 +56,7 @@ def is_prime(n: int) -> bool: dice_agent = Agent( name="dice_agent", # model="gemini-2.5-flash", - model="gemini-2.5-flash-lite", + model="gemini-3-flash-preview", instruction="""You are a helpful assistant that can roll dice and check if numbers are prime. When a user asks you to roll a die, use the roll_die tool with the appropriate number of sides. diff --git a/examples/zero-code-examples/pydantic-ai/run.py b/examples/zero-code-examples/pydantic-ai/run.py index cac2c1c..dfc1e3a 100644 --- a/examples/zero-code-examples/pydantic-ai/run.py +++ b/examples/zero-code-examples/pydantic-ai/run.py @@ -73,6 +73,7 @@ def main(): agent = Agent( "openai:gpt-4o-mini", + # "openai:gpt-5.4-mini-2026-03-17", instructions="You are a helpful assistant. You can roll dice and check if numbers are prime.", ) agent.tool_plain(roll_die) diff --git a/ui/src/components/dashboard/DashboardView.tsx b/ui/src/components/dashboard/DashboardView.tsx index 9057431..b0e05c2 100644 --- a/ui/src/components/dashboard/DashboardView.tsx +++ b/ui/src/components/dashboard/DashboardView.tsx @@ -129,6 +129,17 @@ export const DashboardView: React.FC = () => { setHoveredTraceId(traceId); }; + // The golden/reference trace is plotted in the performance charts (as a + // baseline) but excluded from pass/fail scoring - summary stats and the + // results table. Charts get the full set plus the golden id for labeling. + const goldenTraceId = state.goldenTraceId; + const scoredResults = goldenTraceId + ? state.results.filter((r) => r.traceId !== goldenTraceId) + : state.results; + const scoredRows = Array.from(state.tableRows.values()).filter( + (r) => !goldenTraceId || r.traceId !== goldenTraceId, + ); + return (
@@ -178,9 +189,13 @@ export const DashboardView: React.FC = () => { {state.tableRows.size > 0 && ( <> - {state.results.length > 0 && } + {scoredResults.length > 0 && } - + {state.isEvaluating && (
@@ -192,7 +207,7 @@ export const DashboardView: React.FC = () => { )} = ({ traceResults, hoveredTraceId }) => { +export const PerformanceCharts: React.FC = ({ traceResults, hoveredTraceId, goldenTraceId }) => { const tracesWithPerf = traceResults.filter(tr => tr.performanceMetrics); if (tracesWithPerf.length === 0) { return null; } + const isGolden = (tr: TraceResult) => goldenTraceId != null && tr.traceId === goldenTraceId; + const sortedTraces = [...tracesWithPerf].sort((a, b) => { + // Golden reference first, so it reads as the baseline the runs compare to. + if (isGolden(a) !== isGolden(b)) return isGolden(a) ? -1 : 1; const aSession = a.sessionId || a.traceId; const bSession = b.sessionId || b.traceId; return aSession.localeCompare(bSession); }); - const labels = sortedTraces.map(tr => tr.sessionId || tr.traceId.substring(0, 12)); + const labels = sortedTraces.map(tr => { + const base = tr.sessionId || tr.traceId.substring(0, 12); + return isGolden(tr) ? `${base} (reference)` : base; + }); const hoveredIndex = hoveredTraceId ? sortedTraces.findIndex(tr => tr.traceId === hoveredTraceId) diff --git a/ui/src/components/dashboard/TraceTable.tsx b/ui/src/components/dashboard/TraceTable.tsx index 75f9c66..890cf4e 100644 --- a/ui/src/components/dashboard/TraceTable.tsx +++ b/ui/src/components/dashboard/TraceTable.tsx @@ -3,8 +3,19 @@ import { Table } from 'antd'; import type { ColumnsType } from 'antd/es/table'; import { css } from '@emotion/react'; import { Clock, User, Cpu, MessageSquare, CheckCircle2, XCircle, Loader2, ChevronRight, ChevronDown } from 'lucide-react'; -import type { TraceTableRow, Annotation } from '../../lib/types'; +import type { TraceTableRow, Annotation, TrajectoryComparison } from '../../lib/types'; import { formatTimestamp } from '../../lib/utils'; +import { TrajectoryComparisonDetails } from '../inspector/TrajectoryComparisonDetails'; + +const TRAJECTORY_METRIC = 'tool_trajectory_avg_score'; + +function getTrajectoryComparisons(record: TraceTableRow): TrajectoryComparison[] { + return record.metricResults.get(TRAJECTORY_METRIC)?.details?.comparisons ?? []; +} + +function hasTrajectoryMismatch(record: TraceTableRow): boolean { + return getTrajectoryComparisons(record).some(c => !c.matched); +} interface TraceTableProps { rows: TraceTableRow[]; @@ -121,6 +132,19 @@ const tableStyle = css` white-space: nowrap; } + .diff-badge { + font-size: 10px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--status-failure); + background: rgba(255, 87, 87, 0.12); + border: 1px solid rgba(255, 87, 87, 0.3); + border-radius: 4px; + padding: 1px 6px; + cursor: pointer; + } + @keyframes spin { from { transform: rotate(0deg); @@ -238,27 +262,35 @@ export const TraceTable: React.FC = ({ const numTurns = record.invocations?.length || record.numInvocations || 1; const isExpanded = expandedRowKeys.includes(record.traceId); + const mismatch = hasTrajectoryMismatch(record); + const expandable = numTurns > 1 || mismatch; + + const toggle = (e: React.MouseEvent) => { + e.stopPropagation(); + setExpandedRowKeys( + isExpanded + ? expandedRowKeys.filter(k => k !== record.traceId) + : [...expandedRowKeys, record.traceId], + ); + }; - if (numTurns > 1) { + if (expandable) { return (
- { - e.stopPropagation(); - if (isExpanded) { - setExpandedRowKeys(expandedRowKeys.filter(k => k !== record.traceId)); - } else { - setExpandedRowKeys([...expandedRowKeys, record.traceId]); - } - }} - style={{ cursor: 'pointer', display: 'flex', alignItems: 'center' }} - > + {isExpanded ? : } - - {numTurns} turns - + {numTurns > 1 ? ( + {numTurns} turns + ) : ( + {record.userInputPreview} + )} + {mismatch && ( + + diff + + )}
); } @@ -385,7 +417,9 @@ export const TraceTable: React.FC = ({ const expandedRowRender = (record: TraceTableRow) => { const invocations = record.invocations || []; - if (invocations.length === 0) return null; + const comparisons = getTrajectoryComparisons(record); + const showDiff = comparisons.some(c => !c.matched); + if (invocations.length === 0 && !showDiff) return null; return (
= ({
); })} + {showDiff && ( +
0 ? '16px' : '0' }}> + +
+ )}
); }; @@ -524,7 +563,7 @@ export const TraceTable: React.FC = ({ } }, expandedRowRender, - rowExpandable: (record) => (record.invocations?.length || 0) > 1, + rowExpandable: (record) => (record.invocations?.length || 0) > 1 || hasTrajectoryMismatch(record), expandIcon: () => null, // Hide default expand icon since we show it in the Conversation column }} onRow={(record) => ({ diff --git a/ui/src/components/streaming/LiveStreamingView.tsx b/ui/src/components/streaming/LiveStreamingView.tsx index a51adac..9f1758b 100644 --- a/ui/src/components/streaming/LiveStreamingView.tsx +++ b/ui/src/components/streaming/LiveStreamingView.tsx @@ -441,10 +441,12 @@ export function LiveStreamingView() { if (import.meta.env.DEV) { console.log('Fetching traces for all sessions'); } - // The golden session defines the eval set, so scoring it against itself - // is a trivial pass. Exclude it so only the agents being meaningfully - // evaluated against the golden are scored and surfaced. - const sessionIds = Array.from(activeSessions.keys()).filter(id => id !== selectedGoldenId); + // Include the golden alongside the other sessions so its performance is + // plotted as a reference in the charts. It's tagged via goldenTraceId so + // the results view keeps it out of pass/fail scoring (scoring it against + // the eval set derived from itself is a trivial pass). + const sessionIds = Array.from(activeSessions.keys()); + const goldenTraceId = activeSessions.get(selectedGoldenId)?.traceId ?? null; const traceFiles = await Promise.all( sessionIds.map(async (sessionId) => { @@ -483,6 +485,7 @@ export function LiveStreamingView() { actions.setEvaluationOrigin('streaming'); actions.setTraceFiles(validTraceFiles); actions.setEvalSet(evalSetFile); + actions.setGoldenTraceId(goldenTraceId); actions.setCurrentView('upload'); } catch (error) { diff --git a/ui/src/context/TraceContext.tsx b/ui/src/context/TraceContext.tsx index dee1d79..aeb52b2 100644 --- a/ui/src/context/TraceContext.tsx +++ b/ui/src/context/TraceContext.tsx @@ -32,6 +32,9 @@ export interface TraceState { currentView: ViewType; evaluationOrigin: ViewType | null; selectedTraceId: string | null; + // Trace id of the golden/reference session in the current results, if any. + // Plotted in the performance charts but excluded from pass/fail scoring. + goldenTraceId: string | null; version: string | null; // Streaming state @@ -63,6 +66,7 @@ export interface TraceContextType { removeSession: (sessionId: string) => void; clearAllSessions: () => void; selectTrace: (traceId: string | null) => void; + setGoldenTraceId: (traceId: string | null) => void; clearResults: () => void; // Annotation queue actions diff --git a/ui/src/context/TraceProvider.tsx b/ui/src/context/TraceProvider.tsx index 556bee1..c5e3d0a 100644 --- a/ui/src/context/TraceProvider.tsx +++ b/ui/src/context/TraceProvider.tsx @@ -30,6 +30,7 @@ export const TraceProvider: React.FC = ({ children }) => { currentView: 'welcome', evaluationOrigin: null, selectedTraceId: null, + goldenTraceId: null, version: null, streamingSessions: new Map(), annotationQueues: [], @@ -292,11 +293,15 @@ export const TraceProvider: React.FC = ({ children }) => { selectTrace: (traceId: string | null) => setState((prev) => ({ ...prev, selectedTraceId: traceId })), + setGoldenTraceId: (traceId: string | null) => + setState((prev) => ({ ...prev, goldenTraceId: traceId })), + clearResults: () => setState((prev) => ({ ...prev, results: [], errors: [], + goldenTraceId: null, currentView: 'upload', })),