(feature): add initial Runs dashboard

examples/dice_agent/agent.py

@@ -56,7 +56,7 @@ def is_prime(n: int) -> bool:
 dice_agent = Agent(
     name="dice_agent",
     # model="gemini-2.5-flash",
-    model="gemini-2.5-flash-lite",
+    model="gemini-3-flash-preview",
     instruction="""You are a helpful assistant that can roll dice and check if numbers are prime.
 
 When a user asks you to roll a die, use the roll_die tool with the appropriate number of sides.

examples/zero-code-examples/adk/run.py

@@ -44,6 +44,7 @@ async def main():
     endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318")
     print(f"OTLP endpoint: {endpoint}")
 
+    os.environ.setdefault("OTEL_SERVICE_NAME", "adk-agent")
     os.environ.setdefault(
         "OTEL_RESOURCE_ATTRIBUTES",
         "agentevals.eval_set_id=dice_agent_eval,agentevals.session_name=adk-zero-code",

examples/zero-code-examples/langchain/run.py

@@ -48,6 +48,7 @@ def main():
 
     os.environ["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = "true"
 
+    os.environ.setdefault("OTEL_SERVICE_NAME", "langchain-agent")
     os.environ.setdefault(
         "OTEL_RESOURCE_ATTRIBUTES",
         "agentevals.eval_set_id=langchain_agent_eval,agentevals.session_name=langchain-zero-code",

examples/zero-code-examples/ollama/run.py

@@ -112,6 +112,7 @@ def main():
     print(f"Local model: {model}")
 
     os.environ["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = "true"
+    os.environ.setdefault("OTEL_SERVICE_NAME", "ollama-agent")
     os.environ.setdefault(
         "OTEL_RESOURCE_ATTRIBUTES",
         "agentevals.eval_set_id=langchain_local_ollama_openai_eval,agentevals.session_name=langchain-ollama-openai-zero-code",

examples/zero-code-examples/openai-agents/run.py

@@ -58,6 +58,7 @@ def main():
     os.environ.setdefault("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "span_and_event")
     os.environ.setdefault("OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental")
 
+    os.environ.setdefault("OTEL_SERVICE_NAME", "openai-agents-agent")
     os.environ.setdefault(
         "OTEL_RESOURCE_ATTRIBUTES",
         "agentevals.eval_set_id=openai_agents_eval,agentevals.session_name=openai-agents-zero-code",

examples/zero-code-examples/pydantic-ai/run.py

@@ -54,6 +54,7 @@ def main():
     endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318")
     print(f"OTLP endpoint: {endpoint}")
 
+    os.environ.setdefault("OTEL_SERVICE_NAME", "pydantic-ai-agent")
     os.environ.setdefault(
         "OTEL_RESOURCE_ATTRIBUTES",
         "agentevals.eval_set_id=pydantic_ai_eval,agentevals.session_name=pydantic-ai-zero-code",
@@ -72,6 +73,7 @@ def main():
 
     agent = Agent(
         "openai:gpt-4o-mini",
+        # "openai:gpt-5.4-mini-2026-03-17",
         instructions="You are a helpful assistant. You can roll dice and check if numbers are prime.",
     )
     agent.tool_plain(roll_die)

examples/zero-code-examples/strands/run.py

@@ -40,6 +40,7 @@ def main():
     endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318")
     print(f"OTLP endpoint: {endpoint}")
 
+    os.environ.setdefault("OTEL_SERVICE_NAME", "strands-agent")
     os.environ.setdefault(
         "OTEL_RESOURCE_ATTRIBUTES",
         "agentevals.eval_set_id=strands_agent_eval,agentevals.session_name=strands-zero-code",

src/agentevals/api/streaming_routes.py

@@ -7,16 +7,15 @@
 import logging
 from typing import TYPE_CHECKING
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, Request
 from fastapi.responses import FileResponse
 from pydantic import BaseModel, ConfigDict, Field
 
-from ..config import BuiltinMetricDef, EvalRunConfig, EvaluatorDef
+from ..config import BuiltinMetricDef, EvalParams, EvalRunConfig, EvaluatorDef
 from ..converter import convert_traces
 from ..loader.otlp import OtlpJsonLoader
-from ..runner import run_evaluation
+from ..runner import RunResult, run_evaluation
 from ..trace_attrs import OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_REQUEST_MODEL
-from ..utils.log_enrichment import enrich_spans_with_logs
 from .dependencies import require_trace_manager
 from .models import (
     CreateEvalSetData,
@@ -167,9 +166,47 @@ async def create_eval_set_from_session(
     return await _do_create_eval_set(request, manager)
 
 
+async def _persist_sessions_run(
+    http_request: Request,
+    *,
+    evaluators: list[EvaluatorDef],
+    eval_set_dict: dict,
+    session_ids: list[str],
+    run_results: list[RunResult],
+    session_errors: list[str],
+) -> None:
+    """Best-effort: persist a UI-driven session evaluation as a single Run row
+    (plus Result rows) when ``app.state.run_service`` is configured (postgres
+    backend), mirroring the offline ``/api/evaluate`` persistence.
+
+    Every evaluated session's traces are aggregated into one run so the run
+    history counts one evaluation per "Evaluate" click, the same way one
+    offline upload of N traces produces one run. No-op (and never raises) on
+    the memory backend or on persistence failure, so the live eval results are
+    always returned to the caller regardless."""
+    service = getattr(http_request.app.state, "run_service", None)
+    if service is None:
+        return
+    trace_results = [tr for rr in run_results for tr in rr.trace_results]
+    if not trace_results:
+        return
+    combined = RunResult(trace_results=trace_results, errors=session_errors)
+    try:
+        await service.record_eval_run(
+            params=EvalParams(evaluators=evaluators),
+            eval_set_dict=eval_set_dict,
+            trace_format="otlp-json",
+            upload_filenames=session_ids,
+            run_result=combined,
+        )
+    except Exception:
+        logger.exception("failed to persist evaluate-sessions run; results still returned to caller")
+
+
 @streaming_router.post("/evaluate-sessions", response_model=StandardResponse[EvaluateSessionsData])
 async def evaluate_sessions(
     request: EvaluateSessionsRequest,
+    http_request: Request,
     manager: StreamingTraceManager = Depends(require_trace_manager),
 ):
     """Evaluate all sessions against a golden session converted to EvalSet."""
@@ -200,7 +237,7 @@ async def evaluate_sessions(
 
         sem = asyncio.Semaphore(5)
 
-        async def eval_one_session(session_id: str, session) -> SessionEvalResult:
+        async def eval_one_session(session_id: str, session) -> tuple[SessionEvalResult, RunResult | None]:
             async with sem:
                 try:
                     trace_file = await manager._save_spans_to_temp_file(session)
@@ -216,34 +253,47 @@ async def eval_one_session(session_id: str, session) -> SessionEvalResult:
 
                     if eval_result.trace_results:
                         trace_result = eval_result.trace_results[0]
-                        return SessionEvalResult(
-                            session_id=session_id,
-                            trace_id=trace_result.trace_id,
-                            num_invocations=trace_result.num_invocations,
-                            metric_results=[
-                                {
-                                    "metricName": mr.metric_name,
-                                    "score": mr.score,
-                                    "evalStatus": mr.eval_status,
-                                    "error": mr.error,
-                                    "perInvocationScores": mr.per_invocation_scores,
-                                    "details": mr.details,
-                                }
-                                for mr in trace_result.metric_results
-                            ],
+                        return (
+                            SessionEvalResult(
+                                session_id=session_id,
+                                trace_id=trace_result.trace_id,
+                                num_invocations=trace_result.num_invocations,
+                                metric_results=[
+                                    {
+                                        "metricName": mr.metric_name,
+                                        "score": mr.score,
+                                        "evalStatus": mr.eval_status,
+                                        "error": mr.error,
+                                        "perInvocationScores": mr.per_invocation_scores,
+                                        "details": mr.details,
+                                    }
+                                    for mr in trace_result.metric_results
+                                ],
+                            ),
+                            eval_result,
                         )
                     else:
                         logger.warning("No trace results for session %s", session_id)
-                        return SessionEvalResult(session_id=session_id, error="No trace results")
+                        return SessionEvalResult(session_id=session_id, error="No trace results"), None
 
                 except Exception as exc:
                     logger.error(f"Failed to evaluate session {session_id}: {exc}", exc_info=True)
-                    return SessionEvalResult(session_id=session_id, error=str(exc))
+                    return SessionEvalResult(session_id=session_id, error=str(exc)), None
 
-        results = await asyncio.gather(*[eval_one_session(sid, sess) for sid, sess in sessions_to_evaluate])
+        evaluated = await asyncio.gather(*[eval_one_session(sid, sess) for sid, sess in sessions_to_evaluate])
+        results = [session_result for session_result, _ in evaluated]
 
         logger.info("Evaluation complete. Total results: %d", len(results))
 
+        await _persist_sessions_run(
+            http_request,
+            evaluators=request.evaluators,
+            eval_set_dict=eval_set_response.data.eval_set,
+            session_ids=[sid for sid, _ in sessions_to_evaluate],
+            run_results=[run_result for _, run_result in evaluated if run_result is not None],
+            session_errors=[f"{r.session_id}: {r.error}" for r in results if r.error],
+        )
+
         return StandardResponse(
             data=EvaluateSessionsData(
                 golden_session_id=request.golden_session_id,
@@ -344,12 +394,6 @@ async def get_trace(
         raise HTTPException(status_code=404, detail="Session not found")
 
     try:
-        import tempfile
-
-        temp_file = tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False)
-
-        unified_trace_id = session.trace_id
-
         has_genai_spans = any(
             span.get("attributes", [])
             and any(
@@ -366,23 +410,20 @@ async def get_trace(
                 request.session_id,
             )
 
-        enriched_spans = enrich_spans_with_logs(session.spans, session.logs)
-
-        for span in enriched_spans:
-            span_copy = span.copy()
-            span_copy["traceId"] = unified_trace_id
-            temp_file.write(json.dumps(span_copy) + "\n")
-
-        temp_file.close()
-
-        with open(temp_file.name) as f:  # noqa: ASYNC230
+        # Reuse the canonical serializer so the trace handed to the UI (and
+        # re-uploaded to /api/evaluate) carries service.name, exactly like the
+        # evaluate-sessions path. Serializing spans here independently would
+        # drop the resource attribute and lose agent identity on the run.
+        trace_file = await manager._save_spans_to_temp_file(session)
+        with open(trace_file) as f:  # noqa: ASYNC230
             trace_content = f.read()
+        num_spans = sum(1 for line in trace_content.splitlines() if line.strip())
 
         return StandardResponse(
             data=GetTraceData(
                 session_id=request.session_id,
                 trace_content=trace_content,
-                num_spans=len(enriched_spans),
+                num_spans=num_spans,
             )
         )
 

src/agentevals/run/result_builder.py

@@ -99,27 +99,63 @@ def build_results(run_id: UUID, params: EvalParams, run_result: RunResult) -> li
     return out
 
 
+def _status_key(mr: MetricResult) -> str:
+    """Map a :class:`MetricResult` onto a :class:`ResultStatus` string,
+    matching :func:`result_from_metric_result` so the summary counts and the
+    persisted ``result`` rows never disagree."""
+    if mr.error:
+        return "errored"
+    status = (mr.eval_status or "").upper()
+    if status == "PASSED":
+        return "passed"
+    if status == "FAILED":
+        return "failed"
+    return "skipped"
+
+
 def summarize_run_result(run_result: RunResult) -> dict[str, Any]:
     """Summary blob persisted alongside the run row.
 
     Counts mirror :class:`agentevals.storage.models.ResultStatus` values so a
     caller polling ``GET /api/runs/{id}`` can compute pass/fail rates without
     fetching the full result list.
+
+    ``per_metric`` aggregates the same statuses by ``evaluator_name`` plus a
+    mean ``score``, so the run-history dashboard can chart per-metric trends
+    across runs from the list response alone, without an N+1 over
+    ``/api/runs/{id}/results``. ``avg_score`` is ``None`` when no invocation of
+    that evaluator produced a numeric score (e.g. it only errored).
     """
     counts = {"passed": 0, "failed": 0, "errored": 0, "skipped": 0}
+    per_metric: dict[str, dict[str, Any]] = {}
+    agents: set[str] = set()
     for tr in run_result.trace_results:
+        identity = tr.service_name or tr.agent_name
+        if identity:
+            agents.add(identity)
         for mr in tr.metric_results:
-            if mr.error:
-                counts["errored"] += 1
-            elif (mr.eval_status or "").upper() == "PASSED":
-                counts["passed"] += 1
-            elif (mr.eval_status or "").upper() == "FAILED":
-                counts["failed"] += 1
-            else:
-                counts["skipped"] += 1
+            key = _status_key(mr)
+            counts[key] += 1
+
+            metric = per_metric.setdefault(
+                mr.metric_name,
+                {"passed": 0, "failed": 0, "errored": 0, "skipped": 0, "_score_sum": 0.0, "_score_count": 0},
+            )
+            metric[key] += 1
+            if mr.score is not None:
+                metric["_score_sum"] += mr.score
+                metric["_score_count"] += 1
+
+    for metric in per_metric.values():
+        score_count = metric.pop("_score_count")
+        score_sum = metric.pop("_score_sum")
+        metric["avg_score"] = (score_sum / score_count) if score_count else None
+
     return {
         "trace_count": len(run_result.trace_results),
         "result_counts": counts,
+        "per_metric": per_metric,
+        "agents": sorted(agents),
         "errors": list(run_result.errors),
         "performance_metrics": run_result.performance_metrics,
     }

src/agentevals/runner.py

@@ -22,7 +22,7 @@
 from .converter import ConversionResult, convert_traces
 from .loader import load_traces
 from .loader.base import Trace
-from .trace_metrics import _calc_percentiles, extract_performance_metrics
+from .trace_metrics import _calc_percentiles, extract_agent_identity, extract_performance_metrics
 
 logger = logging.getLogger(__name__)
 
@@ -50,6 +50,8 @@ class TraceResult(BaseModel):
     metric_results: list[MetricResult] = Field(default_factory=list)
     conversion_warnings: list[str] = Field(default_factory=list)
     performance_metrics: dict[str, Any] | None = None
+    service_name: str | None = None
+    agent_name: str | None = None
 
 
 class RunResult(BaseModel):
@@ -111,7 +113,7 @@ async def _evaluate_trace_bounded(idx: int, conv_result: ConversionResult) -> Tr
 
             trace = trace_map.get(conv_result.trace_id)
 
-            return await _evaluate_trace(
+            trace_result = await _evaluate_trace(
                 conv_result=conv_result,
                 evaluators=config.evaluators,
                 eval_set=eval_set,
@@ -121,6 +123,11 @@ async def _evaluate_trace_bounded(idx: int, conv_result: ConversionResult) -> Tr
                 trace=trace,
                 performance_metrics=perf_metrics_map.get(conv_result.trace_id),
             )
+            if trace is not None:
+                identity = extract_agent_identity(trace)
+                trace_result.service_name = identity["service_name"]
+                trace_result.agent_name = identity["agent_name"]
+            return trace_result
 
     trace_results = await asyncio.gather(
         *[_evaluate_trace_bounded(idx, conv_result) for idx, conv_result in enumerate(conversion_results)],

src/agentevals/streaming/ws_server.py

@@ -28,7 +28,7 @@
 )
 from ..loader.base import Trace
 from ..loader.otlp import OtlpJsonLoader
-from ..trace_attrs import OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_REQUEST_MODEL
+from ..trace_attrs import OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_REQUEST_MODEL, OTEL_SERVICE_NAME
 from ..utils.log_enrichment import enrich_spans_with_logs
 from .incremental_processor import IncrementalInvocationExtractor
 from .session import TraceSession
@@ -658,10 +658,21 @@ async def _save_spans_to_temp_file(self, session: TraceSession) -> Path:
 
         enriched_spans = enrich_spans_with_logs(session.spans, session.logs, session.session_id)
 
+        # service.name is an OTel resource attribute, so it lives on the session
+        # (lifted from resource attrs at ingest) rather than on individual spans.
+        # The JSONL trace format has no resource envelope, so re-attach it to each
+        # span here; otherwise it's lost on reload and runs can't group by agent.
+        service_name = (session.metadata or {}).get(OTEL_SERVICE_NAME)
+
         with open(temp_file, "w") as f:  # noqa: ASYNC230
             for span in enriched_spans:
                 span_copy = span.copy()
                 span_copy["traceId"] = session.trace_id
+                if service_name:
+                    attrs = list(span_copy.get("attributes", []))
+                    if not any(a.get("key") == OTEL_SERVICE_NAME for a in attrs):
+                        attrs.append({"key": OTEL_SERVICE_NAME, "value": {"stringValue": service_name}})
+                        span_copy["attributes"] = attrs
                 f.write(json.dumps(span_copy) + "\n")
 
         return temp_file