From 2b1e3ad901d15e1ce81a2af9843ba35e8f7f4c5a Mon Sep 17 00:00:00 2001
From: Peter Tomko <tomko5peter@gmail.com>
Date: Wed, 5 Aug 2026 14:02:51 +0200
Subject: [PATCH 1/2] feat(gooddata-eval): capture conversation_id/response_id
 through the agentic-CLI path

8010bd44 wired reasoning_steps through cli/agentic_runner.py -> evaluate_agentic_*,
but conversation_id/response_id stayed unset on ItemReport for every agentic kind
(agentic_alert_skill/agentic_metric_skill/agentic_conversation) -- each ChatResult
already carries both, and conversation_id was already threaded up to the
Alert/Metric/ConversationRunResult layer, but neither ever reached the top-level
evaluate_agentic_* return value or its failure exception, so run_agentic_items had
nothing to read.

Mirrors the reasoning_steps idiom exactly: widens each evaluate_agentic_*'s return
from list[str] to (reasoning_steps, conversation_id, response_id), attaches all
three to the raised exception on failure, and has run_agentic_items unpack either
form (tuple or the untouched kinds' bare list/None) onto ItemReport.conversation_id
/response_id. response_id is new at the RunResult layer for all three kinds --
captured as the last non-null value across a run's turns, same pattern already
used for reasoning_steps accumulation.

general_question/guardrail/search_tool/visualization untouched (already populated
via the single-turn runner.py path, not this one).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 .../src/gooddata_eval/cli/agentic_runner.py   | 16 +++++++++++----
 .../gooddata_eval/core/agentic/alert_skill.py | 20 ++++++++++++++-----
 .../core/agentic/conversation.py              | 19 +++++++++++++-----
 .../core/agentic/metric_skill.py              | 20 ++++++++++++++-----
 .../tests/test_agentic_alert_skill.py         |  6 +++++-
 .../tests/test_agentic_conversation.py        |  8 +++++++-
 .../tests/test_agentic_metric_skill.py        |  6 +++++-
 .../tests/test_agentic_runner.py              | 12 ++++++++++-
 8 files changed, 84 insertions(+), 23 deletions(-)

diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py
index 1526c309d..02c599326 100644
--- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py
+++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py
@@ -86,11 +86,12 @@ def _dispatch_agentic(
     model_version_override: str | None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
-) -> list[str] | None:
+) -> tuple[list[str], str, str | None] | list[str] | None:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind.
 
-    Returns whatever that function returns -- only alert_skill/metric_skill/conversation
-    currently return their reasoning_steps; the rest still return None (unchanged).
+    Returns whatever that function returns -- alert_skill/metric_skill/conversation return
+    their (reasoning_steps, conversation_id, response_id); the rest still return None
+    (unchanged).
     """
     kind = item.test_kind
     eo = item.expected_output
@@ -232,16 +233,23 @@ def run_agentic_items(
         )
         t0 = time.perf_counter()
         try:
-            reasoning_steps = _dispatch_agentic(
+            outcome = _dispatch_agentic(
                 item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id
             )
+            reasoning_steps, conversation_id, response_id = (
+                outcome if isinstance(outcome, tuple) else (outcome, None, None)
+            )
             item_report.pass_at_k = True
             item_report.runs = k
             item_report.reasoning_steps = reasoning_steps or []
+            item_report.conversation_id = conversation_id
+            item_report.response_id = response_id
         except AssertionError as exc:
             item_report.pass_at_k = False
             item_report.runs = k
             item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
+            item_report.conversation_id = getattr(exc, "conversation_id", None)
+            item_report.response_id = getattr(exc, "response_id", None)
             print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
         except Exception as exc:
             item_report.error = f"{type(exc).__name__}: {exc}"
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py
index 532abced8..4eb4d4b5b 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py
@@ -303,6 +303,7 @@ class AlertRunResult:
     eval: AlertEvaluation
     actual_alert_arguments: dict
     reasoning_steps: list[str] = field(default_factory=list)
+    response_id: str | None = None
 
 
 @dataclass
@@ -452,6 +453,7 @@ def _run_once(conv_id: str) -> AlertRunResult:
             actual_args: dict = {}
             tool_called = False
             reasoning_steps: list[str] = []
+            response_id: str | None = None
             # conversation_history stores prior turns for GPT-4o context.
             # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply.
             conversation_history: list = []
@@ -460,6 +462,7 @@ def _run_once(conv_id: str) -> AlertRunResult:
             for _iteration in range(max_iterations):
                 chat_result = client.send_message(conv_id, current_question)
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
+                response_id = chat_result.response_id or response_id
                 alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or [])
                 if tool_called:
                     alert_id_to_delete = alert_id
@@ -496,6 +499,7 @@ def _run_once(conv_id: str) -> AlertRunResult:
                 eval=ev,
                 actual_alert_arguments=actual_args,
                 reasoning_steps=reasoning_steps,
+                response_id=response_id,
             )
         finally:
             if alert_id_to_delete:
@@ -547,6 +551,8 @@ class AlertSkillAssertionError(AssertionError):
 
     __tracebackhide__ = True
     reasoning_steps: list[str]
+    conversation_id: str
+    response_id: str | None
 
 
 def evaluate_agentic_alert_skill(
@@ -566,12 +572,14 @@ def evaluate_agentic_alert_skill(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> list[str]:
+) -> tuple[list[str], str, str | None]:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.
 
-    Returns the best run's reasoning_steps on success; on failure the same list is attached
-    to the raised exception as ``.reasoning_steps`` (mirrors the `conversation_id`-on-exception
-    idiom in `ChatClient.ask()`) so callers can retrieve it either way.
+    Returns the best run's (reasoning_steps, conversation_id, response_id) on success; on
+    failure the same three values are attached to the raised exception as
+    ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the
+    `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them
+    either way.
     """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
@@ -655,5 +663,7 @@ def evaluate_agentic_alert_skill(
             f"Actual args: {best.actual_alert_arguments}"
         )
         exc.reasoning_steps = best.reasoning_steps
+        exc.conversation_id = best.conversation_id
+        exc.response_id = best.response_id
         raise exc
-    return summary.best.reasoning_steps
+    return summary.best.reasoning_steps, summary.best.conversation_id, summary.best.response_id
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py
index 51fd1d4e9..c0c00c362 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py
@@ -266,6 +266,7 @@ class ConversationResult:
     conversation_success: bool
     total_clarification_turns: int
     reasoning_steps: list[str] = field(default_factory=list)
+    response_id: str | None = None
 
 
 def run_agentic_conversation(
@@ -298,6 +299,7 @@ def run_agentic_conversation(
     # the end — a later turn may $ref a metric an earlier turn created.
     created_metric_ids: list[str] = []
     reasoning_steps: list[str] = []
+    response_id: str | None = None
 
     try:
         if initial_conversation_id is not None:
@@ -335,6 +337,7 @@ def run_agentic_conversation(
                 final_result = chat_result
                 all_tool_calls.extend(chat_result.tool_call_events or [])
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
+                response_id = chat_result.response_id or response_id
 
                 if _check_output_present(resolved_turn, chat_result):
                     break
@@ -399,6 +402,7 @@ def run_agentic_conversation(
         conversation_success=conversation_success,
         total_clarification_turns=total_clarification_turns,
         reasoning_steps=reasoning_steps,
+        response_id=response_id,
     )
 
 
@@ -407,6 +411,8 @@ class ConversationAssertionError(AssertionError):
 
     __tracebackhide__ = True
     reasoning_steps: list[str]
+    conversation_id: str
+    response_id: str | None
 
 
 def evaluate_agentic_conversation(
@@ -424,12 +430,13 @@ def evaluate_agentic_conversation(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> list[str]:
+) -> tuple[list[str], str, str | None]:
     """Run conversation evaluation, log to Langfuse, and raise on failure.
 
-    Returns the conversation's reasoning_steps on success; on failure the same list is
-    attached to the raised exception as ``.reasoning_steps`` (mirrors the
-    `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve it
+    Returns the conversation's (reasoning_steps, conversation_id, response_id) on success;
+    on failure the same three values are attached to the raised exception as
+    ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the
+    `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them
     either way.
     """
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -514,5 +521,7 @@ def evaluate_agentic_conversation(
             f"Failed turns: {[t.turn_id for t in failed_turns]}"
         )
         exc.reasoning_steps = result.reasoning_steps
+        exc.conversation_id = result.conversation_id
+        exc.response_id = result.response_id
         raise exc
-    return result.reasoning_steps
+    return result.reasoning_steps, result.conversation_id, result.response_id
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py
index 2db890209..813ce8e38 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py
@@ -130,6 +130,7 @@ class MetricRunResult:
     maql_correct: bool
     total_turns: float
     reasoning_steps: list[str] = field(default_factory=list)
+    response_id: str | None = None
 
 
 @dataclass
@@ -207,12 +208,14 @@ def _execute_single_metric_run(
     turns = 0
     current_question = question
     reasoning_steps: list[str] = []
+    response_id: str | None = None
 
     try:
         for _iteration in range(max_iterations):
             turns += 1
             chat_result = client.send_message(conversation_id, current_question)
             reasoning_steps.extend(chat_result.reasoning_steps or [])
+            response_id = chat_result.response_id or response_id
             candidate = _extract_metric_result(chat_result.tool_call_events or [])
             if candidate is not None:
                 metric_result = candidate
@@ -240,6 +243,7 @@ def _execute_single_metric_run(
             maql_correct=maql_correct,
             total_turns=float(turns),
             reasoning_steps=reasoning_steps,
+            response_id=response_id,
         )
     finally:
         if metric_id_to_delete:
@@ -311,6 +315,8 @@ class MetricSkillAssertionError(AssertionError):
 
     __tracebackhide__ = True
     reasoning_steps: list[str]
+    conversation_id: str
+    response_id: str | None
 
 
 def evaluate_agentic_metric_skill(
@@ -330,12 +336,14 @@ def evaluate_agentic_metric_skill(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> list[str]:
+) -> tuple[list[str], str, str | None]:
     """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure.
 
-    Returns the best run's reasoning_steps on success; on failure the same list is attached
-    to the raised exception as ``.reasoning_steps`` (mirrors the `conversation_id`-on-exception
-    idiom in `ChatClient.ask()`) so callers can retrieve it either way.
+    Returns the best run's (reasoning_steps, conversation_id, response_id) on success; on
+    failure the same three values are attached to the raised exception as
+    ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the
+    `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them
+    either way.
     """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
@@ -408,5 +416,7 @@ def evaluate_agentic_metric_skill(
             f"Actual MAQL: {best.actual_maql}."
         )
         exc.reasoning_steps = best.reasoning_steps
+        exc.conversation_id = best.conversation_id
+        exc.response_id = best.response_id
         raise exc
-    return summary.best.reasoning_steps
+    return summary.best.reasoning_steps, summary.best.conversation_id, summary.best.response_id
diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py
index 140c62fd4..0fc6a385f 100644
--- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py
+++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py
@@ -565,7 +565,7 @@ def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass():
         patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client),
         patch("gooddata_eval.core.agentic.alert_skill._delete_alert"),
     ):
-        reasoning = evaluate_agentic_alert_skill(
+        reasoning, conversation_id, response_id = evaluate_agentic_alert_skill(
             host="http://host",
             token="tok",
             workspace_id="ws1",
@@ -576,6 +576,8 @@ def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass():
         )
 
     assert reasoning == ["thinking about it"]
+    assert conversation_id == "conv-1"
+    assert response_id is None
 
 
 def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_fail():
@@ -604,3 +606,5 @@ def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_f
             max_iterations=1,
         )
     assert exc_info.value.reasoning_steps == ["confused thinking"]
+    assert exc_info.value.conversation_id == "conv-1"
+    assert exc_info.value.response_id is None
diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py
index cc4ab0791..3815e867c 100644
--- a/packages/gooddata-eval/tests/test_agentic_conversation.py
+++ b/packages/gooddata-eval/tests/test_agentic_conversation.py
@@ -532,6 +532,7 @@ def test_evaluate_agentic_conversation_returns_reasoning_steps_on_pass():
     chat_result.created_visualizations = [MagicMock()]
     chat_result.tool_call_events = [tc]
     chat_result.reasoning_steps = ["thinking about it"]
+    chat_result.response_id = "resp-1"
     mock_client.send_message.return_value = chat_result
 
     fixture = ConversationFixture(
@@ -547,13 +548,15 @@ def test_evaluate_agentic_conversation_returns_reasoning_steps_on_pass():
         ],
     )
     with patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client):
-        reasoning = evaluate_agentic_conversation(
+        reasoning, conversation_id, response_id = evaluate_agentic_conversation(
             host="http://host",
             token="tok",
             workspace_id="ws1",
             fixture=fixture,
         )
     assert reasoning == ["thinking about it"]
+    assert conversation_id == "conv-1"
+    assert response_id == "resp-1"
 
 
 def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_fail():
@@ -568,6 +571,7 @@ def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_
     chat_result.tool_call_events = [tc]
     chat_result.alert_proposals = []
     chat_result.reasoning_steps = ["confused thinking"]
+    chat_result.response_id = "resp-2"
     mock_client.send_message.return_value = chat_result
 
     fixture = ConversationFixture(
@@ -594,3 +598,5 @@ def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_
             max_clarification_turns=0,
         )
     assert exc_info.value.reasoning_steps == ["confused thinking"]
+    assert exc_info.value.conversation_id == "conv-1"
+    assert exc_info.value.response_id == "resp-2"
diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py
index ccc24299c..dd855d8f2 100644
--- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py
+++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py
@@ -381,7 +381,7 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass():
         }
     )
     with patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client):
-        reasoning = evaluate_agentic_metric_skill(
+        reasoning, conversation_id, response_id = evaluate_agentic_metric_skill(
             host="http://host/api/v1/actions/workspaces/ws1/ai",
             token="tok",
             workspace_id="ws1",
@@ -391,6 +391,8 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass():
             max_iterations=1,
         )
     assert reasoning == ["thinking about it"]
+    assert conversation_id == "conv-1"
+    assert response_id is None
 
 
 def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_fail():
@@ -417,3 +419,5 @@ def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_
             max_iterations=1,
         )
     assert exc_info.value.reasoning_steps == ["confused thinking"]
+    assert exc_info.value.conversation_id == "conv-1"
+    assert exc_info.value.response_id is None
diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py
index fda604542..925d323d7 100644
--- a/packages/gooddata-eval/tests/test_agentic_runner.py
+++ b/packages/gooddata-eval/tests/test_agentic_runner.py
@@ -108,7 +108,7 @@ def _item(test_kind: str = "agentic_alert_skill") -> DatasetItem:
 def test_run_agentic_items_surfaces_reasoning_steps_on_pass():
     with patch(
         "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill",
-        return_value=["it created the alert"],
+        return_value=(["it created the alert"], "conv-1", "resp-1"),
     ):
         report = run_agentic_items(
             [_item()],
@@ -119,11 +119,15 @@ def test_run_agentic_items_surfaces_reasoning_steps_on_pass():
         )
     assert report.items[0].pass_at_k is True
     assert report.items[0].reasoning_steps == ["it created the alert"]
+    assert report.items[0].conversation_id == "conv-1"
+    assert report.items[0].response_id == "resp-1"
 
 
 def test_run_agentic_items_surfaces_reasoning_steps_from_exception_on_fail():
     exc = AlertSkillAssertionError("nope")
     exc.reasoning_steps = ["it got confused"]
+    exc.conversation_id = "conv-2"
+    exc.response_id = "resp-2"
     with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=exc):
         report = run_agentic_items(
             [_item()],
@@ -134,6 +138,8 @@ def test_run_agentic_items_surfaces_reasoning_steps_from_exception_on_fail():
         )
     assert report.items[0].pass_at_k is False
     assert report.items[0].reasoning_steps == ["it got confused"]
+    assert report.items[0].conversation_id == "conv-2"
+    assert report.items[0].response_id == "resp-2"
 
 
 def test_run_agentic_items_defaults_reasoning_steps_to_empty_when_exception_has_none():
@@ -149,6 +155,8 @@ def test_run_agentic_items_defaults_reasoning_steps_to_empty_when_exception_has_
             run_ts="2026-01-01",
         )
     assert report.items[0].reasoning_steps == []
+    assert report.items[0].conversation_id is None
+    assert report.items[0].response_id is None
 
 
 def test_run_agentic_items_defaults_reasoning_steps_to_empty_for_untouched_kinds():
@@ -163,3 +171,5 @@ def test_run_agentic_items_defaults_reasoning_steps_to_empty_for_untouched_kinds
         )
     assert report.items[0].pass_at_k is True
     assert report.items[0].reasoning_steps == []
+    assert report.items[0].conversation_id is None
+    assert report.items[0].response_id is None

From 8e7d622e15cbd1ce1aa849b3df2256eb19f0b54a Mon Sep 17 00:00:00 2001
From: Peter Tomko <tomko5peter@gmail.com>
Date: Thu, 20 Aug 2026 13:46:47 +0200
Subject: [PATCH 2/2] refactor(gooddata-eval): replace tuple return with
 AgenticEvalOutcome dataclass

evaluate_agentic_alert_skill/metric_skill/conversation returned a bare
tuple[list[str], str, str | None], and _dispatch_agentic's return type
was a tuple | list[str] | None union unpacked via isinstance(outcome,
tuple). Introduced AgenticEvalOutcome (reasoning_steps, conversation_id,
response_id) in core/models.py and used it as the return type instead.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .../src/gooddata_eval/cli/agentic_runner.py       | 15 +++++++++------
 .../src/gooddata_eval/core/agentic/alert_skill.py | 15 ++++++++++-----
 .../gooddata_eval/core/agentic/conversation.py    | 15 ++++++++++-----
 .../gooddata_eval/core/agentic/metric_skill.py    | 15 ++++++++++-----
 .../src/gooddata_eval/core/models.py              |  8 ++++++++
 .../tests/test_agentic_alert_skill.py             |  8 ++++----
 .../tests/test_agentic_conversation.py            |  8 ++++----
 .../tests/test_agentic_metric_skill.py            |  8 ++++----
 .../gooddata-eval/tests/test_agentic_runner.py    |  6 ++++--
 9 files changed, 63 insertions(+), 35 deletions(-)

diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py
index 02c599326..24fbad1a9 100644
--- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py
+++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py
@@ -16,7 +16,7 @@
 from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import CreatedVisualization, DatasetItem
+from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
 
@@ -86,11 +86,11 @@ def _dispatch_agentic(
     model_version_override: str | None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
-) -> tuple[list[str], str, str | None] | list[str] | None:
+) -> AgenticEvalOutcome | list[str] | None:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind.
 
     Returns whatever that function returns -- alert_skill/metric_skill/conversation return
-    their (reasoning_steps, conversation_id, response_id); the rest still return None
+    an AgenticEvalOutcome; the rest still return None
     (unchanged).
     """
     kind = item.test_kind
@@ -236,9 +236,12 @@ def run_agentic_items(
             outcome = _dispatch_agentic(
                 item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id
             )
-            reasoning_steps, conversation_id, response_id = (
-                outcome if isinstance(outcome, tuple) else (outcome, None, None)
-            )
+            if isinstance(outcome, AgenticEvalOutcome):
+                reasoning_steps = outcome.reasoning_steps
+                conversation_id = outcome.conversation_id
+                response_id = outcome.response_id
+            else:
+                reasoning_steps, conversation_id, response_id = outcome, None, None
             item_report.pass_at_k = True
             item_report.runs = k
             item_report.reasoning_steps = reasoning_steps or []
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py
index 4eb4d4b5b..e21737371 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py
@@ -14,7 +14,7 @@
 from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import ToolCallEvent
+from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent
 
 try:
     from openai import OpenAI as _OpenAI
@@ -572,11 +572,12 @@ def evaluate_agentic_alert_skill(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> tuple[list[str], str, str | None]:
+) -> AgenticEvalOutcome:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.
 
-    Returns the best run's (reasoning_steps, conversation_id, response_id) on success; on
-    failure the same three values are attached to the raised exception as
+    Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an
+    AgenticEvalOutcome on success; on failure the same three values are attached to the
+    raised exception as
     ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the
     `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them
     either way.
@@ -666,4 +667,8 @@ def evaluate_agentic_alert_skill(
         exc.conversation_id = best.conversation_id
         exc.response_id = best.response_id
         raise exc
-    return summary.best.reasoning_steps, summary.best.conversation_id, summary.best.response_id
+    return AgenticEvalOutcome(
+        reasoning_steps=summary.best.reasoning_steps,
+        conversation_id=summary.best.conversation_id,
+        response_id=summary.best.response_id,
+    )
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py
index c0c00c362..95821bb43 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py
@@ -15,7 +15,7 @@
 from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import ChatResult, ToolCallEvent
+from gooddata_eval.core.models import AgenticEvalOutcome, ChatResult, ToolCallEvent
 from gooddata_eval.core.scoring import (
     check_filters,
     check_viz_type,
@@ -430,11 +430,12 @@ def evaluate_agentic_conversation(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> tuple[list[str], str, str | None]:
+) -> AgenticEvalOutcome:
     """Run conversation evaluation, log to Langfuse, and raise on failure.
 
-    Returns the conversation's (reasoning_steps, conversation_id, response_id) on success;
-    on failure the same three values are attached to the raised exception as
+    Returns the conversation's outcome (reasoning_steps, conversation_id, response_id) as
+    an AgenticEvalOutcome on success; on failure the same three values are attached to the
+    raised exception as
     ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the
     `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them
     either way.
@@ -524,4 +525,8 @@ def evaluate_agentic_conversation(
         exc.conversation_id = result.conversation_id
         exc.response_id = result.response_id
         raise exc
-    return result.reasoning_steps, result.conversation_id, result.response_id
+    return AgenticEvalOutcome(
+        reasoning_steps=result.reasoning_steps,
+        conversation_id=result.conversation_id,
+        response_id=result.response_id,
+    )
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py
index 813ce8e38..a699ebe63 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py
@@ -12,7 +12,7 @@
 
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import ToolCallEvent
+from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent
 
 try:
     from openai import OpenAI as _OpenAI
@@ -336,11 +336,12 @@ def evaluate_agentic_metric_skill(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> tuple[list[str], str, str | None]:
+) -> AgenticEvalOutcome:
     """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure.
 
-    Returns the best run's (reasoning_steps, conversation_id, response_id) on success; on
-    failure the same three values are attached to the raised exception as
+    Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an
+    AgenticEvalOutcome on success; on failure the same three values are attached to the
+    raised exception as
     ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the
     `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them
     either way.
@@ -419,4 +420,8 @@ def evaluate_agentic_metric_skill(
         exc.conversation_id = best.conversation_id
         exc.response_id = best.response_id
         raise exc
-    return summary.best.reasoning_steps, summary.best.conversation_id, summary.best.response_id
+    return AgenticEvalOutcome(
+        reasoning_steps=summary.best.reasoning_steps,
+        conversation_id=summary.best.conversation_id,
+        response_id=summary.best.response_id,
+    )
diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py
index 40f7ede96..0c44cc114 100644
--- a/packages/gooddata-eval/src/gooddata_eval/core/models.py
+++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py
@@ -107,6 +107,14 @@ class ChatResult(BaseModel):
     turn_wall_clock_sec: float | None = None
 
 
+class AgenticEvalOutcome(BaseModel):
+    """Reasoning trace and trace-lookup IDs returned by an evaluate_agentic_* call on success."""
+
+    reasoning_steps: list[str] = Field(default_factory=list)
+    conversation_id: str | None = None
+    response_id: str | None = None
+
+
 class SummaryInput(BaseModel):
     """Structured input for the `dashboard_summary` test kind.
 
diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py
index 0fc6a385f..acc0b5669 100644
--- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py
+++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py
@@ -565,7 +565,7 @@ def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass():
         patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client),
         patch("gooddata_eval.core.agentic.alert_skill._delete_alert"),
     ):
-        reasoning, conversation_id, response_id = evaluate_agentic_alert_skill(
+        outcome = evaluate_agentic_alert_skill(
             host="http://host",
             token="tok",
             workspace_id="ws1",
@@ -575,9 +575,9 @@ def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass():
             max_iterations=1,
         )
 
-    assert reasoning == ["thinking about it"]
-    assert conversation_id == "conv-1"
-    assert response_id is None
+    assert outcome.reasoning_steps == ["thinking about it"]
+    assert outcome.conversation_id == "conv-1"
+    assert outcome.response_id is None
 
 
 def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_fail():
diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py
index 3815e867c..a6368683b 100644
--- a/packages/gooddata-eval/tests/test_agentic_conversation.py
+++ b/packages/gooddata-eval/tests/test_agentic_conversation.py
@@ -548,15 +548,15 @@ def test_evaluate_agentic_conversation_returns_reasoning_steps_on_pass():
         ],
     )
     with patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client):
-        reasoning, conversation_id, response_id = evaluate_agentic_conversation(
+        outcome = evaluate_agentic_conversation(
             host="http://host",
             token="tok",
             workspace_id="ws1",
             fixture=fixture,
         )
-    assert reasoning == ["thinking about it"]
-    assert conversation_id == "conv-1"
-    assert response_id == "resp-1"
+    assert outcome.reasoning_steps == ["thinking about it"]
+    assert outcome.conversation_id == "conv-1"
+    assert outcome.response_id == "resp-1"
 
 
 def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_fail():
diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py
index dd855d8f2..684436ee1 100644
--- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py
+++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py
@@ -381,7 +381,7 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass():
         }
     )
     with patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client):
-        reasoning, conversation_id, response_id = evaluate_agentic_metric_skill(
+        outcome = evaluate_agentic_metric_skill(
             host="http://host/api/v1/actions/workspaces/ws1/ai",
             token="tok",
             workspace_id="ws1",
@@ -390,9 +390,9 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass():
             k=1,
             max_iterations=1,
         )
-    assert reasoning == ["thinking about it"]
-    assert conversation_id == "conv-1"
-    assert response_id is None
+    assert outcome.reasoning_steps == ["thinking about it"]
+    assert outcome.conversation_id == "conv-1"
+    assert outcome.response_id is None
 
 
 def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_fail():
diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py
index 925d323d7..627afebd9 100644
--- a/packages/gooddata-eval/tests/test_agentic_runner.py
+++ b/packages/gooddata-eval/tests/test_agentic_runner.py
@@ -5,7 +5,7 @@
 import pytest
 from gooddata_eval.cli.agentic_runner import _dispatch_agentic, run_agentic_items
 from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError
-from gooddata_eval.core.models import DatasetItem
+from gooddata_eval.core.models import AgenticEvalOutcome, DatasetItem
 
 
 def test_dispatch_agentic_passes_agent_id_through_to_alert_skill():
@@ -108,7 +108,9 @@ def _item(test_kind: str = "agentic_alert_skill") -> DatasetItem:
 def test_run_agentic_items_surfaces_reasoning_steps_on_pass():
     with patch(
         "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill",
-        return_value=(["it created the alert"], "conv-1", "resp-1"),
+        return_value=AgenticEvalOutcome(
+            reasoning_steps=["it created the alert"], conversation_id="conv-1", response_id="resp-1"
+        ),
     ):
         report = run_agentic_items(
             [_item()],