From 4585e38f4c5b172aebc1af87044496290ae18853 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 09:43:15 +0200 Subject: [PATCH 1/3] feat(gooddata-eval): support targeting a specific AI Hub agent GoodData has no admin-settable "default agent": when a conversation doesn't name one, the platform picks whichever agent was last used or last edited in that workspace. Orgs with multiple AI Hub agents (e.g. one scoped to visualization only) can end up silently evaluating the wrong one -- a metric_skill/alert_skill item run against a visualization-only agent never passes, regardless of question quality. ChatClient never sent an agentId at all, so there was no way to pick. - ChatClient gains an `agent_id` param; `create_conversation()` sends `{"agentId": ...}` only when set -- omitted entirely when None, so existing behavior (platform's own default-agent resolution) is unchanged unless the caller opts in. - Threaded through all 7 run_agentic_*/evaluate_agentic_* pairs, the agentic-dispatch layer (_dispatch_agentic/run_agentic_items), and the non-agentic ChatClient construction in cli/main.py. - New `gd-eval run --agent-id ID` flag (or `GD_EVAL_AGENT_ID` env var, same precedence convention as --token/GOODDATA_TOKEN). - README: new flags-table row + a "Targeting a specific AI Hub agent" section with real usage examples. - Tests: ChatClient POST-body shape (with/without agent_id), CLI arg parsing, flag/env-var/unset precedence into the constructed ChatClient, and _dispatch_agentic threading it to evaluate_agentic_*. Co-Authored-By: Claude Sonnet 5 --- packages/gooddata-eval/README.md | 35 ++++++ .../src/gooddata_eval/cli/agentic_runner.py | 13 ++- .../src/gooddata_eval/cli/main.py | 14 +++ .../gooddata_eval/core/agentic/alert_skill.py | 7 +- .../core/agentic/conversation.py | 7 +- .../core/agentic/general_question.py | 7 +- .../gooddata_eval/core/agentic/guardrail.py | 7 +- .../core/agentic/metric_skill.py | 7 +- .../gooddata_eval/core/agentic/search_tool.py | 7 +- .../core/agentic/visualization.py | 7 +- .../src/gooddata_eval/core/chat/sse_client.py | 7 +- .../src/gooddata_eval/core/config.py | 1 + .../tests/test_agentic_runner.py | 50 ++++++++ packages/gooddata-eval/tests/test_cli.py | 110 ++++++++++++++++++ .../gooddata-eval/tests/test_sse_client.py | 26 +++++ 15 files changed, 296 insertions(+), 9 deletions(-) create mode 100644 packages/gooddata-eval/tests/test_agentic_runner.py diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index b17db60cb..e5f321027 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -62,6 +62,40 @@ When the same model id is offered by multiple providers, use the Both provider name and provider id are accepted as the prefix. +### Targeting a specific AI Hub agent + +GoodData has no admin-settable "default agent": when a conversation doesn't +name one, the platform picks whichever agent was last used or last edited in +that workspace. If your org has several AI Hub agents configured (e.g. one +scoped to visualization only, another with every skill enabled), evaluating +without `--agent-id` can silently exercise the wrong one — a +`metric_skill`/`alert_skill` item run against a visualization-only agent will +never pass, no matter how well-formed the question is. + +```bash +export GD_EVAL_AGENT_ID='eval-all-skills' + +gd-eval run \ + --host https://your.gooddata.cloud \ + --workspace ecommerce_demo \ + --dataset ./my-dataset \ + --model gpt-5.2 \ + --runs 1 \ + --json results.json +``` + +Or pass it explicitly instead of via the env var: + +```bash +gd-eval run \ + --host https://your.gooddata.cloud \ + --workspace ecommerce_demo \ + --dataset ./my-dataset \ + --agent-id eval-all-skills \ + --model gpt-5.2 \ + --runs 1 +``` + ### All flags #### Connection @@ -72,6 +106,7 @@ Both provider name and provider id are accepted as the prefix. | `--token TOKEN` | `GOODDATA_TOKEN` | API token. Pass via flag or env var. | | `--profile NAME` | — | Profile name in `~/.gooddata/profiles.yaml` (same file as the `gdc` CLI). | | `--workspace ID` | — | **Required.** Workspace id to evaluate against. | +| `--agent-id ID` | `GD_EVAL_AGENT_ID` | AI Hub agent every conversation should target. GoodData has no admin-settable default agent — without this, each conversation falls back to whichever agent the platform's last-used/last-edited heuristic resolves, which may not have every skill under test enabled. | #### Dataset source (pick one) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 7147af183..b379fadd9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -85,6 +85,7 @@ def _dispatch_agentic( run_ts: str, model_version_override: str | None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> None: """Call the appropriate evaluate_agentic_* function for the item's test_kind.""" kind = item.test_kind @@ -106,6 +107,7 @@ def _dispatch_agentic( question=item.question, expected_outputs=_parse_visualization_expected(eo), k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_metric_skill": @@ -116,6 +118,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, (dict, list)) else {}, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_alert_skill": @@ -126,6 +129,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, dict) else {}, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_search": @@ -139,6 +143,7 @@ def _dispatch_agentic( question=item.question, expected_tool_call=expected_args, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_general_question": @@ -149,6 +154,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, str) else str(eo), k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_guardrail": @@ -159,6 +165,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, str) else str(eo), k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_kda_skill": @@ -178,6 +185,7 @@ def _dispatch_agentic( token=token, workspace_id=workspace_id, fixture=ConversationFixture.model_validate(fixture_data), + agent_id=agent_id, **lf_kw, ) else: @@ -197,6 +205,7 @@ def run_agentic_items( run_ts: str, on_item_start: Any = None, on_item_done: Any = None, + agent_id: str | None = None, ) -> EvalReport: """Run agentic items through evaluate_agentic_* and return an EvalReport.""" langfuse = make_langfuse_client() if use_langfuse else None @@ -219,7 +228,9 @@ def run_agentic_items( ) t0 = time.perf_counter() try: - _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort) + _dispatch_agentic( + item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id + ) item_report.pass_at_k = True item_report.runs = k except AssertionError as exc: diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 9465602c0..1e40efd77 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -2,6 +2,7 @@ """`gd-eval` command-line entry point.""" import argparse +import os import sys import threading from datetime import datetime, timezone @@ -117,6 +118,16 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).", ) + run.add_argument( + "--agent-id", + dest="agent_id", + help=( + "AI Hub agent id every conversation should target (or set GD_EVAL_AGENT_ID). " + "GoodData has no admin-settable default agent -- without this, each conversation " + "falls back to whichever agent the platform's last-used/last-edited heuristic " + "resolves, which may not have every skill under test enabled." + ), + ) models_cmd = sub.add_parser("models", help="List LLM providers and models configured in the org.") models_cmd.add_argument("--host", help="GoodData host URL.") models_cmd.add_argument("--token", help="API token (or set GOODDATA_TOKEN).") @@ -347,6 +358,7 @@ def on_langfuse_item_done( run_ts=run_ts, on_item_start=on_item_start, on_item_done=on_item_done, + agent_id=config.agent_id, ) # --- non-agentic items (single-turn, use Evaluator) --- @@ -357,6 +369,7 @@ def on_langfuse_item_done( workspace_id=config.workspace_id, preserve_failed=config.preserve_failed, reasoning_effort=config.reasoning_effort, + agent_id=config.agent_id, ), SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id), ) @@ -449,6 +462,7 @@ def main(argv: list[str] | None = None) -> int: kind=args.kind, preserve_failed=args.preserve_failed, reasoning_effort=args.reasoning_effort, + agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"), ) return _run(config) except ( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 3583780bd..e94584f39 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -434,11 +434,14 @@ def run_agentic_alert_skill( max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticAlertSummary: """Run the alert-skill agentic evaluation K times and return a summary.""" expected = _normalize_expected_output(expected_output) run_results: list[AlertRunResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) sdk = GoodDataSdk.create(host, token) def _run_once(conv_id: str) -> AlertRunResult: @@ -550,6 +553,7 @@ def evaluate_agentic_alert_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "alert_skill", @@ -577,6 +581,7 @@ def evaluate_agentic_alert_skill( max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 876bbb1c6..1e50352c2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -275,6 +275,7 @@ def run_agentic_conversation( max_clarification_turns: int = _DEFAULT_MAX_CLARIFICATION_TURNS, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> ConversationResult: """Run a multi-turn, multi-skill conversation evaluation (no K-runs). @@ -282,7 +283,9 @@ def run_agentic_conversation( trigger up to *max_clarification_turns* additional rounds of simulated-user replies before the agent produces the expected output. """ - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) sdk = GoodDataSdk.create(host, token) turn_results: list[TurnResult] = [] turn_outputs: dict[str, dict] = {} @@ -408,6 +411,7 @@ def evaluate_agentic_conversation( fixture: ConversationFixture, max_clarification_turns: int = _DEFAULT_MAX_CLARIFICATION_TURNS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "conversation", @@ -433,6 +437,7 @@ def evaluate_agentic_conversation( max_clarification_turns=max_clarification_turns, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index f2aa494ff..d710b2a45 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -73,10 +73,13 @@ def run_agentic_general_question( k: int = _DEFAULT_K, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticGeneralQuestionSummary: """Run the general-question agentic evaluation K times and return a summary.""" run_results: list[GeneralQuestionResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) judge = LLMJudge(_GENERAL_QUESTION_EVALUATION_STEPS, model="gpt-4o") try: @@ -149,6 +152,7 @@ def evaluate_agentic_general_question( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "general_question", @@ -175,6 +179,7 @@ def evaluate_agentic_general_question( k=k, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index ae74da5b4..aea303360 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -70,10 +70,13 @@ def run_agentic_guardrail( k: int = _DEFAULT_K, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticGuardrailSummary: """Run the guardrail agentic evaluation K times and return a summary.""" run_results: list[GuardrailResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) judge = LLMJudge(_GUARDRAIL_EVALUATION_STEPS, model="gpt-4o") try: @@ -146,6 +149,7 @@ def evaluate_agentic_guardrail( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "guardrail", @@ -172,6 +176,7 @@ def evaluate_agentic_guardrail( k=k, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index dc12a2b7f..6d882d286 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -249,6 +249,7 @@ def run_agentic_metric_skill( max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticMetricSummary: """Run the metric-skill agentic evaluation K times and return a summary. @@ -257,7 +258,9 @@ def run_agentic_metric_skill( """ expected_outputs: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output] run_results: list[MetricRunResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) sdk = GoodDataSdk.create(host, token) try: @@ -311,6 +314,7 @@ def evaluate_agentic_metric_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "metric_skill", @@ -338,6 +342,7 @@ def evaluate_agentic_metric_skill( max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index 25053e17b..955818c0e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -69,11 +69,14 @@ def run_agentic_search_tool( k: int = _DEFAULT_K, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticSearchSummary: """Run the search-tool agentic evaluation K times (single-turn each).""" run_results: list[SearchResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: @@ -140,6 +143,7 @@ def evaluate_agentic_search_tool( expected_tool_call: dict, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "search", @@ -166,6 +170,7 @@ def evaluate_agentic_search_tool( k=k, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 3548b22b7..60134779a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -205,6 +205,7 @@ def run_agentic_visualization( max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticRunSummary: """Run K independent conversations and return evaluation results. @@ -213,7 +214,9 @@ def run_agentic_visualization( fresh conversations. Caller-supplied conversations are not deleted; all conversations created by this function are deleted on completion. """ - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) run_results: list[RunResult] = [] try: @@ -260,6 +263,7 @@ def evaluate_agentic_visualization( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "visualization", @@ -289,6 +293,7 @@ def evaluate_agentic_visualization( max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index 562b466b7..d131b82cc 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -293,6 +293,7 @@ def __init__( timeout: float = 300.0, preserve_failed: bool = False, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ): """Create a chat client bound to one workspace. @@ -307,10 +308,14 @@ def __init__( self._client = httpx.Client(timeout=timeout) self._preserve_failed = preserve_failed self._reasoning_effort = normalize_reasoning_effort(reasoning_effort) + self._agent_id = agent_id def create_conversation(self) -> str: def _do() -> str: - resp = self._client.post(self._base, headers={**self._auth, "Content-Type": "application/json"}) + body = {"agentId": self._agent_id} if self._agent_id else {} + resp = self._client.post( + self._base, headers={**self._auth, "Content-Type": "application/json"}, json=body + ) resp.raise_for_status() body = resp.json() if "conversationId" not in body: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index 8cb07794f..06a2dd926 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -44,3 +44,4 @@ class RunConfig: kind: str = "visualization" preserve_failed: bool = False reasoning_effort: ReasoningEffort | None = None + agent_id: str | None = None diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py new file mode 100644 index 000000000..dbbc3443e --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -0,0 +1,50 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +from unittest.mock import patch + +from gooddata_eval.cli.agentic_runner import _dispatch_agentic +from gooddata_eval.core.models import DatasetItem + + +def test_dispatch_agentic_passes_agent_id_through_to_alert_skill(): + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind="agentic_alert_skill", + question="Alert me when spend exceeds 100", + expected_output={"Operator": "GREATER_THAN", "Threshold": 100}, + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + agent_id="agent-1", + ) + assert mock_eval.call_args.kwargs["agent_id"] == "agent-1" + + +def test_dispatch_agentic_omits_agent_id_by_default(): + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind="agentic_metric_skill", + question="Create a metric for total spend", + expected_output={"maql": "SELECT {metric/spend}"}, + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_metric_skill") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + ) + assert mock_eval.call_args.kwargs["agent_id"] is None diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 3f42a3f4e..9956ad31f 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -26,6 +26,16 @@ def test_build_run_config_requires_a_source(): cli_main.parse_args(["run", "--host", "h", "--workspace", "w"]) +def test_parse_args_agent_id_flag(): + args = cli_main.parse_args(["run", "--host", "h", "--workspace", "w", "--dataset", "d", "--agent-id", "agent-1"]) + assert args.agent_id == "agent-1" + + +def test_parse_args_agent_id_defaults_to_none(): + args = cli_main.parse_args(["run", "--host", "h", "--workspace", "w", "--dataset", "d"]) + assert args.agent_id is None + + def test_cli_run_end_to_end(monkeypatch, tmp_path, fixtures_dir): # Stub connection + model activation + chat backend so no network is needed. monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) @@ -93,6 +103,106 @@ def _fake_run( assert orjson.loads(out.read_bytes())["runs"]["Test Provider/gpt-5.2"]["summary"]["passed"] == 1 +def _stub_run_for_agent_id_test(monkeypatch, seen_chat_client_kwargs): + monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) + + class _FakeController: + def __init__(self, *a, **k): ... + def get_active(self): + return ActiveLlmProvider(provider_id="prov", default_model_id="gpt-5.2") + + def resolve_and_activate(self, requested, provider=None): + return ResolvedModel( + provider_id="prov", model_id=requested or "gpt-5.2", switched=False, provider_name="Test Provider" + ) + + def restore(self, original): ... + def close(self): ... + + monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) + + def _fake_run(items, backend, *, runs, model, workspace_id, **kw): + return EvalReport(model=model, workspace_id=workspace_id, items=[]) + + monkeypatch.setattr(cli_main, "run_items", _fake_run) + + def _spy_chat_client(**kwargs): + seen_chat_client_kwargs.update(kwargs) + return object() + + monkeypatch.setattr(cli_main, "ChatClient", _spy_chat_client) + + +def test_cli_run_passes_agent_id_flag_to_chat_client(monkeypatch, tmp_path, fixtures_dir): + seen = {} + _stub_run_for_agent_id_test(monkeypatch, seen) + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--agent-id", + "agent-1", + "--json", + str(tmp_path / "res.json"), + ] + ) + assert exit_code == 0 + assert seen["agent_id"] == "agent-1" + + +def test_cli_run_falls_back_to_agent_id_env_var(monkeypatch, tmp_path, fixtures_dir): + monkeypatch.setenv("GD_EVAL_AGENT_ID", "agent-from-env") + seen = {} + _stub_run_for_agent_id_test(monkeypatch, seen) + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--json", + str(tmp_path / "res.json"), + ] + ) + assert exit_code == 0 + assert seen["agent_id"] == "agent-from-env" + + +def test_cli_run_agent_id_omitted_when_unset(monkeypatch, tmp_path, fixtures_dir): + monkeypatch.delenv("GD_EVAL_AGENT_ID", raising=False) + seen = {} + _stub_run_for_agent_id_test(monkeypatch, seen) + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--json", + str(tmp_path / "res.json"), + ] + ) + assert exit_code == 0 + assert seen["agent_id"] is None + + def test_cli_operational_error_exits_nonzero(monkeypatch, fixtures_dir): def _boom(host, token, profile): raise ConnectionError_("Missing token.") diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 88be58abe..6361410ba 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -388,6 +388,32 @@ def handler(request): assert sleeps == [] +def test_create_conversation_omits_agent_id_by_default(): + # No agent_id given -> unchanged, existing behavior: GoodData's own + # last-used/last-edited default-agent resolution still applies. + seen = {} + + def handler(request): + seen["body"] = request.content + return httpx.Response(200, json={"conversationId": "abc"}) + + client = _client_with_handler(handler) + client.create_conversation() + assert seen["body"] in (b"", b"{}") + + +def test_create_conversation_sends_agent_id_when_given(): + seen = {} + + def handler(request): + seen["body"] = json.loads(request.content) + return httpx.Response(200, json={"conversationId": "abc"}) + + client = _client_with_handler(handler, agent_id="agent-123") + client.create_conversation() + assert seen["body"] == {"agentId": "agent-123"} + + def test_int_env_uses_default_when_unset(monkeypatch): monkeypatch.delenv("GD_TEST_INT", raising=False) assert sse_mod._int_env("GD_TEST_INT", 5) == 5 From fd4922564b896ac85e64acd075f541582f6fc5fe Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 09:48:36 +0200 Subject: [PATCH 2/3] fix: satisfy ruff format/lint on CI - sse_client.py: reformat create_conversation() to ruff's line-length rule (the PR's own change). - test_cli.py: drop an unused `original_chat_client` local -- pre-existing on master (same line, unrelated to this PR), but ruff check runs whole-file and blocks this PR's lint-and-format-check job since this test function lives in a file the PR also touches. Co-Authored-By: Claude Sonnet 5 --- .../gooddata-eval/src/gooddata_eval/core/chat/sse_client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index d131b82cc..d1e375c97 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -313,9 +313,7 @@ def __init__( def create_conversation(self) -> str: def _do() -> str: body = {"agentId": self._agent_id} if self._agent_id else {} - resp = self._client.post( - self._base, headers={**self._auth, "Content-Type": "application/json"}, json=body - ) + resp = self._client.post(self._base, headers={**self._auth, "Content-Type": "application/json"}, json=body) resp.raise_for_status() body = resp.json() if "conversationId" not in body: From f2764acb6cb6d57fe73c5e245e776bd606421dc6 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 09:53:46 +0200 Subject: [PATCH 3/3] test: cover agent_id threading for every agentic test kind codecov flagged one uncovered line in _dispatch_agentic's agent_id threading -- the earlier tests only exercised 2 of 7 kind branches. Parametrized test now covers all 7 (vis_agentic, agentic_visualization, agentic_search, agentic_general_question, agentic_guardrail, agentic_conversation, plus the two already covered). Co-Authored-By: Claude Sonnet 5 --- .../tests/test_agentic_runner.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index dbbc3443e..3cbff3137 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -1,6 +1,7 @@ # (C) 2026 GoodData Corporation. All rights reserved. from unittest.mock import patch +import pytest from gooddata_eval.cli.agentic_runner import _dispatch_agentic from gooddata_eval.core.models import DatasetItem @@ -48,3 +49,45 @@ def test_dispatch_agentic_omits_agent_id_by_default(): model_version_override=None, ) assert mock_eval.call_args.kwargs["agent_id"] is None + + +_MIN_VIZ = {"id": "v1", "type": "table", "query": {"fields": {}, "filter_by": {}}, "metrics": [], "view_by": []} +_MIN_CONVERSATION_FIXTURE = { + "id": "c1", + "expected_skills": ["visualization"], + "turns": [{"turn_id": "t1", "message": "hi", "expected_skill": "visualization"}], +} + + +@pytest.mark.parametrize( + ("kind", "expected_output", "target"), + [ + ("vis_agentic", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), + ("agentic_visualization", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), + ("agentic_search", {"tool_call": {"function_arguments": {}}}, "evaluate_agentic_search_tool"), + ("agentic_general_question", "What is X?", "evaluate_agentic_general_question"), + ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), + ("agentic_conversation", {"fixture": _MIN_CONVERSATION_FIXTURE}, "evaluate_agentic_conversation"), + ], +) +def test_dispatch_agentic_passes_agent_id_through_for_every_kind(kind, expected_output, target): + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind=kind, + question="q", + expected_output=expected_output, + ) + with patch(f"gooddata_eval.cli.agentic_runner.{target}") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + agent_id="agent-1", + ) + assert mock_eval.call_args.kwargs["agent_id"] == "agent-1"