From c5e39eba8f36d97c2d8c907544fd61988b1cb0c1 Mon Sep 17 00:00:00 2001 From: Andrew Klatzke Date: Mon, 4 May 2026 14:31:59 -0800 Subject: [PATCH] feat: ability to specify variation key on config or from_options for optimization package --- .../optimization/src/ldai_optimizer/client.py | 93 +++++- .../src/ldai_optimizer/dataclasses.py | 6 + .../src/ldai_optimizer/ld_api_client.py | 24 ++ packages/optimization/tests/test_client.py | 315 ++++++++++++++++++ .../optimization/tests/test_ld_api_client.py | 75 ++++- 5 files changed, 499 insertions(+), 14 deletions(-) diff --git a/packages/optimization/src/ldai_optimizer/client.py b/packages/optimization/src/ldai_optimizer/client.py index c7927d4c..a7006553 100644 --- a/packages/optimization/src/ldai_optimizer/client.py +++ b/packages/optimization/src/ldai_optimizer/client.py @@ -824,29 +824,61 @@ async def _evaluate_acceptance_judge( return dataclasses.replace(judge_result, duration_ms=judge_duration_ms, usage=judge_response.usage) async def _get_agent_config( - self, agent_key: str, context: Context + self, + agent_key: str, + context: Context, + variation_key: Optional[str] = None, + project_key: Optional[str] = None, + api_client: Optional["LDApiClient"] = None, + base_url: Optional[str] = None, ) -> AIAgentConfig: """ Fetch the agent configuration, replacing the instructions with the raw variation template so that {{placeholder}} tokens are preserved for client-side interpolation. agent_config() is called normally so we get a fully populated AIAgentConfig - (including the tracker). We then call variation() separately to retrieve the - unrendered instruction template and swap it in, keeping everything else intact. + (including the tracker). When variation_key is set, the specific variation's + data (instructions, model, tools) is fetched via the REST API and used as the + base instead of the SDK-evaluated default. Otherwise, variation() is called to + retrieve the unrendered instruction template for the SDK-evaluated variation. :param agent_key: The key for the agent to get the configuration for :param context: The evaluation context + :param variation_key: If set, fetch this specific variation from the API as the base. + :param project_key: Required when variation_key is set. + :param api_client: Optional pre-built LDApiClient to reuse (e.g. from optimize_from_config). + :param base_url: Optional base URL override for a newly created LDApiClient. :return: AIAgentConfig with raw {{placeholder}} instruction templates intact """ try: agent_config = self._ldClient.agent_config(agent_key, context) - # variation() returns the raw JSON before chevron.render(), so instructions - # still contain {{placeholder}} tokens rather than empty strings. - raw_variation = self._ldClient._client.variation(agent_key, context, {}) - raw_instructions = raw_variation.get( - "instructions", agent_config.instructions - ) + if variation_key: + # Fetch the specific variation from the REST API so instructions, + # model, and tools all come from the requested base variation rather + # than whatever the SDK evaluates for the given context. + client = api_client or LDApiClient( + self._api_key, # type: ignore[arg-type] + **({"base_url": base_url} if base_url else {}), + ) + variation_data = client.get_ai_config_variation(project_key, agent_key, variation_key) # type: ignore[arg-type] + raw_instructions = variation_data.get("instructions") or "" + raw_tools = variation_data.get("tools") or [] + model_config_key = variation_data.get("modelConfigKey") or "" + if model_config_key: + agent_config = dataclasses.replace( + agent_config, + model=ModelConfig(name=model_config_key, parameters={}), + ) + else: + # variation() returns the raw JSON before chevron.render(), so instructions + # still contain {{placeholder}} tokens rather than empty strings. + raw_variation = self._ldClient._client.variation(agent_key, context, {}) + raw_instructions = raw_variation.get( + "instructions", agent_config.instructions + ) + raw_tools = raw_variation.get("tools", []) + if not raw_instructions: raise ValueError( f"Agent '{agent_key}' has no instructions configured. " @@ -854,7 +886,6 @@ async def _get_agent_config( ) self._initial_instructions = raw_instructions - raw_tools = raw_variation.get("tools", []) self._initial_tool_keys = [ t["key"] for t in raw_tools @@ -888,9 +919,24 @@ async def optimize_from_options( raise ValueError( "auto_commit requires project_key to be set on OptimizationOptions" ) + if options.variation_key: + if not self._has_api_key: + raise ValueError( + "variation_key requires LAUNCHDARKLY_API_KEY to be set" + ) + if not options.project_key: + raise ValueError( + "variation_key requires project_key to be set on OptimizationOptions" + ) self._agent_key = agent_key context = random.choice(options.context_choices) - agent_config = await self._get_agent_config(agent_key, context) + agent_config = await self._get_agent_config( + agent_key, + context, + variation_key=options.variation_key, + project_key=options.project_key, + base_url=options.base_url, + ) result = await self._run_optimization(agent_config, options) if options.auto_commit and self._last_run_succeeded and self._last_succeeded_context: self._commit_variation( @@ -926,9 +972,24 @@ async def optimize_from_ground_truth_options( raise ValueError( "auto_commit requires project_key to be set on GroundTruthOptimizationOptions" ) + if options.variation_key: + if not self._has_api_key: + raise ValueError( + "variation_key requires LAUNCHDARKLY_API_KEY to be set" + ) + if not options.project_key: + raise ValueError( + "variation_key requires project_key to be set on GroundTruthOptimizationOptions" + ) self._agent_key = agent_key context = random.choice(options.context_choices) - agent_config = await self._get_agent_config(agent_key, context) + agent_config = await self._get_agent_config( + agent_key, + context, + variation_key=options.variation_key, + project_key=options.project_key, + base_url=options.base_url, + ) result = await self._run_ground_truth_optimization(agent_config, options) if options.auto_commit and self._last_run_succeeded and self._last_succeeded_context: self._commit_variation( @@ -1425,7 +1486,13 @@ async def optimize_from_config( context = random.choice(options.context_choices) # _get_agent_config calls _initialize_class_members_from_config internally; # _run_optimization calls it again to reset history before the loop starts. - agent_config = await self._get_agent_config(self._agent_key, context) + agent_config = await self._get_agent_config( + self._agent_key, + context, + variation_key=config.get("variationKey"), + project_key=options.project_key, + api_client=api_client, + ) optimization_options = self._build_options_from_config( config, options, api_client, optimization_key, run_id, model_configs diff --git a/packages/optimization/src/ldai_optimizer/dataclasses.py b/packages/optimization/src/ldai_optimizer/dataclasses.py index 9e52e046..490e34d8 100644 --- a/packages/optimization/src/ldai_optimizer/dataclasses.py +++ b/packages/optimization/src/ldai_optimizer/dataclasses.py @@ -342,6 +342,9 @@ class OptimizationOptions: project_key: Optional[str] = None # required when auto_commit=True output_key: Optional[str] = None # variation key/name; auto-generated if omitted base_url: Optional[str] = None # override to target a non-default LD instance + # When set, uses this specific variation as the base instead of the SDK-evaluated default. + # Requires LAUNCHDARKLY_API_KEY to be set and project_key to be provided. + variation_key: Optional[str] = None on_passing_result: Optional[Callable[[OptimizationContext], None]] = None on_failing_result: Optional[Callable[[OptimizationContext], None]] = None # called to provide status updates during the optimization flow @@ -434,6 +437,9 @@ class GroundTruthOptimizationOptions: project_key: Optional[str] = None # required when auto_commit=True output_key: Optional[str] = None # variation key/name; auto-generated if omitted base_url: Optional[str] = None # override to target a non-default LD instance + # When set, uses this specific variation as the base instead of the SDK-evaluated default. + # Requires LAUNCHDARKLY_API_KEY to be set and project_key to be provided. + variation_key: Optional[str] = None token_limit: Optional[int] = None # stop the run when total token usage reaches this value def __post_init__(self): diff --git a/packages/optimization/src/ldai_optimizer/ld_api_client.py b/packages/optimization/src/ldai_optimizer/ld_api_client.py index 3efa725d..d1afbaa6 100644 --- a/packages/optimization/src/ldai_optimizer/ld_api_client.py +++ b/packages/optimization/src/ldai_optimizer/ld_api_client.py @@ -90,6 +90,7 @@ class AgentOptimizationConfig(_AgentOptimizationConfigRequired, total=False): groundTruthResponses: List[str] metricKey: str tokenLimit: int + variationKey: str # --------------------------------------------------------------------------- @@ -287,6 +288,29 @@ def get_ai_config(self, project_key: str, config_key: str) -> Any: path = f"/api/v2/projects/{project_key}/ai-configs/{config_key}" return self._request("GET", path, extra_headers={"LD-API-Version": "beta"}) + def get_ai_config_variation( + self, project_key: str, config_key: str, variation_key: str + ) -> Dict[str, Any]: + """Fetch a specific variation of an AI config by key. + + Returns the first (latest) item from the variations response. + + :param project_key: LaunchDarkly project key. + :param config_key: Key of the AI Config (aiConfigKey). + :param variation_key: Key of the specific variation to fetch. + :return: The variation dict (first item from the ``items`` array). + :raises LDApiError: If the variation is not found or the request fails. + """ + path = f"/api/v2/projects/{project_key}/ai-configs/{config_key}/variations/{variation_key}" + result = self._request("GET", path, extra_headers={"LD-API-Version": "beta"}) + items = result.get("items") if isinstance(result, dict) else None + if not items: + raise LDApiError( + f"Variation '{variation_key}' not found for AI config '{config_key}'.", + path=path, + ) + return items[0] + def create_ai_config_variation( self, project_key: str, config_key: str, payload: Dict[str, Any] ) -> Any: diff --git a/packages/optimization/tests/test_client.py b/packages/optimization/tests/test_client.py index c441eedc..5c4a90f1 100644 --- a/packages/optimization/tests/test_client.py +++ b/packages/optimization/tests/test_client.py @@ -4404,3 +4404,318 @@ async def test_optimization_key_in_post_url_uses_string_key_not_uuid(self): assert opt_key_arg == "my-optimization", ( f"Expected string key 'my-optimization', got '{opt_key_arg}'" ) + + +# --------------------------------------------------------------------------- +# variation_key in _get_agent_config +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +class TestGetAgentConfigVariationKey: + _VARIATION_DATA = { + "key": "my-variation", + "instructions": "Custom variation instructions.", + "modelConfigKey": "OpenAI.gpt-4o-mini", + "tools": [{"key": "search-tool", "version": 1}], + } + + def _make_client_with_key(self) -> OptimizationClient: + with patch.dict("os.environ", {"LAUNCHDARKLY_API_KEY": "test-api-key"}): + return OptimizationClient(_make_ldai_client()) + + async def test_sdk_path_used_when_no_variation_key(self): + """Without variation_key, the existing SDK variation() call is used.""" + client = _make_client() + await client._get_agent_config("test-agent", LD_CONTEXT) + client._ldClient._client.variation.assert_called_once_with("test-agent", LD_CONTEXT, {}) + + async def test_api_path_used_when_variation_key_set(self): + """With variation_key, get_ai_config_variation is called instead of SDK variation().""" + client = self._make_client_with_key() + mock_api = MagicMock() + mock_api.get_ai_config_variation.return_value = self._VARIATION_DATA + + await client._get_agent_config( + "test-agent", LD_CONTEXT, + variation_key="my-variation", + project_key="my-project", + api_client=mock_api, + ) + + mock_api.get_ai_config_variation.assert_called_once_with( + "my-project", "test-agent", "my-variation" + ) + client._ldClient._client.variation.assert_not_called() + + async def test_instructions_come_from_api_variation(self): + client = self._make_client_with_key() + mock_api = MagicMock() + mock_api.get_ai_config_variation.return_value = self._VARIATION_DATA + + await client._get_agent_config( + "test-agent", LD_CONTEXT, + variation_key="my-variation", + project_key="my-project", + api_client=mock_api, + ) + + assert client._initial_instructions == "Custom variation instructions." + + async def test_model_replaced_from_api_variation_model_config_key(self): + client = self._make_client_with_key() + mock_api = MagicMock() + mock_api.get_ai_config_variation.return_value = self._VARIATION_DATA + + config = await client._get_agent_config( + "test-agent", LD_CONTEXT, + variation_key="my-variation", + project_key="my-project", + api_client=mock_api, + ) + + assert config.model is not None + assert config.model.name == "OpenAI.gpt-4o-mini" + + async def test_tool_keys_extracted_from_api_variation(self): + client = self._make_client_with_key() + mock_api = MagicMock() + mock_api.get_ai_config_variation.return_value = self._VARIATION_DATA + + await client._get_agent_config( + "test-agent", LD_CONTEXT, + variation_key="my-variation", + project_key="my-project", + api_client=mock_api, + ) + + assert client._initial_tool_keys == ["search-tool"] + + async def test_api_client_built_from_base_url_when_not_provided(self): + """When no api_client is passed, a new LDApiClient is created with the given base_url.""" + client = self._make_client_with_key() + variation_data = {**self._VARIATION_DATA} + + with patch("ldai_optimizer.client.LDApiClient") as mock_cls: + mock_instance = MagicMock() + mock_instance.get_ai_config_variation.return_value = variation_data + mock_cls.return_value = mock_instance + + await client._get_agent_config( + "test-agent", LD_CONTEXT, + variation_key="my-variation", + project_key="my-project", + base_url="https://staging.launchdarkly.com", + ) + + mock_cls.assert_called_once_with( + "test-api-key", base_url="https://staging.launchdarkly.com" + ) + + async def test_api_error_propagates_without_sdk_fallback(self): + """If the API call fails, the error propagates — no silent fallback to SDK default.""" + from ldai_optimizer.ld_api_client import LDApiError + client = self._make_client_with_key() + mock_api = MagicMock() + mock_api.get_ai_config_variation.side_effect = LDApiError( + "Variation 'bad-key' not found", status_code=404 + ) + + with pytest.raises(Exception): + await client._get_agent_config( + "test-agent", LD_CONTEXT, + variation_key="bad-key", + project_key="my-project", + api_client=mock_api, + ) + + client._ldClient._client.variation.assert_not_called() + + +# --------------------------------------------------------------------------- +# variation_key in optimize_from_options +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +class TestVariationKeyInOptimizeFromOptions: + def _make_client_with_key(self) -> OptimizationClient: + with patch.dict("os.environ", {"LAUNCHDARKLY_API_KEY": "test-api-key"}): + return OptimizationClient(_make_ldai_client()) + + def _make_client_without_key(self) -> OptimizationClient: + client = OptimizationClient(_make_ldai_client()) + client._has_api_key = False + client._api_key = None + return client + + async def test_variation_key_forwarded_to_get_agent_config(self): + client = self._make_client_with_key() + options = _make_options(variation_key="custom-var", project_key="my-project") + + with patch.object(client, "_get_agent_config", wraps=client._get_agent_config) as spy: + mock_api = MagicMock() + mock_api.get_ai_config_variation.return_value = { + "key": "custom-var", + "instructions": AGENT_INSTRUCTIONS, + "modelConfigKey": "OpenAI.gpt-4o", + "tools": [], + } + with patch("ldai_optimizer.client.LDApiClient", return_value=mock_api): + await client.optimize_from_options("test-agent", options) + + call_kwargs = spy.call_args[1] + assert call_kwargs.get("variation_key") == "custom-var" + assert call_kwargs.get("project_key") == "my-project" + + async def test_raises_when_variation_key_set_without_api_key(self): + client = self._make_client_without_key() + options = _make_options(variation_key="custom-var", project_key="my-project") + + with pytest.raises(ValueError, match="LAUNCHDARKLY_API_KEY"): + await client.optimize_from_options("test-agent", options) + + async def test_raises_when_variation_key_set_without_project_key(self): + client = self._make_client_with_key() + options = _make_options(variation_key="custom-var", project_key=None) + + with pytest.raises(ValueError, match="project_key"): + await client.optimize_from_options("test-agent", options) + + async def test_no_validation_error_when_variation_key_not_set(self): + """Omitting variation_key should not trigger any new validation errors.""" + client = self._make_client_without_key() + options = _make_options() # variation_key=None by default + + result = await client.optimize_from_options("test-agent", options) + assert result is not None + + +# --------------------------------------------------------------------------- +# variation_key in optimize_from_ground_truth_options +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +class TestVariationKeyInOptimizeFromGroundTruthOptions: + def _make_client_with_key(self) -> OptimizationClient: + with patch.dict("os.environ", {"LAUNCHDARKLY_API_KEY": "test-api-key"}): + return OptimizationClient(_make_ldai_client()) + + def _make_client_without_key(self) -> OptimizationClient: + client = OptimizationClient(_make_ldai_client()) + client._has_api_key = False + client._api_key = None + return client + + async def test_variation_key_forwarded_to_get_agent_config(self): + client = self._make_client_with_key() + opts = _make_gt_options(variation_key="custom-var", project_key="my-project") + + with patch.object(client, "_get_agent_config", wraps=client._get_agent_config) as spy: + mock_api = MagicMock() + mock_api.get_ai_config_variation.return_value = { + "key": "custom-var", + "instructions": AGENT_INSTRUCTIONS, + "modelConfigKey": "OpenAI.gpt-4o", + "tools": [], + } + with patch("ldai_optimizer.client.LDApiClient", return_value=mock_api): + await client.optimize_from_ground_truth_options("test-agent", opts) + + call_kwargs = spy.call_args[1] + assert call_kwargs.get("variation_key") == "custom-var" + assert call_kwargs.get("project_key") == "my-project" + + async def test_raises_when_variation_key_set_without_api_key(self): + client = self._make_client_without_key() + opts = _make_gt_options(variation_key="custom-var", project_key="my-project") + + with pytest.raises(ValueError, match="LAUNCHDARKLY_API_KEY"): + await client.optimize_from_ground_truth_options("test-agent", opts) + + async def test_raises_when_variation_key_set_without_project_key(self): + client = self._make_client_with_key() + opts = _make_gt_options(variation_key="custom-var", project_key=None) + + with pytest.raises(ValueError, match="project_key"): + await client.optimize_from_ground_truth_options("test-agent", opts) + + +# --------------------------------------------------------------------------- +# variation_key in optimize_from_config +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +class TestVariationKeyInOptimizeFromConfig: + def _make_client_with_key(self) -> OptimizationClient: + with patch.dict("os.environ", {"LAUNCHDARKLY_API_KEY": "test-api-key"}): + return OptimizationClient(_make_ldai_client()) + + async def test_variation_key_from_config_forwarded_to_get_agent_config(self): + """variationKey from the fetched AgentOptimization config is passed to _get_agent_config.""" + client = self._make_client_with_key() + api_config = dict(_API_CONFIG, variationKey="base-variation") + mock_api = _make_mock_api_client() + mock_api.get_agent_optimization = MagicMock(return_value=api_config) + mock_api.get_ai_config_variation = MagicMock(return_value={ + "key": "base-variation", + "instructions": AGENT_INSTRUCTIONS, + "modelConfigKey": "OpenAI.gpt-4o", + "tools": [], + }) + + with patch("ldai_optimizer.client.LDApiClient", return_value=mock_api): + with patch.object(client, "_get_agent_config", wraps=client._get_agent_config) as spy: + await client.optimize_from_config("my-opt", _make_from_config_options()) + + call_kwargs = spy.call_args[1] + assert call_kwargs.get("variation_key") == "base-variation" + assert call_kwargs.get("project_key") == "my-project" + + async def test_variation_key_none_when_not_in_config(self): + """When variationKey is absent from the config, None is passed to _get_agent_config.""" + client = self._make_client_with_key() + api_config = dict(_API_CONFIG) # no variationKey + assert "variationKey" not in api_config + mock_api = _make_mock_api_client() + mock_api.get_agent_optimization = MagicMock(return_value=api_config) + + with patch("ldai_optimizer.client.LDApiClient", return_value=mock_api): + with patch.object(client, "_get_agent_config", wraps=client._get_agent_config) as spy: + await client.optimize_from_config("my-opt", _make_from_config_options()) + + call_kwargs = spy.call_args[1] + assert call_kwargs.get("variation_key") is None + + async def test_prebuilt_api_client_passed_to_get_agent_config(self): + """The api_client constructed in optimize_from_config is reused in _get_agent_config.""" + client = self._make_client_with_key() + mock_api = _make_mock_api_client() + mock_api.get_agent_optimization = MagicMock(return_value=dict(_API_CONFIG)) + + with patch("ldai_optimizer.client.LDApiClient", return_value=mock_api): + with patch.object(client, "_get_agent_config", wraps=client._get_agent_config) as spy: + await client.optimize_from_config("my-opt", _make_from_config_options()) + + call_kwargs = spy.call_args[1] + assert call_kwargs.get("api_client") is mock_api + + async def test_api_variation_instructions_used_as_base(self): + """When variationKey is set, the base instructions come from the API variation.""" + client = self._make_client_with_key() + api_config = dict(_API_CONFIG, variationKey="base-variation") + mock_api = _make_mock_api_client() + mock_api.get_agent_optimization = MagicMock(return_value=api_config) + mock_api.get_ai_config_variation = MagicMock(return_value={ + "key": "base-variation", + "instructions": "Instructions from specific variation.", + "modelConfigKey": "OpenAI.gpt-4o", + "tools": [], + }) + + with patch("ldai_optimizer.client.LDApiClient", return_value=mock_api): + await client.optimize_from_config("my-opt", _make_from_config_options()) + + assert client._initial_instructions == "Instructions from specific variation." diff --git a/packages/optimization/tests/test_ld_api_client.py b/packages/optimization/tests/test_ld_api_client.py index 4faa750b..e2c11cb8 100644 --- a/packages/optimization/tests/test_ld_api_client.py +++ b/packages/optimization/tests/test_ld_api_client.py @@ -65,11 +65,17 @@ def test_valid_config_is_returned_unchanged(self): def test_optional_fields_not_required(self): config = _make_config() - # groundTruthResponses and metricKey are optional — should not raise + # groundTruthResponses, metricKey, and variationKey are optional — should not raise assert "groundTruthResponses" not in config assert "metricKey" not in config + assert "variationKey" not in config _parse_agent_optimization(config) # must not raise + def test_variation_key_passes_through_when_present(self): + config = _make_config(variationKey="my-base-variation") + result = _parse_agent_optimization(config) + assert result.get("variationKey") == "my-base-variation" + def test_raises_on_non_dict_input(self): with pytest.raises(ValueError, match="Expected a JSON object"): _parse_agent_optimization(["not", "a", "dict"]) @@ -369,3 +375,70 @@ def test_succeeds_on_retry_after_transient_error(self): result = client._request("GET", "/path") assert result == {"result": "ok"} assert mock_open.call_count == 2 + + +# --------------------------------------------------------------------------- +# LDApiClient.get_ai_config_variation +# --------------------------------------------------------------------------- + + +class TestGetAiConfigVariation: + _VARIATION = { + "key": "my-variation", + "name": "My Variation", + "instructions": "You are a helpful assistant.", + "modelConfigKey": "OpenAI.gpt-4o", + "tools": [{"key": "search-tool", "version": 1}], + } + + def test_requests_correct_path(self): + client = LDApiClient("test-key") + response = {"items": [self._VARIATION], "totalCount": 1} + with patch("urllib.request.urlopen", return_value=_mock_urlopen(response)) as mock_open: + client.get_ai_config_variation("my-project", "my-agent", "my-variation") + req: urllib.request.Request = mock_open.call_args[0][0] + assert ( + "/api/v2/projects/my-project/ai-configs/my-agent/variations/my-variation" + in req.full_url + ) + + def test_sends_ld_api_version_header(self): + client = LDApiClient("test-key") + response = {"items": [self._VARIATION], "totalCount": 1} + with patch("urllib.request.urlopen", return_value=_mock_urlopen(response)) as mock_open: + client.get_ai_config_variation("my-project", "my-agent", "my-variation") + req: urllib.request.Request = mock_open.call_args[0][0] + assert req.get_header("Ld-api-version") == "beta" + + def test_returns_first_item_from_response(self): + client = LDApiClient("test-key") + second_variation = {**self._VARIATION, "name": "Older version"} + response = {"items": [self._VARIATION, second_variation], "totalCount": 2} + with patch("urllib.request.urlopen", return_value=_mock_urlopen(response)): + result = client.get_ai_config_variation("my-project", "my-agent", "my-variation") + assert result["name"] == "My Variation" + assert result["instructions"] == "You are a helpful assistant." + assert result["modelConfigKey"] == "OpenAI.gpt-4o" + + def test_raises_ld_api_error_when_items_is_empty(self): + client = LDApiClient("test-key") + response = {"items": [], "totalCount": 0} + with patch("urllib.request.urlopen", return_value=_mock_urlopen(response)): + with pytest.raises(LDApiError, match="not found"): + client.get_ai_config_variation("my-project", "my-agent", "missing-variation") + + def test_raises_ld_api_error_when_response_has_no_items_key(self): + client = LDApiClient("test-key") + with patch("urllib.request.urlopen", return_value=_mock_urlopen({})): + with pytest.raises(LDApiError, match="not found"): + client.get_ai_config_variation("my-project", "my-agent", "missing-variation") + + def test_raises_ld_api_error_on_http_404(self): + client = LDApiClient("test-key") + http_error = urllib.error.HTTPError( + url="http://x", code=404, msg="Not Found", hdrs=MagicMock(), fp=BytesIO(b"not found") + ) + with patch("urllib.request.urlopen", side_effect=http_error): + with pytest.raises(LDApiError) as exc_info: + client.get_ai_config_variation("my-project", "my-agent", "bad-key") + assert exc_info.value.status_code == 404