diff --git a/py/src/braintrust/parameters.py b/py/src/braintrust/parameters.py index 595ba3ce..beb0dc50 100644 --- a/py/src/braintrust/parameters.py +++ b/py/src/braintrust/parameters.py @@ -246,6 +246,28 @@ def validate_json_schema(parameters: dict[str, Any], schema: ParametersSchema) - return candidate +def _rehydrate_remote_parameters( + parameters: dict[str, Any], + schema: ParametersSchema, +) -> ValidatedParameters: + properties = schema.get("properties") + if not isinstance(properties, dict): + return parameters + + result: ValidatedParameters = dict(parameters) + for name, property_schema in properties.items(): + if not isinstance(property_schema, dict) or name not in result: + continue + + if property_schema.get("x-bt-type") == "prompt": + prompt_data = result[name] + if not isinstance(prompt_data, dict): + raise ValueError(f"Invalid parameter '{name}': prompt value must be an object") + result[name] = _create_prompt(name, prompt_data) + + return result + + def _validate_local_parameters( parameters: dict[str, Any], parameter_schema: EvalParameters, @@ -334,7 +356,8 @@ def validate_parameters( if isinstance(parameter_schema, RemoteEvalParameters): merged = dict(parameter_schema.data) merged.update(parameters) - return validate_json_schema(merged, parameter_schema.schema) + validated = validate_json_schema(merged, parameter_schema.schema) + return _rehydrate_remote_parameters(validated, parameter_schema.schema) return _validate_local_parameters(parameters, parameter_schema) diff --git a/py/src/braintrust/test_parameters.py b/py/src/braintrust/test_parameters.py index 6f9b6ea0..74e7f2a1 100644 --- a/py/src/braintrust/test_parameters.py +++ b/py/src/braintrust/test_parameters.py @@ -90,7 +90,7 @@ def test_validate_remote_parameters_merges_saved_data_and_runtime_overrides(): } -def test_validate_remote_parameters_keeps_prompt_values_as_dicts(): +def test_validate_remote_parameters_rehydrates_prompt_values(): parameters = RemoteEvalParameters( id="params-123", project_id="project-123", @@ -122,7 +122,61 @@ def test_validate_remote_parameters_keeps_prompt_values_as_dicts(): result = validate_parameters({}, parameters) - assert isinstance(result["main"], dict) + assert hasattr(result["main"], "build") + built = result["main"].build(input="test input") + assert built["messages"] == [{"role": "user", "content": "test input"}] + assert built["model"] == "gpt-5-mini" + + +def test_validate_remote_parameters_allows_prompt_overrides(): + parameters = RemoteEvalParameters( + id="params-123", + project_id="project-123", + name="Saved parameters", + slug="saved-parameters", + version="v1", + schema={ + "type": "object", + "properties": { + "main": { + "type": "object", + "x-bt-type": "prompt", + }, + }, + "additionalProperties": True, + }, + data={ + "main": { + "prompt": { + "type": "chat", + "messages": [{"role": "user", "content": "{{input}}"}], + }, + "options": { + "model": "gpt-5-mini", + }, + }, + }, + ) + + result = validate_parameters( + { + "main": { + "prompt": { + "type": "chat", + "messages": [{"role": "user", "content": "override {{input}}"}], + }, + "options": { + "model": "gpt-5-nano", + }, + } + }, + parameters, + ) + + assert hasattr(result["main"], "build") + built = result["main"].build(input="test input") + assert built["messages"] == [{"role": "user", "content": "override test input"}] + assert built["model"] == "gpt-5-nano" @pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed")