From 58461d0f421718e958b302114532fab6692918af Mon Sep 17 00:00:00 2001 From: Joshua Wootonn Date: Fri, 27 Mar 2026 10:35:31 -0500 Subject: [PATCH 1/2] Rehydrate remote prompt parameters --- js/src/eval-parameters.ts | 28 ++++++++++++++++++-- js/src/parameters.test.ts | 55 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/js/src/eval-parameters.ts b/js/src/eval-parameters.ts index e33d5c911..166f763dd 100644 --- a/js/src/eval-parameters.ts +++ b/js/src/eval-parameters.ts @@ -150,6 +150,30 @@ function validateParametersWithJsonSchema>( throw Error(`Invalid parameters: ${errorMessages}`); } - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return parameters as T; + return rehydrateRemoteParameters(parameters, schema) as T; +} + +function rehydrateRemoteParameters( + parameters: Record, + schema: Record, +): Record { + const schemaProperties = schema.properties; + if (typeof schemaProperties !== "object" || schemaProperties === null) { + return parameters; + } + + return Object.fromEntries( + Object.entries(parameters).map(([name, value]) => { + const propertySchema = Reflect.get(schemaProperties, name); + if (typeof propertySchema !== "object" || propertySchema === null) { + return [name, value]; + } + + if (Reflect.get(propertySchema, "x-bt-type") === "prompt") { + return [name, Prompt.fromPromptData(name, promptDataSchema.parse(value))]; + } + + return [name, value]; + }), + ); } diff --git a/js/src/parameters.test.ts b/js/src/parameters.test.ts index dbba8ea49..da1a8caf8 100644 --- a/js/src/parameters.test.ts +++ b/js/src/parameters.test.ts @@ -1,5 +1,7 @@ import { expect, test, beforeAll } from "vitest"; +import { validateParameters } from "./eval-parameters"; import { runEvaluator } from "./framework"; +import { RemoteEvalParameters } from "./logger"; import { z } from "zod/v3"; import { type ProgressReporter } from "./reporters/types"; import { configureNode } from "./node/config"; @@ -87,6 +89,59 @@ test("prompt parameter is passed correctly", async () => { expect(result.results[0].output).toBe("test input"); }); +test("remote prompt parameter is rehydrated correctly", async () => { + const parameters = new RemoteEvalParameters< + true, + true, + { + main: { + build: (args: { input: string }) => { + messages: Array<{ role: string; content: string }>; + model?: string; + }; + }; + } + >({ + id: "11111111-1111-4111-8111-111111111111", + _xact_id: "v1", + project_id: "22222222-2222-4222-8222-222222222222", + name: "Saved parameters", + slug: "saved-parameters", + function_type: "parameters", + function_data: { + type: "parameters", + data: { + main: { + prompt: { + type: "chat", + messages: [{ role: "user", content: "{{input}}" }], + }, + options: { + model: "gpt-5-mini", + }, + }, + }, + __schema: { + type: "object", + properties: { + main: { + type: "object", + "x-bt-type": "prompt", + }, + }, + additionalProperties: true, + }, + }, + }); + + const validated = await validateParameters({}, parameters); + + expect(validated.main.build({ input: "test input" })).toMatchObject({ + messages: [{ role: "user", content: "test input" }], + model: "gpt-5-mini", + }); +}); + test("custom parameter values override defaults", async () => { const result = await runEvaluator( null, From 960940206dbebd4f13ec69d720d6306f0bde8242 Mon Sep 17 00:00:00 2001 From: Joshua Wootonn Date: Fri, 27 Mar 2026 11:24:23 -0500 Subject: [PATCH 2/2] Format remote prompt rehydration changes --- js/src/eval-parameters.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/js/src/eval-parameters.ts b/js/src/eval-parameters.ts index 166f763dd..8a3792f62 100644 --- a/js/src/eval-parameters.ts +++ b/js/src/eval-parameters.ts @@ -170,7 +170,10 @@ function rehydrateRemoteParameters( } if (Reflect.get(propertySchema, "x-bt-type") === "prompt") { - return [name, Prompt.fromPromptData(name, promptDataSchema.parse(value))]; + return [ + name, + Prompt.fromPromptData(name, promptDataSchema.parse(value)), + ]; } return [name, value];