diff --git a/js/src/eval-parameters.ts b/js/src/eval-parameters.ts index e33d5c91..8a3792f6 100644 --- a/js/src/eval-parameters.ts +++ b/js/src/eval-parameters.ts @@ -150,6 +150,33 @@ function validateParametersWithJsonSchema>( throw Error(`Invalid parameters: ${errorMessages}`); } - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return parameters as T; + return rehydrateRemoteParameters(parameters, schema) as T; +} + +function rehydrateRemoteParameters( + parameters: Record, + schema: Record, +): Record { + const schemaProperties = schema.properties; + if (typeof schemaProperties !== "object" || schemaProperties === null) { + return parameters; + } + + return Object.fromEntries( + Object.entries(parameters).map(([name, value]) => { + const propertySchema = Reflect.get(schemaProperties, name); + if (typeof propertySchema !== "object" || propertySchema === null) { + return [name, value]; + } + + if (Reflect.get(propertySchema, "x-bt-type") === "prompt") { + return [ + name, + Prompt.fromPromptData(name, promptDataSchema.parse(value)), + ]; + } + + return [name, value]; + }), + ); } diff --git a/js/src/parameters.test.ts b/js/src/parameters.test.ts index dbba8ea4..da1a8caf 100644 --- a/js/src/parameters.test.ts +++ b/js/src/parameters.test.ts @@ -1,5 +1,7 @@ import { expect, test, beforeAll } from "vitest"; +import { validateParameters } from "./eval-parameters"; import { runEvaluator } from "./framework"; +import { RemoteEvalParameters } from "./logger"; import { z } from "zod/v3"; import { type ProgressReporter } from "./reporters/types"; import { configureNode } from "./node/config"; @@ -87,6 +89,59 @@ test("prompt parameter is passed correctly", async () => { expect(result.results[0].output).toBe("test input"); }); +test("remote prompt parameter is rehydrated correctly", async () => { + const parameters = new RemoteEvalParameters< + true, + true, + { + main: { + build: (args: { input: string }) => { + messages: Array<{ role: string; content: string }>; + model?: string; + }; + }; + } + >({ + id: "11111111-1111-4111-8111-111111111111", + _xact_id: "v1", + project_id: "22222222-2222-4222-8222-222222222222", + name: "Saved parameters", + slug: "saved-parameters", + function_type: "parameters", + function_data: { + type: "parameters", + data: { + main: { + prompt: { + type: "chat", + messages: [{ role: "user", content: "{{input}}" }], + }, + options: { + model: "gpt-5-mini", + }, + }, + }, + __schema: { + type: "object", + properties: { + main: { + type: "object", + "x-bt-type": "prompt", + }, + }, + additionalProperties: true, + }, + }, + }); + + const validated = await validateParameters({}, parameters); + + expect(validated.main.build({ input: "test input" })).toMatchObject({ + messages: [{ role: "user", content: "test input" }], + model: "gpt-5-mini", + }); +}); + test("custom parameter values override defaults", async () => { const result = await runEvaluator( null,