Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 29 additions & 2 deletions js/src/eval-parameters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,33 @@
throw Error(`Invalid parameters: ${errorMessages}`);
}

// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
return parameters as T;
return rehydrateRemoteParameters(parameters, schema) as T;

Check warning on line 153 in js/src/eval-parameters.ts

View workflow job for this annotation

GitHub Actions / lint

Do not use any type assertions
}

function rehydrateRemoteParameters(
parameters: Record<string, unknown>,
schema: Record<string, unknown>,
): Record<string, unknown> {
const schemaProperties = schema.properties;
if (typeof schemaProperties !== "object" || schemaProperties === null) {
return parameters;
}

return Object.fromEntries(
Object.entries(parameters).map(([name, value]) => {
const propertySchema = Reflect.get(schemaProperties, name);
if (typeof propertySchema !== "object" || propertySchema === null) {
return [name, value];
}

if (Reflect.get(propertySchema, "x-bt-type") === "prompt") {
return [
name,
Prompt.fromPromptData(name, promptDataSchema.parse(value)),
];
}

return [name, value];
}),
);
}
55 changes: 55 additions & 0 deletions js/src/parameters.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { expect, test, beforeAll } from "vitest";
import { validateParameters } from "./eval-parameters";
import { runEvaluator } from "./framework";
import { RemoteEvalParameters } from "./logger";
import { z } from "zod/v3";
import { type ProgressReporter } from "./reporters/types";
import { configureNode } from "./node/config";
Expand Down Expand Up @@ -87,6 +89,59 @@ test("prompt parameter is passed correctly", async () => {
expect(result.results[0].output).toBe("test input");
});

test("remote prompt parameter is rehydrated correctly", async () => {
const parameters = new RemoteEvalParameters<
true,
true,
{
main: {
build: (args: { input: string }) => {
messages: Array<{ role: string; content: string }>;
model?: string;
};
};
}
>({
id: "11111111-1111-4111-8111-111111111111",
_xact_id: "v1",
project_id: "22222222-2222-4222-8222-222222222222",
name: "Saved parameters",
slug: "saved-parameters",
function_type: "parameters",
function_data: {
type: "parameters",
data: {
main: {
prompt: {
type: "chat",
messages: [{ role: "user", content: "{{input}}" }],
},
options: {
model: "gpt-5-mini",
},
},
},
__schema: {
type: "object",
properties: {
main: {
type: "object",
"x-bt-type": "prompt",
},
},
additionalProperties: true,
},
},
});

const validated = await validateParameters({}, parameters);

expect(validated.main.build({ input: "test input" })).toMatchObject({
messages: [{ role: "user", content: "test input" }],
model: "gpt-5-mini",
});
});

test("custom parameter values override defaults", async () => {
const result = await runEvaluator(
null,
Expand Down
Loading