Updates to match https://github.com/braintrustdata/braintrust-spec/pull/2/changes

Author: aswink

js/dev/server.ts

@@ -117,7 +117,7 @@ export function runDevServer(
 
         evalDefs[name] = {
           parameters,
-          scores: evaluator.scores.map((score, idx) => ({
+          scores: (evaluator.scores ?? []).map((score, idx) => ({
             name: scorerName(score, idx),
           })),
         };
@@ -209,7 +209,7 @@ export function runDevServer(
           {
             ...evaluator,
             data: evalData.data,
-            scores: evaluator.scores.concat(
+            scores: (evaluator.scores ?? []).concat(
               scores?.map((score) =>
                 makeScorer(
                   state,

js/src/framework.test.ts

@@ -1498,8 +1498,7 @@ test("classifier-only evaluator populates classifications field", async () => {
     {
       data: [{ input: "hello", expected: "greeting" }],
       task: (input) => input,
-      scores: [],
-      classifications: [
+      classifiers: [
         () => ({
           name: "category",
           id: "greeting",
@@ -1551,8 +1550,7 @@ test("multiple classifiers returning the same name append items correctly", asyn
     {
       data: [{ input: "hello" }],
       task: (input) => input,
-      scores: [],
-      classifications: [
+      classifiers: [
         () => [
           { name: "category", id: "greeting", label: "Greeting" },
           { name: "category", id: "informal", label: "Informal" },
@@ -1586,7 +1584,7 @@ test("mixed evaluator populates both scores and classifications", async () => {
           score: args.output === args.expected ? 1 : 0,
         }),
       ],
-      classifications: [
+      classifiers: [
         () => ({ name: "category", id: "greeting", label: "Greeting" }),
       ],
     },
@@ -1606,8 +1604,7 @@ test("malformed classifier output fails clearly", async () => {
     {
       data: [{ input: "hello" }],
       task: (input) => input,
-      scores: [],
-      classifications: [() => ({}) as never],
+      classifiers: [() => ({}) as never],
     },
     { noSendLogs: true, returnResults: true },
   );

js/src/framework.ts

@@ -219,13 +219,13 @@ type ErrorScoreHandler = (args: {
   unhandledScores: string[];
 }) => Record<string, number> | undefined | void;
 
-export interface Evaluator<
+type EvaluatorBase<
   Input,
   Output,
   Expected,
   Metadata extends BaseMetadata = DefaultMetadataType,
   Parameters extends EvalParameters = EvalParameters,
-> {
+> = {
   /**
    * A function that returns a list of inputs, expected outputs, and metadata.
    */
@@ -236,17 +236,6 @@ export interface Evaluator<
    */
   task: EvalTask<Input, Output, Expected, Metadata, Parameters>;
 
-  /**
-   * A set of functions that take an input, output, and expected value and return a {@link Score}.
-   */
-  scores: EvalScorer<Input, Output, Expected, Metadata>[];
-
-  /**
-   * A set of functions that take an input, output, and expected value and return a
-   * {@link Classification}. Results are recorded under the `classifications` column.
-   */
-  classifications?: EvalClassifier<Input, Output, Expected, Metadata>[];
-
   /**
    * A set of parameters that will be passed to the evaluator.
    * Can be:
@@ -364,7 +353,42 @@ export interface Evaluator<
    * Flushes spans before calling scoring functions
    */
   flushBeforeScoring?: boolean;
-}
+};
+
+/**
+ * Defines an evaluator. At least one of `scores` or `classifiers` must be provided.
+ */
+export type Evaluator<
+  Input,
+  Output,
+  Expected,
+  Metadata extends BaseMetadata = DefaultMetadataType,
+  Parameters extends EvalParameters = EvalParameters,
+> = EvaluatorBase<Input, Output, Expected, Metadata, Parameters> &
+  (
+    | {
+        /**
+         * A set of functions that take an input, output, and expected value and return a {@link Score}.
+         */
+        scores: EvalScorer<Input, Output, Expected, Metadata>[];
+        /**
+         * A set of functions that take an input, output, and expected value and return a
+         * {@link Classification}. Results are recorded under the `classifications` column.
+         */
+        classifiers?: EvalClassifier<Input, Output, Expected, Metadata>[];
+      }
+    | {
+        /**
+         * A set of functions that take an input, output, and expected value and return a {@link Score}.
+         */
+        scores?: EvalScorer<Input, Output, Expected, Metadata>[];
+        /**
+         * A set of functions that take an input, output, and expected value and return a
+         * {@link Classification}. Results are recorded under the `classifications` column.
+         */
+        classifiers: EvalClassifier<Input, Output, Expected, Metadata>[];
+      }
+  );
 
 export class EvalResultWithSummary<
   Input,
@@ -1023,6 +1047,11 @@ export async function runEvaluator(
   enableCache = true,
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
 ): Promise<EvalResultWithSummary<any, any, any, any>> {
+  if (!evaluator.scores && !evaluator.classifiers) {
+    throw new Error(
+      "Evaluator must include at least one of `scores` or `classifiers`",
+    );
+  }
   return await runEvaluatorInternal(
     experiment,
     evaluator,
@@ -1237,8 +1266,8 @@ async function runEvaluatorInternal(
           let tags: string[] = [...(datum.tags ?? [])];
           const scores: Record<string, number | null> = {};
           const classifications: Record<string, ClassificationItem[]> = {};
-          const scorerNames = evaluator.scores.map(scorerName);
-          const classifierNames = (evaluator.classifications ?? []).map(
+          const scorerNames = (evaluator.scores ?? []).map(scorerName);
+          const classifierNames = (evaluator.classifiers ?? []).map(
             classifierName,
           );
           let unhandledScores: string[] | null = scorerNames;
@@ -1317,7 +1346,7 @@ async function runEvaluatorInternal(
 
             const [scoreResults, classificationResults] = await Promise.all([
               Promise.all(
-                evaluator.scores.map((score, score_idx) =>
+                (evaluator.scores ?? []).map((score, score_idx) =>
                   runInScorerSpan(
                     rootSpan,
                     scorerNames[score_idx],
@@ -1371,7 +1400,7 @@ async function runEvaluatorInternal(
                 ),
               ),
               Promise.all(
-                (evaluator.classifications ?? []).map((classifier, idx) =>
+                (evaluator.classifiers ?? []).map((classifier, idx) =>
                   runInScorerSpan(
                     rootSpan,
                     classifierNames[idx],

js/src/parameters.test.ts

@@ -26,7 +26,7 @@ test("parameters are passed to task", async () => {
         return output;
       },
       scores: [],
-      classifications: [],
+      classifiers: [],
       parameters: {
         prefix: z.string().default("start:"),
         suffix: z.string().default(":end"),
@@ -60,7 +60,7 @@ test("prompt parameter is passed correctly", async () => {
         return input;
       },
       scores: [],
-      classifications: [],
+      classifiers: [],
       parameters: {
         main: {
           type: "prompt",
@@ -101,7 +101,7 @@ test("custom parameter values override defaults", async () => {
         return output;
       },
       scores: [],
-      classifications: [],
+      classifiers: [],
       parameters: {
         prefix: z.string().default("start:"),
         suffix: z.string().default(":end"),
@@ -134,7 +134,7 @@ test("array parameter is handled correctly", async () => {
         return input;
       },
       scores: [],
-      classifications: [],
+      classifiers: [],
       parameters: {
         items: z.array(z.string()).default(["item1", "item2"]),
       },
@@ -165,7 +165,7 @@ test("object parameter is handled correctly", async () => {
         return input;
       },
       scores: [],
-      classifications: [],
+      classifiers: [],
       parameters: {
         config: z
           .object({
@@ -201,7 +201,7 @@ test("model parameter defaults to configured value", async () => {
         return input;
       },
       scores: [],
-      classifications: [],
+      classifiers: [],
       parameters: {
         model: {
           type: "model",
@@ -230,7 +230,7 @@ test("model parameter is required when default is missing", async () => {
         data: [{ input: "test" }],
         task: async (input: string) => input,
         scores: [],
-        classifications: [],
+        classifiers: [],
         parameters: {
           model: {
             type: "model",