Outcome 5 (1/2): replication statistics aggregator

evals/src/analysis/replication-stats.ts

@@ -0,0 +1,119 @@
+/**
+ * Aggregates K replicated A/B comparisons of the same scenario and harness into
+ * per-dimension lift statistics (GOAL.md Outcome 5). Pure and deterministic —
+ * no I/O, no clock dependence beyond the report timestamp.
+ *
+ * Lift is reported as mean ± sample standard deviation across replications, and
+ * is only flagged as a signal when the absolute mean lift exceeds one SD. A
+ * t-statistic is reported alongside so a caller can compare it to the K=5,
+ * one-tailed α=0.05 critical value (t > 2.13, df=4).
+ */
+import type { ComparisonResult, DimensionScore } from '../domain/result.js';
+import type { DimensionLiftStat, ReplicationReport, ReplicationSignificance } from '../domain/replication.js';
+
+/** One-tailed α=0.05 critical t by degrees of freedom (df = n − 1). */
+const T_CRITICAL_ONE_TAILED_05: Readonly<Record<number, number>> = {
+  1: 6.314,
+  2: 2.920,
+  3: 2.353,
+  4: 2.132,
+  5: 2.015,
+  6: 1.943,
+  7: 1.895,
+  8: 1.860,
+  9: 1.833,
+  10: 1.812,
+  15: 1.753,
+  20: 1.725,
+  30: 1.697,
+};
+
+function mean(xs: readonly number[]): number {
+  return xs.length === 0 ? 0 : xs.reduce((sum, x) => sum + x, 0) / xs.length;
+}
+
+function sampleStdDev(xs: readonly number[]): number {
+  if (xs.length < 2) return 0;
+  const m = mean(xs);
+  const variance = xs.reduce((sum, x) => sum + (x - m) ** 2, 0) / (xs.length - 1);
+  return Math.sqrt(variance);
+}
+
+function scoreByDimension(scores: readonly DimensionScore[]): Map<string, DimensionScore> {
+  const map = new Map<string, DimensionScore>();
+  for (const score of scores) map.set(score.dimension, score);
+  return map;
+}
+
+/** Dimensions present in the jumboScores of every replication, in first-replication order. */
+function dimensionsInEveryReplication(comparisons: readonly ComparisonResult[]): string[] {
+  if (comparisons.length === 0) return [];
+  const [first, ...rest] = comparisons;
+  let common = new Set(first.jumboScores.map((s) => s.dimension));
+  for (const comparison of rest) {
+    const here = new Set(comparison.jumboScores.map((s) => s.dimension));
+    common = new Set([...common].filter((d) => here.has(d)));
+  }
+  return first.jumboScores.map((s) => s.dimension).filter((d) => common.has(d));
+}
+
+export function aggregateReplications(comparisons: readonly ComparisonResult[]): ReplicationReport {
+  const k = comparisons.length;
+  const createdAt = new Date().toISOString();
+  const significance: ReplicationSignificance = {
+    rule: 'isSignal = |meanLift| > sdLift',
+    tCriticalOneTailed05: T_CRITICAL_ONE_TAILED_05[k - 1] ?? null,
+    note: `Lift is a signal only when |meanLift| exceeds one SD. For K=5 (df=4) the one-tailed alpha=0.05 t-threshold is 2.13; current K=${k} (df=${Math.max(k - 1, 0)}).`,
+  };
+
+  if (k === 0) {
+    return { scenarioId: '', harness: '', k: 0, dimensions: [], significance, createdAt };
+  }
+
+  const jumboMaps = comparisons.map((c) => scoreByDimension(c.jumboScores));
+  const baselineMaps = comparisons.map((c) => scoreByDimension(c.baselineScores));
+
+  const dimensions: DimensionLiftStat[] = dimensionsInEveryReplication(comparisons).map((dimension) => {
+    const jumboVals: number[] = [];
+    const baselineVals: number[] = [];
+    const lifts: number[] = [];
+    for (let i = 0; i < k; i++) {
+      const jumbo = jumboMaps[i].get(dimension);
+      const baseline = baselineMaps[i].get(dimension);
+      if (!jumbo || !baseline) continue;
+      // N/A markers (e.g. token-efficiency without output-equivalence) carry
+      // maxScore 0 and are excluded from this dimension's statistics.
+      if (jumbo.maxScore === 0) continue;
+      jumboVals.push(jumbo.score);
+      baselineVals.push(baseline.score);
+      lifts.push(jumbo.score - baseline.score);
+    }
+
+    const applicable = lifts.length;
+    const meanLift = mean(lifts);
+    const sdLift = sampleStdDev(lifts);
+    const tStatistic = sdLift > 0 && applicable >= 2 ? meanLift / (sdLift / Math.sqrt(applicable)) : 0;
+    const isSignal = applicable >= 2 && Math.abs(meanLift) > sdLift;
+
+    return {
+      dimension,
+      k,
+      applicableReplications: applicable,
+      meanJumbo: mean(jumboVals),
+      meanBaseline: mean(baselineVals),
+      meanLift,
+      sdLift,
+      tStatistic,
+      isSignal,
+    };
+  });
+
+  return {
+    scenarioId: comparisons[0].scenarioId,
+    harness: comparisons[0].harness,
+    k,
+    dimensions,
+    significance,
+    createdAt,
+  };
+}

evals/src/domain/index.ts

@@ -18,3 +18,4 @@ export * from './session.js';
 export * from './heartbeat.js';
 export * from './result.js';
 export * from './result-factories.js';
+export * from './replication.js';

evals/src/domain/replication.ts

@@ -0,0 +1,42 @@
+/**
+ * Statistics over K replicated A/B comparisons of the same scenario and harness
+ * (GOAL.md Outcome 5). Lift is reported as mean ± standard deviation, never a
+ * single-point estimate, and is only a "signal" when it exceeds one standard
+ * deviation of its own distribution across replications.
+ */
+
+export interface DimensionLiftStat {
+  readonly dimension: string;
+  /** Total replications in the batch. */
+  readonly k: number;
+  /** Replications where this dimension was applicable (excludes N/A, e.g. token-efficiency with maxScore 0). */
+  readonly applicableReplications: number;
+  readonly meanJumbo: number;
+  readonly meanBaseline: number;
+  /** mean over applicable replications of (jumboScore − baselineScore). */
+  readonly meanLift: number;
+  /** Sample (n−1) standard deviation of the per-replication lifts; 0 when fewer than 2 applicable. */
+  readonly sdLift: number;
+  /** meanLift / (sdLift / sqrt(applicable)); 0 when sdLift is 0 or fewer than 2 applicable. */
+  readonly tStatistic: number;
+  /** True only when |meanLift| > sdLift (GOAL.md: a lift is a signal only when it exceeds one SD). */
+  readonly isSignal: boolean;
+}
+
+export interface ReplicationSignificance {
+  /** The rule used for `isSignal`. */
+  readonly rule: string;
+  /** One-tailed α=0.05 critical t for df = k−1, or null when df is outside the lookup table. */
+  readonly tCriticalOneTailed05: number | null;
+  readonly note: string;
+}
+
+export interface ReplicationReport {
+  readonly scenarioId: string;
+  readonly harness: string;
+  /** Number of replications aggregated. */
+  readonly k: number;
+  readonly dimensions: readonly DimensionLiftStat[];
+  readonly significance: ReplicationSignificance;
+  readonly createdAt: string;
+}

evals/tests/unit/replication-stats.test.ts

@@ -0,0 +1,127 @@
+import { describe, it, expect } from '@jest/globals';
+import { aggregateReplications } from '../../src/analysis/replication-stats.js';
+import type { ComparisonResult, DimensionScore } from '../../src/domain/index.js';
+
+/** Builds a minimal ComparisonResult carrying only the per-dimension scores the aggregator reads. */
+function comparison(
+  dims: Record<string, { jumbo: number; baseline: number; maxScore?: number }>,
+): ComparisonResult {
+  const score = (dimension: string, value: number, maxScore: number): DimensionScore => ({
+    dimension,
+    score: value,
+    maxScore,
+    details: '',
+  });
+  const jumboScores = Object.entries(dims).map(([d, v]) => score(d, v.jumbo, v.maxScore ?? 1));
+  const baselineScores = Object.entries(dims).map(([d, v]) => score(d, v.baseline, v.maxScore ?? 1));
+  const deltas = Object.entries(dims).map(([d, v]) => score(d, v.jumbo - v.baseline, v.maxScore ?? 1));
+  return {
+    id: 'c',
+    scenarioId: 'scenario-1',
+    harness: 'claude-code',
+    jumboResult: { id: 'j', scenarioId: 'scenario-1', harness: 'claude-code', sessionRecords: [], createdAt: 't', tampered: false, tamperLog: [] },
+    baselineResult: { id: 'b', scenarioId: 'scenario-1', harness: 'claude-code', sessionRecords: [], createdAt: 't', tampered: false, tamperLog: [] },
+    jumboScores,
+    baselineScores,
+    deltas,
+    createdAt: 't',
+    tampered: false,
+    tamperLog: [],
+  };
+}
+
+function dim(report: ReturnType<typeof aggregateReplications>, name: string) {
+  const d = report.dimensions.find((x) => x.dimension === name);
+  if (!d) throw new Error(`dimension ${name} not in report`);
+  return d;
+}
+
+describe('aggregateReplications', () => {
+  it('computes mean lift, sample (n-1) SD, and arm means across replications', () => {
+    const report = aggregateReplications([
+      comparison({ 'file-accuracy': { jumbo: 0.8, baseline: 0.6 } }),
+      comparison({ 'file-accuracy': { jumbo: 0.9, baseline: 0.5 } }),
+      comparison({ 'file-accuracy': { jumbo: 1.0, baseline: 0.4 } }),
+    ]);
+    expect(report.k).toBe(3);
+    expect(report.scenarioId).toBe('scenario-1');
+    expect(report.harness).toBe('claude-code');
+
+    const fa = dim(report, 'file-accuracy');
+    expect(fa.meanJumbo).toBeCloseTo(0.9, 6);
+    expect(fa.meanBaseline).toBeCloseTo(0.5, 6);
+    expect(fa.meanLift).toBeCloseTo(0.4, 6); // lifts [0.2, 0.4, 0.6]
+    expect(fa.sdLift).toBeCloseTo(0.2, 6); // sample SD of [0.2,0.4,0.6]
+    expect(fa.applicableReplications).toBe(3);
+    expect(fa.tStatistic).toBeCloseTo(0.4 / (0.2 / Math.sqrt(3)), 4);
+  });
+
+  it('flags a signal only when |mean lift| exceeds one SD', () => {
+    // lifts [0.3, 0.5] -> mean 0.4, sd 0.1414 -> signal
+    const signal = aggregateReplications([
+      comparison({ d: { jumbo: 0.3, baseline: 0 } }),
+      comparison({ d: { jumbo: 0.5, baseline: 0 } }),
+    ]);
+    expect(dim(signal, 'd').isSignal).toBe(true);
+
+    // lifts [0, 0.4] -> mean 0.2, sd 0.2828 -> not a signal
+    const noise = aggregateReplications([
+      comparison({ d: { jumbo: 0, baseline: 0 } }),
+      comparison({ d: { jumbo: 0.4, baseline: 0 } }),
+    ]);
+    expect(dim(noise, 'd').isSignal).toBe(false);
+  });
+
+  it('treats K=1 as no SD and never a signal', () => {
+    const report = aggregateReplications([comparison({ d: { jumbo: 1, baseline: 0 } })]);
+    const d = dim(report, 'd');
+    expect(report.k).toBe(1);
+    expect(d.sdLift).toBe(0);
+    expect(d.tStatistic).toBe(0);
+    expect(d.isSignal).toBe(false);
+    expect(d.applicableReplications).toBe(1);
+  });
+
+  it('excludes N/A (maxScore 0) token-efficiency replications and records the applicable count', () => {
+    const report = aggregateReplications([
+      comparison({ 'token-efficiency': { jumbo: 0.5, baseline: 0, maxScore: 1 } }),
+      comparison({ 'token-efficiency': { jumbo: 0.3, baseline: 0, maxScore: 1 } }),
+      comparison({ 'token-efficiency': { jumbo: 0, baseline: 0, maxScore: 0 } }), // N/A
+    ]);
+    const te = dim(report, 'token-efficiency');
+    expect(te.k).toBe(3);
+    expect(te.applicableReplications).toBe(2);
+    expect(te.meanLift).toBeCloseTo(0.4, 6); // mean of [0.5, 0.3]
+  });
+
+  it('aggregates multiple dimensions independently', () => {
+    const report = aggregateReplications([
+      comparison({ a: { jumbo: 1, baseline: 0 }, b: { jumbo: 0.2, baseline: 0.2 } }),
+      comparison({ a: { jumbo: 1, baseline: 0 }, b: { jumbo: 0.4, baseline: 0.4 } }),
+    ]);
+    expect(dim(report, 'a').meanLift).toBeCloseTo(1, 6);
+    expect(dim(report, 'b').meanLift).toBeCloseTo(0, 6);
+    expect(dim(report, 'a').isSignal).toBe(true); // lift 1, sd 0
+    expect(dim(report, 'b').isSignal).toBe(false); // lift 0
+  });
+
+  it('only includes dimensions present in every replication', () => {
+    const report = aggregateReplications([
+      comparison({ a: { jumbo: 1, baseline: 0 }, b: { jumbo: 1, baseline: 0 } }),
+      comparison({ a: { jumbo: 1, baseline: 0 } }), // no b
+    ]);
+    expect(report.dimensions.map((d) => d.dimension)).toEqual(['a']);
+  });
+
+  it('records the K=5 significance threshold note', () => {
+    const report = aggregateReplications([
+      comparison({ a: { jumbo: 1, baseline: 0 } }),
+      comparison({ a: { jumbo: 1, baseline: 0 } }),
+      comparison({ a: { jumbo: 1, baseline: 0 } }),
+      comparison({ a: { jumbo: 1, baseline: 0 } }),
+      comparison({ a: { jumbo: 1, baseline: 0 } }),
+    ]);
+    expect(report.significance.tCriticalOneTailed05).toBeCloseTo(2.132, 2); // df = 4
+    expect(report.significance.note).toContain('2.13');
+  });
+});