From bbac8ebfb6b64d6fde7cf7bbb87cba1b083ddf62 Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Mon, 3 Aug 2026 21:32:07 +0000 Subject: [PATCH] test(eval): actually isolate eval repositories The isolateRepo: true override in evals/it.ts had no effect: vitest's extend() does not remap dependency references of fixtures defined in an earlier layer, so the repo fixture still saw isolateRepo: false and every eval ran against the shared repository. Evals polluted each other's state (leftover previews, locales, and types), causing spurious failures. Replace the dead override with it.scoped({ isolateRepo: true }), the one vitest path that remaps fixture deps. It runs at module scope in evals/it.ts, which re-executes during each eval file's collection, so every eval test gets a throwaway repository. Co-Authored-By: Claude Fable 5 --- evals/it.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evals/it.ts b/evals/it.ts index 8a29774..e5976c5 100644 --- a/evals/it.ts +++ b/evals/it.ts @@ -41,10 +41,8 @@ declare module "vitest" { } export const it = base.extend<{ - isolateRepo: boolean; agent: (prompt: string) => Promise; }>({ - isolateRepo: true, agent: async ({ home, project, login, task, repo, token, host, password }, use) => { await login(); @@ -102,6 +100,8 @@ export const it = base.extend<{ }, }); +it.scoped({ isolateRepo: true }); + expect.extend({ toHaveRun(result: AgentResult, bin: string, positionals: string[] = []) { const pass = result.commands.some((command) => {