From bfbf124db740eb5ddb412ff5c46b562526bfaa3f Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Wed, 5 Aug 2026 15:53:11 +0200 Subject: [PATCH] Add sandboxed benchmark skill --- .agents/skills/sandbox-bench/SKILL.md | 172 ++++ .../sandbox-bench/references/methodology.md | 91 ++ .../skills/sandbox-bench/scripts/collect.mjs | 121 +++ .../skills/sandbox-bench/scripts/config.mjs | 102 +++ .../sandbox-bench/scripts/sandbox-bench.mjs | 848 ++++++++++++++++++ .../skills/sandbox-bench/scripts/status.mjs | 54 ++ .../sandbox-bench/scripts/summarize.mjs | 273 ++++++ 7 files changed, 1661 insertions(+) create mode 100644 .agents/skills/sandbox-bench/SKILL.md create mode 100644 .agents/skills/sandbox-bench/references/methodology.md create mode 100644 .agents/skills/sandbox-bench/scripts/collect.mjs create mode 100644 .agents/skills/sandbox-bench/scripts/config.mjs create mode 100644 .agents/skills/sandbox-bench/scripts/sandbox-bench.mjs create mode 100644 .agents/skills/sandbox-bench/scripts/status.mjs create mode 100644 .agents/skills/sandbox-bench/scripts/summarize.mjs diff --git a/.agents/skills/sandbox-bench/SKILL.md b/.agents/skills/sandbox-bench/SKILL.md new file mode 100644 index 00000000..92779bb3 --- /dev/null +++ b/.agents/skills/sandbox-bench/SKILL.md @@ -0,0 +1,172 @@ +--- +name: sandbox-bench +description: Run framework-tracker framework, app, package, and version benchmark matrices on identical Vercel Sandbox VMs, including Next.js stable/canary comparisons and cross-framework SSR, load, browser-rendering, build, or install measurements. Use when asked to run, compare, repeat, or validate framework-tracker benchmarks remotely, on Vercel Sandbox, across independent machines, or without tying up the local workstation. +--- + +# Run framework-tracker benchmarks on Vercel Sandbox + +Use the repository's `pnpm benchmark` CLI as the source of truth for benchmark +selection and version overrides. Use this skill's runner only for remote +provisioning, identical setup snapshots, independent VM replication, +collection, and boot-level summaries. + +Run every command below from the framework-tracker repository root. + +## Start by checking configuration and active work + +Before launching resources, run: + +```sh +node .agents/skills/sandbox-bench/scripts/config.mjs show +node .agents/skills/sandbox-bench/scripts/status.mjs +``` + +If configuration reports `NOT CONFIGURED`, ask which Vercel **team** and +**project** should own and bill the sandboxes. Never guess or commit them. +Save the answer outside the repository: + +```sh +node .agents/skills/sandbox-bench/scripts/config.mjs set \ + team= project= +``` + +The Vercel CLI session must have access to that scope. Verify failures with: + +```sh +vercel whoami --scope +vercel sandbox list --team --project +``` + +Stop after a persistent authorization failure instead of retrying resource +creation. Configuration is stored in +`~/.config/framework-tracker-sandbox-bench/config.json`; results and snapshot +IDs are stored in `~/.cache/framework-tracker-sandbox-bench/`. + +## Resolve the comparison matrix + +Pass any framework or package selector accepted by `pnpm benchmark`. Append +`@` to make one case use a specific framework version. + +- `next@stable` is normalized to `next@latest`. +- Use `next@canary` for the current npm canary. +- Prefer exact app selectors, such as `app-next-js@canary`, for app-only runtime + comparisons. +- A framework selector such as `next@canary` may produce both starter and app + results, depending on which measurements those packages configure. +- Each versioned target becomes a separate invocation so the local CLI's + one-framework-per-`--version` invariant remains intact. +- Use the resolved `frameworkVersion` in raw results when reporting dist-tags. + +Inspect available local cases when uncertain: + +```sh +pnpm benchmark --list +``` + +Always inspect a no-op plan before creating billable resources: + +```sh +node .agents/skills/sandbox-bench/scripts/sandbox-bench.mjs \ + app-next-js@canary app-tanstack-start-react \ + --measurement ssrLoad,ssrRequestThroughput \ + --vms 8 --dry-run +``` + +The source bundle contains tracked changes plus non-ignored untracked files, so +the sandbox measures the current working tree. The runner prints a source +fingerprint and snapshots one prepared checkout; every measurement VM boots +from that snapshot. Browser measurements automatically add Google Chrome. + +## Launch the run + +Typical runtime comparison: + +```sh +node .agents/skills/sandbox-bench/scripts/sandbox-bench.mjs \ + app-next-js@canary app-tanstack-start-react \ + --measurement \ + ssrLoad,ssrRequestThroughput,clientSideRendered,serverSideRendered \ + --vms 8 --runs 5 --label next-canary-vs-tanstack +``` + +Next.js stable versus canary: + +```sh +node .agents/skills/sandbox-bench/scripts/sandbox-bench.mjs \ + app-next-js@stable app-next-js@canary \ + --measurement ssrLoad,ssrRequestThroughput \ + --vms 8 --label next-stable-vs-canary +``` + +Useful options: + +- `--vms ` controls independent boots. Use one only for a smoke test, eight + for exploration, and sixteen when a small effect may drive a decision. +- `--concurrency ` controls how many sandboxes run at once and defaults to + five, which fits Hobby project limits without changing the boot count. +- `--blocks ` repeats the full matrix inside each boot. More blocks improve + a boot's estimate but do not increase the independent sample count. +- `--runs ` forwards the run count to benchmarks that support it. +- `--vcpus ` defaults to 8. Keep it identical across compared cases. +- `--no-cache` forces a fresh setup snapshot after debugging setup changes. +- `--keep` retains measurement VMs after collection; omit it normally. + +Run the launcher as a background task for long matrices and return to it on +completion. Relay the printed run directory immediately. Do not stop a run +because partial output appears favorable; only `summary.md` after all planned +VMs finish is reportable. + +## Read and report results + +The launcher writes: + +- `meta.json`: source fingerprint, snapshot, cases, allocation, and per-VM + status; +- `summary.md`: absolute boot-level means and paired comparisons; +- `vm-*/bench-output/`: raw JSON and per-case logs; and +- `vm-*/bench.log`: the full VM transcript. + +Rebuild a summary without rerunning: + +```sh +node .agents/skills/sandbox-bench/scripts/summarize.mjs +``` + +Read [references/methodology.md](references/methodology.md) before interpreting +or reporting a run. In particular: + +- Treat the VM boot—not internal iterations or requests—as `n`. +- Positive deltas mean only "candidate numeric value is larger"; interpret + whether that is better from the metric. +- Require boot-level `p < 0.01`, a 95% CI excluding zero, an A/A-validated + team/project, and an independent confirmation before a decision-driving + claim. +- Report every unmatched metric as "no detected difference," not a small win + or loss. +- State VM count, vCPUs, Node/Chrome versions, source fingerprint, exact + resolved framework versions, and that percentages are platform-specific. + +## Recover or diagnose failures + +Start with: + +```sh +node .agents/skills/sandbox-bench/scripts/status.mjs +``` + +The runner detaches benchmark work inside each VM, polls it, downloads results, +and removes VMs after collection. If the local launcher is interrupted, the +remote work can continue until the configured sandbox timeout. Reconnect and +collect every completed VM recorded in a run with: + +```sh +node .agents/skills/sandbox-bench/scripts/collect.mjs +``` + +The collector leaves still-running VMs alone, updates `meta.json`, removes +collected VMs, and regenerates `summary.md` once the matrix is complete. Use the +scoped `vercel sandbox list`, `exec`, and `cp` commands for deeper inspection; +remove only exact `ftrack-*` sandbox names after recovery. + +For a setup or benchmark failure, inspect the relevant per-case log and report +the failure instead of producing a comparison from an incomplete matrix. diff --git a/.agents/skills/sandbox-bench/references/methodology.md b/.agents/skills/sandbox-bench/references/methodology.md new file mode 100644 index 00000000..16fbfcbb --- /dev/null +++ b/.agents/skills/sandbox-bench/references/methodology.md @@ -0,0 +1,91 @@ +# Framework-tracker sandbox benchmark methodology + +## Use the VM boot as the independent sample + +JIT state, heap layout, CPU placement, and host conditions are shared by every +measurement inside one sandbox boot. Repetitions within a boot improve that +boot's estimate, but they are not independent evidence. + +The sandbox runner therefore: + +- prepares one content-addressed source snapshot so every VM starts from the + same files, Node version, pnpm version, root dependencies, and (when needed) + Chrome installation; +- runs every matrix case inside every VM; +- reverses case order across VM/block parity to reduce linear order effects; +- averages blocks within a VM; and +- computes confidence intervals and paired deltas across VM boots. + +Do not report the framework-tracker CLI's internal browser runs or load-test +requests as independent samples. The number of completed VMs is `n`. + +## Choose comparable cases + +- Prefer exact app packages when comparing runtime behavior, for example + `app-next-js@canary` and `app-tanstack-start-react`. +- Use framework selectors when starter and app results are both relevant. +- The runner maps `@stable` to npm's `latest` tag. Record the resolved + `frameworkVersion` from each result rather than assuming a dist-tag stayed + fixed. +- Keep measurements, `--runs`, VM size, source fingerprint, Chrome version, + and sandbox project identical across cases. +- Do not compare results from different run directories as though they were + paired. Host and platform changes can dominate small framework effects. + +## Allocation and claims + +Use one VM only for setup smoke tests. Use at least eight boots for exploratory +comparisons and sixteen when a small result may drive a decision. More internal +`--runs` cannot replace more independent boots. + +Before making claims for a new Vercel team/project or after a platform, +benchmark-app, Node, Chrome, or runner change, run an A/A calibration by +specifying the same target twice. No metric should reach boot-level `p < 0.01` +beyond the nominal false-positive rate. + +Treat a result as detected only when all of these hold: + +1. The source fingerprint, snapshot, case arguments, and completed VM count are + correct. +2. The boot-level paired comparison has `p < 0.01`. +3. The 95% confidence interval excludes zero. +4. Per-boot signs are not dominated by one outlier. +5. A decision-driving result repeats in an independent run. + +Everything else is "no detected difference," not a small win or loss. + +## Metric interpretation + +The generated summary uses a mechanical sign convention: a positive delta +means the candidate produced a larger number. Interpret direction per metric: + +- Higher is generally better for `opsPerSec`, `peakRequestsPerSec`, and similar + throughput metrics. +- Lower is generally better for latency, paint timing, INP, build/install time, + output size, and error counts. +- Body size, sample count, total requests, peak workers, and framework version + metadata describe the run; they are not automatically performance wins. +- Browser FP/FCP/INP values are noisy. Check the Chrome version and require + stronger replication than for deterministic build size. +- `ssrLoadTests.stages` is intentionally omitted from the compact summary. + Inspect the raw per-VM `ci-stats.json` files when load-curve shape matters. + +## Reporting template + +Lead with what ran and the independent sample count: + +```markdown +## Next.js canary vs TanStack Start — Vercel Sandbox, 8 boots + +Source fingerprint: ``; Node 24; 8 vCPUs; measurements: +`ssrLoad`, `ssrRequestThroughput`. + +| metric | candidate delta | 95% CI | p | verdict | +| ------------------ | --------------: | -----: | -----: | -------- | +| app SSR throughput | +12.4% | ±3.1% | 0.0008 | detected | + +No detected difference: ... +``` + +State that magnitudes are platform-specific. Link or provide the run directory +and retain `meta.json`, `summary.md`, per-case logs, and raw result JSON files. diff --git a/.agents/skills/sandbox-bench/scripts/collect.mjs b/.agents/skills/sandbox-bench/scripts/collect.mjs new file mode 100644 index 00000000..399ed67d --- /dev/null +++ b/.agents/skills/sandbox-bench/scripts/collect.mjs @@ -0,0 +1,121 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process' +import fs from 'node:fs' +import path from 'node:path' +import { promisify } from 'node:util' +import { loadConfig, sandboxScope } from './config.mjs' + +const execFileP = promisify(execFile) + +function usage() { + console.error('Usage: node collect.mjs [--keep]') +} + +async function main() { + const runDir = process.argv[2] + const keep = process.argv.includes('--keep') + if (!runDir || runDir.startsWith('-')) { + usage() + process.exit(1) + } + + const metaPath = path.join(runDir, 'meta.json') + const meta = JSON.parse(fs.readFileSync(metaPath, 'utf8')) + const config = loadConfig() + const scope = sandboxScope(config) + const sandbox = async (args) => { + const scoped = ['sandbox', ...args] + const separator = scoped.indexOf('--') + scoped.splice(separator < 0 ? scoped.length : separator, 0, ...scope) + const { stdout, stderr } = await execFileP(config.vercelBin, scoped, { + maxBuffer: 64 * 1024 * 1024, + }) + return `${stdout}\n${stderr}` + } + const save = () => + fs.writeFileSync(metaPath, `${JSON.stringify(meta, null, 2)}\n`) + let pending = 0 + let failures = 0 + + for (const [index, state] of Object.entries(meta.vms)) { + if (state.phase === 'done') continue + let done + try { + const output = await sandbox([ + 'exec', + state.vm, + '--timeout', + '2m', + '--', + 'bash', + '-c', + 'cat /vercel/sandbox/bench.done 2>/dev/null || echo no', + ]) + done = output.match(/(?:^|\n)(no|\d+)(?:\n|$)/)?.[1] + } catch (error) { + console.error(`${state.vm}: could not read status: ${error.message}`) + failures += 1 + continue + } + if (done === 'no' || done === undefined) { + console.log(`${state.vm}: still running`) + pending += 1 + continue + } + + const archive = path.join(runDir, `vm-${index}.tgz`) + const directory = path.join(runDir, `vm-${index}`) + try { + await sandbox([ + 'cp', + `${state.vm}:/vercel/sandbox/bench-results.tgz`, + archive, + ]) + fs.mkdirSync(directory, { recursive: true }) + await execFileP('tar', ['-xzf', archive, '-C', directory]) + const exitCode = Number( + fs.readFileSync(path.join(directory, 'bench.exit'), 'utf8').trim(), + ) + state.phase = exitCode === 0 ? 'done' : 'failed' + state.exitCode = exitCode + state.updatedAt = new Date().toISOString() + save() + console.log(`${state.vm}: collected (exit ${exitCode})`) + if (exitCode !== 0) failures += 1 + if (!keep) { + await sandbox(['rm', state.vm]) + } + } catch (error) { + console.error(`${state.vm}: collection failed: ${error.message}`) + failures += 1 + } + } + + const states = Object.values(meta.vms) + if ( + pending === 0 && + failures === 0 && + states.length > 0 && + states.every((state) => state.phase === 'done') + ) { + meta.phase = 'complete' + save() + const summary = await execFileP( + process.execPath, + [new URL('./summarize.mjs', import.meta.url).pathname, runDir], + { maxBuffer: 64 * 1024 * 1024 }, + ) + process.stdout.write(summary.stdout) + process.stderr.write(summary.stderr) + } else if (failures > 0) { + meta.phase = 'failed' + save() + process.exitCode = 1 + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error) + process.exitCode = 1 +}) diff --git a/.agents/skills/sandbox-bench/scripts/config.mjs b/.agents/skills/sandbox-bench/scripts/config.mjs new file mode 100644 index 00000000..ad8a102d --- /dev/null +++ b/.agents/skills/sandbox-bench/scripts/config.mjs @@ -0,0 +1,102 @@ +#!/usr/bin/env node + +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +const CONFIG_DIR = path.join( + os.homedir(), + '.config', + 'framework-tracker-sandbox-bench', +) +const CONFIG_FILE = path.join(CONFIG_DIR, 'config.json') + +function expandHome(value) { + if (!value) return value + return value.startsWith('~/') + ? path.join(os.homedir(), value.slice(2)) + : value +} + +export function loadConfig({ requireScope = true } = {}) { + const saved = fs.existsSync(CONFIG_FILE) + ? JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8')) + : {} + const config = { + ...saved, + team: process.env.FRAMEWORK_TRACKER_BENCH_TEAM ?? saved.team, + project: process.env.FRAMEWORK_TRACKER_BENCH_PROJECT ?? saved.project, + cacheDir: expandHome( + process.env.FRAMEWORK_TRACKER_BENCH_CACHE ?? + saved.cacheDir ?? + '~/.cache/framework-tracker-sandbox-bench', + ), + vercelBin: + process.env.FRAMEWORK_TRACKER_BENCH_VERCEL_BIN ?? + saved.vercelBin ?? + 'vercel', + } + + if (requireScope && (!config.team || !config.project)) { + throw new Error( + 'sandbox-bench is not configured: team and project are required.\n' + + 'Ask which Vercel team and project should be billed, then run:\n' + + ` node ${path.relative(process.cwd(), new URL(import.meta.url).pathname)} ` + + 'set team= project=', + ) + } + + return config +} + +export function saveConfig(patch) { + fs.mkdirSync(CONFIG_DIR, { recursive: true }) + const current = fs.existsSync(CONFIG_FILE) + ? JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8')) + : {} + const next = { ...current, ...patch } + fs.writeFileSync(CONFIG_FILE, `${JSON.stringify(next, null, 2)}\n`) + return next +} + +export function sandboxScope(config) { + return ['--scope', config.team, '--project', config.project] +} + +function isMain() { + if (!process.argv[1]) return false + try { + return ( + fs.realpathSync(process.argv[1]) === + fs.realpathSync(new URL(import.meta.url).pathname) + ) + } catch { + return false + } +} + +if (isMain()) { + const [command, ...args] = process.argv.slice(2) + + if (command === 'show') { + const config = loadConfig({ requireScope: false }) + console.log(JSON.stringify(config, null, 2)) + if (!config.team || !config.project) { + console.error('\nNOT CONFIGURED: team/project missing.') + process.exitCode = 2 + } + } else if (command === 'set') { + const patch = {} + for (const argument of args) { + const separator = argument.indexOf('=') + if (separator < 1) { + throw new Error(`Expected key=value, received "${argument}"`) + } + patch[argument.slice(0, separator)] = argument.slice(separator + 1) + } + console.log(JSON.stringify(saveConfig(patch), null, 2)) + } else { + console.error('Usage: node config.mjs show | set key=value [key=value...]') + process.exitCode = 1 + } +} diff --git a/.agents/skills/sandbox-bench/scripts/sandbox-bench.mjs b/.agents/skills/sandbox-bench/scripts/sandbox-bench.mjs new file mode 100644 index 00000000..8c636cf7 --- /dev/null +++ b/.agents/skills/sandbox-bench/scripts/sandbox-bench.mjs @@ -0,0 +1,848 @@ +#!/usr/bin/env node + +import crypto from 'node:crypto' +import { execFile, spawn } from 'node:child_process' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { promisify } from 'node:util' +import { loadConfig, sandboxScope } from './config.mjs' + +const execFileP = promisify(execFile) +const SETUP_VERSION = 'framework-tracker-node24-pnpm11-chrome1' +const RESULT_FILES = ['ci-stats.json', 'install-stats.json', 'build-stats.json'] +const MEASUREMENTS = new Set([ + 'install', + 'build', + 'ssrRequestThroughput', + 'ssrLoad', + 'clientSideRendered', + 'serverSideRendered', +]) + +function usage() { + console.log(`Run framework-tracker benchmark matrices on Vercel Sandbox. + +Usage: + node sandbox-bench.mjs [options] + +Targets: + Framework or package selectors accepted by "pnpm benchmark". + Append @ to run that framework with a temporary version. + "stable" is normalized to the npm "latest" tag. + +Options: + -m, --measurement Comma-separated benchmark types; repeatable + -r, --runs Override the benchmark's internal run count + --vms Independent sandbox boots (default: 8) + --concurrency Concurrent sandboxes (default: 5) + --blocks Matrix repetitions inside each boot (default: 1) + --vcpus vCPUs per sandbox (default: 8) + --timeout Sandbox timeout, such as 2h or 5h (default: 5h) + --label Human-readable run label + --no-cache Build a fresh setup snapshot + --keep Keep measurement sandboxes after collection + --dry-run Print the resolved plan without creating resources + -h, --help Show this help + +Examples: + node sandbox-bench.mjs next@latest next@canary \\ + -m ssrLoad,ssrRequestThroughput --vms 8 + node sandbox-bench.mjs app-next-js@canary app-tanstack-start-react \\ + -m ssrLoad,ssrRequestThroughput,clientSideRendered,serverSideRendered + node sandbox-bench.mjs astro next nuxt -m build --vms 4 --runs 3`) +} + +function optionValue(args, index, option) { + const value = args[index + 1] + if (!value || value.startsWith('-')) { + throw new Error(`${option} requires a value`) + } + return value +} + +function positiveInteger(value, option) { + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`${option} must be a positive integer`) + } + return parsed +} + +function splitValues(value) { + return value + .split(',') + .map((item) => item.trim()) + .filter(Boolean) +} + +function slug(value) { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 48) +} + +function parseTarget(spec, index) { + const separator = spec.lastIndexOf('@') + let selector = spec + let version + if (separator > 0) { + selector = spec.slice(0, separator) + version = spec.slice(separator + 1) + if (!version) throw new Error(`Missing version in target "${spec}"`) + if (/[\s\x00-\x1f]/.test(version)) { + throw new Error(`Invalid framework version in target "${spec}"`) + } + } + if (!/^[a-z0-9][a-z0-9-]*$/.test(selector)) { + throw new Error(`Invalid framework or package selector "${selector}"`) + } + if (version === 'stable') version = 'latest' + return { + id: `case-${index + 1}-${slug(spec)}`, + label: spec, + selector, + version, + } +} + +function parseArgs(argv) { + const options = { + targets: [], + measurements: [], + runs: undefined, + vms: 8, + concurrency: 5, + blocks: 1, + vcpus: 8, + timeout: '5h', + label: 'matrix', + noCache: false, + keep: false, + dryRun: false, + } + + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index] + if (argument === '-h' || argument === '--help') return null + if (argument === '--dry-run') { + options.dryRun = true + continue + } + if (argument === '--no-cache') { + options.noCache = true + continue + } + if (argument === '--keep') { + options.keep = true + continue + } + if ( + argument === '-m' || + argument === '--measurement' || + argument === '--measurements' + ) { + options.measurements.push( + ...splitValues(optionValue(argv, index, argument)), + ) + index += 1 + continue + } + if (argument.startsWith('--measurement=')) { + options.measurements.push( + ...splitValues(argument.slice('--measurement='.length)), + ) + continue + } + if (argument === '-r' || argument === '--runs') { + options.runs = positiveInteger( + optionValue(argv, index, argument), + argument, + ) + index += 1 + continue + } + if ( + argument === '--vms' || + argument === '--concurrency' || + argument === '--blocks' || + argument === '--vcpus' + ) { + options[argument.slice(2)] = positiveInteger( + optionValue(argv, index, argument), + argument, + ) + index += 1 + continue + } + if (argument === '--timeout' || argument === '--label') { + options[argument.slice(2)] = optionValue(argv, index, argument) + index += 1 + continue + } + if (argument.startsWith('-')) { + throw new Error(`Unknown option: ${argument}`) + } + options.targets.push(argument) + } + + if (options.targets.length === 0) { + throw new Error('Provide at least one framework or package selector') + } + for (const measurement of options.measurements) { + if (!MEASUREMENTS.has(measurement)) { + throw new Error( + `Unknown measurement "${measurement}". Expected: ${[...MEASUREMENTS].join(', ')}`, + ) + } + } + if (!/^\d+(m|h)$/.test(options.timeout)) { + throw new Error( + '--timeout must use minutes or hours, for example 45m or 5h', + ) + } + + options.measurements = [...new Set(options.measurements)] + options.cases = options.targets.map(parseTarget) + options.label = slug(options.label) || 'matrix' + return options +} + +function shellQuote(value) { + return `'${String(value).replaceAll("'", "'\\''")}'` +} + +function timeoutMs(value) { + const count = Number(value.slice(0, -1)) + return count * (value.endsWith('h') ? 60 * 60_000 : 60_000) +} + +async function repoRoot() { + const { stdout } = await execFileP('git', ['rev-parse', '--show-toplevel'], { + maxBuffer: 1024 * 1024, + }) + const root = stdout.trim() + const packageJson = JSON.parse( + fs.readFileSync(path.join(root, 'package.json'), 'utf8'), + ) + if (packageJson.name !== 'framework-tracker') { + throw new Error(`Expected the framework-tracker repository, found ${root}`) + } + return root +} + +async function createSourceArchive(root, tempDir) { + const { stdout } = await execFileP( + 'git', + ['ls-files', '-z', '--cached', '--others', '--exclude-standard'], + { cwd: root, encoding: 'buffer', maxBuffer: 64 * 1024 * 1024 }, + ) + const files = stdout + .toString('utf8') + .split('\0') + .filter(Boolean) + .filter((file) => fs.existsSync(path.join(root, file))) + .sort() + if (files.length === 0) throw new Error('No repository files to upload') + + const listPath = path.join(tempDir, 'source-files') + const archivePath = path.join(tempDir, 'framework-tracker.tgz') + const fingerprint = crypto.createHash('sha256') + for (const file of files) { + const absolutePath = path.join(root, file) + const stat = fs.lstatSync(absolutePath) + fingerprint.update(`${file}\0${stat.mode}\0`) + fingerprint.update( + stat.isSymbolicLink() + ? fs.readlinkSync(absolutePath) + : fs.readFileSync(absolutePath), + ) + fingerprint.update('\0') + } + fs.writeFileSync(listPath, `${files.join('\0')}\0`) + await execFileP( + 'tar', + ['--no-xattrs', '-czf', archivePath, '--null', '-T', listPath], + { + cwd: root, + env: { ...process.env, COPYFILE_DISABLE: '1' }, + maxBuffer: 64 * 1024 * 1024, + }, + ) + const hash = fingerprint.digest('hex').slice(0, 20) + return { archivePath, hash, files: files.length } +} + +function makeSandboxClient(config) { + const scope = sandboxScope(config) + + async function sandbox(args, options = {}) { + const scoped = ['sandbox', ...args] + const separator = scoped.indexOf('--') + scoped.splice(separator < 0 ? scoped.length : separator, 0, ...scope) + const { stdout, stderr } = await execFileP(config.vercelBin, scoped, { + maxBuffer: 64 * 1024 * 1024, + ...options, + }) + return `${stdout}\n${stderr}` + } + + function streamExec(vm, duration, script, tag) { + return new Promise((resolve, reject) => { + const child = spawn( + config.vercelBin, + [ + 'sandbox', + 'exec', + vm, + ...scope, + '--timeout', + duration, + '--', + 'bash', + '-c', + script, + ], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ) + let output = '' + for (const [stream, marker] of [ + [child.stdout, ''], + [child.stderr, '!'], + ]) { + stream.on('data', (chunk) => { + output += chunk + process.stderr.write( + String(chunk) + .split('\n') + .filter(Boolean) + .map((line) => `[${tag}${marker}] ${line}\n`) + .join(''), + ) + }) + } + child.on('error', reject) + child.on('exit', (code) => { + if (code === 0) resolve(output) + else reject(new Error(`${tag} exited ${code}\n${output.slice(-2000)}`)) + }) + }) + } + + async function remove(vm) { + try { + await sandbox(['rm', vm]) + } catch (error) { + console.error(`warning: could not remove ${vm}: ${error.message}`) + } + } + + return { sandbox, streamExec, remove } +} + +async function cachedSnapshot(client, config, key) { + const cacheFile = path.join(config.cacheDir, 'snapshots', key) + if (!fs.existsSync(cacheFile)) return undefined + const snapshot = fs.readFileSync(cacheFile, 'utf8').trim() + if (!snapshot) return undefined + try { + const output = await client.sandbox(['snapshots', 'list', '--limit', '50']) + return output.includes(snapshot) ? snapshot : undefined + } catch { + return undefined + } +} + +function saveSnapshot(config, key, snapshot) { + const directory = path.join(config.cacheDir, 'snapshots') + fs.mkdirSync(directory, { recursive: true }) + fs.writeFileSync(path.join(directory, key), `${snapshot}\n`) +} + +async function prepareSnapshot(client, config, options, source, needsChrome) { + const key = crypto + .createHash('sha256') + .update(`${SETUP_VERSION}:${source.hash}:chrome=${needsChrome}`) + .digest('hex') + .slice(0, 24) + if (!options.noCache) { + const cached = await cachedSnapshot(client, config, key) + if (cached) { + console.error(`reusing setup snapshot ${cached}`) + return cached + } + } + + const vm = `ftrack-setup-${Date.now().toString(36)}` + console.error(`creating setup sandbox ${vm}`) + await client.sandbox([ + 'create', + '--name', + vm, + '--runtime', + 'node24', + '--vcpus', + String(options.vcpus), + '--timeout', + '45m', + '--non-persistent', + '--network-policy', + 'allow-all', + '--tag', + 'purpose=framework-tracker-bench', + '--silent', + ]) + + try { + await client.sandbox([ + 'cp', + source.archivePath, + `${vm}:/vercel/sandbox/framework-tracker.tgz`, + ]) + if (needsChrome) { + console.error('installing Google Chrome for browser measurements') + await client.sandbox([ + 'exec', + vm, + '--timeout', + '15m', + '--sudo', + '--', + 'dnf', + 'install', + '-y', + '-q', + 'https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm', + ]) + } + await client.streamExec( + vm, + '25m', + `set -e +mkdir -p /vercel/sandbox/framework-tracker +cd /vercel/sandbox/framework-tracker +tar -xzf ../framework-tracker.tgz +rm -f ../framework-tracker.tgz +npm install -g pnpm@11.1.1 +pnpm install --frozen-lockfile +node --version +pnpm --version`, + 'setup', + ) + const output = await client.sandbox([ + 'snapshot', + vm, + '--stop', + '--expiration', + '7d', + ]) + const snapshot = output.match(/snap_[A-Za-z0-9]+/)?.[0] + if (!snapshot) { + throw new Error( + `Could not parse snapshot id from:\n${output.slice(-1000)}`, + ) + } + saveSnapshot(config, key, snapshot) + console.error(`prepared snapshot ${snapshot}`) + return snapshot + } finally { + await client.remove(vm) + } +} + +function benchmarkArgs(testCase, options) { + const args = testCase.selector === 'all' ? ['--all'] : [testCase.selector] + if (options.measurements.length > 0) { + args.push('--measurement', options.measurements.join(',')) + } + if (options.runs !== undefined) { + args.push('--runs', String(options.runs)) + } + if (testCase.version) { + args.push('--version', testCase.version) + } + return args +} + +async function validateCases(root, options) { + const cli = path.join( + root, + 'packages', + 'stats-generator', + 'src', + 'run-benchmarks.ts', + ) + for (const testCase of options.cases) { + try { + await execFileP( + process.execPath, + [cli, ...benchmarkArgs(testCase, options), '--dry-run'], + { + cwd: root, + env: { + ...process.env, + npm_config_userconfig: path.join(root, '.npmrc'), + }, + maxBuffer: 16 * 1024 * 1024, + }, + ) + } catch (error) { + throw new Error( + `Invalid benchmark case "${testCase.label}": ${ + error.stderr?.trim() || error.stdout?.trim() || error.message + }`, + ) + } + } +} + +function remoteScript(vmIndex, options) { + const commands = [] + for (let block = 1; block <= options.blocks; block += 1) { + const reverse = (vmIndex + block) % 2 === 1 + const ordered = reverse ? [...options.cases].reverse() : options.cases + for (const testCase of ordered) { + const invocation = [ + 'pnpm', + 'benchmark', + ...benchmarkArgs(testCase, options), + ] + .map(shellQuote) + .join(' ') + commands.push(` +printf '%s\\n' ${shellQuote(`CASE ${testCase.label} block ${block}`)} +find packages -mindepth 2 -maxdepth 2 -type f \\( ${RESULT_FILES.map((file) => `-name ${shellQuote(file)}`).join(' -o ')} \\) -delete +DEST="$OUTPUT/block-${block}/${testCase.id}" +mkdir -p "$DEST" +set +e +${invocation} 2>&1 | tee "$DEST/benchmark.log" +CODE=\${PIPESTATUS[0]} +set -e +if [ "$CODE" -ne 0 ]; then + printf '%s exit %s\\n' ${shellQuote( + `benchmark failed: ${testCase.label} block ${block}`, + )} "$CODE" + exit "$CODE" +fi +find packages -mindepth 2 -maxdepth 2 -type f \\( ${RESULT_FILES.map((file) => `-name ${shellQuote(file)}`).join(' -o ')} \\) -print0 > /tmp/framework-tracker-result-files +while IFS= read -r -d '' FILE; do + mkdir -p "$DEST/$(dirname "$FILE")" + cp "$FILE" "$DEST/$FILE" +done < /tmp/framework-tracker-result-files +`) + } + } + + return `#!/usr/bin/env bash +set -euo pipefail +ROOT=/vercel/sandbox/framework-tracker +OUTPUT=/vercel/sandbox/bench-output +rm -rf "$OUTPUT" +mkdir -p "$OUTPUT" +finish() { + CODE=$? + trap - EXIT + printf '%s\\n' "$CODE" > /vercel/sandbox/bench.exit + tar -czf /vercel/sandbox/bench-results.tgz -C /vercel/sandbox bench-output bench.log bench.exit 2>/dev/null || true + printf '%s\\n' "$CODE" > /vercel/sandbox/bench.done + exit "$CODE" +} +trap finish EXIT +cd "$ROOT" +export CI=1 +export NO_COLOR=1 +if [ -x /opt/google/chrome/chrome ]; then + export CHROME_PATH=/opt/google/chrome/chrome +fi +echo "vm=${vmIndex} cpu=$(grep -m1 'model name' /proc/cpuinfo | cut -d: -f2- | sed 's/^ //')" +echo "node=$(node --version) pnpm=$(pnpm --version)" +${commands.join('\n')} +echo "all benchmark cases complete" + ` +} + +async function validateRemoteScript(options) { + const directory = fs.mkdtempSync( + path.join(os.tmpdir(), 'framework-tracker-bash-check-'), + ) + const file = path.join(directory, 'run.sh') + try { + fs.writeFileSync(file, remoteScript(1, options)) + await execFileP('bash', ['-n', file]) + } finally { + fs.rmSync(directory, { recursive: true, force: true }) + } +} + +async function sleep(milliseconds) { + await new Promise((resolve) => setTimeout(resolve, milliseconds)) +} + +async function collectVm(client, vm, runDir, index) { + const archive = path.join(runDir, `vm-${index}.tgz`) + const directory = path.join(runDir, `vm-${index}`) + await client.sandbox([ + 'cp', + `${vm}:/vercel/sandbox/bench-results.tgz`, + archive, + ]) + fs.mkdirSync(directory, { recursive: true }) + await execFileP('tar', ['-xzf', archive, '-C', directory]) + const exitCode = Number( + fs.readFileSync(path.join(directory, 'bench.exit'), 'utf8').trim(), + ) + return exitCode +} + +async function runVm(client, snapshot, options, runDir, index, updateStatus) { + const vm = `ftrack-${options.label}-${index}-${Date.now().toString(36)}` + updateStatus(index, { vm, phase: 'creating' }) + await client.sandbox([ + 'create', + '--name', + vm, + '--snapshot', + snapshot, + '--vcpus', + String(options.vcpus), + '--timeout', + options.timeout, + '--non-persistent', + '--network-policy', + 'allow-all', + '--tag', + 'purpose=framework-tracker-bench', + '--silent', + ]) + + let collected = false + try { + const scriptPath = path.join(runDir, `vm-${index}.sh`) + fs.writeFileSync(scriptPath, remoteScript(index, options)) + await client.sandbox([ + 'cp', + scriptPath, + `${vm}:/vercel/sandbox/run-benchmarks.sh`, + ]) + await client.sandbox([ + 'exec', + vm, + '--timeout', + '2m', + '--', + 'bash', + '-c', + 'rm -f /vercel/sandbox/bench.done /vercel/sandbox/bench.exit /vercel/sandbox/bench.log; nohup bash /vercel/sandbox/run-benchmarks.sh >/vercel/sandbox/bench.log 2>&1 & echo started', + ]) + updateStatus(index, { vm, phase: 'running' }) + + const deadline = Date.now() + timeoutMs(options.timeout) + let failures = 0 + let lastLine = '' + while (Date.now() < deadline) { + await sleep(30_000) + let output + try { + output = await client.sandbox([ + 'exec', + vm, + '--timeout', + '2m', + '--', + 'bash', + '-c', + 'printf \'@@DONE %s\\n\' "$(cat /vercel/sandbox/bench.done 2>/dev/null || echo no)"; tail -n 8 /vercel/sandbox/bench.log 2>/dev/null || true', + ]) + failures = 0 + } catch (error) { + failures += 1 + if (failures >= 6) throw error + continue + } + const done = output.match(/@@DONE (\S+)/)?.[1] + const progress = output + .split('\n') + .filter( + (line) => + line && + !line.startsWith('@@DONE') && + !line.startsWith('Vercel CLI') && + !line.startsWith('$ bash -c') && + !line.startsWith('- Fetching'), + ) + .at(-1) + if (progress && progress !== lastLine) { + lastLine = progress + console.error(`[vm${index}] ${progress}`) + } + if (done !== 'no' && done !== undefined) { + updateStatus(index, { vm, phase: 'collecting' }) + const exitCode = await collectVm(client, vm, runDir, index) + collected = true + if (exitCode !== 0) { + throw new Error(`${vm} benchmark exited ${exitCode}`) + } + updateStatus(index, { vm, phase: 'done' }) + return + } + } + throw new Error(`${vm} exceeded local deadline ${options.timeout}`) + } finally { + if (!collected) { + try { + await collectVm(client, vm, runDir, index) + } catch {} + } + if (!options.keep) await client.remove(vm) + } +} + +async function main() { + const options = parseArgs(process.argv.slice(2)) + if (!options) { + usage() + return + } + const root = await repoRoot() + await validateCases(root, options) + await validateRemoteScript(options) + const config = loadConfig({ requireScope: !options.dryRun }) + const needsChrome = + options.measurements.length === 0 || + options.measurements.some( + (measurement) => + measurement === 'clientSideRendered' || + measurement === 'serverSideRendered', + ) + const plan = { + cases: options.cases.map((testCase) => ({ + ...testCase, + benchmarkArgs: benchmarkArgs(testCase, options), + })), + measurements: + options.measurements.length > 0 ? options.measurements : ['configured'], + vms: options.vms, + concurrency: options.concurrency, + blocks: options.blocks, + vcpus: options.vcpus, + timeout: options.timeout, + needsChrome, + } + console.log(JSON.stringify(plan, null, 2)) + if (options.dryRun) { + console.log('Dry run complete; no Vercel resources were created.') + return + } + + const tempDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'framework-tracker-sandbox-bench-'), + ) + const runId = `${new Date().toISOString().replace(/[:.]/g, '-')}-${options.label}` + const runDir = path.join(config.cacheDir, 'runs', runId) + fs.mkdirSync(runDir, { recursive: true }) + const status = { + runId, + runDir, + createdAt: new Date().toISOString(), + phase: 'preparing', + sourceHash: null, + snapshot: null, + plan, + vms: {}, + } + const writeStatus = () => + fs.writeFileSync( + path.join(runDir, 'meta.json'), + `${JSON.stringify(status, null, 2)}\n`, + ) + const updateStatus = (index, patch) => { + status.vms[index] = { + ...status.vms[index], + ...patch, + updatedAt: new Date().toISOString(), + } + writeStatus() + } + writeStatus() + console.error(`run directory: ${runDir}`) + + try { + const source = await createSourceArchive(root, tempDir) + status.sourceHash = source.hash + writeStatus() + console.error( + `packed ${source.files} files; source fingerprint ${source.hash}`, + ) + const client = makeSandboxClient(config) + const snapshot = await prepareSnapshot( + client, + config, + options, + source, + needsChrome, + ) + status.snapshot = snapshot + status.phase = 'running' + writeStatus() + + const results = Array(options.vms) + let nextVm = 1 + const runWorker = async () => { + while (nextVm <= options.vms) { + const index = nextVm + nextVm += 1 + try { + await runVm(client, snapshot, options, runDir, index, updateStatus) + results[index - 1] = { status: 'fulfilled' } + } catch (reason) { + results[index - 1] = { status: 'rejected', reason } + } + } + } + await Promise.all( + Array.from( + { length: Math.min(options.concurrency, options.vms) }, + runWorker, + ), + ) + const failures = results + .map((result, index) => ({ result, index: index + 1 })) + .filter(({ result }) => result.status === 'rejected') + if (failures.length > 0) { + status.phase = 'failed' + writeStatus() + throw new Error( + failures + .map( + ({ result, index }) => + `vm${index}: ${result.status === 'rejected' ? result.reason.message : ''}`, + ) + .join('\n'), + ) + } + + status.phase = 'complete' + writeStatus() + const summary = await execFileP( + process.execPath, + [new URL('./summarize.mjs', import.meta.url).pathname, runDir], + { maxBuffer: 64 * 1024 * 1024 }, + ) + process.stdout.write(summary.stdout) + process.stderr.write(summary.stderr) + } catch (error) { + status.phase = 'failed' + status.error = error instanceof Error ? error.message : String(error) + writeStatus() + throw error + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }) + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error) + process.exitCode = 1 +}) diff --git a/.agents/skills/sandbox-bench/scripts/status.mjs b/.agents/skills/sandbox-bench/scripts/status.mjs new file mode 100644 index 00000000..31d8d6a9 --- /dev/null +++ b/.agents/skills/sandbox-bench/scripts/status.mjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process' +import fs from 'node:fs' +import path from 'node:path' +import { promisify } from 'node:util' +import { loadConfig, sandboxScope } from './config.mjs' + +const execFileP = promisify(execFile) + +function recentRuns(cacheDir) { + const runsDir = path.join(cacheDir, 'runs') + if (!fs.existsSync(runsDir)) return [] + return fs + .readdirSync(runsDir) + .map((name) => path.join(runsDir, name, 'meta.json')) + .filter((file) => fs.existsSync(file)) + .map((file) => JSON.parse(fs.readFileSync(file, 'utf8'))) + .sort((left, right) => right.createdAt.localeCompare(left.createdAt)) + .slice(0, 10) +} + +async function main() { + const config = loadConfig() + const runs = recentRuns(config.cacheDir) + if (runs.length === 0) { + console.log('No local runs recorded.') + } else { + for (const run of runs) { + const phases = Object.values(run.vms).reduce((counts, vm) => { + counts[vm.phase] = (counts[vm.phase] ?? 0) + 1 + return counts + }, {}) + console.log( + `${run.runId} run=${run.phase ?? 'unknown'} ${Object.entries(phases) + .map(([phase, count]) => `${phase}=${count}`) + .join(' ')}\n ${run.runDir}`, + ) + } + } + + const { stdout, stderr } = await execFileP( + config.vercelBin, + ['sandbox', 'list', '--limit', '50', ...sandboxScope(config)], + { maxBuffer: 16 * 1024 * 1024 }, + ) + console.log('\nVercel sandboxes:') + console.log(`${stdout}${stderr}`.trim()) +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error) + process.exitCode = 1 +}) diff --git a/.agents/skills/sandbox-bench/scripts/summarize.mjs b/.agents/skills/sandbox-bench/scripts/summarize.mjs new file mode 100644 index 00000000..67c10519 --- /dev/null +++ b/.agents/skills/sandbox-bench/scripts/summarize.mjs @@ -0,0 +1,273 @@ +#!/usr/bin/env node + +import fs from 'node:fs' +import path from 'node:path' + +const T975 = [ + 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, + 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, + 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042, +] + +function tCritical975(df) { + if (df < 1) return Infinity + if (df <= 30) return T975[df - 1] + return 1.96 + 2.4 / df +} + +function tTestP(values) { + const count = values.length + if (count < 2 || values.some((value) => !Number.isFinite(value))) return 1 + const mean = values.reduce((sum, value) => sum + value, 0) / count + const standardDeviation = Math.sqrt( + values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (count - 1), + ) + if (standardDeviation === 0) return mean === 0 ? 1 : 0 + const t = Math.abs(mean / (standardDeviation / Math.sqrt(count))) + const degrees = count - 1 + if (degrees === 1) { + return Math.min(1, Math.max(0, 1 - (2 / Math.PI) * Math.atan(t))) + } + if (!Number.isFinite(t) || t > 45) return 0 + const pdf = (value) => + Math.exp(-((degrees + 1) / 2) * Math.log(1 + (value * value) / degrees)) + let tail = 0 + let normalizer = 0 + const step = 0.001 + for (let value = t; value < t + 60; value += step) { + tail += pdf(value + step / 2) * step + } + for (let value = 0; value < 80; value += step) { + normalizer += pdf(value + step / 2) * step + } + return Math.min(1, tail / normalizer) +} + +function stats(values) { + if (values.length === 0) return undefined + const mean = values.reduce((sum, value) => sum + value, 0) / values.length + if (values.length === 1) { + return { mean, ci95: Infinity, p: 1, count: 1 } + } + const standardDeviation = Math.sqrt( + values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / + (values.length - 1), + ) + return { + mean, + ci95: + (tCritical975(values.length - 1) * standardDeviation) / + Math.sqrt(values.length), + p: tTestP(values), + count: values.length, + } +} + +function walk(directory) { + const files = [] + if (!fs.existsSync(directory)) return files + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const file = path.join(directory, entry.name) + if (entry.isDirectory()) files.push(...walk(file)) + else files.push(file) + } + return files +} + +function flattenNumbers(value, prefix = '', output = {}) { + if (typeof value === 'number' && Number.isFinite(value)) { + output[prefix] = value + return output + } + if (!value || typeof value !== 'object' || Array.isArray(value)) return output + for (const [key, child] of Object.entries(value)) { + const next = prefix ? `${prefix}.${key}` : key + if (next === 'ssrLoadTests.stages') continue + flattenNumbers(child, next, output) + } + return output +} + +function packageRole(packageName) { + if (packageName.startsWith('app-')) return 'app' + if (packageName.startsWith('starter-')) return 'starter' + return packageName +} + +function formatNumber(value) { + const absolute = Math.abs(value) + if (absolute >= 1000000) return value.toFixed(0) + if (absolute >= 1000) return value.toFixed(1) + if (absolute >= 10) return value.toFixed(2) + return value.toFixed(3) +} + +function formatP(value) { + if (value < 0.0001) return '<0.0001' + return value.toFixed(4) +} + +function parseSamples(runDir, meta) { + const caseLabels = new Map( + meta.plan.cases.map((testCase) => [testCase.id, testCase.label]), + ) + const samples = [] + for (const file of walk(runDir)) { + if (!file.endsWith('.json')) continue + const relative = path.relative(runDir, file) + const match = relative.match( + /^vm-(\d+)\/bench-output\/block-(\d+)\/(case-[^/]+)\/packages\/([^/]+)\/(ci-stats|install-stats|build-stats)\.json$/, + ) + if (!match) continue + const [, vm, block, caseId, packageName, fileType] = match + const data = JSON.parse(fs.readFileSync(file, 'utf8')) + for (const [metric, value] of Object.entries(flattenNumbers(data))) { + samples.push({ + vm: Number(vm), + block: Number(block), + caseId, + caseLabel: caseLabels.get(caseId) ?? caseId, + packageName, + key: `${packageRole(packageName)}/${fileType}:${metric}`, + value, + }) + } + } + return samples +} + +function bootMeans(samples) { + const grouped = new Map() + for (const sample of samples) { + const key = `${sample.caseId}\0${sample.key}\0${sample.vm}` + const values = grouped.get(key) ?? [] + values.push(sample.value) + grouped.set(key, values) + } + const means = new Map() + for (const [key, values] of grouped) { + means.set( + key, + values.reduce((sum, value) => sum + value, 0) / values.length, + ) + } + return means +} + +function summarizeAbsolute(samples, means, cases) { + const lines = [ + '## Boot-level means', + '', + '| case | metric | mean | 95% CI | boots |', + '|---|---|---:|---:|---:|', + ] + const metrics = [...new Set(samples.map((sample) => sample.key))].sort() + for (const testCase of cases) { + for (const metric of metrics) { + const values = [] + const seenVms = new Set() + for (const sample of samples) { + if (sample.caseId !== testCase.id || sample.key !== metric) continue + if (seenVms.has(sample.vm)) continue + const value = means.get(`${testCase.id}\0${metric}\0${sample.vm}`) + if (value !== undefined) { + seenVms.add(sample.vm) + values.push(value) + } + } + const result = stats(values) + if (!result) continue + lines.push( + `| ${testCase.label} | \`${metric}\` | ${formatNumber(result.mean)} | ${ + result.ci95 === Infinity ? 'n/a' : `±${formatNumber(result.ci95)}` + } | ${result.count} |`, + ) + } + } + return lines +} + +function summarizeComparisons(samples, means, cases) { + if (cases.length < 2) return [] + const base = cases[0] + const lines = [ + '', + `## Paired comparisons against ${base.label}`, + '', + 'Positive deltas mean the candidate produced a larger numeric value. The VM boot is the unit of replication.', + '', + '| candidate | metric | delta | 95% CI | p | boots |', + '|---|---|---:|---:|---:|---:|', + ] + const metrics = [...new Set(samples.map((sample) => sample.key))].sort() + const vmIds = [...new Set(samples.map((sample) => sample.vm))].sort( + (left, right) => left - right, + ) + + for (const candidate of cases.slice(1)) { + for (const metric of metrics) { + const deltas = [] + for (const vm of vmIds) { + const baseValue = means.get(`${base.id}\0${metric}\0${vm}`) + const candidateValue = means.get(`${candidate.id}\0${metric}\0${vm}`) + if ( + baseValue === undefined || + candidateValue === undefined || + baseValue === 0 + ) { + continue + } + deltas.push((candidateValue - baseValue) / Math.abs(baseValue)) + } + const result = stats(deltas) + if (!result) continue + lines.push( + `| ${candidate.label} | \`${metric}\` | ${(result.mean * 100).toFixed(2)}% | ${ + result.ci95 === Infinity + ? 'n/a' + : `±${(result.ci95 * 100).toFixed(2)}%` + } | ${formatP(result.p)} | ${result.count} |`, + ) + } + } + return lines +} + +function main() { + const runDir = process.argv[2] + if (!runDir) { + console.error('Usage: node summarize.mjs ') + process.exit(1) + } + const meta = JSON.parse( + fs.readFileSync(path.join(runDir, 'meta.json'), 'utf8'), + ) + const samples = parseSamples(runDir, meta) + if (samples.length === 0) { + throw new Error(`No benchmark result JSON files found in ${runDir}`) + } + const means = bootMeans(samples) + const lines = [ + `# ${meta.runId}`, + '', + `Source: \`${meta.sourceHash}\` `, + `Snapshot: \`${meta.snapshot}\` `, + `Completed boots: ${Object.values(meta.vms).filter((vm) => vm.phase === 'done').length}`, + '', + ...summarizeAbsolute(samples, means, meta.plan.cases), + ...summarizeComparisons(samples, means, meta.plan.cases), + '', + 'Treat p-values as descriptive until this team/project has passed an A/A calibration. Confirm decision-driving findings with an independent run.', + '', + ] + const output = `${lines.join('\n')}\n` + fs.writeFileSync(path.join(runDir, 'summary.md'), output) + console.log(output) +} + +try { + main() +} catch (error) { + console.error(error instanceof Error ? error.message : error) + process.exitCode = 1 +}