benchmark: replace simulation with walltime macro benches + e2e correctness

.github/workflows/benchmark.yml

@@ -17,7 +17,7 @@ concurrency:
 jobs:
   build:
     name: Build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: read # clone repo
     steps:
@@ -32,17 +32,7 @@ jobs:
         with:
           deno-version: v2.x
 
-      - name: Cache WASM
-        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
-        id: wasm-cache
-        with:
-          path: |
-            clayterm.wasm
-            wasm.ts
-          key: wasm-${{ hashFiles('Makefile', 'src/**', 'tasks/bundle-wasm.ts') }}
-
       - name: Build WASM
-        if: steps.wasm-cache.outputs.cache-hit != 'true'
         run: make
 
       - name: Cache dependencies
@@ -63,52 +53,7 @@ jobs:
           retention-days: 1
           path: wasm.ts
 
-  simulation:
-    name: Run benchmarks (simulation)
-    needs: build
-    runs-on: codspeed-macro
-    permissions:
-      contents: read # clone repo
-      id-token: write # upload benchmark results to codspeed
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-
-      - name: Setup Deno
-        uses: denoland/setup-deno@667a34cdef165d8d2b2e98dde39547c9daac7282 # v2.0.4
-        with:
-          deno-version: v2.x
-
-      - name: Setup Node
-        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
-        with:
-          node-version: 24
-
-      - name: Download build artifact
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: bench-build
-
-      - name: Restore dependencies
-        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
-        id: deno-cache
-        with:
-          path: node_modules
-          key: deno-${{ hashFiles('deno.lock') }}
-
-      - name: Install dependencies
-        if: steps.deno-cache.outputs.cache-hit != 'true'
-        run: deno install
-
-      - name: Run benchmarks
-        uses: CodSpeedHQ/action@9d332c4d90b43981c3e55ae8e38e68709996240f # v4.17.0
-        with:
-          mode: simulation
-          # IMPORTANT! deno task bench fails in CI due to incompatible V8 bindings
-          run: node bench/mod.ts
-
-  walltime:
+  benchmarks:
     name: Run benchmarks (walltime)
     needs: build
     runs-on: codspeed-macro
@@ -146,8 +91,12 @@ jobs:
         if: steps.deno-cache.outputs.cache-hit != 'true'
         run: deno install
 
-      - name: Run process startup benchmarks
+      - name: Run walltime benchmarks
         uses: CodSpeedHQ/action@9d332c4d90b43981c3e55ae8e38e68709996240f # v4.17.0
         with:
           mode: walltime
-          run: node bench/startup.bench.ts
+          run: |
+            node bench/startup.bench.ts
+            node bench/throughput.bench.ts
+            node bench/render.bench.ts
+            node bench/ops.bench.ts

bench/fixtures/utils.ts

@@ -1,6 +1,17 @@
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
 
+export { withCodSpeed } from "@codspeed/tinybench-plugin";
+
+// CodSpeed walltime reads `task.result.latency` which
+// tinybench only sets for async tasks
+export function sync(fn: () => void): () => Promise<void> {
+  return () => {
+    fn();
+    return Promise.resolve();
+  };
+}
+
 export const fixture = (name: string) => {
   return new URL(`./${name}/mod.ts`, import.meta.url);
 };

bench/mod.ts

@@ -1,3 +1,3 @@
-import "./input.bench.ts";
+import "./throughput.bench.ts";
 import "./render.bench.ts";
 import "./ops.bench.ts";

bench/ops.bench.ts

@@ -1,20 +1,8 @@
 import { Bench } from "tinybench";
-import { withCodSpeed } from "@codspeed/tinybench-plugin";
+import { sync, withCodSpeed } from "./fixtures/utils.ts";
 import { close, fixed, grow, open, pack, rgba, text } from "../ops.ts";
 import type { Op } from "../ops.ts";
 
-function makeBuf(size: number): ArrayBuffer {
-  return new ArrayBuffer(size);
-}
-
-let simpleOps: Op[] = [
-  open("root", {
-    layout: { width: grow(), height: grow(), direction: "ttb" },
-  }),
-  text("Hello, World!"),
-  close(),
-];
-
 let complexOps: Op[] = [
   open("root", {
     layout: { width: grow(), height: grow(), direction: "ttb" },
@@ -104,21 +92,23 @@ let listOps: Op[] = [
   close(),
 ];
 
-let bench = withCodSpeed(new Bench());
+let buf = new ArrayBuffer(32768);
+
+let bench = withCodSpeed(new Bench({ name: "ops" }));
 
 bench
-  .add("simple tree (root + text)", () => {
-    let buf = makeBuf(4096);
-    pack(simpleOps, buf, 0);
-  })
-  .add("complex layout (header + sidebar + main + footer)", () => {
-    let buf = makeBuf(8192);
-    pack(complexOps, buf, 0);
-  })
-  .add("large list (50 items)", () => {
-    let buf = makeBuf(32768);
-    pack(listOps, buf, 0);
-  });
+  .add(
+    "pack complex layout",
+    sync(() => {
+      for (let i = 0; i < 1500; i++) pack(complexOps, buf, 0);
+    }),
+  )
+  .add(
+    "pack large list",
+    sync(() => {
+      for (let i = 0; i < 250; i++) pack(listOps, buf, 0);
+    }),
+  );
 
 await bench.run();
 console.table(bench.table());

bench/render.bench.ts

@@ -1,44 +1,10 @@
 import { Bench } from "tinybench";
-import { withCodSpeed } from "@codspeed/tinybench-plugin";
+import { sync, withCodSpeed } from "./fixtures/utils.ts";
 import { createTerm } from "../term.ts";
 import { close, fixed, grow, open, rgba, text } from "../ops.ts";
 import type { Op } from "../ops.ts";
 
 let term = await createTerm({ width: 80, height: 24 });
-let termPtr = await createTerm({ width: 80, height: 24 });
-
-let helloOps: Op[] = [
-  open("root", {
-    layout: { width: grow(), height: grow(), direction: "ttb" },
-  }),
-  text("Hello, World!"),
-  close(),
-];
-
-let borderedOps: Op[] = [
-  open("root", {
-    layout: { width: grow(), height: grow(), direction: "ttb" },
-  }),
-  open("box", {
-    layout: {
-      width: grow(),
-      height: grow(),
-      padding: { left: 1, right: 1, top: 1, bottom: 1 },
-      direction: "ttb",
-    },
-    border: {
-      color: rgba(0, 255, 0),
-      left: 1,
-      right: 1,
-      top: 1,
-      bottom: 1,
-    },
-    cornerRadius: { tl: 1, tr: 1, bl: 1, br: 1 },
-  }),
-  text("Bordered content"),
-  close(),
-  close(),
-];
 
 let dashboardOps: Op[] = [
   open("root", {
@@ -134,25 +100,25 @@ let uiOps: Op[] = [
   close(),
 ];
 
-let bench = withCodSpeed(new Bench());
+let bench = withCodSpeed(new Bench({ name: "render" }));
 
 bench
-  .add("simple text", () => {
-    term.render(helloOps);
-  })
-  .add("bordered box with corner radius", () => {
-    term.render(borderedOps);
-  })
-  .add("dashboard layout", () => {
-    term.render(dashboardOps);
-  })
-  .add("diff render (second frame)", () => {
-    term.render(dashboardOps);
-    term.render(dashboardOps);
-  })
-  .add("render with pointer hit testing", () => {
-    termPtr.render(uiOps, { pointer: { x: 10, y: 1, down: false } });
-  });
+  .add(
+    "render mixed frames",
+    sync(() => {
+      for (let i = 0; i < 250; i++) {
+        term.render(i % 2 === 0 ? dashboardOps : uiOps);
+      }
+    }),
+  )
+  .add(
+    "render steady diff",
+    sync(() => {
+      for (let i = 0; i < 250; i++) {
+        term.render(dashboardOps);
+      }
+    }),
+  );
 
 await bench.run();
 console.table(bench.table());

bench/startup.bench.ts

@@ -1,6 +1,5 @@
 import { Bench } from "tinybench";
-import { withCodSpeed } from "@codspeed/tinybench-plugin";
-import { spawnFixture } from "./fixtures/utils.ts";
+import { spawnFixture, withCodSpeed } from "./fixtures/utils.ts";
 
 let bench = withCodSpeed(new Bench({ name: "startup" }));
 

bench/throughput.bench.ts

@@ -0,0 +1,49 @@
+import { Bench } from "tinybench";
+import { sync, withCodSpeed } from "./fixtures/utils.ts";
+import { createInput } from "../input.ts";
+
+function str(s: string): Uint8Array {
+  return new TextEncoder().encode(s);
+}
+
+function concat(parts: Uint8Array[]): Uint8Array {
+  let len = parts.reduce((n, p) => n + p.length, 0);
+  let out = new Uint8Array(len);
+  let off = 0;
+  for (let p of parts) {
+    out.set(p, off);
+    off += p.length;
+  }
+  return out;
+}
+
+let unit = concat([
+  str("the quick brown fox "),
+  new Uint8Array([0x1b, 0x5b, 0x41]), // ArrowUp
+  str("\x1b[<0;40;12M"), // SGR mouse press
+  new Uint8Array([0xe4, 0xb8, 0xad]), // 中
+  str("\x1b[97;3u"), // Kitty a+Alt
+  new Uint8Array([0xf0, 0x9f, 0x8e, 0x89]), // 🎉
+]);
+let corpus = concat(new Array(1000).fill(unit));
+
+let READ = 64;
+
+let input = await createInput({ escLatency: 25 });
+
+let bench = withCodSpeed(new Bench({ name: "throughput" }));
+
+bench.add(
+  "input throughput (mixed corpus, chunked read loop)",
+  sync(() => {
+    let dispatched = 0;
+    for (let off = 0; off < corpus.length; off += READ) {
+      let { events } = input.scan(corpus.subarray(off, off + READ));
+      dispatched += events.length;
+    }
+    if (dispatched === 0) throw new Error("expected events");
+  }),
+);
+
+await bench.run();
+console.table(bench.table());