From c337964f9670c7976deaeb28bb5f0064563b29a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Sun, 16 Aug 2026 18:43:55 +0200 Subject: [PATCH 1/3] test(bench): add an on-device performance regression harness Adds apps/benchmarks, a headless Expo app that runs the task pipelines against deterministic synthetic inputs and reports load time, inference latency and peak memory as JSON, plus a driver that collects a run and a comparator that diffs two runs and fails on regressions. Built to bracket an ExecuTorch bump: run the suite on 1.3.1, bump, run it again on the same device, compare. Three things the design turns on: - A raw-execute pass isolates ExecuTorch from the pipeline. Task timings fold model.execute together with preprocessing and post-processing, which are TypeScript and unaffected by a bump. The pass loads the .pte on its own and sizes its tensors from model.schema, so it covers every method a program exports and needs no per-model wiring. - Memory is sampled in a pass of its own. Reading total PSS on Android walks /proc/self/smaps and costs milliseconds, which would otherwise land in the inference numbers. - Inputs are pure functions of their parameters. Post-processing cost is input-dependent, so a harness reading a photo off the device would move for reasons unrelated to the change under test. The comparator refuses to diff runs from different devices, and reports a metric whose workload changed as INCOMPARABLE rather than as a delta. Refs #1078 --- .cspell-wordlist.txt | 17 + apps/benchmarks/.gitignore | 3 + apps/benchmarks/App.tsx | 128 +++++++ apps/benchmarks/README.md | 132 ++++++++ apps/benchmarks/app.json | 45 +++ .../benchmarks/assets/icons/adaptive-icon.png | Bin 0 -> 17547 bytes apps/benchmarks/assets/icons/favicon.png | Bin 0 -> 1466 bytes apps/benchmarks/assets/icons/icon.png | Bin 0 -> 22380 bytes apps/benchmarks/assets/icons/splash.png | Bin 0 -> 47346 bytes apps/benchmarks/babel.config.js | 7 + apps/benchmarks/index.ts | 5 + apps/benchmarks/metro.config.js | 9 + .../modules/bench-probe/android/build.gradle | 25 ++ .../modules/benchprobe/BenchProbeModule.kt | 54 +++ .../bench-probe/expo-module.config.json | 9 + apps/benchmarks/modules/bench-probe/index.ts | 42 +++ .../bench-probe/ios/BenchProbe.podspec | 20 ++ .../bench-probe/ios/BenchProbeModule.swift | 64 ++++ apps/benchmarks/package.json | 50 +++ apps/benchmarks/scripts/compare.mjs | 231 +++++++++++++ apps/benchmarks/scripts/run-benchmarks.mjs | 201 +++++++++++ apps/benchmarks/src/config.ts | 70 ++++ apps/benchmarks/src/inputs.ts | 153 +++++++++ apps/benchmarks/src/memory.ts | 73 ++++ apps/benchmarks/src/nativeForward.ts | 317 ++++++++++++++++++ apps/benchmarks/src/report.ts | 112 +++++++ apps/benchmarks/src/runner.ts | 201 +++++++++++ apps/benchmarks/src/stats.ts | 73 ++++ apps/benchmarks/src/suite.ts | 306 +++++++++++++++++ apps/benchmarks/src/time.ts | 97 ++++++ apps/benchmarks/tsconfig.json | 19 ++ yarn.lock | 20 ++ 32 files changed, 2483 insertions(+) create mode 100644 apps/benchmarks/.gitignore create mode 100644 apps/benchmarks/App.tsx create mode 100644 apps/benchmarks/README.md create mode 100644 apps/benchmarks/app.json create mode 100644 apps/benchmarks/assets/icons/adaptive-icon.png create mode 100644 apps/benchmarks/assets/icons/favicon.png create mode 100644 apps/benchmarks/assets/icons/icon.png create mode 100644 apps/benchmarks/assets/icons/splash.png create mode 100644 apps/benchmarks/babel.config.js create mode 100644 apps/benchmarks/index.ts create mode 100644 apps/benchmarks/metro.config.js create mode 100644 apps/benchmarks/modules/bench-probe/android/build.gradle create mode 100644 apps/benchmarks/modules/bench-probe/android/src/main/java/expo/modules/benchprobe/BenchProbeModule.kt create mode 100644 apps/benchmarks/modules/bench-probe/expo-module.config.json create mode 100644 apps/benchmarks/modules/bench-probe/index.ts create mode 100644 apps/benchmarks/modules/bench-probe/ios/BenchProbe.podspec create mode 100644 apps/benchmarks/modules/bench-probe/ios/BenchProbeModule.swift create mode 100644 apps/benchmarks/package.json create mode 100644 apps/benchmarks/scripts/compare.mjs create mode 100644 apps/benchmarks/scripts/run-benchmarks.mjs create mode 100644 apps/benchmarks/src/config.ts create mode 100644 apps/benchmarks/src/inputs.ts create mode 100644 apps/benchmarks/src/memory.ts create mode 100644 apps/benchmarks/src/nativeForward.ts create mode 100644 apps/benchmarks/src/report.ts create mode 100644 apps/benchmarks/src/runner.ts create mode 100644 apps/benchmarks/src/stats.ts create mode 100644 apps/benchmarks/src/suite.ts create mode 100644 apps/benchmarks/src/time.ts create mode 100644 apps/benchmarks/tsconfig.json diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 964521ab13..aba2c06f21 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -314,3 +314,20 @@ Partitioner denoised ttfa TTFA +smaps +xcrun +simctl +greppable +denormal +denormals +formants +stdev +interquartile +untimed +colormapping +inlines +jetsam +utsname +phys +benchprobe +adb diff --git a/apps/benchmarks/.gitignore b/apps/benchmarks/.gitignore new file mode 100644 index 0000000000..257b2fe827 --- /dev/null +++ b/apps/benchmarks/.gitignore @@ -0,0 +1,3 @@ +# Raw output of individual runs. A run worth keeping gets copied into +# `baselines/` by hand, so that the committed numbers are ones somebody chose. +results/ diff --git a/apps/benchmarks/App.tsx b/apps/benchmarks/App.tsx new file mode 100644 index 0000000000..2555b28576 --- /dev/null +++ b/apps/benchmarks/App.tsx @@ -0,0 +1,128 @@ +/** + * The harness UI. + * + * Deliberately thin. The app's product is the JSON on stdout and at the + * collector; this screen exists so a human watching a 20-minute run on a desk + * can see which case is executing and spot a failure without tailing a log. + */ + +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import { ScrollView, StyleSheet, Text, TouchableOpacity, View } from 'react-native'; +import { SafeAreaProvider, SafeAreaView } from 'react-native-safe-area-context'; + +import { config } from './src/config'; +import { runSuite } from './src/runner'; +import type { CaseResult } from './src/report'; +import { selectCases } from './src/suite'; + +type Phase = { readonly caseId: string; readonly phase: string } | null; + +export default function App() { + const [running, setRunning] = useState(false); + const [done, setDone] = useState(false); + const [phase, setPhase] = useState(null); + const [results, setResults] = useState([]); + const [fatal, setFatal] = useState(null); + const started = useRef(false); + + const planned = selectCases(config.suite, config.only); + + const start = useCallback(async () => { + if (started.current) return; + started.current = true; + setRunning(true); + setFatal(null); + setResults([]); + setDone(false); + + try { + await runSuite({ + onPhase: (caseId, name) => setPhase({ caseId, phase: name }), + onCase: (result) => setResults((previous) => [...previous, result]), + }); + setDone(true); + } catch (error) { + setFatal(String(error)); + } finally { + setPhase(null); + setRunning(false); + started.current = false; + } + }, []); + + useEffect(() => { + if (config.autostart) start(); + }, [start]); + + return ( + + + ExecuTorch benchmarks + + {config.label} · {config.only.length > 0 ? 'custom' : config.suite} · {config.iterations}{' '} + iterations · {planned.length} cases + + sink: {config.sink ?? 'console only'} + + + {running ? 'Running…' : 'Run suite'} + + + {phase && ( + + {phase.caseId} — {phase.phase} + + )} + {done && Run complete.} + {fatal && {fatal}} + + + {planned.map((benchCase) => { + const result = results.find((entry) => entry.id === benchCase.id); + return ( + + {benchCase.id} + {!result && pending} + {result?.status === 'error' && {result.error}} + {result?.status === 'ok' && ( + + pipeline {result.pipeline?.median ?? 0} ms · load {result.taskLoadMs} ms + {result.memory ? ` · peak ${result.memory.peakMb} MB` : ''} + + )} + + ); + })} + + + + ); +} + +const styles = StyleSheet.create({ + container: { flex: 1, backgroundColor: '#fff', paddingHorizontal: 16 }, + title: { fontSize: 20, fontWeight: '700', marginTop: 12 }, + meta: { fontSize: 12, color: '#666', marginTop: 4 }, + button: { + marginTop: 16, + backgroundColor: '#001a72', + borderRadius: 10, + paddingVertical: 12, + alignItems: 'center', + }, + buttonDisabled: { backgroundColor: '#9aa0b4' }, + buttonText: { color: '#fff', fontWeight: '600' }, + phase: { marginTop: 12, fontSize: 13, color: '#001a72' }, + done: { marginTop: 12, fontSize: 13, fontWeight: '600', color: '#2b8a3e' }, + error: { marginTop: 4, fontSize: 12, color: '#c92a2a' }, + list: { flex: 1, marginTop: 16 }, + listContent: { paddingBottom: 24 }, + row: { paddingVertical: 8, borderBottomWidth: 1, borderBottomColor: '#f1f3f5' }, + rowId: { fontSize: 13, fontWeight: '600' }, + rowPending: { fontSize: 12, color: '#adb5bd' }, + rowStats: { fontSize: 12, color: '#495057' }, +}); diff --git a/apps/benchmarks/README.md b/apps/benchmarks/README.md new file mode 100644 index 0000000000..2cd0875975 --- /dev/null +++ b/apps/benchmarks/README.md @@ -0,0 +1,132 @@ +# Performance benchmarks + +An on-device harness that measures model load time, inference latency and peak +memory for the task pipelines, and a comparator that diffs two runs and fails on +regressions. + +Its first job is bracketing an ExecuTorch bump: run the suite on 1.3.1, bump, +run it again on the same device, and compare. + +## Why on-device + +The numbers that matter come from the backends an ExecuTorch bump actually +changes — CoreML on the Apple Neural Engine, XNNPACK on an ARM core, Vulkan on +an Android GPU. A host-side benchmark on a CI runner exercises none of them: it +links a separately built desktop ExecuTorch against x86 XNNPACK, on a shared +runner whose noise floor is wider than most regressions. So the harness runs on +a real device and is triggered by hand, rather than running on every pull +request and being ignored. + +## Running a suite + +```bash +# Android, quick tier, results tagged "et-1.3.1" +yarn bench --platform android --label et-1.3.1 + +# iOS, everything +yarn bench --platform ios --suite full --label et-1.3.1 + +# A single case, more iterations +yarn bench --platform android --only classification/efficientnet-v2-s-xnnpack-int8 --iterations 50 +``` + +`yarn bench` starts a collector on port 8099, sets the app's `EXPO_PUBLIC_BENCH_*` +variables, builds and launches the app, and writes +`results/