diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 964521ab13..aba2c06f21 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -314,3 +314,20 @@ Partitioner denoised ttfa TTFA +smaps +xcrun +simctl +greppable +denormal +denormals +formants +stdev +interquartile +untimed +colormapping +inlines +jetsam +utsname +phys +benchprobe +adb diff --git a/apps/benchmarks/.gitignore b/apps/benchmarks/.gitignore new file mode 100644 index 0000000000..257b2fe827 --- /dev/null +++ b/apps/benchmarks/.gitignore @@ -0,0 +1,3 @@ +# Raw output of individual runs. A run worth keeping gets copied into +# `baselines/` by hand, so that the committed numbers are ones somebody chose. +results/ diff --git a/apps/benchmarks/App.tsx b/apps/benchmarks/App.tsx new file mode 100644 index 0000000000..2555b28576 --- /dev/null +++ b/apps/benchmarks/App.tsx @@ -0,0 +1,128 @@ +/** + * The harness UI. + * + * Deliberately thin. The app's product is the JSON on stdout and at the + * collector; this screen exists so a human watching a 20-minute run on a desk + * can see which case is executing and spot a failure without tailing a log. + */ + +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import { ScrollView, StyleSheet, Text, TouchableOpacity, View } from 'react-native'; +import { SafeAreaProvider, SafeAreaView } from 'react-native-safe-area-context'; + +import { config } from './src/config'; +import { runSuite } from './src/runner'; +import type { CaseResult } from './src/report'; +import { selectCases } from './src/suite'; + +type Phase = { readonly caseId: string; readonly phase: string } | null; + +export default function App() { + const [running, setRunning] = useState(false); + const [done, setDone] = useState(false); + const [phase, setPhase] = useState(null); + const [results, setResults] = useState([]); + const [fatal, setFatal] = useState(null); + const started = useRef(false); + + const planned = selectCases(config.suite, config.only); + + const start = useCallback(async () => { + if (started.current) return; + started.current = true; + setRunning(true); + setFatal(null); + setResults([]); + setDone(false); + + try { + await runSuite({ + onPhase: (caseId, name) => setPhase({ caseId, phase: name }), + onCase: (result) => setResults((previous) => [...previous, result]), + }); + setDone(true); + } catch (error) { + setFatal(String(error)); + } finally { + setPhase(null); + setRunning(false); + started.current = false; + } + }, []); + + useEffect(() => { + if (config.autostart) start(); + }, [start]); + + return ( + + + ExecuTorch benchmarks + + {config.label} · {config.only.length > 0 ? 'custom' : config.suite} · {config.iterations}{' '} + iterations · {planned.length} cases + + sink: {config.sink ?? 'console only'} + + + {running ? 'Running…' : 'Run suite'} + + + {phase && ( + + {phase.caseId} — {phase.phase} + + )} + {done && Run complete.} + {fatal && {fatal}} + + + {planned.map((benchCase) => { + const result = results.find((entry) => entry.id === benchCase.id); + return ( + + {benchCase.id} + {!result && pending} + {result?.status === 'error' && {result.error}} + {result?.status === 'ok' && ( + + pipeline {result.pipeline?.median ?? 0} ms · load {result.taskLoadMs} ms + {result.memory ? ` · peak ${result.memory.peakMb} MB` : ''} + + )} + + ); + })} + + + + ); +} + +const styles = StyleSheet.create({ + container: { flex: 1, backgroundColor: '#fff', paddingHorizontal: 16 }, + title: { fontSize: 20, fontWeight: '700', marginTop: 12 }, + meta: { fontSize: 12, color: '#666', marginTop: 4 }, + button: { + marginTop: 16, + backgroundColor: '#001a72', + borderRadius: 10, + paddingVertical: 12, + alignItems: 'center', + }, + buttonDisabled: { backgroundColor: '#9aa0b4' }, + buttonText: { color: '#fff', fontWeight: '600' }, + phase: { marginTop: 12, fontSize: 13, color: '#001a72' }, + done: { marginTop: 12, fontSize: 13, fontWeight: '600', color: '#2b8a3e' }, + error: { marginTop: 4, fontSize: 12, color: '#c92a2a' }, + list: { flex: 1, marginTop: 16 }, + listContent: { paddingBottom: 24 }, + row: { paddingVertical: 8, borderBottomWidth: 1, borderBottomColor: '#f1f3f5' }, + rowId: { fontSize: 13, fontWeight: '600' }, + rowPending: { fontSize: 12, color: '#adb5bd' }, + rowStats: { fontSize: 12, color: '#495057' }, +}); diff --git a/apps/benchmarks/README.md b/apps/benchmarks/README.md new file mode 100644 index 0000000000..702e1bbce0 --- /dev/null +++ b/apps/benchmarks/README.md @@ -0,0 +1,141 @@ +# Performance benchmarks + +An on-device harness that measures model load time, inference latency and peak +memory for the task pipelines, and a comparator that diffs two runs and fails on +regressions. + +Its first job is bracketing an ExecuTorch bump: run the suite on 1.3.1, bump, +run it again on the same device, and compare. + +## Why on-device + +The numbers that matter come from the backends an ExecuTorch bump actually +changes — CoreML on the Apple Neural Engine, XNNPACK on an ARM core, Vulkan on +an Android GPU. A host-side benchmark on a CI runner exercises none of them: it +links a separately built desktop ExecuTorch against x86 XNNPACK, on a shared +runner whose noise floor is wider than most regressions. So the harness runs on +a real device and is triggered by hand, rather than running on every pull +request and being ignored. + +## Running a suite + +```bash +# Android, quick tier, results tagged "et-1.3.1" +yarn bench --platform android --label et-1.3.1 + +# iOS, everything +yarn bench --platform ios --suite full --label et-1.3.1 + +# A single case, more iterations +yarn bench --platform android --only classification/efficientnet-v2-s-xnnpack-int8 --iterations 50 +``` + +`yarn bench` starts a collector on port 8099, sets the app's `EXPO_PUBLIC_BENCH_*` +variables, builds and launches the app, and writes +`results/