[RNE Rewrite] test: add an on-device performance regression harness

.cspell-wordlist.txt

@@ -314,3 +314,20 @@ Partitioner
 denoised
 ttfa
 TTFA
+smaps
+xcrun
+simctl
+greppable
+denormal
+denormals
+formants
+stdev
+interquartile
+untimed
+colormapping
+inlines
+jetsam
+utsname
+phys
+benchprobe
+adb

apps/benchmarks/.gitignore

@@ -0,0 +1,3 @@
+# Raw output of individual runs. A run worth keeping gets copied into
+# `baselines/` by hand, so that the committed numbers are ones somebody chose.
+results/

apps/benchmarks/App.tsx

@@ -0,0 +1,128 @@
+/**
+ * The harness UI.
+ *
+ * Deliberately thin. The app's product is the JSON on stdout and at the
+ * collector; this screen exists so a human watching a 20-minute run on a desk
+ * can see which case is executing and spot a failure without tailing a log.
+ */
+
+import React, { useCallback, useEffect, useRef, useState } from 'react';
+import { ScrollView, StyleSheet, Text, TouchableOpacity, View } from 'react-native';
+import { SafeAreaProvider, SafeAreaView } from 'react-native-safe-area-context';
+
+import { config } from './src/config';
+import { runSuite } from './src/runner';
+import type { CaseResult } from './src/report';
+import { selectCases } from './src/suite';
+
+type Phase = { readonly caseId: string; readonly phase: string } | null;
+
+export default function App() {
+  const [running, setRunning] = useState(false);
+  const [done, setDone] = useState(false);
+  const [phase, setPhase] = useState<Phase>(null);
+  const [results, setResults] = useState<CaseResult[]>([]);
+  const [fatal, setFatal] = useState<string | null>(null);
+  const started = useRef(false);
+
+  const planned = selectCases(config.suite, config.only);
+
+  const start = useCallback(async () => {
+    if (started.current) return;
+    started.current = true;
+    setRunning(true);
+    setFatal(null);
+    setResults([]);
+    setDone(false);
+
+    try {
+      await runSuite({
+        onPhase: (caseId, name) => setPhase({ caseId, phase: name }),
+        onCase: (result) => setResults((previous) => [...previous, result]),
+      });
+      setDone(true);
+    } catch (error) {
+      setFatal(String(error));
+    } finally {
+      setPhase(null);
+      setRunning(false);
+      started.current = false;
+    }
+  }, []);
+
+  useEffect(() => {
+    if (config.autostart) start();
+  }, [start]);
+
+  return (
+    <SafeAreaProvider>
+      <SafeAreaView style={styles.container}>
+        <Text style={styles.title}>ExecuTorch benchmarks</Text>
+        <Text style={styles.meta}>
+          {config.label} · {config.only.length > 0 ? 'custom' : config.suite} · {config.iterations}{' '}
+          iterations · {planned.length} cases
+        </Text>
+        <Text style={styles.meta}>sink: {config.sink ?? 'console only'}</Text>
+
+        <TouchableOpacity
+          style={[styles.button, running && styles.buttonDisabled]}
+          onPress={start}
+          disabled={running}
+        >
+          <Text style={styles.buttonText}>{running ? 'Running…' : 'Run suite'}</Text>
+        </TouchableOpacity>
+
+        {phase && (
+          <Text style={styles.phase}>
+            {phase.caseId} — {phase.phase}
+          </Text>
+        )}
+        {done && <Text style={styles.done}>Run complete.</Text>}
+        {fatal && <Text style={styles.error}>{fatal}</Text>}
+
+        <ScrollView style={styles.list} contentContainerStyle={styles.listContent}>
+          {planned.map((benchCase) => {
+            const result = results.find((entry) => entry.id === benchCase.id);
+            return (
+              <View key={benchCase.id} style={styles.row}>
+                <Text style={styles.rowId}>{benchCase.id}</Text>
+                {!result && <Text style={styles.rowPending}>pending</Text>}
+                {result?.status === 'error' && <Text style={styles.error}>{result.error}</Text>}
+                {result?.status === 'ok' && (
+                  <Text style={styles.rowStats}>
+                    pipeline {result.pipeline?.median ?? 0} ms · load {result.taskLoadMs} ms
+                    {result.memory ? ` · peak ${result.memory.peakMb} MB` : ''}
+                  </Text>
+                )}
+              </View>
+            );
+          })}
+        </ScrollView>
+      </SafeAreaView>
+    </SafeAreaProvider>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: { flex: 1, backgroundColor: '#fff', paddingHorizontal: 16 },
+  title: { fontSize: 20, fontWeight: '700', marginTop: 12 },
+  meta: { fontSize: 12, color: '#666', marginTop: 4 },
+  button: {
+    marginTop: 16,
+    backgroundColor: '#001a72',
+    borderRadius: 10,
+    paddingVertical: 12,
+    alignItems: 'center',
+  },
+  buttonDisabled: { backgroundColor: '#9aa0b4' },
+  buttonText: { color: '#fff', fontWeight: '600' },
+  phase: { marginTop: 12, fontSize: 13, color: '#001a72' },
+  done: { marginTop: 12, fontSize: 13, fontWeight: '600', color: '#2b8a3e' },
+  error: { marginTop: 4, fontSize: 12, color: '#c92a2a' },
+  list: { flex: 1, marginTop: 16 },
+  listContent: { paddingBottom: 24 },
+  row: { paddingVertical: 8, borderBottomWidth: 1, borderBottomColor: '#f1f3f5' },
+  rowId: { fontSize: 13, fontWeight: '600' },
+  rowPending: { fontSize: 12, color: '#adb5bd' },
+  rowStats: { fontSize: 12, color: '#495057' },
+});

apps/benchmarks/README.md

@@ -0,0 +1,141 @@
+# Performance benchmarks
+
+An on-device harness that measures model load time, inference latency and peak
+memory for the task pipelines, and a comparator that diffs two runs and fails on
+regressions.
+
+Its first job is bracketing an ExecuTorch bump: run the suite on 1.3.1, bump,
+run it again on the same device, and compare.
+
+## Why on-device
+
+The numbers that matter come from the backends an ExecuTorch bump actually
+changes — CoreML on the Apple Neural Engine, XNNPACK on an ARM core, Vulkan on
+an Android GPU. A host-side benchmark on a CI runner exercises none of them: it
+links a separately built desktop ExecuTorch against x86 XNNPACK, on a shared
+runner whose noise floor is wider than most regressions. So the harness runs on
+a real device and is triggered by hand, rather than running on every pull
+request and being ignored.
+
+## Running a suite
+
+```bash
+# Android, quick tier, results tagged "et-1.3.1"
+yarn bench --platform android --label et-1.3.1
+
+# iOS, everything
+yarn bench --platform ios --suite full --label et-1.3.1
+
+# A single case, more iterations
+yarn bench --platform android --only classification/efficientnet-v2-s-xnnpack-int8 --iterations 50
+```
+
+`yarn bench` starts a collector on port 8099, sets the app's `EXPO_PUBLIC_BENCH_*`
+variables, builds and launches the app, and writes
+`results/<label>-<platform>-<device>.json` when the run ends. On Android it sets
+up `adb reverse` for you; on an iOS device it binds the collector to the host's
+LAN address (override with `--host`).
+
+To drive the app yourself, pass `--no-launch` and the script prints the
+environment to start it with.
+
+Options: `--suite quick|full`, `--only <ids>`, `--iterations N`, `--warmup N`,
+`--no-memory`, `--no-native`, `--port N`, `--out <path>`.
+
+## Comparing two runs
+
+```bash
+yarn bench:compare results/et-1.3.1-ios-iPhone17,1.json results/et-1.4.1-ios-iPhone17,1.json
+```
+
+Prints a per-metric table and exits 1 if anything regressed past tolerance
+(inference 10%, load 15%, memory 10% — override with `--inference N`, `--load N`,
+`--memory N`).
+
+Two guards keep the output honest:
+
+- **Device mismatch is fatal.** Comparing an iPhone run against a Pixel run is
+  meaningless; pass `--allow-device-mismatch` if you know what you are doing.
+- **A metric whose workload changed is reported as `INCOMPARABLE`, not as a
+  delta.** If a pipeline decoded 14 tokens in one run and 19 in the other, it did
+  different work, and the ratio of the two timings measures nothing.
+
+A delta inside the run's own interquartile range is reported but not failed —
+the two runs cannot distinguish it from scheduling jitter.
+
+Copy a run you want to keep into `baselines/`; `results/` is gitignored.
+
+## What gets measured
+
+Per case:
+
+| Metric | What it covers |
+| --- | --- |
+| `load.native` | `loadModel` on the `.pte` alone |
+| `load.task` | The pipeline's `create` — load, schema validation, tensor pre-allocation |
+| `execute.<method>` | Raw `model.execute`, per exported method, no pipeline around it |
+| `pipeline.median` | The task's synchronous entry point end to end: preprocessing, execute, post-processing |
+| `memory.loaded` | Process footprint once the pipeline is ready |
+| `memory.peak` | Peak footprint during inference |
+| `memory.disposed` | Footprint after `dispose` — a leak shows up as a case that never returns to baseline |
+
+The raw-execute pass is the one to watch for an ExecuTorch bump. A pipeline
+timing folds `model.execute` together with preprocessing and post-processing,
+which are TypeScript and did not change; `execute.<method>` is ExecuTorch and
+nothing else. Its input and output tensors are derived from `model.schema`, so
+it works for any `.pte` in the registry, including every method a multi-method
+program exports. Methods whose schema cannot be pinned to concrete shapes are
+reported as skipped, with the reason.
+
+**`execute.<method>` and `pipeline.median` are not comparable to each other.**
+Where a model declares a dynamic dimension, the raw pass takes it at the top of
+its declared domain, so it measures the worst case the model can be asked for.
+The pipeline feeds whatever the input actually needs. On all-MiniLM-L6-v2 that
+is the difference between a 254-token forward and a 20-token one, and the raw
+number comes out several times the pipeline's. Each is comparable against itself
+across runs, which is all the comparator asks of them. The resolved shapes are
+recorded per method in the report, so it is always visible what was run.
+
+Memory is sampled in a separate pass from the timings. Reading total PSS on
+Android walks `/proc/self/smaps` and costs milliseconds; polling that during a
+15 ms inference would land in the numbers.
+
+## Determinism
+
+Every input is synthetic and is a pure function of its parameters, so two runs
+feed byte-identical data to the models. That matters because post-processing
+cost is input-dependent — an NMS pass over 200 candidate boxes is not the work
+of one over 3 — and a harness that picked a photo from the gallery would move
+for reasons unrelated to the change under test. See `src/inputs.ts`.
+
+The exception is speech-to-text. The synthetic waveform is voice-shaped but is
+not speech, so Whisper's decoder emits far fewer tokens than a real clip would
+and the pipeline figure is dominated by the encoder. Compare its
+`execute.<method>` numbers rather than its pipeline number.
+
+## Tiers
+
+`quick` (the default) is the small models — classification, selfie segmentation,
+style transfer, BlazeFace, MiniLM embeddings, FSMN VAD. Roughly 150 MB of
+downloads, a couple of minutes on device. `full` adds SSDLite, YOLO26, CLIP, the
+privacy filter, Whisper tiny and Supertonic TTS.
+
+## Adding a case
+
+Add an entry to `CASES` in `src/suite.ts`. The runner derives everything else —
+what to download, which methods to benchmark, what to sample — from the config
+and the model's schema. A case needs an id, its registry config, the pipeline's
+`create`, and a `run` worklet that returns the iteration's workload size.
+
+Pipelines with no synchronous entry point (Supertonic streams through four
+sub-models with JS-thread orchestration between chunks) set `mode: 'async'` and
+provide `runAsync`. Those timings include a thread hop per call, so they are
+comparable across runs but not against worklet-timed cases.
+
+## The native probe
+
+`modules/bench-probe` is a local Expo module reading the process footprint:
+`task_vm_info.phys_footprint` on iOS (what jetsam measures an app against) and
+total PSS on Android. Both count the resident pages of a memory-mapped `.pte`,
+which the native-heap counters miss entirely — and that is most of a model's
+footprint.

apps/benchmarks/app.json

@@ -0,0 +1,45 @@
+{
+  "expo": {
+    "name": "benchmarks",
+    "slug": "benchmarks",
+    "version": "1.0.0",
+    "orientation": "portrait",
+    "icon": "./assets/icons/icon.png",
+    "userInterfaceStyle": "light",
+    "newArchEnabled": true,
+    "scheme": "rne-benchmarks",
+    "splash": {
+      "image": "./assets/icons/splash.png",
+      "resizeMode": "contain",
+      "backgroundColor": "#ffffff"
+    },
+    "ios": {
+      "supportsTablet": true,
+      "bundleIdentifier": "com.anonymous.benchmarks"
+    },
+    "android": {
+      "adaptiveIcon": {
+        "foregroundImage": "./assets/icons/adaptive-icon.png",
+        "backgroundColor": "#ffffff"
+      },
+      "package": "com.anonymous.benchmarks",
+      "usesCleartextTraffic": true
+    },
+    "web": {
+      "favicon": "./assets/icons/favicon.png"
+    },
+    "plugins": [
+      [
+        "expo-build-properties",
+        {
+          "android": {
+            "minSdkVersion": 26
+          },
+          "ios": {
+            "deploymentTarget": "17.0"
+          }
+        }
+      ]
+    ]
+  }
+}

apps/benchmarks/babel.config.js

@@ -0,0 +1,7 @@
+module.exports = function (api) {
+  api.cache(true);
+  return {
+    presets: ['babel-preset-expo'],
+    plugins: ['react-native-worklets/plugin'],
+  };
+};

apps/benchmarks/index.ts

@@ -0,0 +1,5 @@
+import { registerRootComponent } from 'expo';
+
+import App from './App';
+
+registerRootComponent(App);

apps/benchmarks/metro.config.js

@@ -0,0 +1,9 @@
+// Learn more https://docs.expo.io/guides/customizing-metro
+const { getDefaultConfig } = require('expo/metro-config');
+
+/** @type {import('expo/metro-config').MetroConfig} */
+const config = getDefaultConfig(__dirname);
+
+config.resolver.assetExts.push('pte');
+
+module.exports = config;

apps/benchmarks/modules/bench-probe/android/build.gradle

@@ -0,0 +1,25 @@
+apply plugin: 'com.android.library'
+
+group = 'expo.modules.benchprobe'
+version = '1.0.0'
+
+def expoModulesCorePlugin = new File(
+  project(":expo-modules-core").projectDir.absolutePath,
+  "ExpoModulesCorePlugin.gradle"
+)
+apply from: expoModulesCorePlugin
+applyKotlinExpoModulesCorePlugin()
+useCoreDependencies()
+useDefaultAndroidSdkVersions()
+useExpoPublishing()
+
+android {
+  namespace "expo.modules.benchprobe"
+  defaultConfig {
+    versionCode 1
+    versionName "1.0.0"
+  }
+  lintOptions {
+    abortOnError false
+  }
+}