Skip to content

Commit 050cfce

Browse files
committed
feat: fidelity eval harness (movit-eval) + fix inverted abduction
- packages/movit-eval: headless probe runs the real parser → timeline → FK → ground-lock pipeline without WebGL and snapshots world-space bones at each phase end - geometric metrics (torso pitch, knee flexion, spine curl, feet height) + generic invariants (parse clean, no clamps, above floor, contacts planted per support type) + per-movement biomech signatures - npm run eval scores spec/examples: 97/97; wired into vitest so CI gates - extract ground-lock solver out of the viewer (movit-render/groundlock.ts) so eval and render share one implementation - harness immediately caught abduct/adduct sign inversion (arms crossed through the torso on lateral raise) — flipped in joints.ts, verified in playground
1 parent f1182f3 commit 050cfce

16 files changed

Lines changed: 776 additions & 104 deletions

File tree

package-lock.json

Lines changed: 18 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@
1414
"test:watch": "vitest",
1515
"coverage": "vitest run --coverage",
1616
"dev": "npm run dev -w playground",
17+
"eval": "tsx packages/movit-eval/src/cli.ts",
1718
"build": "npm run build -w playground",
18-
"typecheck": "tsc --noEmit -p packages/movit-parser && tsc --noEmit -p packages/movit-share && tsc --noEmit -p packages/movit-render && tsc --noEmit -p packages/movit-language && tsc --noEmit -p packages/movit-lsp && tsc --noEmit -p packages/movit-mcp && tsc --noEmit -p playground && tsc --noEmit -p editors/vscode"
19+
"typecheck": "tsc --noEmit -p packages/movit-parser && tsc --noEmit -p packages/movit-share && tsc --noEmit -p packages/movit-render && tsc --noEmit -p packages/movit-eval && tsc --noEmit -p packages/movit-language && tsc --noEmit -p packages/movit-lsp && tsc --noEmit -p packages/movit-mcp && tsc --noEmit -p playground && tsc --noEmit -p editors/vscode"
1920
},
2021
"devDependencies": {
2122
"@vitest/coverage-v8": "^2.1.8",
23+
"tsx": "^4.19.2",
2224
"typescript": "^5.7.2",
2325
"vitest": "^2.1.8"
2426
}

packages/movit-eval/package.json

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"name": "movit-eval",
3+
"version": "0.1.0",
4+
"description": "Fidelity eval harness for Movit: headless kinematic probing + geometric invariant scoring.",
5+
"license": "MIT",
6+
"type": "module",
7+
"main": "./src/index.ts",
8+
"types": "./src/index.ts",
9+
"exports": {
10+
".": "./src/index.ts"
11+
},
12+
"dependencies": {
13+
"movit-parser": "0.1.0",
14+
"movit-render": "0.1.0",
15+
"three": "^0.171.0"
16+
},
17+
"devDependencies": {
18+
"@types/three": "^0.171.0"
19+
}
20+
}

packages/movit-eval/src/checks.ts

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
/**
2+
* Invariant checks — the "does it look like the movement it claims to be"
3+
* layer. Generic checks apply to every document; movement checks encode the
4+
* biomechanical signature of specific canonical movements (verified against
5+
* the playground render).
6+
*/
7+
8+
import type { PhasePose, ProbeResult } from "./probe.js";
9+
import {
10+
feetHeight,
11+
heightOf,
12+
kneeFlexionDeg,
13+
lowestPoint,
14+
segmentTiltDeg,
15+
spineCurlDeg,
16+
torsoPitchDeg,
17+
} from "./metrics.js";
18+
19+
export interface CheckOutcome {
20+
id: string;
21+
pass: boolean;
22+
/** Human-readable measurement, e.g. "torso pitch 68.2° (want ≥ 55)". */
23+
detail: string;
24+
}
25+
26+
export interface MovementChecks {
27+
/** Matches ProbeResult docs by fixture file name (without extension). */
28+
movement: string;
29+
checks: ((result: ProbeResult) => CheckOutcome)[];
30+
}
31+
32+
/** Find a phase by name; throws a failing outcome path if missing. */
33+
function phase(result: ProbeResult, name: string): PhasePose | null {
34+
return result.phases.find((p) => p.name === name) ?? null;
35+
}
36+
37+
/** Build a check on one named phase with a measured value and a predicate. */
38+
export function phaseCheck(
39+
id: string,
40+
phaseName: string,
41+
measure: (pose: PhasePose) => number,
42+
test: (value: number) => boolean,
43+
want: string,
44+
): (result: ProbeResult) => CheckOutcome {
45+
return (result) => {
46+
const p = phase(result, phaseName);
47+
if (!p) return { id, pass: false, detail: `phase "${phaseName}" not found` };
48+
const value = measure(p);
49+
return {
50+
id,
51+
pass: test(value),
52+
detail: `${value.toFixed(1)} (want ${want})`,
53+
};
54+
};
55+
}
56+
57+
/** Generic invariants every well-formed movement must satisfy. */
58+
export function genericChecks(result: ProbeResult): CheckOutcome[] {
59+
const out: CheckOutcome[] = [
60+
{
61+
id: "parses-clean",
62+
pass: result.ok && result.errors.length === 0,
63+
detail: result.errors.map((e) => e.message).join("; ") || "no errors",
64+
},
65+
{
66+
id: "no-clamp-warnings",
67+
pass: result.warnings.length === 0,
68+
detail:
69+
result.warnings
70+
.map((w) => `${w.joint} ${w.action} ${w.requested}${w.clamped}`)
71+
.join("; ") || "no warnings",
72+
},
73+
];
74+
for (const p of result.phases) {
75+
out.push({
76+
id: `above-floor:${p.name}`,
77+
pass: lowestPoint(p) > -0.05,
78+
detail: `lowest bone ${lowestPoint(p).toFixed(3)}m (want > -0.05)`,
79+
});
80+
const hands = p.groundLock.includes("hands");
81+
const feet = p.groundLock.includes("feet");
82+
if (feet && !hands) {
83+
// Standing support: the ankle joints sit on the floor.
84+
out.push({
85+
id: `feet-planted:${p.name}`,
86+
pass: Math.abs(feetHeight(p)) < 0.03,
87+
detail: `avg ankle height ${feetHeight(p).toFixed(3)}m (want |·| < 0.03)`,
88+
});
89+
} else if (hands && feet) {
90+
// Plank-style support pivots on the TOES (ankles ride above the floor);
91+
// the solver drives the hands to the ground, so assert that instead.
92+
const handY = (heightOf(p, "wrist_left") + heightOf(p, "wrist_right")) / 2;
93+
out.push({
94+
id: `hands-planted:${p.name}`,
95+
pass: Math.abs(handY) < 0.03,
96+
detail: `avg wrist height ${handY.toFixed(3)}m (want |·| < 0.03)`,
97+
});
98+
}
99+
}
100+
return out;
101+
}
102+
103+
/**
104+
* Movement-specific signatures. Thresholds were set from the verified
105+
* playground renders with margin; a regression that flattens a deadlift into
106+
* a leg-swing (the pre-hinge bug) fails these loudly.
107+
*/
108+
export const MOVEMENT_CHECKS: MovementChecks[] = [
109+
{
110+
movement: "deadlift",
111+
checks: [
112+
phaseCheck("torso-hinged", "Hinge down", torsoPitchDeg, (v) => v >= 55, "≥ 55° pitch"),
113+
phaseCheck(
114+
"soft-knees-only",
115+
"Hinge down",
116+
(p) => kneeFlexionDeg(p, "left"),
117+
(v) => v < 35,
118+
"< 35° knee flexion",
119+
),
120+
phaseCheck(
121+
"legs-vertical",
122+
"Hinge down",
123+
(p) => segmentTiltDeg(p, "knee_left", "hip_left"),
124+
(v) => v < 15,
125+
"< 15° leg tilt",
126+
),
127+
phaseCheck("flat-back", "Hinge down", spineCurlDeg, (v) => v < 15, "< 15° spine curl"),
128+
phaseCheck("stands-back-up", "Stand tall", torsoPitchDeg, (v) => v < 10, "< 10° pitch"),
129+
],
130+
},
131+
{
132+
movement: "forward-fold",
133+
checks: [
134+
phaseCheck("deep-hinge", "Fold", torsoPitchDeg, (v) => v >= 75, "≥ 75° pitch"),
135+
phaseCheck(
136+
"head-drops",
137+
"Fold",
138+
(p) => heightOf(p, "head"),
139+
(v) => v < 1.0,
140+
"head < 1.0m",
141+
),
142+
phaseCheck(
143+
"arms-hang",
144+
"Fold",
145+
(p) => heightOf(p, "wrist_left"),
146+
(v) => v < 0.7,
147+
"wrist < 0.7m",
148+
),
149+
],
150+
},
151+
{
152+
movement: "squat",
153+
checks: [
154+
phaseCheck(
155+
"pelvis-drops",
156+
"Descend",
157+
(p) => heightOf(p, "pelvis"),
158+
(v) => v < 0.75,
159+
"pelvis < 0.75m",
160+
),
161+
phaseCheck(
162+
"knees-bend-deep",
163+
"Descend",
164+
(p) => kneeFlexionDeg(p, "left"),
165+
(v) => v >= 80,
166+
"≥ 80° knee flexion",
167+
),
168+
phaseCheck(
169+
"stands-back-up",
170+
"Drive up",
171+
(p) => heightOf(p, "pelvis"),
172+
(v) => v > 0.9,
173+
"pelvis > 0.9m",
174+
),
175+
],
176+
},
177+
{
178+
movement: "roll-down",
179+
checks: [
180+
// The pre-fix rig curled the spine BACKWARD (-Z); forward is +Z.
181+
phaseCheck(
182+
"curls-forward",
183+
"Roll down",
184+
(p) => p.bones.get("head")![2],
185+
(v) => v > 0.1,
186+
"head z > 0.1m (forward)",
187+
),
188+
phaseCheck("spine-curls", "Roll down", spineCurlDeg, (v) => v >= 30, "≥ 30° spine curl"),
189+
],
190+
},
191+
{
192+
movement: "biceps-curl",
193+
checks: [
194+
phaseCheck(
195+
"forearm-raises",
196+
"Curl",
197+
(p) => heightOf(p, "wrist_left"),
198+
(v) => v > 1.0,
199+
"wrist > 1.0m",
200+
),
201+
],
202+
},
203+
{
204+
movement: "lateral-raise",
205+
checks: [
206+
phaseCheck(
207+
"arms-out-to-sides",
208+
"Raise",
209+
(p) => Math.abs(p.bones.get("wrist_left")![0]),
210+
(v) => v > 0.5,
211+
"wrist |x| > 0.5m",
212+
),
213+
],
214+
},
215+
];

packages/movit-eval/src/cli.ts

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
/**
2+
* Eval CLI: score the canonical spec/examples fixtures.
3+
* Run from the repo root: npm run eval
4+
* Exits non-zero if any check fails, so it can gate CI.
5+
*/
6+
7+
import { fileURLToPath } from "node:url";
8+
import { dirname, resolve } from "node:path";
9+
import { loadFixtures } from "./generator.js";
10+
import { renderReport, runEval } from "./report.js";
11+
12+
const here = dirname(fileURLToPath(import.meta.url));
13+
const examplesDir = resolve(here, "../../../spec/examples");
14+
15+
const report = runEval(loadFixtures(examplesDir));
16+
console.log(renderReport(report));
17+
18+
const { checksPassed, checksTotal } = report.summary;
19+
process.exit(checksPassed === checksTotal ? 0 : 1);
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/**
2+
* Movement sources for the harness.
3+
*
4+
* The fixture generator reads the canonical `spec/examples` — the regression
5+
* baseline. LLM generators implement the same interface so "ask model X for a
6+
* deadlift, score what it wrote" plugs straight into `runEval` (they need API
7+
* keys, so they live with the caller, not here).
8+
*/
9+
10+
import { readdirSync, readFileSync } from "node:fs";
11+
import { basename, join } from "node:path";
12+
import type { MovementSource } from "./report.js";
13+
14+
export interface MovementGenerator {
15+
/** Produce `.movit` source for the named movements. */
16+
generate(movements: readonly string[]): Promise<MovementSource[]>;
17+
}
18+
19+
/** Load every `.movit` fixture in a directory (movement id = file stem). */
20+
export function loadFixtures(dir: string): MovementSource[] {
21+
return readdirSync(dir)
22+
.filter((f) => f.endsWith(".movit"))
23+
.sort()
24+
.map((f) => ({
25+
movement: basename(f, ".movit"),
26+
source: readFileSync(join(dir, f), "utf8"),
27+
}));
28+
}

packages/movit-eval/src/index.ts

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/** movit-eval — public API. */
2+
3+
export { probeMovement } from "./probe.js";
4+
export type { ProbeResult, PhasePose, Vec3 } from "./probe.js";
5+
export {
6+
angleBetweenDeg,
7+
bone,
8+
feetHeight,
9+
heightOf,
10+
jointAngleDeg,
11+
kneeFlexionDeg,
12+
lowestPoint,
13+
segmentTiltDeg,
14+
spineCurlDeg,
15+
torsoPitchDeg,
16+
} from "./metrics.js";
17+
export { genericChecks, phaseCheck, MOVEMENT_CHECKS } from "./checks.js";
18+
export type { CheckOutcome, MovementChecks } from "./checks.js";
19+
export { runEval, renderReport } from "./report.js";
20+
export type { EvalReport, MovementReport, MovementSource } from "./report.js";
21+
export { loadFixtures } from "./generator.js";
22+
export type { MovementGenerator } from "./generator.js";

0 commit comments

Comments
 (0)