From 9b784d2fd709d6a6930d424cc83060a7f0709686 Mon Sep 17 00:00:00 2001 From: a-baran-orhan Date: Mon, 13 Jul 2026 01:10:35 +0300 Subject: [PATCH] Ground Xbot across every movement --- packages/posecode-eval/src/checks.ts | 2 +- packages/posecode-eval/src/probe.ts | 7 +- packages/posecode-render/src/character.ts | 75 ++++++++++++++++++- packages/posecode-render/src/index.ts | 74 ++++++++++++++++-- .../posecode-render/test/xbot-asset.test.ts | 58 ++++++++++++++ playground/src/main.ts | 54 ++++++++++++- 6 files changed, 256 insertions(+), 14 deletions(-) diff --git a/packages/posecode-eval/src/checks.ts b/packages/posecode-eval/src/checks.ts index bd22e4c..da9eeab 100644 --- a/packages/posecode-eval/src/checks.ts +++ b/packages/posecode-eval/src/checks.ts @@ -94,7 +94,7 @@ export function genericChecks(result: ProbeResult): CheckOutcome[] { // must actually rest on the floor — not hover above it. Guards the // levitating-squat/deadlift regression where levelPlantedFeet lifted the // sole after ground-lock and an up-only clamp left the whole figure floating. - if (p.groundLock.length > 0) { + if (p.floorBound) { out.push({ id: `grounded-not-floating:${p.name}`, pass: p.meshMinY < 0.02, diff --git a/packages/posecode-eval/src/probe.ts b/packages/posecode-eval/src/probe.ts index c877fcb..d7e2ba0 100644 --- a/packages/posecode-eval/src/probe.ts +++ b/packages/posecode-eval/src/probe.ts @@ -41,6 +41,8 @@ export interface PhasePose { rootYaw: number; /** True when the phase relies on pins/reach-IK the probe cannot solve. */ usesSceneIk: boolean; + /** Whether the phase should rest on the floor (no elevated prop/grip support). */ + floorBound: boolean; /** * Height of the lowest visible-mesh point above the floor after the full * contact solve. ~0 for a grounded pose; a positive value means the figure @@ -175,8 +177,8 @@ export function probeMovement(source: string): ProbeResult { // airborne, so only rescue parts that dip below y=0. Mirror index.ts. m.root.updateMatrixWorld(true); const box = new THREE.Box3().setFromObject(m.root); - const planted = info.groundLock.length > 0; - if (box.min.y < 0 || (planted && box.min.y > 0)) { + const floorBound = info.grips.length === 0 && !info.pins.some((pin) => pin.anchor !== "floor"); + if (box.min.y < 0 || (floorBound && box.min.y > 0)) { m.root.position.y -= box.min.y; m.root.updateMatrixWorld(true); } @@ -191,6 +193,7 @@ export function probeMovement(source: string): ProbeResult { rootOffset: [info.rootOffset.x, 0, info.rootOffset.z], rootYaw: info.rootYaw, usesSceneIk: info.pins.length > 0 || info.reaches.length > 0 || info.grips.length > 0, + floorBound, meshMinY: Number.isFinite(finalBox.min.y) ? finalBox.min.y : 0, bones: snapshotBones(m.bones), boneQuaternions: snapshotBoneQuaternions(m.bones), diff --git a/packages/posecode-render/src/character.ts b/packages/posecode-render/src/character.ts index efe9c0f..6879683 100644 --- a/packages/posecode-render/src/character.ts +++ b/packages/posecode-render/src/character.ts @@ -73,6 +73,10 @@ export interface Character { sync(driver: Mannequin): void; /** Restore solved terminal contacts after a mocap layer has overwritten them. */ correctContacts(driver: Mannequin, boneIds: readonly string[]): void; + /** Precise CPU-skinned world bounds for diagnostics/export validation. */ + getBounds(): THREE.Box3; + /** Fast sampled visible-surface correction; returns the applied Y delta. */ + reconcileFloor(floorY?: number): number; /** * The character's first skinned mesh, the retarget target for mocap clips * (see clips.ts). Null on bare skeletons, which then can't play clips. @@ -414,19 +418,86 @@ export function rigCharacter(charScene: THREE.Object3D): Character { // Surface for the optional mocap-clip layer (clips.ts): the retarget target // mesh and the set of bones sync() rewrites each frame. let skinnedMesh: THREE.SkinnedMesh | null = null; + const skinnedMeshes: THREE.SkinnedMesh[] = []; charScene.traverse((o) => { - if (!skinnedMesh && (o as THREE.SkinnedMesh).isSkinnedMesh) { - skinnedMesh = o as THREE.SkinnedMesh; + if ((o as THREE.SkinnedMesh).isSkinnedMesh) { + const skin = o as THREE.SkinnedMesh; + skinnedMeshes.push(skin); + if (!skinnedMesh) skinnedMesh = skin; } }); const drivenNodes = new Set(mapped.map((m) => m.node)); for (const ph of phalanges) drivenNodes.add(ph.node); + function getBounds(): THREE.Box3 { + group.updateMatrixWorld(true); + const box = new THREE.Box3().makeEmpty(); + const vertex = new THREE.Vector3(); + let foundSkin = false; + for (const mesh of skinnedMeshes) { + foundSkin = true; + mesh.skeleton.update(); + const positions = mesh.geometry.getAttribute("position"); + for (let i = 0; i < positions.count; i++) { + vertex.fromBufferAttribute(positions, i); + mesh.applyBoneTransform(i, vertex).applyMatrix4(mesh.matrixWorld); + box.expandByPoint(vertex); + } + } + return foundSkin ? box : new THREE.Box3().setFromObject(group); + } + + // A dense uniform sample plus every rest-pose axis extremum. Xbot's 28k + // vertices reduce to ~3.6k skin transforms per frame while retaining sole, + // back, head, hand, and limb surface coverage under arbitrary articulation. + const floorSamples = skinnedMeshes.map((mesh) => { + const positions = mesh.geometry.getAttribute("position"); + const indices = new Set(); + for (let i = 0; i < positions.count; i += 8) indices.add(i); + indices.add(positions.count - 1); + for (const axis of ["x", "y", "z"] as const) { + let min = Infinity; + let max = -Infinity; + let minIndex = 0; + let maxIndex = 0; + for (let i = 0; i < positions.count; i++) { + const value = axis === "x" ? positions.getX(i) : axis === "y" ? positions.getY(i) : positions.getZ(i); + if (value < min) { min = value; minIndex = i; } + if (value > max) { max = value; maxIndex = i; } + } + indices.add(minIndex); + indices.add(maxIndex); + } + return { mesh, indices: [...indices] }; + }); + + function reconcileFloor(floorY = 0): number { + group.updateMatrixWorld(true); + const vertex = new THREE.Vector3(); + let minY = Infinity; + for (const { mesh, indices } of floorSamples) { + mesh.skeleton.update(); + const positions = mesh.geometry.getAttribute("position"); + for (const index of indices) { + vertex.fromBufferAttribute(positions, index); + mesh.applyBoneTransform(index, vertex).applyMatrix4(mesh.matrixWorld); + minY = Math.min(minY, vertex.y); + } + } + if (!Number.isFinite(minY)) return 0; + const delta = floorY - minY; + group.position.y += delta; + group.updateMatrixWorld(true); + return delta; + } + return { group, proportions, sync, correctContacts, + getBounds, + reconcileFloor, skinnedMesh, drivenNodes, dispose() { diff --git a/packages/posecode-render/src/index.ts b/packages/posecode-render/src/index.ts index 2b7df5a..1c462d3 100644 --- a/packages/posecode-render/src/index.ts +++ b/packages/posecode-render/src/index.ts @@ -58,6 +58,8 @@ export interface Viewer { /** True while a retargeted mocap clip is driving (or fading over) the pose. */ get clipActive(): boolean; getTimeline(): TimelineInfo | null; + /** Precise visible world bounds; intended for audits and deterministic export. */ + getVisibleBounds(): THREE.Box3; /** * Render the current time synchronously and return the frame as a PNG data * URL. Works without preserveDrawingBuffer because the read happens in the @@ -223,6 +225,10 @@ export function createViewer( const url = name ? opts.clips?.[name] : undefined; if (!name || !url || !character?.skinnedMesh) { clipTargetWeight = 0; + clipWeight = 0; + clipLayer?.dispose(); + clipLayer = null; + clipLayerName = null; return; } if (clipLayerName === name && clipLayer) { @@ -607,8 +613,10 @@ export function createViewer( } function frame(): void { + let solvedInfo: ReturnType["sample"]> | null = null; if (timeline) { const info = timeline.sample(time, mannequin.bones); + solvedInfo = info; // Life layer rides on wall-clock time (not timeline time) so the figure // keeps breathing and blinking while paused or scrubbing. applyLife(performance.now() / 1000); @@ -667,15 +675,18 @@ export function createViewer( // never recover it and the whole figure floated (squat, deadlift, // good-morning, forward-fold, plank, …). // - // A phase with NO ground-lock may be intentionally airborne (a prone - // "superman" lift, a jump), so it stays up-only: never yank a lifted body - // down, only rescue parts that dip below y=0. Pinned phases with a - // fixed-height anchor (a low chair seat) also rely on this up-only rescue - // as the legs fold. + // Explicit elevated support is the opt-out: bar grips and non-floor pins + // (box/chair) preserve their solved height. Everything else remains + // floor-bound; airborne choreography should use a future explicit flight + // contact rather than arise accidentally from missing `ground-lock`. mannequin.root.updateMatrixWorld(true); const box = new THREE.Box3().setFromObject(mannequin.root); - const planted = info.groundLock.length > 0; - if (box.min.y < 0 || (planted && box.min.y > 0)) { + // Unless an elevated prop/grip is carrying the body, the movement is + // floor-bound even when the author omitted `ground-lock`. This prevents + // ordinary curls, lunges, stretches, and transitions from inheriting a + // floating root when their FK pose raises the previous lowest point. + const floorBound = isFloorBound(info); + if (box.min.y < 0 || (floorBound && box.min.y > 0)) { mannequin.root.position.y -= box.min.y; mannequin.root.updateMatrixWorld(true); } @@ -694,8 +705,18 @@ export function createViewer( const gap = clipTargetWeight - clipWeight; clipWeight += Math.sign(gap) * Math.min(Math.abs(gap), step); clipLayer.apply(time, clipWeight); - if (clipWeight > 0) character.group.updateMatrixWorld(true); + if (clipWeight > 0) { + character.group.updateMatrixWorld(true); + // Mocap is layered after procedural grounding and can add hip/root bob + // that lifts a planted foot. Restore declared terminal contacts on the + // visible character without removing motion from unconstrained limbs. + if (solvedInfo) character.correctContacts(mannequin, contactBoneIds(solvedInfo)); + } } + // Final visible-surface grounding. The hidden driver uses calibrated proxy + // geometry; a segmented skin can have a different lowest point as limbs + // rotate. Reconcile the actual skinned surface after every animation layer. + if (character && solvedInfo && isFloorBound(solvedInfo)) character.reconcileFloor(); frameDt = 0; if (easeCamera) { controls.target.lerp(desiredTarget, 0.07); @@ -824,6 +845,9 @@ export function createViewer( segments: timeline.segments, }; }, + getVisibleBounds() { + return character?.getBounds() ?? new THREE.Box3().setFromObject(mannequin.root); + }, captureFrame() { frame(); return renderer.domElement.toDataURL("image/png"); @@ -882,6 +906,40 @@ export function createViewer( return api; } +/** True unless a grip or non-floor pin intentionally suspends/supports the body. */ +function isFloorBound(info: { + grips: readonly unknown[]; + pins: readonly { anchor: string }[]; +}): boolean { + return info.grips.length === 0 && !info.pins.some((pin) => pin.anchor !== "floor"); +} + +/** Driver terminal bones that must survive a mocap layer unchanged. */ +function contactBoneIds(info: { + groundLock: readonly string[]; + grips: readonly { effector: string }[]; + pins: readonly { effector: string }[]; + reaches: readonly { effector: string; target: string }[]; +}): string[] { + const ids = new Set(); + const addEffector = (effector: string): void => { + if (effector === "feet" || effector === "foot_left") ids.add("ankle_left"); + if (effector === "feet" || effector === "foot_right") ids.add("ankle_right"); + if (effector === "hands" || effector === "hand_left") ids.add("wrist_left"); + if (effector === "hands" || effector === "hand_right") ids.add("wrist_right"); + if (effector === "forearms") { + ids.add("elbow_left"); + ids.add("elbow_right"); + } + }; + for (const group of info.groundLock) addEffector(group); + for (const contact of [...info.grips, ...info.pins]) addEffector(contact.effector); + for (const reach of info.reaches) { + if (reach.target === "floor") addEffector(reach.effector); + } + return [...ids]; +} + /** * Show or hide a figure's meshes. The skeleton keeps driving animation and * bounding-box grounding regardless, so hiding only the meshes lets a hidden diff --git a/packages/posecode-render/test/xbot-asset.test.ts b/packages/posecode-render/test/xbot-asset.test.ts index e7d0b45..97cab17 100644 --- a/packages/posecode-render/test/xbot-asset.test.ts +++ b/packages/posecode-render/test/xbot-asset.test.ts @@ -2,6 +2,11 @@ import fs from "node:fs"; import { describe, expect, it } from "vitest"; import * as THREE from "three"; import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"; +import { rigCharacter } from "../src/character.js"; +import { buildMannequin } from "../src/mannequin.js"; +import { groundFigure } from "../src/groundlock.js"; +import { buildTimeline } from "../src/timeline.js"; +import { parse } from "posecode-parser"; const ASSET = new URL("../../../playground/public/models/xbot.glb", import.meta.url); @@ -38,4 +43,57 @@ describe("normalized Xbot runtime asset", () => { ); expect(vertices).toBeLessThan(30_000); }); + + it("matches the grounded driver's visible floor exactly", async () => { + const character = rigCharacter(await loadXbot()); + const driver = buildMannequin(undefined, character.proportions); + groundFigure(driver); + character.sync(driver); + + const driverMin = new THREE.Box3().setFromObject(driver.root).min.y; + const characterMin = character.getBounds().min.y; + expect(driverMin).toBeCloseTo(0, 4); + expect(characterMin).toBeCloseTo(driverMin, 3); + }); + + it("grounds Xbot's exact skinned surface in every floor-bound canonical phase", async () => { + const character = rigCharacter(await loadXbot()); + const examples = new URL("../../../spec/examples/", import.meta.url); + const files = fs.readdirSync(examples).filter((name) => name.endsWith(".posecode")); + const failures: string[] = []; + + for (const file of files) { + const { ir, errors } = parse(fs.readFileSync(new URL(file, examples), "utf8")); + expect(errors, file).toEqual([]); + if (!ir) continue; + const timeline = buildTimeline(ir); + const base = timeline.basePose.root; + for (let phaseIndex = 0; phaseIndex < timeline.segments.length; phaseIndex++) { + const authored = ir.phases[phaseIndex]!; + const floorBound = authored.grips.length === 0 && + !authored.pins.some((pin) => pin.anchor !== "floor"); + if (!floorBound) continue; + + const driver = buildMannequin(undefined, character.proportions); + driver.root.position.set(...(base?.position ?? [0, 0, 0])); + const [rx, ry, rz] = base?.rotationDeg ?? [0, 0, 0]; + driver.root.rotation.set( + THREE.MathUtils.degToRad(rx), + THREE.MathUtils.degToRad(ry), + THREE.MathUtils.degToRad(rz), + ); + const segment = timeline.segments[phaseIndex]!; + timeline.sample(segment.end - 1e-4, driver.bones); + groundFigure(driver); + character.sync(driver); + character.reconcileFloor(); + const minY = character.getBounds().min.y; + if (Math.abs(minY) >= 0.01) { + failures.push(`${file}:${segment.name} minY=${minY.toFixed(4)}`); + } + } + } + + expect(failures).toEqual([]); + }, 20_000); }); diff --git a/playground/src/main.ts b/playground/src/main.ts index 8490cbe..1ab5aba 100644 --- a/playground/src/main.ts +++ b/playground/src/main.ts @@ -548,6 +548,8 @@ void import("posecode-render").then(({ createViewer }) => { // procedural mannequin (debugging aid, and a fallback link for slow pages). const classicFigure = new URLSearchParams(location.search).get("figure") === "classic"; + const groundingAuditMode = + import.meta.env.DEV && new URLSearchParams(location.search).get("audit") === "grounding"; viewer = createViewer(canvas, { autoRotate: false, ...(classicFigure @@ -563,10 +565,60 @@ void import("posecode-render").then(({ createViewer }) => { // clips.ts). Only fetched when a loaded movement names the clip, so this // never slows the default page. Disabled with the classic figure, which has // no skinned mesh to retarget onto. - ...(classicFigure ? {} : { clips: SHOWCASE_CLIPS }), + ...(classicFigure || groundingAuditMode ? {} : { clips: SHOWCASE_CLIPS }), }); // Exposed for capture/e2e tooling (frame capture drives README GIFs). (window as unknown as Record).__posecodeViewer = viewer; + if (import.meta.env.DEV) { + // Full-library visual-grounding audit for local regression work. It uses + // the real viewer, Xbot skin, contact solver, and timeline rather than a + // parallel approximation. Elevated grip/prop phases are reported but do + // not fail the floor threshold. + const auditGrounding = () => { + const results: Array<{ + movement: string; + phase: string; + floorBound: boolean; + minY: number; + }> = []; + viewer!.pause(); + for (const preset of PRESETS) { + const parsed = parse(preset.source); + if (!parsed.ir) continue; + viewer!.load(parsed.ir); + const segments = viewer!.getTimeline()?.segments ?? []; + for (let i = 0; i < segments.length; i++) { + const segment = segments[i]!; + const phase = parsed.ir.phases[i]!; + viewer!.seek(Math.max(segment.start, segment.end - 1e-4)); + viewer!.captureFrame(); + const elevated = phase.grips.length > 0 || phase.pins.some((pin) => pin.anchor !== "floor"); + results.push({ + movement: preset.id, + phase: segment.name, + floorBound: !elevated, + minY: viewer!.getVisibleBounds().min.y, + }); + } + } + return results; + }; + (window as unknown as Record).__posecodeAuditGrounding = auditGrounding; + if (new URLSearchParams(location.search).get("audit") === "grounding") { + const publishAudit = (): void => { + if (!viewer!.characterActive) { + window.setTimeout(publishAudit, 100); + return; + } + const output = document.createElement("script"); + output.id = "grounding-audit"; + output.type = "application/json"; + output.textContent = JSON.stringify(auditGrounding()); + document.body.append(output); + }; + window.setTimeout(publishAudit, 0); + } + } wireViewer(viewer); recompile(); });