diff --git a/.claude/launch.json b/.claude/launch.json index 023d455..2c00d09 100644 --- a/.claude/launch.json +++ b/.claude/launch.json @@ -4,8 +4,8 @@ { "name": "playground", "runtimeExecutable": "npm", - "runtimeArgs": ["run", "dev", "-w", "playground"], - "port": 5173, + "runtimeArgs": ["run", "dev", "-w", "playground", "--", "--port", "5199", "--strictPort"], + "port": 5199, "autoPort": true } ] diff --git a/packages/posecode-language/src/vocab.ts b/packages/posecode-language/src/vocab.ts index 95d2d95..0d3e563 100644 --- a/packages/posecode-language/src/vocab.ts +++ b/packages/posecode-language/src/vocab.ts @@ -22,7 +22,7 @@ export const REACH_EFFECTORS = EFFECTOR_NAMES; export const PROPS = ["chair", "wall", "bar", "box", "dip-bars"]; /** Top-level directives (excluding the `posecode` header keyword). */ -export const TOP_KEYWORDS = ["rig", "prop", "pose", "step", "repeat"]; +export const TOP_KEYWORDS = ["rig", "prop", "pose", "clip", "step", "repeat"]; /** Keywords valid as step children. */ export const CHILD_KEYWORDS = ["ground-lock", "reach", "pin", "turn", "travel", "cue"]; @@ -34,6 +34,7 @@ export const KEYWORD_DOCS: Record = { prop: "Adds a scene object: `prop chair | wall | bar | box | dip-bars`. Supplies reach/pin anchors.", pose: "Sets the starting pose: `pose start = standing | neutral | plank | supine | prone | seated`.", start: "Used in `pose start = `.", + clip: 'Optional mocap clip: `clip "walk"`. A renderer with a matching retargeted animation plays it crossfaded over the procedural pose; others ignore it.', step: 'A movement phase: `step "" :`.', repeat: "How many times the movement loops.", "ground-lock": "Pins effectors (hands / feet) to the floor for this phase.", diff --git a/packages/posecode-parser/src/clamp.ts b/packages/posecode-parser/src/clamp.ts index d8f3e77..9e4327d 100644 --- a/packages/posecode-parser/src/clamp.ts +++ b/packages/posecode-parser/src/clamp.ts @@ -52,6 +52,7 @@ export function resolve(ast: AstDoc): ResolveResult { rig: ast.rig, ...(ast.startPose ? { startPose: ast.startPose } : {}), props: ast.props, + ...(ast.clip ? { clip: ast.clip } : {}), repeat: ast.repeat, phases, }; diff --git a/packages/posecode-parser/src/parser.ts b/packages/posecode-parser/src/parser.ts index 8bbe340..9ba8a76 100644 --- a/packages/posecode-parser/src/parser.ts +++ b/packages/posecode-parser/src/parser.ts @@ -50,6 +50,8 @@ export interface AstDoc { rig: string; startPose?: string; props: string[]; + /** Optional mocap clip name (`clip ""`), resolved to an asset by hosts. */ + clip?: string; repeat: number; steps: AstStep[]; } @@ -124,6 +126,13 @@ export function parseToAst(source: string): ParseAstResult { else doc.props.push(p); break; } + case "clip": { + // `clip ""`: an optional mocap clip the renderer may play + // (retargeted) instead of / blended with the procedural phases. + if (t[1]?.type === "str") doc.clip = t[1].value; + else errors.push({ line: ln.line, message: 'expected `clip ""`' }); + break; + } case "pose": { // `pose start = ` const name = t.length > 0 ? t[t.length - 1] : undefined; diff --git a/packages/posecode-parser/src/types.ts b/packages/posecode-parser/src/types.ts index 9a1a587..0539c9d 100644 --- a/packages/posecode-parser/src/types.ts +++ b/packages/posecode-parser/src/types.ts @@ -82,6 +82,14 @@ export interface PosecodeIR { startPose?: string; /** Scene props declared with `prop `, e.g. ["chair", "bar"]. */ props: string[]; + /** + * Optional mocap clip name declared with `clip ""`. A renderer MAY + * play a retargeted animation clip of this name (resolved by the host to an + * asset URL) instead of, or blended with, the procedural phase keyframes. + * Renderers without a matching clip ignore it: phases always fully describe + * the movement, so the procedural path remains the source of truth. + */ + clip?: string; repeat: number; phases: Phase[]; } diff --git a/packages/posecode-parser/test/parse.test.ts b/packages/posecode-parser/test/parse.test.ts index 9df9f92..31c6540 100644 --- a/packages/posecode-parser/test/parse.test.ts +++ b/packages/posecode-parser/test/parse.test.ts @@ -209,3 +209,36 @@ describe("reach/pin effectors", () => { expect(errors[0]!.message).toContain("tentacle"); }); }); + +describe("clip directive", () => { + const doc = (clipLine: string): string => + [ + 'posecode exercise "Walk"', + " rig humanoid", + " pose start = standing", + clipLine, + ' step "Step" 1s linear:', + " hips: flex 20", + " repeat 1", + ].join("\n"); + + it("parses a document-level clip name into the IR", () => { + const { ir, errors } = parse(doc(' clip "walk"')); + expect(errors).toEqual([]); + expect(ir!.clip).toBe("walk"); + }); + + it("omits clip from the IR when the directive is absent", () => { + const { ir, errors } = parse(doc("")); + expect(errors).toEqual([]); + expect(ir!.clip).toBeUndefined(); + }); + + it("rejects a clip directive without a quoted name", () => { + const { ir, errors } = parse(doc(" clip walk")); + expect(ir).toBeNull(); + expect(errors).toHaveLength(1); + expect(errors[0]!.line).toBe(4); + expect(errors[0]!.message).toContain("clip"); + }); +}); diff --git a/packages/posecode-render/src/character.ts b/packages/posecode-render/src/character.ts index e77cb5c..7635d8b 100644 --- a/packages/posecode-render/src/character.ts +++ b/packages/posecode-render/src/character.ts @@ -71,6 +71,13 @@ export interface Character { proportions: Proportions; /** Copy the driver's current pose onto the character skeleton. */ sync(driver: Mannequin): void; + /** + * The character's first skinned mesh, the retarget target for mocap clips + * (see clips.ts). Null on bare skeletons, which then can't play clips. + */ + skinnedMesh: THREE.SkinnedMesh | null; + /** Bones `sync` writes every frame; the mocap layer blends against these. */ + drivenNodes: ReadonlySet; /** Free GPU resources. */ dispose(): void; } @@ -372,10 +379,23 @@ export function rigCharacter(charScene: THREE.Object3D): Character { group.updateMatrixWorld(true); } + // Surface for the optional mocap-clip layer (clips.ts): the retarget target + // mesh and the set of bones sync() rewrites each frame. + let skinnedMesh: THREE.SkinnedMesh | null = null; + charScene.traverse((o) => { + if (!skinnedMesh && (o as THREE.SkinnedMesh).isSkinnedMesh) { + skinnedMesh = o as THREE.SkinnedMesh; + } + }); + const drivenNodes = new Set(mapped.map((m) => m.node)); + for (const ph of phalanges) drivenNodes.add(ph.node); + return { group, proportions, sync, + skinnedMesh, + drivenNodes, dispose() { group.traverse((o) => { const mesh = o as THREE.Mesh; diff --git a/packages/posecode-render/src/clips.ts b/packages/posecode-render/src/clips.ts new file mode 100644 index 0000000..731ff80 --- /dev/null +++ b/packages/posecode-render/src/clips.ts @@ -0,0 +1,256 @@ +/** + * Optional mocap-clip layer: play a retargeted animation clip (e.g. a Mixamo + * walk) on the skinned character instead of — or crossfaded with — the + * procedural DSL keyframes. + * + * Pipeline: `loadClipSource` fetches an FBX/GLB and picks its longest + * AnimationClip; `retargetMocapClip` bakes it onto the character's skeleton + * with SkeletonUtils.retargetClip (both rigs follow Mixamo naming, so bones + * pair up by suffix); `createClipLayer` plays the result through a + * THREE.AnimationMixer and blends it over whatever pose the procedural path + * already wrote this frame. The procedural path stays the source of truth: + * any missing asset, missing character, or retarget failure simply leaves the + * clip layer off. + */ + +import * as THREE from "three"; +import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"; +import { FBXLoader } from "three/examples/jsm/loaders/FBXLoader.js"; +import { + retargetClip, + type RetargetClipOptions, +} from "three/examples/jsm/utils/SkeletonUtils.js"; + +/** Strip the mixamo namespace, mirroring character.ts's bone-name matching. */ +function plainName(name: string): string { + return name.replace(/^mixamorig\d*:?/i, ""); +} + +export interface ClipSource { + /** The loaded asset's scene root (holds the source skeleton). */ + root: THREE.Object3D; + /** The longest animation found in the asset. */ + clip: THREE.AnimationClip; +} + +/** + * Load a mocap asset (.fbx or .glb/.gltf) and pick its longest clip. Rejects + * when the asset has no animations; callers treat any rejection as "keep the + * procedural path". + */ +export async function loadClipSource(url: string): Promise { + const isFbx = /\.fbx(\?.*)?$/i.test(url); + let root: THREE.Object3D; + let animations: THREE.AnimationClip[]; + if (isFbx) { + const group = await new FBXLoader().loadAsync(url); + root = group; + animations = group.animations; + } else { + const gltf = await new GLTFLoader().loadAsync(url); + root = gltf.scene; + animations = gltf.animations; + } + const clip = [...animations].sort((a, b) => b.duration - a.duration)[0]; + if (!clip) throw new Error(`clip asset has no animations: ${url}`); + return { root, clip }; +} + +/** + * Bake `clip` (animating the bones under `sourceRoot`) onto the target + * character's skeleton. Returns a new clip whose tracks bind as + * `.bones[]` on a mixer rooted at the target SkinnedMesh. + * + * Beyond the raw SkeletonUtils bake this: + * - maps bones by plain Mixamo name so namespace prefixes ("mixamorig:", + * "mixamorig1") never break the pairing; + * - snapshots and restores every target bone transform: retargetClip resets + * the skeleton to its BIND pose (the T-pose), which would silently destroy + * the anatomical rest calibration character.ts applied at load; + * - drops tracks for target bones with no source counterpart (retargetClip + * emits bind-pose constants for those, which would snap fingers or helper + * bones into the T-pose at full weight); + * - pins the hip X/Z translation to the rest stance while keeping the + * vertical bob, so a traveling source clip plays in place and composes with + * the DSL's own `travel`/`turn` root choreography. + */ +export function retargetMocapClip( + target: THREE.SkinnedMesh, + sourceRoot: THREE.Object3D, + clip: THREE.AnimationClip, +): THREE.AnimationClip { + sourceRoot.updateMatrixWorld(true); + const sourceBones: THREE.Bone[] = []; + sourceRoot.traverse((n) => { + if ((n as THREE.Bone).isBone) sourceBones.push(n as THREE.Bone); + }); + if (sourceBones.length === 0) throw new Error("clip source has no skeleton"); + const sourceByPlain = new Map(sourceBones.map((b) => [plainName(b.name), b.name])); + const sourceHips = sourceBones.find((b) => plainName(b.name) === "Hips"); + if (!sourceHips) throw new Error("clip source has no Hips bone"); + + const targetBones = target.skeleton.bones; + const targetHips = targetBones.find((b) => plainName(b.name) === "Hips"); + if (!targetHips) throw new Error("clip target has no Hips bone"); + const mappedNames = new Set( + targetBones + .filter((b) => sourceByPlain.has(plainName(b.name))) + .map((b) => b.name), + ); + + // Hip scale: both heights measured in each rig's own track units, so the + // baked hip translation lands in the target's local space. + target.updateMatrixWorld(true); + const targetSpace = target.matrixWorld.clone().invert(); + const targetHipY = targetHips + .getWorldPosition(new THREE.Vector3()) + .applyMatrix4(targetSpace).y; + const sourceHipY = sourceHips.getWorldPosition(new THREE.Vector3()).y; + const scale = + Math.abs(sourceHipY) > 1e-6 && Math.abs(targetHipY) > 1e-6 + ? targetHipY / sourceHipY + : 1; + + // retargetClip resets the target skeleton to bind pose and leaves it at the + // clip's last frame: preserve the calibrated rest across the bake. + const saved = targetBones.map((b) => ({ + bone: b, + pos: b.position.clone(), + quat: b.quaternion.clone(), + scale: b.scale.clone(), + })); + + let baked: THREE.AnimationClip; + try { + const options: RetargetClipOptions = { + // Unmatched target bones map to "" (matches nothing); their bind-pose + // filler tracks are dropped below. + getBoneName: (bone) => sourceByPlain.get(plainName(bone.name)) ?? "", + hip: sourceHips.name, + scale, + }; + baked = retargetClip(target, new THREE.Skeleton(sourceBones), clip, options); + } finally { + for (const s of saved) { + s.bone.position.copy(s.pos); + s.bone.quaternion.copy(s.quat); + s.bone.scale.copy(s.scale); + } + target.updateMatrixWorld(true); + } + + const tracks = baked.tracks.filter((t) => { + const m = /^\.bones\[(.+)\]\./.exec(t.name); + return m !== null && mappedNames.has(m[1]!); + }); + + // Play in place: pin hip X/Z to the rest stance, re-anchor the bob at the + // rest height so proportion differences never sink or float the figure. + const hipTrackName = `.bones[${targetHips.name}].position`; + const rest = targetHips.position; + for (const t of tracks) { + if (t.name !== hipTrackName) continue; + const v = t.values; + const y0 = v[1] ?? rest.y; + for (let i = 0; i < v.length / 3; i++) { + v[i * 3] = rest.x; + v[i * 3 + 1] = rest.y + (v[i * 3 + 1]! - y0); + v[i * 3 + 2] = rest.z; + } + } + + return new THREE.AnimationClip(clip.name, clip.duration, tracks); +} + +export interface ClipLayer { + /** + * Pose the clip-driven bones at `timeSec` (looped), blended over the pose + * the procedural path wrote this frame by `weight` (0 = untouched + * procedural, 1 = pure clip). The caller refreshes world matrices after. + */ + apply(timeSec: number, weight: number): void; + /** Release mixer bindings. The layer must not be applied afterwards. */ + dispose(): void; +} + +/** + * Wrap a retargeted clip in a mixer + blend layer. + * + * `syncDriven` are the bones the procedural sync writes every frame; their + * current pose is the blend partner. Clip bones OUTSIDE that set (spine + * in-betweens, toes…) have no per-frame procedural writer, so their blend + * partner is the calibrated rest captured here — and they are restored to it + * whenever the weight hits zero, so a faded-out clip can't leave a stale pose. + */ +export function createClipLayer( + target: THREE.SkinnedMesh, + clip: THREE.AnimationClip, + syncDriven: ReadonlySet, +): ClipLayer { + const mixer = new THREE.AnimationMixer(target); + mixer.clipAction(clip).play(); + + const byName = new Map(target.skeleton.bones.map((b) => [b.name, b])); + const drivenNames = new Set(); + for (const t of clip.tracks) { + const m = /^\.bones\[(.+)\]\./.exec(t.name); + if (m && byName.has(m[1]!)) drivenNames.add(m[1]!); + } + interface Driven { + bone: THREE.Object3D; + synced: boolean; + rest: { pos: THREE.Vector3; quat: THREE.Quaternion }; + snap: { pos: THREE.Vector3; quat: THREE.Quaternion }; + } + const driven: Driven[] = [...drivenNames].map((name) => { + const bone = byName.get(name)!; + return { + bone, + synced: syncDriven.has(bone), + rest: { pos: bone.position.clone(), quat: bone.quaternion.clone() }, + snap: { pos: new THREE.Vector3(), quat: new THREE.Quaternion() }, + }; + }); + + // True while unsynced bones may hold clip pose (needs a restore at w=0). + let dirty = false; + + return { + apply(timeSec: number, weight: number): void { + if (weight <= 0) { + if (dirty) { + for (const d of driven) { + if (d.synced) continue; + d.bone.position.copy(d.rest.pos); + d.bone.quaternion.copy(d.rest.quat); + } + dirty = false; + } + return; + } + // Blend partner: this frame's procedural pose for synced bones, the + // calibrated rest for the others (reset first — the mixer wrote clip + // pose into them last frame and nothing else ever rewrites them). + for (const d of driven) { + if (!d.synced) { + d.bone.position.copy(d.rest.pos); + d.bone.quaternion.copy(d.rest.quat); + } + d.snap.pos.copy(d.bone.position); + d.snap.quat.copy(d.bone.quaternion); + } + mixer.setTime(timeSec); + if (weight < 1) { + for (const d of driven) { + d.bone.position.lerp(d.snap.pos, 1 - weight); + d.bone.quaternion.slerp(d.snap.quat, 1 - weight); + } + } + dirty = true; + }, + dispose(): void { + mixer.stopAllAction(); + mixer.uncacheRoot(target); + }, + }; +} diff --git a/packages/posecode-render/src/index.ts b/packages/posecode-render/src/index.ts index 8fe07fb..6168f8e 100644 --- a/packages/posecode-render/src/index.ts +++ b/packages/posecode-render/src/index.ts @@ -19,6 +19,13 @@ import { buildTimeline, type BuiltTimeline, type PhaseSegment } from "./timeline import { solveCCD, type JointLimits } from "./ik.js"; import { buildProps, type PropScene } from "./props.js"; import { loadCharacter, type Character } from "./character.js"; +import { + loadClipSource, + retargetMocapClip, + createClipLayer, + type ClipLayer, + type ClipSource, +} from "./clips.js"; import { depenetrate } from "./depenetrate.js"; const DEG = Math.PI / 180; @@ -47,6 +54,8 @@ export interface Viewer { get time(): number; /** True once the skinned character (characterUrl) is loaded and visible. */ get characterActive(): boolean; + /** True while a retargeted mocap clip is driving (or fading over) the pose. */ + get clipActive(): boolean; getTimeline(): TimelineInfo | null; /** * Render the current time synchronously and return the frame as a PNG data @@ -72,6 +81,15 @@ export interface ViewerOptions { * skeleton, rebuilt to the character's exact proportions (see character.ts). */ characterUrl?: string; + /** + * Mocap clip library: clip name (as written in a document's `clip ""` + * directive) → FBX/GLB asset URL. When a loaded document names a clip found + * here and the skinned character is active, the viewer retargets the clip + * onto the character and crossfades it over the procedural pose. Documents + * naming clips absent from this map — and any load/retarget failure — play + * the procedural keyframes as always, so clips can never blank a movement. + */ + clips?: Record; } export function createViewer( @@ -162,6 +180,58 @@ export function createViewer( // bounding-box grounding), and the character mirrors it every frame. let character: Character | null = null; + // Mocap-clip layer (optional, character-only). When the loaded document + // names a clip present in opts.clips, the asset is fetched once, retargeted + // onto the character skeleton, and crossfaded over the procedural pose. The + // weight eases toward its target each frame, so switching documents (or a + // clip arriving mid-play) fades rather than pops; every failure path leaves + // the procedural keyframes driving the figure. + const CLIP_FADE_PER_SEC = 2.5; // full crossfade in ~0.4s + let clipLayer: ClipLayer | null = null; + let clipLayerName: string | null = null; + let clipWeight = 0; + let clipTargetWeight = 0; + let clipToken = 0; + const clipSources = new Map>(); + + /** (Re)aim the clip layer at the current document's `clip` request. */ + function requestClip(ir: PosecodeIR | null): void { + clipToken++; + const token = clipToken; + const name = ir?.clip; + const url = name ? opts.clips?.[name] : undefined; + if (!name || !url || !character?.skinnedMesh) { + clipTargetWeight = 0; + return; + } + if (clipLayerName === name && clipLayer) { + clipTargetWeight = 1; + return; + } + clipTargetWeight = 0; // fade out whatever plays while the new clip loads + let source = clipSources.get(url); + if (!source) { + source = loadClipSource(url); + clipSources.set(url, source); + } + source + .then((src) => { + const mesh = character?.skinnedMesh; + if (token !== clipToken || !mesh || !character) return; + const retargeted = retargetMocapClip(mesh, src.root, src.clip); + clipLayer?.dispose(); + clipLayer = createClipLayer(mesh, retargeted, character.drivenNodes); + clipLayerName = name; + clipWeight = 0; + clipTargetWeight = 1; + }) + .catch(() => { + // Missing/broken clip asset: the procedural keyframes keep playing. + // Deliberately silent, matching the characterUrl fallback. + clipSources.delete(url); + }); + } + // --- Life layer: breathing + blinking so the figure reads as alive even // when the movement is paused. Both are MESH-only effects. Breathing must // never rotate skeleton bones: an earlier version breathed via tiny @@ -481,6 +551,17 @@ export function createViewer( } // Mirror the fully-solved driver pose onto the skinned character. character?.sync(mannequin); + // Mocap layer: ease the crossfade weight, then blend the retargeted clip + // over the procedural pose sync() just wrote. Timeline time drives the + // mixer so pause/scrub/export stay deterministic. + if (character && clipLayer) { + const step = frameDt * CLIP_FADE_PER_SEC; + const gap = clipTargetWeight - clipWeight; + clipWeight += Math.sign(gap) * Math.min(Math.abs(gap), step); + clipLayer.apply(time, clipWeight); + if (clipWeight > 0) character.group.updateMatrixWorld(true); + } + frameDt = 0; if (easeCamera) { controls.target.lerp(desiredTarget, 0.07); camera.position.lerp(desiredPos, 0.07); @@ -493,9 +574,13 @@ export function createViewer( let raf = 0; let lastT = performance.now(); + // Wall-clock delta consumed by frame() for the clip crossfade; zeroed after + // each frame so captureFrame() renders without advancing the fade. + let frameDt = 0; function loopFn(now: number): void { const dt = Math.min(0.05, (now - lastT) / 1000); lastT = now; + frameDt = dt; if (playing && timeline) { time += dt * speed; if (time >= timeline.duration) { @@ -544,6 +629,7 @@ export function createViewer( captureGroundTargets(); baseRootPos.copy(mannequin.root.position); baseRootQuat.copy(mannequin.root.quaternion); + requestClip(ir); frameCamera(); }, play() { @@ -580,6 +666,9 @@ export function createViewer( get characterActive() { return character !== null; }, + get clipActive() { + return clipLayer !== null && (clipWeight > 0 || clipTargetWeight > 0); + }, getTimeline() { if (!timeline) return null; return { @@ -604,6 +693,7 @@ export function createViewer( dispose() { cancelAnimationFrame(raf); controls.dispose(); + clipLayer?.dispose(); character?.dispose(); renderer.dispose(); }, @@ -671,5 +761,12 @@ export { buildTimeline } from "./timeline.js"; export { solveCCD, type IkChain, type JointLimits } from "./ik.js"; export { buildProps, type PropScene } from "./props.js"; export { loadCharacter, rigCharacter, type Character } from "./character.js"; +export { + loadClipSource, + retargetMocapClip, + createClipLayer, + type ClipLayer, + type ClipSource, +} from "./clips.js"; export { depenetrate } from "./depenetrate.js"; export type { PhaseSegment } from "./timeline.js"; diff --git a/packages/posecode-render/test/clips.test.ts b/packages/posecode-render/test/clips.test.ts new file mode 100644 index 0000000..e99c666 --- /dev/null +++ b/packages/posecode-render/test/clips.test.ts @@ -0,0 +1,203 @@ +import { describe, it, expect } from "vitest"; +import * as THREE from "three"; +import { retargetMocapClip, createClipLayer } from "../src/clips.js"; + +const DEG = Math.PI / 180; + +/** + * A minimal Mixamo-convention T-pose rig: enough bones to exercise name + * mapping, hip travel stripping, and quaternion retargeting. World positions + * given; locals are diffs since every rest rotation is identity. + */ +function makeRig(extraBone?: string): { + scene: THREE.Group; + bones: Map; +} { + const world = new Map([ + ["Hips", [0, 1.0, 0]], + ["Spine", [0, 1.2, 0]], + ["LeftArm", [0.2, 1.4, 0]], + ["LeftForeArm", [0.45, 1.4, 0]], + ["LeftHand", [0.7, 1.4, 0]], + ["LeftUpLeg", [0.1, 0.95, 0]], + ["LeftLeg", [0.1, 0.5, 0]], + ["LeftFoot", [0.1, 0.1, 0]], + ]); + const parents = new Map([ + ["Spine", "Hips"], + ["LeftArm", "Spine"], + ["LeftForeArm", "LeftArm"], + ["LeftHand", "LeftForeArm"], + ["LeftUpLeg", "Hips"], + ["LeftLeg", "LeftUpLeg"], + ["LeftFoot", "LeftLeg"], + ]); + if (extraBone) { + world.set(extraBone, [0, 1.0, -0.3]); + parents.set(extraBone, "Hips"); + } + const scene = new THREE.Group(); + const bones = new Map(); + for (const [name, pos] of world) { + const b = new THREE.Bone(); + b.name = `mixamorig${name}`; + const parent = parents.get(name); + const base = parent ? world.get(parent)! : [0, 0, 0]; + b.position.set(pos[0] - base[0], pos[1] - base[1], pos[2] - base[2]); + (parent ? bones.get(parent)! : scene).add(b); + bones.set(name, b); + } + scene.updateMatrixWorld(true); + return { scene, bones }; +} + +/** Bind a bare SkinnedMesh to the rig so it can be a retarget target. */ +function makeSkinnedTarget(rig: ReturnType): THREE.SkinnedMesh { + const mesh = new THREE.SkinnedMesh( + new THREE.BufferGeometry(), + new THREE.MeshBasicMaterial(), + ); + rig.scene.add(mesh); + rig.scene.updateMatrixWorld(true); + mesh.bind(new THREE.Skeleton([...rig.bones.values()])); + return mesh; +} + +/** A 1s source clip: left arm raises 90° about Z, hips bob and travel. */ +function makeSourceClip(): THREE.AnimationClip { + const q0 = new THREE.Quaternion(); + const q1 = new THREE.Quaternion().setFromEuler(new THREE.Euler(0, 0, 90 * DEG)); + return new THREE.AnimationClip("walk", 1, [ + new THREE.QuaternionKeyframeTrack( + "mixamorigLeftArm.quaternion", + [0, 1], + [q0.x, q0.y, q0.z, q0.w, q1.x, q1.y, q1.z, q1.w], + ), + new THREE.VectorKeyframeTrack( + "mixamorigHips.position", + [0, 1], + [0, 1.0, 0, 0.5, 1.1, 2.0], + ), + ]); +} + +function retargeted(extraTargetBone?: string): { + clip: THREE.AnimationClip; + target: THREE.SkinnedMesh; + rig: ReturnType; +} { + const source = makeRig(); + const rig = makeRig(extraTargetBone); + const target = makeSkinnedTarget(rig); + const clip = retargetMocapClip(target, source.scene, makeSourceClip()); + return { clip, target, rig }; +} + +function track(clip: THREE.AnimationClip, name: string): THREE.KeyframeTrack | undefined { + return clip.tracks.find((t) => t.name === name); +} + +describe("retargetMocapClip", () => { + it("emits mixer-ready .bones[] tracks for bones the source animates", () => { + const { clip } = retargeted(); + expect(track(clip, ".bones[mixamorigLeftArm].quaternion")).toBeDefined(); + expect(track(clip, ".bones[mixamorigHips].position")).toBeDefined(); + expect(clip.duration).toBeCloseTo(1, 2); + }); + + it("drops tracks for target bones with no source counterpart", () => { + const { clip } = retargeted("Tail"); + expect(track(clip, ".bones[mixamorigTail].quaternion")).toBeUndefined(); + expect(track(clip, ".bones[mixamorigTail].position")).toBeUndefined(); + }); + + it("preserves the target's calibrated bone locals (bake must not disturb the pose)", () => { + const source = makeRig(); + const rig = makeRig(); + const target = makeSkinnedTarget(rig); + // Simulate the character calibration: a non-bind rest on the arm. + const arm = rig.bones.get("LeftArm")!; + arm.quaternion.setFromEuler(new THREE.Euler(0, 0, -80 * DEG)); + rig.scene.updateMatrixWorld(true); + const before = new Map( + [...rig.bones.values()].map((b) => [ + b.name, + { pos: b.position.clone(), quat: b.quaternion.clone() }, + ]), + ); + retargetMocapClip(target, source.scene, makeSourceClip()); + for (const [name, rest] of before) { + const bone = [...rig.bones.values()].find((b) => b.name === name)!; + expect(bone.position.distanceTo(rest.pos)).toBeLessThan(1e-6); + expect(Math.abs(bone.quaternion.dot(rest.quat))).toBeGreaterThan(1 - 1e-6); + } + }); + + it("pins hip X/Z to the rest stance but keeps the vertical bob", () => { + const { clip, rig } = retargeted(); + const pos = track(clip, ".bones[mixamorigHips].position")!; + const rest = rig.bones.get("Hips")!.position; + const v = pos.values; + const frames = v.length / 3; + let minY = Infinity; + let maxY = -Infinity; + for (let i = 0; i < frames; i++) { + expect(v[i * 3]).toBeCloseTo(rest.x, 5); + expect(v[i * 3 + 2]).toBeCloseTo(rest.z, 5); + minY = Math.min(minY, v[i * 3 + 1]!); + maxY = Math.max(maxY, v[i * 3 + 1]!); + } + // Source bobs 0.1 up over the clip; rigs are congruent so the delta carries. + expect(maxY - minY).toBeCloseTo(0.1, 2); + expect(minY).toBeCloseTo(rest.y, 5); + }); +}); + +describe("createClipLayer", () => { + function layerSetup(): { + rig: ReturnType; + layer: ReturnType; + } { + const { clip, target, rig } = retargeted(); + // Pretend the procedural sync drives only the arm; hips are clip-only. + const syncDriven = new Set([rig.bones.get("LeftArm")!]); + const layer = createClipLayer(target, clip, syncDriven); + return { rig, layer }; + } + + it("poses clip-driven bones exactly at full weight", () => { + const { rig, layer } = layerSetup(); + layer.apply(0.5, 1); + const arm = rig.bones.get("LeftArm")!; + const angle = new THREE.Euler().setFromQuaternion(arm.quaternion, "XYZ").z / DEG; + expect(angle).toBeCloseTo(45, 0); + }); + + it("blends halfway between the current pose and the clip at weight 0.5", () => { + const { rig, layer } = layerSetup(); + layer.apply(0.5, 0.5); + const arm = rig.bones.get("LeftArm")!; + const angle = new THREE.Euler().setFromQuaternion(arm.quaternion, "XYZ").z / DEG; + expect(angle).toBeCloseTo(22.5, 0); + }); + + it("restores non-sync bones to rest when the weight reaches zero", () => { + const { rig, layer } = layerSetup(); + const hips = rig.bones.get("Hips")!; + const rest = hips.position.clone(); + // Mid-clip (t=1 would wrap to the bob-free first frame under LoopRepeat). + layer.apply(0.5, 1); + expect(hips.position.distanceTo(rest)).toBeGreaterThan(0.01); + layer.apply(0.5, 0); + expect(hips.position.distanceTo(rest)).toBeLessThan(1e-6); + }); + + it("keeps procedural bones untouched at zero weight", () => { + const { rig, layer } = layerSetup(); + const arm = rig.bones.get("LeftArm")!; + arm.quaternion.setFromEuler(new THREE.Euler(0, 0, 10 * DEG)); + layer.apply(0.5, 0); + const angle = new THREE.Euler().setFromQuaternion(arm.quaternion, "XYZ").z / DEG; + expect(angle).toBeCloseTo(10, 3); + }); +}); diff --git a/playground/public/clips/walk.fbx b/playground/public/clips/walk.fbx new file mode 120000 index 0000000..36ccada --- /dev/null +++ b/playground/public/clips/walk.fbx @@ -0,0 +1 @@ +../../../Ch36_nonPBR.fbx \ No newline at end of file diff --git a/playground/src/main.ts b/playground/src/main.ts index 1d9defe..27bcca8 100644 --- a/playground/src/main.ts +++ b/playground/src/main.ts @@ -517,6 +517,12 @@ void import("posecode-render").then(({ createViewer }) => { viewer = createViewer(canvas, { autoRotate: false, ...(classicFigure ? {} : { characterUrl: "/models/character.glb" }), + // Mocap-clip library: a document's `clip ""` directive picks a + // retargeted animation from here, crossfaded over the procedural pose (see + // clips.ts). Only fetched when a loaded movement names the clip, so this + // never slows the default page. Disabled with the classic figure, which has + // no skinned mesh to retarget onto. + ...(classicFigure ? {} : { clips: { walk: "/clips/walk.fbx" } }), }); // Exposed for capture/e2e tooling (frame capture drives README GIFs). (window as unknown as Record).__posecodeViewer = viewer; diff --git a/spec/SPEC.md b/spec/SPEC.md index 1c2f7af..06ac3b3 100644 --- a/spec/SPEC.md +++ b/spec/SPEC.md @@ -23,10 +23,11 @@ Posecode is line- and indentation-oriented. Comments start with `#` or `//`. document = header { directive } ; header = "posecode" kind STRING ; kind = "exercise" | "stretch" | "posture" ; (* free-form word *) -directive = rig | prop | pose | step | repeat ; +directive = rig | prop | pose | clip | step | repeat ; rig = "rig" WORD ; prop = "prop" WORD ; (* chair|wall|bar|box|dip-bars, repeatable *) pose = "pose" "start" "=" WORD ; (* neutral|standing|plank|supine|prone|seated *) +clip = "clip" STRING ; (* optional mocap clip; renderer may retarget & blend *) repeat = "repeat" NUMBER ; step = "step" STRING DURATION easing ":" { child } ; easing = "linear" | "ease-in" | "ease-out" | "ease-in-out" ; diff --git a/spec/examples/walk-cycle.posecode b/spec/examples/walk-cycle.posecode index 2a2fc6c..8519216 100644 --- a/spec/examples/walk-cycle.posecode +++ b/spec/examples/walk-cycle.posecode @@ -1,6 +1,7 @@ posecode exercise "Walk & turn" rig humanoid pose start = standing + clip "walk" step "Step right" 0.7s ease-in-out: hip_right: flex 30