From f340305d259703d8d40bf9ab92be154e3fc2bf2c Mon Sep 17 00:00:00 2001 From: Marcel Wiessler Date: Tue, 28 Jul 2026 15:51:03 +0200 Subject: [PATCH] Support scaled cameras (WebXR rig scaling): render transform, sort/LOD thresholds, XR camera choice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gaussian splat scenes break under a uniformly scaled camera — the common WebXR pattern where an engine scales the user rig to let people resize a placed scene in AR/VR: 1. Rendering: the accum-to-view transform is decomposed into position + quaternion, discarding the scale. The translation keeps the scale factor but the rotated center term and the splat extents don't, so the scene warps instead of scaling. New renderToViewScale uniform (exact for uniform scale: T*R*S decompose => view = s*R*center + t); the covariance branch already carries the full basis and is unchanged. 2. Sort/LOD triggers: the view-changed epsilon (1mm) and the LOD update ramp (1m) are world-unit constants. At rig scale s, physical head movement maps to s-times less/more world movement — scaled-up scenes never re-sort ('frozen' blend order, foveation holes). Both now scale with the camera's world scale. The LOD pixel cut is deliberately NOT scaled: perspective projection cancels a uniform camera scale for size-at-distance; the apparent-size change comes entirely from the world-space distances the cut already measures. 3. XR update camera: updates ran on the ArrayCamera container, often in a setTimeout outside the render pass — where the parentless container's getWorldPosition recomputes matrixWorld from the raw reference-space local matrix, discarding the rig transform entirely. Use the per-eye camera instead (valid at any time when the engine keeps it parented; half an IPD off head center). Unscaled cameras: behavior is bit-identical (scale factor 1 everywhere). --- src/SparkRenderer.ts | 33 +++++++++++++++++++++++++++------ src/shaders/splatVertex.glsl | 6 +++++- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/SparkRenderer.ts b/src/SparkRenderer.ts index 6c72c595..5284e3a4 100644 --- a/src/SparkRenderer.ts +++ b/src/SparkRenderer.ts @@ -22,6 +22,14 @@ import { uploadU32DataTextureRows, } from "./utils"; +const renderToViewScaleTmp = new THREE.Vector3(); + +// Average (uniform) world scale of a camera +function getCameraWorldScale(camera: THREE.Camera): number { + const s = camera.getWorldScale(renderToViewScaleTmp); + return (s.x + s.y + s.z) / 3; +} + export interface SparkRendererOptions { /** * Pass in your THREE.WebGLRenderer instance so Spark can perform work @@ -621,6 +629,8 @@ export class SparkRenderer extends THREE.Mesh { renderToViewQuat: { value: new THREE.Quaternion() }, // SplatAccumulator to view transformation translation renderToViewPos: { value: new THREE.Vector3() }, + // SplatAccumulator to view transformation uniform scale (camera world scale) + renderToViewScale: { value: 1 }, renderToViewBasis: { value: new THREE.Matrix3() }, renderToViewOffset: { value: new THREE.Vector3() }, // Maximum distance (in stddevs) from Gsplat center to render @@ -780,8 +790,13 @@ export class SparkRenderer extends THREE.Mesh { accumToCamera.decompose( this.uniforms.renderToViewPos.value, this.uniforms.renderToViewQuat.value, - new THREE.Vector3(), + renderToViewScaleTmp, ); + this.uniforms.renderToViewScale.value = + (renderToViewScaleTmp.x + + renderToViewScaleTmp.y + + renderToViewScaleTmp.z) / + 3; this.uniforms.renderToViewBasis.value.setFromMatrix4(accumToCamera); this.uniforms.maxStdDev.value = spark.maxStdDev; @@ -829,8 +844,10 @@ export class SparkRenderer extends THREE.Mesh { if (spark.autoUpdate && isNewFrame) { const preUpdate = spark.preUpdate && !renderer.xr.isPresenting; + // Use the per-eye XR camera: getWorldPosition on the parentless ArrayCamera + // container falls back to the raw reference-space pose, losing any rig transform const useCamera = renderer.xr.isPresenting - ? renderer.xr.getCamera() + ? (renderer.xr.getCamera().cameras[0] ?? renderer.xr.getCamera()) : camera; if (preUpdate) { spark.updateInternal({ @@ -913,9 +930,10 @@ export class SparkRenderer extends THREE.Mesh { const center = camera.getWorldPosition(new THREE.Vector3()); const dir = camera.getWorldDirection(new THREE.Vector3()); + // scale the world-units epsilon with the camera scale (~1mm of physical movement) const viewChanged = - center.distanceTo(this.sortedCenter) > 0.001 || - dir.dot(this.sortedDir) < 0.999; + center.distanceTo(this.sortedCenter) > + 0.001 * getCameraWorldScale(camera) || dir.dot(this.sortedDir) < 0.999; const next = this.accumulators.pop(); if (!next) { @@ -1159,11 +1177,14 @@ export class SparkRenderer extends THREE.Mesh { pixelScaleLimit = Math.min(pxX, pxY); } + // NOTE: pixelScaleLimit needs no camera-scale term (perspective cancels it) pixelScaleLimit *= this.lodRenderScale; const viewPos = new THREE.Vector3(); const viewQuat = new THREE.Quaternion(); - this.current.viewToWorld.decompose(viewPos, viewQuat, new THREE.Vector3()); + const viewScale = new THREE.Vector3(); + this.current.viewToWorld.decompose(viewPos, viewQuat, viewScale); + const viewCamScale = (viewScale.x + viewScale.y + viewScale.z) / 3; if (this.lodPosOverride) { viewPos.copy(this.lodPosOverride); @@ -1181,7 +1202,7 @@ export class SparkRenderer extends THREE.Mesh { } const distance = viewPos.distanceTo(this.lastLod.pos); - const distanceRamp = Math.max(0.0, 1.0 - distance / 1.0); + const distanceRamp = Math.max(0.0, 1.0 - distance / viewCamScale); const dot = viewQuat.dot(this.lastLod.quat); const quatRamp = Math.max(0.0, 1.0 - (1.0 - dot) / 0.01); const similarity = distanceRamp * quatRamp; diff --git a/src/shaders/splatVertex.glsl b/src/shaders/splatVertex.glsl index ffbf93a0..32b1c605 100644 --- a/src/shaders/splatVertex.glsl +++ b/src/shaders/splatVertex.glsl @@ -15,6 +15,8 @@ flat out float adjustedStdDev; uniform vec2 renderSize; uniform vec4 renderToViewQuat; uniform vec3 renderToViewPos; +// Uniform scale of the render-to-view transform (1.0 unless the camera is scaled) +uniform float renderToViewScale; uniform mat3 renderToViewBasis; uniform float maxStdDev; uniform float minPixelRadius; @@ -120,8 +122,10 @@ void main() { adjustedStdDev = maxStdDev + 0.7 * (rgba.a - 1.0); } + // Apply the camera scale (the basis branch already carries the full matrix incl. scale) + scales *= renderToViewScale; // Compute the view space center of the splat - vec3 viewCenter = (!enableCovSplats ? quatVec(renderToViewQuat, center) : (renderToViewBasis * center)) + renderToViewPos; + vec3 viewCenter = (!enableCovSplats ? (renderToViewScale * quatVec(renderToViewQuat, center)) : (renderToViewBasis * center)) + renderToViewPos; // Discard splats behind the camera if (viewCenter.z >= 0.0) {