Skip to content

Support scaled cameras (WebXR rig scaling) #403

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
marwie wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Support scaled cameras (WebXR rig scaling) #403

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 27 additions & 6 deletions src/SparkRenderer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,14 @@ import {
uploadU32DataTextureRows,
} from "./utils";

const renderToViewScaleTmp = new THREE.Vector3();

// Average (uniform) world scale of a camera
function getCameraWorldScale(camera: THREE.Camera): number {
const s = camera.getWorldScale(renderToViewScaleTmp);
return (s.x + s.y + s.z) / 3;
}

export interface SparkRendererOptions {
/**
* Pass in your THREE.WebGLRenderer instance so Spark can perform work
Expand Down Expand Up @@ -621,6 +629,8 @@ export class SparkRenderer extends THREE.Mesh {
renderToViewQuat: { value: new THREE.Quaternion() },
// SplatAccumulator to view transformation translation
renderToViewPos: { value: new THREE.Vector3() },
// SplatAccumulator to view transformation uniform scale (camera world scale)
renderToViewScale: { value: 1 },
renderToViewBasis: { value: new THREE.Matrix3() },
renderToViewOffset: { value: new THREE.Vector3() },
// Maximum distance (in stddevs) from Gsplat center to render
Expand Down Expand Up @@ -780,8 +790,13 @@ export class SparkRenderer extends THREE.Mesh {
accumToCamera.decompose(
this.uniforms.renderToViewPos.value,
this.uniforms.renderToViewQuat.value,
new THREE.Vector3(),
renderToViewScaleTmp,
);
this.uniforms.renderToViewScale.value =
(renderToViewScaleTmp.x +
renderToViewScaleTmp.y +
renderToViewScaleTmp.z) /
3;
this.uniforms.renderToViewBasis.value.setFromMatrix4(accumToCamera);

this.uniforms.maxStdDev.value = spark.maxStdDev;
Expand Down Expand Up @@ -829,8 +844,10 @@ export class SparkRenderer extends THREE.Mesh {

if (spark.autoUpdate && isNewFrame) {
const preUpdate = spark.preUpdate && !renderer.xr.isPresenting;
// Use the per-eye XR camera: getWorldPosition on the parentless ArrayCamera
// container falls back to the raw reference-space pose, losing any rig transform
const useCamera = renderer.xr.isPresenting
? renderer.xr.getCamera()
? (renderer.xr.getCamera().cameras[0] ?? renderer.xr.getCamera())
: camera;
if (preUpdate) {
spark.updateInternal({
Expand Down Expand Up @@ -913,9 +930,10 @@ export class SparkRenderer extends THREE.Mesh {
const center = camera.getWorldPosition(new THREE.Vector3());
const dir = camera.getWorldDirection(new THREE.Vector3());

// scale the world-units epsilon with the camera scale (~1mm of physical movement)
const viewChanged =
center.distanceTo(this.sortedCenter) > 0.001 ||
dir.dot(this.sortedDir) < 0.999;
center.distanceTo(this.sortedCenter) >
0.001 * getCameraWorldScale(camera) || dir.dot(this.sortedDir) < 0.999;

const next = this.accumulators.pop();
if (!next) {
Expand Down Expand Up @@ -1159,11 +1177,14 @@ export class SparkRenderer extends THREE.Mesh {
pixelScaleLimit = Math.min(pxX, pxY);
}

// NOTE: pixelScaleLimit needs no camera-scale term (perspective cancels it)
pixelScaleLimit *= this.lodRenderScale;

const viewPos = new THREE.Vector3();
const viewQuat = new THREE.Quaternion();
this.current.viewToWorld.decompose(viewPos, viewQuat, new THREE.Vector3());
const viewScale = new THREE.Vector3();
this.current.viewToWorld.decompose(viewPos, viewQuat, viewScale);
const viewCamScale = (viewScale.x + viewScale.y + viewScale.z) / 3;

if (this.lodPosOverride) {
viewPos.copy(this.lodPosOverride);
Expand All @@ -1181,7 +1202,7 @@ export class SparkRenderer extends THREE.Mesh {
}

const distance = viewPos.distanceTo(this.lastLod.pos);
const distanceRamp = Math.max(0.0, 1.0 - distance / 1.0);
const distanceRamp = Math.max(0.0, 1.0 - distance / viewCamScale);
const dot = viewQuat.dot(this.lastLod.quat);
const quatRamp = Math.max(0.0, 1.0 - (1.0 - dot) / 0.01);
const similarity = distanceRamp * quatRamp;
Expand Down
6 changes: 5 additions & 1 deletion src/shaders/splatVertex.glsl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ flat out float adjustedStdDev;
uniform vec2 renderSize;
uniform vec4 renderToViewQuat;
uniform vec3 renderToViewPos;
// Uniform scale of the render-to-view transform (1.0 unless the camera is scaled)
uniform float renderToViewScale;
uniform mat3 renderToViewBasis;
uniform float maxStdDev;
uniform float minPixelRadius;
Expand Down Expand Up @@ -120,8 +122,10 @@ void main() {
adjustedStdDev = maxStdDev + 0.7 * (rgba.a - 1.0);
}

// Apply the camera scale (the basis branch already carries the full matrix incl. scale)
scales *= renderToViewScale;
// Compute the view space center of the splat
vec3 viewCenter = (!enableCovSplats ? quatVec(renderToViewQuat, center) : (renderToViewBasis * center)) + renderToViewPos;
vec3 viewCenter = (!enableCovSplats ? (renderToViewScale * quatVec(renderToViewQuat, center)) : (renderToViewBasis * center)) + renderToViewPos;

// Discard splats behind the camera
if (viewCenter.z >= 0.0) {
Expand Down