diff --git a/Sources/CShaderTypes/ShaderTypes.h b/Sources/CShaderTypes/ShaderTypes.h index 3411912a..40d13823 100644 --- a/Sources/CShaderTypes/ShaderTypes.h +++ b/Sources/CShaderTypes/ShaderTypes.h @@ -486,20 +486,28 @@ typedef struct{ float opacity; }GaussianSplat; +// position stays full float precision (fed directly into world/view/clip-space matrix math, +// where half-precision error would visibly drift); covariance and color/opacity are +// half-precision, matching the density another native Metal/simd Gaussian-splat renderer +// (MetalSplatter) uses for the same fields. simd_half3/simd_half4 pack tightly (8 bytes each, +// no padding between them) unlike simd_float3 (16-byte-aligned even for a 3-component vector), +// so this is 48 bytes total per splat vs. the previous 128 — not just "half the bytes" of the +// 3 vector fields, but a ~2.7x reduction overall once the float3 alignment padding that also +// disappears is accounted for. colorAndOpacity folds opacity into color's 4th component +// (again matching MetalSplatter) since a lone scalar field here would otherwise force the +// same kind of alignment padding this change is trying to eliminate. typedef struct{ simd_float3 position; - float opacity; - simd_float3 color; - float _pad0; - simd_float3 covA; - float _pad1; - simd_float3 covB; - float _pad2; + simd_half3 covA; + simd_half3 covB; + simd_half4 colorAndOpacity; // .xyz = SH0 base color, .w = opacity }EncodedGaussianSplat; -// Higher-order SH coefficients are stored in a separate packed half buffer. +// Higher-order SH coefficients are stored in a separate packed byte buffer, +// quantized to a fixed [-1, 1] range: byte = round(clamp(x,-1,1)*127)+128, +// dequantized on read as (byte-128)/128 — see loadGaussianSHCoefficient. // Layout per splat: R[1...n], G[1...n], B[1...n]. The DC coefficient remains -// represented by EncodedGaussianSplat.color. +// represented by EncodedGaussianSplat.colorAndOpacity.xyz. typedef struct{ uint degree; uint coefficientsPerChannel; @@ -507,17 +515,26 @@ typedef struct{ uint _pad0; }GaussianSHMetadata; -// Per-splat conic/radius/color, computed once per splat per frame by the gaussianPreprocess +// Per-splat conic/axes/color, computed once per splat per frame by the gaussianPreprocess // compute kernel instead of redundantly 4x per splat (once per instanced quad vertex) in the // draw vertex shader — see gaussianPreprocess in Gaussians.metal. Indexed by the same // original splat index as EncodedGaussianSplat. +// +// axis1/axis2 are the projected covariance ellipse's two (orthogonal) kGaussianQuadSigma-sigma +// semi-axis vectors in screen pixels — i.e. eigenvectors of the 2D covariance scaled by +// kGaussianQuadSigma*sqrt(eigenvalue) — used to build a tight, rotated quad instead of an +// axis-aligned bounding box. An axis-aligned box has +// to cover a rotated ellipse's full extent along screen X/Y, which for an anisotropic splat +// (the common case — Gaussians are oriented however the surface they came from sits) can be +// several times larger in area than the ellipse itself, costing that many more rasterized/ +// shaded fragments regardless of how cheap the per-fragment TBDR blend itself is. typedef struct{ simd_float3 conic; float _pad0; simd_float3 color; float _pad1; - simd_float2 radius; - simd_float2 _pad2; + simd_float2 axis1; + simd_float2 axis2; }GaussianPrecomputedSplat; typedef enum{ @@ -540,6 +557,7 @@ typedef enum{ gaussianTBDRRenderViewPortIndex, gaussianTBDRRenderReverseZIndex, gaussianTBDRRenderPrecomputedIndex, + gaussianTBDRRenderDebugColorIndex, }GaussianTBDRRenderBufferIndices; typedef enum{ diff --git a/Sources/Sandbox/GameScene.swift b/Sources/Sandbox/GameScene.swift index a4457fa2..bfc49e90 100644 --- a/Sources/Sandbox/GameScene.swift +++ b/Sources/Sandbox/GameScene.swift @@ -35,18 +35,6 @@ setSceneReady(success) } */ - /* - let sceneRoot = createEntity() - setEntityStreamScene( - entityId: sceneRoot, - url: URL(fileURLWithPath: "/path/to/local/json") - ) { success in - if success { - loadSceneAuthored(url: URL(fileURLWithPath: "/path/to/local/json")) - } - setSceneReady(success) - } - */ // Uncomment to render a streamed scene } @@ -57,6 +45,7 @@ InputSystem.shared.registerKeyboardEvents() InputSystem.shared.registerMouseEvents() bypassPostProcessing = false + setSpatialDebug(.lodLevels(false)) } private func setupDefaultSceneObjects() { diff --git a/Sources/UntoldEngine/ECS/Components.swift b/Sources/UntoldEngine/ECS/Components.swift index a35deaff..da93ba37 100644 --- a/Sources/UntoldEngine/ECS/Components.swift +++ b/Sources/UntoldEngine/ECS/Components.swift @@ -412,6 +412,23 @@ public enum LODResidencyState { case loading // Mesh is being loaded } +/// Shared by `LODLevel` and `GaussianLODLevel` so `isLODLevelResident`/`findFallbackLODLevel` +/// (`LODSystem.swift`) can implement residency/fallback selection once for both mesh and +/// Gaussian LOD instead of each component re-declaring the same algorithm. +protocol LODResidencyLevel { + var residencyState: LODResidencyState { get } + /// Whether this level actually has a usable payload — `residencyState` alone isn't + /// trusted, mirroring the double-check both components already made before this was shared. + var isPopulated: Bool { get } +} + +/// Shared by `LODLevel` and `GaussianLODLevel` so `selectLODIndex` (`LODSystem.swift`) can pick +/// a distance-based tier once for both mesh and Gaussian LOD instead of each system +/// re-declaring the same threshold/hysteresis algorithm. +protocol LODDistanceLevel { + var maxDistance: Float { get } +} + public struct LODLevel { public var mesh: [Mesh] // Meshes for this lod public var maxDistance: Float // Switch to next LOD beyond this @@ -431,6 +448,14 @@ public struct LODLevel { } } +extension LODLevel: LODResidencyLevel { + var isPopulated: Bool { + !mesh.isEmpty + } +} + +extension LODLevel: LODDistanceLevel {} + public class LODComponent: Component { public var lodLevels: [LODLevel] = [] // Sorted by distance (LOD0 first) public var currentLOD: Int = 0 // Active LOD index (what's actually rendering) @@ -451,26 +476,106 @@ public class LODComponent: Component { /// Check if the desired LOD level has a resident mesh public func isLODResident(_ lodIndex: Int) -> Bool { - guard lodIndex >= 0, lodIndex < lodLevels.count else { return false } - let level = lodLevels[lodIndex] - return level.residencyState == .resident && !level.mesh.isEmpty + isLODLevelResident(lodLevels, lodIndex) } /// Find the best available fallback LOD (coarser than desired) public func findFallbackLOD(from desiredIndex: Int) -> Int? { - // Try coarser LODs first (higher index = lower detail) - for i in (desiredIndex + 1) ..< lodLevels.count { - if isLODResident(i) { - return i - } - } - // Then try finer LODs (lower index = higher detail) - for i in (0 ..< desiredIndex).reversed() { - if isLODResident(i) { - return i - } + findFallbackLODLevel(lodLevels, from: desiredIndex) + } +} + +// MARK: - Progressive Gaussian LOD Component + +/// One pre-baked quality tier for a progressive Gaussian splat asset. +/// LOD0 is expected to be the full-resolution tier; later indices are progressively coarser. +public struct GaussianLODLevel { + public var buffers: GaussianComponent? + public var maxDistance: Float + public var url: URL? + public var residencyState: LODResidencyState = .unknown + var loadTask: Task? + /// Bake-time mean of this tier's kept splats' squared major-axis extent — see + /// `estimatedGaussianOverdraw`. Baked into the `.untoldgs` file itself + /// (`UntoldGSFormat`/`bakeGaussianSplatProgressiveTiers`) and populated automatically by + /// `loadGaussianLODLevel` once this tier's file is actually read. `nil` only before that — + /// i.e. this tier hasn't loaded yet — in which case `clampGaussianLODForOverdraw` falls + /// back to distance-only LOD selection for it. + public var meanSquaredSplatExtent: Float? + + public init(maxDistance: Float, url: URL? = nil) { + self.maxDistance = maxDistance + self.url = url + } +} + +extension GaussianLODLevel: LODResidencyLevel { + var isPopulated: Bool { + buffers != nil + } +} + +extension GaussianLODLevel: LODDistanceLevel {} + +/// Runtime state for a progressively streamed Gaussian splat prop. +/// The renderer still consumes a normal `GaussianComponent`; this component owns the +/// per-tier residency and `GaussianLODSystem` copies the selected tier onto the live +/// `GaussianComponent`. +public class GaussianLODComponent: Component { + public var lodLevels: [GaussianLODLevel] = [] + public var currentLOD: Int = -1 + public var desiredLOD: Int = 0 + public var forcedLOD: Int? + public var isUsingFallback: Bool = false + + /// The LOD index pure distance+hysteresis selection last landed on, before the + /// overdraw-aware clamp is applied — see `GaussianLODSystem.selectDesiredLOD`. Kept + /// separate from `desiredLOD` (the final, possibly overdraw-forced-coarser target used for + /// residency/streaming/`applyLOD`) so an overdraw-forced tier doesn't retroactively bias + /// the hysteresis anchor `selectLODIndex` uses next frame — otherwise a frame where overdraw + /// forces LOD 3 would make LOD 3 "the tier we're logically at" for the *next* frame's + /// distance-only hysteresis math too, even though distance alone would have picked LOD 1. + var distanceSelectedLOD: Int = 0 + + /// Distance to camera the last time this entity's LOD was fully re-evaluated — see + /// `GaussianLODSystem.update`'s per-entity fast path, which forces a refresh when this + /// entity (not just the camera) has moved enough to plausibly cross a LOD threshold, + /// independent of the camera-movement/frame-interval throttle. `nil` forces an evaluation + /// the first time this entity is seen. + var lastEvaluatedDistance: Float? + + /// `true` when the caller explicitly supplied a `boundingBoxHalfExtent` (always the case + /// for the streaming path, optional for the non-streaming path). When `false`, + /// `loadGaussianLODLevel` auto-populates `LocalTransformComponent.boundingBox` from the + /// first (coarsest) tier's real splat data once it loads, instead of leaving the entity on + /// its default placeholder box forever. + var hasExplicitBoundingBox: Bool = false + + /// Last state GaussianLODSystem's diagnostic log reported for this entity — used only to + /// dedup consecutive identical log lines (log on change, not every evaluation). + var lastLoggedDesiredLOD: Int = -2 + var lastLoggedActualLOD: Int = -2 + + public required init() {} + + public func isLODResident(_ lodIndex: Int) -> Bool { + isLODLevelResident(lodLevels, lodIndex) + } + + public func findFallbackLOD(from desiredIndex: Int) -> Int? { + findFallbackLODLevel(lodLevels, from: desiredIndex) + } + + /// Cancels in-flight loads and drops residency for every tier. Does not touch + /// currentLOD/desiredLOD — callers decide those based on whether this is a + /// stream-out (reset to coarsest) or full teardown (component removed right after). + func releaseAllLevelResources() { + for index in lodLevels.indices { + lodLevels[index].loadTask?.cancel() + lodLevels[index].loadTask = nil + lodLevels[index].buffers = nil + lodLevels[index].residencyState = .notResident } - return nil } } diff --git a/Sources/UntoldEngine/Renderer/RenderPasses.swift b/Sources/UntoldEngine/Renderer/RenderPasses.swift index aa17f747..626101fb 100644 --- a/Sources/UntoldEngine/Renderer/RenderPasses.swift +++ b/Sources/UntoldEngine/Renderer/RenderPasses.swift @@ -4540,6 +4540,19 @@ public enum RenderPasses { index: Int(gaussianTBDRRenderPrecomputedIndex.rawValue) ) + var gaussianLODDebugColor = simd_float4(0, 0, 0, 0) + if SpatialDebugVisualization.shared.colorRenderablesByLOD, + let gaussianLOD = scene.get(component: GaussianLODComponent.self, for: entityId) + { + let color = lodDebugColor(for: gaussianLOD.currentLOD) + gaussianLODDebugColor = simd_float4(color.x, color.y, color.z, 1.0) + } + renderEncoder.setVertexBytes( + &gaussianLODDebugColor, + length: MemoryLayout.stride, + index: Int(gaussianTBDRRenderDebugColorIndex.rawValue) + ) + renderEncoder.drawPrimitivesTracked(type: .triangleStrip, vertexStart: 0, vertexCount: 4, diff --git a/Sources/UntoldEngine/Renderer/UntoldEngine.swift b/Sources/UntoldEngine/Renderer/UntoldEngine.swift index 3a93d12e..6d810337 100644 --- a/Sources/UntoldEngine/Renderer/UntoldEngine.swift +++ b/Sources/UntoldEngine/Renderer/UntoldEngine.swift @@ -594,6 +594,7 @@ public class UntoldRenderer: NSObject, MTKViewDelegate { // 3. LOD selection (decides which representation is active, checks residency) LODSystem.shared.update(deltaTime: fixedStep) + GaussianLODSystem.shared.update(deltaTime: fixedStep) // 4. Flush events (residency and LOD change events are processed) SystemEventBus.shared.flushEvents() diff --git a/Sources/UntoldEngine/Shaders/Gaussians.metal b/Sources/UntoldEngine/Shaders/Gaussians.metal index da919d01..ea83e2eb 100644 --- a/Sources/UntoldEngine/Shaders/Gaussians.metal +++ b/Sources/UntoldEngine/Shaders/Gaussians.metal @@ -41,6 +41,16 @@ constant float GAUSSIAN_SH_C3[7] = { // cost per splat. constant float kGaussianMaxScreenRadius = 128.0f; +// How many standard deviations out the rendered quad extends along each principal axis. +// fragmentGaussianTBDRShader discards any fragment whose alpha falls below 1/255 (any dimmer +// than that rounds to nothing in 8-bit output anyway) — so the quad edge needs alpha = +// opacity*exp(-0.5*k^2) to already be under that bar for a fully-opaque splat, i.e. k > +// sqrt(2*ln(255)) ≈ 3.33, or the geometric edge itself becomes a faint but visible boundary +// (most noticeable on large, high-opacity, texturally-flat splats — e.g. sky/cloud splats — +// where there's nothing else nearby to mask a subtle discontinuity). 3.5 clears that with a +// small margin; going further trades quad area (~k^2) for diminishing returns. +constant float kGaussianQuadSigma = 3.5f; + // Hard ceiling on how many splats may blend into a single pixel. [[raster_order_group(0)]] // forces every fragment touching a given pixel to execute serially (a correct ordered // read-modify-write into the imageblock), so per-pixel overdraw isn't just extra work — it's @@ -54,8 +64,15 @@ constant uchar kGaussianMaxBlendedSplatsPerPixel = 64; inline uint unpackIndex(uint64_t packed) { return (uint)(packed & 0xffffffffu); } inline uint unpackDepthKey(uint64_t packed) { return (uint)(packed >> 32); } +// Dequantizes a byte packed by quantizeGaussianSHCoefficient (Swift) back +// into the fixed [-1, 1] range. +inline float dequantizeGaussianSHCoefficient(uchar packed) +{ + return (float(packed) - 128.0f) / 128.0f; +} + float3 loadGaussianSHCoefficient( - const device half *coefficients, + const device uchar *coefficients, constant GaussianSHMetadata &metadata, uint splatIndex, uint coefficientIndex) @@ -64,15 +81,15 @@ float3 loadGaussianSHCoefficient( uint splatBase = splatIndex * metadata.higherOrderCoefficientsPerSplat; uint offset = coefficientIndex - 1; return float3( - coefficients[splatBase + offset], - coefficients[splatBase + perChannel + offset], - coefficients[splatBase + 2 * perChannel + offset] + dequantizeGaussianSHCoefficient(coefficients[splatBase + offset]), + dequantizeGaussianSHCoefficient(coefficients[splatBase + perChannel + offset]), + dequantizeGaussianSHCoefficient(coefficients[splatBase + 2 * perChannel + offset]) ); } float3 evaluateGaussianSphericalHarmonics( float3 baseColor, - const device half *coefficients, + const device uchar *coefficients, constant GaussianSHMetadata &metadata, uint splatIndex, float3 direction) @@ -134,7 +151,7 @@ float3 gaussianSRGBToLinear(float3 color) // Diagnostic entry point for validating the packed GPU SH contract against // the exact evaluator used by the Gaussian vertex shader. kernel void gaussianSphericalHarmonicsDiagnostic( - const device half *coefficients [[buffer(0)]], + const device uchar *coefficients [[buffer(0)]], constant GaussianSHMetadata &metadata [[buffer(1)]], constant float4 &baseColor [[buffer(2)]], constant float4 &direction [[buffer(3)]], @@ -208,9 +225,18 @@ float3 computeCov2D(float4 splatCenter, return float3(cov[0][0], cov[0][1], cov[1][1]); } -// Compute inverse covariance and a per-axis screen-space half-extent (~3σ) in pixels +// Compute inverse covariance (conic) and the two orthogonal kGaussianQuadSigma-sigma semi-axis +// vectors (in screen pixels) of the projected covariance ellipse, via eigen-decomposition of +// the symmetric 2×2 [[a,b],[b,c]] matrix. axis1/axis2 point along the ellipse's true principal directions — +// used to build a tight, rotated quad instead of an axis-aligned bounding box, which for an +// anisotropic, non-axis-aligned splat (the common case: Gaussians are oriented however the +// surface they came from sits) can be several times larger in area than the ellipse itself, +// costing that many more rasterized/shaded fragments regardless of how cheap the per-fragment +// TBDR blend itself is. Eigenvector formula matches the standard closed-form solution for a +// symmetric 2×2 matrix (as used by e.g. MetalSplatter's decomposeCovariance). float3 computeInverseCovarianceConic(float3 cov2D, - thread float2 &radius, + thread float2 &axis1, + thread float2 &axis2, thread bool &valid) { float a = cov2D.x; @@ -218,12 +244,71 @@ float3 computeInverseCovarianceConic(float3 cov2D, float c = cov2D.z; float det = a * c - b * b; - if (det == 0.0f) { + // cov2D is provably PSD (built from a congruence transform of a PSD 3D covariance, plus a + // positive diagonal dilation), so its determinant is mathematically always >= 0 — a + // negative reading can only come from float round-off on a near-singular matrix. Guarding + // <= 0 (not just == 0) catches that case before it flips the sign of detInv/conic, which + // would otherwise invert the falloff (alpha growing instead of decaying away from center). + if (det <= 0.0f) { valid = false; - radius = float2(0.0f); + axis1 = float2(0.0f); + axis2 = float2(0.0f); return float3(0.0f); } + float trace = a + c; + float mean = 0.5f * trace; + // Discriminant of the characteristic polynomial — mathematically always >= 0 for a real + // symmetric matrix; the max() guards only against float round-off near-singular matrices. + // The 0.1 floor (matching the reference implementation) keeps the eigenvector computation + // below numerically stable when the ellipse is nearly circular (b~0, a~c). + float dist = max(0.1f, sqrt(max(mean * mean - det, 0.0f))); + float lambda1 = mean + dist; + float lambda2 = max(mean - dist, 0.0f); + + float2 eigenvector1; + if (b == 0.0f) { + eigenvector1 = (a > c) ? float2(1.0f, 0.0f) : float2(0.0f, 1.0f); + } else { + eigenvector1 = normalize(float2(b, c - (mean - dist))); + } + // The second eigenvector of a symmetric 2x2 matrix is always orthogonal to the first. + float2 eigenvector2 = float2(eigenvector1.y, -eigenvector1.x); + + float radius1 = kGaussianQuadSigma * sqrt(lambda1); + float radius2 = kGaussianQuadSigma * sqrt(lambda2); + + // A splat whose true kGaussianQuadSigma-sigma extent along either principal axis exceeds + // kGaussianMaxScreenRadius (very close to the camera — radius grows ~1/distance) needs + // its rendered quad clamped down for overdraw reasons, but the falloff must be clamped + // along with it, or the (smaller) quad sits within the Gaussian's near-flat peak and + // never reaches the part of the curve that actually decays — visually a hard-edged, + // nearly-opaque block instead of a soft blob. + // + // Each axis is clamped independently (not by a single shared ratio) so an elongated + // splat's short axis isn't shrunk just because its long axis needed clamping — matching + // how the previous axis-aligned implementation clamped x and y separately. The clamped + // covariance is then reconstructed from the (independently-scaled) eigenvalues and the + // unchanged eigenvector directions — M = R·diag(λ1,λ2)·Rᵀ — so conic, radius, and the + // falloff all agree on the same (possibly non-uniformly-shrunk) ellipse. det(M) = λ1·λ2 + // regardless of rotation, since R is orthogonal (det(R)·det(Rᵀ) = 1). + float clampedRadius1 = min(radius1, kGaussianMaxScreenRadius); + float clampedRadius2 = min(radius2, kGaussianMaxScreenRadius); + if (clampedRadius1 < radius1 || clampedRadius2 < radius2) { + float scale1 = clampedRadius1 / radius1; + float scale2 = clampedRadius2 / radius2; + float clampedLambda1 = lambda1 * scale1 * scale1; + float clampedLambda2 = lambda2 * scale2 * scale2; + + a = clampedLambda1 * eigenvector1.x * eigenvector1.x + clampedLambda2 * eigenvector2.x * eigenvector2.x; + b = clampedLambda1 * eigenvector1.x * eigenvector1.y + clampedLambda2 * eigenvector2.x * eigenvector2.y; + c = clampedLambda1 * eigenvector1.y * eigenvector1.y + clampedLambda2 * eigenvector2.y * eigenvector2.y; + det = clampedLambda1 * clampedLambda2; + + radius1 = clampedRadius1; + radius2 = clampedRadius2; + } + float detInv = 1.0f / det; float3 conic = float3( @@ -232,23 +317,14 @@ float3 computeInverseCovarianceConic(float3 cov2D, a * detInv ); - // Tight axis-aligned half-extent covering ~3σ along each screen axis independently. - // This is the *exact* per-axis extent of the ellipse {d : dᵀ·conic·d <= 9} — it depends - // only on cov2D's diagonal (a, c), not its off-diagonal correlation term (b), regardless - // of how the ellipse is rotated. That makes it strictly tighter than (or equal to) the - // previous square sized by the larger eigenvalue: an anisotropic splat (e.g. a thin, - // flat-surface splat) no longer forces both screen axes out to the size of its longest - // axis, so fewer wasted fragments get rasterized, shaded, and discarded. - radius = float2( - min(ceil(3.0f * sqrt(a)), kGaussianMaxScreenRadius), - min(ceil(3.0f * sqrt(c)), kGaussianMaxScreenRadius) - ); + axis1 = eigenvector1 * radius1; + axis2 = eigenvector2 * radius2; valid = true; return conic; } -// Computes conic/radius/color once per visible splat per frame — the same quantities the +// Computes conic/axes/color once per visible splat per frame — the same quantities the // draw vertex shader used to recompute redundantly on every one of its 4 instanced quad // vertices. Writes into a buffer indexed by original splat index; the vertex shader then // just reads by index instead of redoing the Jacobian/covariance math and SH evaluation 4x. @@ -259,7 +335,7 @@ kernel void gaussianPreprocess( const device uint *visibleIndices [[buffer(gaussianPreprocessVisibleIndicesIndex)]], const device uint *visibleCount [[buffer(gaussianPreprocessVisibleCountIndex)]], constant float2 &viewport [[buffer(gaussianPreprocessViewportIndex)]], - const device half *shCoefficients [[buffer(gaussianPreprocessSHIndex)]], + const device uchar *shCoefficients [[buffer(gaussianPreprocessSHIndex)]], constant GaussianSHMetadata &shMetadata [[buffer(gaussianPreprocessSHMetadataIndex)]], constant float3 &localCameraPosition [[buffer(gaussianPreprocessLocalCameraIndex)]], device GaussianPrecomputedSplat *precomputed [[buffer(gaussianPreprocessOutputIndex)]], @@ -274,7 +350,8 @@ kernel void gaussianPreprocess( GaussianPrecomputedSplat out; out.conic = float3(0.0f); - out.radius = float2(0.0f); + out.axis1 = float2(0.0f); + out.axis2 = float2(0.0f); out.color = float3(0.0f); float3 centerLocal = splat.position; @@ -288,9 +365,9 @@ kernel void gaussianPreprocess( } float3x3 cov3D = float3x3( - splat.covA.x, splat.covA.y, splat.covA.z, - splat.covA.y, splat.covB.x, splat.covB.y, - splat.covA.z, splat.covB.y, splat.covB.z + float(splat.covA.x), float(splat.covA.y), float(splat.covA.z), + float(splat.covA.y), float(splat.covB.x), float(splat.covB.y), + float(splat.covA.z), float(splat.covB.y), float(splat.covB.z) ); float3 cov2D = computeCov2D(float4(centerLocal, 1.0), @@ -299,19 +376,21 @@ kernel void gaussianPreprocess( uniforms.projectionMatrix, viewport); - float2 radius = float2(0.0f); + float2 axis1 = float2(0.0f); + float2 axis2 = float2(0.0f); bool valid = true; - float3 conic = computeInverseCovarianceConic(cov2D, radius, valid); + float3 conic = computeInverseCovarianceConic(cov2D, axis1, axis2, valid); - if (!valid || radius.x <= 0.0f || radius.y <= 0.0f) { + if (!valid || (axis1.x == 0.0f && axis1.y == 0.0f) || (axis2.x == 0.0f && axis2.y == 0.0f)) { precomputed[splatIndex] = out; return; } out.conic = conic; - out.radius = radius; + out.axis1 = axis1; + out.axis2 = axis2; out.color = gaussianSRGBToLinear(evaluateGaussianSphericalHarmonics( - splat.color, + float3(splat.colorAndOpacity.xyz), shCoefficients, shMetadata, splatIndex, @@ -377,6 +456,7 @@ vertex GaussianOutData vertexGaussianTBDRShader( constant Uniforms &uniforms [[buffer(gaussianTBDRRenderUniformIndex)]], constant float2 &viewport [[buffer(gaussianTBDRRenderViewPortIndex)]], const device GaussianPrecomputedSplat *precomputedSplats [[buffer(gaussianTBDRRenderPrecomputedIndex)]], + constant float4 &debugColor [[buffer(gaussianTBDRRenderDebugColorIndex)]], uint vid [[vertex_id]], uint iid [[instance_id]]) { @@ -392,10 +472,11 @@ vertex GaussianOutData vertexGaussianTBDRShader( const EncodedGaussianSplat splat = splats[splatIndex]; const GaussianPrecomputedSplat precomputed = precomputedSplats[splatIndex]; - // radius <= 0 means gaussianPreprocess found this splat invalid this frame (behind the - // camera, or a degenerate covariance) — same condition the old inline computation + // Zero axis vectors mean gaussianPreprocess found this splat invalid this frame (behind + // the camera, or a degenerate covariance) — same condition the old inline computation // guarded against. - if (precomputed.radius.x <= 0.0f || precomputed.radius.y <= 0.0f) { + if ((precomputed.axis1.x == 0.0f && precomputed.axis1.y == 0.0f) || + (precomputed.axis2.x == 0.0f && precomputed.axis2.y == 0.0f)) { return out; } @@ -418,11 +499,16 @@ vertex GaussianOutData vertexGaussianTBDRShader( out.conic = precomputed.conic; - float2 ndcOffset = quad * precomputed.radius * 2.0f / viewport; + // Tight, rotated quad along the ellipse's true principal axes (see + // computeInverseCovarianceConic) instead of an axis-aligned bounding box — avoids + // rasterizing/shading several times more fragments than necessary for anisotropic, + // non-axis-aligned splats. + float2 pixelOffset = quad.x * precomputed.axis1 + quad.y * precomputed.axis2; + float2 ndcOffset = pixelOffset * 2.0f / viewport; out.position = centerClip; out.position.xy += ndcOffset * centerClip.w; - out.color = precomputed.color; - out.alpha = splat.opacity; + out.color = debugColor.w > 0.0f ? debugColor.xyz : precomputed.color; + out.alpha = float(splat.colorAndOpacity.w); out.valid = true; return out; @@ -452,9 +538,10 @@ fragment GaussianTBDRFragmentStore fragmentGaussianTBDRShader( } // Evaluate the Gaussian falloff before touching the opaque-depth texture: this is pure - // ALU (no memory fetch), and most of a splat's rasterized area — out near the 3σ quad - // edge — has negligible alpha. Rejecting those tail fragments here means they never pay - // for the depth-texture read at all, on top of never reaching the blend math below. + // ALU (no memory fetch), and most of a splat's rasterized area — out near the quad edge + // (kGaussianQuadSigma sigma out) — has negligible alpha. Rejecting those tail fragments + // here means they never pay for the depth-texture read at all, on top of never reaching + // the blend math below. const float projYSign = 1.0f; float2 d = calcScreenSpaceDelta(in.position.xy, in.coordxy, projYSign); float power = calcPowerFromConic(in.conic, d); diff --git a/Sources/UntoldEngine/Systems/GaussianLODSystem.swift b/Sources/UntoldEngine/Systems/GaussianLODSystem.swift new file mode 100644 index 00000000..14241bbd --- /dev/null +++ b/Sources/UntoldEngine/Systems/GaussianLODSystem.swift @@ -0,0 +1,286 @@ +// +// GaussianLODSystem.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import Foundation +import simd + +/// Distance-driven tier selection for progressively streamed Gaussian splats. +public class GaussianLODSystem: @unchecked Sendable { + public static let shared = GaussianLODSystem() + private init() {} + + private var frameCounter = 0 + private var lastCameraPosition: simd_float3 = .zero + private var hasRunOnce = false + + public func reset() { + frameCounter = 0 + lastCameraPosition = .zero + hasRunOnce = false + } + + public func update(deltaTime _: Float) { + frameCounter &+= 1 + + guard let camera = CameraSystem.shared.activeCamera, + let cameraComponent = scene.get(component: CameraComponent.self, for: camera) + else { return } + + let cameraPosition = SceneRootTransform.shared.effectiveCameraPosition(cameraComponent.localPosition) + + // This global check only catches camera movement — it decides whether to force *every* + // entity to re-evaluate this frame (the frame-interval throttle still applies per-entity + // below either way). An entity that itself moved (e.g. a splat prop being dragged while + // the camera stays put) is caught by updateEntityLOD's own per-entity displacement check, + // not by this one. + let forceFullReevaluation = lodShouldRunThisFrame( + frameCounter: frameCounter, + hasRunOnce: hasRunOnce, + interval: LODConfig.shared.lodUpdateFrameInterval, + cameraPosition: cameraPosition, + lastCameraPosition: lastCameraPosition, + displacementThreshold: LODConfig.shared.minimumCameraDisplacementForLODUpdate + ) + + // Only advance on a qualifying frame — lodShouldRunThisFrame's cumulative-displacement + // fast path needs lastCameraPosition to still reflect the last qualifying frame, not + // every frame, or slow continuous camera motion below the per-frame threshold could + // never accumulate enough to trip it. Mirrors the mesh LODSystem.update()'s guarded form. + if forceFullReevaluation { + hasRunOnce = true + lastCameraPosition = cameraPosition + } + + let lodId = getComponentId(for: GaussianLODComponent.self) + let transformId = getComponentId(for: WorldTransformComponent.self) + let entities = queryEntitiesWithComponentIds([lodId, transformId], in: scene) + + for entityId in entities { + updateEntityLOD(entityId: entityId, cameraPosition: cameraPosition, forceFullReevaluation: forceFullReevaluation) + } + } + + private func updateEntityLOD(entityId: EntityID, cameraPosition: simd_float3, forceFullReevaluation: Bool) { + guard let lodComponent = scene.get(component: GaussianLODComponent.self, for: entityId), + !lodComponent.lodLevels.isEmpty + else { return } + + let distance = entityDistanceToCamera(entityId: entityId, cameraPosition: cameraPosition) + + // Cheap per-entity fast path: even when the global camera-driven throttle above says + // "skip this frame," still refresh if this specific entity's distance to the camera has + // moved enough to plausibly cross a LOD threshold — otherwise dragging a prop while the + // camera stays still could go unnoticed for up to lodUpdateFrameInterval frames, or + // (if the drag ends before a throttled frame lands) not be picked up at all until + // something else nudges the throttle. + let entityMoved: Bool + if let lastEvaluatedDistance = lodComponent.lastEvaluatedDistance { + entityMoved = abs(distance - lastEvaluatedDistance) > LODConfig.shared.minimumEntityDisplacementForLODUpdate + } else { + entityMoved = true + } + + guard forceFullReevaluation || entityMoved else { return } + lodComponent.lastEvaluatedDistance = distance + + let distanceSelectedLOD = selectDesiredLOD(distance: distance, lodComponent: lodComponent) + lodComponent.distanceSelectedLOD = distanceSelectedLOD + + var desiredLOD = distanceSelectedLOD + + if let localTransform = scene.get(component: LocalTransformComponent.self, for: entityId) { + let halfExtent = (localTransform.boundingBox.max - localTransform.boundingBox.min) * 0.5 + let boundingRadius = simd_length(halfExtent) + let tanHalfFovY = renderInfo.perspectiveSpace[1][1] > 0 ? 1.0 / renderInfo.perspectiveSpace[1][1] : 0 + let fovY = 2 * atan(tanHalfFovY) + let viewportHeight = renderInfo.viewPort?.y ?? 0 + + desiredLOD = clampGaussianLODForOverdraw( + desiredLOD: desiredLOD, + lodComponent: lodComponent, + distance: distance, + fovY: fovY, + viewportHeight: viewportHeight, + boundingRadius: boundingRadius, + budget: LODConfig.shared.gaussianOverdrawBudget + ) + } + + lodComponent.desiredLOD = desiredLOD + + if !lodComponent.isLODResident(desiredLOD) { + GeometryStreamingSystem.shared.requestGaussianLODLevelLoad(entityId: entityId, lodIndex: desiredLOD) + } + + let actualLOD: Int + if lodComponent.isLODResident(desiredLOD) { + lodComponent.isUsingFallback = false + actualLOD = desiredLOD + } else if let fallback = lodComponent.findFallbackLOD(from: desiredLOD) { + lodComponent.isUsingFallback = true + actualLOD = fallback + } else { + return + } + + if Logger.isEnabled(category: .gaussian), + desiredLOD != lodComponent.lastLoggedDesiredLOD || actualLOD != lodComponent.lastLoggedActualLOD + { + lodComponent.lastLoggedDesiredLOD = desiredLOD + lodComponent.lastLoggedActualLOD = actualLOD + Logger.log( + message: String( + format: "[Gaussian][LOD] entity=%llu distance=%.2f desiredLOD=%d actualLOD=%d currentLOD=%d fallback=%@ residency=%@", + entityId, + distance, + desiredLOD, + actualLOD, + lodComponent.currentLOD, + lodComponent.isUsingFallback ? "true" : "false", + lodComponent.lodLevels.map { "\($0.residencyState)" }.joined(separator: ",") + ), + category: LogCategory.gaussian.rawValue + ) + } + + applyLOD(entityId: entityId, newLOD: actualLOD) + } + + private func selectDesiredLOD(distance: Float, lodComponent: GaussianLODComponent) -> Int { + // lodComponent.distanceSelectedLOD still holds the previous frame's pure distance-based + // decision here — the caller overwrites it with this call's result right after — so + // passing it as currentLOD gives selectLODIndex the hysteresis reference point it + // needs, same as the mesh LODSystem.updateEntityLOD -> selectLODLevel call. Deliberately + // NOT lodComponent.desiredLOD: that field can hold an overdraw-forced-coarser value the + // hysteresis math was never designed to anchor on — see distanceSelectedLOD's doc comment. + selectLODIndex( + levels: lodComponent.lodLevels, + distance: distance, + currentLOD: lodComponent.distanceSelectedLOD, + forcedLOD: lodComponent.forcedLOD, + lodBias: LODConfig.shared.lodBias, + hysteresis: LODConfig.shared.hysteresis, + globalDistances: LODConfig.shared.lodDistances + ) + } + + private func applyLOD(entityId: EntityID, newLOD: Int) { + guard let lodComponent = scene.get(component: GaussianLODComponent.self, for: entityId), + newLOD >= 0, + newLOD < lodComponent.lodLevels.count, + let source = lodComponent.lodLevels[newLOD].buffers + else { return } + + if newLOD == lodComponent.currentLOD, scene.get(component: GaussianComponent.self, for: entityId) != nil { + return + } + + withWorldMutationGate { + // Reuse the entity's existing GaussianComponent if it already has one — scene.assign + // unconditionally re-initializes the component slot, which would drop the previous + // instance (and every Metal buffer it retained) without releasing it. + guard let destination = scene.get(component: GaussianComponent.self, for: entityId) + ?? scene.assign(to: entityId, component: GaussianComponent.self) + else { + return + } + + copyGaussianComponentBuffers(from: source, to: destination) + lodComponent.currentLOD = newLOD + SystemIntegrationMonitor.shared.recordLODSwitch() + } + } +} + +/// Estimated mean overdraw (blended fragments per pixel) across a Gaussian entity's screen +/// footprint, for one candidate LOD tier. Mirrors the perspective size-at-distance factor +/// `computeCov2D` (`Gaussians.metal`) derives from the projection matrix +/// (`focalY = viewport.y * projectionMatrix[1][1] * 0.5`), reused here CPU-side to turn a +/// tier's bake-time `meanSquaredSplatExtent` (see `GaussianLODTier`) into an estimate of how +/// many splats overlap per pixel — the actual GPU cost driver behind the engine's serial +/// per-pixel TBDR blend (`kGaussianMaxBlendedSplatsPerPixel`), which pure distance-based LOD +/// selection has no visibility into. +func estimatedGaussianOverdraw( + splatCount: Int, + meanSquaredSplatExtent: Float, + distance: Float, + fovY: Float, + viewportHeight: Float, + boundingRadius: Float +) -> Float { + guard splatCount > 0, distance > 0, boundingRadius > 0, viewportHeight > 0 else { return 0 } + let tanHalfFovY = tan(fovY * 0.5) + guard tanHalfFovY > 0 else { return 0 } + + let projectionScaleFactor = viewportHeight / (2 * tanHalfFovY * distance) + let projectedAreaPerSplat = projectionScaleFactor * projectionScaleFactor * meanSquaredSplatExtent + let footprintRadius = boundingRadius * projectionScaleFactor + let objectScreenFootprintArea = Float.pi * footprintRadius * footprintRadius + + guard objectScreenFootprintArea > 0 else { return 0 } + return (Float(splatCount) * projectedAreaPerSplat) / objectScreenFootprintArea +} + +/// Walks from `desiredLOD` (the distance/hysteresis-based choice from `selectDesiredLOD`) +/// toward coarser tiers (higher index, never finer) while `estimatedGaussianOverdraw` for the +/// candidate exceeds `budget`, using each candidate's bake-time `meanSquaredSplatExtent` and +/// its currently-known resident `splatCount`. Bails out and returns `desiredLOD` unchanged the +/// moment a candidate is missing either value — an un-baked `meanSquaredSplatExtent` or a +/// not-yet-resident tier (unknown splat count) — so entities without bake-time stats, or a +/// tier this walk reaches before it has ever loaded, fall back to pure distance-based +/// selection exactly as before this feature existed. +func clampGaussianLODForOverdraw( + desiredLOD: Int, + lodComponent: GaussianLODComponent, + distance: Float, + fovY: Float, + viewportHeight: Float, + boundingRadius: Float, + budget: Float +) -> Int { + guard desiredLOD >= 0, desiredLOD < lodComponent.lodLevels.count else { return desiredLOD } + + var candidate = desiredLOD + while candidate < lodComponent.lodLevels.count { + let level = lodComponent.lodLevels[candidate] + guard let meanSquaredSplatExtent = level.meanSquaredSplatExtent, + let splatCount = level.buffers?.splatCount + else { + return desiredLOD + } + + let overdraw = estimatedGaussianOverdraw( + splatCount: Int(splatCount), + meanSquaredSplatExtent: meanSquaredSplatExtent, + distance: distance, + fovY: fovY, + viewportHeight: viewportHeight, + boundingRadius: boundingRadius + ) + if overdraw <= budget { + return candidate + } + candidate += 1 + } + return lodComponent.lodLevels.count - 1 +} + +func copyGaussianComponentBuffers(from source: GaussianComponent, to destination: GaussianComponent) { + destination.splatCount = source.splatCount + destination.visibleSplatCountForRendering = source.splatCount + destination.gaussianSortedIndices = source.gaussianSortedIndices + destination.gaussianVisibleIndices = source.gaussianVisibleIndices + destination.gaussianVisibleCount = source.gaussianVisibleCount + destination.encodedSplatData = source.encodedSplatData + destination.gaussianPrecomputedData = source.gaussianPrecomputedData + destination.sphericalHarmonicsData = source.sphericalHarmonicsData + destination.sphericalHarmonicsMetadata = source.sphericalHarmonicsMetadata + destination.spaceUniform = source.spaceUniform +} diff --git a/Sources/UntoldEngine/Systems/GeometryStreamingSystem+GaussianStreaming.swift b/Sources/UntoldEngine/Systems/GeometryStreamingSystem+GaussianStreaming.swift index f8ecd50d..bafcccb1 100644 --- a/Sources/UntoldEngine/Systems/GeometryStreamingSystem+GaussianStreaming.swift +++ b/Sources/UntoldEngine/Systems/GeometryStreamingSystem+GaussianStreaming.swift @@ -27,59 +27,212 @@ extension GeometryStreamingSystem { /// incrementing it here would inflate the numerator past a denominator it never grew, /// corrupting the tile's `visualState` completion fraction. func loadGaussianStreamingEntity(entityId: EntityID, streaming: StreamingComponent, isNearBand: Bool) { + if scene.get(component: GaussianLODComponent.self, for: entityId) != nil { + loadGaussianProgressiveStreamingEntity(entityId: entityId, streaming: streaming, isNearBand: isNearBand) + return + } + let filename = streaming.assetFilename let ext = streaming.assetExtension + dispatchGaussianLoad(entityId: entityId, streaming: streaming, isNearBand: isNearBand) { + await setEntityGaussianAsync(entityId: entityId, filename: filename, withExtension: ext) + } + } + + /// Loads only the coarsest tier for a progressive Gaussian prop. Finer tiers are pulled + /// later by `GaussianLODSystem` as the camera moves close enough to need them. + func loadGaussianProgressiveStreamingEntity(entityId: EntityID, streaming: StreamingComponent, isNearBand: Bool) { + dispatchGaussianLoad(entityId: entityId, streaming: streaming, isNearBand: isNearBand) { + await self.loadInitialGaussianProgressiveTier(entityId: entityId) + } + } + + /// Runs `loader`, then applies its result through the shared completion path (state + /// transition, residency event, monitor bookkeeping, active/near-band load release) and + /// records load-time instrumentation — shared by both the plain and progressive Gaussian + /// load entry points above, which previously duplicated this whole body and had already + /// drifted (the progressive path was missing this timing instrumentation). + private func dispatchGaussianLoad( + entityId: EntityID, + streaming: StreamingComponent, + isNearBand: Bool, + loader: @escaping @Sendable () async -> Bool + ) { let task = Task { let asyncLoadStart = CFAbsoluteTimeGetCurrent() - let success = await setEntityGaussianAsync(entityId: entityId, filename: filename, withExtension: ext) + let success = await loader() let asyncLoadMs = (CFAbsoluteTimeGetCurrent() - asyncLoadStart) * 1000.0 var applyMs: Double = 0 withWorldMutationGate { let applyStart = CFAbsoluteTimeGetCurrent() + completeGaussianStreamingLoad(entityId: entityId, isNearBand: isNearBand, success: success) + applyMs = (CFAbsoluteTimeGetCurrent() - applyStart) * 1000.0 + } + recordLoadCompletion(success: success, asyncLoadMs: asyncLoadMs, applyMs: applyMs, wasLODReload: false) + } + + streaming.loadTask = task + } - // Guard against the cooperative-cancellation race, same as loadMesh's mesh path: - // unloadGaussian may have freed this entity while the load was in flight. - guard scene.exists(entityId) else { - releaseActiveLoad(entityId: entityId) - if isNearBand { releaseNearBandLoad(entityId: entityId) } - return + /// Applies a completed Gaussian load's result to `StreamingComponent`/event bus/monitor. + /// Must be called from within `withWorldMutationGate`. + private func completeGaussianStreamingLoad( + entityId: EntityID, + isNearBand: Bool, + success: Bool + ) { + // Guard against the cooperative-cancellation race, same as loadMesh's mesh path: + // unloadGaussian may have freed this entity while the load was in flight. + guard scene.exists(entityId) else { + releaseActiveLoad(entityId: entityId) + if isNearBand { releaseNearBandLoad(entityId: entityId) } + return + } + + guard let s = scene.get(component: StreamingComponent.self, for: entityId), + s.state == .loading + else { + releaseActiveLoad(entityId: entityId) + if isNearBand { releaseNearBandLoad(entityId: entityId) } + return + } + + if success { + s.state = .loaded + s.lastVisibleFrame = currentFrame + + let event = AssetResidencyChangedEvent( + entityId: entityId, + assetURL: URL(fileURLWithPath: ""), + meshName: s.assetFilename, + isResident: true + ) + SystemEventBus.shared.queueResidencyChange(event) + markLoadedStreamingEntity(entityId) + SystemIntegrationMonitor.shared.recordStreamingLoad() + } else { + s.state = .unloaded + handleError(.meshStreamingFailed, entityId) + } + releaseActiveLoad(entityId: entityId) + if isNearBand { releaseNearBandLoad(entityId: entityId) } + } + + func loadInitialGaussianProgressiveTier(entityId: EntityID) async -> Bool { + let coarsestIndex: Int? = withWorldMutationGate { + guard let lod = scene.get(component: GaussianLODComponent.self, for: entityId), + !lod.lodLevels.isEmpty + else { return nil } + let index = lod.lodLevels.count - 1 + // Mark the slot .loading synchronously, same as requestGaussianLODLevelLoad does for + // on-demand tiers — loadGaussianLODLevel's completion only commits while this still + // reads .loading, so unloadGaussianProgressive resetting it to .notResident in the + // meantime turns a stale completion into a safe no-op instead of resurrecting state. + lod.lodLevels[index].residencyState = .loading + return index + } + guard let coarsestIndex else { return false } + return await loadGaussianLODLevel(entityId: entityId, lodIndex: coarsestIndex, makeCurrent: true) + } + + public func requestGaussianLODLevelLoad(entityId: EntityID, lodIndex: Int) { + withWorldMutationGate { + guard scene.exists(entityId), + let lod = scene.get(component: GaussianLODComponent.self, for: entityId), + lodIndex >= 0, + lodIndex < lod.lodLevels.count + else { return } + + guard lod.lodLevels[lodIndex].residencyState != .resident, + lod.lodLevels[lodIndex].residencyState != .loading + else { return } + + lod.lodLevels[lodIndex].residencyState = .loading + let task = Task { [weak self] in + _ = await self?.loadGaussianLODLevel(entityId: entityId, lodIndex: lodIndex, makeCurrent: false) + } + lod.lodLevels[lodIndex].loadTask = task + } + } + + @discardableResult + func loadGaussianLODLevel(entityId: EntityID, lodIndex: Int, makeCurrent: Bool) async -> Bool { + let url: URL? = withWorldMutationGate { + guard scene.exists(entityId), + let lod = scene.get(component: GaussianLODComponent.self, for: entityId), + lodIndex >= 0, + lodIndex < lod.lodLevels.count + else { return nil } + return lod.lodLevels[lodIndex].url + } + guard let url, let built = buildGaussianComponentFromUntoldGS(url: url) else { + withWorldMutationGate { + if let lod = scene.get(component: GaussianLODComponent.self, for: entityId), + lodIndex >= 0, + lodIndex < lod.lodLevels.count, + lod.lodLevels[lodIndex].residencyState == .loading + { + lod.lodLevels[lodIndex].residencyState = .notResident + lod.lodLevels[lodIndex].loadTask = nil } + } + return false + } - guard let s = scene.get(component: StreamingComponent.self, for: entityId), - s.state == .loading - else { - releaseActiveLoad(entityId: entityId) - if isNearBand { releaseNearBandLoad(entityId: entityId) } - return + return withWorldMutationGate { + // residencyState == .loading confirms this is still the authoritative load for the + // slot: unloadGaussianProgressive/removeEntityGaussianLOD reset it to .notResident on + // teardown, and Task.cancel() is only cooperative (never observed mid-await here), so + // without this re-check a cancelled-but-still-running load could resurrect buffers/ + // residency on an entity the streaming system already considers unloaded. + guard scene.exists(entityId), + let lod = scene.get(component: GaussianLODComponent.self, for: entityId), + lodIndex >= 0, + lodIndex < lod.lodLevels.count, + lod.lodLevels[lodIndex].residencyState == .loading + else { return false } + + lod.lodLevels[lodIndex].buffers = built.component + lod.lodLevels[lodIndex].residencyState = .resident + lod.lodLevels[lodIndex].loadTask = nil + // Baked into the .untoldgs file itself (see UntoldGSFormat) — no caller-supplied + // value needed, and it can't drift out of sync with the tier it describes. + lod.lodLevels[lodIndex].meanSquaredSplatExtent = built.meanSquaredSplatExtent + + if makeCurrent { + // The entity's very first tier to ever load — if the caller didn't supply a + // real box up front (setEntityGaussianProgressive's boundingBoxHalfExtent is + // optional; setEntityGaussianProgressiveStreamable's is required and already + // marks hasExplicitBoundingBox, so this never overwrites it), auto-populate one + // from this tier's actual splat data instead of leaving the entity on its + // default placeholder box forever. + if !lod.hasExplicitBoundingBox, + let local = scene.get(component: LocalTransformComponent.self, for: entityId) + { + local.boundingBox = built.boundingBox } - if success { - s.state = .loaded - s.lastVisibleFrame = currentFrame - - let event = AssetResidencyChangedEvent( - entityId: entityId, - assetURL: URL(fileURLWithPath: ""), - meshName: s.assetFilename, - isResident: true - ) - SystemEventBus.shared.queueResidencyChange(event) - markLoadedStreamingEntity(entityId) - SystemIntegrationMonitor.shared.recordStreamingLoad() - } else { - s.state = .unloaded - handleError(.meshStreamingFailed, entityId) + // Reuse the entity's existing GaussianComponent if it already has one — scene.assign + // unconditionally re-initializes the component slot, which would drop the previous + // instance (and every Metal buffer it retained) without releasing it. + if let live = scene.get(component: GaussianComponent.self, for: entityId) + ?? scene.assign(to: entityId, component: GaussianComponent.self) + { + copyGaussianComponentBuffers(from: built.component, to: live) + lod.currentLOD = lodIndex } - releaseActiveLoad(entityId: entityId) - if isNearBand { releaseNearBandLoad(entityId: entityId) } - applyMs = (CFAbsoluteTimeGetCurrent() - applyStart) * 1000.0 } - recordLoadCompletion(success: success, asyncLoadMs: asyncLoadMs, applyMs: applyMs, wasLODReload: false) - } - streaming.loadTask = task + var totalBytes = 0 + for level in lod.lodLevels { + guard let buffers = level.buffers else { continue } + totalBytes += gaussianComponentEstimatedBytes(buffers) + } + MemoryBudgetManager.shared.registerMesh(entityId: entityId, meshSizeBytes: totalBytes) + return true + } } /// Tears down a streamed Gaussian-splat entity's GPU resources and removes it from the @@ -100,6 +253,15 @@ extension GeometryStreamingSystem { streaming.loadTask?.cancel() streaming.loadTask = nil + // No-op for a plain (non-progressive) splat entity, since it has no + // GaussianLODComponent — folds the progressive-only per-tier teardown in here + // instead of a separate unloadGaussianProgressive sibling function. + if let lod = scene.get(component: GaussianLODComponent.self, for: entityId) { + lod.releaseAllLevelResources() + lod.currentLOD = -1 + lod.desiredLOD = max(0, lod.lodLevels.count - 1) + } + removeEntityGaussian(entityId: entityId) // removeEntityGaussian already unregisters from MemoryBudgetManager, but the call @@ -124,3 +286,25 @@ extension GeometryStreamingSystem { updateLastUnloadDuration(unloadMs) } } + +private func gaussianComponentEstimatedBytes(_ component: GaussianComponent) -> Int { + var total = 0 + total += component.encodedSplatData?.length ?? 0 + total += component.sphericalHarmonicsData?.length ?? 0 + for buffer in component.gaussianSortedIndices { + total += buffer?.length ?? 0 + } + for buffer in component.gaussianVisibleIndices { + total += buffer?.length ?? 0 + } + for buffer in component.gaussianVisibleCount { + total += buffer?.length ?? 0 + } + for buffer in component.gaussianPrecomputedData { + total += buffer?.length ?? 0 + } + for buffer in component.spaceUniform { + total += buffer?.length ?? 0 + } + return total +} diff --git a/Sources/UntoldEngine/Systems/LODConfig.swift b/Sources/UntoldEngine/Systems/LODConfig.swift index 6ec151fe..ede66199 100644 --- a/Sources/UntoldEngine/Systems/LODConfig.swift +++ b/Sources/UntoldEngine/Systems/LODConfig.swift @@ -65,4 +65,21 @@ public struct LODConfig { /// Camera must move at least this many world units since the last LOD update /// before a new update is forced ahead of the frame-interval throttle. public var minimumCameraDisplacementForLODUpdate: Float = 0.5 + + /// An entity's distance-to-camera must change by at least this many world units since its + /// own last LOD evaluation before a refresh is forced ahead of the frame-interval throttle + /// — mirrors `minimumCameraDisplacementForLODUpdate` but for the target moving instead of + /// the camera (e.g. dragging a Gaussian splat prop while the camera stays put, which the + /// camera-only fast path can't see). + public var minimumEntityDisplacementForLODUpdate: Float = 0.5 + + /// Ceiling on `estimatedGaussianOverdraw` (mean blended fragments per pixel across a + /// Gaussian entity's screen footprint) before `GaussianLODSystem` forces a coarser tier + /// than pure distance-based selection would pick. This is a starting guess, not a derived + /// constant — the engine's serial TBDR blend caps at `kGaussianMaxBlendedSplatsPerPixel` + /// (64) per pixel, but sustained GPU frame-time overrun (the actual failure mode this + /// guards against) was observed well below that cap. Tune on-device by watching GPU frame + /// time while varying this value; only takes effect for LOD levels with a non-nil + /// `GaussianLODLevel.meanSquaredSplatExtent` supplied. + public var gaussianOverdrawBudget: Float = 12.0 } diff --git a/Sources/UntoldEngine/Systems/LODSystem.swift b/Sources/UntoldEngine/Systems/LODSystem.swift index 1591a73f..65066e0c 100644 --- a/Sources/UntoldEngine/Systems/LODSystem.swift +++ b/Sources/UntoldEngine/Systems/LODSystem.swift @@ -115,7 +115,7 @@ public class LODSystem: @unchecked Sendable { } // Calculate distance - let distance = calculateDistance(entityId: entityId, cameraPosition: cameraPosition) + let distance = entityDistanceToCamera(entityId: entityId, cameraPosition: cameraPosition) // Select desired LOD level based on distance let desiredLOD = selectLODLevel( @@ -161,58 +161,16 @@ public class LODSystem: @unchecked Sendable { return lodComponent.currentLOD } - private func calculateDistance(entityId: EntityID, cameraPosition: simd_float3) -> Float { - guard let worldTransform = scene.get(component: WorldTransformComponent.self, for: entityId), - let localTransform = scene.get(component: LocalTransformComponent.self, for: entityId) - else { return 0.0 } - - // Get entity center from AABB - let boundingBox = localTransform.boundingBox - let localCenter = (boundingBox.min + boundingBox.max) * 0.5 - - // Transform to world space - let worldCenter = worldTransform.space * simd_float4(localCenter, 1.0) - - // Calculate distance from camera to entity center - return simd_distance(cameraPosition, simd_float3(worldCenter.x, worldCenter.y, worldCenter.z)) - } - private func selectLODLevel(distance: Float, lodComponent: LODComponent, currentLOD: Int) -> Int { - // Check for forced LOD override - if let forced = lodComponent.forcedLOD, forced >= 0 { - return min(forced, lodComponent.lodLevels.count - 1) - } - - // Apply LOD bias - let adjustedDistance = distance * LODConfig.shared.lodBias - let globalDistances = LODConfig.shared.lodDistances - - // Find appropriate LOD level. - // Per-level maxDistance takes priority; fall back to LODConfig.lodDistances[index] - // when maxDistance is 0 (unset). This lets users configure distances either - // per-level at registration time OR globally via LODConfig. - for (index, lodLevel) in lodComponent.lodLevels.enumerated() { - let baseThreshold: Float - if lodLevel.maxDistance > 0 { - baseThreshold = lodLevel.maxDistance - } else if index < globalDistances.count { - baseThreshold = globalDistances[index] - } else { - continue // No threshold available for this level — skip - } - - // Apply hysteresis when switching to higher detail (prevents flickering) - let threshold = index < currentLOD - ? baseThreshold - LODConfig.shared.hysteresis - : baseThreshold - - if adjustedDistance <= threshold { - return index - } - } - - // Beyond all thresholds, use lowest LOD - return lodComponent.lodLevels.count - 1 + selectLODIndex( + levels: lodComponent.lodLevels, + distance: distance, + currentLOD: currentLOD, + forcedLOD: lodComponent.forcedLOD, + lodBias: LODConfig.shared.lodBias, + hysteresis: LODConfig.shared.hysteresis, + globalDistances: LODConfig.shared.lodDistances + ) } private func applyLOD(entityId: EntityID, newLOD: Int, deltaTime _: Float) { @@ -304,3 +262,77 @@ func shouldDeferLODSelectionDuringTransition( ) -> Bool { fadeTransitionsEnabled && previousLOD != nil } + +/// Shared by `LODComponent` and `GaussianLODComponent` — see `LODResidencyLevel`. +func isLODLevelResident(_ levels: [some LODResidencyLevel], _ index: Int) -> Bool { + guard index >= 0, index < levels.count else { return false } + let level = levels[index] + return level.residencyState == .resident && level.isPopulated +} + +/// Shared by `LODComponent` and `GaussianLODComponent` — see `LODResidencyLevel`. Prefers a +/// coarser resident level (higher index, lower detail) over a finer one, since showing +/// something-but-coarser while the desired level streams in beats swapping to a different +/// detail level entirely. +func findFallbackLODLevel(_ levels: [some LODResidencyLevel], from desiredIndex: Int) -> Int? { + for i in (desiredIndex + 1) ..< levels.count where isLODLevelResident(levels, i) { + return i + } + for i in (0 ..< desiredIndex).reversed() where isLODLevelResident(levels, i) { + return i + } + return nil +} + +/// Shared by `LODSystem` and `GaussianLODSystem` — distance from the camera to an entity's +/// local-space bounding-box center, in world space. +func entityDistanceToCamera(entityId: EntityID, cameraPosition: simd_float3) -> Float { + guard let worldTransform = scene.get(component: WorldTransformComponent.self, for: entityId), + let localTransform = scene.get(component: LocalTransformComponent.self, for: entityId) + else { return 0.0 } + + let boundingBox = localTransform.boundingBox + let localCenter = (boundingBox.min + boundingBox.max) * 0.5 + let worldCenter = worldTransform.space * simd_float4(localCenter, 1.0) + return simd_distance(cameraPosition, simd_float3(worldCenter.x, worldCenter.y, worldCenter.z)) +} + +/// Shared by `LODSystem.selectLODLevel` and `GaussianLODSystem.selectDesiredLOD` — walks +/// `levels` in distance order and returns the first whose threshold isn't yet exceeded by +/// `distance * lodBias`. Applies `hysteresis` when a candidate level would mean switching to +/// higher detail (lower index) than `currentLOD`, so a distance oscillating right at a +/// threshold doesn't flip the selection every re-evaluation. Falls back to +/// `globalDistances[index]` when a level's own `maxDistance` is unset (0). +func selectLODIndex( + levels: [some LODDistanceLevel], + distance: Float, + currentLOD: Int, + forcedLOD: Int?, + lodBias: Float, + hysteresis: Float, + globalDistances: [Float] +) -> Int { + if let forced = forcedLOD, forced >= 0 { + return min(forced, levels.count - 1) + } + + let adjustedDistance = distance * lodBias + + for (index, level) in levels.enumerated() { + let baseThreshold: Float + if level.maxDistance > 0 { + baseThreshold = level.maxDistance + } else if index < globalDistances.count { + baseThreshold = globalDistances[index] + } else { + continue + } + + let threshold = index < currentLOD ? baseThreshold - hysteresis : baseThreshold + if adjustedDistance <= threshold { + return index + } + } + + return levels.count - 1 +} diff --git a/Sources/UntoldEngine/Systems/RegistrationSystem.swift b/Sources/UntoldEngine/Systems/RegistrationSystem.swift index dbcfbbaf..93530c74 100644 --- a/Sources/UntoldEngine/Systems/RegistrationSystem.swift +++ b/Sources/UntoldEngine/Systems/RegistrationSystem.swift @@ -174,6 +174,10 @@ private func registerComponentCleanupHandlers() { removeEntityLOD(entityId: entityId) } + ComponentRegistry.register(componentType: GaussianLODComponent.self, handlerId: "gaussianLOD", priority: 30) { entityId in + removeEntityGaussianLOD(entityId: entityId) + } + ComponentRegistry.register(componentType: GaussianComponent.self, handlerId: "gaussian", priority: 30) { entityId in removeEntityGaussian(entityId: entityId) } @@ -3198,7 +3202,7 @@ public func loadRawMesh( /// Built Metal resources for a parsed Gaussian splat asset, ready to attach to an entity. /// Shared by `setEntityGaussian` (synchronous) and `setEntityGaussianAsync` (off-thread) so /// there is a single implementation of the PLY-parse/buffer-build/SH-pack pipeline. -private struct GaussianLoadResult { +struct GaussianLoadResult { let splatCount: UInt // One buffer per in-flight frame slot (see the comment on GaussianComponent's matching // fields) — written fresh every frame by the cull/depth-key/radix-sort passes, so a @@ -3216,35 +3220,242 @@ private struct GaussianLoadResult { let spaceUniform: [MTLBuffer?] /// Sum of all GPU buffer bytes above, for `MemoryBudgetManager` registration. let estimatedGPUBytes: Int + /// Local-space bounding box computed from the actual loaded splat positions, for + /// `LocalTransformComponent.boundingBox` — see `computeGaussianSplatBoundingBox`. + let boundingBox: (min: simd_float3, max: simd_float3) +} + +public enum UntoldGSError: Error, CustomStringConvertible { + case badMagic + case unsupportedVersion(UInt32) + case truncated + case sizeMismatch(String) + + public var description: String { + switch self { + case .badMagic: "Not an Untold Gaussian splat file" + case let .unsupportedVersion(version): "Unsupported Untold Gaussian splat version \(version)" + case .truncated: "Untold Gaussian splat file is truncated" + case let .sizeMismatch(reason): "Untold Gaussian splat size mismatch: \(reason)" + } + } } -/// Reads a `.ply` Gaussian splat asset from disk and builds its GPU buffers. -/// Returns `nil` on any failure, calling `handleError` internally — callers just guard-return. -private func buildGaussianLoadResult(filename: String, withExtension: String) -> GaussianLoadResult? { - guard let url = LoadingSystem.shared.resourceURL(forResource: filename, withExtension: withExtension, subResource: nil) else { - handleError(.filenameNotFound, filename) - return nil +public struct UntoldGSAsset { + public let encodedSplats: [EncodedGaussianSplat] + public let shCoefficients: [UInt8] + public let shMetadata: GaussianSHMetadata? + /// Mean of this tier's splats' squared major-axis extent, baked in by + /// `bakeGaussianSplatProgressiveTiers` — see `estimatedGaussianOverdraw`. 0 for files + /// baked before this field existed (indistinguishable from a real 0, but a real 0 can only + /// happen for a tier with no splats, which never gets written). + public let meanSquaredSplatExtent: Float + /// Asset-level local-space bounding box (shared by every tier of the same bake, not + /// per-tier — see `bakeGaussianSplatProgressiveTiers`), baked in at version 2. Lets any + /// registration path — including streaming, which needs a real box before it can decide + /// whether to load anything — read a real box via `UntoldGSFormat.readHeader` without a + /// caller-supplied value. + public let boundingBoxMin: simd_float3 + public let boundingBoxMax: simd_float3 + + public var splatCount: Int { + encodedSplats.count + } +} + +public enum UntoldGSFormat { + private static let magic: UInt32 = 0x5347_5455 // "UTGS" + // v2 appended boundingBoxMin/boundingBoxMax (24 bytes) after the v1 header — every v1 field + // offset is unchanged. No dual-version reader: .untoldgs is a regeneratable cache of the + // source .ply, not hand-authored data, so a version bump just means "re-bake," the same way + // an EncodedGaussianSplat layout change already does. + private static let version: UInt32 = 2 + private static let headerByteCount = 72 + + public static func write( + encodedSplats: [EncodedGaussianSplat], + sphericalHarmonics: PackedGaussianSphericalHarmonics?, + meanSquaredSplatExtent: Float = 0, + boundingBoxMin: simd_float3, + boundingBoxMax: simd_float3, + to url: URL + ) throws { + var data = Data() + appendUInt32(magic, to: &data) + appendUInt32(version, to: &data) + appendUInt64(UInt64(encodedSplats.count), to: &data) + appendUInt32(sphericalHarmonics?.metadata.degree ?? 0, to: &data) + appendUInt32(sphericalHarmonics?.metadata.coefficientsPerChannel ?? 0, to: &data) + appendUInt32(sphericalHarmonics?.metadata.higherOrderCoefficientsPerSplat ?? 0, to: &data) + appendFloat(meanSquaredSplatExtent, to: &data) + appendUInt64(UInt64(encodedSplats.count * MemoryLayout.stride), to: &data) + appendUInt64(UInt64(sphericalHarmonics?.coefficients.count ?? 0), to: &data) + appendFloat(boundingBoxMin.x, to: &data) + appendFloat(boundingBoxMin.y, to: &data) + appendFloat(boundingBoxMin.z, to: &data) + appendFloat(boundingBoxMax.x, to: &data) + appendFloat(boundingBoxMax.y, to: &data) + appendFloat(boundingBoxMax.z, to: &data) + + encodedSplats.withUnsafeBytes { data.append(contentsOf: $0) } + if let sphericalHarmonics { + data.append(contentsOf: sphericalHarmonics.coefficients) + } + + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try data.write(to: url, options: .atomic) + } + + /// Check just enough to identify the version before checking the full v2 header length — an + /// old, valid-but-shorter v1 file (48 bytes) must report .unsupportedVersion (a clear + /// "re-bake me" signal), not .truncated, which would otherwise fire first purely because + /// it's shorter than the current header size. Shared by read() and readHeader() so both + /// report the same error for the same malformed input. + private static func validateMagicAndVersion(_ data: Data) throws { + guard data.count >= 8 else { throw UntoldGSError.truncated } + let magicValue = readUInt32(data, at: 0) + guard magicValue == magic else { throw UntoldGSError.badMagic } + let versionValue = readUInt32(data, at: 4) + guard versionValue == version else { throw UntoldGSError.unsupportedVersion(versionValue) } + } + + /// Reads only `boundingBoxMin`/`boundingBoxMax` from the fixed-size header via a bounded + /// `FileHandle` read — not `Data(contentsOf:)`, which would pull the entire (potentially + /// multi-megabyte) splat/SH payload into memory just to look at 24 header bytes. Lets + /// registration paths (including streaming, which needs a real box before it can decide + /// whether to load anything) get one synchronously without a caller-supplied value. + public static func readHeader(from url: URL) throws -> (boundingBoxMin: simd_float3, boundingBoxMax: simd_float3) { + guard let fileHandle = FileHandle(forReadingAtPath: url.path) else { + throw UntoldGSError.truncated + } + defer { try? fileHandle.close() } + + let data = try (fileHandle.read(upToCount: headerByteCount)) ?? Data() + try validateMagicAndVersion(data) + guard data.count >= headerByteCount else { throw UntoldGSError.truncated } + + let boundingBoxMin = simd_float3(readFloat(data, at: 48), readFloat(data, at: 52), readFloat(data, at: 56)) + let boundingBoxMax = simd_float3(readFloat(data, at: 60), readFloat(data, at: 64), readFloat(data, at: 68)) + return (boundingBoxMin, boundingBoxMax) + } + + public static func read(from url: URL) throws -> UntoldGSAsset { + let data = try Data(contentsOf: url) + try validateMagicAndVersion(data) + guard data.count >= headerByteCount else { throw UntoldGSError.truncated } + + let splatCountRaw = readUInt64(data, at: 8) + let shDegree = readUInt32(data, at: 16) + let shCoefficientsPerChannel = readUInt32(data, at: 20) + let shHigherOrderPerSplat = readUInt32(data, at: 24) + let meanSquaredSplatExtent = readFloat(data, at: 28) + let encodedByteCountRaw = readUInt64(data, at: 32) + let shByteCountRaw = readUInt64(data, at: 40) + let boundingBoxMin = simd_float3(readFloat(data, at: 48), readFloat(data, at: 52), readFloat(data, at: 56)) + let boundingBoxMax = simd_float3(readFloat(data, at: 60), readFloat(data, at: 64), readFloat(data, at: 68)) + + // Validate every header-declared count against the actual file size using + // overflow-checked UInt64 arithmetic before converting anything to Int — a corrupt or + // malicious header can declare values that overflow a plain multiply/add or don't fit + // Int, and an unchecked Int(...) conversion would trap the process instead of throwing + // a catchable UntoldGSError. + let stride = UInt64(MemoryLayout.stride) + let (expectedEncodedBytes, splatByteOverflow) = splatCountRaw.multipliedReportingOverflow(by: stride) + guard !splatByteOverflow, encodedByteCountRaw == expectedEncodedBytes else { + throw UntoldGSError.sizeMismatch("encoded splat bytes \(encodedByteCountRaw), expected \(expectedEncodedBytes)") + } + + let (headerPlusEncoded, headerOverflow) = UInt64(headerByteCount).addingReportingOverflow(encodedByteCountRaw) + let (totalExpectedBytes, totalOverflow) = headerPlusEncoded.addingReportingOverflow(shByteCountRaw) + guard !headerOverflow, !totalOverflow, UInt64(data.count) == totalExpectedBytes else { + throw UntoldGSError.sizeMismatch("file has \(data.count) bytes, expected \(totalExpectedBytes)") + } + + // Both counts are now provably <= data.count (a valid Int), so these conversions + // cannot trap. + guard let encodedByteCount = Int(exactly: encodedByteCountRaw), + let shByteCount = Int(exactly: shByteCountRaw) + else { + throw UntoldGSError.sizeMismatch("header-declared byte counts do not fit in memory") + } + + let encodedStart = headerByteCount + let encodedEnd = encodedStart + encodedByteCount + let encodedSplats = data[encodedStart ..< encodedEnd].withUnsafeBytes { rawBuffer in + Array(rawBuffer.bindMemory(to: EncodedGaussianSplat.self)) + } + + let shStart = encodedEnd + let shCoefficients = shByteCount > 0 ? Array(data[shStart ..< shStart + shByteCount]) : [] + let shMetadata: GaussianSHMetadata? = shByteCount > 0 + ? GaussianSHMetadata( + degree: shDegree, + coefficientsPerChannel: shCoefficientsPerChannel, + higherOrderCoefficientsPerSplat: shHigherOrderPerSplat, + _pad0: 0 + ) + : nil + + return UntoldGSAsset( + encodedSplats: encodedSplats, + shCoefficients: shCoefficients, + shMetadata: shMetadata, + meanSquaredSplatExtent: meanSquaredSplatExtent, + boundingBoxMin: boundingBoxMin, + boundingBoxMax: boundingBoxMax + ) } - // Attempt to read Gaussian splats, handling errors internally - let asset: GaussianSplatAsset - do { - asset = try PLYReader.readGaussianAsset(from: url) - } catch { - handleError(.assetDataMissing, "Failed to read Gaussian splats from \(filename): \(error.localizedDescription)") - return nil + private static func appendUInt32(_ value: UInt32, to data: inout Data) { + var littleEndian = value.littleEndian + withUnsafeBytes(of: &littleEndian) { data.append(contentsOf: $0) } + } + + private static func appendFloat(_ value: Float, to data: inout Data) { + appendUInt32(value.bitPattern, to: &data) + } + + private static func readFloat(_ data: Data, at offset: Int) -> Float { + Float(bitPattern: readUInt32(data, at: offset)) } - let splats = asset.splats - // Check if we exceed the buffer capacity - guard splats.count <= Int(maxNumOfGaussians) else { - handleError(.bufferAllocationFailed, "Too many Gaussian splats: \(splats.count) exceeds maximum \(maxNumOfGaussians)") + private static func appendUInt64(_ value: UInt64, to data: inout Data) { + var littleEndian = value.littleEndian + withUnsafeBytes(of: &littleEndian) { data.append(contentsOf: $0) } + } + + private static func readUInt32(_ data: Data, at offset: Int) -> UInt32 { + data.withUnsafeBytes { rawBuffer in + UInt32(littleEndian: rawBuffer.loadUnaligned(fromByteOffset: offset, as: UInt32.self)) + } + } + + private static func readUInt64(_ data: Data, at offset: Int) -> UInt64 { + data.withUnsafeBytes { rawBuffer in + UInt64(littleEndian: rawBuffer.loadUnaligned(fromByteOffset: offset, as: UInt64.self)) + } + } +} + +/// Builds GPU buffers from already-encoded Gaussian splat data. +/// Returns `nil` on any failure, calling `handleError` internally — callers just guard-return. +func buildGaussianLoadResult( + encodedSplats: [EncodedGaussianSplat], + packedSphericalHarmonics: PackedGaussianSphericalHarmonics?, + meanSquaredSplatExtent: Float = 0, + sourceDescription: String +) -> GaussianLoadResult? { + guard encodedSplats.count <= Int(maxNumOfGaussians) else { + handleError(.bufferAllocationFailed, "Too many Gaussian splats: \(encodedSplats.count) exceeds maximum \(maxNumOfGaussians)") return nil } - let splatCount = UInt(splats.count) + let splatCount = UInt(encodedSplats.count) guard splatCount > 0 else { - handleError(.assetDataMissing, "Gaussian splat file contains no vertices: \(filename)") + handleError(.assetDataMissing, "Gaussian splat file contains no vertices: \(sourceDescription)") return nil } @@ -3286,13 +3497,8 @@ private func buildGaussianLoadResult(filename: String, withExtension: String) -> return nil } - let encodedPointer = encodedSplatBuffer.contents().bindMemory( - to: EncodedGaussianSplat.self, - capacity: splats.count - ) - - for (index, splat) in splats.enumerated() { - encodedPointer[index] = encodeGaussianSplatForTBDR(splat) + encodedSplats.withUnsafeBytes { bytes in + encodedSplatBuffer.contents().copyMemory(from: bytes.baseAddress!, byteCount: bytes.count) } var gaussianPrecomputedData: [MTLBuffer] = [] @@ -3307,21 +3513,11 @@ private func buildGaussianLoadResult(filename: String, withExtension: String) -> gaussianPrecomputedData.append(precomputedSlot) } - let packedSphericalHarmonics: PackedGaussianSphericalHarmonics? - do { - packedSphericalHarmonics = try asset.sphericalHarmonics.map { - try packGaussianSphericalHarmonics($0, splatCount: splats.count) - } - } catch { - handleError(.assetDataMissing, "Failed to pack spherical harmonics from \(filename): \(error.localizedDescription)") - return nil - } - let sphericalHarmonicsBuffer: MTLBuffer? if let packedSphericalHarmonics, !packedSphericalHarmonics.coefficients.isEmpty { sphericalHarmonicsBuffer = renderInfo.device.makeBuffer( bytes: packedSphericalHarmonics.coefficients, - length: packedSphericalHarmonics.coefficients.count * MemoryLayout.stride, + length: packedSphericalHarmonics.coefficients.count * MemoryLayout.stride, options: .storageModeShared ) guard sphericalHarmonicsBuffer != nil else { @@ -3367,10 +3563,125 @@ private func buildGaussianLoadResult(filename: String, withExtension: String) -> sphericalHarmonicsBuffer: sphericalHarmonicsBuffer, sphericalHarmonicsMetadata: packedSphericalHarmonics?.metadata, spaceUniform: spaceUniform, - estimatedGPUBytes: estimatedGPUBytes + estimatedGPUBytes: estimatedGPUBytes, + // Splat centers alone under-size the true silhouette wherever a large-scale splat sits + // near the edge — pad uniformly by an approximate per-splat radius derived from the + // tier's mean squared extent (sqrt of the mean of major-axis², i.e. an RMS radius), + // since only splat centers/positions (not per-splat scale) are available post-encode. + boundingBox: computeGaussianSplatPositionBoundingBox( + encodedSplats.map(\.position), + padding: meanSquaredSplatExtent > 0 ? sqrt(meanSquaredSplatExtent) : 0 + ) ) } +/// Min/max bounding box over a set of local-space splat positions, expanded by `padding` in +/// every direction (see call site doc for why: splat centers alone under-size the true +/// silhouette). Mirrors `Mesh.computeMeshBoundingBox`'s shape/purpose for the mesh path. +func computeGaussianSplatPositionBoundingBox(_ positions: [simd_float3], padding: Float = 0) -> (min: simd_float3, max: simd_float3) { + guard !positions.isEmpty else { return (min: .zero, max: .zero) } + var boundsMin = simd_float3(repeating: .infinity) + var boundsMax = simd_float3(repeating: -.infinity) + for position in positions { + boundsMin = simd_min(boundsMin, position) + boundsMax = simd_max(boundsMax, position) + } + let paddingVector = simd_float3(repeating: padding) + return (min: boundsMin - paddingVector, max: boundsMax + paddingVector) +} + +/// Min/max bounding box over a set of source `GaussianSplat`s (bake-time, pre-encode form), +/// expanded per-splat by its own major-axis extent (`gaussianMajorAxis`) rather than just its +/// center — a splat whose center sits near the silhouette boundary but has a large individual +/// scale visually extends past a centers-only box. Used for the asset-level box in +/// `bakeGaussianSplatProgressiveTiers`, where the position hasn't been encoded into +/// `EncodedGaussianSplat` yet and real per-splat scale is still available. +func computeGaussianSplatBoundingBox(_ splats: [GaussianSplat]) -> (min: simd_float3, max: simd_float3) { + guard !splats.isEmpty else { return (min: .zero, max: .zero) } + var boundsMin = simd_float3(repeating: .infinity) + var boundsMax = simd_float3(repeating: -.infinity) + for splat in splats { + let center = simd_float3(splat.center.x, splat.center.y, splat.center.z) + let radius = simd_float3(repeating: gaussianMajorAxis(splat)) + boundsMin = simd_min(boundsMin, center - radius) + boundsMax = simd_max(boundsMax, center + radius) + } + return (min: boundsMin, max: boundsMax) +} + +/// Reads a `.ply` Gaussian splat asset from disk and builds its GPU buffers. +/// Returns `nil` on any failure, calling `handleError` internally — callers just guard-return. +private func buildGaussianLoadResult(filename: String, withExtension: String) -> GaussianLoadResult? { + guard let url = LoadingSystem.shared.resourceURL(forResource: filename, withExtension: withExtension, subResource: nil) else { + handleError(.filenameNotFound, filename) + return nil + } + + do { + return try buildGaussianLoadResultFromPLY(url: url, sourceDescription: filename) + } catch { + handleError(.assetDataMissing, "Failed to read Gaussian splats from \(filename): \(error.localizedDescription)") + return nil + } +} + +func buildGaussianLoadResultFromPLY(url: URL, sourceDescription: String) throws -> GaussianLoadResult? { + let asset = try PLYReader.readGaussianAsset(from: url) + let encodedSplats = asset.splats.map(encodeGaussianSplatForTBDR) + + // Reported here, distinctly from a raw PLY-parse failure (which propagates to the + // caller's catch instead), so a debugger sees "failed to pack SH" rather than a generic + // "failed to read" message when the file parses fine but has bad SH data (e.g. NaN). + let packedSphericalHarmonics: PackedGaussianSphericalHarmonics? + do { + packedSphericalHarmonics = try asset.sphericalHarmonics.map { + try packGaussianSphericalHarmonics($0, splatCount: asset.splats.count) + } + } catch { + handleError(.assetDataMissing, "Failed to pack spherical harmonics from \(sourceDescription): \(error.localizedDescription)") + return nil + } + + return buildGaussianLoadResult( + encodedSplats: encodedSplats, + packedSphericalHarmonics: packedSphericalHarmonics, + meanSquaredSplatExtent: meanSquaredSplatExtent(asset.splats, keeping: Array(asset.splats.indices)), + sourceDescription: sourceDescription + ) +} + +func buildGaussianComponentFromUntoldGS(url: URL) -> ( + component: GaussianComponent, + estimatedGPUBytes: Int, + meanSquaredSplatExtent: Float, + boundingBox: (min: simd_float3, max: simd_float3) +)? { + let asset: UntoldGSAsset + do { + asset = try UntoldGSFormat.read(from: url) + } catch { + handleError(.assetDataMissing, "Failed to read .untoldgs Gaussian tier from \(url.lastPathComponent): \(error)") + return nil + } + + let packedSphericalHarmonics = asset.shMetadata.map { + PackedGaussianSphericalHarmonics(coefficients: asset.shCoefficients, metadata: $0) + } + + guard let result = buildGaussianLoadResult( + encodedSplats: asset.encodedSplats, + packedSphericalHarmonics: packedSphericalHarmonics, + meanSquaredSplatExtent: asset.meanSquaredSplatExtent, + sourceDescription: url.lastPathComponent + ) else { + return nil + } + + let component = GaussianComponent() + copyGaussianLoadResult(result, to: component) + return (component, result.estimatedGPUBytes, asset.meanSquaredSplatExtent, result.boundingBox) +} + /// Registers `GaussianComponent` on `entityId` from a built `GaussianLoadResult` and records /// its GPU footprint with `MemoryBudgetManager`. Must be called from within a world-mutation /// gate (`withWorldMutationGate`). @@ -3382,6 +3693,15 @@ private func applyGaussianLoadResult(_ result: GaussianLoadResult, to entityId: return } + copyGaussianLoadResult(result, to: gaussianComponent) + MemoryBudgetManager.shared.registerMesh(entityId: entityId, meshSizeBytes: result.estimatedGPUBytes) + + if let localTransform = scene.get(component: LocalTransformComponent.self, for: entityId) { + localTransform.boundingBox = result.boundingBox + } +} + +func copyGaussianLoadResult(_ result: GaussianLoadResult, to gaussianComponent: GaussianComponent) { gaussianComponent.splatCount = result.splatCount gaussianComponent.visibleSplatCountForRendering = result.splatCount gaussianComponent.gaussianSortedIndices = result.gaussianSortedIndices.map { $0 as MTLBuffer? } @@ -3392,10 +3712,28 @@ private func applyGaussianLoadResult(_ result: GaussianLoadResult, to entityId: gaussianComponent.sphericalHarmonicsData = result.sphericalHarmonicsBuffer gaussianComponent.sphericalHarmonicsMetadata = result.sphericalHarmonicsMetadata gaussianComponent.spaceUniform = result.spaceUniform +} - MemoryBudgetManager.shared.registerMesh(entityId: entityId, meshSizeBytes: result.estimatedGPUBytes) +public enum GaussianSource { + case single(filename: String, withExtension: String) + /// No `boundingBoxHalfExtent` here: both `setEntityGaussian(source:)` and + /// `setEntityGaussianStreaming(source:options:)` can read a real box baked into the + /// `.untoldgs` header itself (see `UntoldGSFormat.readHeader`) — an explicit override, when + /// one is genuinely needed, is a parameter on the underlying registration path instead + /// (`GaussianStreamingOptions.boundingBoxHalfExtent` for streaming). Overdraw-aware LOD + /// stats (`meanSquaredSplatExtent`) are baked directly into each `.untoldgs` tier by + /// `bakeGaussianSplatProgressiveTiers` and read automatically when a tier loads — nothing to + /// pass here either. + case progressive( + baseFilename: String, + withExtension: String = "untoldgs", + levelCount: Int, + maxDistances: [Float] + ) } +public typealias GaussianStreamingSource = GaussianSource + public func setEntityGaussian(entityId: EntityID, filename: String, withExtension: String) { guard let result = buildGaussianLoadResult(filename: filename, withExtension: withExtension) else { return @@ -3406,6 +3744,25 @@ public func setEntityGaussian(entityId: EntityID, filename: String, withExtensio } } +/// Registers a Gaussian splat entity that is always present, either as one whole asset or +/// as a progressive multi-tier `.untoldgs` asset. Progressive entities do not require a +/// streamed tile scene: the coarsest tier is loaded immediately, then `GaussianLODSystem` +/// requests finer tiers based on camera distance. +public func setEntityGaussian(entityId: EntityID, source: GaussianSource) { + switch source { + case let .single(filename, ext): + setEntityGaussian(entityId: entityId, filename: filename, withExtension: ext) + case let .progressive(baseFilename, ext, levelCount, maxDistances): + setEntityGaussianProgressive( + entityId: entityId, + baseFilename: baseFilename, + withExtension: ext, + levelCount: levelCount, + maxDistances: maxDistances + ) + } +} + /// Asynchronously reads and encodes a `.ply` Gaussian splat asset and attaches it to /// `entityId` without blocking the main thread. Parsing, per-splat encoding, and spherical- /// harmonics packing all run before the world-mutation gate is acquired; only the final @@ -3438,6 +3795,117 @@ public func setEntityGaussianAsync( return true } +public struct GaussianStreamingOptions { + public var streamingRadius: Float + public var unloadRadius: Float + /// Explicit override for the entity's local-space bounding box. Optional: `.untoldgs` + /// sources (single-file or progressive) can read a real box baked into the file itself — + /// see `UntoldGSFormat.readHeader` — so this is only required for a raw `.ply` `.single` + /// source, which has no baked box to fall back to. + public var boundingBoxHalfExtent: simd_float3? + public var priority: Int + + public init( + streamingRadius: Float = 100.0, + unloadRadius: Float = 150.0, + boundingBoxHalfExtent: simd_float3? = nil, + priority: Int = 0 + ) { + self.streamingRadius = streamingRadius + self.unloadRadius = unloadRadius + self.boundingBoxHalfExtent = boundingBoxHalfExtent + self.priority = priority + } +} + +/// Registers a distance-streamed Gaussian splat entity, either as one whole asset or as a +/// progressive multi-tier asset. Prefer this API for new call sites. +public func setEntityGaussianStreaming( + entityId: EntityID, + source: GaussianSource, + options: GaussianStreamingOptions +) { + switch source { + case let .single(filename, ext): + setEntityGaussianStreamable( + entityId: entityId, + filename: filename, + withExtension: ext, + streamingRadius: options.streamingRadius, + unloadRadius: options.unloadRadius, + boundingBoxHalfExtent: options.boundingBoxHalfExtent, + priority: options.priority + ) + case let .progressive(baseFilename, ext, levelCount, maxDistances): + setEntityGaussianProgressiveStreamable( + entityId: entityId, + baseFilename: baseFilename, + withExtension: ext, + levelCount: levelCount, + maxDistances: maxDistances, + streamingRadius: options.streamingRadius, + unloadRadius: options.unloadRadius, + boundingBoxHalfExtent: options.boundingBoxHalfExtent, + priority: options.priority + ) + } +} + +/// Registers a tile-independent progressive Gaussian splat entity. Not part of the public API +/// — reached only through `setEntityGaussian(entityId:source:)`'s `.progressive` case, which is +/// the entry point callers should use. +/// +/// The expected files are `_lod0.untoldgs`, `_lod1.untoldgs`, +/// etc. LOD0 is full detail; the highest index is the coarsest tier and is loaded +/// immediately so the entity can become visible before finer tiers finish loading. +func setEntityGaussianProgressive( + entityId: EntityID, + baseFilename: String, + withExtension ext: String = "untoldgs", + levelCount: Int, + maxDistances: [Float] +) { + guard configureEntityGaussianProgressiveLOD( + entityId: entityId, + baseFilename: baseFilename, + withExtension: ext, + levelCount: levelCount, + maxDistances: maxDistances, + errorPrefix: "setEntityGaussianProgressive" + ) else { return } + + // Read the box baked into the coarsest tier's header (cheap, synchronous) so the entity has + // a real box from frame 1 instead of waiting on the async coarsest-tier load below to + // populate one via GeometryStreamingSystem+GaussianStreaming.swift's hasExplicitBoundingBox + // fallback. If unavailable (e.g. a pre-v2 .untoldgs file), that fallback still applies + // unchanged. No caller-supplied override here — GaussianSource.progressive has no + // boundingBoxHalfExtent of its own to forward (see its doc comment). + let coarsestTierURL = gaussianProgressiveTierURL(baseFilename: baseFilename, withExtension: ext, levelCount: levelCount, tierIndex: levelCount - 1) + if let box = resolveGaussianBoundingBox(override: nil, untoldgsURL: coarsestTierURL), + let local = scene.get(component: LocalTransformComponent.self, for: entityId) + { + local.boundingBox = box + scene.get(component: GaussianLODComponent.self, for: entityId)?.hasExplicitBoundingBox = true + } + + // Store the load on the coarsest tier's loadTask, mirroring requestGaussianLODLevelLoad's + // pattern, so removeEntityGaussianLOD can cancel it if entityId is destroyed while this is + // still in flight. The assignment happens inside the same gate that spawns the Task: since + // the Task's own body needs this same lock to touch scene state, it can't run ahead and + // clear loadTask before this closure finishes assigning it. + withWorldMutationGate { + guard let lod = scene.get(component: GaussianLODComponent.self, for: entityId), + !lod.lodLevels.isEmpty + else { return } + + let coarsestIndex = lod.lodLevels.count - 1 + let task = Task { + _ = await GeometryStreamingSystem.shared.loadInitialGaussianProgressiveTier(entityId: entityId) + } + lod.lodLevels[coarsestIndex].loadTask = task + } +} + /// Registers `entityId` as a distance-streamed Gaussian-splat prop, so /// `GeometryStreamingSystem` loads/unloads it based on camera distance the same way it /// does the surrounding tile geometry — rather than loading it immediately the way @@ -3448,18 +3916,25 @@ public func setEntityGaussianAsync( /// which tile it belongs to, via `findTileEntity(containing:)`. If no tile is found there, /// this logs a warning and leaves `entityId` as a plain, non-streaming entity. /// -/// `boundingBoxHalfExtent` has no default on purpose: `GeometryStreamingSystem`'s frustum -/// gate needs a real local-space volume on the entity before it ever loads (the splat's -/// true extent isn't known until the asset is parsed). A zero-size placeholder collapses -/// the gate to a single exact point, making re-streaming unreliable once the camera moves -/// away and back — pick a box roughly matching the prop's real-world size. -public func setEntityGaussianStreamable( +/// `boundingBoxHalfExtent` is optional: a `.untoldgs` source has a real box baked into its +/// header (see `UntoldGSFormat.readHeader`), read synchronously here since +/// `GeometryStreamingSystem`'s frustum gate needs a real local-space volume on the entity +/// before it ever loads. A raw `.ply` source has no baked box, so an explicit value is still +/// required there — omitting it leaves the entity non-streaming rather than registering a +/// zero-size placeholder, which would collapse the gate to a single exact point and make +/// re-streaming unreliable once the camera moves away and back. +/// +/// Not part of the public API — reached only through +/// `setEntityGaussianStreaming(entityId:source:options:)`'s `.single` case, which forwards +/// `GaussianStreamingOptions.boundingBoxHalfExtent` here; that's the entry point callers should +/// use. +func setEntityGaussianStreamable( entityId: EntityID, filename: String, withExtension ext: String, streamingRadius: Float = 100.0, unloadRadius: Float = 150.0, - boundingBoxHalfExtent: simd_float3, + boundingBoxHalfExtent: simd_float3? = nil, priority: Int = 0 ) { guard let local = scene.get(component: LocalTransformComponent.self, for: entityId) else { @@ -3472,7 +3947,14 @@ public func setEntityGaussianStreamable( return } - local.boundingBox = (min: -boundingBoxHalfExtent, max: boundingBoxHalfExtent) + let untoldgsURL = ext.lowercased() == "untoldgs" + ? LoadingSystem.shared.resourceURL(forResource: filename, withExtension: ext, subResource: nil) + : nil + guard let box = resolveGaussianBoundingBox(override: boundingBoxHalfExtent, untoldgsURL: untoldgsURL) else { + Logger.logWarning(message: "[RegistrationSystem] setEntityGaussianStreamable: no boundingBoxHalfExtent supplied and no baked box available for '\(filename).\(ext)' — entity left non-streaming. A raw .ply source requires an explicit boundingBoxHalfExtent.") + return + } + local.boundingBox = box setParent(childId: entityId, parentId: tileEntity) OctreeSystem.shared.registerEntity(entityId) @@ -3487,13 +3969,443 @@ public func setEntityGaussianStreamable( } } -struct PackedGaussianSphericalHarmonics { - let coefficients: [Float16] - let metadata: GaussianSHMetadata +/// Registers `entityId` as a distance-streamed progressive Gaussian splat prop. +/// +/// The expected files are `_lod0.untoldgs`, `_lod1.untoldgs`, +/// etc. LOD0 is full detail; the highest index is the coarsest tier and is loaded first +/// when the prop enters streaming range. Finer tiers are requested later by +/// `GaussianLODSystem` based on camera distance. +/// +/// Not part of the public API — reached only through +/// `setEntityGaussianStreaming(entityId:source:options:)`'s `.progressive` case, which forwards +/// `GaussianStreamingOptions.boundingBoxHalfExtent` here; that's the entry point callers should +/// use. +func setEntityGaussianProgressiveStreamable( + entityId: EntityID, + baseFilename: String, + withExtension ext: String = "untoldgs", + levelCount: Int, + maxDistances: [Float], + streamingRadius: Float = 100.0, + unloadRadius: Float = 150.0, + boundingBoxHalfExtent: simd_float3? = nil, + priority: Int = 0 +) { + guard let local = scene.get(component: LocalTransformComponent.self, for: entityId) else { + handleError(.noLocalTransformComponent, entityId) + return + } + guard let tileEntity = findTileEntity(containing: local.position) else { + Logger.logWarning(message: "[RegistrationSystem] setEntityGaussianProgressiveStreamable: no tile found containing position \(local.position) for entity \(entityId) — entity left non-streaming.") + return + } + + // Resolve LOD levels (and validate levelCount/maxDistances/tier files) before touching + // parent/octree/box state, so a validation failure here leaves the entity exactly as it + // was before this call — no half-registered state to clean up. + guard configureEntityGaussianProgressiveLOD( + entityId: entityId, + baseFilename: baseFilename, + withExtension: ext, + levelCount: levelCount, + maxDistances: maxDistances, + errorPrefix: "setEntityGaussianProgressiveStreamable" + ) else { return } + + // configureEntityGaussianProgressiveLOD already confirmed every tier's file exists (it + // resolves and validates each URL, including the coarsest), so this is a header-parse + // concern only, not a file-lookup one. + let coarsestTierURL = scene.get(component: GaussianLODComponent.self, for: entityId)?.lodLevels.last?.url + guard let box = resolveGaussianBoundingBox(override: boundingBoxHalfExtent, untoldgsURL: coarsestTierURL) else { + Logger.logWarning(message: "[RegistrationSystem] setEntityGaussianProgressiveStreamable: no boundingBoxHalfExtent supplied and no baked box available for '\(baseFilename)' — entity left non-streaming.") + removeEntityGaussianLOD(entityId: entityId) + return + } + local.boundingBox = box + // boundingBoxHalfExtent may come from either an explicit override or the baked header — + // mark it explicit either way so loadGaussianLODLevel never overwrites it with a redundant + // auto-computed one once the first tier actually loads. + scene.get(component: GaussianLODComponent.self, for: entityId)?.hasExplicitBoundingBox = true + + setParent(childId: entityId, parentId: tileEntity) + OctreeSystem.shared.registerEntity(entityId) + + if let streaming = scene.assign(to: entityId, component: StreamingComponent.self) { + streaming.assetKind = .gaussianSplat + streaming.assetFilename = baseFilename + streaming.assetExtension = ext + streaming.streamingRadius = streamingRadius + streaming.unloadRadius = unloadRadius + streaming.priority = priority + } +} + +/// Resolves the on-disk URL for one progressive tier, given the same `_lod` +/// naming `configureEntityGaussianProgressiveLOD` uses (or just `baseFilename` when +/// `levelCount == 1`, matching `bakeGaussianSplatProgressiveTiers`'s single-tier output). +/// Factored out so callers can resolve a specific tier's URL (typically the coarsest, for a +/// bounding-box header read) without going through full LOD-component setup first. +private func gaussianProgressiveTierURL( + baseFilename: String, + withExtension ext: String, + levelCount: Int, + tierIndex: Int +) -> URL? { + let filename = levelCount == 1 ? baseFilename : "\(baseFilename)_lod\(tierIndex)" + return LoadingSystem.shared.resourceURL(forResource: filename, withExtension: ext, subResource: nil) +} + +/// Resolves a local-space bounding box for Gaussian entity registration: an explicit +/// caller-supplied `override` always wins; otherwise, if `untoldgsURL` points at a real, +/// header-readable `.untoldgs` file, reads its baked box (see `UntoldGSFormat.readHeader`) — +/// cheap enough to call synchronously at registration time. Returns `nil` when neither is +/// available (e.g. a raw `.ply` source with no override) — callers decide how to handle that. +private func resolveGaussianBoundingBox( + override: simd_float3?, + untoldgsURL: URL? +) -> (min: simd_float3, max: simd_float3)? { + if let override { + return (min: -override, max: override) + } + guard let untoldgsURL, let header = try? UntoldGSFormat.readHeader(from: untoldgsURL) else { + return nil + } + return (min: header.boundingBoxMin, max: header.boundingBoxMax) +} + +@discardableResult +private func configureEntityGaussianProgressiveLOD( + entityId: EntityID, + baseFilename: String, + withExtension ext: String, + levelCount: Int, + maxDistances: [Float], + errorPrefix: String +) -> Bool { + guard levelCount > 0 else { + handleError(.assetDataMissing, "\(errorPrefix): levelCount must be at least 1, got \(levelCount)") + return false + } + guard maxDistances.count == levelCount else { + handleError(.assetDataMissing, "\(errorPrefix): maxDistances must have \(levelCount) entries, got \(maxDistances.count)") + return false + } + + var levels: [GaussianLODLevel] = [] + for index in 0 ..< levelCount { + guard let url = gaussianProgressiveTierURL(baseFilename: baseFilename, withExtension: ext, levelCount: levelCount, tierIndex: index) else { + let filename = levelCount == 1 ? baseFilename : "\(baseFilename)_lod\(index)" + handleError(.filenameNotFound, filename) + return false + } + // meanSquaredSplatExtent is populated automatically by loadGaussianLODLevel when this + // tier's .untoldgs file is actually read — it's baked into the file, not caller-supplied. + levels.append(GaussianLODLevel(maxDistance: maxDistances[index], url: url)) + } + + guard let lodComponent = scene.assign(to: entityId, component: GaussianLODComponent.self) else { + return false + } + lodComponent.lodLevels = levels + lodComponent.currentLOD = -1 + lodComponent.desiredLOD = levelCount - 1 + lodComponent.isUsingFallback = false + return true +} + +/// Largest per-axis scale magnitude of a splat, used both as the size term in +/// `gaussianImportanceScore` and, aggregated across a tier, as +/// `GaussianLODTier.meanSquaredSplatExtent` for overdraw estimation. +func gaussianMajorAxis(_ splat: GaussianSplat) -> Float { + max(abs(splat.scale.x), max(abs(splat.scale.y), abs(splat.scale.z))) +} + +private func gaussianImportanceScore(_ splat: GaussianSplat) -> Float { + let majorAxis = gaussianMajorAxis(splat) + return splat.opacity * majorAxis * majorAxis +} + +private struct GaussianSpatialBucketKey: Hashable { + let x: Int + let y: Int + let z: Int +} + +private func spatiallyInterleavedGaussianRanking(_ splats: [GaussianSplat]) -> [Int] { + guard splats.count > 1 else { return Array(splats.indices) } + + var minBounds = simd_float3(Float.greatestFiniteMagnitude, Float.greatestFiniteMagnitude, Float.greatestFiniteMagnitude) + var maxBounds = simd_float3(-Float.greatestFiniteMagnitude, -Float.greatestFiniteMagnitude, -Float.greatestFiniteMagnitude) + for splat in splats { + let center = simd_float3(splat.center.x, splat.center.y, splat.center.z) + minBounds = simd_min(minBounds, center) + maxBounds = simd_max(maxBounds, center) + } + + let extent = maxBounds - minBounds + let occupiedAxisCount = [extent.x, extent.y, extent.z].filter { $0 > 0.0001 }.count + guard occupiedAxisCount > 0 else { + return splats.indices.sorted { + gaussianImportanceScore(splats[$0]) > gaussianImportanceScore(splats[$1]) + } + } + + let targetCellCount = max(8, min(512, splats.count / 24)) + let cellsPerAxis = max(1, Int(ceil(pow(Double(targetCellCount), 1.0 / Double(occupiedAxisCount))))) + let safeExtent = simd_float3( + max(extent.x, 0.0001), + max(extent.y, 0.0001), + max(extent.z, 0.0001) + ) + + var buckets: [GaussianSpatialBucketKey: [Int]] = [:] + for index in splats.indices { + let center = simd_float3(splats[index].center.x, splats[index].center.y, splats[index].center.z) + let normalized = (center - minBounds) / safeExtent + let maxCellIndex = cellsPerAxis - 1 + let cellX = min(maxCellIndex, max(0, Int(normalized.x * Float(cellsPerAxis)))) + let cellY = min(maxCellIndex, max(0, Int(normalized.y * Float(cellsPerAxis)))) + let cellZ = min(maxCellIndex, max(0, Int(normalized.z * Float(cellsPerAxis)))) + let key = GaussianSpatialBucketKey(x: cellX, y: cellY, z: cellZ) + buckets[key, default: []].append(index) + } + + let sortedBuckets = buckets.mapValues { indices in + indices.sorted { + gaussianImportanceScore(splats[$0]) > gaussianImportanceScore(splats[$1]) + } + } + + let bucketCenters = sortedBuckets.mapValues { indices in + var center = simd_float3.zero + for index in indices { + center += simd_float3(splats[index].center.x, splats[index].center.y, splats[index].center.z) + } + return center / Float(max(1, indices.count)) + } + + var remainingBuckets = Array(sortedBuckets.keys) + var bucketOrder: [GaussianSpatialBucketKey] = [] + if let first = remainingBuckets.max(by: { lhs, rhs in + guard let lhsIndex = sortedBuckets[lhs]?.first, + let rhsIndex = sortedBuckets[rhs]?.first + else { return false } + return gaussianImportanceScore(splats[lhsIndex]) < gaussianImportanceScore(splats[rhsIndex]) + }) { + bucketOrder.append(first) + remainingBuckets.removeAll { $0 == first } + } + + while !remainingBuckets.isEmpty { + let next = remainingBuckets.max { lhs, rhs in + let lhsDistance = nearestSelectedBucketDistanceSquared(lhs, centers: bucketCenters, selected: bucketOrder) + let rhsDistance = nearestSelectedBucketDistanceSquared(rhs, centers: bucketCenters, selected: bucketOrder) + if lhsDistance == rhsDistance { + let lhsIndex = sortedBuckets[lhs]?.first ?? 0 + let rhsIndex = sortedBuckets[rhs]?.first ?? 0 + return gaussianImportanceScore(splats[lhsIndex]) < gaussianImportanceScore(splats[rhsIndex]) + } + return lhsDistance < rhsDistance + }! + bucketOrder.append(next) + remainingBuckets.removeAll { $0 == next } + } + + var ranking: [Int] = [] + ranking.reserveCapacity(splats.count) + var depth = 0 + while ranking.count < splats.count { + var appendedThisRound = false + for key in bucketOrder { + guard let indices = sortedBuckets[key], depth < indices.count else { continue } + ranking.append(indices[depth]) + appendedThisRound = true + } + guard appendedThisRound else { break } + depth += 1 + } + + return ranking +} + +private func nearestSelectedBucketDistanceSquared( + _ key: GaussianSpatialBucketKey, + centers: [GaussianSpatialBucketKey: simd_float3], + selected: [GaussianSpatialBucketKey] +) -> Float { + guard let center = centers[key], !selected.isEmpty else { return Float.greatestFiniteMagnitude } + var best = Float.greatestFiniteMagnitude + for selectedKey in selected { + guard let selectedCenter = centers[selectedKey] else { continue } + best = min(best, simd_distance_squared(center, selectedCenter)) + } + return best +} + +private func subsetSphericalHarmonics( + _ sh: GaussianSphericalHarmonics, + keeping indices: [Int] +) -> GaussianSphericalHarmonics { + let perSplat = sh.coefficientsPerSplat + var subset: [Float] = [] + subset.reserveCapacity(indices.count * perSplat) + for index in indices { + let base = index * perSplat + subset.append(contentsOf: sh.coefficients[base ..< base + perSplat]) + } + return GaussianSphericalHarmonics( + degree: sh.degree, + coefficientsPerChannel: sh.coefficientsPerChannel, + coefficients: subset + ) +} + +/// One baked `.untoldgs` tier plus the bake-time statistic needed for overdraw estimation — +/// see `estimatedGaussianOverdraw`. +public struct GaussianLODTier { + public let url: URL + public let meanSquaredSplatExtent: Float +} + +/// Result of `bakeGaussianSplatProgressiveTiers`: the baked tiers plus a single asset-level +/// bounding box (from the full, unsubsetted source splats) shared by all tiers so it stays +/// stable across LOD switches. +public struct GaussianProgressiveBakeResult { + public let tiers: [GaussianLODTier] + public let boundingBoxMin: simd_float3 + public let boundingBoxMax: simd_float3 +} + +private func meanSquaredSplatExtent(_ splats: [GaussianSplat], keeping indices: [Int]) -> Float { + guard !indices.isEmpty else { return 0 } + let sumOfSquares = indices.reduce(Float(0)) { partial, index in + let majorAxis = gaussianMajorAxis(splats[index]) + return partial + majorAxis * majorAxis + } + return sumOfSquares / Float(indices.count) +} + +/// Bakes progressive `.untoldgs` Gaussian tiers from a source `.ply`. +/// +/// `lodFractions` are ordered finest to coarsest. With `[1.0, 0.5, 0.25]`, output files are +/// `_lod0.untoldgs`, `_lod1.untoldgs`, and `_lod2.untoldgs`. +/// +/// Throws `UntoldGSError.sizeMismatch` if `plyURL` contains no splats, regardless of +/// `lodFractions` — including the single-tier (`[1.0]`) case, which needs the same guard since +/// it now also computes an asset-level bounding box that's meaningless for zero splats. +public func bakeGaussianSplatProgressiveTiers( + plyURL: URL, + outputBaseURL: URL, + lodFractions: [Float] +) throws -> GaussianProgressiveBakeResult { + guard !lodFractions.isEmpty else { + throw UntoldGSError.sizeMismatch("lodFractions must contain at least one entry") + } + + let asset = try PLYReader.readGaussianAsset(from: plyURL) + guard !asset.splats.isEmpty else { + throw UntoldGSError.sizeMismatch("source .ply contains no splats") + } + let assetBoundingBox = computeGaussianSplatBoundingBox(asset.splats) + + if lodFractions == [1.0] { + let resultURL = outputBaseURL + let allIndices = Array(asset.splats.indices) + let encodedSplats = asset.splats.map(encodeGaussianSplatForTBDR) + let packedSphericalHarmonics = try asset.sphericalHarmonics.map { + try packGaussianSphericalHarmonics($0, splatCount: asset.splats.count) + } + let tierExtent = meanSquaredSplatExtent(asset.splats, keeping: allIndices) + try UntoldGSFormat.write( + encodedSplats: encodedSplats, + sphericalHarmonics: packedSphericalHarmonics, + meanSquaredSplatExtent: tierExtent, + boundingBoxMin: assetBoundingBox.min, + boundingBoxMax: assetBoundingBox.max, + to: resultURL + ) + return GaussianProgressiveBakeResult( + tiers: [GaussianLODTier(url: resultURL, meanSquaredSplatExtent: tierExtent)], + boundingBoxMin: assetBoundingBox.min, + boundingBoxMax: assetBoundingBox.max + ) + } + + let rankedIndices = spatiallyInterleavedGaussianRanking(asset.splats) + + let baseWithoutExtension = outputBaseURL.deletingPathExtension() + let baseName = baseWithoutExtension.lastPathComponent + let baseDirectory = baseWithoutExtension.deletingLastPathComponent() + + var tiers: [GaussianLODTier] = [] + for (tierIndex, fraction) in lodFractions.enumerated() { + let clampedFraction = min(max(fraction, 0), 1) + let keepCount = max(1, Int((Float(asset.splats.count) * clampedFraction).rounded(.up))) + let keptIndices = Array(rankedIndices.prefix(keepCount)) + let encodedSplats = keptIndices.map { encodeGaussianSplatForTBDR(asset.splats[$0]) } + let packedSphericalHarmonics = try asset.sphericalHarmonics.map { sh in + try packGaussianSphericalHarmonics( + subsetSphericalHarmonics(sh, keeping: keptIndices), + splatCount: keptIndices.count + ) + } + let tierURL = baseDirectory + .appendingPathComponent("\(baseName)_lod\(tierIndex)") + .appendingPathExtension("untoldgs") + let tierExtent = meanSquaredSplatExtent(asset.splats, keeping: keptIndices) + try UntoldGSFormat.write( + encodedSplats: encodedSplats, + sphericalHarmonics: packedSphericalHarmonics, + meanSquaredSplatExtent: tierExtent, + boundingBoxMin: assetBoundingBox.min, + boundingBoxMax: assetBoundingBox.max, + to: tierURL + ) + tiers.append(GaussianLODTier(url: tierURL, meanSquaredSplatExtent: tierExtent)) + } + return GaussianProgressiveBakeResult( + tiers: tiers, + boundingBoxMin: assetBoundingBox.min, + boundingBoxMax: assetBoundingBox.max + ) +} + +public func bakeGaussianSplatProgressiveTiers( + plyURL: URL, + outputBaseURL: URL, + levelCount: Int +) throws -> GaussianProgressiveBakeResult { + guard levelCount > 0 else { + throw UntoldGSError.sizeMismatch("levelCount must be at least 1, got \(levelCount)") + } + let fractions = (0 ..< levelCount).map { Float(1.0) / Float(1 << $0) } + return try bakeGaussianSplatProgressiveTiers( + plyURL: plyURL, + outputBaseURL: outputBaseURL, + lodFractions: fractions + ) +} + +public struct PackedGaussianSphericalHarmonics { + public let coefficients: [UInt8] + public let metadata: GaussianSHMetadata +} + +/// Quantizes a higher-order SH coefficient into the GPU's fixed [-1, 1] byte +/// contract. Mirrors `loadGaussianSHCoefficient`'s dequantization in +/// Gaussians.metal: `(byte - 128) / 128`. Values outside [-1, 1] are clamped +/// rather than rejected — real trained assets occasionally have rare +/// higher-order outliers (e.g. strong specular splats), and clamping only +/// caps the affected highlight rather than discarding the whole asset. +func quantizeGaussianSHCoefficient(_ value: Float) -> UInt8 { + let clamped = min(max(value, -1), 1) + return UInt8(clamping: Int(clamped * 127) + 128) } /// Packs higher-order SH coefficients to the GPU contract while leaving DC /// color in `EncodedGaussianSplat`. Input and output are both channel-major. +/// Higher-order coefficients are quantized to one byte each; see +/// `quantizeGaussianSHCoefficient`. func packGaussianSphericalHarmonics( _ sphericalHarmonics: GaussianSphericalHarmonics, splatCount: Int @@ -3511,7 +4423,7 @@ func packGaussianSphericalHarmonics( } let outputPerSplat = higherOrderPerChannel * 3 - var packed: [Float16] = [] + var packed: [UInt8] = [] packed.reserveCapacity(splatCount * outputPerSplat) for splatIndex in 0 ..< splatCount { @@ -3519,11 +4431,11 @@ func packGaussianSphericalHarmonics( for channel in 0 ..< 3 { let channelBase = splatBase + channel * coefficientsPerChannel for coefficient in 1 ..< coefficientsPerChannel { - let value = Float16(sphericalHarmonics.coefficients[channelBase + coefficient]) + let value = sphericalHarmonics.coefficients[channelBase + coefficient] guard value.isFinite else { - throw PLYError.invalidData("Spherical-harmonic coefficient cannot be represented as Float16") + throw PLYError.invalidData("Spherical-harmonic coefficient is not finite") } - packed.append(value) + packed.append(quantizeGaussianSHCoefficient(value)) } } } @@ -3552,13 +4464,11 @@ private func encodeGaussianSplatForTBDR(_ splat: GaussianSplat) -> EncodedGaussi return EncodedGaussianSplat( position: simd_float3(splat.center.x, splat.center.y, splat.center.z), - opacity: splat.opacity, - color: simd_float3(splat.color.x, splat.color.y, splat.color.z), - _pad0: 0.0, - covA: simd_float3(covariance[0, 0], covariance[0, 1], covariance[0, 2]), - _pad1: 0.0, - covB: simd_float3(covariance[1, 1], covariance[1, 2], covariance[2, 2]), - _pad2: 0.0 + covA: simd_half3(Float16(covariance[0, 0]), Float16(covariance[0, 1]), Float16(covariance[0, 2])), + covB: simd_half3(Float16(covariance[1, 1]), Float16(covariance[1, 2]), Float16(covariance[2, 2])), + colorAndOpacity: simd_half4( + Float16(splat.color.x), Float16(splat.color.y), Float16(splat.color.z), Float16(splat.opacity) + ) ) } @@ -3697,6 +4607,14 @@ func removeEntityLOD(entityId: EntityID) { } } +func removeEntityGaussianLOD(entityId: EntityID) { + if let lodComponent = scene.get(component: GaussianLODComponent.self, for: entityId) { + lodComponent.releaseAllLevelResources() + lodComponent.lodLevels.removeAll() + scene.remove(component: GaussianLODComponent.self, from: entityId) + } +} + func removeEntityGaussian(entityId: EntityID) { if let gaussianComponent = scene.get(component: GaussianComponent.self, for: entityId) { // Release Metal buffers diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.air b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.air index 8a065de1..7c86ab89 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.air and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.air differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib index a9e51512..d89baea2 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-ios.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib index 3561c1aa..76233700 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-iossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.air b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.air index 5c27a6a2..bc9fb16e 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.air and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.air differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib index 0a15f88c..5bc4a997 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvos.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.air b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.air index 272c8e66..33f2676f 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.air and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.air differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib index 3172dcff..dbdf9f3f 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-tvossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.air b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.air index 30199897..faf45f4f 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.air and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.air differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib index ca764a5f..ef946a5d 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xros.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.air b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.air index 462faa19..f2895a42 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.air and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.air differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib index 87a7c8fc..3d1b20a8 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels-xrossim.metallib differ diff --git a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib index 4563fca0..94f8c320 100644 Binary files a/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib and b/Sources/UntoldEngine/UntoldEngineKernels/UntoldEngineKernels.metallib differ diff --git a/Tests/UntoldEngineRenderTests/DeviceRadixSortTest.swift b/Tests/UntoldEngineRenderTests/DeviceRadixSortTest.swift index 644115bf..4bd4fd9d 100644 --- a/Tests/UntoldEngineRenderTests/DeviceRadixSortTest.swift +++ b/Tests/UntoldEngineRenderTests/DeviceRadixSortTest.swift @@ -861,13 +861,9 @@ final class DeviceRadixSortTest: BaseRenderSetup { let splats = positions.map { pos in EncodedGaussianSplat( position: pos, - opacity: 1.0, - color: simd_float3(1, 0, 0), - _pad0: 0.0, - covA: simd_float3(1, 0, 0), - _pad1: 0.0, - covB: simd_float3(1, 0, 1), - _pad2: 0.0 + covA: simd_half3(1, 0, 0), + covB: simd_half3(1, 0, 1), + colorAndOpacity: simd_half4(1, 0, 0, 1) ) } diff --git a/Tests/UntoldEngineRenderTests/GaussianProgressiveLODTest.swift b/Tests/UntoldEngineRenderTests/GaussianProgressiveLODTest.swift new file mode 100644 index 00000000..d557ecd7 --- /dev/null +++ b/Tests/UntoldEngineRenderTests/GaussianProgressiveLODTest.swift @@ -0,0 +1,729 @@ +// +// GaussianProgressiveLODTest.swift +// UntoldEngine +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import CShaderTypes +import simd +@testable import UntoldEngine +import XCTest + +@MainActor +final class GaussianProgressiveLODTest: BaseRenderSetup { + private var savedLODUpdateFrameInterval = 1 + private var savedMinimumEntityDisplacementForLODUpdate: Float = 0.5 + private var savedMinimumCameraDisplacementForLODUpdate: Float = 0.5 + private var savedGaussianOverdrawBudget: Float = 12.0 + private var savedHysteresis: Float = 5.0 + + override func setUp() async throws { + try await super.setUp() + savedLODUpdateFrameInterval = LODConfig.shared.lodUpdateFrameInterval + savedMinimumEntityDisplacementForLODUpdate = LODConfig.shared.minimumEntityDisplacementForLODUpdate + savedMinimumCameraDisplacementForLODUpdate = LODConfig.shared.minimumCameraDisplacementForLODUpdate + savedGaussianOverdrawBudget = LODConfig.shared.gaussianOverdrawBudget + savedHysteresis = LODConfig.shared.hysteresis + LODConfig.shared.lodUpdateFrameInterval = 1 + GeometryStreamingSystem.shared.reset() + GeometryStreamingSystem.shared.enabled = true + GeometryStreamingSystem.shared.updateInterval = 0 + GeometryStreamingSystem.shared.enableFrustumGate = false + GeometryStreamingSystem.shared.maxQueryRadius = 500 + MemoryBudgetManager.shared.clear() + GaussianLODSystem.shared.reset() + } + + override func tearDown() async throws { + GeometryStreamingSystem.shared.reset() + GeometryStreamingSystem.shared.enabled = false + GeometryStreamingSystem.shared.updateInterval = 0.1 + GeometryStreamingSystem.shared.enableFrustumGate = true + LODConfig.shared.lodUpdateFrameInterval = savedLODUpdateFrameInterval + LODConfig.shared.minimumEntityDisplacementForLODUpdate = savedMinimumEntityDisplacementForLODUpdate + LODConfig.shared.minimumCameraDisplacementForLODUpdate = savedMinimumCameraDisplacementForLODUpdate + LODConfig.shared.gaussianOverdrawBudget = savedGaussianOverdrawBudget + LODConfig.shared.hysteresis = savedHysteresis + MemoryBudgetManager.shared.clear() + GaussianLODSystem.shared.reset() + try await super.tearDown() + } + + private func testPLYURL() throws -> URL { + try XCTUnwrap( + LoadingSystem.shared.resourceURL(forResource: "test_gaussians", withExtension: "ply", subResource: nil) + ) + } + + private func makeContainingTile() -> EntityID { + let tile = createEntity() + scene.assign(to: tile, component: TileComponent.self)?.state = .parsed + scene.get(component: LocalTransformComponent.self, for: tile)?.boundingBox = ( + min: simd_float3(-500, -500, -500), + max: simd_float3(500, 500, 500) + ) + OctreeSystem.shared.registerEntity(tile) + return tile + } + + private func bakeProgressiveBase(levelCount: Int) throws -> String { + let output = FileManager.default.temporaryDirectory + .appendingPathComponent("GaussianProgressiveLODTest-\(UUID().uuidString)") + .appendingPathExtension("untoldgs") + _ = try bakeGaussianSplatProgressiveTiers( + plyURL: testPLYURL(), + outputBaseURL: output, + levelCount: levelCount + ) + return output.deletingPathExtension().path + } + + func testBakeProgressiveTiersWritesNestedCounts() throws { + let output = FileManager.default.temporaryDirectory + .appendingPathComponent("GaussianProgressiveBake-\(UUID().uuidString)") + .appendingPathExtension("untoldgs") + let bakeResult = try bakeGaussianSplatProgressiveTiers( + plyURL: testPLYURL(), + outputBaseURL: output, + levelCount: 3 + ) + + let tiers = try bakeResult.tiers.map { try UntoldGSFormat.read(from: $0.url) } + XCTAssertEqual(tiers.count, 3) + XCTAssertGreaterThan(tiers[0].splatCount, tiers[1].splatCount) + XCTAssertGreaterThan(tiers[1].splatCount, tiers[2].splatCount) + } + + func testBakeProgressiveTiersPreservesSpatialCoverageInCoarseTier() throws { + let ply = FileManager.default.temporaryDirectory + .appendingPathComponent("GaussianSpatialCoverage-\(UUID().uuidString)") + .appendingPathExtension("ply") + let output = FileManager.default.temporaryDirectory + .appendingPathComponent("GaussianSpatialCoverage-\(UUID().uuidString)") + .appendingPathExtension("untoldgs") + + let lines = [ + "ply", + "format ascii 1.0", + "element vertex 8", + "property float x", + "property float y", + "property float z", + "property float scale_0", + "property float scale_1", + "property float scale_2", + "property float f_dc_0", + "property float f_dc_1", + "property float f_dc_2", + "property float opacity", + "property float rot_0", + "property float rot_1", + "property float rot_2", + "property float rot_3", + "end_header", + "0 0 0 0 0 0 0 0 0 5 1 0 0 0", + "0 1 0 0 0 0 0 0 0 5 1 0 0 0", + "0 2 0 0 0 0 0 0 0 5 1 0 0 0", + "0 3 0 0 0 0 0 0 0 5 1 0 0 0", + "0 4 0 0 0 0 0 0 0 5 1 0 0 0", + "0 5 0 0 0 0 0 0 0 5 1 0 0 0", + "10 0 0 -4 -4 -4 0 0 0 -4 1 0 0 0", + "10 1 0 -4 -4 -4 0 0 0 -4 1 0 0 0", + ] + try lines.joined(separator: "\n").write(to: ply, atomically: true, encoding: .utf8) + + let bakeResult = try bakeGaussianSplatProgressiveTiers( + plyURL: ply, + outputBaseURL: output, + lodFractions: [1.0, 0.25] + ) + let coarse = try UntoldGSFormat.read(from: bakeResult.tiers[1].url) + XCTAssertEqual(coarse.splatCount, 2) + + let xs = coarse.encodedSplats.map(\.position.x) + XCTAssertTrue(xs.contains { $0 < 1 }, "Coarse tier should keep coverage from the dense left cluster") + XCTAssertTrue(xs.contains { $0 > 9 }, "Coarse tier should keep coverage from the sparse right cluster") + } + + func testProgressiveStreamingLoadsCoarsestFirstThenRefinesOnDemand() async throws { + _ = makeContainingTile() + let base = try bakeProgressiveBase(levelCount: 3) + + let entity = createEntity() + translateTo(entityId: entity, position: .zero) + setEntityGaussianStreaming( + entityId: entity, + source: .progressive( + baseFilename: base, + levelCount: 3, + maxDistances: [20, 50, .greatestFiniteMagnitude] + ), + options: GaussianStreamingOptions( + streamingRadius: 200, + unloadRadius: 300, + boundingBoxHalfExtent: simd_float3(1, 1, 1) + ) + ) + + GeometryStreamingSystem.shared.update(cameraPosition: .zero, deltaTime: 0.1) + await scene.get(component: StreamingComponent.self, for: entity)?.loadTask?.value + + let lod = try XCTUnwrap(scene.get(component: GaussianLODComponent.self, for: entity)) + XCTAssertEqual(lod.currentLOD, 2) + XCTAssertEqual(lod.lodLevels[2].residencyState, .resident) + XCTAssertEqual(lod.lodLevels[1].residencyState, .unknown) + XCTAssertEqual(lod.lodLevels[0].residencyState, .unknown) + + let coarseCount = try XCTUnwrap(lod.lodLevels[2].buffers?.splatCount) + XCTAssertEqual(scene.get(component: GaussianComponent.self, for: entity)?.splatCount, coarseCount) + + let camera = createEntity() + scene.assign(to: camera, component: CameraComponent.self)?.localPosition = .zero + _ = scene.assign(to: camera, component: LocalTransformComponent.self) + CameraSystem.shared.activeCamera = camera + + GaussianLODSystem.shared.update(deltaTime: 0.1) + await lod.lodLevels[0].loadTask?.value + GaussianLODSystem.shared.update(deltaTime: 0.1) + + XCTAssertEqual(lod.lodLevels[0].residencyState, .resident) + XCTAssertEqual(lod.currentLOD, 0) + XCTAssertEqual(scene.get(component: GaussianComponent.self, for: entity)?.splatCount, lod.lodLevels[0].buffers?.splatCount) + } + + func testProgressiveGaussianDoesNotRequireTileStreaming() async throws { + let base = try bakeProgressiveBase(levelCount: 3) + + let entity = createEntity() + translateTo(entityId: entity, position: .zero) + setEntityGaussian( + entityId: entity, + source: .progressive( + baseFilename: base, + levelCount: 3, + maxDistances: [20, 50, .greatestFiniteMagnitude] + ) + ) + + let lod = try XCTUnwrap(scene.get(component: GaussianLODComponent.self, for: entity)) + XCTAssertNil(scene.get(component: StreamingComponent.self, for: entity)) + + for _ in 0 ..< 20 where lod.currentLOD != 2 { + try await Task.sleep(nanoseconds: 10_000_000) + } + + XCTAssertEqual(lod.currentLOD, 2) + XCTAssertEqual(lod.lodLevels[2].residencyState, .resident) + XCTAssertEqual(scene.get(component: GaussianComponent.self, for: entity)?.splatCount, lod.lodLevels[2].buffers?.splatCount) + + let camera = createEntity() + scene.assign(to: camera, component: CameraComponent.self)?.localPosition = .zero + _ = scene.assign(to: camera, component: LocalTransformComponent.self) + CameraSystem.shared.activeCamera = camera + + GaussianLODSystem.shared.update(deltaTime: 0.1) + await lod.lodLevels[0].loadTask?.value + GaussianLODSystem.shared.update(deltaTime: 0.1) + + XCTAssertEqual(lod.lodLevels[0].residencyState, .resident) + XCTAssertEqual(lod.currentLOD, 0) + XCTAssertEqual(scene.get(component: GaussianComponent.self, for: entity)?.splatCount, lod.lodLevels[0].buffers?.splatCount) + } + + func testBakeProgressiveTiersReturnsAssetLevelBoundingBox() throws { + let asset = try PLYReader.readGaussianAsset(from: testPLYURL()) + let expected = computeGaussianSplatBoundingBox(asset.splats) + + let output = FileManager.default.temporaryDirectory + .appendingPathComponent("GaussianBakeBoundingBox-\(UUID().uuidString)") + .appendingPathExtension("untoldgs") + let bakeResult = try bakeGaussianSplatProgressiveTiers( + plyURL: testPLYURL(), + outputBaseURL: output, + levelCount: 3 + ) + + // The box must reflect the full source asset, not just the coarsest (most-subsetted) tier. + XCTAssertEqual(bakeResult.boundingBoxMin, expected.min) + XCTAssertEqual(bakeResult.boundingBoxMax, expected.max) + } + + func testSetEntityGaussianAutoComputesBoundingBox() throws { + // setEntityGaussian's auto-computed box goes through buildGaussianLoadResult, which + // only has post-encode splat positions (no per-splat scale) -- it pads a centers-only + // box by an RMS radius derived from the mean squared extent, rather than the exact + // per-splat expansion computeGaussianSplatBoundingBox(_ splats:) does at bake time. + let asset = try PLYReader.readGaussianAsset(from: testPLYURL()) + let allIndices = Array(asset.splats.indices) + let positions = asset.splats.map { simd_float3($0.center.x, $0.center.y, $0.center.z) } + let extentSum = allIndices.reduce(Float(0)) { partial, index in + let majorAxis = gaussianMajorAxis(asset.splats[index]) + return partial + majorAxis * majorAxis + } + let meanSquaredExtent = extentSum / Float(allIndices.count) + let expected = computeGaussianSplatPositionBoundingBox(positions, padding: sqrt(meanSquaredExtent)) + + let entity = createEntity() + setEntityGaussian(entityId: entity, filename: "test_gaussians", withExtension: "ply") + + let boundingBox = try XCTUnwrap(scene.get(component: LocalTransformComponent.self, for: entity)?.boundingBox) + XCTAssertNotEqual(boundingBox.min, boundingBox.max, "Bounding box should not collapse to a single point") + XCTAssertEqual(boundingBox.min, expected.min) + XCTAssertEqual(boundingBox.max, expected.max) + } + + func testMeanSquaredSplatExtentRoundTripsThroughUntoldGSFile() throws { + let output = FileManager.default.temporaryDirectory + .appendingPathComponent("GaussianMeanSquaredExtentRoundTrip-\(UUID().uuidString)") + .appendingPathExtension("untoldgs") + let bakeResult = try bakeGaussianSplatProgressiveTiers( + plyURL: testPLYURL(), + outputBaseURL: output, + levelCount: 3 + ) + + for tier in bakeResult.tiers { + let readBack = try UntoldGSFormat.read(from: tier.url) + XCTAssertGreaterThan(tier.meanSquaredSplatExtent, 0, "bake-time stat should be a real, non-degenerate value for a real asset") + XCTAssertEqual(readBack.meanSquaredSplatExtent, tier.meanSquaredSplatExtent, "the value baked into the file must match what bakeGaussianSplatProgressiveTiers reported") + } + } + + func testProgressiveTierAutoPopulatesMeanSquaredSplatExtentOnLoad() async throws { + let base = try bakeProgressiveBase(levelCount: 3) + + let entity = createEntity() + translateTo(entityId: entity, position: .zero) + setEntityGaussianProgressive( + entityId: entity, + baseFilename: base, + levelCount: 3, + maxDistances: [20, 50, .greatestFiniteMagnitude] + ) + + let lod = try XCTUnwrap(scene.get(component: GaussianLODComponent.self, for: entity)) + + // Nothing was passed by the caller -- before the coarsest tier's async load completes, + // its stat is unknown. + XCTAssertNil(lod.lodLevels[2].meanSquaredSplatExtent) + + for _ in 0 ..< 20 where lod.currentLOD != 2 { + try await Task.sleep(nanoseconds: 10_000_000) + } + + // Auto-populated from the .untoldgs file itself once it's actually read -- no caller + // input required. + let loadedExtent = try XCTUnwrap(lod.lodLevels[2].meanSquaredSplatExtent) + XCTAssertGreaterThan(loadedExtent, 0) + } + + // MARK: - Overdraw-aware LOD clamping + + func testEstimatedGaussianOverdrawMatchesHandComputedFormula() { + // distance=10, fovY=90deg (tanHalfFovY=1), viewportHeight=1000 -> + // projectionScaleFactor = 1000 / (2 * 1 * 10) = 50 + let overdraw = estimatedGaussianOverdraw( + splatCount: 100, + meanSquaredSplatExtent: 0.04, + distance: 10, + fovY: .pi / 2, + viewportHeight: 1000, + boundingRadius: 2 + ) + // projectedAreaPerSplat = 50^2 * 0.04 = 100 + // footprintRadius = 2 * 50 = 100 -> area = pi * 100^2 = pi * 10000 + // overdraw = (100 * 100) / (pi * 10000) = 10000 / (pi * 10000) = 1/pi + XCTAssertEqual(overdraw, 1 / Float.pi, accuracy: 0.0001) + } + + func testEstimatedGaussianOverdrawIsZeroForDegenerateInputs() { + XCTAssertEqual(estimatedGaussianOverdraw(splatCount: 0, meanSquaredSplatExtent: 1, distance: 10, fovY: 1, viewportHeight: 1000, boundingRadius: 1), 0) + XCTAssertEqual(estimatedGaussianOverdraw(splatCount: 100, meanSquaredSplatExtent: 1, distance: 0, fovY: 1, viewportHeight: 1000, boundingRadius: 1), 0) + XCTAssertEqual(estimatedGaussianOverdraw(splatCount: 100, meanSquaredSplatExtent: 1, distance: 10, fovY: 1, viewportHeight: 1000, boundingRadius: 0), 0) + } + + func testClampGaussianLODForOverdrawWalksToCoarserTierWhenOverBudget() { + let lodComponent = GaussianLODComponent() + var level0 = GaussianLODLevel(maxDistance: 20) + level0.meanSquaredSplatExtent = 1.0 + level0.buffers = GaussianComponent() + level0.buffers?.splatCount = 1_000_000 // deliberately huge -> massive overdraw at any sane distance + + var level1 = GaussianLODLevel(maxDistance: 50) + level1.meanSquaredSplatExtent = 0.0001 + level1.buffers = GaussianComponent() + level1.buffers?.splatCount = 10 // tiny -> comfortably under budget + + lodComponent.lodLevels = [level0, level1] + + let clamped = clampGaussianLODForOverdraw( + desiredLOD: 0, + lodComponent: lodComponent, + distance: 10, + fovY: .pi / 2, + viewportHeight: 1000, + boundingRadius: 2, + budget: 12.0 + ) + XCTAssertEqual(clamped, 1) + } + + func testClampGaussianLODForOverdrawIsNoOpWithoutBakedStats() { + let lodComponent = GaussianLODComponent() + var level0 = GaussianLODLevel(maxDistance: 20) + level0.buffers = GaussianComponent() + level0.buffers?.splatCount = 1_000_000 // huge, but no meanSquaredSplatExtent supplied + + lodComponent.lodLevels = [level0] + + let clamped = clampGaussianLODForOverdraw( + desiredLOD: 0, + lodComponent: lodComponent, + distance: 10, + fovY: .pi / 2, + viewportHeight: 1000, + boundingRadius: 2, + budget: 12.0 + ) + XCTAssertEqual(clamped, 0, "Without a baked meanSquaredSplatExtent, selection must fall back to the distance-based choice unchanged") + } + + func testClampGaussianLODForOverdrawStopsAtCoarsestTierWhenStillOverBudget() { + let lodComponent = GaussianLODComponent() + var level0 = GaussianLODLevel(maxDistance: 20) + level0.meanSquaredSplatExtent = 1.0 + level0.buffers = GaussianComponent() + level0.buffers?.splatCount = 1_000_000 + + var level1 = GaussianLODLevel(maxDistance: 50) + level1.meanSquaredSplatExtent = 1.0 + level1.buffers = GaussianComponent() + level1.buffers?.splatCount = 1_000_000 // still huge -> still over budget + + lodComponent.lodLevels = [level0, level1] + + let clamped = clampGaussianLODForOverdraw( + desiredLOD: 0, + lodComponent: lodComponent, + distance: 10, + fovY: .pi / 2, + viewportHeight: 1000, + boundingRadius: 2, + budget: 12.0 + ) + XCTAssertEqual(clamped, 1, "Should clamp to the coarsest available tier even if it's still over budget") + } + + func testDraggingEntityForcesLODRefreshEvenWhenCameraIsStationary() async throws { + LODConfig.shared.lodUpdateFrameInterval = 4 // matches the real production default + + let base = try bakeProgressiveBase(levelCount: 3) + + let entity = createEntity() + translateTo(entityId: entity, position: simd_float3(0, 0, 5)) + setEntityGaussianProgressive( + entityId: entity, + baseFilename: base, + levelCount: 3, + maxDistances: [10, 50, .greatestFiniteMagnitude] + ) + + let camera = createEntity() + scene.assign(to: camera, component: CameraComponent.self)?.localPosition = .zero + _ = scene.assign(to: camera, component: LocalTransformComponent.self) + CameraSystem.shared.activeCamera = camera + + let lod = try XCTUnwrap(scene.get(component: GaussianLODComponent.self, for: entity)) + + // The non-streaming progressive path always loads the coarsest tier first, + // independent of distance (see setEntityGaussianProgressive) -- wait that out. + for _ in 0 ..< 20 where lod.currentLOD != 2 { + try await Task.sleep(nanoseconds: 10_000_000) + } + XCTAssertEqual(lod.currentLOD, 2) + + // Drive it to LOD0 (distance 5, within maxDistances[0] = 10). This requires an actual + // tier load, so a handful of update() calls (crossing the interval) plus an await is + // expected here -- this part isn't what's under test. + for _ in 0 ..< 5 { + GaussianLODSystem.shared.update(deltaTime: 0.1) + } + await lod.lodLevels[0].loadTask?.value + for _ in 0 ..< 5 { + GaussianLODSystem.shared.update(deltaTime: 0.1) + } + XCTAssertEqual(lod.currentLOD, 0) + + // "Drag" the entity back out to distance 60 in one move. LOD2 (coarsest) is already + // resident from the very first load, so applying it needs no further async load -- + // just a single post-drag update() call. That call lands on frameCounter 11 (11 % 4 != + // 0, so the interval hasn't elapsed) with the camera stationary, so only the per-entity + // fast path can catch this. Without it, this would silently do nothing until the + // interval happened to elapse or the entity was dragged again. + translateTo(entityId: entity, position: simd_float3(0, 0, 60)) + GaussianLODSystem.shared.update(deltaTime: 0.1) + + XCTAssertEqual(lod.currentLOD, 2, "A single post-drag update should pick up the entity's own displacement without waiting on the camera-driven throttle") + } + + func testCameraDriftAccumulatesAcrossThrottledFramesInsteadOfResettingEachFrame() throws { + // Disable the per-entity fast path so only the camera-driven cumulative-displacement + // fast path can possibly catch this, and set the interval high enough that it can't + // fire on its own within the handful of update() calls below. + LODConfig.shared.minimumEntityDisplacementForLODUpdate = 1000 + LODConfig.shared.minimumCameraDisplacementForLODUpdate = 1.0 + LODConfig.shared.lodUpdateFrameInterval = 1000 + // Not testing hysteresis here -- the default (5.0) is larger than this test's near-tier + // threshold (3), which would make switching back to LOD0 mathematically impossible + // (3 - 5 = -2, an unreachable distance) and has nothing to do with what's under test. + LODConfig.shared.hysteresis = 0 + + let entity = createEntity() + translateTo(entityId: entity, position: simd_float3(0, 0, 5)) + scene.get(component: LocalTransformComponent.self, for: entity)?.boundingBox = ( + min: simd_float3(-0.1, -0.1, -0.1), max: simd_float3(0.1, 0.1, 0.1) + ) + + let lod = try XCTUnwrap(scene.assign(to: entity, component: GaussianLODComponent.self)) + var near = GaussianLODLevel(maxDistance: 3) + near.buffers = GaussianComponent() + near.residencyState = .resident + var far = GaussianLODLevel(maxDistance: .greatestFiniteMagnitude) + far.buffers = GaussianComponent() + far.residencyState = .resident + lod.lodLevels = [near, far] + lod.currentLOD = 1 + lod.desiredLOD = 1 + lod.distanceSelectedLOD = 1 + + let camera = createEntity() + scene.assign(to: camera, component: CameraComponent.self)?.localPosition = .zero + _ = scene.assign(to: camera, component: LocalTransformComponent.self) + CameraSystem.shared.activeCamera = camera + + // Entity sits at distance 5 (over the "near" tier's threshold of 3) -- move the camera + // toward it in many small steps (0.6 each), each individually under the 1.0 + // displacement threshold, ending well past distance 3 (final camera z = 6.0, so final + // distance = 1.0). Only the cumulative fast path can possibly catch any of this (the + // per-entity path is disabled above and the interval, 1000, never elapses in 10 calls). + // If lastCameraPosition were being reset every frame instead of only on qualifying + // frames, this fast path could never fire at all and lod.currentLOD would still be 1 + // after every one of these calls, regardless of how far the camera cumulatively moved. + for step in 1 ... 10 { + let cameraComponent = scene.get(component: CameraComponent.self, for: camera) + cameraComponent?.localPosition = simd_float3(0, 0, Float(step) * 0.6) + GaussianLODSystem.shared.update(deltaTime: 0.1) + } + + XCTAssertEqual(lod.currentLOD, 0, "Cumulative camera displacement across throttled-out frames should still trip the fast path") + } + + func testProgressiveEntityWithoutExplicitBoundingBoxAutoResolvesFromBakedHeaderImmediately() throws { + // Since M2, the box comes from the coarsest tier's baked v2 header (see + // UntoldGSFormat.readHeader) synchronously at registration time -- it's the exact + // asset-level box every tier's header carries (see bakeGaussianSplatProgressiveTiers), + // not an approximation derived from whichever tier happens to load first, and it's + // available immediately, with no need to wait for the coarsest tier's async load. + let asset = try PLYReader.readGaussianAsset(from: testPLYURL()) + let trueBox = computeGaussianSplatBoundingBox(asset.splats) + + let base = try bakeProgressiveBase(levelCount: 3) + let entity = createEntity() + translateTo(entityId: entity, position: .zero) + // Deliberately omit boundingBoxHalfExtent. + setEntityGaussianProgressive(entityId: entity, baseFilename: base, levelCount: 3, maxDistances: [20, 50, .greatestFiniteMagnitude]) + + let boundingBox = try XCTUnwrap(scene.get(component: LocalTransformComponent.self, for: entity)?.boundingBox) + XCTAssertEqual(boundingBox.min, trueBox.min) + XCTAssertEqual(boundingBox.max, trueBox.max) + XCTAssertEqual(scene.get(component: GaussianLODComponent.self, for: entity)?.hasExplicitBoundingBox, true, "Should be marked explicit so the async coarsest-tier load never overwrites it with a redundant auto-computed box") + } + + func testOverdrawClampDoesNotContaminateDistanceHysteresisAnchor() throws { + LODConfig.shared.gaussianOverdrawBudget = 0.001 // force the clamp to walk to the coarsest tier + + let entity = createEntity() + translateTo(entityId: entity, position: simd_float3(0, 0, 1)) + scene.get(component: LocalTransformComponent.self, for: entity)?.boundingBox = ( + min: simd_float3(-1, -1, -1), max: simd_float3(1, 1, 1) + ) + + let lod = try XCTUnwrap(scene.assign(to: entity, component: GaussianLODComponent.self)) + var level0 = GaussianLODLevel(maxDistance: 10) + level0.meanSquaredSplatExtent = 1.0 + level0.buffers = GaussianComponent() + level0.buffers?.splatCount = 100 + + var level1 = GaussianLODLevel(maxDistance: 50) + level1.meanSquaredSplatExtent = 0.0001 + level1.buffers = GaussianComponent() + level1.buffers?.splatCount = 1 + + lod.lodLevels = [level0, level1] + // Stale anchor simulating "last frame we were logically at LOD 1" -- pure distance (1 + // unit, well under level0's threshold of 10) should move this back to LOD 0. + lod.desiredLOD = 1 + lod.distanceSelectedLOD = 1 + + let camera = createEntity() + scene.assign(to: camera, component: CameraComponent.self)?.localPosition = .zero + _ = scene.assign(to: camera, component: LocalTransformComponent.self) + CameraSystem.shared.activeCamera = camera + + GaussianLODSystem.shared.update(deltaTime: 0.1) + + XCTAssertEqual(lod.desiredLOD, 1, "The tiny overdraw budget should force the actually-applied target to the coarser tier") + XCTAssertEqual(lod.distanceSelectedLOD, 0, "distanceSelectedLOD must track the pure distance pick, uncontaminated by the overdraw-forced value") + } + + func testBakeProgressiveTiersThrowsOnEmptyPLYRegardlessOfLodFractions() throws { + let ply = FileManager.default.temporaryDirectory + .appendingPathComponent("GaussianEmptyPLY-\(UUID().uuidString)") + .appendingPathExtension("ply") + let lines = [ + "ply", "format ascii 1.0", "element vertex 0", + "property float x", "property float y", "property float z", + "property float scale_0", "property float scale_1", "property float scale_2", + "property float f_dc_0", "property float f_dc_1", "property float f_dc_2", + "property float opacity", + "property float rot_0", "property float rot_1", "property float rot_2", "property float rot_3", + "end_header", + ] + // Trailing newline required: PLYReader's header scan finds "end_header" by looking for + // the newline that terminates it, so a file ending right after that line with none + // would never be recognized. + try (lines.joined(separator: "\n") + "\n").write(to: ply, atomically: true, encoding: .utf8) + + let output = FileManager.default.temporaryDirectory + .appendingPathComponent("GaussianEmptyPLYBake-\(UUID().uuidString)") + .appendingPathExtension("untoldgs") + + XCTAssertThrowsError(try bakeGaussianSplatProgressiveTiers(plyURL: ply, outputBaseURL: output, lodFractions: [1.0])) { error in + guard case UntoldGSError.sizeMismatch = error else { + XCTFail("Expected .sizeMismatch, got \(error)") + return + } + } + } + + func testComputeGaussianSplatBoundingBoxExpandsBySplatScaleNotJustCenter() { + // A single splat sitting exactly at the origin with a large scale should produce a box + // reflecting its true extent, not a degenerate single point. + let splat = GaussianSplat( + center: simd_float4(0, 0, 0, 1), + scale: simd_float4(2, 0.5, 0.5, 1), + color: simd_float4(1, 1, 1, 1), + quat: simd_float4(0, 0, 0, 1), + opacity: 1 + ) + let box = computeGaussianSplatBoundingBox([splat]) + XCTAssertEqual(box.min, simd_float3(-2, -2, -2)) + XCTAssertEqual(box.max, simd_float3(2, 2, 2)) + } + + // MARK: - UntoldGSFormat corrupt-header handling + + func testReadThrowsInsteadOfTrappingOnOverflowingHeaderCounts() throws { + // Hand-crafted 72-byte v2 header (magic "UTGS", version 2) declaring a splatCount of + // UInt64.max, which overflows when multiplied by EncodedGaussianSplat's stride to + // compute the expected encoded-splat byte count. Before the overflow-checked rewrite, + // the unchecked `Int(UInt64)`/`*` in UntoldGSFormat.read would trap the process on a + // file like this instead of throwing a catchable error. Must be a well-formed v2 header + // (real version, real byte count) so this test actually exercises the overflow guard in + // .sizeMismatch, not just the unrelated .unsupportedVersion check. + var bytes: [UInt8] = [] + bytes += [0x55, 0x54, 0x47, 0x53] // magic "UTGS", little-endian + bytes += [2, 0, 0, 0] // version = 2 + bytes += [UInt8](repeating: 0xFF, count: 8) // splatCount = UInt64.max + bytes += [0, 0, 0, 0] // shDegree + bytes += [0, 0, 0, 0] // shCoefficientsPerChannel + bytes += [0, 0, 0, 0] // shHigherOrderCoefficientsPerSplat + bytes += [0, 0, 0, 0] // meanSquaredSplatExtent (0.0 as Float bit pattern) + bytes += [UInt8](repeating: 0, count: 8) // encodedByteCount + bytes += [UInt8](repeating: 0, count: 8) // shByteCount + bytes += [UInt8](repeating: 0, count: 24) // boundingBoxMin/boundingBoxMax (6 floats) + XCTAssertEqual(bytes.count, 72) + + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("UntoldGSFormat-overflow-\(UUID().uuidString)") + .appendingPathExtension("untoldgs") + try Data(bytes).write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + + XCTAssertThrowsError(try UntoldGSFormat.read(from: url)) { error in + guard case UntoldGSError.sizeMismatch = error else { + XCTFail("Expected .sizeMismatch, got \(error)") + return + } + } + } + + func testReadRejectsV1HeaderAsUnsupportedVersion() throws { + // A well-formed pre-bounding-box v1 file (48-byte header, no boundingBoxMin/Max) must + // fail loudly with .unsupportedVersion rather than silently misreading 24 bytes of + // splat/SH payload as a bounding box. .untoldgs is a regeneratable cache of the source + // .ply — the fix for a file like this is re-running `untoldengine export`, not a + // best-effort read. + var bytes: [UInt8] = [] + bytes += [0x55, 0x54, 0x47, 0x53] // magic "UTGS", little-endian + bytes += [1, 0, 0, 0] // version = 1 + bytes += [UInt8](repeating: 0, count: 8) // splatCount = 0 + bytes += [0, 0, 0, 0] // shDegree + bytes += [0, 0, 0, 0] // shCoefficientsPerChannel + bytes += [0, 0, 0, 0] // shHigherOrderCoefficientsPerSplat + bytes += [0, 0, 0, 0] // meanSquaredSplatExtent + bytes += [UInt8](repeating: 0, count: 8) // encodedByteCount = 0 + bytes += [UInt8](repeating: 0, count: 8) // shByteCount = 0 + XCTAssertEqual(bytes.count, 48) + + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("UntoldGSFormat-v1-\(UUID().uuidString)") + .appendingPathExtension("untoldgs") + try Data(bytes).write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + + XCTAssertThrowsError(try UntoldGSFormat.read(from: url)) { error in + guard case let UntoldGSError.unsupportedVersion(version) = error else { + XCTFail("Expected .unsupportedVersion, got \(error)") + return + } + XCTAssertEqual(version, 1) + } + } + + func testWriteBakesBoundingBoxIntoHeaderAtExpectedOffsets() throws { + let output = FileManager.default.temporaryDirectory + .appendingPathComponent("UntoldGSFormat-boxheader-\(UUID().uuidString)") + .appendingPathExtension("untoldgs") + let bakeResult = try bakeGaussianSplatProgressiveTiers( + plyURL: testPLYURL(), + outputBaseURL: output, + lodFractions: [1.0] + ) + let tierURL = try XCTUnwrap(bakeResult.tiers.first?.url) + + // Read the raw bytes independently of UntoldGSFormat.read to guard the actual on-disk + // contract (version + fixed offsets), not just whatever read() happens to decode. + let data = try Data(contentsOf: tierURL) + let versionBits = data[4 ..< 8].withUnsafeBytes { $0.loadUnaligned(fromByteOffset: 0, as: UInt32.self) } + XCTAssertEqual(UInt32(littleEndian: versionBits), 2) + + func readFloat(at offset: Int) -> Float { + let bits = data[offset ..< offset + 4].withUnsafeBytes { $0.loadUnaligned(fromByteOffset: 0, as: UInt32.self) } + return Float(bitPattern: UInt32(littleEndian: bits)) + } + let boundingBoxMin = simd_float3(readFloat(at: 48), readFloat(at: 52), readFloat(at: 56)) + let boundingBoxMax = simd_float3(readFloat(at: 60), readFloat(at: 64), readFloat(at: 68)) + XCTAssertEqual(boundingBoxMin, bakeResult.boundingBoxMin) + XCTAssertEqual(boundingBoxMax, bakeResult.boundingBoxMax) + + // Round-trip through the public read() API should agree with the raw bytes. + let readBack = try UntoldGSFormat.read(from: tierURL) + XCTAssertEqual(readBack.boundingBoxMin, bakeResult.boundingBoxMin) + XCTAssertEqual(readBack.boundingBoxMax, bakeResult.boundingBoxMax) + } +} diff --git a/Tests/UntoldEngineRenderTests/GaussianRenderingTest.swift b/Tests/UntoldEngineRenderTests/GaussianRenderingTest.swift index 337f1799..a621b750 100644 --- a/Tests/UntoldEngineRenderTests/GaussianRenderingTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianRenderingTest.swift @@ -245,7 +245,7 @@ final class GaussianRenderingTest: BaseRenderSetup { return } component.sphericalHarmonicsData = renderInfo.device.makeBuffer( - length: MemoryLayout.stride, + length: MemoryLayout.stride, options: .storageModeShared ) component.sphericalHarmonicsMetadata = GaussianSHMetadata( @@ -261,6 +261,8 @@ final class GaussianRenderingTest: BaseRenderSetup { } func testPackedSphericalHarmonicsRoundTripsThroughMetalBuffer() throws { + // Coefficients 0.125...1.5 span in-range and clamped values so the + // round trip exercises both regimes. let sphericalHarmonics = GaussianSphericalHarmonics( degree: 1, coefficientsPerChannel: 4, @@ -269,15 +271,15 @@ final class GaussianRenderingTest: BaseRenderSetup { let packed = try packGaussianSphericalHarmonics(sphericalHarmonics, splatCount: 1) guard let buffer = renderInfo.device.makeBuffer( bytes: packed.coefficients, - length: packed.coefficients.count * MemoryLayout.stride, + length: packed.coefficients.count * MemoryLayout.stride, options: .storageModeShared ) else { XCTFail("Expected SH buffer allocation") return } - XCTAssertEqual(buffer.length, 9 * MemoryLayout.stride) - let pointer = buffer.contents().bindMemory(to: Float16.self, capacity: 9) + XCTAssertEqual(buffer.length, 9 * MemoryLayout.stride) + let pointer = buffer.contents().bindMemory(to: UInt8.self, capacity: 9) let roundTripped = (0 ..< 9).map { pointer[$0] } XCTAssertEqual(roundTripped, packed.coefficients) } @@ -299,16 +301,20 @@ final class GaussianRenderingTest: BaseRenderSetup { let packed = try packGaussianSphericalHarmonics(sphericalHarmonics, splatCount: 1) let baseColor = simd_float4(0.35, 0.5, 0.65, 1) let direction = simd_float4(0.25, -0.5, 0.75, 0) + // Dequantize the same way loadGaussianSHCoefficient does on the GPU, + // so the CPU and GPU sides evaluate the same quantized inputs and can + // be compared without any tolerance for the quantization step itself. + let dequantized = packed.coefficients.map { (Float($0) - 128) / 128 } let cpuResult = evaluateGaussianSphericalHarmonics( baseColor: simd_float3(baseColor.x, baseColor.y, baseColor.z), - higherOrderCoefficients: packed.coefficients.map(Float.init), + higherOrderCoefficients: dequantized, degree: 3, direction: simd_float3(direction.x, direction.y, direction.z) ) guard let coefficients = renderInfo.device.makeBuffer( bytes: packed.coefficients, - length: packed.coefficients.count * MemoryLayout.stride, + length: packed.coefficients.count * MemoryLayout.stride, options: .storageModeShared ), let output = renderInfo.device.makeBuffer( length: 2 * MemoryLayout.stride, @@ -371,11 +377,11 @@ final class GaussianRenderingTest: BaseRenderSetup { XCTAssertEqual(metadata.degree, 3) XCTAssertEqual(metadata.coefficientsPerChannel, 16) XCTAssertEqual(metadata.higherOrderCoefficientsPerSplat, 45) - XCTAssertEqual(coefficientBuffer.length, splatCount * 45 * MemoryLayout.stride) + XCTAssertEqual(coefficientBuffer.length, splatCount * 45 * MemoryLayout.stride) XCTAssertEqual(splatBuffer.length, splatCount * MemoryLayout.stride) let coefficients = coefficientBuffer.contents().bindMemory( - to: Float16.self, + to: UInt8.self, capacity: splatCount * 45 ) let sampledValues = [0, splatCount * 45 / 2, splatCount * 45 - 1].map { coefficients[$0] } diff --git a/Tests/UntoldEngineRenderTests/GaussianStreamingTest.swift b/Tests/UntoldEngineRenderTests/GaussianStreamingTest.swift index a8669330..ed216056 100644 --- a/Tests/UntoldEngineRenderTests/GaussianStreamingTest.swift +++ b/Tests/UntoldEngineRenderTests/GaussianStreamingTest.swift @@ -148,10 +148,10 @@ final class GaussianStreamingTest: BaseRenderSetup { XCTAssertEqual(component.assetKind, .mesh, "❌ Default assetKind should be .mesh so existing mesh streaming call sites are unaffected") } - // MARK: - setEntityGaussianStreamable / findTileEntity + // MARK: - setEntityGaussianStreaming / findTileEntity /// End-to-end check of the consolidated API: given a positioned entity and a tile - /// whose bounds contain that position, `setEntityGaussianStreamable` should parent it + /// whose bounds contain that position, `setEntityGaussianStreaming` should parent it /// under that tile, register it with the octree, and configure it as a real gaussian /// streaming candidate — equivalent to `makeUnloadedGaussianEntity`'s manual assembly, /// but via the one public call callers are meant to use. @@ -168,13 +168,14 @@ final class GaussianStreamingTest: BaseRenderSetup { let entity = createEntity() translateTo(entityId: entity, position: .zero) - setEntityGaussianStreamable( + setEntityGaussianStreaming( entityId: entity, - filename: "test_gaussians", - withExtension: "ply", - streamingRadius: 10.0, - unloadRadius: 20.0, - boundingBoxHalfExtent: simd_float3(1, 1, 1) + source: .single(filename: "test_gaussians", withExtension: "ply"), + options: GaussianStreamingOptions( + streamingRadius: 10.0, + unloadRadius: 20.0, + boundingBoxHalfExtent: simd_float3(1, 1, 1) + ) ) XCTAssertEqual( @@ -201,23 +202,166 @@ final class GaussianStreamingTest: BaseRenderSetup { /// If no tile contains the entity's position, the entity should be left as a plain, /// non-streaming entity rather than crashing or silently half-configuring it. - func testSetEntityGaussianStreamable_noContainingTileLeavesEntityNonStreaming() { + func testSetEntityGaussianStreaming_noContainingTileLeavesEntityNonStreaming() { let entity = createEntity() translateTo(entityId: entity, position: simd_float3(1000, 1000, 1000)) // no tile out here + setEntityGaussianStreaming( + entityId: entity, + source: .single(filename: "test_gaussians", withExtension: "ply"), + options: GaussianStreamingOptions( + boundingBoxHalfExtent: simd_float3(1, 1, 1) + ) + ) + + XCTAssertNil( + scene.get(component: StreamingComponent.self, for: entity), + "❌ Entity should not get a StreamingComponent when no containing tile is found" + ) + } + + /// Creates a tile at the origin with bounds large enough to contain any entity position + /// this file's streaming-registration tests use, and registers it in the octree. + @discardableResult + private func makeContainingTileAtOrigin() -> EntityID { + let tileRoot = createEntity() + if let tileComp = scene.assign(to: tileRoot, component: TileComponent.self) { + tileComp.state = .parsed + } + if let tileLocal = scene.get(component: LocalTransformComponent.self, for: tileRoot) { + tileLocal.boundingBox = (min: simd_float3(-10, -10, -10), max: simd_float3(10, 10, 10)) + } + OctreeSystem.shared.registerEntity(tileRoot) + return tileRoot + } + + // MARK: - Auto-resolved bounding box (baked .untoldgs header) + + /// `.untoldgs` sources have a real box baked into their v2 header (see + /// `UntoldGSFormat.readHeader`) -- `setEntityGaussianStreamable` should read it + /// synchronously at registration time when no explicit override is supplied, rather than + /// leaving the entity with a degenerate placeholder box. + func testSetEntityGaussianStreamable_autoResolvesBoundingBoxFromBakedUntoldgsHeader() throws { + makeContainingTileAtOrigin() + + let plyURL = try XCTUnwrap(LoadingSystem.shared.resourceURL(forResource: "test_gaussians", withExtension: "ply", subResource: nil)) + let asset = try PLYReader.readGaussianAsset(from: plyURL) + let trueBox = computeGaussianSplatBoundingBox(asset.splats) + + let output = FileManager.default.temporaryDirectory + .appendingPathComponent("GaussianStreamingTest-\(UUID().uuidString)") + .appendingPathExtension("untoldgs") + _ = try bakeGaussianSplatProgressiveTiers(plyURL: plyURL, outputBaseURL: output, lodFractions: [1.0]) + + let entity = createEntity() + translateTo(entityId: entity, position: .zero) + + // Deliberately omit boundingBoxHalfExtent. + setEntityGaussianStreamable( + entityId: entity, + filename: output.deletingPathExtension().path, + withExtension: "untoldgs" + ) + + let boundingBox = try XCTUnwrap(scene.get(component: LocalTransformComponent.self, for: entity)?.boundingBox) + XCTAssertEqual(boundingBox.min, trueBox.min) + XCTAssertEqual(boundingBox.max, trueBox.max) + XCTAssertNotNil(scene.get(component: StreamingComponent.self, for: entity), "❌ Entity should still register for streaming once the box resolves from the baked header") + } + + /// A raw `.ply` source has no baked header to fall back to, so omitting + /// `boundingBoxHalfExtent` there should still leave the entity non-streaming (with a + /// warning), not register a degenerate placeholder box. + func testSetEntityGaussianStreamable_rawPLYWithoutOverrideLeavesEntityNonStreaming() { + makeContainingTileAtOrigin() + + let entity = createEntity() + translateTo(entityId: entity, position: .zero) + + // Deliberately omit boundingBoxHalfExtent for a raw .ply source. setEntityGaussianStreamable( entityId: entity, filename: "test_gaussians", - withExtension: "ply", - boundingBoxHalfExtent: simd_float3(1, 1, 1) + withExtension: "ply" ) XCTAssertNil( scene.get(component: StreamingComponent.self, for: entity), - "❌ Entity should not get a StreamingComponent when no containing tile is found" + "❌ Entity should not get a StreamingComponent when no override is given and the source has no baked box" ) } + /// Same auto-resolve behavior as above, for the progressive-tier streaming entry point -- + /// the box should come from the coarsest tier's baked header and be marked explicit so it's + /// never overwritten by the (now effectively redundant) async-load fallback. + func testSetEntityGaussianProgressiveStreamable_autoResolvesBoundingBoxFromBakedHeader() throws { + makeContainingTileAtOrigin() + + let plyURL = try XCTUnwrap(LoadingSystem.shared.resourceURL(forResource: "test_gaussians", withExtension: "ply", subResource: nil)) + let asset = try PLYReader.readGaussianAsset(from: plyURL) + let trueBox = computeGaussianSplatBoundingBox(asset.splats) + + let output = FileManager.default.temporaryDirectory + .appendingPathComponent("GaussianStreamingTest-\(UUID().uuidString)") + .appendingPathExtension("untoldgs") + _ = try bakeGaussianSplatProgressiveTiers(plyURL: plyURL, outputBaseURL: output, levelCount: 3) + + let entity = createEntity() + translateTo(entityId: entity, position: .zero) + + // Deliberately omit boundingBoxHalfExtent. + setEntityGaussianProgressiveStreamable( + entityId: entity, + baseFilename: output.deletingPathExtension().path, + levelCount: 3, + maxDistances: [20, 50, .greatestFiniteMagnitude] + ) + + let boundingBox = try XCTUnwrap(scene.get(component: LocalTransformComponent.self, for: entity)?.boundingBox) + XCTAssertEqual(boundingBox.min, trueBox.min) + XCTAssertEqual(boundingBox.max, trueBox.max) + XCTAssertNotNil(scene.get(component: StreamingComponent.self, for: entity)) + XCTAssertEqual(scene.get(component: GaussianLODComponent.self, for: entity)?.hasExplicitBoundingBox, true) + } + + /// A pre-v2 (48-byte, no baked box) `.untoldgs` tier can't have its header read as a box, + /// so with no override supplied, registration should fail cleanly -- no `StreamingComponent`, + /// and no half-configured `GaussianLODComponent` left behind for `GaussianLODSystem` to keep + /// processing every frame on an entity that never actually streams. + func testSetEntityGaussianProgressiveStreamable_unreadableHeaderLeavesNoLeakedComponent() throws { + makeContainingTileAtOrigin() + + var bytes: [UInt8] = [] + bytes += [0x55, 0x54, 0x47, 0x53] // magic "UTGS" + bytes += [1, 0, 0, 0] // version = 1 (pre-box format) + bytes += [UInt8](repeating: 0, count: 8) // splatCount = 0 + bytes += [0, 0, 0, 0] // shDegree + bytes += [0, 0, 0, 0] // shCoefficientsPerChannel + bytes += [0, 0, 0, 0] // shHigherOrderCoefficientsPerSplat + bytes += [0, 0, 0, 0] // meanSquaredSplatExtent + bytes += [UInt8](repeating: 0, count: 8) // encodedByteCount = 0 + bytes += [UInt8](repeating: 0, count: 8) // shByteCount = 0 + + let output = FileManager.default.temporaryDirectory + .appendingPathComponent("GaussianStreamingTest-v1header-\(UUID().uuidString)") + .appendingPathExtension("untoldgs") + try Data(bytes).write(to: output) + defer { try? FileManager.default.removeItem(at: output) } + + let entity = createEntity() + translateTo(entityId: entity, position: .zero) + + setEntityGaussianProgressiveStreamable( + entityId: entity, + baseFilename: output.deletingPathExtension().path, + levelCount: 1, + maxDistances: [.greatestFiniteMagnitude] + ) + + XCTAssertNil(scene.get(component: StreamingComponent.self, for: entity), "❌ Should not register for streaming") + XCTAssertNil(scene.get(component: GaussianLODComponent.self, for: entity), "❌ Should not leave a half-configured GaussianLODComponent behind") + } + /// `findTileEntity` should return the tile whose bounds contain the query point, and /// `nil` when nothing does. func testFindTileEntity_returnsContainingTileOrNil() { diff --git a/Tests/UntoldEngineTests/PLYReaderTest.swift b/Tests/UntoldEngineTests/PLYReaderTest.swift index f4302068..5335e402 100644 --- a/Tests/UntoldEngineTests/PLYReaderTest.swift +++ b/Tests/UntoldEngineTests/PLYReaderTest.swift @@ -341,12 +341,13 @@ final class PLYReaderTest: XCTestCase { for splatIndex in sampleIndices { let sourceBase = splatIndex * 48 let packedBase = splatIndex * 45 - let higherOrder = packed.coefficients[packedBase ..< packedBase + 45].map(Float.init) + let higherOrder = packed.coefficients[packedBase ..< packedBase + 45].map { (Float($0) - 128) / 128 } for channel in 0 ..< 3 { for coefficient in 0 ..< 15 { + let sourceValue = sphericalHarmonics.coefficients[sourceBase + channel * 16 + coefficient + 1] XCTAssertEqual( - higherOrder[channel * 15 + coefficient], - Float(Float16(sphericalHarmonics.coefficients[sourceBase + channel * 16 + coefficient + 1])) + packed.coefficients[packedBase + channel * 15 + coefficient], + quantizeGaussianSHCoefficient(sourceValue) ) } } @@ -365,7 +366,7 @@ final class PLYReaderTest: XCTestCase { print("Gaussian diagnostic: splats=\(asset.splats.count), degree=\(sphericalHarmonics.degree), " + "sourceCoefficients=\(sphericalHarmonics.coefficients.count), " - + "packedCoefficients=\(packed.coefficients.count), packedBytes=\(packed.coefficients.count * MemoryLayout.stride)") + + "packedCoefficients=\(packed.coefficients.count), packedBytes=\(packed.coefficients.count * MemoryLayout.stride)") } func test_readGaussianAsset_binaryAndASCIIHaveIdenticalSphericalHarmonics() throws { @@ -428,10 +429,12 @@ final class PLYReaderTest: XCTestCase { } func test_packGaussianSphericalHarmonics_preservesHigherOrderChannelMajorLayout() throws { - XCTAssertEqual(MemoryLayout.stride, 2) + XCTAssertEqual(MemoryLayout.stride, 1) - let first = (0 ..< 12).map { Float($0) } - let second = (100 ..< 112).map { Float($0) } + // Coefficient 0 of each channel (the DC term, dropped by packing) is + // set to an out-of-range value so an off-by-one would be obvious. + let first: [Float] = [999, 0.1, 0.2, 0.3, 999, 0.4, 0.5, 0.6, 999, 0.7, 0.8, 0.9] + let second: [Float] = [999, -0.1, -0.2, -0.3, 999, -0.4, -0.5, -0.6, 999, -0.7, -0.8, -0.9] let sphericalHarmonics = GaussianSphericalHarmonics( degree: 1, coefficientsPerChannel: 4, @@ -440,9 +443,9 @@ final class PLYReaderTest: XCTestCase { let packed = try packGaussianSphericalHarmonics(sphericalHarmonics, splatCount: 2) let expected = [ - 1, 2, 3, 5, 6, 7, 9, 10, 11, - 101, 102, 103, 105, 106, 107, 109, 110, 111, - ].map(Float16.init) + 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, + -0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, + ].map(quantizeGaussianSHCoefficient) XCTAssertEqual(packed.coefficients, expected) XCTAssertEqual(packed.metadata.degree, 1) @@ -465,7 +468,7 @@ final class PLYReaderTest: XCTestCase { XCTAssertEqual(packed.metadata.higherOrderCoefficientsPerSplat, 0) } - func test_packGaussianSphericalHarmonics_rejectsInvalidCountAndFloat16Overflow() { + func test_packGaussianSphericalHarmonics_rejectsInvalidCountAndNonFiniteCoefficients() { let invalidCount = GaussianSphericalHarmonics( degree: 1, coefficientsPerChannel: 4, @@ -473,14 +476,26 @@ final class PLYReaderTest: XCTestCase { ) XCTAssertThrowsError(try packGaussianSphericalHarmonics(invalidCount, splatCount: 1)) - var overflowCoefficients = [Float](repeating: 0, count: 12) - overflowCoefficients[1] = Float.greatestFiniteMagnitude - let overflow = GaussianSphericalHarmonics( + // Out-of-[-1,1] magnitudes are clamped, not rejected — only a + // non-finite coefficient (NaN) can't be meaningfully quantized. + var outOfRangeCoefficients = [Float](repeating: 0, count: 12) + outOfRangeCoefficients[1] = Float.greatestFiniteMagnitude + let outOfRange = GaussianSphericalHarmonics( degree: 1, coefficientsPerChannel: 4, - coefficients: overflowCoefficients + coefficients: outOfRangeCoefficients ) - XCTAssertThrowsError(try packGaussianSphericalHarmonics(overflow, splatCount: 1)) + let packed = try? packGaussianSphericalHarmonics(outOfRange, splatCount: 1) + XCTAssertEqual(packed?.coefficients.first, 255, "Extreme positive values should clamp to the top of the byte range") + + var nanCoefficients = [Float](repeating: 0, count: 12) + nanCoefficients[1] = .nan + let nan = GaussianSphericalHarmonics( + degree: 1, + coefficientsPerChannel: 4, + coefficients: nanCoefficients + ) + XCTAssertThrowsError(try packGaussianSphericalHarmonics(nan, splatCount: 1)) } // MARK: - Test Error Handling diff --git a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/ExportCommand.swift b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/ExportCommand.swift index b99b3d6a..0f8a91eb 100644 --- a/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/ExportCommand.swift +++ b/Tools/UntoldEngineCLI/Sources/UntoldEngineCLI/ExportCommand.swift @@ -10,6 +10,7 @@ import ArgumentParser import Foundation +import UntoldEngine struct ExportCommand: ParsableCommand { static let configuration = CommandConfiguration( @@ -24,16 +25,20 @@ struct ExportCommand: ParsableCommand { references — equivalent to running --compress-geometry followed by `untoldengine texbake --dir` and `untoldengine texbake --patch-refs`. + Gaussian `.ply` inputs skip Blender and export directly to `.untoldgs`. + Example: untoldengine export --input model.usdz --output model.untold --convert-orientation --optimize untoldengine export --input model.blend --output model.untold --convert-orientation --optimize + untoldengine export --input splats.ply --output splats.untoldgs + untoldengine export --input splats.ply --output splats.untoldgs --lod-levels 4 """ ) - @Option(name: .long, help: "Source .usd, .usda, .usdc, .usdz, or .blend asset") + @Option(name: .long, help: "Source .usd, .usda, .usdc, .usdz, .blend, or Gaussian .ply asset") var input: String - @Option(name: .long, help: "Destination .untold file") + @Option(name: .long, help: "Destination .untold or .untoldgs file") var output: String @Option(name: .long, help: "Override the Blender executable path") @@ -81,6 +86,9 @@ struct ExportCommand: ParsableCommand { /// Keep in sync with MAX_BAKE_RESOLUTION in scripts/untoldexplorer.py. static let maxBakeResolution = 8192 + @Option(name: .customLong("lod-levels"), help: "Gaussian .ply export only: number of progressive .untoldgs tiers to generate. Default 1 writes --output directly; values greater than 1 write _lod0.untoldgs, _lod1.untoldgs, ...") + var lodLevels: Int = 1 + func run() throws { let inputURL = resolvePath(input).standardizedFileURL let outputURL = resolvePath(output).standardizedFileURL @@ -96,6 +104,17 @@ struct ExportCommand: ParsableCommand { printInfo("--bake-resolution \(bakeResolution) is very high; clamping to \(clampedBakeResolution).") } + if inputURL.pathExtension.lowercased() == "ply" { + guard outputURL.pathExtension.lowercased() == "untoldgs" else { + throw ExportError.unsupportedPLYExportOutput(outputURL.pathExtension) + } + guard lodLevels > 0 else { + throw ExportError.invalidLODLevels(lodLevels) + } + try runGaussianSplatExport(inputURL: inputURL, outputURL: outputURL) + return + } + let blenderURL = try resolveBlender() let exporterURL = try resolveExporter() @@ -151,6 +170,28 @@ struct ExportCommand: ParsableCommand { } } + private func runGaussianSplatExport(inputURL: URL, outputURL: URL) throws { + printInfo("Exporting Gaussian splats \(inputURL.path)") + let bakeResult = try bakeGaussianSplatProgressiveTiers( + plyURL: inputURL, + outputBaseURL: outputURL, + levelCount: lodLevels + ) + // meanSquaredSplatExtent is baked into each .untoldgs file and read automatically when + // the engine loads it — printed here only as a diagnostic (e.g. to compare density + // across source captures), not something to copy anywhere. + for tier in bakeResult.tiers { + printSuccess("Exported: \(tier.url.path) (meanSquaredSplatExtent: \(tier.meanSquaredSplatExtent))") + } + + // boundingBoxHalfExtent is NOT baked into the files (the engine can auto-compute it for + // non-streaming loads instead) — pass this into setEntityGaussianProgressive/ + // setEntityGaussianStreaming's boundingBoxHalfExtent if you want it set explicitly, e.g. + // for the streaming path, which requires a real box before any tier is ever read. + let halfExtent = (bakeResult.boundingBoxMax - bakeResult.boundingBoxMin) * 0.5 + printInfo("boundingBoxHalfExtent: (\(halfExtent.x), \(halfExtent.y), \(halfExtent.z))") + } + private func optimizeTextures(outputURL: URL) throws { let texturesDir = outputURL.deletingLastPathComponent().appendingPathComponent("Textures") guard validateDirectory(texturesDir) else { @@ -199,6 +240,8 @@ enum ExportError: LocalizedError { case exportFailed(Int32) case optimizeFailed(Int32) case invalidBakeResolution(Int) + case unsupportedPLYExportOutput(String) + case invalidLODLevels(Int) var errorDescription: String? { switch self { @@ -212,6 +255,11 @@ enum ExportError: LocalizedError { return "Texture optimization (texbake) failed with exit status \(status)" case let .invalidBakeResolution(value): return "--bake-resolution must be a positive integer, got \(value)" + case let .unsupportedPLYExportOutput(pathExtension): + let suffix = pathExtension.isEmpty ? "" : ".\(pathExtension)" + return "Gaussian .ply export supports only .untoldgs output, got \(suffix)" + case let .invalidLODLevels(value): + return "--lod-levels must be a positive integer, got \(value)" } } } diff --git a/docs/API/UsingGaussianSystem.md b/docs/API/UsingGaussianSystem.md index 29f14ea9..3882cb0a 100644 --- a/docs/API/UsingGaussianSystem.md +++ b/docs/API/UsingGaussianSystem.md @@ -22,6 +22,15 @@ To display a Gaussian Splat model, load its .ply file and link it to the entity setEntityGaussian(entityId: myEntity, filename: "splat", withExtension: "ply") ``` +You can also use the source-based API: + +```swift +setEntityGaussian( + entityId: myEntity, + source: .single(filename: "splat", withExtension: "ply") +) +``` + Parameters: - entityId: The ID of the entity created earlier. @@ -30,6 +39,44 @@ Parameters: > Note: The Gaussian System renders point cloud data stored in the .ply format. Ensure your Gaussian Splat file is properly formatted and contains the necessary attributes (position, color, opacity, scale, rotation). +Both forms load synchronously and keep the splat resident for the entity's lifetime, and both +compute the entity's `LocalTransformComponent.boundingBox` automatically from the loaded splat +positions — no bounding box parameter is needed for this path. + +--- + +### Step 3: Loading Without Blocking the Main Thread + +`setEntityGaussianAsync` does the same immediate/resident load as `setEntityGaussian`, but +parsing, per-splat encoding, and spherical-harmonics packing all run off the main thread — +only the final component registration touches the world. Use it for a one-off splat load +where you don't want a frame hitch but don't need distance-based streaming. + +```swift +Task { + let ok = await setEntityGaussianAsync( + entityId: myEntity, + filename: "splat", + withExtension: "ply" + ) + if !ok { + print("Failed to load splat") + } +} +``` + +`completion` is an optional alternative to checking the returned `Bool`: + +```swift +await setEntityGaussianAsync( + entityId: myEntity, + filename: "splat", + withExtension: "ply" +) { success in + print(success ? "Loaded" : "Failed to load splat") +} +``` + --- ### Running the Gaussian System @@ -42,6 +89,113 @@ Once everything is set up: --- +## Progressive Gaussian Splats + +Progressive Gaussian loading is available without a tile-streamed scene. Use it when you +want a Gaussian to appear quickly at a coarse tier, then refine toward full resolution as +the camera gets closer. + +Progressive assets use `.untoldgs` tier files: + +```text +_lod0.untoldgs +_lod1.untoldgs +_lod2.untoldgs +... +``` + +`lod0` is the finest/full-resolution tier. Higher LOD numbers are progressively coarser. +The engine loads the coarsest tier first, then `GaussianLODSystem` requests finer tiers +based on camera distance (see [Overdraw-aware LOD selection](#overdraw-aware-lod-selection) +below for a second, distance-independent signal that can also hold an entity on a coarser +tier). + +Generate tiers from a `.ply` source with the exporter: + +```bash +untoldengine export --input "chair.ply" --output "chair.untoldgs" --lod-levels 4 +``` + +The exporter prints a diagnostic `meanSquaredSplatExtent` per tier and a `boundingBoxHalfExtent` +line computed from the full source asset: + +```text +✅ Exported: chair_lod0.untoldgs (meanSquaredSplatExtent: 0.0021) +✅ Exported: chair_lod1.untoldgs (meanSquaredSplatExtent: 0.0087) +✅ Exported: chair_lod2.untoldgs (meanSquaredSplatExtent: 0.0341) +✅ Exported: chair_lod3.untoldgs (meanSquaredSplatExtent: 0.1250) +ℹ️ boundingBoxHalfExtent: (0.42, 0.55, 0.38) +``` + +Both values are baked directly into each tier's `.untoldgs` file (its header carries the +asset-level bounding box alongside `meanSquaredSplatExtent`) and read back automatically when +the engine loads it — nothing here needs to be copied into your code. The console lines are +diagnostics only (e.g. to sanity-check density/size across source captures). + +Then register the entity with the source-based API: + +```swift +let chair = createEntity() +translateTo(entityId: chair, position: simd_float3(0.0, 0.0, -3.0)) + +setEntityGaussian( + entityId: chair, + source: .progressive( + baseFilename: "chair", + levelCount: 4, + maxDistances: [5.0, 15.0, 25.0, .greatestFiniteMagnitude] + ) +) +``` + +`maxDistances` must have one entry per LOD. Each value is the farthest camera distance at +which that LOD is allowed to be selected: + +- `lod0` can be used inside `5.0` units. +- `lod1` can be used from `5.0` to `15.0` units. +- `lod2` can be used from `15.0` to `25.0` units. +- `lod3` is used beyond `25.0` units, or while finer tiers are still loading. + +The system always falls back to the best tier already resident in memory, so the entity can +become visible quickly with the coarsest tier and refine toward `lod0`. + +There's no bounding-box parameter to pass here: the engine reads the box baked into the +coarsest tier's header synchronously at registration time, so the entity has a correct, +exact bounding box from frame one. + +### Overdraw-aware LOD selection + +Distance alone is a proxy for how expensive a Gaussian entity is to render — two assets at +the same distance can have very different overdraw depending on splat density. On top of the +distance/`maxDistances` selection above, `GaussianLODSystem` also estimates the entity's mean +overdraw (blended fragments per pixel across its screen footprint) each LOD update, using +`meanSquaredSplatExtent` values baked into each `.untoldgs` tier by the exporter. If a +distance-selected tier would exceed `LODConfig.shared.gaussianOverdrawBudget` (default `12.0`, +see `LODConfig.swift`), the system walks to a coarser tier instead — it never picks a finer +tier than distance already allows. + +This is fully automatic for any asset baked with the current exporter — there is nothing to +wire up. It only has an effect on tiers that carry a real `meanSquaredSplatExtent`; `.untoldgs` +files baked before this feature existed fall back to pure distance-based selection. + +Tune `LODConfig.shared.gaussianOverdrawBudget` on-device by watching GPU frame time while +varying it — the default is a starting guess, not a derived constant. + +### Debugging progressive LOD + +Gaussian progressive LODs participate in the same LOD debug visualization used by mesh +LODs: + +```swift +setSpatialDebug(.lodLevels(true)) +``` + +When enabled, the renderer tints Gaussian splats by their currently selected progressive +LOD. This is useful for confirming that the engine is switching tiers as the camera moves, +including tiers the overdraw budget forces early. + +--- + ## Streaming Gaussian Splats in Large Scenes `setEntityGaussian` loads a splat immediately and keeps it resident for the lifetime of the @@ -50,15 +204,50 @@ scattered across a large tile-streamed scene (chairs, tables, decor inside a str building). Loading every one of those up front defeats the point of streaming, and the engine has no way to unload them again on its own. -For that case, use `setEntityGaussianStreamable` instead. It registers the entity with -`GeometryStreamingSystem`, which loads and unloads it automatically based on camera -distance — the same way it already handles the surrounding streamed tile geometry. +For that case, register the entity with `GeometryStreamingSystem` instead, via +`setEntityGaussianStreaming`, which loads and unloads it automatically based on camera +distance — the same way it already handles the surrounding streamed tile geometry. It can +stream either one whole Gaussian file or a progressive `.untoldgs` tier set. + +### API overview + +```swift +setEntityGaussianStreaming( + entityId: EntityID, + source: GaussianSource, + options: GaussianStreamingOptions +) +``` + +`GaussianSource` selects what kind of Gaussian asset the streaming system should load: + +```swift +.single(filename: String, withExtension: String) + +.progressive( + baseFilename: String, + withExtension: String = "untoldgs", + levelCount: Int, + maxDistances: [Float] +) +``` + +`GaussianStreamingOptions` controls the entity's streaming behavior: + +```swift +GaussianStreamingOptions( + streamingRadius: Float = 100.0, + unloadRadius: Float = 150.0, + boundingBoxHalfExtent: simd_float3? = nil, + priority: Int = 0 +) +``` ### Prerequisites This only makes sense in a scene that is already using tile-based streaming — i.e. one loaded with `setEntityStreamScene` (see [Using the Geometry Streaming System](UsingGeometryStreamingSystem.md)). -`setEntityGaussianStreamable` attaches the splat to whichever tile stub's bounds contain +`setEntityGaussianStreaming` attaches the splat to whichever tile stub's bounds contain the entity's position, so it needs those tile stubs to already exist. Call it **after** `setEntityStreamScene`'s completion handler has fired — tile stubs are guaranteed to be registered by then. @@ -66,7 +255,7 @@ registered by then. ### Step 1: Create and position the entity Position and orient the entity *before* registering it for streaming — the position at the -time you call `setEntityGaussianStreamable` is what determines which tile it gets attached +time you call `setEntityGaussianStreaming` is what determines which tile it gets attached to. ```swift @@ -78,47 +267,138 @@ rotateTo(entityId: streamSplat, angle: 180.0, axis: simd_float3(1.0, 0.0, 0.0)) ### Step 2: Register it for streaming ```swift -setEntityGaussianStreamable( +setEntityGaussianStreaming( entityId: streamSplat, - filename: "chair", - withExtension: "ply", - streamingRadius: 30.0, - unloadRadius: 45.0, - boundingBoxHalfExtent: simd_float3(0.5, 0.8, 0.5) + source: .single(filename: "chair", withExtension: "untoldgs"), + options: GaussianStreamingOptions( + streamingRadius: 30.0, + unloadRadius: 45.0 + ) ) ``` Parameters: - `entityId`: The entity created and positioned in Step 1. -- `filename` / `withExtension`: Same as `setEntityGaussian` — the `.ply` file to stream in - once the camera is in range. +- `source`: Use `.single(filename:withExtension:)` for a whole `.ply`/`.untoldgs` asset, or + `.progressive(baseFilename:levelCount:maxDistances:)` for progressive tiers named + `_lod0.untoldgs`, `_lod1.untoldgs`, etc. - `streamingRadius`: Distance from the camera at which the splat starts loading. - `unloadRadius`: Distance beyond which the splat unloads. Should be larger than `streamingRadius` to avoid load/unload thrashing at the boundary. -- `boundingBoxHalfExtent`: A local-space half-extent for the entity, roughly matching the - splat's real-world size. **This has no default and must be supplied.** Without a real - bounding box, the streaming system's frustum gate collapses to a zero-extent point at - the entity's exact position — the splat may load once but then fail to reliably - re-stream in after the camera moves away and back, because re-entry depends on the - camera looking at that exact point rather than anywhere near the prop. +- `boundingBoxHalfExtent`: Optional local-space half-extent for the entity, roughly matching + the splat's real-world size. `GeometryStreamingSystem`'s frustum gate needs a real + local-space volume on the entity *before* it ever loads, so a `.untoldgs` source's box is + read from its baked header synchronously at registration time when this is left `nil` — no + value needed for that case. **A raw `.ply` source has no baked header, so this must be + supplied explicitly there** — omitting it leaves the entity non-streaming (logged as a + warning) rather than registering a zero-size placeholder, which would collapse the frustum + gate to a single exact point and make re-streaming unreliable once the camera moves away + and back. When you do need one, the exporter's printed `boundingBoxHalfExtent` diagnostic + (see above) is a good starting value. - `priority`: Optional. Higher-priority entities load first when multiple candidates are in range at once. Defaults to `0`. -> Note: If no tile is found containing the entity's position, `setEntityGaussianStreamable` +> Note: If no tile is found containing the entity's position, `setEntityGaussianStreaming` > logs a warning and leaves the entity as a plain, non-streaming entity (no `StreamingComponent` > is attached) — it will not crash, but it also will not load. Double-check the position > against the streamed scene's tile bounds if this happens. -### Which function should I use? +### Progressive Gaussian splat streaming + +Use `.progressive(...)` with `setEntityGaussianStreaming` when you want tile-driven +load/unload behavior plus the same coarse-to-fine refinement (including the +[overdraw-aware LOD clamp](#overdraw-aware-lod-selection)) described above. + +Progressive tier filenames must follow this pattern: + +```text +_lod0.untoldgs +_lod1.untoldgs +_lod2.untoldgs +... +``` + +For example, if `baseFilename` is `"chair"` and `levelCount` is `4`, the engine expects: + +```text +chair_lod0.untoldgs +chair_lod1.untoldgs +chair_lod2.untoldgs +chair_lod3.untoldgs +``` + +```swift +setEntityGaussianStreaming( + entityId: streamSplat, + source: .progressive( + baseFilename: "chair", + levelCount: 4, + maxDistances: [5.0, 15.0, 25.0, .greatestFiniteMagnitude] + ), + options: GaussianStreamingOptions( + streamingRadius: 30.0, + unloadRadius: 45.0 + ) +) +``` + +`.untoldgs` progressive tiers always have a baked header, so `boundingBoxHalfExtent` can be +omitted here the same way it can for `.single(...)` with a `.untoldgs` file. + +### Putting it together: stream scene + streaming splat + +`setEntityGaussianStreaming` needs the tile stubs `setEntityStreamScene` creates (see +[Prerequisites](#prerequisites) above), so the natural place to register streaming splat props +is inside the same completion handler that loads the streamed tile scene: + +```swift +let sceneRoot = createEntity() +setEntityStreamScene(entityId: sceneRoot, manifest: "dungeon", withExtension: "json") { success in + guard success else { + setSceneReady(false) + return + } + + let splat = createEntity() + translateTo(entityId: splat, position: simd_float3(2.0, 0.0, -4.0)) + rotateBy(entityId: splat, angle: 180.0, axis: simd_float3(1.0, 0.0, 0.0)) + + setEntityGaussianStreaming( + entityId: splat, + source: .progressive( + baseFilename: "pooltable", + levelCount: 4, + maxDistances: [15.0, 25.0, 35.0, .greatestFiniteMagnitude] + ), + options: GaussianStreamingOptions( + streamingRadius: 100.0, + unloadRadius: 140.0 + ) + ) + + setSceneReady(true) +} +``` + +`boundingBoxHalfExtent` is omitted from `GaussianStreamingOptions` here since `pooltable` is a +`.untoldgs` progressive asset — its box comes from the baked header automatically (see +[API overview](#api-overview) above). Guarding on `success` before registering the splat and +calling `setSceneReady` matters: without it, a failed scene load would still try to attach a +streaming prop to tile stubs that were never created, and would report the scene ready when it +isn't. + +--- + +## Which function should I use? -- **`setEntityGaussian`** — load immediately, stays resident. Use for a small number of - splats that should always be visible (e.g. a single hero object, a standalone demo scene). -- **`setEntityGaussianAsync`** — same immediate/resident behavior, but the parse and GPU - upload happen off the main thread. Use for a one-off splat load where you don't want a - frame hitch, but still don't need distance-based unloading. -- **`setEntityGaussianStreamable`** — registers the entity with `GeometryStreamingSystem` - instead of loading anything itself. Use for splat props inside a large, tile-streamed - scene, where you want the same load-when-near/unload-when-far behavior the rest of the - scene already gets. +| Function | Resident/Streamed | LOD | Use when | +|---|---|---|---| +| `setEntityGaussian(entityId:filename:withExtension:)` | Resident, loads immediately (blocks) | None | A small number of splats that should always be visible (a hero object, a standalone demo scene). | +| `setEntityGaussian(entityId:source:)` | Resident | None (`.single`) or progressive (`.progressive`) | Same as above, plus a single call site that can also take `.progressive(...)` for coarse-to-fine refinement without a tile-streamed scene. | +| `setEntityGaussianAsync` | Resident, loads off-thread | None | Same as `setEntityGaussian`, but avoids a frame hitch on a large `.ply`. | +| `setEntityGaussianStreaming(source:options:)` | Streamed via `GeometryStreamingSystem` | None (`.single`) or progressive (`.progressive`) | Props scattered across a tile-streamed scene that should load/unload with camera distance. | +All progressive paths (`setEntityGaussian(source: .progressive(...))` and +`setEntityGaussianStreaming(source: .progressive(...), options:)`) share the same +[overdraw-aware LOD selection](#overdraw-aware-lod-selection) behavior automatically.