Add Gaussian Splatting support (.ply / .splat / .spz)

editor/src/editor/layout/assets-browser.tsx

@@ -896,6 +896,9 @@ export class EditorAssetsBrowser extends Component<IEditorAssetsBrowserProps, IE
 			case ".gltf":
 			case ".ms3d":
 			case ".babylon":
+			case ".ply":
+			case ".splat":
+			case ".spz":
 				return <MeshSelectable {...props} />;
 
 			case ".material":

editor/src/editor/layout/graph.tsx

@@ -58,6 +58,7 @@ import {
 	isCollisionInstancedMesh,
 	isCollisionMesh,
 	isEditorCamera,
+	isGaussianSplattingMesh,
 	isInstancedMesh,
 	isLight,
 	isMesh,
@@ -499,7 +500,7 @@ export class EditorGraph extends Component<IEditorGraphProps, IEditorGraphState>
 								}
 							}
 
-							if (isAbstractMesh(node)) {
+							if (isAbstractMesh(node) && !isGaussianSplattingMesh(node)) {
 								this.props.editor.layout.preview.scene.lights
 									.map((light) => light.getShadowGenerator())
 									.forEach((generator) => generator?.getShadowMap()?.renderList?.push(node));

editor/src/editor/layout/inspector/mesh/mesh.tsx

@@ -29,7 +29,7 @@ import { registerUndoRedo } from "../../../../tools/undoredo";
 import { waitNextAnimationFrame } from "../../../../tools/tools";
 import { onNodeModifiedObservable } from "../../../../tools/observables";
 import { updateIblShadowsRenderPipeline } from "../../../../tools/light/ibl";
-import { isAbstractMesh, isInstancedMesh, isMesh } from "../../../../tools/guards/nodes";
+import { isAbstractMesh, isGaussianSplattingMesh, isInstancedMesh, isMesh } from "../../../../tools/guards/nodes";
 import { updateAllLights, updateLightShadowMapRefreshRate, updatePointLightShadowMapRenderListPredicate } from "../../../../tools/light/shadows";
 
 import { applyMaterialAssetToObject } from "../../preview/import/material";
@@ -100,6 +100,10 @@ export class EditorMeshInspector extends Component<IEditorInspectorImplementatio
 	}
 
 	public render(): ReactNode {
+		// Gaussian splatting meshes have a quad geometry but no standard material/shadow/collision semantics,
+		// so the geometry-derived sections below are hidden for them.
+		const isGaussianSplatting = isGaussianSplattingMesh(this.props.object);
+
 		return (
 			<>
 				<EditorInspectorSectionField title="Common">
@@ -159,14 +163,14 @@ export class EditorMeshInspector extends Component<IEditorInspectorImplementatio
 					/>
 				</EditorInspectorSectionField>
 
-				{this.props.object.geometry && (
+				{this.props.object.geometry && !isGaussianSplatting && (
 					<>
 						<EditorMeshCollisionInspector {...this.props} />
 						<EditorMeshPhysicsInspector mesh={this.props.object} />
 					</>
 				)}
 
-				{this.props.editor.layout.preview.scene.lights.length > 0 && this.props.object.geometry && (
+				{this.props.editor.layout.preview.scene.lights.length > 0 && this.props.object.geometry && !isGaussianSplatting && (
 					<EditorInspectorSectionField title="Shadows">
 						<EditorInspectorSwitchField
 							label="Cast Shadows"
@@ -214,7 +218,11 @@ export class EditorMeshInspector extends Component<IEditorInspectorImplementatio
 			}
 		});
 
-		this.props.editor.layout.preview.selectionOutlineLayer.addSelection(this.props.object);
+		// The selection outline draws the mesh through a depth/outline pass that can't handle the thin-instance
+		// splat layout, so Gaussian splatting meshes are not added to it.
+		if (!isGaussianSplattingMesh(this.props.object)) {
+			this.props.editor.layout.preview.selectionOutlineLayer.addSelection(this.props.object);
+		}
 	}
 
 	public componentWillUnmount(): void {
@@ -236,7 +244,8 @@ export class EditorMeshInspector extends Component<IEditorInspectorImplementatio
 	}
 
 	private _getMaterialComponent(): ReactNode {
-		if (!this.props.object.geometry) {
+		// Gaussian splatting meshes manage their own internal material; there is nothing user-editable here.
+		if (!this.props.object.geometry || isGaussianSplattingMesh(this.props.object)) {
 			return;
 		}
 

editor/src/editor/layout/preview.tsx

@@ -1349,6 +1349,9 @@ export class EditorPreview extends Component<IEditorPreviewProps, IEditorPreview
 				case ".ms3d":
 				case ".blend":
 				case ".babylon":
+				case ".ply":
+				case ".splat":
+				case ".spz":
 					this.importSceneFile(absolutePath, ev.shiftKey).then((result) => {
 						if (pick.pickedPoint) {
 							result?.meshes.forEach((m) => !m.parent && m.position.addInPlace(pick.pickedPoint!));

editor/src/editor/layout/preview/import/import.ts

@@ -23,7 +23,7 @@ import {
 } from "babylonjs";
 
 import { UniqueNumber } from "../../../../tools/tools";
-import { isMesh } from "../../../../tools/guards/nodes";
+import { isGaussianSplattingMesh, isMesh } from "../../../../tools/guards/nodes";
 import { isSprite } from "../../../../tools/guards/sprites";
 import { isTexture } from "../../../../tools/guards/texture";
 import { executeSimpleWorker } from "../../../../tools/worker";
@@ -72,6 +72,14 @@ export async function loadImportedSceneFile(scene: Scene, absolutePath: string)
 	try {
 		result = await ImportMeshAsync(basename(absolutePath), scene, {
 			rootUrl: join(dirname(absolutePath), "/"),
+			// Keep Gaussian splatting data in RAM. Otherwise the splat buffer is freed right after being
+			// uploaded to the GPU, and `GaussianSplattingMesh.serialize()` (used when saving/exporting) would
+			// have nothing to write, producing an empty splat on reload. Ignored by non-splat loaders.
+			pluginOptions: {
+				splat: {
+					keepInRam: true,
+				},
+			},
 		});
 		// result = await SceneLoader.ImportMeshAsync("", join(dirname(absolutePath), "/"), basename(absolutePath), scene);
 	} catch (e) {
@@ -92,7 +100,17 @@ export async function loadImportedSceneFile(scene: Scene, absolutePath: string)
 	result.meshes.forEach((mesh) => {
 		configureImportedNodeIds(mesh);
 
-		mesh.receiveShadows = true;
+		// Gaussian splatting meshes render through thin instances with a custom splat buffer and cannot be
+		// drawn by the standard shadow/depth passes, so they must not participate in shadows.
+		mesh.receiveShadows = !isGaussianSplattingMesh(mesh);
+
+		if (isGaussianSplattingMesh(mesh)) {
+			// Gaussian splatting assets have no `__root__`, so apply the editor's centimeters convention (the
+			// same x100 scaling that glTF receives through its root) directly to the splat mesh. The SPLAT
+			// loader already orients the splat (it bakes `scaling.y = -1` to convert from the Y-down splat
+			// convention), so we only scale here and keep that orientation untouched.
+			mesh.scaling.scaleInPlace(100);
+		}
 
 		if (mesh.skeleton) {
 			mesh.skeleton.id = Tools.RandomId();
@@ -127,6 +145,11 @@ export async function loadImportedSceneFile(scene: Scene, absolutePath: string)
 		}
 
 		result.meshes.forEach((mesh) => {
+			// Gaussian splatting meshes can't be rendered into shadow maps (see above).
+			if (isGaussianSplattingMesh(mesh)) {
+				return;
+			}
+
 			shadowMap.renderList!.push(mesh);
 		});
 	});

editor/src/project/add/configure.ts

@@ -1,16 +1,21 @@
 import { AbstractMesh, Tools, Node } from "babylonjs";
 
 import { UniqueNumber } from "../../tools/tools";
+import { isGaussianSplattingMesh } from "../../tools/guards/nodes";
 
 import { Editor } from "../../editor/main";
 
 export function configureAddedMesh(editor: Editor, mesh: AbstractMesh, parent?: Node) {
-	mesh.receiveShadows = true;
+	// Gaussian splatting meshes render through thin instances and can't be drawn by the shadow/depth pass,
+	// so they neither receive shadows nor go into shadow-map render lists.
+	const isGaussianSplatting = isGaussianSplattingMesh(mesh);
+
+	mesh.receiveShadows = !isGaussianSplatting;
 	mesh.id = Tools.RandomId();
 	mesh.uniqueId = UniqueNumber.Get();
 	mesh.parent = parent ?? null;
 
-	if (mesh.geometry) {
+	if (mesh.geometry && !isGaussianSplatting) {
 		mesh.geometry.id = Tools.RandomId();
 		mesh.geometry.uniqueId = UniqueNumber.Get();
 

editor/src/project/export/export.tsx

@@ -11,7 +11,7 @@ import { getCollisionMeshFor } from "../../tools/mesh/collision";
 import { storeTexturesBaseSize } from "../../tools/material/texture";
 import { extractNodeMaterialTextures } from "../../tools/material/extract";
 import { createDirectoryIfNotExist, normalizedGlob } from "../../tools/fs";
-import { isCollisionMesh, isEditorCamera, isMesh } from "../../tools/guards/nodes";
+import { isCollisionMesh, isEditorCamera, isGaussianSplattingMesh, isMesh } from "../../tools/guards/nodes";
 import { extractNodeParticleSystemSetTextures, extractParticleSystemTextures } from "../../tools/particles/extract";
 
 import { taaPipelineCameraConfigurations } from "../../editor/rendering/taa";
@@ -113,7 +113,9 @@ async function _exportProject(editor: Editor, options: IExportProjectOptions): P
 
 	storeTexturesBaseSize(scene);
 
-	scene.meshes.forEach((mesh) => (mesh.doNotSerialize = mesh.metadata?.doNotSerialize ?? false));
+	// Keep internal hidden meshes (e.g. the GaussianSplattingMesh per-camera render proxies, which already
+	// set doNotSerialize) out of the export so they are not written as standalone meshes.
+	scene.meshes.forEach((mesh) => (mesh.doNotSerialize = mesh.reservedDataStore?.hidden || (mesh.metadata?.doNotSerialize ?? false)));
 	scene.lights.forEach((light) => (light.doNotSerialize = light.metadata?.doNotSerialize ?? false));
 	scene.cameras.forEach((camera) => (camera.doNotSerialize = camera.metadata?.doNotSerialize ?? false));
 	scene.transformNodes.forEach((transformNode) => (transformNode.doNotSerialize = transformNode.metadata?.doNotSerialize ?? false));
@@ -221,6 +223,24 @@ async function _exportProject(editor: Editor, options: IExportProjectOptions): P
 				}
 			}
 
+			// Gaussian splatting meshes embed their splat data inline and rebuild their own quad geometry at
+			// parse time (the loader skips importing geometry for them), so they must not go through the
+			// geometry externalization/delay-loading pipeline.
+			if (instantiatedMesh && isGaussianSplattingMesh(instantiatedMesh)) {
+				let geometryIndex = -1;
+				do {
+					geometryIndex = data.geometries?.vertexData?.findIndex((g) => g.id === mesh.geometryId) ?? -1;
+					if (geometryIndex !== -1) {
+						data.geometries!.vertexData!.splice(geometryIndex, 1);
+					}
+				} while (geometryIndex !== -1);
+
+				delete mesh.geometryId;
+				delete mesh.geometryUniqueId;
+				delete mesh.delayLoadingFile;
+				return;
+			}
+
 			const geometry = data.geometries?.vertexData?.find((v) => v.id === mesh.geometryId);
 
 			if (geometry) {

editor/src/project/load/plugins/meshes.ts

@@ -6,7 +6,7 @@ import { Scene, Constants, Matrix, Mesh, SceneLoader, MultiMaterial, Geometry }
 import { ISceneLoaderPluginOptions } from "../scene";
 
 import { wait } from "../../../tools/tools";
-import { isCollisionMesh, isMesh } from "../../../tools/guards/nodes";
+import { isCollisionMesh, isGaussianSplattingMesh, isMesh } from "../../../tools/guards/nodes";
 import { isMultiMaterial, isNodeMaterial } from "../../../tools/guards/material";
 import { parsePhysicsAggregate } from "../../../tools/physics/serialization/aggregate";
 import { configureSimultaneousLightsForMaterial, normalizeNodeMaterialUniqueIds } from "../../../tools/material/material";
@@ -40,7 +40,7 @@ export async function loadMeshes(meshesFiles: string[], scene: Scene, options: I
 					}
 
 					result.meshes.forEach((m) => {
-						if (!isMesh(m)) {
+						if (!isMesh(m) && !isGaussianSplattingMesh(m)) {
 							return;
 						}
 
@@ -109,7 +109,9 @@ export async function loadMeshes(meshesFiles: string[], scene: Scene, options: I
 
 						options.loadResult.meshes.push(m);
 
-						if (m.material) {
+						// Gaussian splatting meshes manage their own internal material recreated at parse time,
+						// so the regular material reconciliation below does not apply to them.
+						if (m.material && !isGaussianSplattingMesh(m)) {
 							const material = isMultiMaterial(m.material)
 								? data.multiMaterials?.find((d) => d.id === m.material!.id)
 								: data.materials?.find((d) => d.id === m.material!.id);

editor/src/project/load/plugins/shadow-generators.ts

@@ -5,6 +5,8 @@ import { Scene, ShadowGenerator, CascadedShadowGenerator, RenderTargetTexture }
 
 import { Editor } from "../../../editor/main";
 
+import { isGaussianSplattingMesh } from "../../../tools/guards/nodes";
+
 import { ISceneLoaderPluginOptions } from "../scene";
 
 export async function loadShadowGenerators(editor: Editor, shadowGeneratorFiles: string[], scene: Scene, options: ISceneLoaderPluginOptions) {
@@ -33,6 +35,12 @@ export async function loadShadowGenerators(editor: Editor, shadowGeneratorFiles:
 				const shadowMap = shadowGenerator.getShadowMap();
 				if (shadowMap) {
 					shadowMap.refreshRate = data.refreshRate ?? RenderTargetTexture.REFRESHRATE_RENDER_ONEVERYFRAME;
+
+					// Gaussian splatting meshes can't be rendered into shadow maps (their thin-instance splat
+					// layout breaks the depth pass), so drop any that an older project persisted in the render list.
+					if (shadowMap.renderList) {
+						shadowMap.renderList = shadowMap.renderList.filter((mesh) => !isGaussianSplattingMesh(mesh));
+					}
 				}
 			} catch (e) {
 				editor.layout.console.error(`Failed to load shadow generator file "${file}": ${e.message}`);

editor/src/project/save/scene.ts

@@ -19,7 +19,7 @@ import { isSpriteManagerNode, isSpriteMapNode } from "../../tools/guards/sprites
 import { serializePhysicsAggregate } from "../../tools/physics/serialization/aggregate";
 import { isAnimationGroupFromSceneLink, isFromSceneLink } from "../../tools/scene/scene-link";
 import { isGPUParticleSystem, isNodeParticleSystemSetMesh, isParticleSystem } from "../../tools/guards/particles";
-import { isAnyTransformNode, isClusteredLightContainer, isCollisionMesh, isEditorCamera, isMesh, isTransformNode } from "../../tools/guards/nodes";
+import { isAnyTransformNode, isClusteredLightContainer, isCollisionMesh, isEditorCamera, isGaussianSplattingMesh, isMesh, isTransformNode } from "../../tools/guards/nodes";
 
 import { taaPipelineCameraConfigurations } from "../../editor/rendering/taa";
 import { vlsPostProcessCameraConfigurations } from "../../editor/rendering/vls";
@@ -73,7 +73,7 @@ export async function saveScene(editor: Editor, projectPath: string, scenePath:
 
 	const scene = editor.layout.preview.scene;
 	const meshesToSave = scene.meshes.filter((mesh) => {
-		if ((!isMesh(mesh) && !isCollisionMesh(mesh)) || mesh._masterMesh || isFromSceneLink(mesh) || !isNodeVisibleInGraph(mesh)) {
+		if ((!isMesh(mesh) && !isCollisionMesh(mesh) && !isGaussianSplattingMesh(mesh)) || mesh._masterMesh || isFromSceneLink(mesh) || !isNodeVisibleInGraph(mesh)) {
 			return false;
 		}
 
@@ -99,7 +99,7 @@ export async function saveScene(editor: Editor, projectPath: string, scenePath:
 	// Write geometries and meshes
 	await Promise.all(
 		meshesToSave.map(async (mesh) => {
-			if ((!isMesh(mesh) && !isCollisionMesh(mesh)) || mesh._masterMesh || isFromSceneLink(mesh) || !isNodeVisibleInGraph(mesh)) {
+			if ((!isMesh(mesh) && !isCollisionMesh(mesh) && !isGaussianSplattingMesh(mesh)) || mesh._masterMesh || isFromSceneLink(mesh) || !isNodeVisibleInGraph(mesh)) {
 				return;
 			}
 
@@ -131,6 +131,21 @@ export async function saveScene(editor: Editor, projectPath: string, scenePath:
 					data.metadata = meshToSerialize.metadata;
 					data.basePoseMatrix = meshToSerialize.getPoseMatrix().asArray();
 
+					// Gaussian splatting meshes embed their splat data inline in the serialized JSON and recreate
+					// their own material and quad geometry when parsed (the loader skips importing geometry for
+					// them). So the serialized material and geometry references are unused: dropping them avoids
+					// writing/delay-loading a useless `.babylonbinarymeshdata` file for the splatting quad.
+					if (isGaussianSplattingMesh(meshToSerialize)) {
+						delete data.materials;
+						delete data.geometries;
+
+						data.meshes?.forEach((m) => {
+							delete m.geometryId;
+							delete m.geometryUniqueId;
+							delete m.delayLoadingFile;
+						});
+					}
+
 					// Handle case where the mesh is a collision mesh
 					if (isCollisionMesh(meshToSerialize)) {
 						data.isCollisionMesh = true;

editor/src/tools/assets/extensions.ts

@@ -1,6 +1,7 @@
 export const assetsImageExtensions = [".png", ".webp", ".jpg", ".bmp", ".jpeg"];
 export const assetsAudioExtensions = [".mp3", ".wav", ".wave", ".ogg"];
 export const assetsVideoExtensions = [".mp4", ".webm", ".ogg"];
-export const assetsModelExtensions = [".gltf", ".glb", ".obj", ".babylon", ".stl", ".3ds", ".fbx"];
+export const assetsGaussianSplattingExtensions = [".ply", ".splat", ".spz"];
+export const assetsModelExtensions = [".gltf", ".glb", ".obj", ".babylon", ".stl", ".3ds", ".fbx", ...assetsGaussianSplattingExtensions];
 
 export const assetsAllSupportedExtensions = [...assetsImageExtensions, ...assetsAudioExtensions, ...assetsVideoExtensions, ...assetsModelExtensions];

editor/src/tools/guards/nodes.ts

@@ -15,6 +15,7 @@ import {
 	HemisphericLight,
 	Skeleton,
 	ClusteredLightContainer,
+	GaussianSplattingMesh,
 } from "babylonjs";
 
 import { EditorCamera } from "../../editor/nodes/camera";
@@ -35,12 +36,21 @@ export function isAbstractMesh(object: any): object is Mesh {
 		case "GroundMesh":
 		case "InstancedMesh":
 		case "NodeParticleSystemSetMesh":
+		case "GaussianSplattingMesh":
 			return true;
 	}
 
 	return false;
 }
 
+/**
+ * Returns wether or not the given object is a GaussianSplattingMesh.
+ * @param object defines the reference to the object to test its class name.
+ */
+export function isGaussianSplattingMesh(object: any): object is GaussianSplattingMesh {
+	return object.getClassName?.() === "GaussianSplattingMesh";
+}
+
 /**
  * Returns wether or not the given object is a Mesh.
  * @param object defines the reference to the object to test its class name.

editor/src/tools/node/metadata.ts

@@ -66,6 +66,13 @@ export function setNodeSerializable(node: Node, value: boolean): void {
  * @param node defines the reference to the node to check.
  */
 export function isNodeVisibleInGraph(node: Node): boolean {
+	// Internal nodes flagged hidden through Babylon's reservedDataStore convention (e.g. the per-camera
+	// proxy meshes a GaussianSplattingMesh creates at render time) are implementation details and must
+	// never show in the graph nor be saved.
+	if (node.reservedDataStore?.hidden) {
+		return false;
+	}
+
 	const value = ensureNodeMetadata(node).notVisibleInGraph;
 	return value === undefined ? true : !value;
 }

tools/src/loading/loader.ts

@@ -1,3 +1,7 @@
+// Registers the GaussianSplattingMesh parser (Mesh._GaussianSplattingMeshParser) so the .babylon scene
+// loader can reconstruct Gaussian splatting meshes serialized inline by the editor.
+import "@babylonjs/core/Meshes/GaussianSplatting/gaussianSplattingMesh";
+
 import { Scene } from "@babylonjs/core/scene";
 import { Vector3 } from "@babylonjs/core/Maths/math.vector";
 import { Constants } from "@babylonjs/core/Engines/constants";