diff --git a/.gitignore b/.gitignore index 96474c02..1520f8ba 100755 --- a/.gitignore +++ b/.gitignore @@ -94,6 +94,7 @@ docs/* !docs/AUTO_UPDATER_DESIGN.md !docs/IMAGE_TO_3D_SPIKE_764.md !docs/IMAGE_TO_3D_QUALITY.md +!docs/TRIPOSG_EXPORT_NOTES.md !docs/MESH_SEGMENTATION_STRATEGY.md # minisign — never commit secret keys diff --git a/CLAUDE.md b/CLAUDE.md index bd78621f..365cf666 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -132,6 +132,7 @@ qtmesh generate3d image.png --texture-size 2048 -o out.glb # quality pass (ON b qtmesh generate3d image.png --no-smooth --no-refine --no-bake-texture -o out.glb # raw marching-cubes output with per-vertex color (pre-quality-pass behavior) qtmesh generate3d image.png --upscale-texture -o out.glb # + Real-ESRGAN 2x on the baked diffuse (sharper color; upscale model downloads on demand) qtmesh generate3d image.png --no-pbr -o out.glb # skip the PBR stage (#404 normal+roughness synthesized from the baked diffuse and bound into the material — ON by default; the polished-surface look; writes *_normal/_roughness.png sidecars) +qtmesh generate3d image.png --backend triposg --flow-steps 25 --guidance 7 -o out.glb # TripoSG backend (1.5B rectified-flow DiT, MIT): higher-fidelity GEOMETRY, slower; models download on first use; --guidance 0 disables CFG. TripoSG is GEOMETRY-ONLY (no colour decoder) — a colour bake queries TripoSR's image-conditioned colour field on the same image + projects the input photo onto the visible front (the back is inferred, so it's approximate; a "Generate texture (AI)" option in the GUI does a front-photo + SD-generated-back multi-view bake for a better back). int8 tier is DROPPED for TripoSG (fp32 only — quantized geometry degrades to blobs, no ARM speed win) qtmesh segment model.fbx # AI part segmentation: per-part vertex/face counts (head/torso/arm/leg) (#410) qtmesh segment model.fbx --json # full vertex/face → label arrays + per-part summary (stable schema) qtmesh segment model.fbx --no-model --up-axis y # force the deterministic geometric fallback (skip the ONNX model) @@ -326,7 +327,7 @@ Three singletons manage core state. All run on the main thread. Access via `Clas - **AutoRig** (`src/AutoRig.h/cpp`, issue #407): native automatic rigging — predicts a skeleton for an unrigged mesh. The issue proposed wrapping **Pinocchio** (Baran & Popović, SIGGRAPH 2007), but Pinocchio's **core library is LGPL-2.1-or-later** (only its demo CLI is MIT). Statically vendoring LGPL imposes relink / object-file obligations that conflict with this project's statically-linked, permissively-redistributed binaries (Homebrew / Snap / WinGet / Docker) — the same reason #401 (Instant Meshes / QuadriFlow) and #402 (libigl BBW needs GPL TetGen) shipped native heuristics. Pinocchio's *algorithm* (embed a skeleton template into the mesh interior) is published and unencumbered; only its code is LGPL, so this is a from-scratch native implementation with **zero new dependencies**. Pipeline: (1) read mesh vertices → AABB; (2) each built-in template (humanoid 19-bone / biped / quadruped / generic) is a proportional joint graph in a normalised unit box; map every joint into the AABB; (3) recentre flagged joints (spine, limb roots) toward the centroid of the vertices in a thin slab at the joint's up-height — pulls the spine onto the medial line and lands limb roots inside the silhouette. `rigEntity()` builds an `Ogre::Skeleton` (parent-relative bone positions, `setBindingPose`), binds via `mesh->_notifySkeleton` **+ `entity->_initialise(true)`** — the re-initialise is REQUIRED or both exporters (FBXExporter and the Assimp glTF/FBX path gate on `entity->hasSkeleton()`) silently drop the new rig. Pure-data core (`templateJoints` / `fitTemplate`) is unit-tested without GL. Surfaced via `qtmesh rig [--skeleton T] [--skin] [--up-axis x|y|z] -o out` (`CLIPipeline::cmdRig`, optionally chains `SkinWeights::computeAndApply` for one-click rig+skin), MCP `auto_rig` `{template, skin?, up_axis?, output_path?}` (`MCPServer::toolAutoRig`), and the **Animation Mode → Mode Tools → "Rigging" section → "Auto-Rig…" button** (`qml/AutoRigDialog.qml`, driven by `AutoRigController` singleton, gated on `hasRiggableSelection` — a static/skeleton-less mesh; already-rigged meshes show the "Skinning" section instead). Sentry breadcrumb category `ai.assist.auto_rig`. **Quality limits** (documented per the issue, like Pinocchio): heuristic embedding — works best on roughly upright, single-component, manifold, T/A-pose meshes with +Y up; it does not detect limbs from topology, so exotic proportions or non-upright poses can misplace joints. Verified end-to-end: a static OBJ → 19-bone humanoid + skin → glTF export with 1 skin / 17 joints. **Mixamo-style marker placement** (refinement over the proportional fit): the user clicks the 10 humanoid markers on the mesh surface in the viewport (chin, L/R shoulder, L/R wrist, L/R hip, L/R knee, hips/pelvis — `AutoRig::humanoidMarkerOrder()`), and each placed marker anchors its joint while the limb/spine chains interpolate between the anchors so the rig follows actual body proportions instead of the fixed template. The pure-data core is `AutoRig::fitTemplateWithMarkers` and it does **coherent inference**, not per-marker patching: it runs `fitTemplate` for a proportional baseline (and to read the template's segment vectors / lateral offsets), then **resolves an anchor for every key joint** (Head, L/R Shoulder, L/R Hand, Hips, L/R UpLeg, L/R Knee) as *marked → inferred-from-marked-neighbours → template* and lays the dependent chains (spine, arms, legs) from those anchors — so a partial marker set yields an anatomically-sane skeleton instead of mixing marked anchors with stranded template joints (no shoulder-above-head). Inference: Hips ← midpoint of marked up-legs + template socket→pelvis rise; Head ← template offset above resolved Hips; UpLeg ← mirror the other up-leg across the pelvis, else pelvis + template socket offset; Shoulder ← along the live Hips→Head line at the template shoulder-height fraction + template lateral offset, else mirror the other; Hand ← shoulder + template arm vector (marked shoulder + skipped wrist still lays a full arm); Knee/foot ← clamped to the mesh AABB so an inferred leg never punches through the model: when the knee is skipped, the foot is dropped straight to the mesh FLOOR (`mn[up]`) below the up-leg and the knee placed halfway between (template thigh-vector extrapolation, which used to shoot feet past the lower limit, is only used for the small forward knee nudge); a marked knee keeps its position with the foot extrapolated below but still floor-clamped. Mirroring reflects across the sagittal plane (side axis auto-detected). An empty marker set early-returns `fitTemplate` unchanged; `report.markersApplied` counts only user-placed markers. (Legacy per-marker description retained below for the chain mechanics.) The old behaviour was: Hips→anchor pelvis AND carry the thigh roots (LeftUpLeg/RightUpLeg, children of Hips) by the same delta so the whole pelvis+thigh cluster moves as a unit — unless an explicit hip marker overrides; Chin→anchor Head AND lay the spine straight up from the pelvis — Spine/Chest/Neck distributed evenly between Hips and Head by index (cartoon torso lengths vary too much for a proportional guess); L/R shoulder→anchor the arm-chain attach point (applied before the wrist so the chain lays from the marked shoulder); L/R wrist→`layChain` lays the WHOLE arm straight from the shoulder anchor — Shoulder[anchor]→Arm[⅓]→ForeArm[⅔]→Hand[marker] — distributing every intermediate joint so the entire arm reaches the wrist, not just the hand; L/R hip→anchor the thigh root/hip socket (applied before the knee, overrides the hips-carry — needed for cartoon legs that splay at odd angles); L/R knee→`layLeg` anchors the knee at the marker and continues the foot below it along the thigh→knee direction (so the whole leg — hip socket → knee → foot — follows the marked hip + knee)). `layChain` is generic (anchor-first, marker-last, evens the middle by index) so adding more chain joints is a one-line change. Every marker is OPTIONAL — unset markers keep the template fit (`report.markersApplied` counts the placed ones; an empty marker set is bit-identical to `fitTemplate`). The viewport flow lives in `AutoRigController` (marker-session state machine: `beginMarkerPlacement`/`skipCurrentMarker`/`undoLastMarker`/`cancelMarkerPlacement`/`commitMarkerRig`); clicks are routed in by `TransformOperator::mousePressEvent` (checked **before** the knife/select paths when `markerMode()` is true), ray-cast to the mesh surface (`getCameraToViewportRay` → Möller-Trumbore against world-space triangles), stored in mesh-local space, and shown as unlit-yellow `PT_SPHERE` overlays. **The whole UI is inline in the Inspector's "Rigging" section** (`riggingToolsComponent` in `qml/PropertiesPanel.qml`) — there is no separate dialog (the old `AutoRigDialog.qml` was removed). It show/hides smartly: idle shows the two entry points ("Place markers…" / "Auto-Rig (template)"), a skin checkbox, and an "Advanced options" checkbox that reveals the template + up-axis pickers; while `markerMode` is active it swaps to the per-marker guidance label + Skip/Undo/Cancel/"Rig from markers" controls. Rig state + the `runAutoRig`/`runMarkerRig` helpers live on the `PropertiesPanel` root; the section's `onSectionVisibleChanged` cancels any active marker session if the section disappears (mode change / deselect / re-rig), replacing the dialog's old `onClosing` cancel. No CLI/MCP marker surface — guided placement is inherently interactive. **UniRig ML backend** (issue #408): `AutoRig::Algorithm {Pinocchio, UniRig}` selects the skeleton-prediction backend (default Pinocchio — offline, deterministic). **UniRig** (Zhang et al., *"One Model to Rig Them All"*, SIGGRAPH 2025, VAST-AI-Research/UniRig — **MIT code + MIT weights**, trained on Articulation-XL2.0 **CC-BY-4.0**) is an autoregressive transformer that predicts a skeleton from the mesh geometry, handling arbitrary/non-humanoid topology better than the fixed template. It's the **second ONNX consumer** after #404 PbrMapSynth. **RigNet was rejected** for #408 (GPL code + unlicensed weights + non-public ModelsResource dataset — fails the project's permissive-redistribution bar); UniRig is the clean permissively-licensed alternative (see `THIRD_PARTY_AI_MODELS.md`). `UniRigPredictor` (`src/UniRigPredictor.h/cpp`, Ogre-free + unit-tested) is the C++/ONNX runtime that ports UniRig's skeleton stage: (1) surface-sample up to 65536 points + normals, normalise into a centred unit box (+Y up); (2) run the **Michelangelo encoder** (`encoder.onnx`, pc[1,N,3]+feats[1,N,3] → latent prefix); (3) **greedy/constrained autoregressive decode** over the ~350M causal-LM (`decoder.onnx`) with a manual KV-cache + the tokenizer's next-possible-token validity mask (a documented simplification of UniRig's beam+sampling — deterministic + exportable, still yields a valid tree); (4) the **exact tokenizer FSM** from `src/tokenizer/tokenizer_part.py` (256 coord bins, `continuous_range [-1,1]`, `undiscretize(t)=(t+0.5)/256*2-1`, branch/parent rules, vocab 267) → joints (de-normalised) + parent indices, parent-before-child ordered for Ogre. The detokenizer + `undiscretize` are public statics (`UniRigPredictor::detokenize`/`undiscretize`) so they're unit-tested without ONNX. Everything is `ENABLE_ONNX`-guarded; **two** model files (`AppData/ai_models/unirig/{encoder,decoder}.onnx`) download on first use via `ModelDownloader` (`ensureModelBlocking`, 180s timeout, returns the encoder path only when BOTH exist; base URL override `QTMESH_UNIRIG_MODEL_BASE_URL` / `QSettings ai/unirigModelBaseUrl`, offline guard `QTMESH_UNIRIG_NO_DOWNLOAD`). **UniRig falls back to Pinocchio** (logged in `report.fallbackReason`) when ONNX is off / the models are missing/offline/not-yet-hosted / prediction is unusable — reliable offline. **Design contract / hosting status:** UniRig is an autoregressive HF `AutoModelForCausalLM` + a Michelangelo perceiver — no single-graph ONNX export exists upstream; `scripts/export-unirig-onnx.py` (one-time, offline, NOT shipped) exports the encoder + a KV-cache decoder to the I/O `UniRigPredictor` targets (via `optimum`, with a hand-rolled fallback). Until the exported `.onnx` files are hosted on the HF models repo, the download 404s and the Pinocchio fallback runs — the plumbing + runtime are complete and ship today; hosting the export lights up the ML path with no code change. UniRig is marker-incompatible (markers are a template concept), so a marker-driven call always uses the template. Surfaced via CLI `qtmesh rig --algo pinocchio|unirig` (`CLIPipeline::cmdRig`; `rignet` accepted as a deprecated alias), MCP `auto_rig` `algo` param (`MCPServer::toolAutoRig`), and the Inspector Rigging-section **Algorithm** segmented picker; the report carries `algorithmUsed` + `fallbackReason`, and the Sentry `ai.assist.auto_rig` breadcrumb records the `algo`. (The early `qml/AutoRigDialog.qml` reference above is stale — the UI is inline in `qml/PropertiesPanel.qml`'s Rigging section.) - **QuadRetopo** (`src/QuadRetopo.h/cpp`, issue #401): triangle-pairing quad-dominant retopology. The issue proposed wrapping Instant Meshes (Wenzel Jakob), but Instant Meshes ships as a research GUI app with no clean C++ library API and has been dormant since 2016. QuadriFlow (the production-grade alternative used by Blender 3.0+) requires Boost + Eigen + LEMON — heavy deps the project doesn't currently use. This first slice ships a native triangle-pairing backend with **zero new dependencies**: walks every interior edge whose two adjacent faces are triangles and scores the merge by (1) coplanarity (dot product of triangle normals; default `maxAngleDeg=25°`), (2) quad shape (deviation of interior angles from 90°; default `shapeToleranceDeg=65°`), (3) aspect ratio (longest/shortest edge; default `maxAspectRatio=6.0`). Pairs are taken greedily best-first; each triangle claimed at most once. Quads are emitted with opposing-corner winding `(opposing0, sharedA, opposing1, sharedB)`. Output goes through `EditableSubMesh::faces` → `triangulateFaces` (fan retri for GPU) → `writeNgonFacesToMesh` (n-gon binding for exporters / Edit Mode). **No new vertices** are introduced, so UVs and skin weights survive unchanged. Backends are pluggable via the `Algorithm` enum (only `TrianglePair` implemented; future `QuadriFlow` / `InstantMeshes` slot in here). Surfaced via `qtmesh retopo --target-faces N --max-angle DEG -o out`, MCP `retopologize`, and the **Material Mode → Mode Tools → "Quad Retopology…" button** (`qml/QuadRetopoDialog.qml`, driven by `QuadRetopoController` singleton). Sentry breadcrumb category `ai.assist.retopo`. Verified on Rumba Dancing.fbx: 10,220 tris → 6,032 faces (4188 quads + 1844 tris), 82% quad dominance. Hard lower bound on face count is ~50% of input (every triangle paired); strict gates typically land 60-70%. - **MeshSegmenter** (`src/MeshSegmenter.h/cpp`, issue #410): AI mesh part segmentation — predicts a semantic part label (head/torso/left+right arm/left+right leg) per vertex + per face. The **fourth ONNX consumer**; powers Edit-Mode "Select by part", per-part material assignment, and auto-rig priors. **Geometric fallback is first-class** (always compiled, Ogre-free): `segmentGeometric` does connected-component islands (`connectedComponents`, union-find) + an up-axis/lateral spatial heuristic (top→head, lower→legs, mid-sides→arms, centre→torso), overridable per-vertex by rig bone-proximity hints — used automatically when the build lacks ONNX, the model is missing/un-downloadable, or inference fails (`Result::usedModel`/`fallbackReason` report which ran). The **ONNX path** (`#ifdef ENABLE_ONNX`, PointNet++-style) normalises → deterministic point sample → `[1,N,3]` → per-point argmax over the part channels → scatters labels back to all vertices by nearest sampled point (runtime I/O-name discovery, channels-first/last handling, CoreML EP). `ensureModelBlocking()` downloads `meshseg.onnx` to `AppData/ai_models/segment/` (override `QTMESH_SEGMENT_MODEL_BASE_URL` / `QSettings ai/segmentModelBaseUrl`; offline guard `QTMESH_SEGMENT_NO_DOWNLOAD`; non-ONNX `#ifndef` guard) — the #408/#409 pattern. Pure-data helpers (`connectedComponents`, `facesFromVertexLabels`) are unit-tested without Ogre/GL. Surfaced via **CLI `qtmesh segment [--json] [--no-model] [--up-axis x|y|z]`** (`CLIPipeline::cmdSegment` — text per-part counts or full label arrays), the **MCP `segment_mesh` tool** (`MCPServer::toolSegmentMesh`, args `{entity_name?, no_model?}`, heavy), and the **Edit Mode → "Select by Part (AI)" button** (`EditModeController::selectByPart()` → selects all faces matching the selected face's part, or the largest part if none selected; pushes via `selectFace` so the existing highlight refreshes). Sentry breadcrumb `ai.assist.segment`. **Model: ours (v2), trained on surface-sampled synthetic bodies (humanoid/chibi/quadruped/biped-tail plans) + mined CC0 Quaternius rigs** (rig bone-weight → part; ShapeNet-Part/PartNet are non-commercial and rejected) via `scripts/export-meshseg-onnx.py` (offline, not shipped) — the v2 loader canonicalises arbitrarily-oriented mined clouds from their own labels and geometrically fixes miner side errors; hosted on the HF models repo under `segment/` (see `THIRD_PARTY_AI_MODELS.md` + `docs/MESH_SEGMENTATION_STRATEGY.md` for the v1 failure analysis, accuracy numbers, and the multi-category roadmap). **Three-tier dispatch in `selectByPart`**: (1) **rig-prior** — if the mesh is SKINNED, label each vertex by the part of the bone it's most-weighted to (`AutoRig::rigPriorPartLabels` → `MeshSegmenter::partForBoneName`); EXACT and handles non-human anatomy (ears/snout→head, tail→torso, paws→leg) the coordinate model can't. Used when it resolves ≥70% of vertices. (2) **ONNX model** (UNrigged meshes). (3) **geometric fallback**. The ONNX path also applies `Options::upAxis` by remapping the sampled point cloud to the model's +Y-up training frame before inference (and in the nearest-point scatter), so X/Z-up meshes aren't mislabelled. **Continual-training miner** (the "train further as we gather data" loop): `qtmesh segment --dump-training-data out.json` runs the rig-prior path and writes the normalised point cloud + EXACT per-vertex labels (schema `qtmesh-meshseg-training-v1`) — every rigged asset becomes one free, exactly-labelled sample. `scripts/export-meshseg-onnx.py --real-data ` MIXES those mined JSONs (with yaw/tilt/jitter aug) into the synthetic set and retrains; gains land on the MODEL path used for unrigged meshes (rigged meshes already use the exact rig-prior path in-app). `AutoRig::rigPriorPartLabels` is the shared extractor for the GUI fast-path and the miner, so the in-app selection and mined ground truth are bit-identical. -- **Image-to-3D (TripoSR)** (`src/ImageTo3D/`, epic #764): single-image → 3D mesh generation via **TripoSR** (Tripo AI + Stability AI, **MIT code AND MIT weights**, HF `stabilityai/TripoSR`). The **fifth ONNX consumer** (after #404/#408/#409/#410); all files live in the `src/ImageTo3D/` feature folder. MIT code+weights is the deciding factor for redistribution (Homebrew/Snap/WinGet/Docker) — the bar UniRig #408 cleared and non-commercial SF3D failed. **`MeshGenPredictor`** (Ogre-free + unit-tested) runs two exported ONNX graphs — encoder `image[1,3,512,512]→scene_codes[1,3,40,64,64]` (triplane) and per-point decoder `scene_codes+points[1,P,3]→density[1,P,1],color[1,P,3]` — GENERATING query points per chunk (not the whole `res³` grid up front — that would OOM at 512) and extracting the surface with **`MarchingCubes`** (native Lorensen impl, public-domain tables, zero deps; TripoSR's `torchmcubes` is torch/GPU-only). Surface = MC on `density − threshold` at iso 0 (threshold 25.0, radius 0.87); our MC is inside-positive so `extract()` emits `v0,v2,v1` (flipped winding) to keep faces OUTWARD (else the mesh renders inside-out). **Model size tiers** (`MeshGenPredictor::Quality {Fp32,Int8}` → `triposr_encoder{,_int8}.onnx`): fp32 ~1.68 GB (best), int8 ~430 MB (slight quality loss); user-selectable, downloads on demand. (fp16 was dropped — TripoSR's attention has a hardcoded Cast-to-float32 the ONNX fp16 converters can't rewrite; int8 is smaller anyway.) **`MeshGenBuilder`** (the ONLY Ogre-touching piece) turns the arrays into an `Ogre::Mesh` (POSITION + accumulated per-vertex NORMAL + optional DIFFUSE `VET_COLOUR` with a lit vertex-color material; 16-/32-bit index by vertex count; validates index data first), **bakes -90°X + +90°Y** into positions+normals so the model stands upright and faces forward, uses a UNIQUE per-call node/mesh name, and returns the SceneNode for export. **Background removal:** `BackgroundRemover` (6th ONNX consumer) runs **U²-Net** (Apache-2.0, rembg's model) to isolate the subject: `[1,3,320,320]`→`[1,1,320,320]` saliency, then composites over **gray 128** (not white — white → a reconstructed wall) and crops/re-pads to the subject at 0.85 foreground ratio (TripoSR's `resize_foreground`). Model `ai_models/rembg/u2net.onnx` (`QTMESH_REMBG_MODEL_BASE_URL`/`ai/rembgModelBaseUrl`; guard `QTMESH_REMBG_NO_DOWNLOAD`); falls back to the raw image if unavailable. Everything `ENABLE_ONNX`-guarded; **no fallback** (generative), so a non-ONNX build / missing model returns a clear error (never crashes). Models under `ai_models/triposr/` download on first use (`ensureModelBlocking(q)`; `QTMESH_TRIPOSR_MODEL_BASE_URL`/`ai/triposrModelBaseUrl`; guard `QTMESH_TRIPOSR_NO_DOWNLOAD`), OR can be **pre-downloaded from the AI Settings modal's Download tab** (tier picker + progress bar). **Export is `scripts/export-triposr-onnx.py`** (offline, not shipped; `transformers==4.35.0`, `torchmcubes` stub, frozen ViT pos-encoding; emits the int8 variant unless `--no-quant` — see `docs/IMAGE_TO_3D_SPIKE_764.md`). Surfaced via **CLI `qtmesh generate3d [-o out.glb] [--resolution 16..1024] [--no-color] [--remove-bg] [--quality fp32|int8]`** (`CLIPipeline::cmdGenerate3d`), **MCP `generate_mesh_from_image`** (`MCPServer::toolGenerateMeshFromImage`, args `{image_path, output?, resolution?, vertex_color?, remove_bg?, quality?}`, heavy, ONNX-guarded schema), and the **Object Mode Tools → "AI: Image → 3D" inspector section** (`qml/PropertiesPanel.qml` → **`MeshGenController`**, a QML_SINGLETON that runs the whole pipeline on a WORKER THREAD — UI stays responsive — with a select-image→preview→generate flow, resolution + model-tier dropdowns, progress bar, and cancel; mesh construction is marshalled back to the main thread). Sentry breadcrumb `ai.assist.image_to_3d`. Verified end-to-end on macOS. **Models are HOSTED** on the `fernandotonon/QtMeshEditor-models` HF repo (`triposr/triposr_encoder.onnx` + `triposr_encoder_int8.onnx` + `triposr_decoder.onnx`, `rembg/u2net.onnx`) via `scripts/upload-triposr-models.sh` — first use downloads them; if ever absent, every surface reports a clean "not yet hosted" message (no crash). Design/spike note: `docs/IMAGE_TO_3D_SPIKE_764.md`; slices A #765 (spike) → B #766 predictor → C #767 mesh build → D #768 surfaces → E #769 tiers/pre-download/hosting/docs (all in PR #785). **Quality pass (post-#785, ON by default)**: after marching cubes the predictor runs (a) **`MeshRefine::taubinSmooth`** — Taubin λ|μ smoothing (volume-preserving, kills the res³-grid stair-stepping), (b) **`MeshRefine::isoProjectStep`** — one Newton step per vertex back onto the decoder's true iso-surface using forward-difference gradients from 4 extra decoder probes/vertex (recovers grid-quantized detail; both pure-data + unit-tested in `MeshRefine_test.cpp`), and (c) **`MeshGenBaker`** — xatlas auto-unwrap + UV-space triangle rasterization + per-texel decoder colour queries + chart-border dilation, producing UV0 + a real diffuse TEXTURE (default 1024²) instead of per-vertex colour — colour sharpness then scales with texture size, not vertex density (pure-data behind a `ColorSampler` callback; `MeshGenBaker_test.cpp`). `MeshGenBuilder` gained the textured path: saves the baked PNG (AppData/generated_textures/ or the export dir when given), registers the dir as a resource location, and binds a lit material with a named `diffuse_map` TUS. Bake failure falls back to vertex colours with `Result::warning` set (never fails the generation). **PBR stage (d, ON by default)**: `MeshGenBuilder::BuildOptions::generatePbrMaps` chains **#404 PBR map synthesis** onto the baked diffuse — normal + roughness PNGs written next to it (height skipped, no consumer) and bound into the material via the same recipe as the Material Editor's "Generate PBR maps from diffuse" button (canonical `normal_map`/`roughness` TUS + `wirePbrSlotsForFFP` + `RTShaderHelper::applyNormalMap` — without applyNormalMap the bind is invisible in the viewport — + recompile). This is what turns the flat diffuse-only result into a polished, surface-detailed one; fails soft to diffuse-only when the PBRify models are unavailable. The exported material references all three maps (FBX embeds them; the PNGs land next to the export). **Every stage is user-selectable**: GUI checkboxes in the AI section (Remove background / Smooth / Refine / Bake texture / PBR maps / Upscale 2×) feed an options QVariantMap into `MeshGenController::generateSelected`; CLI `--no-smooth --no-refine --no-bake-texture --no-pbr --texture-size N --upscale-texture`; MCP `smooth/refine/bake_texture/generate_pbr/texture_size/upscale_texture`. The GUI runs the upscale on the WORKER thread (model pre-ensured on the main thread) and the PBR synthesis on the main thread inside buildSceneNode (small models, Material-Editor precedent). Roadmap for bigger jumps (TripoSG MIT backend, input-image projection): `docs/IMAGE_TO_3D_QUALITY.md`. +- **Image-to-3D (TripoSR)** (`src/ImageTo3D/`, epic #764): single-image → 3D mesh generation via **TripoSR** (Tripo AI + Stability AI, **MIT code AND MIT weights**, HF `stabilityai/TripoSR`). The **fifth ONNX consumer** (after #404/#408/#409/#410); all files live in the `src/ImageTo3D/` feature folder. MIT code+weights is the deciding factor for redistribution (Homebrew/Snap/WinGet/Docker) — the bar UniRig #408 cleared and non-commercial SF3D failed. **`MeshGenPredictor`** (Ogre-free + unit-tested) runs two exported ONNX graphs — encoder `image[1,3,512,512]→scene_codes[1,3,40,64,64]` (triplane) and per-point decoder `scene_codes+points[1,P,3]→density[1,P,1],color[1,P,3]` — GENERATING query points per chunk (not the whole `res³` grid up front — that would OOM at 512) and extracting the surface with **`MarchingCubes`** (native Lorensen impl, public-domain tables, zero deps; TripoSR's `torchmcubes` is torch/GPU-only). Surface = MC on `density − threshold` at iso 0 (threshold 25.0, radius 0.87); our MC is inside-positive so `extract()` emits `v0,v2,v1` (flipped winding) to keep faces OUTWARD (else the mesh renders inside-out). **Model size tiers** (`MeshGenPredictor::Quality {Fp32,Int8}` → `triposr_encoder{,_int8}.onnx`): fp32 ~1.68 GB (best), int8 ~430 MB (slight quality loss); user-selectable, downloads on demand. (fp16 was dropped — TripoSR's attention has a hardcoded Cast-to-float32 the ONNX fp16 converters can't rewrite; int8 is smaller anyway.) **`MeshGenBuilder`** (the ONLY Ogre-touching piece) turns the arrays into an `Ogre::Mesh` (POSITION + accumulated per-vertex NORMAL + optional DIFFUSE `VET_COLOUR` with a lit vertex-color material; 16-/32-bit index by vertex count; validates index data first), **bakes -90°X + +90°Y** into positions+normals so the model stands upright and faces forward, uses a UNIQUE per-call node/mesh name, and returns the SceneNode for export. **Background removal:** `BackgroundRemover` (6th ONNX consumer) runs **U²-Net** (Apache-2.0, rembg's model) to isolate the subject: `[1,3,320,320]`→`[1,1,320,320]` saliency, then composites over **gray 128** (not white — white → a reconstructed wall) and crops/re-pads to the subject at 0.85 foreground ratio (TripoSR's `resize_foreground`). Model `ai_models/rembg/u2net.onnx` (`QTMESH_REMBG_MODEL_BASE_URL`/`ai/rembgModelBaseUrl`; guard `QTMESH_REMBG_NO_DOWNLOAD`); falls back to the raw image if unavailable. Everything `ENABLE_ONNX`-guarded; **no fallback** (generative), so a non-ONNX build / missing model returns a clear error (never crashes). Models under `ai_models/triposr/` download on first use (`ensureModelBlocking(q)`; `QTMESH_TRIPOSR_MODEL_BASE_URL`/`ai/triposrModelBaseUrl`; guard `QTMESH_TRIPOSR_NO_DOWNLOAD`), OR can be **pre-downloaded from the AI Settings modal's Download tab** (tier picker + progress bar). **Export is `scripts/export-triposr-onnx.py`** (offline, not shipped; `transformers==4.35.0`, `torchmcubes` stub, frozen ViT pos-encoding; emits the int8 variant unless `--no-quant` — see `docs/IMAGE_TO_3D_SPIKE_764.md`). Surfaced via **CLI `qtmesh generate3d [-o out.glb] [--resolution 16..1024] [--no-color] [--remove-bg] [--quality fp32|int8]`** (`CLIPipeline::cmdGenerate3d`), **MCP `generate_mesh_from_image`** (`MCPServer::toolGenerateMeshFromImage`, args `{image_path, output?, resolution?, vertex_color?, remove_bg?, quality?}`, heavy, ONNX-guarded schema), and the **Object Mode Tools → "AI: Image → 3D" inspector section** (`qml/PropertiesPanel.qml` → **`MeshGenController`**, a QML_SINGLETON that runs the whole pipeline on a WORKER THREAD — UI stays responsive — with a select-image→preview→generate flow, resolution + model-tier dropdowns, progress bar, and cancel; mesh construction is marshalled back to the main thread). Sentry breadcrumb `ai.assist.image_to_3d`. Verified end-to-end on macOS. **Models are HOSTED** on the `fernandotonon/QtMeshEditor-models` HF repo (`triposr/triposr_encoder.onnx` + `triposr_encoder_int8.onnx` + `triposr_decoder.onnx`, `rembg/u2net.onnx`) via `scripts/upload-triposr-models.sh` — first use downloads them; if ever absent, every surface reports a clean "not yet hosted" message (no crash). Design/spike note: `docs/IMAGE_TO_3D_SPIKE_764.md`; slices A #765 (spike) → B #766 predictor → C #767 mesh build → D #768 surfaces → E #769 tiers/pre-download/hosting/docs (all in PR #785). **Quality pass (post-#785, ON by default)**: after marching cubes the predictor runs (a) **`MeshRefine::taubinSmooth`** — Taubin λ|μ smoothing (volume-preserving, kills the res³-grid stair-stepping), (b) **`MeshRefine::isoProjectStep`** — one Newton step per vertex back onto the decoder's true iso-surface using forward-difference gradients from 4 extra decoder probes/vertex (recovers grid-quantized detail; both pure-data + unit-tested in `MeshRefine_test.cpp`), and (c) **`MeshGenBaker`** — xatlas auto-unwrap + UV-space triangle rasterization + per-texel decoder colour queries + chart-border dilation, producing UV0 + a real diffuse TEXTURE (default 1024²) instead of per-vertex colour — colour sharpness then scales with texture size, not vertex density (pure-data behind a `ColorSampler` callback; `MeshGenBaker_test.cpp`). `MeshGenBuilder` gained the textured path: saves the baked PNG (AppData/generated_textures/ or the export dir when given), registers the dir as a resource location, and binds a lit material with a named `diffuse_map` TUS. Bake failure falls back to vertex colours with `Result::warning` set (never fails the generation). **PBR stage (d, ON by default)**: `MeshGenBuilder::BuildOptions::generatePbrMaps` chains **#404 PBR map synthesis** onto the baked diffuse — normal + roughness PNGs written next to it (height skipped, no consumer) and bound into the material via the same recipe as the Material Editor's "Generate PBR maps from diffuse" button (canonical `normal_map`/`roughness` TUS + `wirePbrSlotsForFFP` + `RTShaderHelper::applyNormalMap` — without applyNormalMap the bind is invisible in the viewport — + recompile). This is what turns the flat diffuse-only result into a polished, surface-detailed one; fails soft to diffuse-only when the PBRify models are unavailable. The exported material references all three maps (FBX embeds them; the PNGs land next to the export). **Every stage is user-selectable**: GUI checkboxes in the AI section (Remove background / Smooth / Refine / Bake texture / PBR maps / Upscale 2×) feed an options QVariantMap into `MeshGenController::generateSelected`; CLI `--no-smooth --no-refine --no-bake-texture --no-pbr --texture-size N --upscale-texture`; MCP `smooth/refine/bake_texture/generate_pbr/texture_size/upscale_texture`. The GUI runs the upscale on the WORKER thread (model pre-ensured on the main thread) and the PBR synthesis on the main thread inside buildSceneNode (small models, Material-Editor precedent). **TripoSG backend** (`src/ImageTo3D/TripoSGPredictor.{h,cpp}`, the SEVENTH ONNX consumer): `MeshGenPredictor::Options::backend {TripoSR|TripoSG}` dispatches to **TripoSG** (VAST-AI, SIGGRAPH 2025, **MIT code + MIT weights**, geometry ≈ commercial Tripo 2.0) — a 1.5B rectified-flow DiT over an SDF VAE, run as FOUR exported graphs (`scripts/export-triposg-onnx.py`, offline dev tool; measured contract in `docs/TRIPOSG_EXPORT_NOTES.md`): DINOv2-224 image encoder (mean/std baked in; CFG uncond = zeros) → **C++ Euler flow loop** over the DiT step graph (σᵢ = 1−i/N, timestep = 1000·σ, update `x += (σᵢ−σᵢ₊₁)·v` — sign is OPPOSITE of stock diffusers FlowMatchEuler; CFG as two B=1 calls, guidance 7.0, steps knob default 25) → VAE latent kv-cache graph (run ONCE per generation) → per-point field decoder (already inside-positive, iso 0, bounds ±1.005) → the same native MarchingCubes + smooth/reproject polish. Geometry-only (no colour decoder): bake/PBR/upscale stages are TripoSR-only; background removal for TripoSG composites over WHITE (its reference pipeline) vs TripoSR's gray-128. fp32 DiT ships as `.onnx`+`.onnx.data` (>2 GB external weights) with an int8 single-file tier mapped from `Quality::Int8`. Models under `ai_models/triposg/` download on first use (`QTMESH_TRIPOSG_MODEL_BASE_URL`/`ai/triposgModelBaseUrl`; guard `QTMESH_TRIPOSG_NO_DOWNLOAD`); clean "not hosted yet" error until the export is run + hosted. Surfaced via CLI `--backend triposr|triposg --flow-steps N`, MCP `backend`/`flow_steps` args, and the GUI Backend dropdown (the step list gains a "Denoise (flow steps)" row via `Stage::Denoise`). Roadmap/audit: `docs/IMAGE_TO_3D_QUALITY.md`. **TripoSG post-integration updates (supersede the "geometry-only / int8 tier / white-bg / disabled texture checkboxes" claims above):** (1) **int8 tier DROPPED** — even per-channel-quantized, the 1.5B DiT degrades to blobs over the 25-step CFG flow loop (live-verified), and dynamic-int8 MatMuls are no faster than fp32 on ARM; all surfaces force fp32 (CLI prints a note; the GUI Model picker collapses to "fp32 (only option for TripoSG)" and locks; the `quality` param now only selects the TripoSR tier used for the colour bake). (2) **Colour** — TripoSG has no colour decoder, so `MeshGenPredictor::colorizeWithTripoSR` bakes colour by (a) projecting the actual input PHOTO onto the visible front (depth-buffer-gated front-most-surface test; camera looks toward +Z so nearest = max z; soft depth-band crossfade to the field) and (b) filling occluded/back texels from **TripoSR's image-conditioned colour field** (the TripoSG mesh mapped into TripoSR's native frame + per-axis affine-fit onto its occupied bounds). The front is photo-accurate; the back is inferred/approximate. Falls soft to a shared neutral **lit clay material** (`MeshGen/NeutralClay`) on any failure. Texture/PBR/upscale stages + their GUI checkboxes are ENABLED for TripoSG (route through the colour bake). (3) **AI texture (GUI, `ENABLE_STABLE_DIFFUSION`)** — a "Generate texture (AI, front photo + generated back)" checkbox runs the existing **multi-view depth-ControlNet bake** (`MaterialEditorQML::generateMeshTextureMultiView`, `MultiViewTextureBaker`) after the mesh builds, with the input photo PINNED as the front view (img2img is disabled on Metal, so the photo is injected as a filled view rather than an init image) and back/sides SD-generated; needs a loaded SD model. (4) **Orientation** — TripoSG output is already +Y-up (`Result::bakeTripoSROrientation=false` skips the TripoSR -90°X/+90°Y bake); its decoder field is negated at the sample site (exported graph lands OUTSIDE-positive → inverted winding otherwise). (5) **Memory/speed** — decoder chunk hard-capped at 8192 pts (cross-attention to 2048 kv tokens; TripoSR's 262144 chunk OOM-killed at ~90 GB); ONNX sessions staged (opened/released per stage, ~1 GB peak vs the >4 GB sum); the ~48 MB point decoder can run on the CoreML GPU via `QTMESH_TRIPOSG_COREML_DECODER=1` (default CPU — per-call kv re-upload made GPU slower); `--guidance` knob (CLI/MCP). Next speed win: hierarchical extraction (coarse grid → refine near surface). SF3D (non-commercial) and Hunyuan3D (EU-excluded) rejected for the texture upgrade; MV-Adapter (VAST-AI, Apache-2.0) is the tracked multi-view candidate. - **UvUnwrap** (`src/UvUnwrap.h/cpp`, issue #400): xatlas-backed automatic UV unwrap. xatlas is the MIT library Blender and Godot use under the hood — single-translation-unit `xatlas.cpp` vendored via FetchContent and wrapped in an inline `add_library(xatlas STATIC …)` target (no upstream CMake config). Pipeline: extract (positions, indices) per submesh → `xatlas::AddMesh` → `xatlas::Generate` → for each output mesh, rebuild a single-binding VertexData copying every source attribute from `xref` (input vertex id) and overwriting the target UV channel with `xatlas::Vertex::uv / atlas.{width,height}`. Skinned-mesh bone assignments survive the seam splits because we rebuild `SubMesh::BoneAssignmentList` against the new vertex IDs via xref; for shared-vertex meshes the source assignments come from `Mesh::getBoneAssignments()`, not `SubMesh::getBoneAssignments()`. Surfaced via `qtmesh uv --unwrap`/`--info`, MCP `auto_uv_unwrap` / `uv_unwrap_selection`, and the **Material Mode → Mode Tools → "Auto UV Unwrap…" button** (`qml/UvUnwrapDialog.qml`, driven by `UvUnwrapController` singleton). Sentry breadcrumb category `mesh.uv.unwrap`. The unwrap also erases `qtme.faces.` n-gon bindings (they reference source vertex IDs and become stale). **GUI-safe entry point** (`unwrapEntityToFile`): live skinned meshes cannot survive in-place vertex-data mutation because the active `Ogre::SkeletonInstance` caches the hardware blend buffer and picks up stale state on the first frame after the swap. The GUI path snapshots `vertexData` / `indexData` / `mBoneAssignments` / `blendIndexToBoneIndexMap` for every submesh + the mesh's shared maps, calls `unwrapEntityKeepingOriginals` (which deliberately leaks its own allocations rather than freeing the originals), exports the unwrapped result, then restores the snapshot pointer-for-pointer (deleting only the unwrap's leaked allocations) and pastes the index maps back directly — `_compileBoneAssignments` is NOT called on restore because it would re-pack BLEND_INDICES/WEIGHTS bytes against the live buffer and shatter the on-screen mesh. CLI path uses the destructive `unwrapEntity` since the process exits before rendering. - **UV Editor** (`src/UVEditorController.h/cpp`, issues #463–#465): dedicated UV editing mode (Material Mode toolbar → UV Editor). **UVEditorController** (QML_SINGLETON) owns the 2D UV viewport overlay, island selection, transform gizmos (translate/rotate/scale UVs), pin/sew/split, seam marking in Edit Mode, geometric projection (View/Box/Cylinder/Sphere/Reset), and partial xatlas unwrap of selected faces. Core math lives in `UVTransform`, `UvProject`, `UvSeamData`/`UvSeamOps`, and undo via `UVEditCommand` / `UvSeamCommands`. **Headless parity** (#465) is centralized in `UvPipeline` (`src/UvPipeline.h/cpp`): `analyzeEntity` (channel info + island count + AABB overlap upper bound), `projectEntity`, `parseSeamEdgeList`/`setSeamsOnEntity`, `unwrapEntity`, and `unwrapTriangles` (face-mask partial unwrap). CLI: `qtmesh uv --info`, `--project`, `--set-seams`, `--unwrap`. MCP: `uv_info`, `uv_project`, `uv_set_seams`, `uv_unwrap_selection` (+ existing `auto_uv_unwrap`). Sentry categories: `mesh.uv.transform`, `mesh.uv.pin`, `mesh.uv.sew`, `mesh.uv.split`, `mesh.uv.seam`, `mesh.uv.project`, `mesh.uv.unwrap`, `mesh.uv.unwrap_selected`, `mesh.uv.info`. Keyboard shortcuts (UV Editor active): `G` translate, `R` rotate, `S` scale, `P` pin toggle, projection buttons in toolbar; `Tab` exits back to Object mode. - **ExportOptimizer** (`src/ExportOptimizer.h/cpp`, issue #399): Pipeline that runs `meshopt_optimizeVertexCache` → `meshopt_optimizeOverdraw` (threshold 1.05) → `meshopt_optimizeVertexFetchRemap` on every submesh of an entity. Surfaced through the **Inspector validation flow** — the "Optimize Geometry (cache + overdraw + fetch)" button in `PropertiesPanel.qml` runs it via `MeshValidator::optimizeVertexCache`. NOT hooked into `MeshImporterExporter::exporter` by default (an earlier draft did this and crashed on macOS during a normal export — silent buffer mutation during export is dangerous; explicit user invocation via the validation button is safer). Vertex-fetch is skipped when the submesh uses `useSharedVertices` since remapping shared verts would scramble other submeshes' indices. `qtmesh info --json` includes `submeshAcmr[]` per submesh so downstream tooling can decide whether to recommend re-optimization. Sentry breadcrumb category `ai.assist.optimize_export`. diff --git a/CMakeLists.txt b/CMakeLists.txt index d062819d..f781bbaf 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,7 +47,12 @@ set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) # Building directories if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "debug") MESSAGE("DEBUG COMPILATION") - ADD_DEFINITIONS("-DDEBUG") + # NOTE: do NOT use ADD_DEFINITIONS("-DDEBUG") here — it's directory-scoped + # and leaks into the FetchContent'd llama.cpp/mtmd subdir, where + # mtmd-audio.cpp uses DEBUG as an ordinary identifier and fails to compile. + # Apply -DDEBUG to OUR targets only, via this marker + target_compile_ + # definitions on the app/test targets (see src/CMakeLists.txt). + set(QTMESH_DEFINE_DEBUG ON) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib-debug) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib-debug) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/debug) @@ -168,6 +173,24 @@ if(ENABLE_LOCAL_LLM) set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) set(LLAMA_BUILD_SERVER OFF CACHE BOOL "" FORCE) set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) + # The `mtmd` multimodal library (tools/mtmd) powers image captioning + # (SmolVLM, #764). It only builds under LLAMA_BUILD_TOOLS + LLAMA_BUILD_COMMON + # — but enabling all llama tools also builds CLI executables + # (cvector-generator, export-lora, llama-*-cli) that FAIL TO LINK on + # Windows-MinGW, where GGML_CPU is off (undefined ggml_backend_cpu_init / + # ggml_get_f32_nd). We don't ship those tools. So enable them only where + # they link: NOT on MinGW. The captioner (QTMESH_HAVE_MTMD) degrades + # gracefully when absent, and Windows-MinGW has ONNX off anyway, so the + # image-to-3D texture path there already falls back. + if(WIN32 AND MINGW) + set(LLAMA_BUILD_COMMON OFF CACHE BOOL "" FORCE) + set(LLAMA_BUILD_TOOLS OFF CACHE BOOL "" FORCE) + set(QTMESH_HAVE_MTMD OFF) + else() + set(LLAMA_BUILD_COMMON ON CACHE BOOL "" FORCE) + set(LLAMA_BUILD_TOOLS ON CACHE BOOL "" FORCE) + set(QTMESH_HAVE_MTMD ON) + endif() # Enable Metal on macOS for GPU acceleration if(APPLE) @@ -212,11 +235,20 @@ if(ENABLE_LOCAL_LLM) FetchContent_MakeAvailable(llama_cpp) - # Include llama.cpp headers + # Include llama.cpp headers (+ mtmd multimodal for image captioning where + # available — not on Windows-MinGW, see QTMESH_HAVE_MTMD above). include_directories( ${llama_cpp_SOURCE_DIR}/include ${llama_cpp_SOURCE_DIR}/ggml/include ) + if(QTMESH_HAVE_MTMD) + include_directories( + ${llama_cpp_SOURCE_DIR}/tools/mtmd + ${llama_cpp_SOURCE_DIR}/common + ) + add_definitions(-DENABLE_MTMD) + message(STATUS "mtmd multimodal (image captioning) enabled") + endif() add_definitions(-DENABLE_LOCAL_LLM) message(STATUS "Local LLM support enabled with llama.cpp") @@ -334,6 +366,12 @@ if(NOT TARGET xatlas) add_library(xatlas STATIC ${xatlas_SOURCE_DIR}/source/xatlas/xatlas.cpp) target_include_directories(xatlas PUBLIC ${xatlas_SOURCE_DIR}/source/xatlas) set_target_properties(xatlas PROPERTIES POSITION_INDEPENDENT_CODE ON) + # Build xatlas SINGLE-THREADED (XA_MULTITHREADED=0). Its internal + # std::thread TaskScheduler livelocks on large/organic image-to-3D meshes — + # ComputeCharts spins forever in cthread_yield/swtch_pri, freezing the app + # with an untextured mesh (verified via process sampling, #764). Unwrap is a + # one-off; single-threaded is slower but deadlock-free and correct. + target_compile_definitions(xatlas PRIVATE XA_MULTITHREADED=0) endif() message(STATUS "xatlas enabled (auto UV unwrap)") diff --git a/docs/TRIPOSG_EXPORT_NOTES.md b/docs/TRIPOSG_EXPORT_NOTES.md new file mode 100644 index 00000000..3938cbf0 --- /dev/null +++ b/docs/TRIPOSG_EXPORT_NOTES.md @@ -0,0 +1,370 @@ +# TripoSG → ONNX export notes (feat/triposg-backend) + +Research findings behind `scripts/export-triposg-onnx.py` — the exact inference +architecture of **TripoSG** (VAST-AI-Research/TripoSG, HF `VAST-AI/TripoSG`), +the tensor contract for the C++ ONNX Runtime side, the scheduler math the C++ +loop must implement, and the license verification. All facts below were read +from the upstream sources on 2026-07-02: + +- `triposg/pipelines/pipeline_triposg.py` (pipeline + `__call__` defaults) +- `triposg/schedulers/scheduling_rectified_flow.py` (`RectifiedFlowScheduler`) +- `triposg/models/transformers/triposg_transformer.py` (`TripoSGDiTModel`) +- `triposg/models/autoencoders/autoencoder_kl_triposg.py` (`TripoSGVAEModel`) +- `triposg/inference_utils.py` (`hierarchical_extract_geometry`) +- `scripts/inference_triposg.py`, `scripts/image_process.py` (reference driver) +- HF `VAST-AI/TripoSG`: `model_index.json` + per-component `config.json`s + +--- + +## Pipeline overview (what actually runs at inference) + +Unlike TripoSR (#764 — one feed-forward encoder/decoder pair), TripoSG is a +**rectified-flow diffusion** pipeline over a *vecset* latent (2048 tokens × 64 +channels, no spatial layout): + +```text +input image + │ background removal + white-bg composite + foreground crop (host C++) + │ BitImageProcessor: resize shortest-edge 256 → center-crop 224 + │ → /255 → ImageNet mean/std normalize + ▼ +DINOv2-large (transformers Dinov2Model, 24 layers, hidden 1024) + ▼ image_embeds = last_hidden_state [1, 257, 1024] (CLS token first) +latents x_T = N(0,1) [1, 2048, 64] + │ loop over N=50 timesteps (rectified flow, CFG scale 7.0): + │ v = DiT(cat[x;x], t_i, cat[zeros; image_embeds]) ← doubled batch + │ v = v_uncond + 7.0·(v_cond − v_uncond) + │ x ← x + (σ_i − σ_{i+1})·v + ▼ +x_0 (denoised vecset latent) + │ VAE decode: post_quant → 16 self-attn blocks over the 2048 tokens (ONCE) + │ → per-query: frequency-embed(point) → 1 cross-attn block + │ → norm_out → proj_out(1) → negate + ▼ +sdf(points) INSIDE-POSITIVE, surface at 0 + │ upstream: skimage marching_cubes(grid, level=0) on a dense-then-refined + │ octree grid over bounds (−1.005 … +1.005)³ + ▼ +mesh (trimesh; exported as-is — no vertex transform / face flip in upstream) +``` + +There is **no CLIP branch** — DINOv2 is the only conditioning encoder +(`model_index.json` lists exactly: `feature_extractor_dinov2` / +`image_encoder_dinov2` / `scheduler` / `transformer` / `vae`). + +--- + +## Verified component configs (verbatim from HF `VAST-AI/TripoSG`) + +`transformer/config.json`: + +```json +{ "_class_name": "TripoSGDiTModel", "cross_attention_dim": 1024, + "in_channels": 64, "num_attention_heads": 16, "num_layers": 21, "width": 2048 } +``` + +`vae/config.json`: + +```json +{ "_class_name": "Tripo2VAEModel", "embed_frequency": 8, "embed_include_pi": false, + "embedding_type": "frequency", "in_channels": 3, "latent_channels": 64, + "num_attention_heads": 8, "num_layers_decoder": 16, "num_layers_encoder": 8, + "width_decoder": 1024, "width_encoder": 512 } +``` + +`scheduler/scheduler_config.json`: + +```json +{ "_class_name": "RectifiedFlowScheduler", "num_train_timesteps": 1000, + "shift": 1, "use_dynamic_shifting": false } +``` + +`image_encoder_dinov2/config.json`: `facebook/dinov2-large` — `hidden_size` +1024, `patch_size` 14, `num_hidden_layers` 24, `image_size` 518 (position table +base; interpolated down at runtime). + +`feature_extractor_dinov2/preprocessor_config.json`: `BitImageProcessor`, +resize `shortest_edge: 256` (resample 3 = bicubic), `do_center_crop` to +**224×224**, rescale 1/255, normalize mean `[0.485,0.456,0.406]` / +std `[0.229,0.224,0.225]` (ImageNet). ⇒ DINOv2 runs at **224**, giving +`1 + (224/14)² = 257` tokens. (Contrast TripoSR: 512px, **no** mean/std.) + +Checkpoint sizes (fp32 safetensors): DINOv2 1.22 GB, DiT **5.76 GB** +(~1.44 B params), VAE 0.97 GB. Total ~7.95 GB. + +`__call__` defaults (pipeline_triposg.py, quoted): + +```python +num_inference_steps=50, num_tokens=2048, guidance_scale=7.0, +bounds=(-1.005, -1.005, -1.005, 1.005, 1.005, 1.005), +dense_octree_depth=8, hierarchical_octree_depth=9, flash_octree_depth=9, +use_flash_decoder=True +``` + +(The reference driver `scripts/inference_triposg.py` calls +`pipe(image=img_pil, num_inference_steps=50, guidance_scale=7.0)`.) + +--- + +## Tensor contract (what the exported graphs expose) + +### a. `triposg_image_encoder.onnx` (~1.2 GB fp32) + +| | name | dtype | shape | notes | +|--|------|-------|-------|-------| +| in | `image` | float32 | `[1, 3, 224, 224]` | RGB in `[0,1]`; ImageNet mean/std **baked into the graph** | +| out | `image_embeds` | float32 | `[1, 257, 1024]` | `Dinov2Model(...).last_hidden_state`, CLS first | + +C++ preprocessing to reproduce upstream (`scripts/image_process.py +prepare_image` + `BitImageProcessor`): + +1. Matte the subject (upstream uses **BriaRMBG — non-commercial, do NOT use**; + we substitute the existing Apache-2.0 U²-Net `BackgroundRemover`). +2. Composite over **WHITE** (`bg_color = [1.0, 1.0, 1.0]`) — note TripoSR uses + gray 128; TripoSG's reference is white. +3. Crop to the foreground alpha bbox with **10% padding** + (`padding_ratio=0.1` of the larger dimension, offset to center the smaller + dimension — i.e. a roughly square, centered crop). +4. Resize shortest edge → 256 (bicubic), center-crop 224×224, divide by 255. + (Normalization happens inside the graph.) + +### b. `triposg_dit_step.onnx` (~5.7 GB fp32 → **external-data sidecar**; int8 ~1.5 GB single file) + +| | name | dtype | shape | notes | +|--|------|-------|-------|-------| +| in | `latents` | float32 | `[B, 2048, 64]` | B dynamic; 2 for CFG | +| in | `timestep` | float32 | `[B]` | post-shift `t = 1000·σ_i` (same value repeated) | +| in | `image_embeds` | float32 | `[B, 257, 1024]` | row order **[uncond; cond]**; uncond = all zeros | +| out | `velocity` | float32 | `[B, 2048, 64]` | model prediction ≈ `(x0 − noise)` | + +Upstream CFG (quoted from the pipeline): + +```python +latent_model_input = torch.cat([latents] * 2) +timestep = t.expand(latent_model_input.shape[0]) +noise_pred = self.transformer(latent_model_input, timestep, + encoder_hidden_states=image_embeds, ...)[0] +noise_pred_uncond, noise_pred_image = noise_pred.chunk(2) +noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_image - noise_pred_uncond) +``` + +`encode_image` builds the uncond row as `torch.zeros_like(image_embeds)` and +concatenates `[negative, positive]` — so the C++ side can either run one B=2 +call (upstream-exact) or two B=1 calls (identical math; the export's +`--verify` cross-checks B=1 vs B=2 row 0). + +**>2 GB note:** the fp32 DiT exceeds the 2 GB protobuf limit, so it ships as +`triposg_dit_step.onnx` **plus** `triposg_dit_step.onnx.data` (single +consolidated external-data file, same directory). ONNX Runtime loads it +transparently, but the C++ downloader must fetch **both** files and keep them +side by side. The int8 variant is a single file. + +### c. `triposg_vae_latents.onnx` (~0.9 GB fp32 — run ONCE per generation) + +| | name | dtype | shape | +|--|------|-------|-------| +| in | `latents` | float32 | `[1, 2048, 64]` (the denoised x₀) | +| out | `kv_cache` | float32 | `[1, 2048, 1024]` | + +This is `post_quant` (Linear 64→1024) + the `TripoSGDecoder`'s 16 +self-attention blocks over the latent set — upstream's `kv_cache` (computed on +the first `_decode` chunk, reused for all later chunks; type-hinted +`Optional[torch.Tensor]`, i.e. one tensor, not per-layer pairs). + +### d. `triposg_vae_decoder.onnx` (small — per-chunk query graph) + +| | name | dtype | shape | notes | +|--|------|-------|-------|-------| +| in | `kv_cache` | float32 | `[1, 2048, 1024]` | from graph (c) | +| in | `points` | float32 | `[1, P, 3]` | **P dynamic**; world coords within `(−1.005 … +1.005)` | +| out | `sdf` | float32 | `[1, P, 1]` | **inside-positive**, surface at iso **0.0** | + +Internals: `FrequencyPositionalEmbedding(num_freqs=8, logspace=True, +input_dim=3, include_pi=False)` (3 → 3 + 3·2·8 = 51 channels) → query +projection → **one** cross-attention DiT block against `kv_cache` → `norm_out` +→ `proj_out` (Linear → 1) → `* −1`. + +`--monolithic` additionally emits `triposg_vae_decode_mono.onnx` +(`latents [1,2048,64] + points [1,P,3] → sdf [1,P,1]`) — the unsplit reference +used to validate the kv-split (the script always torch-checks the split +against `vae.decode(latents, sampled_points=pts).sample` and warns loudly on +mismatch). + +--- + +## Scheduler math (the C++ loop contract) + +`RectifiedFlowScheduler` (released config: `num_train_timesteps=1000`, +`shift=1`, `use_dynamic_shifting=false`). Quoted upstream `set_timesteps`: + +```python +timesteps = np.array([(1.0 - i / num_inference_steps) * 1000 for i in range(num_inference_steps)]) +sigmas = timesteps / 1000 +sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) # identity when shift == 1 +timesteps = sigmas * 1000 +self.sigmas = cat([sigmas, zeros(1)]) # σ_N = 0 appended +``` + +and `step`: + +```python +sigma = self.sigmas[i] +sigma_next = self.sigmas[i + 1] +prev_sample = sample + (sigma - sigma_next) * model_output +``` + +So for **N = 50** (default): `σ_i = 1 − i/50` → `1.00, 0.98, …, 0.02`, plus +`σ_50 = 0`; `t_i = 1000·σ_i` → `1000, 980, …, 20`; every step is +`x ← x + 0.02 · v_cfg` (uniform Δσ = 1/N). Initial latents are **pure +N(0,1)** — no `init_noise_sigma` scaling exists, consistent with the training +interpolation `x_t = (1−σ)·x0 + σ·noise` (`scale_noise`, quoted), which at +σ=1 is pure noise. Hence the model output points from noise toward data +(≈ `x0 − ε`), and the `+(σ_i − σ_{i+1})·v` update (σ decreasing) integrates to +x₀ at σ=0. + +⚠ **Sign trap:** diffusers' stock `FlowMatchEulerDiscreteScheduler` writes the +update as `sample + (sigma_next − sigma) * model_output` (its models predict +`ε − x0`). TripoSG's scheduler/model use the **opposite** convention. Implement +exactly the formula above, not the diffusers one. + +C++ pseudo-loop: + +```cpp +latents = randn(1, 2048, 64); // fixed seed for reproducibility +for (i = 0; i < N; ++i) { + float t = 1000.f * sigma[i]; + v_u = dit(latents, t, zeros_embeds); // or one B=2 call + v_c = dit(latents, t, image_embeds); + v = v_u + guidance * (v_c - v_u); // guidance = 7.0 + latents += (sigma[i] - sigma[i + 1]) * v; +} +``` + +CFG is applied when `guidance_scale > 1` (diffusers convention); the reference +driver always uses 7.0. + +--- + +## Surface extraction (host-side marching cubes) + +Upstream defaults to a "flash" octree decoder and offers +`hierarchical_extract_geometry` — both are torch/GPU octree machinery we do +**not** port. The dense path we replicate (from `hierarchical_extract_geometry`): + +- Grid: `linspace(bbox_min, bbox_max, num_cells)` per axis over bounds + `(−1.005, +1.005)` with `dense_octree_depth=8` (≈ 2⁸(+1) = 256/257 samples + per axis; upstream then refines near-surface cells to depth 9 ≈ 512). + For our C++ dense-grid approach: query `resolution³` points, resolution + user-tunable (256 default / 128 preview — matching the TripoSR tiers). +- Field: `grid_logits` from `vae.decode(...).sample` used **directly** — + `skimage.measure.marching_cubes(grid_logits, level=0)` with the default + `gradient_direction="descent"` (object = values **greater** than the level). + I.e. the exported `sdf` output is **inside-positive with the surface at 0** — + the same convention as TripoSR's `density − threshold`, so + `MarchingCubes::extract(field = sdf, isoLevel = 0)` drops straight in. +- Winding: upstream applies no face flip after skimage MC. Our native MC + emitted flipped winding (`v0,v2,v1`) for TripoSR's inside-positive field — + the same flip should apply here, but **verify empirically on the first + end-to-end run** (render both windings once). +- Vertex rescale (upstream): `verts / 2^depth * bbox_size + bbox_min`. With our + MC's `gridMin/gridMax = ±1.005` this is handled by the existing world-box + mapping. +- No threshold constant (unlike TripoSR's 25.0) — the level is plain 0.0 + (`mc_level=0.0` default in `flash_extract_geometry` too). + +--- + +## Licenses (runtime components) — verified 2026-07-02 + +| Component | Source | License | OK? | +|-----------|--------|---------|-----| +| TripoSG code | github.com/VAST-AI-Research/TripoSG `LICENSE` | **MIT** ("Copyright (c) 2025 VAST-AI-Research and contributors") | ✅ | +| TripoSG weights (DiT + VAE) | HF `VAST-AI/TripoSG` | **MIT** (repo license tag: mit) | ✅ | +| DINOv2-large weights | redistributed inside HF `VAST-AI/TripoSG` (`image_encoder_dinov2/`); upstream `facebook/dinov2-large` | **Apache-2.0** upstream; shipped copy sits in the MIT-tagged repo | ✅ (credit Meta AI in THIRD_PARTY_AI_MODELS.md) | +| Background removal | upstream uses **briaai/RMBG-1.4** (`scripts/briarmbg.py`) | **NON-COMMERCIAL** (bria-rmbg-1.4: commercial use requires a paid BRIA agreement) | ❌ **DO NOT SHIP** — substitute our existing U²-Net (Apache-2.0) `BackgroundRemover`, already in the app for TripoSR | +| skimage / torchmcubes MC | upstream extraction | BSD / MIT | N/A — replaced by our native `src/MarchingCubes` | +| ONNX Runtime | existing dependency | MIT | ✅ | + +The only license landmine is **BriaRMBG** — it is quarantined to the upstream +repo's demo scripts and never touches the export or the C++ runtime path. + +--- + +## Export mechanics & risks + +- **Exporter:** legacy TorchScript path (`dynamo=False`), opset **18**, + `do_constant_folding=True` — repo convention (TripoSR used opset 17; 18 adds + nothing risky and is ORT 1.20.1-supported). +- **DINOv2 position-embedding interpolation:** same trap as TripoSR's ViT — + `interpolate_pos_encoding` calls `F.interpolate(bicubic, antialias)` to map + the 518-base table to the 224 input. Fixed input ⇒ the table is a constant; + the script precomputes it and monkeypatches the method + (`freeze_dinov2_pos_encoding`). `# VERIFY:` signature checked against + transformers 4.45.x (the version the HF configs were saved with). +- **>2 GB DiT:** exported via a scratch dir, then consolidated with + `onnx.save_model(save_as_external_data=True, all_tensors_to_one_file=True)` + into `triposg_dit_step.onnx` + `triposg_dit_step.onnx.data`. Hosting and the + C++ downloader must treat the pair atomically. The int8 variant + (MatMul-only dynamic QInt8 — avoids ConvInteger, the TripoSR lesson) is a + single ~1.5 GB file and is the realistic default download tier. +- **VAE kv-split (`# VERIFY:` in the script):** `TripoSGDecoder.forward(sample, + queries, kv_cache)` computes `kv_cache` from `sample` via `blocks[:-1]` when + `None`, and the query path (`proj_query` → final cross-attn block → + `norm_out`/`proj_out` → `*−1`) reads only the cache. The split wrappers rely + on (1) the cache being a single tensor and (2) `sample` being unused when the + cache is provided. The script *always* cross-checks the split against + `vae.decode(latents, sampled_points=pts).sample` in torch and warns loudly; + `--monolithic` provides the unsplit fallback graph. +- **Monolithic trace trick:** `_decode`'s python chunk loop is traced with + `num_chunks = 1<<40` so it unrolls to exactly one iteration and the traced + slice stays valid for any dynamic `P`. +- **Chunk independence:** upstream chunks queries arbitrarily (50 000/chunk) + and concatenates, so per-chunk results are independent by construction — the + dynamic-P graph is exact for any C++ chunk size (verified in `--verify` with + P=777 vs P=4096 prefixes). +- **RoPE / QK-norm / U-Net skips:** `TripoSGDiTModel` uses RMS QK-norm, U-Net + style skip connections between its 21 blocks, and threads an optional + `image_rotary_emb` argument (unused by this pipeline — nothing computes it in + `__call__`). All are ordinary traceable ops; the time embedding + (`Timesteps` sinusoid + GELU MLP, prepended as an extra sequence token and + stripped before `proj_out`) is inside the graph, so C++ passes the plain + float timestep. +- **Compute reality check:** a 1.44 B-param DiT × 2 (CFG) × 50 steps on CPU is + minutes-scale, not seconds — expect ~10–60 min fp32 on a laptop CPU; int8 + + fewer steps (rectified flow degrades gracefully at e.g. N=25) + CoreML EP are + the practical knobs. This is the main product risk vs TripoSR, not a + feasibility risk. +- **Not fetched/verified (low risk, flagged):** the exact + `generate_dense_grid_points_gpu` cell count (2⁸ vs 2⁸+1 samples per axis — + irrelevant to our own grid choice), the upstream repo `requirements.txt` + pins (the script documents `diffusers==0.30.3` from the checkpoint metadata + and `transformers>=4.45`), and the `TripoSGDecoder` internal attribute names + (not needed — the wrappers call whole modules, never sub-attributes). + +--- + +## Live-test findings (2026-07-03, first end-to-end C++ runs) + +- **Per-tensor int8 destroys the geometry.** The shipped + `triposg_dit_step_int8.onnx` (MatMul-only dynamic QInt8, per-tensor) has + max-rel-err 1.67e-02 on a single step, but the error COMPOUNDS over the flow + loop (2 CFG calls/step, amplified 7× by guidance): 4 steps → blobby but + semi-coherent, 8 steps → fragmented, 25 steps → disconnected noise splatter. + The fp32 DiT with the identical C++ loop produces a coherent single figure — + the loop/export/decode are correct; the quantization is the culprit. + `export-triposg-onnx.py` now quantizes `per_channel=True, reduce_range=True`; + the hosted int8 file must be re-exported + re-uploaded before the int8 tier + is trustworthy. Until then fp32 is the only recommended tier. +- **Decoder chunk must stay small (8192).** The VAE decoder cross-attends every + query point to the 2048 kv tokens, so per-Run activation memory is linear in + P with a huge constant: TripoSR's 262144-point chunk transiently allocated + ~90 GB at res 256 and macOS killed the process. `TripoSGPredictor` hard-caps + the chunk at 8192 (a few hundred MB per Run). +- **Sessions are staged, not concurrent.** Encoder (~1.1 GB), DiT (1.3–5.4 GB) + and vae_latents (~769 MB) are each opened when their stage starts and + destroyed when it completes; only the ~48 MB point decoder lives through the + long decode/refine tail. Peak RSS ≈ the largest single stage (measured + ~1.1 GB int8 end-to-end), instead of the >4 GB sum that previously tripped + memory pressure alongside the 90 GB chunk bug. +- **Image preprocessing matches BitImageProcessor now:** resize shortest edge + to 8/7·crop (256 for 224) + centre-crop, not a squash-resize. diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index b7cbd638..20f023cd 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -1530,6 +1530,7 @@ Rectangle { property var mgSteps: [] property int mgActiveIdx: -1 property real mgActiveProgress: -1 // 0..1; < 0 → indeterminate + property bool mgAiPending: false // AI texture queued for onCompleted // A small local button factory (raw QML — the Themed* wrappers blank // this dynamically-loaded panel, so we style raw controls with the @@ -1729,6 +1730,24 @@ Rectangle { } } + // Auto-generated caption of the selected image (SmolVLM), computed + // in the background the moment the image is picked. Shown here so + // the user sees what the model "read" from the image — it becomes + // the texture prompt. "Describing…" while it's still running. + Text { + width: parent.width - 16 + visible: MeshGenController.selectedImagePath.length > 0 + && (MeshGenController.captioning + || MeshGenController.caption.length > 0) + text: MeshGenController.captioning + ? "🔍 Describing image…" + : "📝 " + MeshGenController.caption + color: PropertiesPanelController.textColor + font.pixelSize: 11 + font.italic: MeshGenController.captioning + wrapMode: Text.WordWrap + } + // Resolution picker — marching-cubes grid resolution. Cost grows with // the cube of the value (the decoder queries resolution³ points), so // higher = more detail but much slower. Labels flag the trade-off. @@ -1754,11 +1773,41 @@ Rectangle { } } - // Model quality/size tier — the encoder downloads in the picked - // precision (fp32 best/largest → int8 smallest). index maps 1:1 to the - // MeshGenController quality int (0/1). + // Backend: TripoSR (fast, textured) vs TripoSG (rectified flow — + // higher-fidelity geometry, geometry-only, slower; models download + // on first use). Declared BEFORE the Model row so the tier picker + // can react to it. Row { spacing: 6 + Text { + text: "Backend" + color: PropertiesPanelController.textColor + font.pixelSize: 11 + anchors.verticalCenter: parent.verticalCenter + } + InspectorComboBox { + id: mgBackendCombo + width: 190 + enabled: !MeshGenController.busy + model: ["TripoSR (fast, textured)", "TripoSG (best geometry)"] + currentIndex: 0 + // Switching to TripoSG snaps the tier picker to fp32 (its + // only geometry tier); the int8 option is meaningless there. + onCurrentIndexChanged: { + if (currentIndex === 1) + mgQualityCombo.currentIndex = 0 + } + } + } + + // Model quality/size tier. TripoSR: fp32 (best/largest) vs int8 + // (smallest), 1:1 with the MeshGenController quality int. TripoSG: + // fp32 ONLY — the quantized 1.5B DiT degrades geometry to blobs + // and is no faster on ARM, so the list collapses to the single + // real option and locks. + Row { + spacing: 6 + property bool isSG: mgBackendCombo.currentIndex === 1 Text { text: "Model" color: PropertiesPanelController.textColor @@ -1767,9 +1816,14 @@ Rectangle { } InspectorComboBox { id: mgQualityCombo - width: 150 - enabled: !MeshGenController.busy - model: ["fp32 (best, ~1.7GB)", "int8 (smaller, ~430MB)"] + width: 190 + // TripoSG ships fp32 only (int8 degrades geometry); show + // its size for consistency rather than a placeholder, and + // lock the picker since there's nothing to switch to. + enabled: !MeshGenController.busy && !parent.isSG + model: parent.isSG + ? ["fp32 (best, ~2GB)"] + : ["fp32 (best, ~1.7GB)", "int8 (smaller, ~430MB)"] currentIndex: 0 } } @@ -1793,16 +1847,56 @@ Rectangle { text: "Refine surface (re-project)" checked: true } + // ---- Colour / texture stages, in execution order ---------------- + // TripoSG (geometry-only) gets its colour SOLELY from the AI + // texture pass — so when TripoSG is the backend, that checkbox + // leads and the plain "Bake diffuse" (TripoSR field colour) is + // hidden. TripoSR keeps the classic bake → PBR → upscale chain. + property bool sgSelected: mgBackendCombo.currentIndex === 1 + + // AI texture (multi-view depth-ControlNet): front from the input + // photo, back/sides SD-generated from the shape, then projected. + // For TripoSG this is the only colour source; for TripoSR it's an + // optional higher-quality alternative to the field bake. + InspectorCheck { + id: mgAiTexture + text: "Generate texture (AI, front photo + generated back)" + checked: parent.sgSelected + enabled: !MeshGenController.busy + && MaterialEditorQML.stableDiffusionEnabled + } + Text { + visible: mgAiTexture.checked && !MaterialEditorQML.sdModelLoaded + text: MaterialEditorQML.stableDiffusionEnabled + ? " ⚠ Load a Stable Diffusion model in AI Settings first." + : " ⚠ This build has no Stable Diffusion support." + color: PropertiesPanelController.textColor + font.pixelSize: 10 + wrapMode: Text.WordWrap + width: parent.width - 16 + } + // Plain diffuse bake (TripoSR field colour). Hidden for TripoSG, + // where colour is the AI texture pass only. InspectorCheck { id: mgBake text: "Bake diffuse texture" checked: true + visible: !parent.sgSelected + enabled: !MeshGenController.busy } InspectorCheck { id: mgPbr text: "Generate PBR maps (normal + roughness)" - checked: true - enabled: !MeshGenController.busy && mgBake.checked + // Default OFF for TripoSG — the synthesized normal/roughness + // over its AI texture tends to look worse than the plain + // diffuse (user-verified); ON for TripoSR where it helps. The + // binding re-evaluates when the backend changes; the user can + // still toggle it per run. + checked: !parent.sgSelected + // Valid whenever there's a diffuse to derive from: the plain + // bake OR the AI texture. Runs after whichever produced it. + enabled: !MeshGenController.busy + && (mgAiTexture.checked || mgBake.checked) } InspectorCheck { id: mgUpscale @@ -1819,34 +1913,70 @@ Rectangle { clickEnabled: !MeshGenController.busy && MeshGenController.selectedImagePath.length > 0 onClicked: { + var sg = mgBackendCombo.currentIndex === 1 // TripoSG var steps = [{ key: "prep", label: "Prepare models" }] - if (mgRemoveBg.checked) + // The worker only posts a "background" stage on the TripoSR + // path; TripoSG removes the bg inside its predict() dispatch + // without a discrete progress event, so a "background" row + // there would never resolve and look stuck. + if (mgRemoveBg.checked && !sg) steps.push({ key: "background", label: "Remove background" }) steps.push({ key: "encode", label: "Encode image" }) + if (sg) + steps.push({ key: "denoise", label: "Denoise (flow steps)" }) steps.push({ key: "decode", label: "Reconstruct 3D" }) if (mgRefine.checked) steps.push({ key: "refine", label: "Refine surface" }) - if (mgBake.checked) + // AI texture requested + available? (TripoSG's ONLY colour + // source; optional extra for TripoSR.) + var aiTex = mgAiTexture.checked + && MaterialEditorQML.stableDiffusionEnabled + + // Colour stages. For TripoSG we do NOT bake colour at build + // time (no TripoSR field colouring) — colour is entirely the + // AI texture pass. With no AI, TripoSG ships an uncoloured + // (neutral clay) mesh. TripoSR keeps its own field bake. + var buildBake = mgBake.checked && (!sg || !aiTex) + // PBR: for TripoSG+AI it runs AFTER the AI texture (below), + // not at build time. Otherwise it runs at build from the + // build-time bake. + var buildPbr = mgPbr.checked && buildBake + + if (buildBake) steps.push({ key: "bake", label: "Bake texture" }) - else + else if (!sg && !mgBake.checked) steps.push({ key: "color", label: "Vertex colors" }) - if (mgUpscale.checked && mgBake.checked) + if (mgUpscale.checked && buildBake) steps.push({ key: "upscale", label: "Upscale texture 2×" }) steps.push({ key: "build", - label: (mgPbr.checked && mgBake.checked) - ? "Build mesh + PBR maps" : "Build mesh" }) + label: buildPbr ? "Build mesh + PBR maps" + : "Build mesh" }) + // AI texture pass (after the mesh is built): the front is + // the pinned photo (no SD), the back/sides are generated, + // then projection-baked, then PBR from that final texture. + if (aiTex) { + steps.push({ key: "aitex_gen", + label: "AI texture: generate views" }) + steps.push({ key: "aitex_bake", + label: "AI texture: project + bake" }) + if (mgPbr.checked) + steps.push({ key: "aitex_pbr", + label: "PBR maps from AI texture" }) + } mgRoot.mgSteps = steps mgRoot.mgActiveIdx = 0 mgRoot.mgActiveProgress = -1 + mgRoot.mgAiPending = aiTex // gate onCompleted's AI kickoff MeshGenController.generateSelected( mgResCombo.resValue, mgRemoveBg.checked, mgQualityCombo.currentIndex, { "smooth": mgSmooth.checked, "refine": mgRefine.checked, - "bake_texture": mgBake.checked, - "generate_pbr": mgPbr.checked, - "upscale_texture": mgUpscale.checked + "bake_texture": buildBake, + "generate_pbr": buildPbr, + "upscale_texture": mgUpscale.checked && buildBake, + "backend": sg ? "triposg" : "triposr" }) } } @@ -1957,9 +2087,94 @@ Rectangle { mgRoot.mgActiveIdx = mgRoot.mgSteps.length // all ✓ mgStatus.text = "Done: " + result.vertexCount + " verts, " + result.triangleCount + " tris" + + // Optional AI texture pass: front from the input photo, + // back/sides SD-generated (depth-ControlNet), then PBR from + // that final texture. Runs on the just-built entity. + if (mgRoot.mgAiPending && result.entityName) { + mgRoot.mgAiPending = false + if (!MaterialEditorQML.sdModelLoaded) { + mgStatus.text = "Mesh built; skipped AI texture " + + "(no SD model loaded — see AI Settings)." + return + } + // Select the new entity so the multi-view bake targets + // it, then kick off the generated views (+PBR). + PropertiesPanelController.selectNodeByName(result.entityName) + // Mark the first AI step active (leave build ✓); later + // steps advance on the SD / bake / PBR signals. + var pbrRows = mgPbr.checked ? 3 : 2 + mgRoot.mgActiveIdx = mgRoot.mgSteps.length - pbrRows + mgRoot.mgActiveProgress = -1 + mgStatus.text = "Mesh built — generating AI texture…" + // Pass the caption computed in the BACKGROUND when the + // image was picked (ready by now — no UI-blocking + // captioning here). Empty falls back to a neutral prompt + // inside generateMeshTextureMultiView. + MaterialEditorQML.generateMeshTextureMultiView( + MeshGenController.caption, 512, 512, 0.9, + ["front", "back"], + "", // no photo pinning + mgPbr.checked) + } } function onError(msg) { mgStatus.text = "Error: " + msg } } + // Drive the AI-texture progress rows (generate → bake → optional + // PBR) from the SD + PBR-synth signals. Rows are found by key so + // the optional PBR row doesn't shift the indices. + Connections { + target: MaterialEditorQML + enabled: mgAiTexture.checked + function stepIdx(key) { + for (var i = 0; i < mgRoot.mgSteps.length; i++) + if (mgRoot.mgSteps[i].key === key) return i + return -1 + } + function onSdGenerationProgressChanged() { + var i = stepIdx("aitex_gen") + if (i < 0) return + if (mgRoot.mgActiveIdx < i) mgRoot.mgActiveIdx = i + if (mgRoot.mgActiveIdx === i) + mgRoot.mgActiveProgress = MaterialEditorQML.sdGenerationProgress + } + function onSdTextureGenerated(path) { + // Views done + projected/applied → mark the bake row ✓. + var b = stepIdx("aitex_bake") + if (b >= 0 && mgRoot.mgActiveIdx <= b) { + var p = stepIdx("aitex_pbr") + if (p < 0) { mgRoot.mgActiveIdx = mgRoot.mgSteps.length + mgStatus.text = "AI texture applied." } + else { mgRoot.mgActiveIdx = p // PBR now running + mgRoot.mgActiveProgress = -1 } + } + } + function onPbrSynthCompleted(res) { + if (stepIdx("aitex_pbr") >= 0) { + mgRoot.mgActiveIdx = mgRoot.mgSteps.length // all ✓ + mgStatus.text = "AI texture + PBR applied." + } + } + function onSdGenerationError(msg) { + mgStatus.text = "AI texture error: " + msg + } + function onPbrSynthError(msg) { + mgStatus.text = "PBR error: " + msg + } + // Relay the AI-texture step notices (simplify / unwrap / "view + // N/M — step X/Y" / bake) to the panel status line, so texture + // progress is visible HERE and not only in the Material Editor + // window. Also drive the live progress bar off the SD step + // fraction while the generate view is the active step. + function onSdGenerationNotice(msg) { + mgStatus.text = msg + var i = -1 + for (var k = 0; k < mgRoot.mgSteps.length; k++) + if (mgRoot.mgSteps[k].key === "aitex_gen") { i = k; break } + if (i >= 0 && mgRoot.mgActiveIdx === i) + mgRoot.mgActiveProgress = MaterialEditorQML.sdGenerationProgress + } + } } } diff --git a/scripts/export-triposg-onnx.py b/scripts/export-triposg-onnx.py new file mode 100644 index 00000000..7459492b --- /dev/null +++ b/scripts/export-triposg-onnx.py @@ -0,0 +1,611 @@ +#!/usr/bin/env python3 +"""Export TripoSG (image → 3D shape diffusion) to ONNX graphs for the C++ backend. + +ONE-TIME, OFFLINE developer tool — NOT shipped with the app, NOT wired into CMake +or CI. The app never runs Python: it runs the exported .onnx files in C++ via +ONNX Runtime (CPU / CoreML EP), downloading them on first use. + +MODEL: TripoSG (VAST-AI-Research/TripoSG — "TripoSG: High-Fidelity 3D Shape + Synthesis using Large-Scale Rectified Flow Models", arXiv 2502.06608). + Code **MIT** (repo LICENSE: "MIT License, Copyright (c) 2025 VAST-AI-Research + and contributors"); weights **MIT** (HF `VAST-AI/TripoSG`, license tag: mit). + The bundled image encoder is facebook/dinov2-large (Apache-2.0 upstream, + redistributed inside the MIT-tagged HF repo). Clears QtMeshEditor's + permissive-redistribution bar — see THIRD_PARTY_AI_MODELS.md and + docs/TRIPOSG_EXPORT_NOTES.md. + ⚠ The upstream repo's background-removal helper (scripts/briarmbg.py, + briaai/RMBG-1.4) is NON-COMMERCIAL and must NOT be shipped or exported; + the app substitutes its existing Apache-2.0 U²-Net BackgroundRemover. + +WHY FOUR GRAPHS (unlike TripoSR's clean encoder/decoder pair) + TripoSG is a *diffusion* pipeline — a rectified-flow DiT denoises a 2048-token + vecset latent over N steps, conditioned on DINOv2 image features via + cross-attention, then a vecset VAE decodes SDF values at query points: + + image ── DINOv2-large ──────────────► image_embeds [1,257,1024] (graph a) + latents[1,2048,64] = N(0,1) + loop i = 0..N-1 (C++ owns the scheduler loop): + v = DiT(latents, t_i, embeds) velocity prediction (graph b) + v = v_u + s·(v_c − v_u) classifier-free guidance (C++) + latents += (σ_i − σ_{i+1})·v rectified-flow Euler step (C++) + kv = VAE latent processing(latents) run ONCE (graph c) + sdf(points) = VAE query(kv, points) chunked over the grid (graph d) + mesh = marching cubes on sdf at iso 0 (inside-positive) (host C++) + + The heavy VAE self-attention stack over the 2048 latent tokens must not be + re-run for every grid chunk (a 257³ grid is ~340 chunks of 50k points), so the + VAE decode is split into a run-once "latents" graph (c) and a tiny per-chunk + "query" graph (d) — mirroring upstream's kv_cache. `--monolithic` additionally + emits the unsplit (latents, points) → sdf graph as a correctness reference. + +WHAT IT PRODUCES (the contract the C++ predictor targets; full details + +scheduler math in docs/TRIPOSG_EXPORT_NOTES.md) + + triposg_image_encoder.onnx (~1.2 GB fp32 — DINOv2-large, ImageNet + normalization baked into the graph) + input "image" float32 [1, 3, 224, 224] RGB in [0,1] + output "image_embeds" float32 [1, 257, 1024] last_hidden_state (CLS first) + + triposg_dit_step.onnx (~5.7 GB fp32 → external-data sidecar + triposg_dit_step.onnx.data; int8 ~1.5 GB single file) + input "latents" float32 [B, 2048, 64] B dynamic (2 for CFG: [uncond;cond]) + input "timestep" float32 [B] post-shift t = 1000·σ_i + input "image_embeds" float32 [B, 257, 1024] row 0 = zeros (uncond), row 1 = cond + output "velocity" float32 [B, 2048, 64] model_output ≈ (x0 − noise) + + triposg_vae_latents.onnx (~0.9 GB fp32 — post_quant + 16 self-attn blocks) + input "latents" float32 [1, 2048, 64] the denoised x0 + output "kv_cache" float32 [1, 2048, 1024] processed latent set (run once) + + triposg_vae_decoder.onnx (small — frequency embed + 1 cross-attn block + head) + input "kv_cache" float32 [1, 2048, 1024] + input "points" float32 [1, P, 3] P dynamic; world coords in + bounds (−1.005 … +1.005)³ + output "sdf" float32 [1, P, 1] INSIDE-POSITIVE, surface at 0.0 + (upstream negates its raw logits; + this output == vae.decode().sample) + + + triposg_dit_step_int8.onnx (unless --no-quant; MatMul-only dynamic QInt8) + + triposg_vae_decode_mono.onnx (only with --monolithic; (latents, points) → sdf) + +SAMPLING DEFAULTS (upstream pipeline_triposg.py __call__): + num_inference_steps=50, guidance_scale=7.0, num_tokens=2048, + scheduler = RectifiedFlowScheduler(num_train_timesteps=1000, shift=1) + σ_i = 1 − i/N (shift=1 ⇒ identity shift), σ_N = 0; t_i = 1000·σ_i + step: latents ← latents + (σ_i − σ_{i+1}) · velocity [uniform 0.02 at N=50] + decode bounds (−1.005…1.005)³, dense grid depth 8 (≈257³), MC at iso 0. + +USAGE (offline venv; ~8 GB download from HF `VAST-AI/TripoSG` on first run): + python3 -m venv venv + ./venv/bin/pip install torch --index-url https://download.pytorch.org/whl/cpu + ./venv/bin/pip install "diffusers==0.30.3" "transformers>=4.45,<5" onnx \ + onnxruntime huggingface_hub numpy einops jaxtyping safetensors accelerate + git clone --depth 1 https://github.com/VAST-AI-Research/TripoSG + ./venv/bin/python scripts/export-triposg-onnx.py \ + --triposg ./TripoSG --out dist/triposg_onnx --verify + +Then upload the .onnx (+ the DiT's .onnx.data sidecar) to the HF models repo +under triposg/ (fernandotonon/QtMeshEditor-models) — the TripoSR precedent. +""" +import argparse +import os +import shutil +import sys +import tempfile + +import numpy as np +import torch +import torch.nn as nn + +# --------------------------------------------------------------------------- +# Constants measured from the released VAST-AI/TripoSG checkpoint configs +# (see docs/TRIPOSG_EXPORT_NOTES.md for the verbatim config.json contents). +# --------------------------------------------------------------------------- +NUM_TOKENS = 2048 # pipeline __call__ default num_tokens +LATENT_CHANNELS = 64 # vae/config.json latent_channels == transformer in_channels +COND_TOKENS = 257 # DINOv2-large @224: 1 CLS + (224/14)^2 = 257 +COND_DIM = 1024 # dinov2-large hidden_size == transformer cross_attention_dim +IMAGE_SIZE = 224 # BitImageProcessor: resize shortest edge 256 -> center crop 224 +VAE_WIDTH_DECODER = 1024 # vae/config.json width_decoder (kv_cache channel dim) +IMAGENET_MEAN = [0.485, 0.456, 0.406] +IMAGENET_STD = [0.229, 0.224, 0.225] +DEFAULT_STEPS = 50 +DEFAULT_GUIDANCE = 7.0 +DECODE_BOUNDS = 1.005 # pipeline __call__ default bounds (±1.005 per axis) + +TWO_GB = 2 * 1024 * 1024 * 1024 + + +def log(tag, msg): + print(f"[{tag}] {msg}", flush=True) + + +# --------------------------------------------------------------------------- +# Model loading +# --------------------------------------------------------------------------- +def load_pipeline(triposg_dir, weights_dir): + """Import the triposg package from a git checkout and build the diffusers + pipeline from the HF weights. Returns the TripoSGPipeline on CPU/fp32.""" + sys.path.insert(0, os.path.abspath(triposg_dir)) + + # `triposg.inference_utils` imports `from diso import DiffDMC` at module + # scope. diso is a CUDA-only differentiable-marching-cubes package that + # neither installs on macOS/CPU boxes nor matters here: this export only + # touches the pipeline's MODELS (image encoder / DiT / VAE); the app does + # surface extraction with its own native marching cubes. Stub it out — + # the exact torchmcubes trick the TripoSR exporter uses. + import types # noqa: E402 + if "diso" not in sys.modules: + _diso = types.ModuleType("diso") + class _DiffDMCStub: # noqa: N801 — never instantiated by this export + def __init__(self, *a, **k): + raise RuntimeError("diso stub: not available in this export env") + _diso.DiffDMC = _DiffDMCStub + sys.modules["diso"] = _diso + log("note", "stubbed CUDA-only 'diso' (unused by the export)") + + from triposg.pipelines.pipeline_triposg import TripoSGPipeline # noqa: E402 + + if weights_dir is None: + from huggingface_hub import snapshot_download + weights_dir = snapshot_download("VAST-AI/TripoSG") + log("ok", f"downloaded VAST-AI/TripoSG -> {weights_dir}") + + pipe = TripoSGPipeline.from_pretrained(weights_dir) + pipe = pipe.to("cpu", torch.float32) + for m in (pipe.image_encoder_dinov2, pipe.transformer, pipe.vae): + m.eval() + return pipe + + +def freeze_dinov2_pos_encoding(dinov2, image_size): + """DINOv2's interpolate_pos_encoding resizes its position table (built for + image_size=518, patch 14) to the runtime grid via F.interpolate(bicubic, + antialias) — the same ViT-at-non-native-resolution construct that broke the + TripoSR export trace. The ONNX input size is FIXED (224), so the + interpolated table is a constant: precompute it once in eager mode and + monkeypatch interpolate_pos_encoding to return the frozen tensor. + + # VERIFY: transformers' Dinov2Embeddings.interpolate_pos_encoding(embeddings, + # height, width) only reads embeddings.shape — validated against + # transformers 4.45.x; if a future version changes the signature the + # try/except below falls through to a plain trace (which may or may not + # export depending on the interpolate call).""" + try: + emb = dinov2.embeddings + num_patches = (image_size // dinov2.config.patch_size) ** 2 + with torch.no_grad(): + dummy = torch.zeros(1, 1 + num_patches, dinov2.config.hidden_size) + frozen = emb.interpolate_pos_encoding(dummy, image_size, image_size).detach() + emb.interpolate_pos_encoding = lambda embeddings, height, width: frozen + log("ok", f"froze DINOv2 pos-encoding at {image_size}px " + f"(table {tuple(frozen.shape)})") + except Exception as e: # noqa: BLE001 + log("warn", f"could not freeze DINOv2 pos-encoding ({e}); " + f"attempting plain trace") + + +# --------------------------------------------------------------------------- +# Export wrappers +# --------------------------------------------------------------------------- +class ImageEncoderWrapper(nn.Module): + """image [B,3,224,224] RGB in [0,1] -> image_embeds [B,257,1024]. + + Reproduces pipeline.encode_image minus the PIL BitImageProcessor: the C++ + side composites the U²-Net-matted subject over WHITE, crops to the + foreground bbox (+10% padding, centered square-ish — upstream + scripts/image_process.py prepare_image), resizes shortest-edge-256 + (bicubic) + center-crops 224, and feeds /255 RGB. The ImageNet mean/std + normalization (BitImageProcessor do_normalize) is baked in HERE so the C++ + preprocessing stays a plain [0,1] resize like TripoSR's.""" + + def __init__(self, dinov2): + super().__init__() + self.dinov2 = dinov2 + self.register_buffer("mean", torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1)) + self.register_buffer("std", torch.tensor(IMAGENET_STD).view(1, 3, 1, 1)) + + def forward(self, image): + x = (image - self.mean) / self.std + # pipeline_triposg.encode_image: image_encoder_dinov2(image).last_hidden_state + return self.dinov2(x).last_hidden_state + + +class DiTStepWrapper(nn.Module): + """One denoising step: (latents, timestep, image_embeds) -> velocity. + + Mirrors the pipeline's transformer call exactly: + noise_pred = self.transformer(latent_model_input, timestep, + encoder_hidden_states=image_embeds, + return_dict=False)[0] + The C++ side owns the loop, the CFG combine and the Euler update (see the + module docstring / docs/TRIPOSG_EXPORT_NOTES.md). Batch axis is dynamic so + the caller can run the doubled [uncond; cond] batch in one call (upstream + behavior) or two B=1 calls — mathematically identical since the uncond + embeddings are all-zeros (encode_image: torch.zeros_like(image_embeds)).""" + + def __init__(self, transformer): + super().__init__() + self.transformer = transformer + + def forward(self, latents, timestep, image_embeds): + return self.transformer( + latents, timestep, + encoder_hidden_states=image_embeds, + return_dict=False, + )[0] + + +class VaeLatentsWrapper(nn.Module): + """latents [1,2048,64] -> kv_cache [1,2048,1024]. Run ONCE per generation. + + Reproduces the first-chunk half of TripoSGVAEModel._decode: post_quant(z), + then TripoSGDecoder processes the latent set through its self-attention + blocks and returns the cache (decoder.forward returns (logits, kv_cache); + the kv_cache is the processed latent features the final cross-attention + block attends to). We obtain it by running decoder.forward with a single + throwaway query point. + + # VERIFY: TripoSGDecoder.forward(sample, queries, kv_cache) type-hints + # kv_cache as Optional[torch.Tensor] (a single tensor, not a per-layer + # list) and computes it as blocks[:-1] applied to `sample` when None — + # confirmed from the upstream source read (docs/TRIPOSG_EXPORT_NOTES.md + # §VAE). If the returned cache is NOT a plain [1,2048,1024] tensor, the + # printed contract below will show it and the C++ contract must follow.""" + + def __init__(self, vae): + super().__init__() + self.vae = vae + + def forward(self, latents): + z = self.vae.post_quant(latents) + dummy_q = self.vae.embedder(torch.zeros(1, 1, 3, dtype=latents.dtype)) + _, kv_cache = self.vae.decoder(z, dummy_q, None) + return kv_cache + + +class VaeQueryWrapper(nn.Module): + """(kv_cache [1,2048,1024], points [1,P,3]) -> sdf [1,P,1]. Chunked by C++. + + Reproduces the per-chunk half of _decode: frequency-embed the query points + (FrequencyPositionalEmbedding num_freqs=8, logspace, include_pi=False: + 3 -> 3+3·2·8 = 51 channels), then decoder.forward with the precomputed + kv_cache (which skips the latent self-attention stack). decoder.forward + already negates its raw logits (`logits * -1`), so this output equals + vae.decode(...).sample: INSIDE-POSITIVE, surface at iso 0. + + # VERIFY: when kv_cache is not None, decoder.forward must not read + # `sample` (the upstream first-chunk/kv_cache dance implies it, and we + # pass the kv_cache tensor itself as `sample` so any residual read sees + # sane shapes). The --verify mode cross-checks this graph against + # vae.decode(latents, sampled_points=pts).sample — a mismatch there means + # this assumption broke.""" + + def __init__(self, vae): + super().__init__() + self.vae = vae + + def forward(self, kv_cache, points): + queries = self.vae.embedder(points) + logits, _ = self.vae.decoder(kv_cache, queries, kv_cache) + return logits + + +class VaeDecodeMonolithic(nn.Module): + """(latents [1,2048,64], points [1,P,3]) -> sdf [1,P,1] — the unsplit + reference graph (recomputes the latent stack every call; correctness + baseline for --verify and a fallback if the kv_cache split misbehaves). + + num_chunks is set beyond any realistic P so _decode's python chunking loop + runs exactly ONE iteration during tracing — the traced slice + xyz_samples[:, 0:huge] stays valid for any dynamic P at runtime.""" + + def __init__(self, vae): + super().__init__() + self.vae = vae + + def forward(self, latents, points): + return self.vae._decode(latents, points, + num_chunks=1 << 40, return_dict=False)[0] + + +# --------------------------------------------------------------------------- +# Export helpers +# --------------------------------------------------------------------------- +def export_onnx(module, args_tuple, path, input_names, output_names, + dynamic_axes, opset): + """torch.onnx.export (legacy TorchScript exporter, dynamo=False — the repo + convention; no onnxscript dependency) with >2GB external-data handling: + export into a scratch dir (torch scatters per-tensor external files for + >2GB graphs), then consolidate into .onnx + single .onnx.data.""" + import onnx + + out_dir = os.path.dirname(os.path.abspath(path)) + base = os.path.basename(path) + scratch = tempfile.mkdtemp(prefix="triposg_export_", dir=out_dir) + scratch_path = os.path.join(scratch, base) + try: + torch.onnx.export( + module, args_tuple, scratch_path, + input_names=input_names, output_names=output_names, + dynamic_axes=dynamic_axes, + opset_version=opset, do_constant_folding=True, + dynamo=False, + ) + # Consolidate: single .onnx (+ single .onnx.data only when >2GB). + model = onnx.load(scratch_path, load_external_data=True) + total = sum(os.path.getsize(os.path.join(scratch, f)) + for f in os.listdir(scratch)) + if total >= TWO_GB: + data_name = base + ".data" + onnx.save_model(model, path, save_as_external_data=True, + all_tensors_to_one_file=True, location=data_name, + convert_attribute=False) + log("ok", f"wrote {path} (+ external data sidecar {data_name}, " + f"total ~{total / 1e9:.2f} GB) — host BOTH files together") + else: + onnx.save_model(model, path) + log("ok", f"wrote {path} (~{total / 1e9:.2f} GB)") + finally: + shutil.rmtree(scratch, ignore_errors=True) + + +def print_scheduler_contract(num_steps): + """Print the exact rectified-flow schedule the C++ loop must implement + (RectifiedFlowScheduler with the released config: num_train_timesteps=1000, + shift=1, use_dynamic_shifting=False).""" + shift = 1.0 + sig = np.array([(1.0 - i / num_steps) for i in range(num_steps)]) + sig = shift * sig / (1 + (shift - 1) * sig) # identity at shift=1 + sigmas = np.concatenate([sig, [0.0]]) + timesteps = sig * 1000.0 + log("contract", f"scheduler: N={num_steps} shift={shift}") + log("contract", f" sigmas[0..3]={np.round(sigmas[:4], 4).tolist()} ... " + f"sigmas[-3:]={np.round(sigmas[-3:], 4).tolist()}") + log("contract", f" timesteps[0..3]={np.round(timesteps[:4], 1).tolist()} ... " + f"timesteps[-2:]={np.round(timesteps[-2:], 1).tolist()}") + log("contract", " init: latents = N(0,1) [1,2048,64]") + log("contract", " step: latents += (sigma[i] - sigma[i+1]) * " + "(v_uncond + guidance * (v_cond - v_uncond))") + log("contract", f" defaults: guidance={DEFAULT_GUIDANCE}, " + f"CFG batch order = [uncond(zeros embeds); cond]") + return sigmas, timesteps + + +# --------------------------------------------------------------------------- +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + ap.add_argument("--triposg", required=True, + help="path to a VAST-AI-Research/TripoSG git checkout") + ap.add_argument("--weights", default=None, + help="local VAST-AI/TripoSG weights dir (default: " + "snapshot_download from HF)") + ap.add_argument("--out", default="./out", help="output dir for the .onnx files") + ap.add_argument("--opset", type=int, default=18) + ap.add_argument("--steps", type=int, default=DEFAULT_STEPS, + help="inference steps to print the reference sigma schedule for") + ap.add_argument("--verify", action="store_true", + help="run onnxruntime round-trip checks against torch") + ap.add_argument("--no-quant", action="store_true", + help="skip the int8 DiT variant (export fp32 only)") + ap.add_argument("--monolithic", action="store_true", + help="also export the unsplit (latents, points)->sdf VAE graph") + ap.add_argument("--skip-image-encoder", action="store_true", + help="skip the DINOv2 graph (iterate on DiT/VAE only)") + args = ap.parse_args() + + os.makedirs(args.out, exist_ok=True) + torch.manual_seed(0) + + pipe = load_pipeline(args.triposg, args.weights) + tcfg = pipe.transformer.config + vcfg = pipe.vae.config + log("contract", f"transformer: in_channels={tcfg.in_channels} " + f"width={tcfg.width} layers={tcfg.num_layers} " + f"cross_attention_dim={tcfg.cross_attention_dim}") + log("contract", f"vae: latent_channels={vcfg.latent_channels} " + f"width_decoder={vcfg.width_decoder} " + f"num_layers_decoder={vcfg.num_layers_decoder} " + f"embed_frequency={vcfg.embed_frequency}") + assert tcfg.in_channels == LATENT_CHANNELS + assert vcfg.latent_channels == LATENT_CHANNELS + + print_scheduler_contract(args.steps) + log("contract", f"decode bounds=(-{DECODE_BOUNDS}..+{DECODE_BOUNDS})^3, " + f"dense grid depth 8 (~257^3), sdf INSIDE-POSITIVE, MC iso 0.0") + + # ---- (a) image encoder --------------------------------------------------- + enc_path = os.path.join(args.out, "triposg_image_encoder.onnx") + dummy_img = torch.rand(1, 3, IMAGE_SIZE, IMAGE_SIZE, dtype=torch.float32) + if not args.skip_image_encoder: + freeze_dinov2_pos_encoding(pipe.image_encoder_dinov2, IMAGE_SIZE) + enc = ImageEncoderWrapper(pipe.image_encoder_dinov2).eval() + with torch.no_grad(): + embeds_ref = enc(dummy_img) + log("contract", f"image_embeds shape={tuple(embeds_ref.shape)} " + f"(expect [1,{COND_TOKENS},{COND_DIM}])") + export_onnx(enc, (dummy_img,), enc_path, + ["image"], ["image_embeds"], + {"image": {0: "B"}, "image_embeds": {0: "B"}}, + args.opset) + else: + with torch.no_grad(): + embeds_ref = ImageEncoderWrapper(pipe.image_encoder_dinov2)(dummy_img) + + # ---- (b) DiT denoising step ---------------------------------------------- + dit = DiTStepWrapper(pipe.transformer).eval() + B = 2 # trace with the CFG-doubled batch (upstream shape) + dummy_lat = torch.randn(B, NUM_TOKENS, LATENT_CHANNELS, dtype=torch.float32) + dummy_t = torch.full((B,), 1000.0, dtype=torch.float32) + dummy_cond = torch.cat([torch.zeros_like(embeds_ref), embeds_ref], dim=0) + with torch.no_grad(): + v_ref = dit(dummy_lat, dummy_t, dummy_cond) + log("contract", f"velocity shape={tuple(v_ref.shape)} " + f"(expect [{B},{NUM_TOKENS},{LATENT_CHANNELS}])") + + dit_path = os.path.join(args.out, "triposg_dit_step.onnx") + export_onnx(dit, (dummy_lat, dummy_t, dummy_cond), dit_path, + ["latents", "timestep", "image_embeds"], ["velocity"], + {"latents": {0: "B"}, "timestep": {0: "B"}, + "image_embeds": {0: "B"}, "velocity": {0: "B"}}, + args.opset) + + # ---- int8 tier of the heaviest graph (the ~5.7 GB DiT) -------------------- + # MatMul-only dynamic QInt8 (repo precedent from export-triposr-onnx.py: + # quantizing Conv would emit ConvInteger which our ORT CPU EP lacks; the + # DiT is MatMul-dominated so it still shrinks ~4x to a single-file model). + if not args.no_quant: + try: + from onnxruntime.quantization import quantize_dynamic, QuantType + int8_path = os.path.join(args.out, "triposg_dit_step_int8.onnx") + # per_channel is ESSENTIAL here: per-tensor dynamic quant on this + # DiT compounds over the flow loop (2 CFG calls/step, guidance 7x) + # and the decoded field degenerates into disconnected noise blobs. + # Verified live: fp32 → coherent figure; per-tensor int8 → noise. + quantize_dynamic(dit_path, int8_path, weight_type=QuantType.QInt8, + op_types_to_quantize=["MatMul"], + per_channel=True, reduce_range=True, + use_external_data_format=False) + log("ok", f"wrote {int8_path}") + except Exception as e: # noqa: BLE001 — best-effort; fp32 still ships + log("warn", f"int8 DiT export skipped: {e}") + log("warn", " (if the failure is the >2GB input, retry with " + "onnxruntime>=1.17 which reads external-data models)") + + # ---- (c) VAE latent processing (run once) --------------------------------- + vae_lat = VaeLatentsWrapper(pipe.vae).eval() + lat1 = torch.randn(1, NUM_TOKENS, LATENT_CHANNELS, dtype=torch.float32) + with torch.no_grad(): + kv_ref = vae_lat(lat1) + log("contract", f"kv_cache shape={tuple(kv_ref.shape)} " + f"(expect [1,{NUM_TOKENS},{VAE_WIDTH_DECODER}])") + vae_lat_path = os.path.join(args.out, "triposg_vae_latents.onnx") + export_onnx(vae_lat, (lat1,), vae_lat_path, + ["latents"], ["kv_cache"], {}, + args.opset) + + # ---- (d) VAE point query (chunked) ---------------------------------------- + vae_q = VaeQueryWrapper(pipe.vae).eval() + P = 4096 + dummy_pts = (torch.rand(1, P, 3, dtype=torch.float32) * 2 - 1) * DECODE_BOUNDS + with torch.no_grad(): + sdf_ref = vae_q(kv_ref, dummy_pts) + log("contract", f"sdf shape={tuple(sdf_ref.shape)} (expect [1,{P},1]) " + f"range=({sdf_ref.min():.3f},{sdf_ref.max():.3f})") + vae_q_path = os.path.join(args.out, "triposg_vae_decoder.onnx") + export_onnx(vae_q, (kv_ref, dummy_pts), vae_q_path, + ["kv_cache", "points"], ["sdf"], + {"points": {1: "P"}, "sdf": {1: "P"}}, + args.opset) + + # split-vs-upstream correctness gate (torch-level, cheap, always on): + with torch.no_grad(): + sdf_upstream = pipe.vae.decode(lat1, sampled_points=dummy_pts).sample + split_ok = torch.allclose(sdf_ref, sdf_upstream, atol=1e-4) + log("verify" if split_ok else "warn", + f"kv-split vs vae.decode(): match={split_ok} " + f"max|diff|={float((sdf_ref - sdf_upstream).abs().max()):.3e}") + split_graphs_available = True + if not split_ok: + log("error", "kv-cache split does NOT reproduce upstream decode — the " + "split VAE graphs are WRONG and must not be shipped.") + # Quarantine the mismatched split graphs so a partial/wrong export + # can't be uploaded, then abort — unless --monolithic was requested + # (that path ships the unsplit reference graph instead). + for f in ("triposg_vae_latents.onnx", "triposg_vae_decoder.onnx"): + p = os.path.join(args.out, f) + if os.path.exists(p): + os.remove(p) + log("error", f" removed mismatched {f}") + split_graphs_available = False + if not args.monolithic: + log("error", " re-run with --monolithic to ship the unsplit graph, " + "or fix the split. Aborting.") + sys.exit(1) + log("warn", " --monolithic set: shipping the unsplit graph below.") + + # ---- optional monolithic reference graph ---------------------------------- + if args.monolithic: + mono = VaeDecodeMonolithic(pipe.vae).eval() + mono_path = os.path.join(args.out, "triposg_vae_decode_mono.onnx") + export_onnx(mono, (lat1, dummy_pts), mono_path, + ["latents", "points"], ["sdf"], + {"points": {1: "P"}, "sdf": {1: "P"}}, + args.opset) + + # ---- ORT verification ------------------------------------------------------ + if args.verify: + import onnxruntime as ort + + def sess(p): + return ort.InferenceSession(p, providers=["CPUExecutionProvider"]) + + if not args.skip_image_encoder: + s = sess(enc_path) + e = s.run(None, {"image": dummy_img.numpy()})[0] + rel = np.abs(e - embeds_ref.numpy()).max() / (np.abs(embeds_ref.numpy()).max() + 1e-9) + log("verify", f"ORT image_encoder {e.shape} max-rel-err={rel:.2e}") + + s = sess(dit_path) + v = s.run(None, {"latents": dummy_lat.numpy(), + "timestep": dummy_t.numpy(), + "image_embeds": dummy_cond.numpy()})[0] + rel = np.abs(v - v_ref.numpy()).max() / (np.abs(v_ref.numpy()).max() + 1e-9) + log("verify", f"ORT dit_step {v.shape} max-rel-err={rel:.2e}") + # dynamic-batch check (B=1 must also run — the two-call CFG variant) + v1 = s.run(None, {"latents": dummy_lat[:1].numpy(), + "timestep": dummy_t[:1].numpy(), + "image_embeds": dummy_cond[:1].numpy()})[0] + log("verify", f"ORT dit_step B=1 {v1.shape} " + f"match-B2-row0={np.allclose(v1, v[:1], atol=1e-3)}") + + # The split VAE graphs only exist when the kv-split matched upstream + # (else they were quarantined above). Skip their ORT checks when they + # were removed; the monolithic graph is verified against sdf_upstream + # below in that case. + d = None + if split_graphs_available: + s = sess(vae_lat_path) + kv = s.run(None, {"latents": lat1.numpy()})[0] + rel = np.abs(kv - kv_ref.numpy()).max() / (np.abs(kv_ref.numpy()).max() + 1e-9) + log("verify", f"ORT vae_latents {kv.shape} max-rel-err={rel:.2e}") + + s = sess(vae_q_path) + d = s.run(None, {"kv_cache": kv, "points": dummy_pts.numpy()})[0] + rel = np.abs(d - sdf_ref.numpy()).max() / (np.abs(sdf_ref.numpy()).max() + 1e-9) + log("verify", f"ORT vae_decoder {d.shape} max-rel-err={rel:.2e}") + # dynamic-P check with an odd chunk size + d2 = s.run(None, {"kv_cache": kv, + "points": dummy_pts[:, :777].numpy()})[0] + log("verify", f"ORT vae_decoder P=777 {d2.shape} " + f"match={np.allclose(d2, d[:, :777], atol=1e-3)}") + else: + log("warn", "ORT vae split checks skipped (graphs quarantined).") + + if not args.no_quant and os.path.exists( + os.path.join(args.out, "triposg_dit_step_int8.onnx")): + s = sess(os.path.join(args.out, "triposg_dit_step_int8.onnx")) + vq = s.run(None, {"latents": dummy_lat.numpy(), + "timestep": dummy_t.numpy(), + "image_embeds": dummy_cond.numpy()})[0] + rel = np.abs(vq - v_ref.numpy()).max() / (np.abs(v_ref.numpy()).max() + 1e-9) + log("verify", f"ORT dit_step_int8 {vq.shape} max-rel-err={rel:.2e} " + f"(int8 — expect noticeably larger than fp32)") + + if args.monolithic: + s = sess(os.path.join(args.out, "triposg_vae_decode_mono.onnx")) + dm = s.run(None, {"latents": lat1.numpy(), + "points": dummy_pts.numpy()})[0] + # Compare against the split output when it exists, else against the + # torch upstream reference (the split graphs were quarantined). + ref = d if d is not None else sdf_upstream.numpy() + label = "match-split" if d is not None else "match-upstream" + log("verify", f"ORT vae_decode_mono {dm.shape} " + f"{label}={np.allclose(dm, ref, atol=1e-3)}") + + log("done", "export complete. Host the .onnx files (and the DiT " + ".onnx.data sidecar) under triposg/ on the HF models repo.") + + +if __name__ == "__main__": + main() diff --git a/scripts/run-triposg-export.sh b/scripts/run-triposg-export.sh new file mode 100644 index 00000000..1f137a9c --- /dev/null +++ b/scripts/run-triposg-export.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# One-shot driver for the TripoSG ONNX export (see scripts/export-triposg-onnx.py). +# Creates a DEDICATED venv (do NOT reuse the TripoSR export venv — its +# transformers 4.35 pin conflicts with TripoSG's >=4.45 requirement, and a +# shared install drags huggingface_hub past <1.0 and breaks both), clones the +# upstream repo, runs the export with ORT verification, and leaves the graphs +# in /tmp/triposg_export/dist/triposg_onnx ready for HF upload. +# +# Usage: bash scripts/run-triposg-export.sh +# Runtime: ~8 GB HF download + CPU export — expect 1-2 hours. +set -euo pipefail + +WORK=/tmp/triposg_export +VENV="$WORK/venv" +REPO_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/export-triposg-onnx.py" + +mkdir -p "$WORK" +cd "$WORK" + +if [ ! -x "$VENV/bin/python" ]; then + echo "[venv] creating $VENV" + python3 -m venv "$VENV" +fi + +echo "[deps] installing pinned export stack" +"$VENV/bin/pip" install -q --upgrade pip +"$VENV/bin/pip" install -q torch --index-url https://download.pytorch.org/whl/cpu +# huggingface_hub pinned <1.0: transformers 4.4x and diffusers 0.30.x both +# require it; unpinned installs pull 1.x and break transformers' version gate. +# The tail (trimesh…tqdm) are triposg-package IMPORT-time deps; the CUDA-only +# `diso` is deliberately absent — the export script stubs it. +"$VENV/bin/pip" install -q \ + "diffusers==0.30.3" \ + "transformers==4.46.3" \ + "huggingface_hub==0.26.5" \ + "tokenizers>=0.20,<0.21" \ + onnx onnxruntime numpy einops jaxtyping safetensors accelerate \ + trimesh scipy scikit-image omegaconf typeguard tqdm pillow + +if [ ! -d TripoSG ]; then + echo "[clone] VAST-AI-Research/TripoSG" + git clone --depth 1 https://github.com/VAST-AI-Research/TripoSG +fi + +# HF's xet CDN read-times-out on the multi-GB safetensors from this network. +# Use the classic CDN, a generous read timeout, and a resume-retry loop to +# fetch the snapshot BEFORE the export (downloads resume across attempts), so +# the export itself runs exactly once against the local weights. +export HF_HUB_DISABLE_XET=1 +export HF_HUB_DOWNLOAD_TIMEOUT=120 +echo "[weights] downloading VAST-AI/TripoSG (resumes across retries)" +ok="" +for attempt in 1 2 3 4 5 6 7 8; do + if "$VENV/bin/huggingface-cli" download VAST-AI/TripoSG > /dev/null 2>>"$WORK/download.log"; then + ok=1; break + fi + echo "[weights] attempt $attempt failed — retrying in 20s (see $WORK/download.log)" + sleep 20 +done +[ -n "$ok" ] || { echo "[fatal] weights download kept failing"; exit 1; } +WEIGHTS="$("$VENV/bin/python" - <<'PY' +from huggingface_hub import snapshot_download +print(snapshot_download("VAST-AI/TripoSG", local_files_only=True)) +PY +)" +echo "[weights] snapshot at $WEIGHTS" + +echo "[export] running (log: $WORK/export.log)" +"$VENV/bin/python" "$REPO_SCRIPT" --triposg ./TripoSG --weights "$WEIGHTS" \ + --out dist/triposg_onnx --verify 2>&1 | tee "$WORK/export.log" + +echo "[done] artifacts:" +ls -lh dist/triposg_onnx +echo +echo "Next: upload to fernandotonon/QtMeshEditor-models under triposg/" +echo " (mirror scripts/upload-triposr-models.sh — includes the .onnx.data sidecar)" diff --git a/scripts/upload-caption-models.sh b/scripts/upload-caption-models.sh new file mode 100644 index 00000000..fe883a3f --- /dev/null +++ b/scripts/upload-caption-models.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Upload the SmolVLM image-captioning GGUFs to the QtMeshEditor HF models repo +# (image-to-3D "describe-then-generate" texture prompt, #764). ONE-TIME, run by +# a maintainer with write access. +# +# The app downloads these on first use from +# https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/caption/... +# so the file names below MUST match ImageCaptioner's kModelFile / kMmprojFile. +# +# Source: ggml-org/SmolVLM-500M-Instruct-GGUF (Apache-2.0). Mirror both files +# (the model GGUF + its mmproj vision projector) into caption/ on our repo. +# +# Prereqs: +# huggingface-cli login # token with write access to the models repo +# the two files downloaded to $SRC_DIR (see the download step below) +# +# Usage: +# SRC_DIR=/tmp/smolvlm_dl ./scripts/upload-caption-models.sh +set -euo pipefail + +REPO="${REPO:-fernandotonon/QtMeshEditor-models}" +SRC_DIR="${SRC_DIR:-/tmp/smolvlm_dl}" +HFCLI="${HFCLI:-huggingface-cli}" +command -v "$HFCLI" >/dev/null 2>&1 || HFCLI=/tmp/triposg_export/venv/bin/huggingface-cli + +# One-time source fetch (idempotent — skips files already present): +# HF_HUB_DISABLE_XET=1 "$HFCLI" download ggml-org/SmolVLM-500M-Instruct-GGUF \ +# SmolVLM-500M-Instruct-Q8_0.gguf mmproj-SmolVLM-500M-Instruct-Q8_0.gguf \ +# --local-dir "$SRC_DIR" + +upload() { # + local f="$1" + if [ -f "$SRC_DIR/$f" ]; then + echo ">> uploading $f -> $REPO:caption/$f" + "$HFCLI" upload "$REPO" "$SRC_DIR/$f" "caption/$f" + else + echo "!! skip (missing): $SRC_DIR/$f" + fi +} + +upload "SmolVLM-500M-Instruct-Q8_0.gguf" +upload "mmproj-SmolVLM-500M-Instruct-Q8_0.gguf" + +echo "done. Verify: curl -sI https://huggingface.co/$REPO/resolve/main/caption/mmproj-SmolVLM-500M-Instruct-Q8_0.gguf | head -1" diff --git a/scripts/upload-triposg-models.sh b/scripts/upload-triposg-models.sh new file mode 100644 index 00000000..8fcae947 --- /dev/null +++ b/scripts/upload-triposg-models.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Upload the TripoSG ONNX graphs to the QtMeshEditor HF models repo (image-to-3D +# backend #2). ONE-TIME, run by a maintainer with write access. +# +# The app downloads these on first use from +# https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/triposg/... +# so the file names below MUST match TripoSGPredictor's kImageEncoderFile / +# kDitStepFile (+ .onnx.data sidecar!) / kDitStepInt8File / kVaeLatentsFile / +# kVaeDecoderFile. +# +# Prereqs: +# pip install -U "huggingface_hub[cli]" (or reuse /tmp/triposg_export/venv) +# huggingface-cli login # token with write access +# scripts/run-triposg-export.sh already run → OUT_DIR holds the graphs +# +# Usage: +# OUT_DIR=/tmp/triposg_export/dist/triposg_onnx ./scripts/upload-triposg-models.sh +set -euo pipefail + +REPO="${REPO:-fernandotonon/QtMeshEditor-models}" +OUT_DIR="${OUT_DIR:-/tmp/triposg_export/dist/triposg_onnx}" +HFCLI="${HFCLI:-huggingface-cli}" +command -v "$HFCLI" >/dev/null 2>&1 || HFCLI=/tmp/triposg_export/venv/bin/huggingface-cli + +upload() { # + local src="$1" dst="$2" + if [ -f "$src" ]; then + echo ">> uploading $src -> $REPO:$dst" + "$HFCLI" upload "$REPO" "$src" "$dst" + else + echo "!! skip (missing): $src" + fi +} + +# The fp32 DiT is a graph + external-weights sidecar: BOTH must be hosted +# side-by-side or ONNX Runtime cannot load the graph. +upload "$OUT_DIR/triposg_image_encoder.onnx" "triposg/triposg_image_encoder.onnx" +upload "$OUT_DIR/triposg_dit_step.onnx" "triposg/triposg_dit_step.onnx" +upload "$OUT_DIR/triposg_dit_step.onnx.data" "triposg/triposg_dit_step.onnx.data" +upload "$OUT_DIR/triposg_dit_step_int8.onnx" "triposg/triposg_dit_step_int8.onnx" +upload "$OUT_DIR/triposg_vae_latents.onnx" "triposg/triposg_vae_latents.onnx" +upload "$OUT_DIR/triposg_vae_decoder.onnx" "triposg/triposg_vae_decoder.onnx" + +echo "done. Verify: curl -sI https://huggingface.co/$REPO/resolve/main/triposg/triposg_vae_decoder.onnx | head -1" diff --git a/src/Assimp/MeshProcessor.cpp b/src/Assimp/MeshProcessor.cpp index 1493b6dd..a3427750 100644 --- a/src/Assimp/MeshProcessor.cpp +++ b/src/Assimp/MeshProcessor.cpp @@ -225,13 +225,30 @@ Ogre::MeshPtr MeshProcessor::createMesh(const Ogre::String& name, const Ogre::St // Set the index count indexData->indexCount = subMeshData->indices.size(); - // Create the index buffer and set the index data - Ogre::HardwareIndexBufferSharedPtr ibuf = Ogre::HardwareBufferManager::getSingleton().createIndexBuffer(Ogre::HardwareIndexBuffer::IT_16BIT, indexData->indexCount, Ogre::HardwareBuffer::HBU_STATIC_WRITE_ONLY); - unsigned short* pIndices = static_cast(ibuf->lock(Ogre::HardwareBuffer::HBL_DISCARD)); - - // Set the indices - for(size_t i = 0; i < subMeshData->indices.size(); i++) { - *pIndices++ = subMeshData->indices[i]; + // Create the index buffer. Use 32-bit indices when the submesh has + // more than 65535 vertices — a 16-bit buffer wraps every index past + // 0xFFFF, which collapsed large meshes (e.g. high-res image-to-3D + // output, 80k+ verts) onto their first 65536 vertices and tore them + // apart on import. Match on the VERTEX count (what indices address), + // not the index count. + const bool use32BitIndices = subMeshData->vertices.size() > 65535; + Ogre::HardwareIndexBufferSharedPtr ibuf = + Ogre::HardwareBufferManager::getSingleton().createIndexBuffer( + use32BitIndices ? Ogre::HardwareIndexBuffer::IT_32BIT + : Ogre::HardwareIndexBuffer::IT_16BIT, + indexData->indexCount, Ogre::HardwareBuffer::HBU_STATIC_WRITE_ONLY); + + // Set the indices, writing the matching element width. + if (use32BitIndices) { + uint32_t* pIndices = static_cast( + ibuf->lock(Ogre::HardwareBuffer::HBL_DISCARD)); + for (size_t i = 0; i < subMeshData->indices.size(); i++) + *pIndices++ = subMeshData->indices[i]; + } else { + unsigned short* pIndices = static_cast( + ibuf->lock(Ogre::HardwareBuffer::HBL_DISCARD)); + for (size_t i = 0; i < subMeshData->indices.size(); i++) + *pIndices++ = static_cast(subMeshData->indices[i]); } ibuf->unlock(); diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index f5ce0320..11e67eb3 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -39,6 +39,7 @@ #include "SkinWeights.h" #include "AutoRig.h" #include "ImageTo3D/MeshGenPredictor.h" +#include "ImageTo3D/TripoSGPredictor.h" #include "ImageTo3D/MeshGenBuilder.h" #include "MeshSegmenter.h" #include "MeshDecimator.h" @@ -9068,7 +9069,10 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) bool upscaleTex = false; // optional Real-ESRGAN 2x on the baked texture bool generatePbr = true; // #404 normal+roughness synthesis on the baked diffuse int textureSize = 1024; + int flowSteps = 25; // TripoSG rectified-flow steps + float guidance = 7.0f; // TripoSG CFG scale (0 disables CFG) MeshGenPredictor::Quality quality = MeshGenPredictor::Quality::Fp32; + MeshGenPredictor::Backend backend = MeshGenPredictor::Backend::TripoSR; for (int i = 1; i < argc; ++i) { const QString arg = QString::fromLocal8Bit(argv[i]); @@ -9081,6 +9085,43 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) if (arg == "--no-bake-texture") { bake = false; continue; } if (arg == "--upscale-texture") { upscaleTex = true; continue; } if (arg == "--no-pbr") { generatePbr = false; continue; } + if (arg == "--backend") { + if (i + 1 >= argc) { + err() << "Error: --backend requires triposr or triposg." << Qt::endl; + return 2; + } + const QString b = QString::fromLocal8Bit(argv[++i]).toLower(); + if (b == "triposr") backend = MeshGenPredictor::Backend::TripoSR; + else if (b == "triposg") backend = MeshGenPredictor::Backend::TripoSG; + else { err() << "Error: --backend must be triposr or triposg." << Qt::endl; return 2; } + continue; + } + if (arg == "--flow-steps") { + if (i + 1 >= argc) { + err() << "Error: --flow-steps requires a value (e.g. 10, 25, 50)." << Qt::endl; + return 2; + } + bool okNum = false; + flowSteps = QString::fromLocal8Bit(argv[++i]).toInt(&okNum); + if (!okNum || flowSteps < 1 || flowSteps > 200) { + err() << "Error: --flow-steps must be an integer in [1..200]." << Qt::endl; + return 2; + } + continue; + } + if (arg == "--guidance") { + if (i + 1 >= argc) { + err() << "Error: --guidance requires a value (e.g. 7.0; 0 disables CFG)." << Qt::endl; + return 2; + } + bool okNum = false; + guidance = QString::fromLocal8Bit(argv[++i]).toFloat(&okNum); + if (!okNum || guidance < 0.0f || guidance > 30.0f) { + err() << "Error: --guidance must be in [0..30]." << Qt::endl; + return 2; + } + continue; + } if (arg == "--texture-size") { if (i + 1 >= argc) { err() << "Error: --texture-size requires a value (e.g. 512, 1024, 2048)." << Qt::endl; @@ -9140,7 +9181,8 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) err() << "Usage: qtmesh generate3d [-o out.glb] [--resolution 256] " "[--no-color] [--remove-bg] [--quality fp32|int8] " "[--no-smooth] [--no-refine] [--no-bake-texture] [--texture-size 1024] " - "[--upscale-texture] [--no-pbr]" + "[--upscale-texture] [--no-pbr] " + "[--backend triposr|triposg] [--flow-steps 25] [--guidance 7.0]" << Qt::endl; return 2; } @@ -9168,22 +9210,50 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) err() << "Error: failed to read image: " << inputPath << Qt::endl; return 1; } + const bool useSG = (backend == MeshGenPredictor::Backend::TripoSG); SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.image_to_3d"), - QString("generate3d .%1 res=%2 color=%3") - .arg(fi.suffix()).arg(resolution).arg(vertexColor)); - - // Download the model on first use (blocks; clear message when not hosted). - const QString enc = MeshGenPredictor::ensureModelBlocking(quality); - if (enc.isEmpty() || !MeshGenPredictor::modelsPresent(quality)) { - err() << " (looked for models in: " - << QFileInfo(MeshGenPredictor::encoderModelPath(quality)).absolutePath() - << ")" << Qt::endl; - err() << "Error: TripoSR model unavailable. It downloads on first use from " - "the QtMeshEditor models repo; if it is not hosted yet, export it " - "with scripts/export-triposr-onnx.py and point " - "QTMESH_TRIPOSR_MODEL_BASE_URL (or ai/triposrModelBaseUrl) at it, " - "or drop the files in the ai_models/triposr/ cache." << Qt::endl; - return 1; + QString("generate3d .%1 res=%2 color=%3 backend=%4") + .arg(fi.suffix()).arg(resolution).arg(vertexColor) + .arg(useSG ? QStringLiteral("triposg") : QStringLiteral("triposr"))); + + // Download the chosen backend's models on first use (blocks; clear + // message when not hosted). + if (useSG) { + if (quality == MeshGenPredictor::Quality::Int8) + err() << "Note: --quality int8 is not available for the triposg " + "backend (the quantized DiT degrades geometry); using fp32." + << Qt::endl; + const QString enc = TripoSGPredictor::ensureModelBlocking(false); + if (enc.isEmpty()) { + err() << " (looked for models in: " + << QFileInfo(TripoSGPredictor::imageEncoderPath()).absolutePath() + << ")" << Qt::endl; + err() << "Error: TripoSG models unavailable. They download on first use " + "from the QtMeshEditor models repo; if not hosted yet, export " + "them with scripts/export-triposg-onnx.py and point " + "QTMESH_TRIPOSG_MODEL_BASE_URL (or ai/triposgModelBaseUrl) at " + "them, or drop the files in the ai_models/triposg/ cache." + << Qt::endl; + return 1; + } + // TripoSG's colour bake queries TripoSR's image-conditioned colour + // field — ensure those models too (best-effort: absent models fall + // back to the untextured clay result with a warning). + if (bake) + MeshGenPredictor::ensureModelBlocking(quality); + } else { + const QString enc = MeshGenPredictor::ensureModelBlocking(quality); + if (enc.isEmpty() || !MeshGenPredictor::modelsPresent(quality)) { + err() << " (looked for models in: " + << QFileInfo(MeshGenPredictor::encoderModelPath(quality)).absolutePath() + << ")" << Qt::endl; + err() << "Error: TripoSR model unavailable. It downloads on first use from " + "the QtMeshEditor models repo; if it is not hosted yet, export it " + "with scripts/export-triposr-onnx.py and point " + "QTMESH_TRIPOSR_MODEL_BASE_URL (or ai/triposrModelBaseUrl) at it, " + "or drop the files in the ai_models/triposr/ cache." << Qt::endl; + return 1; + } } if (!initOgreHeadless()) return 1; @@ -9197,6 +9267,9 @@ int CLIPipeline::cmdGenerate3d(int argc, char* argv[]) opts.refineSurface = refine; opts.bakeTexture = bake; opts.textureSize = textureSize; + opts.backend = backend; + opts.flowSteps = flowSteps; + opts.guidanceScale = guidance; MeshGenPredictor::Result res = MeshGenPredictor::predict( image, MeshGenPredictor::encoderModelPath(quality), MeshGenPredictor::decoderModelPath(), opts); diff --git a/src/CLIPipeline_cmdgenerate3d_coverage_test.cpp b/src/CLIPipeline_cmdgenerate3d_coverage_test.cpp new file mode 100644 index 00000000..b0e646f6 --- /dev/null +++ b/src/CLIPipeline_cmdgenerate3d_coverage_test.cpp @@ -0,0 +1,174 @@ +#include + +#include +#include +#include + +#include + +#include "CLIPipeline.h" + +// Coverage tests for CLIPipeline::cmdGenerate3d (image-to-3D, epic #764) — +// specifically the argument parsing added for the TripoSG backend +// (--backend / --flow-steps / --guidance) plus the existing --quality/ +// --resolution validators. +// +// The whole argument-parse loop (every usage-error `return 2`) executes and +// returns BEFORE the input-file existence check, the ONNX-availability guard, +// and any Ogre init — so these branches are fully testable headlessly, and +// the results are independent of whether the binary was built with +// ENABLE_ONNX. A fully-valid argument set with a nonexistent input lands on a +// runtime-error `return 1` (either "image not found" or the "rebuild with +// -DENABLE_ONNX" message — both 1) before any Ogre work. +// +// Return-code contract: 0 = success, 1 = runtime error, 2 = usage error. +// +// Distinct suite name avoids ODR clashes with other CLIPipeline suites. + +namespace { + +int callGenerate3d(const QList& tokens) +{ + std::vector storage; + storage.reserve(tokens.size() + 1); + storage.emplace_back("qtmesh"); // argv[0], ignored by the parser loop + for (const QByteArray& t : tokens) + storage.push_back(t); + + std::vector argv; + argv.reserve(storage.size()); + for (QByteArray& b : storage) + argv.push_back(b.data()); + + return CLIPipeline::cmdGenerate3d(static_cast(argv.size()), argv.data()); +} + +QByteArray nonexistentInput() +{ + return QByteArray("/nonexistent/__qtmesh_cmdgen3d_cov__/no_such_image.png"); +} + +} // namespace + +// --------------------------------------------------------------------------- +// Required-argument checks. +// --------------------------------------------------------------------------- + +TEST(CLIPipelineCmdGenerate3dCoverageTest, NoInputImageReturnsUsageError) +{ + EXPECT_EQ(2, callGenerate3d({"generate3d"})); +} + +// --------------------------------------------------------------------------- +// --backend validation (triposr | triposg). +// --------------------------------------------------------------------------- + +TEST(CLIPipelineCmdGenerate3dCoverageTest, BackendInvalidValueRejected) +{ + EXPECT_EQ(2, callGenerate3d({"generate3d", nonexistentInput(), + "--backend", "foo", "-o", "out.glb"})); +} + +TEST(CLIPipelineCmdGenerate3dCoverageTest, BackendMissingValueRejected) +{ + // --backend as the last token: no value follows. + EXPECT_EQ(2, callGenerate3d({"generate3d", nonexistentInput(), "--backend"})); +} + +TEST(CLIPipelineCmdGenerate3dCoverageTest, BackendTriposrAcceptedThenFileNotFound) +{ + EXPECT_EQ(1, callGenerate3d({"generate3d", nonexistentInput(), + "--backend", "triposr", "-o", "out.glb"})); +} + +TEST(CLIPipelineCmdGenerate3dCoverageTest, BackendTriposgAcceptedThenFileNotFound) +{ + EXPECT_EQ(1, callGenerate3d({"generate3d", nonexistentInput(), + "--backend", "triposg", "-o", "out.glb"})); +} + +// --------------------------------------------------------------------------- +// --flow-steps validation (integer in [1..200]). +// --------------------------------------------------------------------------- + +TEST(CLIPipelineCmdGenerate3dCoverageTest, FlowStepsZeroRejected) +{ + EXPECT_EQ(2, callGenerate3d({"generate3d", nonexistentInput(), + "--flow-steps", "0", "-o", "out.glb"})); +} + +TEST(CLIPipelineCmdGenerate3dCoverageTest, FlowStepsAboveRangeRejected) +{ + EXPECT_EQ(2, callGenerate3d({"generate3d", nonexistentInput(), + "--flow-steps", "500", "-o", "out.glb"})); +} + +TEST(CLIPipelineCmdGenerate3dCoverageTest, FlowStepsNonIntegerRejected) +{ + EXPECT_EQ(2, callGenerate3d({"generate3d", nonexistentInput(), + "--flow-steps", "many", "-o", "out.glb"})); +} + +TEST(CLIPipelineCmdGenerate3dCoverageTest, FlowStepsBoundaryValuesAcceptedThenFileNotFound) +{ + EXPECT_EQ(1, callGenerate3d({"generate3d", nonexistentInput(), + "--backend", "triposg", "--flow-steps", "1", + "-o", "out.glb"})); + EXPECT_EQ(1, callGenerate3d({"generate3d", nonexistentInput(), + "--backend", "triposg", "--flow-steps", "200", + "-o", "out.glb"})); +} + +// --------------------------------------------------------------------------- +// --guidance validation (number in [0..30]). +// --------------------------------------------------------------------------- + +TEST(CLIPipelineCmdGenerate3dCoverageTest, GuidanceNegativeRejected) +{ + EXPECT_EQ(2, callGenerate3d({"generate3d", nonexistentInput(), + "--guidance", "-1", "-o", "out.glb"})); +} + +TEST(CLIPipelineCmdGenerate3dCoverageTest, GuidanceAboveRangeRejected) +{ + EXPECT_EQ(2, callGenerate3d({"generate3d", nonexistentInput(), + "--guidance", "31", "-o", "out.glb"})); +} + +TEST(CLIPipelineCmdGenerate3dCoverageTest, GuidanceNonNumericRejected) +{ + EXPECT_EQ(2, callGenerate3d({"generate3d", nonexistentInput(), + "--guidance", "strong", "-o", "out.glb"})); +} + +TEST(CLIPipelineCmdGenerate3dCoverageTest, GuidanceZeroAcceptedThenFileNotFound) +{ + // 0 is valid (disables CFG) and must be accepted before the file check. + EXPECT_EQ(1, callGenerate3d({"generate3d", nonexistentInput(), + "--backend", "triposg", "--guidance", "0", + "-o", "out.glb"})); +} + +// --------------------------------------------------------------------------- +// --quality validation (fp32 | int8) — existing arg, still parses. +// --------------------------------------------------------------------------- + +TEST(CLIPipelineCmdGenerate3dCoverageTest, QualityInvalidValueRejected) +{ + EXPECT_EQ(2, callGenerate3d({"generate3d", nonexistentInput(), + "--quality", "fp16", "-o", "out.glb"})); +} + +// --------------------------------------------------------------------------- +// Combined valid args + nonexistent file -> runtime error (1). +// --------------------------------------------------------------------------- + +TEST(CLIPipelineCmdGenerate3dCoverageTest, AllNewFlagsValidNonexistentFileReturnsRuntimeError) +{ + EXPECT_EQ(1, callGenerate3d({"generate3d", nonexistentInput(), + "--backend", "triposg", + "--flow-steps", "25", + "--guidance", "7", + "--resolution", "128", + "-o", "out.glb"})); +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9293d27b..b90ee824 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -149,9 +149,11 @@ ImageTo3D/MarchingCubes.cpp ImageTo3D/MeshRefine.cpp ImageTo3D/MeshGenBaker.cpp ImageTo3D/MeshGenPredictor.cpp +ImageTo3D/TripoSGPredictor.cpp ImageTo3D/MeshGenBuilder.cpp ImageTo3D/MeshGenController.cpp ImageTo3D/BackgroundRemover.cpp +ImageTo3D/ImageCaptioner.cpp PbrMapSynth.cpp TextureUpscaler.cpp AIAssistManager.cpp @@ -659,9 +661,20 @@ elseif(WIN32) target_link_libraries(${CMAKE_PROJECT_NAME} advapi32) endif() +# Debug macro, scoped to OUR target only (never the FetchContent deps — see +# the QTMESH_DEFINE_DEBUG note in the top-level CMakeLists). +if(QTMESH_DEFINE_DEBUG) + target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE DEBUG) +endif() + # Link llama.cpp if enabled if(ENABLE_LOCAL_LLM) target_link_libraries(${CMAKE_PROJECT_NAME} llama ggml) + # mtmd = multimodal (image captioning via SmolVLM, #764). Not on Win-MinGW + # (its llama-tools CLIs don't link there); ImageCaptioner degrades gracefully. + if(QTMESH_HAVE_MTMD) + target_link_libraries(${CMAKE_PROJECT_NAME} mtmd) + endif() if(APPLE) target_link_libraries(${CMAKE_PROJECT_NAME} "-framework Metal" @@ -768,6 +781,10 @@ if(BUILD_TESTS) ) target_compile_definitions(UnitTests PRIVATE BATCH_EXPORTER_TEST_SEAM QTMESH_UNIT_TESTS "QTMESH_UT_SOURCE_ROOT=\"${CMAKE_SOURCE_DIR}\"") + # Debug macro, scoped to the test target only (not FetchContent deps). + if(QTMESH_DEFINE_DEBUG) + target_compile_definitions(UnitTests PRIVATE DEBUG) + endif() # Link against Google Test libraries (no gtest_main - we provide our own main) target_link_libraries(UnitTests @@ -801,9 +818,12 @@ if(BUILD_TESTS) target_link_libraries(UnitTests advapi32) endif() - # Link llama.cpp for tests if enabled + # Link llama.cpp for tests if enabled (mtmd for the captioner where present) if(ENABLE_LOCAL_LLM) target_link_libraries(UnitTests llama ggml) + if(QTMESH_HAVE_MTMD) + target_link_libraries(UnitTests mtmd) + endif() endif() # Link stable-diffusion.cpp for tests if enabled diff --git a/src/HDR/HdrEquirectLoader.cpp b/src/HDR/HdrEquirectLoader.cpp index d4f99601..5bac14eb 100644 --- a/src/HDR/HdrEquirectLoader.cpp +++ b/src/HDR/HdrEquirectLoader.cpp @@ -12,6 +12,10 @@ #include #define STB_IMAGE_IMPLEMENTATION +// Make our stb_image symbols TU-local: llama.cpp's mtmd library bundles its +// own stb_image, and without this the two copies collide at link time +// (duplicate _stbi_* symbols) once mtmd is linked for image captioning (#764). +#define STB_IMAGE_STATIC #define STBI_ONLY_HDR #define STBI_NO_BMP #define STBI_NO_PNG diff --git a/src/ImageTo3D/ImageCaptioner.cpp b/src/ImageTo3D/ImageCaptioner.cpp new file mode 100644 index 00000000..bf7d71be --- /dev/null +++ b/src/ImageTo3D/ImageCaptioner.cpp @@ -0,0 +1,239 @@ +#include "ImageCaptioner.h" + +#include +#include +#include + +#ifdef ENABLE_MTMD +#include "ModelDownloader.h" +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#endif + +namespace ImageCaptioner { + +namespace { +// SmolVLM-500M-Instruct (Apache-2.0), the model + its vision projector. Hosted +// on the QtMeshEditor models repo under caption/ (mirrored from ggml-org). +// It's the smallest/fastest tier — captions are terse ("a rabbit"). For richer +// descriptions a future "quality" tier can swap in Moondream2 (~1.7 GB) or +// Qwen2-VL-2B (~1.5 GB), both Apache-2.0 + llama.cpp libmtmd-supported (see the +// #764 captioner research); the detail-seeking prompt in the header pushes the +// 500M model as far as it goes without the extra download. +constexpr const char* kModelFile = "SmolVLM-500M-Instruct-Q8_0.gguf"; +constexpr const char* kMmprojFile = "mmproj-SmolVLM-500M-Instruct-Q8_0.gguf"; +constexpr const char* kDefaultModelBaseUrl = + "https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/caption/"; +constexpr const char* kBaseUrlSettingsKey = "ai/captionModelBaseUrl"; + +QString modelDir() +{ + const QString base = + QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); + return QDir(base).filePath(QStringLiteral("ai_models/caption/")); +} +} // namespace + +QString modelPath() { return QDir(modelDir()).filePath(QString::fromLatin1(kModelFile)); } +QString mmprojPath() { return QDir(modelDir()).filePath(QString::fromLatin1(kMmprojFile)); } + +#ifdef ENABLE_MTMD + +bool isAvailable() { return true; } + +bool modelsPresent() +{ + return QFileInfo::exists(modelPath()) && QFileInfo::exists(mmprojPath()); +} + +QString ensureModelBlocking() +{ + if (modelsPresent()) + return modelPath(); + if (!qEnvironmentVariableIsEmpty("QTMESH_CAPTION_NO_DOWNLOAD")) + return {}; + + QString base; + { + QSettings s; + base = s.value(QString::fromLatin1(kBaseUrlSettingsKey)).toString(); + if (base.isEmpty()) { + const QByteArray env = qgetenv("QTMESH_CAPTION_MODEL_BASE_URL"); + base = env.isEmpty() ? QString::fromLatin1(kDefaultModelBaseUrl) + : QString::fromUtf8(env); + } + } + if (base.isEmpty()) return {}; + if (!base.endsWith('/')) base += '/'; + + auto* dl = ModelDownloader::instance(); + if (!dl) return {}; + + auto downloadOne = [&](const char* fileName, const QString& dest) -> bool { + QDir().mkpath(QFileInfo(dest).absolutePath()); + const QString label = + QStringLiteral("Caption %1").arg(QString::fromLatin1(fileName)); + const QString url = base + QString::fromLatin1(fileName); + QEventLoop loop; + bool ok = false, timedOut = false; + auto onDone = QObject::connect(dl, &ModelDownloader::downloadCompleted, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = true; loop.quit(); } + }); + auto onErr = QObject::connect(dl, &ModelDownloader::downloadError, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = false; loop.quit(); } + }); + QTimer timeout; + timeout.setSingleShot(true); + QObject::connect(&timeout, &QTimer::timeout, &loop, + [&]() { timedOut = true; loop.quit(); }); + timeout.start(1800000); // 30 min — the pair is < 600 MB + dl->startDownload(url, dest, label); + loop.exec(); + QObject::disconnect(onDone); + QObject::disconnect(onErr); + if (timedOut) dl->cancelDownload(); + return ok && !timedOut && QFileInfo::exists(dest); + }; + + if (!QFileInfo::exists(modelPath()) && !downloadOne(kModelFile, modelPath())) return {}; + if (!QFileInfo::exists(mmprojPath()) && !downloadOne(kMmprojFile, mmprojPath())) return {}; + return modelsPresent() ? modelPath() : QString(); +} + +QString caption(const QImage& image, const QString& prompt) +{ + if (image.isNull() || !modelsPresent()) + return {}; + + // ---- Load the text model + its mtmd (vision projector) context ---------- + llama_model_params mparams = llama_model_default_params(); + // Vision projector runs on CPU/accelerator via ggml; the small LM can put a + // few layers on GPU where available. Keep it modest — captioning is one-shot. + mparams.n_gpu_layers = 0; + llama_model* model = llama_model_load_from_file( + modelPath().toUtf8().constData(), mparams); + if (!model) return {}; + + llama_context_params cparams = llama_context_default_params(); + cparams.n_ctx = 4096; // ample for image tokens + a short caption + int threads = QThread::idealThreadCount(); + if (threads <= 0) threads = 4; + if (threads > 16) threads = 16; + cparams.n_threads = threads; + cparams.n_threads_batch = threads; + llama_context* lctx = llama_init_from_model(model, cparams); + if (!lctx) { llama_model_free(model); return {}; } + + mtmd_context_params mparams2 = mtmd_context_params_default(); + mparams2.use_gpu = false; // small projector; CPU is fine + portable + mparams2.print_timings = false; + mparams2.n_threads = threads; + mtmd_context* mctx = mtmd_init_from_file( + mmprojPath().toUtf8().constData(), model, mparams2); + if (!mctx || !mtmd_support_vision(mctx)) { + if (mctx) mtmd_free(mctx); + llama_free(lctx); + llama_model_free(model); + return {}; + } + + QString result; + do { + // ---- Wrap the RGB image as an mtmd_bitmap (RGBRGB… bytes) ----------- + const QImage rgb = image.convertToFormat(QImage::Format_RGB888); + std::vector pix(static_cast(rgb.width()) + * rgb.height() * 3); + for (int y = 0; y < rgb.height(); ++y) { + const uchar* line = rgb.constScanLine(y); + std::copy(line, line + rgb.width() * 3, + pix.data() + static_cast(y) * rgb.width() * 3); + } + mtmd_bitmap* bmp = mtmd_bitmap_init( + static_cast(rgb.width()), + static_cast(rgb.height()), pix.data()); + if (!bmp) break; + + // ---- Build the prompt with the media marker + tokenize -------------- + const QString instruction = prompt.isEmpty() + ? QString::fromLatin1(kDefaultPrompt) : prompt; + // SmolVLM (idefics3) chat format; the media marker is replaced by the + // image chunk during tokenization. + const std::string text = + std::string("<|im_start|>User: ") + mtmd_default_marker() + + instruction.toStdString() + "\nAssistant:"; + + mtmd_input_text itext; + itext.text = text.c_str(); + itext.add_special = true; + itext.parse_special = true; + + mtmd_input_chunks* chunks = mtmd_input_chunks_init(); + const mtmd_bitmap* bmps[1] = { bmp }; + const int32_t trc = mtmd_tokenize(mctx, chunks, &itext, bmps, 1); + if (trc != 0) { + mtmd_input_chunks_free(chunks); + mtmd_bitmap_free(bmp); + break; + } + + // ---- Evaluate text + image chunks (runs the vision encoder) --------- + llama_pos n_past = 0; + const int32_t erc = mtmd_helper_eval_chunks( + mctx, lctx, chunks, /*n_past=*/0, /*seq_id=*/0, + /*n_batch=*/2048, /*logits_last=*/true, &n_past); + mtmd_input_chunks_free(chunks); + mtmd_bitmap_free(bmp); + if (erc != 0) break; + + // ---- Greedy decode the caption -------------------------------------- + const llama_vocab* vocab = llama_model_get_vocab(model); + llama_sampler* smpl = llama_sampler_chain_init( + llama_sampler_chain_default_params()); + llama_sampler_chain_add(smpl, llama_sampler_init_greedy()); + + std::string out; + for (int i = 0; i < 96; ++i) { // room for a detailed comma-list + const llama_token tok = llama_sampler_sample(smpl, lctx, -1); + if (llama_vocab_is_eog(vocab, tok)) break; + char buf[256]; + const int n = llama_token_to_piece(vocab, tok, buf, sizeof(buf), 0, true); + if (n > 0) out.append(buf, n); + llama_batch nb = llama_batch_get_one(const_cast(&tok), 1); + if (llama_decode(lctx, nb) != 0) break; + } + llama_sampler_free(smpl); + + result = QString::fromStdString(out).trimmed(); + // Strip any stray end-marker / role tokens the model may echo. + result.remove(QStringLiteral("")); + result = result.trimmed(); + } while (false); + + mtmd_free(mctx); + llama_free(lctx); + llama_model_free(model); + return result; +} + +#else // !ENABLE_MTMD + +bool isAvailable() { return false; } +bool modelsPresent() { return false; } +QString ensureModelBlocking(){ return {}; } +QString caption(const QImage&, const QString&) { return {}; } + +#endif // ENABLE_MTMD + +} // namespace ImageCaptioner diff --git a/src/ImageTo3D/ImageCaptioner.h b/src/ImageTo3D/ImageCaptioner.h new file mode 100644 index 00000000..72d2c85a --- /dev/null +++ b/src/ImageTo3D/ImageCaptioner.h @@ -0,0 +1,61 @@ +#ifndef IMAGE_CAPTIONER_H +#define IMAGE_CAPTIONER_H + +#include +#include + +// Local image captioning for the image-to-3D "describe-then-generate" texture +// path (epic #764). Given a single RGB image of an isolated object, produces a +// short natural-language description (e.g. "a brown rabbit sitting") to feed +// Stable Diffusion + depth-ControlNet as the texture prompt — so ALL views +// (front + back) are generated from the caption and no photo projection is +// needed (which sidesteps the front/back orientation + registration issues). +// +// Backend: llama.cpp's multimodal `libmtmd` running a SMALL, permissively- +// licensed vision-language model — **SmolVLM-500M-Instruct (Apache-2.0)** by +// default (~550 MB: the GGUF + its mmproj projector). Reuses the existing +// llama.cpp integration (no new runtime; works on macOS/Windows-MinGW/Linux) +// and the ModelDownloader / HF-repo first-use download pattern. +// +// Guarded by ENABLE_LOCAL_LLM. Synchronous + blocking (like the SD CLI +// drivers) — caption() loads model + mmproj, runs one image, returns the text. +// Pure-data (no Ogre); the caption is fed to MaterialEditorQML's multi-view +// texture pass. When the build lacks llama.cpp or the model is unavailable, +// isAvailable()/ensureModelBlocking() report it and caption() returns empty so +// the caller can fall back to a manual/neutral prompt. +namespace ImageCaptioner { + +// True only when built with ENABLE_LOCAL_LLM. +// A DETAIL-seeking prompt: SmolVLM-500M defaults to terse answers ("a +// rabbit"), which makes a weak texture prompt. Explicitly ask for a rich, +// comma-separated visual description (colours, materials, markings, surface) +// and forbid the one-word reply, so the caption feeds SD with real texture cues. +static constexpr const char* kDefaultPrompt = + "Describe this object in detail as a comma-separated visual texture prompt: " + "its type, all colours, materials, surface texture, patterns and markings. " + "Be specific and descriptive, not a single word. Example: 'a fluffy brown " + "and white rabbit, soft short fur, pink inner ears, dark eyes'. " + "Answer with only the description."; + +bool isAvailable(); + +// AppData/ai_models/caption/ paths for the two GGUF files. +QString modelPath(); // the SmolVLM instruct GGUF +QString mmprojPath(); // its vision projector (mmproj) GGUF +bool modelsPresent(); + +// Ensure both files exist, downloading on first use (blocks via a local event +// loop). Returns modelPath() when present, else empty. Honours +// QTMESH_CAPTION_NO_DOWNLOAD and the base-URL override +// (QTMESH_CAPTION_MODEL_BASE_URL / QSettings ai/captionModelBaseUrl). +QString ensureModelBlocking(); + +// Caption `image` (RGB; background-removed subject works best). `prompt` +// overrides the default instruction. Returns a trimmed one-line description, +// or empty on any failure (no model, load error, etc.). Blocking; call off the +// UI thread if responsiveness matters. +QString caption(const QImage& image, const QString& prompt = QString()); + +} // namespace ImageCaptioner + +#endif // IMAGE_CAPTIONER_H diff --git a/src/ImageTo3D/MeshGenBuilder.cpp b/src/ImageTo3D/MeshGenBuilder.cpp index 491a1dc7..8b2d1886 100644 --- a/src/ImageTo3D/MeshGenBuilder.cpp +++ b/src/ImageTo3D/MeshGenBuilder.cpp @@ -128,9 +128,20 @@ Ogre::Mesh* buildMesh(const MeshGenPredictor::Result& result, const QString& mes // step 1: -90° about X to stand it up: (x, y, z) -> (x, z, -y) // step 2: +90° about Y to face forward: (x, y, z) -> (z, y, -x) // Composed: (x, y, z) -> (-y, z, -x). - auto orient = [](float& x, float& y, float& z) { - const float nx = -y, ny = z, nz = -x; - x = nx; y = ny; z = nz; + // TripoSG results are already +Y-up (Result::bakeTripoSROrientation is + // false) — baking the TripoSR frame onto them lays the model on its back. + // BUT TripoSG comes out facing AWAY (its front is -Z here), so rotate it + // 180° about Y so the reconstructed front faces the camera (+Z): + // 180°Y: (x, y, z) -> (-x, y, -z). + const bool bakeOrientation = result.bakeTripoSROrientation; + const bool flipY180 = !bakeOrientation; // TripoSG path only + auto orient = [bakeOrientation, flipY180](float& x, float& y, float& z) { + if (bakeOrientation) { + const float nx = -y, ny = z, nz = -x; + x = nx; y = ny; z = nz; + } else if (flipY180) { + x = -x; z = -z; // 180° about Y — face the front forward + } }; Ogre::Vector3 mn(1e30f, 1e30f, 1e30f), mx(-1e30f, -1e30f, -1e30f); @@ -232,6 +243,27 @@ Ogre::Mesh* buildMesh(const MeshGenPredictor::Result& result, const QString& mes sub->setMaterialName(kMat, Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); } + // Geometry-only result (TripoSG, or TripoSR with every colour stage off): + // without a material the default flat-white one hides all surface relief. + // Assign a shared neutral LIT clay material so the shape actually shades. + if (!hasUv && !hasColor) { + auto& matMgr = Ogre::MaterialManager::getSingleton(); + const char* kMat = "MeshGen/NeutralClay"; + Ogre::MaterialPtr clay = matMgr.getByName(kMat); + if (!clay) { + clay = matMgr.create(kMat, Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + auto* pass = clay->getTechnique(0)->getPass(0); + pass->setLightingEnabled(true); + pass->setDiffuse(Ogre::ColourValue(0.72f, 0.68f, 0.62f)); // warm clay + pass->setAmbient(Ogre::ColourValue(0.35f, 0.33f, 0.30f)); + pass->setSpecular(Ogre::ColourValue(0.15f, 0.15f, 0.15f)); + pass->setShininess(24.0f); + pass->setCullingMode(Ogre::CULL_CLOCKWISE); + clay->compile(); + } + sub->setMaterialName(kMat, Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + } + Ogre::AxisAlignedBox aabb(mn, mx); mesh->_setBounds(aabb); mesh->_setBoundingSphereRadius(0.5f * (mx - mn).length()); @@ -395,6 +427,11 @@ Ogre::SceneNode* buildSceneNode(const MeshGenPredictor::Result& result, // lights identically to an export→reload of it. if (ent) MeshImporterExporter::applyNormalMapsToEntity(ent); + // Both image-to-3D backends reconstruct into a unit-ish box that lands the + // model quite small in the editor scene; scale x2 so it arrives at a + // workable size (matches TripoSR + TripoSG). Applied on the node, so it's + // baked into any export via the node transform. + node->setScale(2.0f, 2.0f, 2.0f); return node; } diff --git a/src/ImageTo3D/MeshGenController.cpp b/src/ImageTo3D/MeshGenController.cpp index 8c8a47c8..e8d7fb39 100644 --- a/src/ImageTo3D/MeshGenController.cpp +++ b/src/ImageTo3D/MeshGenController.cpp @@ -3,12 +3,14 @@ #include "GamificationManager.h" #include "MeshGenPredictor.h" +#include "TripoSGPredictor.h" #include "MeshGenBuilder.h" #include "BackgroundRemover.h" #include "MeshImporterExporter.h" #include "SentryReporter.h" #include "AIAssistManager.h" // ensureUpscaleModel (main-thread model fetch) #include "TextureUpscaler.h" // worker-side Real-ESRGAN 2x on the baked diffuse +#include "ImageCaptioner.h" // background SmolVLM caption of the picked image #include @@ -19,6 +21,7 @@ #include #include // organizationName() — test-harness guard #include +#include #include #include @@ -157,6 +160,51 @@ void MeshGenController::selectImage() QStringLiteral("MeshGenController selectImage %1").arg(QFileInfo(path).fileName())); emit selectedImageChanged(); emit statusMessage(tr("Selected: %1").arg(QFileInfo(path).fileName())); + + // Caption the image in the BACKGROUND now, so it's ready by the time the + // (slow) mesh generation finishes and the AI texture pass needs it — no + // blocking the UI to caption. Shown under the thumbnail as it lands. + m_caption.clear(); + startCaptioning(path); +} + +void MeshGenController::startCaptioning(const QString& path) +{ + if (!ImageCaptioner::isAvailable() || path.isEmpty()) { + m_captioning = false; + emit captionChanged(); + return; + } + m_captionForPath = path; + m_captioning = true; + emit captionChanged(); + + // Detached worker: ensure the model (first-use download) + caption, then + // marshal the result back to the main thread via a queued invocation. + QPointer self(this); + std::thread([self, path]() { + QString cap; + const QString model = ImageCaptioner::ensureModelBlocking(); + if (!model.isEmpty()) { + QImage img(path); + if (!img.isNull()) + cap = ImageCaptioner::caption(img); + } + if (!self) return; + QMetaObject::invokeMethod(self, "setCaptionResult", Qt::QueuedConnection, + Q_ARG(QString, cap), Q_ARG(QString, path)); + }).detach(); +} + +void MeshGenController::setCaptionResult(const QString& caption, const QString& forPath) +{ + // Drop a stale result if the user picked a different image meanwhile. + if (forPath != m_captionForPath) return; + m_caption = caption; + m_captioning = false; + emit captionChanged(); + if (!caption.isEmpty()) + emit statusMessage(tr("Image described: \"%1\"").arg(caption)); } void MeshGenController::generateSelected(int resolution, bool removeBackground, @@ -218,6 +266,13 @@ void MeshGenController::generate(const QString& imagePath, int resolution, m_generatePbr = optBool("generate_pbr", true) && wantBake; const int textureSize = options.contains(QLatin1String("texture_size")) ? options.value(QLatin1String("texture_size")).toInt() : 1024; + // Backend: "triposr" (default, fast + textured) or "triposg" (rectified + // flow — higher-fidelity geometry, geometry-only, slower). + const bool useSG = options.value(QLatin1String("backend")).toString() + .compare(QLatin1String("triposg"), + Qt::CaseInsensitive) == 0; + const int flowSteps = options.contains(QLatin1String("flow_steps")) + ? options.value(QLatin1String("flow_steps")).toInt() : 25; GamificationManager::noteFeature(QStringLiteral("image_to_3d")); @@ -228,17 +283,39 @@ void MeshGenController::generate(const QString& imagePath, int resolution, setBusy(true); emit progress(QStringLiteral("prep"), 0, 1); - // Ensure models on the MAIN thread first — ensureModelBlocking() spins a local - // QEventLoop for the download, which must not run on the worker thread. Once - // present, the worker only reads the files (no event loop needed). + // Ensure the chosen backend's models on the MAIN thread first — + // ensureModelBlocking() spins a local QEventLoop for the download, which + // must not run on the worker thread. Once present, the worker only reads + // the files (no event loop needed). emit statusMessage(tr("Checking model…")); - const QString enc = MeshGenPredictor::ensureModelBlocking(m_quality); - if (enc.isEmpty() || !MeshGenPredictor::modelsPresent(m_quality)) { - setBusy(false); - emit error(tr("TripoSR model unavailable — it downloads on first use; if it " - "is not hosted yet, set QTMESH_TRIPOSR_MODEL_BASE_URL or drop " - "the files in the ai_models/triposr/ cache.")); - return; + if (useSG) { + // TripoSG always runs the fp32 DiT — the int8 tier is dropped + // (quantized geometry degrades to blobs; no ARM speed win). + const QString enc = TripoSGPredictor::ensureModelBlocking(false); + if (enc.isEmpty()) { + setBusy(false); + emit error(tr("TripoSG models unavailable — they download on first " + "use; if not hosted yet, set " + "QTMESH_TRIPOSG_MODEL_BASE_URL or drop the files in " + "the ai_models/triposg/ cache.")); + return; + } + // TripoSG's colour bake queries TripoSR's image-conditioned colour + // field — ensure those models too (best-effort: if unavailable the + // predictor falls back to the clay look with a warning). + if (wantBake) { + emit statusMessage(tr("Checking colour model…")); + MeshGenPredictor::ensureModelBlocking(m_quality); + } + } else { + const QString enc = MeshGenPredictor::ensureModelBlocking(m_quality); + if (enc.isEmpty() || !MeshGenPredictor::modelsPresent(m_quality)) { + setBusy(false); + emit error(tr("TripoSR model unavailable — it downloads on first use; if it " + "is not hosted yet, set QTMESH_TRIPOSR_MODEL_BASE_URL or drop " + "the files in the ai_models/triposr/ cache.")); + return; + } } if (removeBackground) BackgroundRemover::ensureModelBlocking(); // best-effort; falls back if absent @@ -273,7 +350,7 @@ void MeshGenController::generate(const QString& imagePath, int resolution, // connection so the GUI thread updates the bar. m_pending->worker = std::thread([this, image, res, rembg, wantSmooth, wantRefine, wantBake, - textureSize]() { + textureSize, useSG, flowSteps]() { auto post = [this](const QString& stage, int done, int total) { QMetaObject::invokeMethod(this, "progress", Qt::QueuedConnection, Q_ARG(QString, stage), Q_ARG(int, done), Q_ARG(int, total)); @@ -283,7 +360,10 @@ void MeshGenController::generate(const QString& imagePath, int resolution, // worker started), so this thread only reads files — no event loop // needed. (The encode stage is reported by the predictor itself.) QImage subject = image; - if (rembg) { + if (rembg && !useSG) { + // TripoSR path: composite over gray-128 (its training background). + // The TripoSG path leaves removal to the predictor dispatch, which + // composites over WHITE per its reference pipeline. post(QStringLiteral("background"), 0, 1); QMetaObject::invokeMethod(this, "statusMessage", Qt::QueuedConnection, Q_ARG(QString, tr("Removing background…"))); @@ -296,11 +376,17 @@ void MeshGenController::generate(const QString& imagePath, int resolution, MeshGenPredictor::Options opts; opts.sdfResolution = res; opts.vertexColor = true; - opts.removeBackground = false; // already handled above + // TripoSR removal already ran above; TripoSG's white-background + // removal happens inside the predictor dispatch. + opts.removeBackground = rembg && useSG; opts.smoothMesh = wantSmooth; opts.refineSurface = wantRefine; opts.bakeTexture = wantBake; opts.textureSize = textureSize; + opts.backend = useSG ? MeshGenPredictor::Backend::TripoSG + : MeshGenPredictor::Backend::TripoSR; + opts.flowSteps = flowSteps; + opts.quality = m_quality; QMetaObject::invokeMethod(this, "statusMessage", Qt::QueuedConnection, Q_ARG(QString, tr("Reconstructing…"))); @@ -312,11 +398,12 @@ void MeshGenController::generate(const QString& imagePath, int resolution, if (total > 0) { const char* name = nullptr; switch (st) { - case MeshGenPredictor::Stage::Encode: name = "encode"; break; - case MeshGenPredictor::Stage::Decode: name = "decode"; break; - case MeshGenPredictor::Stage::Refine: name = "refine"; break; - case MeshGenPredictor::Stage::Bake: name = "bake"; break; - case MeshGenPredictor::Stage::Color: name = "color"; break; + case MeshGenPredictor::Stage::Encode: name = "encode"; break; + case MeshGenPredictor::Stage::Denoise: name = "denoise"; break; + case MeshGenPredictor::Stage::Decode: name = "decode"; break; + case MeshGenPredictor::Stage::Refine: name = "refine"; break; + case MeshGenPredictor::Stage::Bake: name = "bake"; break; + case MeshGenPredictor::Stage::Color: name = "color"; break; } if (name) post(QString::fromLatin1(name), done, total); } @@ -391,6 +478,13 @@ void MeshGenController::buildOnMainThread() {"vertexCount", r.vertexCount}, {"triangleCount", r.triangleCount}, }; + // Expose the built entity's name so QML can run a follow-up AI texture bake + // on it (the "Generate texture (AI)" option for the geometry-only TripoSG + // backend). The node has exactly one attached Entity. + if (node->numAttachedObjects() > 0) { + if (auto* obj = node->getAttachedObject(0)) + out["entityName"] = QString::fromStdString(obj->getName()); + } if (!r.warning.isEmpty()) out["warning"] = r.warning; emit statusMessage(!r.warning.isEmpty() ? tr("Generated %1 verts, %2 tris (%3)") diff --git a/src/ImageTo3D/MeshGenController.h b/src/ImageTo3D/MeshGenController.h index 560ffd15..f4245727 100644 --- a/src/ImageTo3D/MeshGenController.h +++ b/src/ImageTo3D/MeshGenController.h @@ -38,6 +38,13 @@ class MeshGenController : public QObject // URL the QML Image element can show directly (same idiom as the texture // packer previews). Empty when no image is selected. Q_PROPERTY(QString previewSource READ previewSource NOTIFY selectedImageChanged) + // Auto-generated caption of the selected image (SmolVLM), computed on a + // WORKER thread the moment an image is picked so it's ready by the time the + // (slow) mesh generation finishes — the AI texture pass reuses it instead + // of blocking the UI to caption. The panel shows it under the thumbnail. + // Empty until captioning completes (or if the captioner is unavailable). + Q_PROPERTY(QString caption READ caption NOTIFY captionChanged) + Q_PROPERTY(bool captioning READ captioning NOTIFY captionChanged) public: static MeshGenController* instance(); @@ -48,6 +55,8 @@ class MeshGenController : public QObject bool busy() const { return m_busy; } QString selectedImagePath() const { return m_selectedImage; } QString previewSource() const { return m_previewSource; } + QString caption() const { return m_caption; } + bool captioning() const { return m_captioning; } // Open a native file dialog to pick a source image; stores it as the selected // image and builds the preview thumbnail (does NOT start generation). Returns @@ -102,10 +111,16 @@ class MeshGenController : public QObject void completed(QVariantMap result); // {vertexCount, triangleCount} void error(const QString& message); void modelDownloadFinished(bool ok); // pre-download from AI Settings + void captionChanged(); // caption / captioning state updated private: explicit MeshGenController(QObject* parent = nullptr); + // Kick off SmolVLM captioning of `path` on a detached worker thread; the + // result is marshalled back to the main thread (setCaptionResult). + void startCaptioning(const QString& path); + Q_INVOKABLE void setCaptionResult(const QString& caption, const QString& forPath); + // Runs on the MAIN thread (queued from the worker) to build + attach the mesh. Q_INVOKABLE void buildOnMainThread(); @@ -115,6 +130,11 @@ class MeshGenController : public QObject std::atomic m_cancel{false}; QString m_selectedImage; // currently-selected source image path QString m_previewSource; // data:image/png;base64 thumbnail of it + QString m_caption; // auto-caption of the selected image + bool m_captioning = false; + // Guards against a stale caption landing after the image changed again: + // startCaptioning stamps the path, setCaptionResult drops mismatches. + QString m_captionForPath; MeshGenPredictor::Quality m_quality = MeshGenPredictor::Quality::Fp32; // Parsed pipeline options for the in-flight run (see generateSelected doc). bool m_generatePbr = true; // consumed by buildOnMainThread diff --git a/src/ImageTo3D/MeshGenPredictor.cpp b/src/ImageTo3D/MeshGenPredictor.cpp index 08e384ae..52c3202f 100644 --- a/src/ImageTo3D/MeshGenPredictor.cpp +++ b/src/ImageTo3D/MeshGenPredictor.cpp @@ -1,10 +1,11 @@ #include "MeshGenPredictor.h" #include "MarchingCubes.h" -#include "MeshRefine.h" // Taubin smoothing + iso-surface reprojection -#include "MeshGenBaker.h" // xatlas unwrap + diffuse texture bake -#include "PbrMapSynth.h" // toNCHW (image → planar [0,1]) +#include "MeshRefine.h" // Taubin smoothing + iso-surface reprojection +#include "MeshGenBaker.h" // xatlas unwrap + diffuse texture bake +#include "PbrMapSynth.h" // toNCHW (image → planar [0,1]) #include "BackgroundRemover.h" +#include "TripoSGPredictor.h" // Backend::TripoSG dispatch #include #include @@ -215,6 +216,42 @@ MeshGenPredictor::Result MeshGenPredictor::predict(const QImage& image, { if (image.isNull()) return fail(QStringLiteral("MeshGen: input image is empty.")); + + // ---- Backend dispatch: TripoSG (rectified-flow, geometry-only) ---------- + if (opts.backend == Backend::TripoSG) { + QImage subject = image; + if (opts.removeBackground) { + // TripoSG's reference pipeline composites the cut-out over WHITE + // (unlike TripoSR's gray-128) — see docs/TRIPOSG_EXPORT_NOTES.md. + const QString bgModel = BackgroundRemover::ensureModelBlocking(); + BackgroundRemover::Options bg; + bg.bgR = 255; bg.bgG = 255; bg.bgB = 255; + const BackgroundRemover::Result br = + BackgroundRemover::removeBackground(image, bgModel, bg); + subject = br.image; + } + TripoSGPredictor::Options sg; + sg.sdfResolution = opts.sdfResolution; + sg.flowSteps = opts.flowSteps; + sg.guidanceScale = opts.guidanceScale; + // int8 tier DROPPED for TripoSG: the quantized 1.5B DiT degrades to + // blobs over the flow loop (verified live even with per-channel + // quant), and dynamic-int8 MatMuls are no faster than fp32 on ARM. + // The tier picker stays meaningful for TripoSR only. + sg.useInt8Dit = false; + sg.smoothMesh = opts.smoothMesh; + sg.smoothIterations = opts.smoothIterations; + sg.refineSurface = opts.refineSurface; + sg.chunkPoints = opts.chunkPoints; + Result r = TripoSGPredictor::predict(subject, sg, progress); + // TripoSG's field is already +Y-up — skip the TripoSR frame bake. + r.bakeTripoSROrientation = false; + // TripoSG is geometry-only. Colour comes SOLELY from the AI image + // generation pass (multi-view depth-ControlNet, run later in the GUI + // layer) — no TripoSR field colouring. With no AI texture the mesh + // ships uncoloured (neutral clay material in MeshGenBuilder). + return r; + } if (!QFileInfo::exists(encoderModelPath) || !QFileInfo::exists(decoderModelPath)) return fail(QStringLiteral("MeshGen: TripoSR model not found (not hosted yet? " "see docs/IMAGE_TO_3D_SPIKE_764.md).")); diff --git a/src/ImageTo3D/MeshGenPredictor.h b/src/ImageTo3D/MeshGenPredictor.h index 80f20c3a..4c77bc5c 100644 --- a/src/ImageTo3D/MeshGenPredictor.h +++ b/src/ImageTo3D/MeshGenPredictor.h @@ -50,6 +50,11 @@ class MeshGenPredictor { // that the ONNX fp16 converters can't rewrite cleanly; int8 is smaller anyway.) enum class Quality { Fp32, Int8 }; + // Generation backend. TripoSR = the fast single-pass LRM (default); + // TripoSG = the 1.5B rectified-flow model (higher-fidelity geometry, + // slower, geometry-only — see TripoSGPredictor). Both MIT code+weights. + enum class Backend { TripoSR, TripoSG }; + struct Options { Options(); // out-of-line (same idiom as UniRig::Options) int sdfResolution = 256; // marching-cubes grid resolution (128 = fast) @@ -82,6 +87,14 @@ class MeshGenPredictor { // to vertex colours (Result::warning set) if the unwrap/bake fails. bool bakeTexture = true; int textureSize = 1024; + + // ---- Backend selection ------------------------------------------------ + // TripoSG ignores the colour/bake options (geometry-only model) and + // maps Quality::Int8 onto its int8 DiT tier. flowSteps/guidanceScale + // only apply to TripoSG. + Backend backend = Backend::TripoSR; + int flowSteps = 25; + float guidanceScale = 7.0f; }; struct Result { @@ -98,6 +111,11 @@ class MeshGenPredictor { int vertexCount = 0; int triangleCount = 0; bool usedModel = false; // true iff the ONNX path ran + // TripoSR's reconstruction frame lies on its back + faces 90° off, so + // MeshGenBuilder bakes a fixed -90°X/+90°Y into the vertex data. + // TripoSG's field is already +Y-up (upstream exports the marching-cubes + // trimesh as-is), so its dispatch sets this false to skip the bake. + bool bakeTripoSROrientation = true; }; // True only when built with ENABLE_ONNX. (Model presence is checked per call.) @@ -125,7 +143,8 @@ class MeshGenPredictor { // Pipeline stage identifiers for the progress callback — one per // user-visible step of predict() (the GUI shows a per-step progress list). enum class Stage { - Encode, // TripoSR encoder run (single blocking call: 0/1 → 1/1) + Encode, // image encoder run (single blocking call: 0/1 → 1/1) + Denoise, // TripoSG rectified-flow Euler loop (per-step) Decode, // res³ grid decode (per-chunk; the long one) Refine, // iso-surface reprojection probes (per-chunk) Bake, // texture bake colour queries (per-chunk, baker-reported) diff --git a/src/ImageTo3D/TripoSGPredictor.cpp b/src/ImageTo3D/TripoSGPredictor.cpp new file mode 100644 index 00000000..252e3ab9 --- /dev/null +++ b/src/ImageTo3D/TripoSGPredictor.cpp @@ -0,0 +1,626 @@ +#include "TripoSGPredictor.h" + +#include "MarchingCubes.h" +#include "MeshRefine.h" +#include "PbrMapSynth.h" // toNCHW (image → planar [0,1]) + +#include +#include +#include + +#include +#include +#include +#include + +#ifdef ENABLE_ONNX +#include "ModelDownloader.h" +#include +#include +#include +#include +#include +#include +#endif + +namespace { + +constexpr const char* kImageEncoderFile = "triposg_image_encoder.onnx"; +constexpr const char* kDitStepFile = "triposg_dit_step.onnx"; +constexpr const char* kDitStepDataFile = "triposg_dit_step.onnx.data"; // >2GB external weights +constexpr const char* kDitStepInt8File = "triposg_dit_step_int8.onnx"; +constexpr const char* kVaeLatentsFile = "triposg_vae_latents.onnx"; +constexpr const char* kVaeDecoderFile = "triposg_vae_decoder.onnx"; +constexpr const char* kDefaultModelBaseUrl = + "https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/triposg/"; +constexpr const char* kBaseUrlSettingsKey = "ai/triposgModelBaseUrl"; + +// Query-point half-extent of the VAE's SDF field, measured from the reference +// pipeline (docs/TRIPOSG_EXPORT_NOTES.md): bounds (−1.005, 1.005)³. +constexpr float kRadius = 1.005f; + +// Field convention: the exported decoder graph already NEGATES the raw SDF so +// the field is INSIDE-POSITIVE — the same convention our MarchingCubes uses +// (upstream runs skimage marching_cubes at level 0 on the same negated grid). +constexpr float kIsoLevel = 0.0f; + +// TripoSG's custom RectifiedFlowScheduler: num_train_timesteps=1000, shift=1 +// (identity) → σᵢ = 1 − i/N with σ_N = 0, and the DiT's timestep input is +// 1000·σ. See docs/TRIPOSG_EXPORT_NOTES.md. +constexpr float kNumTrainTimesteps = 1000.0f; + +QString modelDir() +{ + const QString base = + QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); + return QDir(base).filePath(QStringLiteral("ai_models/triposg/")); +} + +} // namespace + +TripoSGPredictor::Options::Options() = default; + +QString TripoSGPredictor::imageEncoderPath() +{ + return QDir(modelDir()).filePath(QString::fromLatin1(kImageEncoderFile)); +} + +QString TripoSGPredictor::ditStepPath(bool int8Tier) +{ + return QDir(modelDir()).filePath( + QString::fromLatin1(int8Tier ? kDitStepInt8File : kDitStepFile)); +} + +QString TripoSGPredictor::vaeLatentsPath() +{ + return QDir(modelDir()).filePath(QString::fromLatin1(kVaeLatentsFile)); +} + +QString TripoSGPredictor::vaeDecoderPath() +{ + return QDir(modelDir()).filePath(QString::fromLatin1(kVaeDecoderFile)); +} + +bool TripoSGPredictor::modelsPresent(bool int8Tier) +{ + if (!QFileInfo::exists(imageEncoderPath()) + || !QFileInfo::exists(ditStepPath(int8Tier)) + || !QFileInfo::exists(vaeLatentsPath()) + || !QFileInfo::exists(vaeDecoderPath())) + return false; + // The fp32 DiT ships as .onnx + .onnx.data (external weights >2GB); the + // graph is unloadable without its sidecar. + if (!int8Tier + && !QFileInfo::exists(QDir(modelDir()).filePath( + QString::fromLatin1(kDitStepDataFile)))) + return false; + return true; +} + +#ifndef ENABLE_ONNX + +bool TripoSGPredictor::isAvailable() { return false; } + +QString TripoSGPredictor::ensureModelBlocking(bool) { return {}; } + +MeshGenPredictor::Result TripoSGPredictor::predict( + const QImage&, const Options&, const MeshGenPredictor::ProgressFn&) +{ + MeshGenPredictor::Result r; + r.error = QStringLiteral( + "TripoSG generation was not built into this binary " + "(rebuild with -DENABLE_ONNX=ON)."); + return r; +} + +#else // ENABLE_ONNX + +bool TripoSGPredictor::isAvailable() { return true; } + +QString TripoSGPredictor::ensureModelBlocking(bool int8Tier) +{ + if (modelsPresent(int8Tier)) + return imageEncoderPath(); + + // Offline / test guard — never hit the network when set. + if (!qEnvironmentVariableIsEmpty("QTMESH_TRIPOSG_NO_DOWNLOAD")) + return {}; + + // Resolve the download base URL (QSettings override → env → default HF repo). + QString base; + { + QSettings s; + base = s.value(QString::fromLatin1(kBaseUrlSettingsKey)).toString(); + if (base.isEmpty()) { + const QByteArray env = qgetenv("QTMESH_TRIPOSG_MODEL_BASE_URL"); + base = env.isEmpty() ? QString::fromLatin1(kDefaultModelBaseUrl) + : QString::fromUtf8(env); + } + } + if (base.isEmpty()) return {}; + if (!base.endsWith('/')) base += '/'; + + auto* dl = ModelDownloader::instance(); + if (!dl) return {}; + + // Download one file, blocking via a local event loop — the exact + // UniRigPredictor::ensureModelBlocking pattern, with a hard timeout so a + // stalled connection can't hang the synchronous call. The DiT step graph + // is the big one (~3 GB fp32), so the cap is generous. + auto downloadOne = [&](const char* fileName, const QString& dest) -> bool { + QDir().mkpath(QFileInfo(dest).absolutePath()); + const QString label = + QStringLiteral("TripoSG %1").arg(QString::fromLatin1(fileName)); + const QString url = base + QString::fromLatin1(fileName); + QEventLoop loop; + bool ok = false, timedOut = false; + auto onDone = QObject::connect(dl, &ModelDownloader::downloadCompleted, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = true; loop.quit(); } + }); + auto onErr = QObject::connect(dl, &ModelDownloader::downloadError, &loop, + [&](const QString& name, const QString&) { + if (name == label) { ok = false; loop.quit(); } + }); + QTimer timeout; + timeout.setSingleShot(true); + QObject::connect(&timeout, &QTimer::timeout, &loop, + [&]() { timedOut = true; loop.quit(); }); + timeout.start(3600000); // 60 min for the ~3 GB DiT on slow links + dl->startDownload(url, dest, label); + loop.exec(); + QObject::disconnect(onDone); + QObject::disconnect(onErr); + if (timedOut) dl->cancelDownload(); + return ok && !timedOut && QFileInfo::exists(dest); + }; + + struct FileSpec { const char* file; QString path; }; + std::vector files{ + { kImageEncoderFile, imageEncoderPath() }, + { kVaeLatentsFile, vaeLatentsPath() }, + { kVaeDecoderFile, vaeDecoderPath() }, + }; + if (int8Tier) { + files.push_back({ kDitStepInt8File, ditStepPath(true) }); + } else { + files.push_back({ kDitStepFile, ditStepPath(false) }); + // fp32 DiT external-weights sidecar (>2GB graphs split the weights out). + files.push_back({ kDitStepDataFile, + QDir(modelDir()).filePath( + QString::fromLatin1(kDitStepDataFile)) }); + } + for (const auto& f : files) { + if (!QFileInfo::exists(f.path) && !downloadOne(f.file, f.path)) + return {}; + } + return modelsPresent(int8Tier) ? imageEncoderPath() : QString(); +} + +namespace { + +MeshGenPredictor::Result fail(const QString& msg) +{ + MeshGenPredictor::Result r; + r.error = msg; + return r; +} + +// TripoSG's RectifiedFlowScheduler sigma schedule (shift=1 → identity): +// σᵢ = 1 − i/N for i in [0, N), with a trailing σ_N = 0 so the last Euler +// step lands exactly on the data sample. +std::vector sigmaSchedule(int steps) +{ + std::vector sigmas; + sigmas.reserve(static_cast(steps) + 1); + for (int i = 0; i < steps; ++i) + sigmas.push_back(1.0f - float(i) / float(steps)); + sigmas.push_back(0.0f); + return sigmas; +} + +struct IoNames { + std::vector holders; + std::vector in, out; +}; + +IoNames ioNames(Ort::Session& s, Ort::AllocatorWithDefaultOptions& alloc) +{ + IoNames n; + for (size_t i = 0; i < s.GetInputCount(); ++i) { + n.holders.push_back(s.GetInputNameAllocated(i, alloc)); + n.in.push_back(n.holders.back().get()); + } + for (size_t i = 0; i < s.GetOutputCount(); ++i) { + n.holders.push_back(s.GetOutputNameAllocated(i, alloc)); + n.out.push_back(n.holders.back().get()); + } + return n; +} + +} // namespace + +MeshGenPredictor::Result TripoSGPredictor::predict( + const QImage& image, const Options& opts, + const MeshGenPredictor::ProgressFn& progress) +{ + using Stage = MeshGenPredictor::Stage; + + if (image.isNull()) + return fail(QStringLiteral("TripoSG: input image is empty.")); + if (!modelsPresent(opts.useInt8Dit)) + return fail(QStringLiteral( + "TripoSG models unavailable — they download on first use; if not " + "hosted yet, export them with scripts/export-triposg-onnx.py and " + "point QTMESH_TRIPOSG_MODEL_BASE_URL at them, or drop the files " + "in the ai_models/triposg/ cache.")); + + const int res = std::max(16, opts.sdfResolution); + const int steps = std::clamp(opts.flowSteps, 1, 200); + + try { + Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "qtmesh_triposg"); + // Two session-option flavours: + // - cpuOnly: for the DiT + the two one-shot graphs. CoreML/MLProgram + // COMPILES the model on session open — for the 1.5-5.4 GB DiT that + // burned minutes of CPU and ~8 GB RSS per generation (sessions are + // staged, so the compile would repeat every run) for an uncertain + // win. Opt back in with QTMESH_TRIPOSG_COREML_DIT=1 to experiment. + // - gpu (MLProgram, ALL compute units): for the ~48 MB point decoder, + // which is invoked ~2000×/run — the one graph where GPU dispatch + // clearly pays and the compile cost is trivial. + Ort::SessionOptions cpuOnly; + cpuOnly.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); + Ort::SessionOptions gpu; + gpu.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); + bool gpuAvailable = false; +#ifdef __APPLE__ + try { + // Legacy "NeuralNetwork" format maps almost none of these ops — + // MLProgram is what actually reaches the M-series GPU/ANE. + std::unordered_map coreml; + coreml["ModelFormat"] = "MLProgram"; + coreml["MLComputeUnits"] = "ALL"; + gpu.AppendExecutionProvider("CoreML", coreml); + gpuAvailable = true; + } catch (const Ort::Exception&) {} +#endif + const bool ditOnGpu = gpuAvailable + && qEnvironmentVariableIntValue("QTMESH_TRIPOSG_COREML_DIT") == 1; + auto open = [&](const QString& p, bool wantGpu = false) { + Ort::SessionOptions& so = (wantGpu && gpuAvailable) ? gpu : cpuOnly; +#ifdef _WIN32 + // Ort::Session takes ORTCHAR_T* (wchar_t*) on Windows — a narrow + // std::string path fails to open (matches the TripoSR path). + const std::wstring s = p.toStdWString(); + return Ort::Session(env, s.c_str(), so); +#else + const std::string s = p.toStdString(); + return Ort::Session(env, s.c_str(), so); +#endif + }; + Ort::AllocatorWithDefaultOptions alloc; + Ort::MemoryInfo mem = + Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + + // The four graphs total ~4-6 GB of weights. Sessions are opened ONE AT + // A TIME and released as soon as their stage completes — holding them + // all made the 25-step run's working set big enough for macOS to + // SIGTERM the process under memory pressure. Peak is now the largest + // single stage (the DiT), not the sum. + + // ---- (1) Image encoder: preprocessed RGB → conditioning tokens ------- + // Conditioning tokens (image_embeds [1,257,1024]). CFG's unconditional + // embedding is simply ZEROS of the same shape (upstream zeros_like). + std::vector cond; + std::vector condShape; + { + Ort::Session imgEnc = open(imageEncoderPath()); + // Input size comes from the graph itself (fallback 224 — the + // BitImageProcessor's crop size; mean/std are baked in). + int imgSize = 224; + { + // Keep the owning TypeInfo alive — GetTensorTypeAndShapeInfo() + // returns an unowned view into it; chaining off the temporary + // dangles and GetShape() segfaults inside GetDimensions. + Ort::TypeInfo ti = imgEnc.GetInputTypeInfo(0); + auto info = ti.GetTensorTypeAndShapeInfo(); + const auto shape = info.GetShape(); + if (shape.size() == 4 && shape[2] > 0) + imgSize = static_cast(shape[2]); + } + // BitImageProcessor recipe: resize SHORTEST edge to 8/7·crop + // (256 for 224), then centre-crop — not a squash-resize (which + // changes the framing the encoder was trained on). + const int resizeEdge = (imgSize * 8) / 7; + QImage pre = image.convertToFormat(QImage::Format_RGB888) + .scaled(resizeEdge, resizeEdge, + Qt::KeepAspectRatioByExpanding, + Qt::SmoothTransformation); + QImage resized = pre.copy((pre.width() - imgSize) / 2, + (pre.height() - imgSize) / 2, + imgSize, imgSize); + // Mean/std normalization baked INTO the exported graph + // (raw [0,1] RGB in). + std::vector imgNCHW = PbrMapSynth::toNCHW(resized, 3); + const int64_t imgShape[4] = {1, 3, imgSize, imgSize}; + Ort::Value imgTensor = Ort::Value::CreateTensor( + mem, imgNCHW.data(), imgNCHW.size(), imgShape, 4); + + IoNames encIo = ioNames(imgEnc, alloc); + if (progress && !progress(Stage::Encode, 0, 1)) + return fail(QStringLiteral("cancelled")); + auto encRes = imgEnc.Run(Ort::RunOptions{nullptr}, encIo.in.data(), + &imgTensor, 1, encIo.out.data(), + encIo.out.size()); + if (progress && !progress(Stage::Encode, 1, 1)) + return fail(QStringLiteral("cancelled")); + + auto condInfo = encRes[0].GetTensorTypeAndShapeInfo(); + condShape = condInfo.GetShape(); + const float* condData = encRes[0].GetTensorData(); + cond.assign(condData, condData + condInfo.GetElementCount()); + } // encoder session (~1.1 GB) released here + std::vector uncond(cond.size(), 0.0f); + std::vector uncondShape = condShape; + + // ---- (2) Rectified-flow Euler loop over the DiT step graph ----------- + // Latent shape from the DiT's `latents` input (e.g. [1, 2048, 64]). + std::vector latShape; + std::vector latents; + { + Ort::Session ditStep = open(ditStepPath(opts.useInt8Dit), ditOnGpu); + IoNames ditIo = ioNames(ditStep, alloc); + { + // Same TypeInfo-lifetime rule as above. + Ort::TypeInfo ti = ditStep.GetInputTypeInfo(0); + auto info = ti.GetTensorTypeAndShapeInfo(); + latShape = info.GetShape(); + for (auto& d : latShape) + if (d < 0) d = 1; // defensive: dynamic dims default to 1 + } + size_t latCount = 1; + for (int64_t d : latShape) latCount *= static_cast(d); + if (latCount <= 1) + return fail(QStringLiteral("TripoSG: could not determine the " + "latent shape from the DiT graph.")); + + // Deterministic gaussian init (same image + seed → same mesh). + // NOT a security context — this is the diffusion latent noise; + // reproducibility REQUIRES a fixed, non-cryptographic PRNG. + latents.resize(latCount); + { + std::mt19937 rng(opts.seed); // NOSONAR - deterministic seed by design (not security) + std::normal_distribution gauss(0.0f, 1.0f); + for (float& v : latents) v = gauss(rng); + } + + const std::vector sigmas = sigmaSchedule(steps); + const bool wantCfg = opts.guidanceScale > 0.0f; + + std::vector vPred(latCount), vUncond(latCount); + for (int i = 0; i < steps; ++i) { + if (progress && !progress(Stage::Denoise, i, steps)) + return fail(QStringLiteral("cancelled")); + + // One DiT evaluation for a given conditioning buffer. Contract: + // (latents[1,2048,64], timestep[1] = 1000·σ, image_embeds + // [1,257,1024]) → velocity[1,2048,64]. CFG runs as two B=1 + // calls (the export's --verify checks this equals the doubled + // batch). + auto evalStep = [&](std::vector& condBuf, + std::vector& condShp, + std::vector& out) { + float tval = kNumTrainTimesteps + * sigmas[static_cast(i)]; + const int64_t tShape[1] = {1}; + + std::vector ins; + ins.push_back(Ort::Value::CreateTensor( + mem, latents.data(), latents.size(), latShape.data(), + latShape.size())); + ins.push_back(Ort::Value::CreateTensor( + mem, &tval, 1, tShape, 1)); + ins.push_back(Ort::Value::CreateTensor( + mem, condBuf.data(), condBuf.size(), condShp.data(), + condShp.size())); + + auto outVals = ditStep.Run( + Ort::RunOptions{nullptr}, ditIo.in.data(), ins.data(), + std::min(ins.size(), ditIo.in.size()), + ditIo.out.data(), 1); + const float* d = outVals[0].GetTensorData(); + out.assign(d, d + latCount); + }; + + evalStep(cond, condShape, vPred); + if (wantCfg) { + // Classifier-free guidance: v = v_u + s·(v_c − v_u), with + // the unconditional pass conditioned on zero embeddings. + evalStep(uncond, uncondShape, vUncond); + for (size_t k = 0; k < latCount; ++k) + vPred[k] = vUncond[k] + + opts.guidanceScale * (vPred[k] - vUncond[k]); + } + + // TripoSG RectifiedFlowScheduler update: x ← x + (σᵢ − σᵢ₊₁)·v. + // NOTE the sign — opposite of diffusers' stock FlowMatchEuler + // (TripoSG's model predicts ≈ x₀ − ε). + const float dSigma = sigmas[static_cast(i)] + - sigmas[static_cast(i) + 1]; + for (size_t k = 0; k < latCount; ++k) + latents[k] += dSigma * vPred[k]; + } + } // DiT session (the largest graph, 1.3-5.4 GB) released here + // Conditioning buffers are only consumed by the DiT. + std::vector().swap(cond); + std::vector().swap(uncond); + if (progress && !progress(Stage::Denoise, steps, steps)) + return fail(QStringLiteral("cancelled")); + + // ---- (3) VAE: precompute the latent kv-cache ONCE, then tile the ------ + // res³ grid through the per-point decoder. The kv-cache split saves + // re-running the VAE's 16-block latent self-attention stack for every + // chunk (~hundreds of chunks at high resolution). + std::vector kvCache; + std::vector kvShape; + { + Ort::Session vaeLat = open(vaeLatentsPath()); + IoNames latIo = ioNames(vaeLat, alloc); + Ort::Value latTensor = Ort::Value::CreateTensor( + mem, latents.data(), latents.size(), latShape.data(), + latShape.size()); + auto kvRes = vaeLat.Run(Ort::RunOptions{nullptr}, latIo.in.data(), + &latTensor, 1, latIo.out.data(), 1); + auto info = kvRes[0].GetTensorTypeAndShapeInfo(); + kvShape = info.GetShape(); + const float* d = kvRes[0].GetTensorData(); + kvCache.assign(d, d + info.GetElementCount()); + } // vae_latents session (~769 MB) released here + std::vector().swap(latents); + + // Only the small per-point decoder (~48 MB) stays alive for the long + // Decode/Refine tail. CPU by DEFAULT despite the ~2000 calls/run: + // every CoreML call re-uploads the 8.4 MB kv-cache input (≈17 GB of + // CPU→GPU traffic at res 256) plus per-dispatch overhead — a live GUI + // run spent >1h in the decode stage where CPU takes ~15-20 min. + // QTMESH_TRIPOSG_COREML_DECODER=1 opts back in for experiments. + const bool decOnGpu = gpuAvailable + && qEnvironmentVariableIntValue("QTMESH_TRIPOSG_COREML_DECODER") == 1; + Ort::Session vaeDec = open(vaeDecoderPath(), decOnGpu); + IoNames vaeIo = ioNames(vaeDec, alloc); + const size_t totalPts = static_cast(res) * res * res; + std::vector field(totalPts, 0.0f); + const float step = (2.0f * kRadius) / float(res - 1); + // HARD cap the chunk — the decoder cross-attends every query point to + // the 2048 kv tokens, so activation memory is linear in P with a huge + // constant (P=262144 transiently allocated ~90 GB and the OS killed + // the process). 8192 keeps each Run's transient in the hundreds of MB. + constexpr int kMaxDecoderChunk = 8192; + const int chunk = std::min( + (opts.chunkPoints > 0) ? opts.chunkPoints + : static_cast(totalPts), + kMaxDecoderChunk); + + // Chunked field sampler shared by the grid pass and the refine pass. + // The exported decoder already emits the INSIDE-POSITIVE field (raw + // SDF negated in-graph), matching our MarchingCubes convention. + std::vector chunkPts(static_cast(chunk) * 3); + auto sampleField = [&](const float* pts, size_t count, float* out, + Stage stage) -> bool { + for (size_t start = 0; start < count; + start += static_cast(chunk)) { + if (progress && !progress(stage, static_cast(start), + static_cast(count))) + return false; + const size_t n = + std::min(static_cast(chunk), count - start); + const int64_t ptShape[3] = {1, static_cast(n), 3}; + Ort::Value ptTensor = Ort::Value::CreateTensor( + mem, const_cast(pts) + start * 3, n * 3, ptShape, 3); + Ort::Value kvTensor = Ort::Value::CreateTensor( + mem, kvCache.data(), kvCache.size(), kvShape.data(), + kvShape.size()); + // Export contract: inputs (kv_cache, points). + Ort::Value ins[] = {std::move(kvTensor), std::move(ptTensor)}; + auto outVals = vaeDec.Run(Ort::RunOptions{nullptr}, + vaeIo.in.data(), ins, 2, + vaeIo.out.data(), 1); + const float* d = outVals[0].GetTensorData(); + // NEGATE into our MarchingCubes' inside-positive convention. + // The exported graph's own negation lands OUTSIDE-positive in + // practice — extracting it un-negated yields the identical + // surface with INVERTED face winding/normals (live-verified: + // GUI models rendered inside-out). The refine pass's Newton + // step is sign-agnostic (f·∇f is even), so this flip is the + // only place the sign matters. + for (size_t k = 0; k < n; ++k) + out[start + k] = -d[k]; + } + return true; + }; + + { + // Generate grid coordinates per chunk (never the full res³ buffer). + for (size_t start = 0; start < totalPts; + start += static_cast(chunk)) { + const size_t n = + std::min(static_cast(chunk), totalPts - start); + for (size_t k = 0; k < n; ++k) { + const size_t lin = start + k; + const int x = static_cast(lin % res); + const int y = static_cast((lin / res) % res); + const int z = static_cast( + lin / (static_cast(res) * res)); + chunkPts[k * 3 + 0] = -kRadius + x * step; + chunkPts[k * 3 + 1] = -kRadius + y * step; + chunkPts[k * 3 + 2] = -kRadius + z * step; + } + if (!sampleField(chunkPts.data(), n, field.data() + start, + Stage::Decode)) + return fail(QStringLiteral("cancelled")); + } + } + + // ---- (4) Surface + polish (same pipeline as TripoSR) ------------------ + const std::array gmin = {-kRadius, -kRadius, -kRadius}; + const std::array gmax = { kRadius, kRadius, kRadius}; + MarchingCubes::Mesh mc = MarchingCubes::extract( + field.data(), res, res, res, kIsoLevel, gmin, gmax, field.size()); + if (mc.vertexCount == 0 || mc.triangleCount == 0) + return fail(QStringLiteral( + "TripoSG: empty surface — try more flow steps or a cleaner " + "input image.")); + + MeshGenPredictor::Result out; + out.positions = std::move(mc.positions); + out.indices = std::move(mc.indices); + out.vertexCount = mc.vertexCount; + out.triangleCount = mc.triangleCount; + out.usedModel = true; + + if (opts.smoothMesh && opts.smoothIterations > 0) + MeshRefine::taubinSmooth(out.positions, out.indices, + opts.smoothIterations); + if (opts.refineSurface && out.vertexCount > 0) { + const size_t nv = static_cast(out.vertexCount); + const float eps = step * 0.5f; + std::vector probes(nv * 4 * 3); + for (size_t v = 0; v < nv; ++v) { + const float* p = out.positions.data() + v * 3; + float* q = probes.data() + v * 12; + for (int k = 0; k < 4; ++k) { + q[k * 3 + 0] = p[0]; + q[k * 3 + 1] = p[1]; + q[k * 3 + 2] = p[2]; + } + q[3] += eps; q[7] += eps; q[11] += eps; + } + std::vector dens(nv * 4); + if (!sampleField(probes.data(), nv * 4, dens.data(), Stage::Refine)) + return fail(QStringLiteral("cancelled")); + std::vector f(nv), grad(nv * 3); + for (size_t v = 0; v < nv; ++v) { + const float f0 = dens[v * 4 + 0]; + f[v] = f0; + grad[v * 3 + 0] = (dens[v * 4 + 1] - f0) / eps; + grad[v * 3 + 1] = (dens[v * 4 + 2] - f0) / eps; + grad[v * 3 + 2] = (dens[v * 4 + 3] - f0) / eps; + } + MeshRefine::isoProjectStep(out.positions, f, grad, step); + } + + // TripoSG is geometry-only: no colours, no uvs — the caller decides + // how to dress it (neutral material / future input-image projection). + out.ok = true; + return out; + } catch (const Ort::Exception& e) { + return fail(QStringLiteral("TripoSG ONNX error: %1") + .arg(QString::fromUtf8(e.what()))); + } catch (const std::exception& e) { + return fail(QStringLiteral("TripoSG error: %1") + .arg(QString::fromUtf8(e.what()))); + } +} + +#endif // ENABLE_ONNX diff --git a/src/ImageTo3D/TripoSGPredictor.h b/src/ImageTo3D/TripoSGPredictor.h new file mode 100644 index 00000000..229fd825 --- /dev/null +++ b/src/ImageTo3D/TripoSGPredictor.h @@ -0,0 +1,110 @@ +#ifndef TRIPOSG_PREDICTOR_H +#define TRIPOSG_PREDICTOR_H + +#include "MeshGenPredictor.h" // shared Result / Stage / ProgressFn contract + +#include +#include + +// TripoSG single-image → 3D geometry backend (follow-up to epic #764). +// +// TripoSG (VAST-AI-Research, SIGGRAPH 2025, "TripoSG: High-Fidelity 3D Shape +// Synthesis using Large-Scale Rectified Flow Models" — MIT code AND MIT +// weights on HF `VAST-AI/TripoSG`) is a 1.5B rectified-flow DiT over an SDF +// VAE's vecset latents. Reported geometry quality ≈ commercial Tripo 2.0 +// (Normal-FID 5.81 vs ~20 for TripoSR-class LRMs) — a full generation ahead +// of the TripoSR fast tier. Same org whose UniRig (#408) we already ship; +// license audit in docs/IMAGE_TO_3D_QUALITY.md. +// +// **Four ONNX graphs** (exported by scripts/export-triposg-onnx.py; the full +// measured contract lives in docs/TRIPOSG_EXPORT_NOTES.md): +// * triposg_image_encoder.onnx — image [1,3,224,224] (raw [0,1] RGB; the +// DINOv2-large mean/std preprocessing is baked into the graph) → +// image_embeds [1,257,1024]. CFG's unconditional embedding is simply +// ZEROS of the same shape (upstream uses zeros_like), so no second output. +// * triposg_dit_step.onnx (+ .onnx.data sidecar — the fp32 graph is >2 GB; +// int8 variant triposg_dit_step_int8.onnx is a single smaller file) — +// ONE denoising step: (latents[B,2048,64], timestep[B]=1000·σ, +// image_embeds[B,257,1024]) → velocity[B,2048,64]. The rectified-flow +// Euler loop runs HERE in C++ (same hand-rolled-loop stance as UniRig's +// KV-cache decode): σᵢ = 1 − i/N, σ_N = 0, and the update is +// latents += (σᵢ − σᵢ₊₁)·v — note the sign is the OPPOSITE of diffusers' +// stock FlowMatchEuler (TripoSG's custom RectifiedFlowScheduler). +// * triposg_vae_latents.onnx — latents → kv_cache [1,2048,1024]; the VAE's +// 16-block latent self-attention stack, run ONCE per generation so the +// per-chunk decode below doesn't re-pay it (~340 chunks at 257³). +// * triposg_vae_decoder.onnx — (kv_cache, points [1,P,3]) → field [1,P,1], +// already negated to our INSIDE-POSITIVE convention, iso 0, bounds +// (−1.005, 1.005)³. Tiled through in chunks exactly like the TripoSR +// decoder; surface extracted with the SAME native MarchingCubes. +// +// TripoSG is GEOMETRY-ONLY (no colour decoder): the texture-bake stage is +// skipped and the result carries no colours/uvs — pair it with the input-image +// projection follow-up for colour. The smoothing/reprojection quality passes +// apply unchanged (the field sampler is the VAE decoder). +// +// Whole file is ENABLE_ONNX-gated like MeshGenPredictor; models download on +// first use from the hosted models repo (override QTMESH_TRIPOSG_MODEL_BASE_URL +// / QSettings ai/triposgModelBaseUrl; offline guard QTMESH_TRIPOSG_NO_DOWNLOAD) +// and every surface reports a clean "not yet hosted" error when absent. +class TripoSGPredictor { +public: + struct Options { + Options(); + int sdfResolution = 256; // marching-cubes grid resolution + // Rectified-flow Euler steps. TripoSG's reference inference uses 50; + // rectified flow degrades gracefully at lower counts — 25 is a good + // CPU default, 10 a fast preview. + int flowSteps = 25; + // Classifier-free guidance scale (reference default 7.0). 0 disables + // the unconditional pass (halves DiT cost, softer geometry). + float guidanceScale = 7.0f; + // Deterministic latent seed — same image + params → same mesh. + unsigned seed = 42; + // Use the int8-quantized DiT step graph (~1.5 GB single file) instead + // of fp32 (~5.8 GB as .onnx + .onnx.data) — the TripoSR-style + // size/quality tier. + bool useInt8Dit = false; + // Post-extraction polish (same semantics as MeshGenPredictor). + bool smoothMesh = true; + int smoothIterations = 6; + bool refineSurface = true; + // Decoder query-point chunk size (bounds memory on the res³ grid). + // MUCH smaller than TripoSR's (262144): TripoSG's decoder is a + // CROSS-ATTENTION from every query point to the 2048 kv tokens, so + // per-Run activation memory scales with P × 2048 × heads — 256k + // points/chunk materialised ~90 GB of attention logits and got the + // process killed. 8192 keeps the transient under a few hundred MB. + // predict() clamps to this cap regardless of what the caller passes. + int chunkPoints = 8192; + }; + + // True only when built with ENABLE_ONNX. + static bool isAvailable(); + + // AppData/ai_models/triposg/ paths for the graphs. + static QString imageEncoderPath(); + static QString ditStepPath(bool int8Tier = false); + static QString vaeLatentsPath(); + static QString vaeDecoderPath(); + + // True when every graph of the chosen DiT tier exists on disk (for the + // fp32 tier this includes the .onnx.data external-weights sidecar). + static bool modelsPresent(bool int8Tier = false); + + // Ensure all graphs exist, downloading any missing one on first use + // (blocks via a local event loop — call on a thread WITH an event loop). + // Returns the encoder path when everything is present, else empty. + static QString ensureModelBlocking(bool int8Tier = false); + + // Run the full TripoSG pipeline. Same Result/Stage/ProgressFn contract as + // MeshGenPredictor::predict (Stage::Encode covers the image encoder, + // Stage::Denoise the flow loop, Stage::Decode the grid decode, + // Stage::Refine the reprojection pass). Never throws. + static MeshGenPredictor::Result predict( + const QImage& image, + const Options& opts = {}, + const MeshGenPredictor::ProgressFn& progress = {}); +}; + +#endif // TRIPOSG_PREDICTOR_H diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 16482f48..1eb506ef 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -18,6 +18,7 @@ #include "MeshImporterExporter.h" #include "CLIPipeline.h" #include "ImageTo3D/MeshGenPredictor.h" +#include "ImageTo3D/TripoSGPredictor.h" #include "ImageTo3D/MeshGenBuilder.h" #include "OgreWidget.h" #include "SpaceCamera.h" @@ -2422,17 +2423,52 @@ QJsonObject MCPServer::toolGenerateMeshFromImage(const QJsonObject &args) const bool upscaleTex = args.value("upscale_texture").toBool(); const bool generatePbr = args.contains("generate_pbr") ? args["generate_pbr"].toBool(true) : true; + if (args.contains("backend")) { + const QString b = args["backend"].toString().toLower(); + if (b == "triposg") opts.backend = MeshGenPredictor::Backend::TripoSG; + else if (b == "triposr" || b.isEmpty()) + opts.backend = MeshGenPredictor::Backend::TripoSR; + else return makeErrorResult("'backend' must be 'triposr' or 'triposg'."); + } + if (args.contains("flow_steps")) { + opts.flowSteps = args["flow_steps"].toInt(25); + if (opts.flowSteps < 1 || opts.flowSteps > 200) + return makeErrorResult("'flow_steps' must be between 1 and 200."); + } + if (args.contains("guidance")) { + opts.guidanceScale = static_cast(args["guidance"].toDouble(7.0)); + if (opts.guidanceScale < 0.0f || opts.guidanceScale > 30.0f) + return makeErrorResult("'guidance' must be between 0 and 30."); + } SentryReporter::addBreadcrumb(QStringLiteral("ai.tool_call"), - QStringLiteral("generate_mesh_from_image %1 res=%2") - .arg(QFileInfo(imagePath).fileName()).arg(opts.sdfResolution)); - - const QString enc = MeshGenPredictor::ensureModelBlocking(opts.quality); - if (enc.isEmpty() || !MeshGenPredictor::modelsPresent(opts.quality)) - return makeErrorResult( - "TripoSR model unavailable — it downloads on first use; if it is not " - "hosted yet, set QTMESH_TRIPOSR_MODEL_BASE_URL / ai/triposrModelBaseUrl " - "or drop the files in the ai_models/triposr/ cache."); + QStringLiteral("generate_mesh_from_image %1 res=%2 backend=%3") + .arg(QFileInfo(imagePath).fileName()).arg(opts.sdfResolution) + .arg(opts.backend == MeshGenPredictor::Backend::TripoSG + ? QStringLiteral("triposg") : QStringLiteral("triposr"))); + + if (opts.backend == MeshGenPredictor::Backend::TripoSG) { + // TripoSG always runs the fp32 DiT (int8 tier dropped — degraded + // geometry, no ARM speed win); 'quality' still selects the TripoSR + // tier used for the colour bake. + const QString enc = TripoSGPredictor::ensureModelBlocking(false); + if (enc.isEmpty()) + return makeErrorResult( + "TripoSG models unavailable — they download on first use; if not " + "hosted yet, set QTMESH_TRIPOSG_MODEL_BASE_URL / ai/triposgModelBaseUrl " + "or drop the files in the ai_models/triposg/ cache."); + // TripoSG's colour bake queries TripoSR's colour field — best-effort + // ensure (absent models fall back to clay with a warning). + if (opts.bakeTexture) + MeshGenPredictor::ensureModelBlocking(opts.quality); + } else { + const QString enc = MeshGenPredictor::ensureModelBlocking(opts.quality); + if (enc.isEmpty() || !MeshGenPredictor::modelsPresent(opts.quality)) + return makeErrorResult( + "TripoSR model unavailable — it downloads on first use; if it is not " + "hosted yet, set QTMESH_TRIPOSR_MODEL_BASE_URL / ai/triposrModelBaseUrl " + "or drop the files in the ai_models/triposr/ cache."); + } QImage image(imagePath); if (image.isNull()) @@ -7299,16 +7335,19 @@ QJsonArray MCPServer::buildToolsList() QJsonObject props; props["image_path"] = QJsonObject{{"type", "string"}, {"description", "Absolute path to the source image (a single object, ideally background-removed). Required."}}; props["output"] = QJsonObject{{"type", "string"}, {"description", "Optional path to save the generated mesh (e.g. /tmp/out.glb). If omitted, the mesh is loaded into the current scene instead."}}; - props["resolution"] = QJsonObject{{"type", "integer"}, {"description", "Marching-cubes grid resolution 16..1024 (default 256; 128 is a fast/preview tier). Higher = more detail + slower. Cost is res^3 floats in RAM: 512~=0.5 GB, 768~=1.7 GB, 1024~=4.3 GB. The encoder input is fixed at 512^2, so detail gains taper off above 512."}}; - props["vertex_color"] = QJsonObject{{"type", "boolean"}, {"description", "Bake TripoSR's predicted per-vertex color (default true)."}}; - props["remove_bg"] = QJsonObject{{"type", "boolean"}, {"description", "Run U²-Net background removal on the image first (default false). Recommended for photos with a background; TripoSR needs an isolated subject. Falls back to the raw image if the model is unavailable."}}; - props["quality"] = QJsonObject{{"type", "string"}, {"enum", QJsonArray{"fp32", "int8"}}, {"description", "Encoder precision/size tier (default fp32). fp32 = best (~1.7GB), int8 = smallest, slight quality loss (~430MB). The chosen tier downloads on demand."}}; + props["resolution"] = QJsonObject{{"type", "integer"}, {"description", "Marching-cubes grid resolution 16..1024 (default 256; 128 is a fast/preview tier). Higher = more detail + slower. Cost is res^3 floats in RAM: 512~=0.5 GB, 768~=1.7 GB, 1024~=4.3 GB. (TripoSR's encoder input is fixed at 512^2, so its detail gains taper off above 512; TripoSG uses a 224^2 DINOv2 encoder and this is purely the extraction grid.)"}}; + props["vertex_color"] = QJsonObject{{"type", "boolean"}, {"description", "TripoSR only: bake its predicted per-vertex color (default true). Ignored by triposg (geometry-only — colour comes from the AI texture pass)."}}; + props["remove_bg"] = QJsonObject{{"type", "boolean"}, {"description", "Run U²-Net background removal on the image first (default false). Recommended for photos with a background; the model needs an isolated subject. Falls back to the raw image if the model is unavailable."}}; + props["quality"] = QJsonObject{{"type", "string"}, {"enum", QJsonArray{"fp32", "int8"}}, {"description", "Precision/size tier, downloaded on demand. TripoSR: fp32 = best (~1.7GB), int8 = smallest with slight quality loss (~430MB). TripoSG: fp32 only (int8 degrades geometry) — int8 is silently upgraded to fp32."}}; props["smooth"] = QJsonObject{{"type", "boolean"}, {"description", "Taubin-smooth the extracted mesh to remove marching-cubes stair-stepping (default true; volume-preserving)."}}; props["refine"] = QJsonObject{{"type", "boolean"}, {"description", "After smoothing, Newton-project each vertex back onto the network's true iso-surface via extra decoder queries (default true; recovers grid-quantized detail)."}}; - props["bake_texture"] = QJsonObject{{"type", "boolean"}, {"description", "Bake a real diffuse texture (xatlas unwrap + per-texel decoder color) instead of per-vertex colors (default true; falls back to vertex colors if the bake fails)."}}; + props["bake_texture"] = QJsonObject{{"type", "boolean"}, {"description", "TripoSR only: bake a real diffuse texture (xatlas unwrap + per-texel decoder color) instead of per-vertex colors (default true; falls back to vertex colors if the bake fails). Ignored by triposg (its colour comes from the GUI AI-texture pass; the CLI/MCP triposg mesh is geometry-only)."}}; props["texture_size"] = QJsonObject{{"type", "integer"}, {"description", "Baked-texture resolution 64..8192 (default 1024)."}}; props["upscale_texture"] = QJsonObject{{"type", "boolean"}, {"description", "Run Real-ESRGAN 2x on the baked diffuse before saving (default false; best-effort — keeps the un-upscaled texture if the upscale model is unavailable)."}}; props["generate_pbr"] = QJsonObject{{"type", "boolean"}, {"description", "Synthesize normal + roughness maps from the baked diffuse (#404 PBRify) and bind them into the material — the polished-surface look (default true; requires bake_texture; fails soft to diffuse-only if the models are unavailable)."}}; + props["backend"] = QJsonObject{{"type", "string"}, {"enum", QJsonArray{"triposr", "triposg"}}, {"description", "Generation backend (default triposr). triposr = fast single-pass LRM with color; triposg = 1.5B rectified-flow model — higher-fidelity GEOMETRY, slower, geometry-only (no texture bake). Both MIT. TripoSG models download on first use."}}; + props["flow_steps"] = QJsonObject{{"type", "integer"}, {"description", "TripoSG rectified-flow Euler steps 1..200 (default 25; 50 = reference quality, 10 = fast preview). Ignored by triposr."}}; + props["guidance"] = QJsonObject{{"type", "number"}, {"description", "TripoSG classifier-free-guidance scale 0..30 (default 7; 0 disables CFG and halves DiT cost). Ignored by triposr."}}; appendTool( "generate_mesh_from_image", "AI image-to-3D mesh generation (epic #764, TripoSR via ONNX): " diff --git a/src/MaterialEditorQML.cpp b/src/MaterialEditorQML.cpp index 74765bad..3117fc56 100644 --- a/src/MaterialEditorQML.cpp +++ b/src/MaterialEditorQML.cpp @@ -9,6 +9,11 @@ #include "MeshDepthRenderer.h" #include "MultiViewTextureBaker.h" #include "TexturePaintBuffer.h" +#include "ImageTo3D/BackgroundRemover.h" +#include "ImageTo3D/ImageCaptioner.h" +#include "UvUnwrap.h" +#include "EditableMesh.h" +#include "MeshDecimator.h" // cap the texture-path mesh before unwrap #include "EmbeddedTextureCache.h" #include #include @@ -4571,6 +4576,10 @@ struct MaterialEditorQML::MultiViewBakeState { std::vector views; // resolved camera views std::vector baked; // accumulated image+matrices size_t current = 0; // index of the view being generated + // Synthesize + bind #404 PBR maps (normal + roughness) FROM the freshly + // baked atlas after the bake applies it — so the maps match the final + // (AI) diffuse, not a throwaway one. Set by the generate3d GUI flow. + bool generatePbrAfter = false; }; namespace { @@ -4584,17 +4593,44 @@ MeshDepthRenderer::View resolveDepthView(const QString& name) if (n == "bottom") return MeshDepthRenderer::bottom(); return MeshDepthRenderer::front(); } + +// True if every submesh of the entity's mesh exposes a UV0 (TEXCOORD 0) in +// whichever vertex data it uses. The multi-view baker projects onto an +// EXISTING UV0 atlas, so a UV-less mesh (e.g. a geometry-only TripoSG result) +// must be unwrapped first. +bool entityHasUv0(Ogre::Entity* entity) +{ + if (!entity || !entity->getMesh()) return false; + const Ogre::MeshPtr& mesh = entity->getMesh(); + for (unsigned short i = 0; i < mesh->getNumSubMeshes(); ++i) { + const Ogre::SubMesh* sm = mesh->getSubMesh(i); + const Ogre::VertexData* vd = sm->useSharedVertices + ? mesh->sharedVertexData : sm->vertexData; + if (!vd || !vd->vertexDeclaration + || !vd->vertexDeclaration->findElementBySemantic( + Ogre::VES_TEXTURE_COORDINATES, 0)) + return false; + } + return true; +} + } // namespace void MaterialEditorQML::generateMeshTextureMultiView(const QString &prompt, int width, int height, double controlStrength, - const QStringList &views) -{ - if (prompt.isEmpty()) { - emit sdGenerationError("Please enter a texture prompt"); - return; - } + const QStringList &views, + const QString &frontPhotoPath, + bool generatePbr) +{ + // No hard requirement for a prompt: the describe-then-generate flow may + // arrive with an empty caption (model not yet downloaded / captioner + // unavailable), and the code below falls back to a neutral texture prompt. + // Just note it so the user knows the result is generic rather than + // image-described. + if (prompt.isEmpty() && frontPhotoPath.isEmpty()) + emit sdGenerationNotice(tr("No image description available — using a " + "neutral texture prompt.")); #ifdef ENABLE_STABLE_DIFFUSION SDManager *sdManager = SDManager::instance(); if (!sdManager->isModelLoaded()) { @@ -4616,7 +4652,36 @@ void MaterialEditorQML::generateMeshTextureMultiView(const QString &prompt, // LCOV_EXCL_START — requires a loaded SD model + a mesh auto st = std::make_unique(); st->entityName = QString::fromStdString(entity->getName()); - st->prompt = prompt; + + // Describe-then-generate: when an input photo is supplied (the image-to-3D + // path) and no explicit prompt was given, CAPTION the photo with the local + // vision model (SmolVLM) and use that as the prompt for EVERY view. This + // replaces the old "pin the raw photo as the front view" approach — all + // views are now SD-generated from the caption + depth, so there's no + // photo-projection orientation/registration artifact, and front + back are + // stylistically consistent (shared caption + locked seed). + QString effectivePrompt = prompt; + if (effectivePrompt.isEmpty() && !frontPhotoPath.isEmpty() + && ImageCaptioner::isAvailable()) { + emit sdGenerationNotice(tr("Describing the image…")); + const QString capModel = ImageCaptioner::ensureModelBlocking(); + if (!capModel.isEmpty()) { + QImage p(frontPhotoPath); + const QString cap = p.isNull() ? QString() + : ImageCaptioner::caption(p); + if (!cap.isEmpty()) { + effectivePrompt = cap; + emit sdGenerationNotice(tr("Image description: \"%1\"").arg(cap)); + } + } + if (effectivePrompt.isEmpty()) + emit sdGenerationNotice(tr("Image captioning unavailable — using a " + "neutral texture prompt.")); + } + st->prompt = effectivePrompt.isEmpty() + ? QStringLiteral("high quality seamless PBR texture, consistent colours, " + "matching the subject, even lighting") + : effectivePrompt; st->width = width > 0 ? width : 512; st->height = height > 0 ? height : 512; st->controlStrength = static_cast(std::clamp(controlStrength, 0.0, 1.0)); @@ -4633,6 +4698,10 @@ void MaterialEditorQML::generateMeshTextureMultiView(const QString &prompt, for (const QString& vn : viewNames) st->views.push_back(resolveDepthView(vn)); + // Describe-then-generate: every view is SD-generated from the caption + // above (no raw-photo pinning), so nothing else to stash here. + st->generatePbrAfter = generatePbr; + if (st->controlNetPath.isEmpty()) { emit sdGenerationNotice( "No ControlNet depth model found — multi-view bake will follow the " @@ -4692,6 +4761,7 @@ void MaterialEditorQML::startNextMultiViewGeneration() MultiViewTextureBaker::View bv; bv.viewProj = rr.projMatrix * rr.viewMatrix; bv.camDirection = rr.camDirection; + s.baked.push_back(bv); // image filled on completion m_sdGenerationProgress = 0.0f; @@ -4718,6 +4788,72 @@ void MaterialEditorQML::finishMultiViewBake() } if (!entity) { emit sdGenerationError("Mesh gone before bake."); emit sdIsGeneratingChanged(); return; } + // The baker projects onto an EXISTING UV0 atlas. A geometry-only mesh + // (TripoSG, or any unwrap-less import) has none, so auto-unwrap it in place + // first via xatlas. Safe here because these are STATIC generated meshes + // (no skeleton — the in-place-mutation caveat that unwrapEntity warns about + // only affects live skinned meshes), and they SHOULD keep the new UVs (the + // texture binds to them and they're carried into any export). + if (!entityHasUv0(entity)) { + // Clean degenerate (zero/near-zero-area) triangles FIRST. Marching + // cubes + Taubin smoothing (image-to-3D) leave slivers that xatlas + // can't parameterize — they collapse to lines in UV and produce the + // "streaks across the atlas + untextured patches" symptom. Removing + // them (plus the proportional weld epsilon inside unwrapEntity) gives + // xatlas clean charts. + { + EditableMesh em; + if (em.loadFromEntity(entity)) { + const int removed = em.removeDegenerateTriangles(1e-5f); + if (removed > 0) { + em.commitToEntity(entity); + emit sdGenerationNotice( + tr("Cleaned %1 degenerate triangles before unwrap.") + .arg(removed)); + } + } + } + // Decimate BEFORE unwrap. Image-to-3D meshes are 80k+ tris; xatlas + // (single-threaded, to avoid its scheduler deadlock) takes many minutes + // on that — effectively a freeze. A texture doesn't need that density: + // ~90% reduction (user-verified as visually fine) drops it to a few + // thousand tris that unwrap + bake in seconds. Reduction is capped for + // meshes that are already small so we don't over-decimate them. + { + EditableMesh em; + int tris = 0; + if (em.loadFromEntity(entity)) + tris = static_cast(em.totalTriangleCount()); // post-cleanup + // Decimate to a fixed TRIANGLE BUDGET (more robust than a flat 90%: + // a huge mesh is reduced hard, a moderate one gently, and small + // meshes are left alone). ~16k tris textures cleanly and unwraps in + // seconds; user-verified ~90% reduction (≈ an 80k→16k mesh) looks + // fine. reductionFromTargetTris returns 0 when already under budget. + constexpr int kTextureTriBudget = 16000; + const double reduction = + MeshDecimator::reductionFromTargetTris(tris, kTextureTriBudget); + if (reduction > 0.01) { + emit sdGenerationNotice( + tr("Simplifying mesh for texturing (%1 → ~%2 tris)…") + .arg(tris).arg(kTextureTriBudget)); + // Meshopt backend, NOT Ogre's MeshLodGenerator — the latter + // SIGSEGVs on these dense image-to-3D meshes (verified); meshopt + // handles them cleanly. + MeshDecimator::decimateEntity(entity, reduction, + MeshDecimator::Algorithm::Meshopt); + } + } + emit sdGenerationNotice("Auto-unwrapping UVs before bake…"); + const UvUnwrapReport ur = UvUnwrap::unwrapEntity(entity); + if (!ur.applied || !entityHasUv0(entity)) { + emit sdGenerationError(QStringLiteral( + "AI texture: auto UV unwrap failed%1 — cannot bake.") + .arg(ur.error.isEmpty() ? QString() : (": " + ur.error))); + emit sdIsGeneratingChanged(); + return; + } + } + QString geoErr; std::vector tris = MultiViewTextureBaker::fromEntity(entity, &geoErr); @@ -4776,6 +4912,13 @@ void MaterialEditorQML::finishMultiViewBake() SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.mesh_texture_multiview"), QStringLiteral("baked %1 texels from %2 views (%3 dilated)") .arg(rep.pixelsWritten).arg(s->baked.size()).arg(rep.pixelsDilated)); + + // PBR AFTER the AI texture (issue: PBR was previously synthesized during + // mesh build from a throwaway diffuse, then the AI bake replaced the + // diffuse — leaving normal/roughness that didn't match the colours). Now + // synthesize + bind from the freshly-applied AI diffuse so the maps match. + if (s->generatePbrAfter && aiPbrAvailable()) + generatePbrFromDiffuse(); #endif } @@ -4804,6 +4947,19 @@ void MaterialEditorQML::onSDGenerationProgress() int total = sdManager->generationTotalSteps(); m_sdGenerationProgress = (total > 0) ? static_cast(step) / total : 0.0f; emit sdGenerationProgressChanged(); + // During a multi-view bake, surface an unambiguous "view N/M — step X/Y" + // status so the (multi-minute) SD generation clearly reads as ALIVE rather + // than frozen. m_multiViewBake->current is the view whose image is pending. + if (m_multiViewBake && total > 0) { + const int nViews = static_cast(m_multiViewBake->views.size()); + const int viewIdx = static_cast(m_multiViewBake->current) + 1; + const QString vn = (m_multiViewBake->current < m_multiViewBake->views.size()) + ? QString::fromLatin1(m_multiViewBake->views[m_multiViewBake->current].name) + : QString(); + emit sdGenerationNotice( + tr("AI texture: %1 view %2/%3 — step %4/%5") + .arg(vn).arg(viewIdx).arg(nViews).arg(step).arg(total)); + } } void MaterialEditorQML::applyTextureToEntityDiffuse(const QString& entityName, diff --git a/src/MaterialEditorQML.h b/src/MaterialEditorQML.h index b9412200..eb79c2dc 100644 --- a/src/MaterialEditorQML.h +++ b/src/MaterialEditorQML.h @@ -606,7 +606,9 @@ public slots: Q_INVOKABLE void generateMeshTextureMultiView(const QString &prompt, int width = 0, int height = 0, double controlStrength = 0.9, - const QStringList &views = {}); + const QStringList &views = {}, + const QString &frontPhotoPath = {}, + bool generatePbr = false); // True when a skinned/static mesh is selected — drives the // "use selected mesh" checkbox enabled state. Q_INVOKABLE bool hasSelectedMesh() const; diff --git a/src/MeshImporterExporter.cpp b/src/MeshImporterExporter.cpp index 30cde991..e214e334 100755 --- a/src/MeshImporterExporter.cpp +++ b/src/MeshImporterExporter.cpp @@ -44,7 +44,10 @@ THE SOFTWARE. #include #include #include +#include #include +#include +#include #include #include #include @@ -766,6 +769,159 @@ static std::vector compactAiMesh(aiMesh* aiM) return remap; } +// Split any aiMesh with > 65535 vertices into ≤65535-vertex chunks before +// export. Assimp's glTF2/OBJ/FBX exporters truncate the VERTEX accessors at +// 65536 while keeping the 32-bit index buffer intact — so indices ≥65536 +// dangle and a large generated mesh (e.g. TripoSG/TripoSR at high resolution, +// 80k+ verts) reimports torn into pieces. Splitting per-triangle-group into +// sub-65536-vertex meshes sidesteps the exporter bug entirely; each chunk owns +// a compact, valid vertex range. Faces are the split unit (each face's 3 verts +// are copied into the chunk with local indices), so no index exceeds the limit. +// +// SCOPE: handles POSITION / NORMAL / TEXCOORD0 / COLOR0 meshes — the generated +// image-to-3D output. Meshes carrying BONES or ANIM(morph) meshes are left +// UNTOUCHED (returned as-is): those need the weight/target arrays remapped per +// chunk, which is a heavier follow-up; skinned meshes rarely exceed the limit +// per submesh in practice. Returns the (possibly multiple) replacement meshes; +// the caller rebuilds scene->mMeshes and the node index arrays. +static std::vector splitLargeAiMesh(aiMesh* aiM) +{ + constexpr unsigned int kMaxVerts = 65535; + if (!aiM || aiM->mNumVertices <= kMaxVerts || aiM->mNumFaces == 0) + return { aiM }; + // Don't risk skinned / morph meshes — leave them for a dedicated pass. + if (aiM->mNumBones > 0 || aiM->mNumAnimMeshes > 0) + return { aiM }; + // Only split all-triangle meshes. readSubmeshGeometry deliberately emits + // quads / n-gons from the cached qtme.faces list; the per-face repack + // below is triangle-only, so splitting a polygon mesh would DROP those + // faces. A >65k-vertex quad/n-gon mesh is rare — leave it intact (correct, + // if still subject to the Assimp exporter cap) rather than lose geometry. + for (unsigned int f = 0; f < aiM->mNumFaces; ++f) + if (aiM->mFaces[f].mNumIndices != 3) + return { aiM }; + + const bool hasN = aiM->mNormals != nullptr; + const bool hasUV = aiM->mTextureCoords[0] != nullptr; + const bool hasC = aiM->mColors[0] != nullptr; + + std::vector out; + unsigned int faceStart = 0; + while (faceStart < aiM->mNumFaces) { + // Greedily pack faces into a chunk until adding one more would exceed + // kMaxVerts unique vertices. Each face contributes ≤3 new verts. + std::vector srcVerts; // chunk-local → source vertex id + std::unordered_map remap; // source → local + std::vector> faces; + unsigned int f = faceStart; + for (; f < aiM->mNumFaces; ++f) { + const aiFace& face = aiM->mFaces[f]; + if (face.mNumIndices != 3) continue; // triangulated on read + // Would this face push us over the limit? (≤3 new verts.) + if (srcVerts.size() + 3 > kMaxVerts && !faces.empty()) + break; + std::array lf{}; + for (int v = 0; v < 3; ++v) { + const unsigned int s = face.mIndices[v]; + auto it = remap.find(s); + unsigned int local; + if (it == remap.end()) { + local = static_cast(srcVerts.size()); + remap.emplace(s, local); + srcVerts.push_back(s); + } else { + local = it->second; + } + lf[v] = local; + } + faces.push_back(lf); + } + + auto* chunk = new aiMesh(); + chunk->mMaterialIndex = aiM->mMaterialIndex; + chunk->mPrimitiveTypes = aiPrimitiveType_TRIANGLE; + chunk->mNumVertices = static_cast(srcVerts.size()); + chunk->mVertices = new aiVector3D[chunk->mNumVertices]; + if (hasN) chunk->mNormals = new aiVector3D[chunk->mNumVertices]; + if (hasUV) { + chunk->mNumUVComponents[0] = aiM->mNumUVComponents[0]; + chunk->mTextureCoords[0] = new aiVector3D[chunk->mNumVertices]; + } + if (hasC) chunk->mColors[0] = new aiColor4D[chunk->mNumVertices]; + for (unsigned int i = 0; i < chunk->mNumVertices; ++i) { + const unsigned int s = srcVerts[i]; + chunk->mVertices[i] = aiM->mVertices[s]; + if (hasN) chunk->mNormals[i] = aiM->mNormals[s]; + if (hasUV) chunk->mTextureCoords[0][i] = aiM->mTextureCoords[0][s]; + if (hasC) chunk->mColors[0][i] = aiM->mColors[0][s]; + } + chunk->mNumFaces = static_cast(faces.size()); + chunk->mFaces = new aiFace[chunk->mNumFaces]; + for (unsigned int i = 0; i < chunk->mNumFaces; ++i) { + chunk->mFaces[i].mNumIndices = 3; + chunk->mFaces[i].mIndices = new unsigned int[3]; + for (int v = 0; v < 3; ++v) + chunk->mFaces[i].mIndices[v] = faces[i][v]; + } + out.push_back(chunk); + faceStart = f; + } + + delete aiM; // replaced by the chunks + return out; +} + +// Rewrite scene->mMeshes (and every node's mMeshes index array) so any mesh +// exceeding the 65535-vertex exporter limit is replaced by its split chunks. +// No-op when nothing exceeds the limit (the common case). +static void splitLargeMeshesForExport(aiScene* scene) +{ + if (!scene || scene->mNumMeshes == 0) return; + bool anyOver = false; + for (unsigned int i = 0; i < scene->mNumMeshes; ++i) + if (scene->mMeshes[i] && scene->mMeshes[i]->mNumVertices > 65535 + && scene->mMeshes[i]->mNumBones == 0 + && scene->mMeshes[i]->mNumAnimMeshes == 0) { anyOver = true; break; } + if (!anyOver) return; + + // Build the new mesh list, recording each old index's replacement range. + std::vector newMeshes; + std::vector> oldToNew(scene->mNumMeshes); + for (unsigned int i = 0; i < scene->mNumMeshes; ++i) { + std::vector parts = splitLargeAiMesh(scene->mMeshes[i]); + for (aiMesh* p : parts) { + oldToNew[i].push_back(static_cast(newMeshes.size())); + newMeshes.push_back(p); + } + } + delete[] scene->mMeshes; + scene->mNumMeshes = static_cast(newMeshes.size()); + scene->mMeshes = new aiMesh*[scene->mNumMeshes]; + for (unsigned int i = 0; i < scene->mNumMeshes; ++i) + scene->mMeshes[i] = newMeshes[i]; + + // Fix up every node's mesh-index array to point at the new indices. + std::function fixNode = [&](aiNode* n) { + if (!n) return; + if (n->mNumMeshes > 0) { + std::vector mapped; + for (unsigned int k = 0; k < n->mNumMeshes; ++k) { + const unsigned int old = n->mMeshes[k]; + if (old < oldToNew.size()) + for (unsigned int ni : oldToNew[old]) mapped.push_back(ni); + } + delete[] n->mMeshes; + n->mNumMeshes = static_cast(mapped.size()); + n->mMeshes = new unsigned int[n->mNumMeshes]; + for (unsigned int k = 0; k < n->mNumMeshes; ++k) + n->mMeshes[k] = mapped[k]; + } + for (unsigned int c = 0; c < n->mNumChildren; ++c) + fixNode(n->mChildren[c]); + }; + fixNode(scene->mRootNode); +} + // Slice A4a: emit per-pose aiAnimMesh entries on an aiMesh, sourced // from the Ogre mesh's pose list filtered to the matching submesh. // `aiAnimMesh::mVertices` carries absolute deformed positions (base @@ -3254,6 +3410,9 @@ int MeshImporterExporter::exporter(const Ogre::SceneNode *_sn, const QString &_u // positions + normals by column, not by UV lookup). if (formatId == "gltf2" || formatId == "glb2") exportFlags |= aiProcess_FlipUVs; + // Split >65535-vertex meshes so Assimp's exporters don't truncate + // the vertex accessors at 65536 (which tears large meshes apart). + splitLargeMeshesForExport(scene); aiReturn result = exporter.Export(scene, formatId.toStdString().c_str(), file.filePath().toStdString().c_str(), exportFlags); @@ -3535,6 +3694,8 @@ int MeshImporterExporter::exportCurrentPose(Ogre::Entity* entity, const QString& // Only apply ConvertToLeftHanded for formats that expect left-handed coords (e.g. FBX). bool rightHanded = (formatId == "x" || formatId == "gltf2" || formatId == "glb2"); unsigned int exportFlags = rightHanded ? 0 : aiProcess_ConvertToLeftHanded; + // Split >65535-vertex meshes (Assimp exporter vertex-accessor cap). + splitLargeMeshesForExport(scene); aiReturn aiResult = exporter.Export(scene, formatId.toStdString().c_str(), file.filePath().toStdString().c_str(), exportFlags); @@ -3819,6 +3980,8 @@ int MeshImporterExporter::sceneExporter(const QString &_uri, const ProgressCallb Assimp::Exporter exporter; + // Split >65535-vertex meshes (Assimp exporter vertex-accessor cap). + splitLargeMeshesForExport(scene); // Both Ogre and glTF are right-handed — no ConvertToLeftHanded aiReturn result = exporter.Export(scene, formatId.toStdString().c_str(), file.filePath().toStdString().c_str(), 0); diff --git a/src/UvUnwrap.cpp b/src/UvUnwrap.cpp index 5dd84ff7..859d1e42 100644 --- a/src/UvUnwrap.cpp +++ b/src/UvUnwrap.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -613,6 +614,26 @@ void restoreMesh(Ogre::Mesh* mesh, MeshSnapshot&& snap) namespace { +// A weld epsilon proportional to the mesh's size, so xatlas merges the +// near-coincident vertices that marching-cubes + smoothing (image-to-3D) +// leave behind (its default ~1.2e-7 is effectively absolute and far too tight +// for such meshes). ~1e-4 of the bounding-box diagonal is small enough to +// preserve real detail but large enough to collapse sliver-producing dupes. +inline float meshWeldEpsilon(const std::vector& positions) +{ + if (positions.size() < 6) return 1.192092896e-07f; + float mn[3] = { positions[0], positions[1], positions[2] }; + float mx[3] = { positions[0], positions[1], positions[2] }; + for (size_t i = 0; i + 2 < positions.size(); i += 3) + for (int a = 0; a < 3; ++a) { + mn[a] = std::min(mn[a], positions[i + a]); + mx[a] = std::max(mx[a], positions[i + a]); + } + const float dx = mx[0] - mn[0], dy = mx[1] - mn[1], dz = mx[2] - mn[2]; + const float diag = std::sqrt(dx * dx + dy * dy + dz * dz); + return std::max(1.192092896e-07f, diag * 1.0e-4f); +} + // Body of the unwrap shared between the destructive and the // keep-originals entry points. When `keepOriginalBuffers` is true, // the old `VertexData` / `IndexData` pointers are NOT deleted when @@ -690,6 +711,14 @@ UvUnwrapReport runUnwrap(Ogre::Entity* entity, decl.vertexPositionStride = sizeof(float) * 3; decl.indexCount = static_cast(geoms[si].indices.size()); decl.indexData = geoms[si].indices.data(); + // Weld near-coincident vertices. xatlas's default epsilon (~1.2e-7) is + // far too tight for marching-cubes + Taubin-smoothed meshes (image-to-3D + // output): near-but-not-exactly-coincident verts stay unmerged, leaving + // zero/near-zero-area sliver triangles that xatlas can't parameterize — + // they collapse to lines in UV space, producing the "star of thin + // streaks across the atlas + untextured patches" symptom. Scale the + // epsilon to the mesh so welding is proportional, not absolute. + decl.epsilon = std::max(1.192092896e-07f, meshWeldEpsilon(geoms[si].positions)); decl.indexFormat = xatlas::IndexFormat::UInt32; if (!faceIgnore.empty()) decl.faceIgnoreData = reinterpret_cast(faceIgnore.data()); @@ -722,7 +751,17 @@ UvUnwrapReport runUnwrap(Ogre::Entity* entity, pack.resolution = static_cast(std::max(64, opts.resolution)); pack.padding = static_cast(std::max(0, opts.padding)); pack.bilinear = true; - xatlas::Generate(atlas, /*chartOptions=*/{}, pack); + // Chart options: keep xatlas's DEFAULT maxIterations (1). A previous + // attempt at maxIterations=2 (for rounder charts) LIVELOCKED xatlas's + // internal TaskScheduler on large/organic image-to-3D meshes — the main + // thread spun forever in ComputeCharts (cthread_yield), freezing the whole + // app with an untextured mesh. The single-iteration path is the tested- + // stable one; sliver charts are already prevented by the proportional weld + // epsilon (MeshDecl.epsilon) + removeDegenerateTriangles before unwrap. + // fixWinding is cheap and only enforces consistent UV winding. + xatlas::ChartOptions chart; + chart.fixWinding = true; + xatlas::Generate(atlas, chart, pack); report.atlasWidth = static_cast(atlas->width); report.atlasHeight = static_cast(atlas->height); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6fc712e5..545b6808 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -155,9 +155,11 @@ if(BUILD_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/MeshRefine.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/MeshGenBaker.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/MeshGenPredictor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/TripoSGPredictor.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/MeshGenBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/MeshGenController.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/BackgroundRemover.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/ImageTo3D/ImageCaptioner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/PbrMapSynth.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/TextureUpscaler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/AIAssistManager.cpp @@ -497,6 +499,11 @@ if(BUILD_TESTS) if(ENABLE_LOCAL_LLM) list(APPEND TEST_SUPPORT_LIBRARIES llama ggml) + # ImageCaptioner.cpp (in the test-common lib) uses mtmd under + # ENABLE_MTMD — link it where available (not Windows-MinGW). + if(QTMESH_HAVE_MTMD) + list(APPEND TEST_SUPPORT_LIBRARIES mtmd) + endif() endif() if(ENABLE_SENTRY) @@ -531,6 +538,15 @@ if(BUILD_TESTS) BATCH_EXPORTER_TEST_SEAM QTMESH_UNIT_TESTS "QTMESH_UT_SOURCE_ROOT=\"${CMAKE_SOURCE_DIR}\"") + # Manager.cpp (compiled into this lib) picks plugins_d.cfg vs plugins.cfg at + # compile time via #ifdef DEBUG. The scoped-DEBUG fix (src/CMakeLists.txt) + # applied DEBUG to QtMeshEditor + UnitTests but NOT here, so the standalone + # per-file test executables (MaterialEditorQML_test, …) looked for the + # nonexistent plugins.cfg in a Debug build → Ogre init failed. Apply DEBUG + # here too (PRIVATE — can't leak into FetchContent deps). + if(QTMESH_DEFINE_DEBUG) + target_compile_definitions(qtmesh_test_common PRIVATE DEBUG) + endif() target_link_libraries(qtmesh_test_common PUBLIC ${TEST_SUPPORT_LIBRARIES}) if(APPLE)