DimSim scene editing/authoring in JS + moving inside dimos

.gitattributes

@@ -18,3 +18,8 @@
 *.foxe filter=lfs diff=lfs merge=lfs -text binary
 docs/capabilities/memory/assets/** filter=lfs diff=lfs merge=lfs -text
 docs/capabilities/memory/assets/.gitattributes -filter -diff -merge text
+# DimSim scene data and agent model
+misc/DimSim/public/sims/*.json filter=lfs diff=lfs merge=lfs -text
+misc/DimSim/public/embodiment/*.glb filter=lfs diff=lfs merge=lfs -text
+misc/DimSim/scenes/**/*.glb filter=lfs diff=lfs merge=lfs -text
+misc/DimSim/scenes/**/*.gltf filter=lfs diff=lfs merge=lfs -text

.github/workflows/dimsim-check.yml

@@ -0,0 +1,45 @@
+name: dimsim-check
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'misc/DimSim/**'
+      - '.github/workflows/dimsim-check.yml'
+  pull_request:
+    paths:
+      - 'misc/DimSim/**'
+      - '.github/workflows/dimsim-check.yml'
+
+permissions: {}
+
+jobs:
+  check:
+    timeout-minutes: 15
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    defaults:
+      run:
+        working-directory: misc/DimSim
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          lfs: true
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+
+      - name: Install npm deps
+        run: npm ci
+
+      - name: Build frontend
+        run: npm run build
+
+      - name: Type-check CLI
+        run: cd cli && deno check cli.ts

.gitignore

@@ -39,10 +39,8 @@ __pycache__
 
 # node env (used by devcontainers cli)
 node_modules
-package.json
-package-lock.json
-!docs/package.json
-!docs/package-lock.json
+/package.json
+/package-lock.json
 
 # Ignore build artifacts
 dist/

.pre-commit-config.yaml

@@ -39,11 +39,13 @@ repos:
         args: [--fix=lf]
         exclude: \.patch$  # keep patch files byte-identical to upstream diffs
       - id: check-json
+        exclude: ^misc/DimSim/public/sims/  # LFS-tracked
       - id: check-toml
       - id: check-yaml
       - id: pretty-format-json
         name: format json
         args: [ --autofix, --no-sort-keys ]
+        exclude: ^misc/DimSim/public/sims/  # LFS-tracked
 
   - repo: https://github.com/editorconfig-checker/editorconfig-checker.python
     rev: 3.4.1

dimos/core/global_config.py

@@ -65,8 +65,9 @@ class GlobalConfig(BaseSettings):
     obstacle_avoidance: bool = True
     detection_model: VlModelName = "moondream"
     listen_host: str = "127.0.0.1"
-    dimsim_scene: str = "apt"
+    dimsim_scene: str = "apartment"
     dimsim_port: int = 8090
+    dimsim_headless: bool = True
 
     model_config = SettingsConfigDict(
         env_file=".env",

dimos/simulation/dimsim/dimsim_process.py

@@ -19,7 +19,6 @@
 import time
 from typing import IO
 
-from dimos.constants import STATE_DIR
 from dimos.core.global_config import GlobalConfig
 from dimos.simulation.dimsim.deno_utils import ensure_deno, ensure_playwright_chromium
 from dimos.utils.logging_config import setup_logger
@@ -28,8 +27,7 @@
 
 _VIDEO_RATE = 50
 _LIDAR_RATE = 1000
-_DIMSIM_REPO_URL = "https://github.com/paul-nechifor/DimSim.git"
-_DIMSIM_REPO_BRANCH = "run-from-repo"
+_DIMSIM_DIR = Path(__file__).resolve().parents[3] / "misc" / "DimSim"
 
 
 class DimSimProcess:
@@ -39,13 +37,14 @@ def __init__(self, global_config: GlobalConfig) -> None:
 
     def start(self) -> None:
         deno_path = ensure_deno()
-        repo_dir = _ensure_repo()
-        base_cmd = _deno_cmd(deno_path, repo_dir)
+        base_cmd = _deno_cmd(deno_path, _resolve_dimsim_dir())
 
         scene = self.global_config.dimsim_scene
         port = self.global_config.dimsim_port
+        headless = self.global_config.dimsim_headless
 
-        ensure_playwright_chromium(deno_path)
+        if headless:
+            ensure_playwright_chromium(deno_path)
         _kill_port_holder(port)
 
         render = os.environ.get("DIMSIM_RENDER", "gpu").strip()
@@ -60,7 +59,7 @@ def start(self) -> None:
             "--port",
             str(port),
             "--no-depth",
-            "--headless",
+            *(("--headless",) if headless else ()),
             "--render",
             render,
             "--image-rate",
@@ -69,6 +68,11 @@ def start(self) -> None:
             str(_LIDAR_RATE),
         ]
 
+        if not headless:
+            logger.info(
+                f"Open http://localhost:{port} in your browser; sensors won't publish until that tab is loaded."
+            )
+
         self.process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
         self._start_log_reader()
@@ -126,28 +130,30 @@ def _kill_port_holder(port: int) -> None:
         logger.warning(f"Failed to check/kill port {port}: {e}")
 
 
-def _ensure_repo() -> Path:
-    repo_dir = STATE_DIR / "dimsim_repo"
-    if (repo_dir / ".git").exists():
-        return repo_dir
-    STATE_DIR.mkdir(parents=True, exist_ok=True)
-    logger.info(f"Cloning DimSim into {repo_dir}")
-    subprocess.run(
-        [
-            "git",
-            "clone",
-            "--depth",
-            "1",
-            "--branch",
-            _DIMSIM_REPO_BRANCH,
-            _DIMSIM_REPO_URL,
-            str(repo_dir),
-        ],
-        check=True,
-    )
-    return repo_dir
+def _resolve_dimsim_dir() -> Path:
+    """Pick the DimSim directory to run.  DIMSIM_LOCAL env overrides the
+    vendored misc/DimSim/ copy — handy when iterating on DimSim itself.
+
+        DIMSIM_LOCAL=1           → ../DimSim sibling of the dimos repo
+        DIMSIM_LOCAL=/some/path  → that path
+        (unset)                  → vendored misc/DimSim/
+    """
+    local = os.environ.get("DIMSIM_LOCAL", "").strip()
+    if not local:
+        return _DIMSIM_DIR
+    if local == "1":
+        dimos_root = Path(__file__).resolve().parents[3]
+        path = dimos_root.parent / "DimSim"
+    else:
+        path = Path(local).expanduser().resolve()
+    if not (path / "cli" / "cli.ts").exists():
+        raise RuntimeError(
+            f"DIMSIM_LOCAL={local} resolved to {path}, but {path}/cli/cli.ts does not exist"
+        )
+    logger.info(f"Using local DimSim from {path}")
+    return path
 
 
 def _deno_cmd(deno_path: str, repo_dir: Path) -> list[str]:
-    cli_ts = repo_dir / "dimos-cli" / "cli.ts"
+    cli_ts = repo_dir / "cli" / "cli.ts"
     return [deno_path, "run", "--allow-all", "--unstable-net", str(cli_ts)]

dimos/simulation/dimsim/scene_client.py

@@ -41,7 +41,7 @@
         "halfHeight": 0.25,
         "lidarMountHeight": 0.35,
         "embodimentType": "quadruped",
-        "avatarUrl": ["/agent-model/unitree_go2.glb", "/agent-model/robot.glb"],
+        "avatarUrl": ["/agent-model/dimsim_unitree_stub.glb"],
         # Physics
         "maxSpeed": 3.0,
         "turnRate": 3.0,
@@ -54,7 +54,7 @@
         "halfHeight": 0.25,
         "lidarMountHeight": 0.35,
         "embodimentType": "quadruped",
-        "avatarUrl": ["/agent-model/unitree_go2.glb", "/agent-model/robot.glb"],
+        "avatarUrl": ["/agent-model/dimsim_unitree_stub.glb"],
         "maxSpeed": 3.0,
         "turnRate": 3.0,
         "gravity": -9.81,
@@ -66,7 +66,7 @@
         "halfHeight": 0.2,
         "lidarMountHeight": 0.35,
         "embodimentType": "quadruped",  # ground physics
-        "avatarUrl": ["/agent-model/robot.glb"],
+        "avatarUrl": ["/agent-model/dimsim_unitree_stub.glb"],
         "maxSpeed": 2.0,
         "turnRate": 2.5,  # differential drive turns by wheel speed diff
         "gravity": -9.81,
@@ -78,7 +78,7 @@
         "halfHeight": 0.4,
         "lidarMountHeight": 0.8,
         "embodimentType": "quadruped",  # ground physics
-        "avatarUrl": ["/agent-model/robot.glb"],
+        "avatarUrl": ["/agent-model/dimsim_unitree_stub.glb"],
         "maxSpeed": 5.0,
         "turnRate": 1.2,  # car-like: slow turn rate (limited steering angle)
         "gravity": -9.81,
@@ -90,7 +90,7 @@
         "halfHeight": 0.25,
         "lidarMountHeight": 0.4,
         "embodimentType": "quadruped",  # ground physics (strafing via cmd_vel.linear.y)
-        "avatarUrl": ["/agent-model/robot.glb"],
+        "avatarUrl": ["/agent-model/dimsim_unitree_stub.glb"],
         "maxSpeed": 2.5,
         "turnRate": 4.0,  # omnidirectional: fast rotation
         "gravity": -9.81,
@@ -102,7 +102,7 @@
         "halfHeight": 0.8,
         "lidarMountHeight": 1.6,
         "embodimentType": "quadruped",  # ground physics
-        "avatarUrl": ["/agent-model/robot.glb"],
+        "avatarUrl": ["/agent-model/dimsim_unitree_stub.glb"],
         "maxSpeed": 1.5,
         "turnRate": 2.0,
         "gravity": -9.81,
@@ -114,7 +114,7 @@
         "halfHeight": 0.15,
         "lidarMountHeight": 0.25,
         "embodimentType": "quadruped",
-        "avatarUrl": ["/agent-model/robot.glb"],
+        "avatarUrl": ["/agent-model/dimsim_unitree_stub.glb"],
         "maxSpeed": 1.0,
         "turnRate": 3.0,
         "gravity": -9.81,
@@ -127,7 +127,7 @@
         "halfHeight": 0.1,
         "lidarMountHeight": 0.15,
         "embodimentType": "drone",
-        "avatarUrl": ["/agent-model/robot.glb"],
+        "avatarUrl": ["/agent-model/dimsim_unitree_stub.glb"],
         "maxSpeed": 5.0,
         "turnRate": 4.0,
         "gravity": 0,  # no gravity in flight
@@ -723,7 +723,7 @@ def set_embodiment(
         - ``"flight"`` — 6DoF, optional gravity, altitude ceiling
 
         **Avatar URL** can be:
-        - Built-in: ``"/agent-model/robot.glb"``
+        - Built-in: ``"/agent-model/dimsim_unitree_stub.glb"``
         - Local asset: ``"/local-assets/my-drone.glb"`` (see :meth:`upload_asset`)
         - Any URL: ``"https://example.com/robot.glb"``
 

dimos/visualization/rerun/websocket_server.py

@@ -151,13 +151,19 @@ def _dispatch(self, raw: str | bytes) -> None:
 
         msg_type = msg.get("type")
 
+        # dict.get's default is only used when the key is missing — if Rerun
+        # sends a 2D-panel click the "z" key is present with value None, and
+        # `float(None)` raises.  Coerce explicitly.
+        def _num(v: Any) -> float:
+            return float(v) if v is not None else 0.0
+
         if msg_type == "click":
             self.clicked_point.publish(
                 PointStamped(
-                    x=float(msg.get("x", 0)),
-                    y=float(msg.get("y", 0)),
-                    z=float(msg.get("z", 0)),
-                    ts=float(msg.get("timestamp_ms", 0)) / 1000.0,
+                    x=_num(msg.get("x")),
+                    y=_num(msg.get("y")),
+                    z=_num(msg.get("z")),
+                    ts=_num(msg.get("timestamp_ms")) / 1000.0,
                     frame_id=str(msg.get("entity_path", "")),
                 )
             )
@@ -166,14 +172,14 @@ def _dispatch(self, raw: str | bytes) -> None:
             self.tele_cmd_vel.publish(
                 Twist(
                     linear=Vector3(
-                        float(msg.get("linear_x", 0)),
-                        float(msg.get("linear_y", 0)),
-                        float(msg.get("linear_z", 0)),
+                        _num(msg.get("linear_x")),
+                        _num(msg.get("linear_y")),
+                        _num(msg.get("linear_z")),
                     ),
                     angular=Vector3(
-                        float(msg.get("angular_x", 0)),
-                        float(msg.get("angular_y", 0)),
-                        float(msg.get("angular_z", 0)),
+                        _num(msg.get("angular_x")),
+                        _num(msg.get("angular_y")),
+                        _num(msg.get("angular_z")),
                     ),
                 )
             )

misc/DimSim/.gitignore

@@ -0,0 +1,9 @@
+node_modules/
+*.tar.gz
+dist/
+.DS_Store
+**/.DS_Store
+.deno/
+scenes.json
+scenes/**/apt.json
+scripts/

misc/DimSim/README.md

@@ -0,0 +1,54 @@
+# DimSim
+
+Browser-based 3D simulator (Three.js + Rapier) plus a Deno bridge that talks LCM/WS to [dimos](https://github.com/dimensionalOS/dimos). Lives inside dimos as `misc/DimSim/`.
+
+```
+src/        — browser engine (vite-bundled)
+cli/        — Deno CLI + bridge server + headless launcher + LCM vendor
+evals/      — eval harness (browser) + runner (Deno) + rubrics
+scenes/     — user-authored scenes (JS) + per-scene eval workflows
+public/     — static assets (agent GLB, logo)
+docs/       — guides
+```
+
+## Run
+
+dimsim is launched by dimos directly when you pick `--simulation dimsim`:
+
+```bash
+cd <dimos-repo>
+.venv/bin/dimos --simulation dimsim --dimsim-scene=apartment run unitree-go2-agentic
+```
+
+On first run, `cli/cli.ts` will build `dist/` via Vite (dimsim ships its frontend as source — Deno+Vite materializes it in ~20s).
+
+## Docs
+
+- [docs/getting-started.md](docs/getting-started.md) — 5-minute tour
+- [docs/scenes.md](docs/scenes.md) — create + edit scenes
+- [docs/evals.md](docs/evals.md) — write eval workflows
+
+## Install the CLI (optional)
+
+If you want `dimsim` as a global command:
+
+```bash
+cd misc/DimSim/cli
+deno install -gAf --unstable-net --name=dimsim --config=./deno.json ./cli.ts
+```
+
+After install:
+
+```bash
+dimsim dev --scene apartment              # standalone dev server + browser
+dimsim eval list                          # list workflows under scenes/*/evals/
+dimsim eval go-to-couch                   # run one workflow against an open sim
+dimsim eval --headless --scene apartment  # full headless run (CI)
+```
+
+## Build manually
+
+```bash
+npm install      # browser deps (three, rapier, spark, vite)
+npm run build    # → dist/
+```