diff --git a/.dockerignore b/.dockerignore index 51d4ec61a457..b718bd108a8a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -23,7 +23,7 @@ recordings/ **/__pycache__/ **/*.egg-info/ # ignore isaac sim symlink -_isaac_sim? +_isaac_sim # Docker history docker/.isaac-lab-docker-history # ignore uv environment diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000000..0e38d5fdd3c8 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,7 @@ +# CI Workflows + +`schedule:` and `workflow_dispatch:` triggers fire **only from the default +branch (`main`)**. A workflow YAML must live on `main` for its cron to +register — the same file on other branches has no effect. `pull_request:` +and `push:` triggers fire from the event branch's file and work normally +on `develop`. diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 013be3a5b126..1d3c0496e1c2 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -132,3 +132,4 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./docs/_build keep_files: false + force_orphan: true diff --git a/.github/workflows/nightly-changelog.yml b/.github/workflows/nightly-changelog.yml index 7e55ad7450c6..839130ffccec 100644 --- a/.github/workflows/nightly-changelog.yml +++ b/.github/workflows/nightly-changelog.yml @@ -6,19 +6,38 @@ # Nightly auto-compile: rolls accumulated fragments under # ``source//changelog.d/`` into per-package ``CHANGELOG.rst`` entries, # bumps each ``extension.toml``, deletes consumed fragments, and pushes the -# result back to ``develop``. Keeps the develop branch's changelog current -# without requiring a maintainer to run ``compile`` by hand. +# result back to each branch in the configured list. Keeps every tracked +# branch's changelog current without requiring a maintainer to run +# ``compile`` by hand. # -# The push uses ``CHANGELOG_PAT`` (a personal access token / fine-grained -# GitHub App token with ``contents:write`` on this repo) when it's -# available so downstream CI runs on the auto-commit. Falls back to -# ``GITHUB_TOKEN`` — sufficient for the push itself, but pushes signed -# with ``GITHUB_TOKEN`` do not trigger workflow runs on the resulting -# commit, which is by design (avoids infinite loops) but means the -# Docker / docs rebuild won't re-trigger off the nightly's auto-commit. +# Scheduled workflow — must live on the default branch (``main``) for the +# cron to register. See ``.github/workflows/README.md``. +# +# Adding a branch to the nightly set is a one-line edit to ``env.CRON_BRANCHES`` +# below. Each target branch uses its own ``tools/changelog/cli.py`` (the +# same copy the PR gate already runs), so the nightly compile honours the +# same rules that validated the fragments. +# +# The push uses a short-lived GitHub App installation token minted from +# ``CHANGELOG_APP_ID`` + ``CHANGELOG_APP_PRIVATE_KEY`` (repo secrets). The +# App must be installed on this repository with ``contents: write`` and +# added to the bypass-actor list of each target branch's ruleset so the +# auto-commit can push directly without satisfying required-checks / +# required-approval gates. +# Commits signed by an App token (unlike ``GITHUB_TOKEN``) are treated as +# external pushes, so they DO trigger downstream workflow runs (Docker +# rebuild, docs, etc.) without needing a separate PAT. name: Nightly Changelog Compilation +# Branches the nightly cron compiles. Single source of truth — append a +# ref here to extend the nightly set (the active release branch belongs +# here). Each branch must carry ``tools/changelog/cli.py`` and the +# isaaclab-bot App must be in its branch-ruleset bypass list. +# Surrounding whitespace per entry is stripped by the resolver below. +env: + CRON_BRANCHES: develop,release/3.0.0-beta2 + on: schedule: # Run nightly at 5 AM UTC (one hour after daily-compatibility, so we @@ -26,44 +45,130 @@ on: - cron: '0 5 * * *' workflow_dispatch: inputs: + branch: + # Manual trigger is always a single branch. Free-text on purpose + # — scales to any branch (e.g. a new ``release/*`` cut from + # develop) without needing to update the workflow. A branch + # that lacks ``tools/changelog/cli.py`` fails the verify step + # below with a clear error, which is the desired failure mode. + description: 'Branch to compile (e.g. develop, release/3.0.0-beta2). Must carry tools/changelog/cli.py.' + required: true + type: string dry_run: description: 'Preview only — do not commit / push' required: false type: boolean default: false -concurrency: - # Only one nightly compile may be in flight at a time. ``cancel-in-progress`` - # is intentionally false: if a previous run is still finishing its push, we - # queue rather than abort it mid-commit. - group: nightly-changelog - cancel-in-progress: false - permissions: - contents: write + # Reduced: the App installation token below carries its own write scope. + # GITHUB_TOKEN only needs read access for the standard checkout machinery. + contents: read jobs: + resolve-branches: + # CSV → JSON array bridge. ``workflow_dispatch`` inputs can only be + # string / bool / choice, so the branch list arrives as a comma- + # separated string; the matrix below needs a JSON list to fan out. + # Mirrors the ``setup-versions`` job in ``daily-compatibility.yml``. + name: Resolve branch list + runs-on: ubuntu-latest + timeout-minutes: 1 + outputs: + branches: ${{ steps.b.outputs.branches }} + steps: + - id: b + env: + # Schedule → the CRON_BRANCHES list. Manual → the single branch + # the maintainer entered. The two paths are intentionally + # asymmetric: cron is the configured set, manual is exactly one + # branch (required input). + BRANCHES: ${{ github.event_name == 'schedule' && env.CRON_BRANCHES || inputs.branch }} + # ``EVENT_NAME`` mirrors ``github.event_name`` so the guard below + # branches on it without re-interpolating into the shell. + EVENT_NAME: ${{ github.event_name }} + run: | + # CSV → JSON array, trimming surrounding whitespace per entry. + # Manual produces a 1-element array; cron produces N elements. + arr=$(echo "$BRANCHES" | tr ',' '\n' | xargs -n1 | jq -R . | jq -s -c .) + # Manual trigger contract: exactly one branch. A maintainer who + # pastes a comma-separated list into the dispatch form should + # see a clear error, not a silent multi-branch fan-out. + if [ "$EVENT_NAME" = "workflow_dispatch" ] && [ "$(echo "$arr" | jq 'length')" -ne 1 ]; then + echo "::error::Manual trigger accepts exactly one branch; got $arr. Fire the workflow separately per branch." + exit 1 + fi + echo "branches=$arr" >> "$GITHUB_OUTPUT" + echo "Resolved branches: $arr" + compile-changelog: - name: Compile changelog fragments + name: Compile changelog fragments (${{ matrix.branch }}) + needs: resolve-branches runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + # Independent branches: one failing shouldn't cancel the others. + # Each matrix entry shows up as a separate job tile in the Actions + # UI, so ``release/3.0.0-beta2`` failing doesn't hide ``develop``'s + # success (and ``gh run rerun --failed`` re-runs only the failed + # entry). Mirrors ``daily-compatibility.yml``'s matrix style. + fail-fast: false + matrix: + branch: ${{ fromJson(needs.resolve-branches.outputs.branches) }} + concurrency: + # Per-branch group: two runs against the same ref queue, but + # different refs (develop and a release branch) compile in parallel. + group: nightly-changelog-${{ matrix.branch }} + cancel-in-progress: false steps: + # Mint a short-lived (1 h) installation access token for the + # ``isaaclab-bot`` GitHub App. The App is on each target branch's + # ruleset bypass list, so its push lands without needing the + # standard required-checks / required-approval pipeline. + - uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + id: app-token + with: + # ``client-id`` (the App's OAuth client ID, ``Iv23...``) supersedes + # the deprecated ``app-id`` integer input as of v3.x. + client-id: ${{ secrets.CHANGELOG_APP_CLIENT_ID }} + private-key: ${{ secrets.CHANGELOG_APP_PRIVATE_KEY }} + # Declare the scope the App must have so token mint fails loudly + # if the App is misconfigured, instead of failing silently at the + # push step. + permission-contents: write + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: - # Operate on develop, not the repo's default branch. Scheduled - # workflows fire from the default branch's workflow file by - # default, but we want the *checkout* to be develop so the - # compile sees develop's accumulated fragments and the push - # writes back to develop. - ref: develop - # Use a PAT so the auto-commit triggers downstream CI; falls back - # to GITHUB_TOKEN which is sufficient for the push itself. - token: ${{ secrets.CHANGELOG_PAT || secrets.GITHUB_TOKEN }} + # Operate on the target branch, not the repo's default branch. + # Scheduled workflows fire from the default branch's workflow + # file, but the *checkout* is the branch we're compiling so the + # compile sees that branch's accumulated fragments and the push + # writes back to it. + ref: ${{ matrix.branch }} + # App token (vs. GITHUB_TOKEN) means the push is signed by + # ``isaaclab-bot`` — the bypass identity — and downstream CI + # workflows DO trigger on the resulting commit. + token: ${{ steps.app-token.outputs.token }} # Full history so the compiler can resolve each fragment's merge # time via ``git log --diff-filter=A --first-parent``. fetch-depth: 0 - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - name: Verify changelog tooling exists on target branch + env: + TARGET_BRANCH: ${{ matrix.branch }} + run: | + # Loud-fail this matrix entry (not the whole run — ``fail-fast: + # false`` keeps siblings going) when the target branch lacks + # ``tools/changelog/cli.py``. A red tile is the desired signal: + # a branch in the nightly set without the compile tooling is a + # configuration error, not a "no-op" condition. + if [ ! -f tools/changelog/cli.py ]; then + echo "::error::Branch '$TARGET_BRANCH' is missing tools/changelog/cli.py — drop it from env.CRON_BRANCHES (or the dispatch input) or restore the tooling on that branch." + exit 1 + fi + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" @@ -77,10 +182,20 @@ jobs: python3 tools/changelog/cli.py compile $ARGS - name: Commit and push if fragments were compiled - if: inputs.dry_run != 'true' + if: ${{ !inputs.dry_run }} + env: + # Pass the matrix branch through an env var rather than + # interpolating ``${{ matrix.branch }}`` directly into ``run:``. + # The interpolation happens *before* shell quoting, so an + # adversarial input could escape the surrounding quotes; the + # env passthrough keeps the value inside the shell's variable + # space where standard quoting protects it. + TARGET_BRANCH: ${{ matrix.branch }} run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + # Author commits as the App's bot user so the GitHub UI attributes + # them correctly. ID 282401363 is isaaclab-bot[bot]'s user ID. + git config user.name "isaaclab-bot[bot]" + git config user.email "282401363+isaaclab-bot[bot]@users.noreply.github.com" git add source/*/changelog.d/ \ source/*/docs/CHANGELOG.rst \ source/*/config/extension.toml @@ -113,7 +228,14 @@ jobs: done } > "$MSG_FILE" git commit -F "$MSG_FILE" - # Push explicitly to develop so we don't accidentally write - # to the source ref of a workflow_dispatch run. - git push origin HEAD:develop + # Rebase onto the target branch's current tip in case a human + # commit landed during this run (~2 min window between + # checkout and push). Without this the push fails non-fast- + # forward and the batch waits for the next run. ``refs/heads/`` + # is explicit so a same-named tag (if one ever exists) can't + # disambiguate the wrong way. + git pull --rebase origin "refs/heads/$TARGET_BRANCH" + # Push explicitly to the target branch so we don't accidentally + # write to the source ref of a workflow_dispatch run. + git push origin "HEAD:refs/heads/$TARGET_BRANCH" fi diff --git a/docs/source/overview/core-concepts/visualization.rst b/docs/source/overview/core-concepts/visualization.rst index 225e4210a986..9159f397d6a6 100644 --- a/docs/source/overview/core-concepts/visualization.rst +++ b/docs/source/overview/core-concepts/visualization.rst @@ -190,16 +190,18 @@ Also, there is a CLI arg ``--max_visible_envs`` that overrides ``VisualizerCfg.m Camera Modes ~~~~~~~~~~~~ +To configure camera modes, including launching a tiled camera view, edit the fields described below in the +``VisualizerCfg`` config class. + The default visualizer camera mode is interactive, with ``eye`` and ``lookat`` specifying the initial pose. Kit and Newton visualizers can also run additional tiled camera image panels. -If ``tiled_cam_view=True`` is set, another window is launched in the visualizer which shows -a non-interactive tiled camera image view. -Kit and Newton cap tiled camera views at 100 tiles. +If ``tiled_cam_view=True`` is set, another window is launched in the visualizer which shows +a non-interactive tiled camera image view. Number of tiles is capped at 100. Note, Kit tiled camera views require launching with ``--enable_cameras``. -.. list-table:: Camera configuration modes +.. list-table:: Camera Modes :header-rows: 1 :widths: 24 30 46 @@ -216,6 +218,15 @@ Note, Kit tiled camera views require launching with ``--enable_cameras``. - ``tiled_cam_view=True``, ``tiled_cam_prim_path="/World/envs/*/Camera"`` - The visualizer displays existing Isaac Lab ``Camera`` sensor output. Generated-camera fields such as ``tiled_cam_eye`` and ``tiled_cam_target_prim_path`` are ignored. +**How to Access the Tiled Camera View in the UI** + +- **Kit Visualizer:** + To display the tiled camera panel, select the "Visualizer Tiled Camera" viewport from the viewport selection menu. + +- **Newton Visualizer:** + To enable or disable the tiled camera panel, use the "Visualizer Tiled Camera" option found in the Tiled Camera View dropdown menu on the left sidebar. + + Video Recording --------------- diff --git a/docs/source/overview/environments.rst b/docs/source/overview/environments.rst index 34cbb7a7624d..384062fd78c0 100644 --- a/docs/source/overview/environments.rst +++ b/docs/source/overview/environments.rst @@ -525,14 +525,11 @@ Environments based on legged locomotion tasks. | |velocity-rough-g1| | |velocity-rough-g1-link| | Track a velocity command on rough terrain with the Unitree G1 robot | **physics=** ``physx``, | | | | | ``newton_mjwarp`` | +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+------------------------------+ - | |velocity-flat-digit| | |velocity-flat-digit-link| | Track a velocity command on flat terrain with the Agility Digit robot | **physics=** ``physx``, | - | | | | ``newton_mjwarp`` | + | |velocity-flat-digit| | |velocity-flat-digit-link| | Track a velocity command on flat terrain with the Agility Digit robot | **physics=** ``physx`` | +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+------------------------------+ - | |velocity-rough-digit| | |velocity-rough-digit-link| | Track a velocity command on rough terrain with the Agility Digit robot | **physics=** ``physx``, | - | | | | ``newton_mjwarp`` | + | |velocity-rough-digit| | |velocity-rough-digit-link| | Track a velocity command on rough terrain with the Agility Digit robot | **physics=** ``physx`` | +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+------------------------------+ - | |tracking-loco-manip-digit| | |tracking-loco-manip-digit-link| | Track a root velocity and hand pose command with the Agility Digit robot | **physics=** ``physx``, | - | | | | ``newton_mjwarp`` | + | |tracking-loco-manip-digit| | |tracking-loco-manip-digit-link| | Track a root velocity and hand pose command with the Agility Digit robot | **physics=** ``physx`` | +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+------------------------------+ .. |velocity-flat-anymal-b-link| replace:: `Isaac-Velocity-Flat-Anymal-B-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/anymal_b/flat_env_cfg.py>`__ @@ -1118,7 +1115,7 @@ inferencing, including reading from an already trained checkpoint and disabling - Isaac-Tracking-LocoManip-Digit-Play-v0 - Manager Based - **rsl_rl** (PPO) - - **physics=** ``physx``, ``newton_mjwarp`` + - **physics=** ``physx`` * - Isaac-Navigation-Flat-Anymal-C-v0 - Isaac-Navigation-Flat-Anymal-C-Play-v0 - Manager Based @@ -1384,7 +1381,7 @@ inferencing, including reading from an already trained checkpoint and disabling - Isaac-Velocity-Flat-Digit-Play-v0 - Manager Based - **rsl_rl** (PPO) - - **physics=** ``physx``, ``newton_mjwarp`` + - **physics=** ``physx`` * - Isaac-Velocity-Flat-G1-v0 - Isaac-Velocity-Flat-G1-Play-v0 - Manager Based @@ -1444,7 +1441,7 @@ inferencing, including reading from an already trained checkpoint and disabling - Isaac-Velocity-Rough-Digit-Play-v0 - Manager Based - **rsl_rl** (PPO) - - **physics=** ``physx``, ``newton_mjwarp`` + - **physics=** ``physx`` * - Isaac-Velocity-Rough-G1-v0 - Isaac-Velocity-Rough-G1-Play-v0 - Manager Based diff --git a/docs/source/refs/issues.rst b/docs/source/refs/issues.rst index 42a78ec27ffb..1ab3d8671401 100644 --- a/docs/source/refs/issues.rst +++ b/docs/source/refs/issues.rst @@ -93,6 +93,30 @@ message and continue with terminating the process. On Windows systems, please us ``Ctrl+Break`` or ``Ctrl+fn+B`` to terminate the process. +Closed-loop articulations on Newton (e.g. Agility Digit) +-------------------------------------------------------- + +Robots whose USD encodes a closed kinematic loop -- such as the achilles rod and +toe push-rods on the Agility Digit -- do not currently run correctly on the +``newton_mjwarp`` physics preset, even though they work on ``physx``. This affects: + +* ``Isaac-Velocity-Flat-Digit-v0`` / ``Isaac-Velocity-Flat-Digit-Play-v0`` +* ``Isaac-Velocity-Rough-Digit-v0`` / ``Isaac-Velocity-Rough-Digit-Play-v0`` +* ``Isaac-Tracking-LocoManip-Digit-v0`` / ``Isaac-Tracking-LocoManip-Digit-Play-v0`` + +The root cause sits inside Newton's :class:`~newton.selection.ArticulationView`. When +it builds its per-link axis it walks each joint in the articulation's joint range and +appends ``model.joint_child[joint_id]`` without deduplicating. A body that is the +child of multiple joints (the standard closed-loop encoding) therefore occupies +multiple slots on that axis, so ``view.link_count`` is larger than the number of +unique physical bodies in the model. On Digit this is 49 link slots versus 43 actual +bodies (the four loop-closed bodies -- left/right ``tarsus`` and left/right +``toe_roll`` -- account for the six extra slots). + +Until the upstream fix lands in Newton, please use the ``physx`` preset for +Digit-based environments. + + URDF Importer: Unresolved references for fixed joints ----------------------------------------------------- diff --git a/source/isaaclab/changelog.d/mtrepte-update_viz_integration_test.rst b/source/isaaclab/changelog.d/mtrepte-update_viz_integration_test.rst new file mode 100644 index 000000000000..7ae85672fba6 --- /dev/null +++ b/source/isaaclab/changelog.d/mtrepte-update_viz_integration_test.rst @@ -0,0 +1,4 @@ +Changed +^^^^^^^ + +* Updated shared visualizer tiled camera defaults for the visualizer integration test coverage. diff --git a/source/isaaclab/changelog.d/peterd-mimic-installs-teleop.rst b/source/isaaclab/changelog.d/peterd-mimic-installs-teleop.rst new file mode 100644 index 000000000000..4074fe7de8de --- /dev/null +++ b/source/isaaclab/changelog.d/peterd-mimic-installs-teleop.rst @@ -0,0 +1,5 @@ +Changed +^^^^^^^ + +* ``./isaaclab.sh -i mimic`` now also installs ``isaaclab_teleop`` as an editable + submodule, since :mod:`isaaclab_mimic` declares it as a required dependency. diff --git a/source/isaaclab/isaaclab/cli/commands/install.py b/source/isaaclab/isaaclab/cli/commands/install.py index 29a774d016be..d13589e5bb54 100644 --- a/source/isaaclab/isaaclab/cli/commands/install.py +++ b/source/isaaclab/isaaclab/cli/commands/install.py @@ -487,7 +487,7 @@ def _install_isaacsim() -> None: # Optional submodules — only installed when explicitly requested or with 'all'. # Maps the short CLI name to one or more source directory names under source/. OPTIONAL_ISAACLAB_SUBMODULES: dict[str, tuple[str, ...]] = { - "mimic": ("isaaclab_mimic",), + "mimic": ("isaaclab_teleop", "isaaclab_mimic"), "teleop": ("isaaclab_teleop",), } diff --git a/source/isaaclab/isaaclab/envs/utils/camera_view.py b/source/isaaclab/isaaclab/envs/utils/camera_view.py index 7b7c093c3c8b..a9a8c5e4da84 100644 --- a/source/isaaclab/isaaclab/envs/utils/camera_view.py +++ b/source/isaaclab/isaaclab/envs/utils/camera_view.py @@ -219,29 +219,34 @@ def prim_world_positions( from isaaclab.sim.views import FrameView - if scene is not None: - positions = _scene_articulation_positions(scene, prim_path_template, env_indices) - if positions is not None: - return positions - xform_cache = UsdGeom.XformCache() positions = [] - for env_id in env_indices: - prim_path = env_path_from_template(prim_path_template, env_id) - try: + try: + for env_id in env_indices: + prim_path = env_path_from_template(prim_path_template, env_id) view = FrameView(prim_path, device="cpu", stage=stage) if view.count != 1: raise RuntimeError(f"expected one prim, got {view.count}") pos_w, _ = view.get_world_poses() pos = pos_w.torch[0].detach().cpu() positions.append((float(pos[0]), float(pos[1]), float(pos[2]))) - except Exception: - prim = stage.GetPrimAtPath(prim_path) - if not prim.IsValid(): - raise RuntimeError(f"tiled_cam_target_prim_path resolved to missing prim: {prim_path!r}.") - transform = xform_cache.GetLocalToWorldTransform(prim) - translation = transform.ExtractTranslation() - positions.append((float(translation[0]), float(translation[1]), float(translation[2]))) + return torch.tensor(positions, dtype=torch.float32) + except Exception: + positions.clear() + + if scene is not None: + positions_tensor = _scene_articulation_positions(scene, prim_path_template, env_indices) + if positions_tensor is not None: + return positions_tensor + + for env_id in env_indices: + prim_path = env_path_from_template(prim_path_template, env_id) + prim = stage.GetPrimAtPath(prim_path) + if not prim.IsValid(): + raise RuntimeError(f"tiled_cam_target_prim_path resolved to missing prim: {prim_path!r}.") + transform = xform_cache.GetLocalToWorldTransform(prim) + translation = transform.ExtractTranslation() + positions.append((float(translation[0]), float(translation[1]), float(translation[2]))) return torch.tensor(positions, dtype=torch.float32) diff --git a/source/isaaclab/isaaclab/utils/string.py b/source/isaaclab/isaaclab/utils/string.py index 4e7790006ade..0625ee4eca6c 100644 --- a/source/isaaclab/isaaclab/utils/string.py +++ b/source/isaaclab/isaaclab/utils/string.py @@ -333,11 +333,11 @@ def resolve_matching_names( When a list of query regular expressions is provided, the function checks each target string against each query regular expression and returns the indices of the matched strings and the matched strings. - If the :attr:`preserve_order` is True, the ordering of the matched indices and names is the same as the order + If the :attr:`preserve_order` is False, the ordering of the matched indices and names is the same as the order of the provided list of strings. This means that the ordering is dictated by the order of the target strings and not the order of the query regular expressions. - If the :attr:`preserve_order` is False, the ordering of the matched indices and names is the same as the order + If the :attr:`preserve_order` is True, the ordering of the matched indices and names is the same as the order of the provided list of query regular expressions. For example, consider the list of strings is ['a', 'b', 'c', 'd', 'e'] and the regular expressions are ['a|c', 'b']. @@ -393,11 +393,11 @@ def resolve_matching_names_values( use it during initialization only (e.g. action/actuator config resolution), so caching would add complexity without a measurable benefit. - If the :attr:`preserve_order` is True, the ordering of the matched indices and names is the same as the order + If the :attr:`preserve_order` is False, the ordering of the matched indices and names is the same as the order of the provided list of strings. This means that the ordering is dictated by the order of the target strings and not the order of the query regular expressions. - If the :attr:`preserve_order` is False, the ordering of the matched indices and names is the same as the order + If the :attr:`preserve_order` is True, the ordering of the matched indices and names is the same as the order of the provided list of query regular expressions. For example, consider the dictionary is {"a|d|e": 1, "b|c": 2}, the list of strings is ['a', 'b', 'c', 'd', 'e']. diff --git a/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py b/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py index e6d84237e8ed..78729f1779a8 100644 --- a/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py +++ b/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py @@ -25,15 +25,7 @@ class VisualizerCfg: RerunVisualizerCfg, or ViserVisualizerCfg (from isaaclab_visualizers.kit/.newton/.rerun/.viser). """ - visualizer_type: str | None = None - """Type identifier (e.g., 'newton', 'rerun', 'viser', 'kit'). Must be overridden by subclasses.""" - - enable_markers: bool = True - """Enable visualization markers (debug drawing).""" - - enable_live_plots: bool = True - """Enable live plotting of data.""" - + # Primary interactive camera settings eye: tuple[float, float, float] = (4.0, -4.0, 3.0) """Interactive visualizer camera eye position in world coordinates.""" @@ -43,6 +35,7 @@ class VisualizerCfg: focal_length: float = 12.0 """Camera focal length in millimeters for visualizer camera views.""" + # Tiled camera settings tiled_cam_view: bool = False """Enable a non-interactive tiled camera image view.""" @@ -63,11 +56,18 @@ class VisualizerCfg: """ tiled_cam_eye: tuple[float, float, float] = (4.0, -4.0, 3.0) - """Eye offset from tiled_cam_target_prim_path for generated tiled cameras.""" + """Offset of the camera eye from tiled_cam_target_prim_path for generated tiled cameras. + + The camera follows the target prim and always maintains this fixed offset relative to it. + """ - tiled_cam_target_prim_path: str = "/World/envs/*/Robot/base" - """Prim path that generated tiled cameras follow and look at.""" + tiled_cam_target_prim_path: str = "/World/envs/*/Robot" + """Prim path that generated tiled cameras follow and look at. + For example, ``"/World/envs/*/Robot"``. + """ + + # Partial visualization settings max_visible_envs: int | None = None """Upper bound on how many envs are shown. @@ -85,6 +85,18 @@ class VisualizerCfg: * Note: ``visible_env_indices`` overrides this field. """ + # Visualization Markers + enable_markers: bool = True + """Enable visualization markers (debug drawing).""" + + # Live Plots + enable_live_plots: bool = True + """Enable live plotting of data.""" + + # Internal + visualizer_type: str | None = None + """Type identifier (e.g., 'newton', 'rerun', 'viser', 'kit'). Must be overridden by subclasses.""" + def get_visualizer_type(self) -> str | None: """Get the visualizer type identifier. diff --git a/source/isaaclab/test/cli/test_install_command_parsing.py b/source/isaaclab/test/cli/test_install_command_parsing.py index d0bccdae4a77..27281ec013f4 100644 --- a/source/isaaclab/test/cli/test_install_command_parsing.py +++ b/source/isaaclab/test/cli/test_install_command_parsing.py @@ -122,7 +122,7 @@ def test_core_submodules_contains_expected_packages(self): def test_optional_submodules_contains_expected_packages(self): assert set(OPTIONAL_ISAACLAB_SUBMODULES.keys()) == {"mimic", "teleop"} - assert OPTIONAL_ISAACLAB_SUBMODULES["mimic"] == ("isaaclab_mimic",) + assert OPTIONAL_ISAACLAB_SUBMODULES["mimic"] == ("isaaclab_teleop", "isaaclab_mimic") assert OPTIONAL_ISAACLAB_SUBMODULES["teleop"] == ("isaaclab_teleop",) def test_valid_extra_features(self): diff --git a/source/isaaclab_mimic/changelog.d/isaaclab-teleop-dep.rst b/source/isaaclab_mimic/changelog.d/isaaclab-teleop-dep.rst new file mode 100644 index 000000000000..ca97da40b172 --- /dev/null +++ b/source/isaaclab_mimic/changelog.d/isaaclab-teleop-dep.rst @@ -0,0 +1,6 @@ +Changed +^^^^^^^ + +* Declared ``isaaclab_teleop`` as a required extension of + ``isaaclab_mimic`` in ``install.py``. ``./isaaclab.sh -i mimic`` + now installs ``isaaclab_teleop`` alongside ``isaaclab_mimic``. diff --git a/source/isaaclab_newton/changelog.d/octi-docs-digit-newton-known-limitation.skip b/source/isaaclab_newton/changelog.d/octi-docs-digit-newton-known-limitation.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_physx/changelog.d/mtrepte-revert-scene-data-rigid-body-view.rst b/source/isaaclab_physx/changelog.d/mtrepte-revert-scene-data-rigid-body-view.rst new file mode 100644 index 000000000000..349b4a2b0a5a --- /dev/null +++ b/source/isaaclab_physx/changelog.d/mtrepte-revert-scene-data-rigid-body-view.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Restored wildcard PhysX scene-data rigid-body view patterns to keep Newton + visualizers updating live PhysX transforms. diff --git a/source/isaaclab_physx/isaaclab_physx/physics/physx_manager.py b/source/isaaclab_physx/isaaclab_physx/physics/physx_manager.py index b0362b809fd9..df30c9e268a7 100644 --- a/source/isaaclab_physx/isaaclab_physx/physics/physx_manager.py +++ b/source/isaaclab_physx/isaaclab_physx/physics/physx_manager.py @@ -173,8 +173,9 @@ def simulation_view(self, simulation_view: omni.physics.tensors.SimulationView | def get_rigid_body_view(self) -> omni.physics.tensors.RigidBodyView | None: """Lazily create a rigid body view covering all rigid bodies in the scene. - Discovers rigid body prims by traversing the USD stage and creates the - PhysX view from their exact prim paths. + Discovers rigid body prims by traversing the USD stage and converts + per-environment paths (``/World/envs/env_N/...``) into wildcard + patterns so a single PhysX view covers every environment instance. """ if self._rigid_body_view is not None: return self._rigid_body_view @@ -186,15 +187,15 @@ def get_rigid_body_view(self) -> omni.physics.tensors.RigidBodyView | None: if stage is None: return None - paths: list[str] = [] + patterns: set[str] = set() for prim in stage.Traverse(): if prim.HasAPI(UsdPhysics.RigidBodyAPI): - paths.append(prim.GetPath().pathString) + patterns.add(re.sub(r"/World/envs/env_\d+", "/World/envs/env_*", prim.GetPath().pathString)) - if not paths: + if not patterns: return None - self._rigid_body_view = self._simulation_view.create_rigid_body_view(paths) + self._rigid_body_view = self._simulation_view.create_rigid_body_view(list(patterns)) return self._rigid_body_view @property diff --git a/source/isaaclab_visualizers/changelog.d/mtrepte-update_viz_integration_test.rst b/source/isaaclab_visualizers/changelog.d/mtrepte-update_viz_integration_test.rst new file mode 100644 index 000000000000..211dfd67851e --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/mtrepte-update_viz_integration_test.rst @@ -0,0 +1,5 @@ +Changed +^^^^^^^ + +* Split visualizer integration coverage into separate interactive and tiled camera cases. +* Renamed the Newton visualizer tiled camera control section to ``Tiled Camera View``. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py index a65c6f78200c..7c4f82234d0f 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py @@ -10,6 +10,7 @@ import logging import math import os +import sys from typing import TYPE_CHECKING import warp as wp @@ -242,7 +243,7 @@ def _render_left_panel(self): # Newton's ImageLogger owns camera-output image windows. Since Isaac Lab overrides # ViewerGL's left panel, explicitly keep the logged-image selector and draw path. if self._image_logger is not None: - self._image_logger.draw_controls() + self._draw_tiled_camera_view_controls() imgui.set_next_item_open(True, imgui.Cond_.appearing) if imgui.collapsing_header("Camera"): @@ -269,6 +270,27 @@ def _render_left_panel(self): self._image_logger.draw() return + def _draw_tiled_camera_view_controls(self) -> None: + """Render Newton ImageLogger controls with Isaac Lab-specific naming.""" + image_logger = self._image_logger + if image_logger is None or not image_logger._images: + return + + imgui = self.ui.imgui + if not imgui.collapsing_header("Tiled Camera View", imgui.TreeNodeFlags_.default_open.value): + return + + names = list(image_logger._images.keys()) + items = ["Hide", *names] + if image_logger._selected is not None and image_logger._selected in names: + current = names.index(image_logger._selected) + 1 + else: + current = 0 + + changed, new_idx = imgui.combo("##tiled_camera_view", current, items) + if changed: + image_logger._selected = None if new_idx == 0 else names[new_idx - 1] + def _prime_image_logger_window_layout(self) -> None: """Make first-open image windows use the available viewer space. @@ -355,9 +377,11 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: self._model = NewtonManager.get_model() self._state = NewtonManager.get_state(self._scene_data_provider) - runtime_headless = self.cfg.headless or not os.environ.get("DISPLAY") + runtime_headless = self.cfg.headless or ( + sys.platform not in ("win32", "darwin") and not os.environ.get("DISPLAY") + ) - # Use pyglet's EGL headless backend when requested or when no X display is available. + # Use pyglet's EGL headless backend when requested or when no Linux X display is available. # This must run before the first ``pyglet.window`` import so ``Window`` resolves to # :class:`~pyglet.window.headless.HeadlessWindow`. if runtime_headless: diff --git a/source/isaaclab_visualizers/setup.py b/source/isaaclab_visualizers/setup.py index a2b2ee093a46..b02846e4b459 100644 --- a/source/isaaclab_visualizers/setup.py +++ b/source/isaaclab_visualizers/setup.py @@ -31,6 +31,7 @@ "rerun": [ "newton[sim] @ git+https://github.com/newton-physics/newton.git@v1.2.0", "rerun-sdk>=0.29.0", + "pyarrow==22.0.0", ], "viser": [ "newton[sim] @ git+https://github.com/newton-physics/newton.git@v1.2.0", diff --git a/source/isaaclab_visualizers/test/test_visualizer_cartpole_integration.py b/source/isaaclab_visualizers/test/test_visualizer_cartpole_integration.py deleted file mode 100644 index bcdeac5e2419..000000000000 --- a/source/isaaclab_visualizers/test/test_visualizer_cartpole_integration.py +++ /dev/null @@ -1,609 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Integration tests: cartpole env + per-backend visualizers (Kit Replicator, tiled camera, GL, Rerun, Viser). - -Visualizer packages use ``logging.getLogger(__name__)``, so loggers are named like -``isaaclab_visualizers.kit.kit_visualizer`` and ``isaaclab.visualizers.base_visualizer``. -:class:`~isaaclab.sim.simulation_context.SimulationContext` uses -``logging.getLogger(__name__)`` → ``isaaclab.sim.simulation_context``. - -We filter :class:`~pytest.LogCaptureFixture` records with :data:`_VIS_LOGGER_PREFIXES` -so only those namespaces count (not Omniverse, PhysX, or unrelated warnings). - -Set :data:`ASSERT_VISUALIZER_WARNINGS` to ``True`` locally or in CI if you want tests to -fail on WARNING-level records from those loggers; by default only ERROR+ fails. -""" - -from __future__ import annotations - -# Pyglet must use HeadlessWindow (EGL) before ``pyglet.window`` is imported so Newton -# ViewerGL can construct without an X11 display (matches ``headless=True`` on NewtonVisualizerCfg). -import pyglet - -pyglet.options["headless"] = True - -from isaaclab.app import AppLauncher - -# launch Kit app -simulation_app = AppLauncher(headless=True, enable_cameras=True).app - -import contextlib -import copy -import logging -import socket - -import numpy as np -import pytest -import torch -import warp as wp -from isaaclab_visualizers.kit import KitVisualizer, KitVisualizerCfg -from isaaclab_visualizers.newton import NewtonVisualizer, NewtonVisualizerCfg -from isaaclab_visualizers.rerun import RerunVisualizer, RerunVisualizerCfg -from isaaclab_visualizers.viser import ViserVisualizer, ViserVisualizerCfg - -import isaaclab.sim as sim_utils -from isaaclab.sim import SimulationContext - -from isaaclab_tasks.direct.cartpole.cartpole_camera_env import CartpoleCameraEnv -from isaaclab_tasks.direct.cartpole.cartpole_camera_presets_env_cfg import CartpoleCameraPresetsEnvCfg -from isaaclab_tasks.manager_based.classic.cartpole.cartpole_env_cfg import CartpolePhysicsCfg - -# When True, tests also fail on WARNING-level records from visualizer-related loggers. -ASSERT_VISUALIZER_WARNINGS = False - -_MAX_NON_BLACK_STEPS = 8 -"""Steps for tiled camera / Rerun / Viser smoke tests (early exit ok when non-black).""" - -_CARTPOLE_INTEGRATION_NUM_ENVS = 1 -"""Vectorized env count for cartpole + visualizer integration tests.""" - -_CARTPOLE_INTEGRATION_VISUALIZER_EYE: tuple[float, float, float] = (3.0, 3.0, 3.0) -"""Passed to :class:`~isaaclab.visualizers.visualizer_cfg.VisualizerCfg` subclasses (``eye``).""" - -_CARTPOLE_INTEGRATION_VISUALIZER_LOOKAT: tuple[float, float, float] = (-4.0, -4.0, 0.0) -"""Passed to visualizer cfgs (``lookat``); also applied to :class:`~isaaclab.envs.common.ViewerCfg` for the env.""" - -# Resolution overrides for this test module (cartpole preset defaults: tiled camera 100×100; Kit helper was 320×240). -_CARTPOLE_KIT_INTEGRATION_RENDER_RESOLUTION: tuple[int, int] = (600, 600) -"""Kit: Replicator ``render_product`` (width, height) for viewport RGB in the motion check.""" - -_CARTPOLE_NEWTON_INTEGRATION_WINDOW_SIZE: tuple[int, int] = (600, 600) -"""Newton: ``NewtonVisualizerCfg`` framebuffer (window_width × window_height) for ``get_frame()``.""" - -_CARTPOLE_TILED_CAMERA_INTEGRATION_WH: tuple[int, int] = (600, 600) -"""Tiled camera per-env tile width/height (preset default is 100×100); keeps ``observation_space`` consistent.""" - -_VIS_FRAME_TEST_STEPS = 60 -"""Steps for Kit / Newton frame capture: no early exit.""" - -# Motion check compares the 2nd vs last captured frame (e.g. 2nd vs 60th when *_STEPS* is 60). -_MOTION_FRAME_EARLY_IDX = 1 -"""0-based index of the *early* frame (2nd capture).""" - -_MOTION_FRAME_LATE_IDX = _VIS_FRAME_TEST_STEPS - 1 -"""0-based index of the *late* frame (e.g. 60th capture when :data:`_VIS_FRAME_TEST_STEPS` is 60).""" - -# Early vs late frame motion: void background stays similar; only count *strongly* differing pixels. -_FRAME_MOTION_CHANNEL_DIFF_THRESHOLD = 50 -"""A pixel counts as differing if max(|ΔR|, |ΔG|, |ΔB|) >= this (0–255 space).""" - -_FRAME_MOTION_MIN_DIFFERING_PIXELS = 100 -"""Minimum number of such pixels between early and late frames (stale/frozen viz should be near zero).""" - -_VIS_LOGGER_PREFIXES = ( - "isaaclab.visualizers", - "isaaclab_visualizers", - "isaaclab.sim.simulation_context", -) - - -def _logger_name_matches_visualizer_scope(logger_name: str) -> bool: - """Return True if *logger_name* is a visualizer / SimulationContext visualizer path.""" - return any(logger_name.startswith(prefix) for prefix in _VIS_LOGGER_PREFIXES) - - -def _assert_no_visualizer_log_issues(caplog: pytest.LogCaptureFixture, *, fail_on_warnings: bool | None = None) -> None: - """Fail if captured records include ERROR/CRITICAL (always) or WARNING (if *fail_on_warnings*). - - *fail_on_warnings* defaults to :data:`ASSERT_VISUALIZER_WARNINGS`. - """ - if fail_on_warnings is None: - fail_on_warnings = ASSERT_VISUALIZER_WARNINGS - - error_logs = [ - r for r in caplog.records if r.levelno >= logging.ERROR and _logger_name_matches_visualizer_scope(r.name) - ] - assert not error_logs, "Visualizer-related error logs: " + "; ".join( - f"{r.name}: {r.getMessage()}" for r in error_logs - ) - - if fail_on_warnings: - warning_logs = [ - r for r in caplog.records if r.levelno == logging.WARNING and _logger_name_matches_visualizer_scope(r.name) - ] - assert not warning_logs, "Visualizer-related warning logs: " + "; ".join( - f"{r.name}: {r.getMessage()}" for r in warning_logs - ) - - -def _configure_sim_for_visualizer_test(env: CartpoleCameraEnv) -> None: - """Settings used by the previous smoke tests; keep RTX sensors enabled for camera paths.""" - env.sim.set_setting("/isaaclab/render/rtx_sensors", True) - env.sim._app_control_on_stop_handle = None # type: ignore[attr-defined] - - -def _find_free_tcp_port(host: str = "127.0.0.1") -> int: - """Ask OS for a currently free local TCP port.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind((host, 0)) - return int(sock.getsockname()[1]) - - -def _allocate_rerun_test_ports(host: str = "127.0.0.1") -> tuple[int, int]: - """Allocate distinct free ports for rerun web and gRPC endpoints.""" - grpc_port = _find_free_tcp_port(host) - web_port = _find_free_tcp_port(host) - while web_port == grpc_port: - web_port = _find_free_tcp_port(host) - return web_port, grpc_port - - -def _cartpole_integration_visualizer_camera_kwargs() -> dict[str, tuple[float, float, float]]: - """Eye/lookat for all :class:`~isaaclab.visualizers.visualizer_cfg.VisualizerCfg` subclasses in these tests.""" - return { - "eye": _CARTPOLE_INTEGRATION_VISUALIZER_EYE, - "lookat": _CARTPOLE_INTEGRATION_VISUALIZER_LOOKAT, - } - - -def _get_visualizer_cfg(visualizer_kind: str): - """Return (visualizer_cfg, expected_visualizer_cls) for the given visualizer kind.""" - cam = _cartpole_integration_visualizer_camera_kwargs() - if visualizer_kind == "newton": - __import__("newton") - nw, nh = _CARTPOLE_NEWTON_INTEGRATION_WINDOW_SIZE - return ( - NewtonVisualizerCfg( - headless=True, - window_width=nw, - window_height=nh, - randomly_sample_visible_envs=False, - **cam, - ), - NewtonVisualizer, - ) - if visualizer_kind == "viser": - __import__("newton") - __import__("viser") - port = _find_free_tcp_port(host="127.0.0.1") - return ( - ViserVisualizerCfg(open_browser=False, port=port, randomly_sample_visible_envs=False, **cam), - ViserVisualizer, - ) - if visualizer_kind == "rerun": - __import__("newton") - __import__("rerun") - web_port, grpc_port = _allocate_rerun_test_ports(host="127.0.0.1") - return ( - RerunVisualizerCfg( - bind_address="127.0.0.1", - open_browser=False, - web_port=web_port, - grpc_port=grpc_port, - randomly_sample_visible_envs=False, - **cam, - ), - RerunVisualizer, - ) - return KitVisualizerCfg(randomly_sample_visible_envs=False, **cam), KitVisualizer - - -def _get_physics_cfg(backend_kind: str): - """Return physics config and expected backend substring for the given backend kind.""" - if backend_kind == "physx": - __import__("isaaclab_physx") - preset = CartpolePhysicsCfg() - physics_cfg = getattr(preset, "physx", None) - if physics_cfg is None: - from isaaclab_physx.physics import PhysxCfg - - physics_cfg = PhysxCfg() - return physics_cfg, "physx" - if backend_kind == "newton": - __import__("newton") - __import__("isaaclab_newton") - preset = CartpolePhysicsCfg() - physics_cfg = getattr(preset, "newton_mjwarp", None) - if physics_cfg is None: - from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg - - physics_cfg = NewtonCfg( - solver_cfg=MJWarpSolverCfg( - njmax=5, - nconmax=3, - cone="pyramidal", - impratio=1, - integrator="implicitfast", - ), - num_substeps=1, - debug_mode=False, - use_cuda_graph=True, - ) - return physics_cfg, "newton" - raise ValueError(f"Unknown backend: {backend_kind!r}") - - -def _assert_non_black_tensor(image_tensor: torch.Tensor, *, min_nonzero_pixels: int = 1) -> None: - """Assert camera-like tensor contains non-black pixels.""" - assert isinstance(image_tensor, torch.Tensor), f"Expected torch.Tensor, got {type(image_tensor)!r}" - assert image_tensor.numel() > 0, "Image tensor is empty." - finite_tensor = torch.where(torch.isfinite(image_tensor), image_tensor, torch.zeros_like(image_tensor)) - if finite_tensor.dtype.is_floating_point: - nonzero = torch.count_nonzero(torch.abs(finite_tensor) > 1e-6).item() - else: - nonzero = torch.count_nonzero(finite_tensor > 0).item() - assert nonzero >= min_nonzero_pixels, "Rendered frame appears black (no non-zero pixels)." - - -def _frame_to_numpy(frame) -> np.ndarray: - """Convert viewer ``get_frame()`` output (numpy, torch, or Warp array) to host ``numpy.ndarray``. - - ``np.asarray(wp.array)`` is unsafe: NumPy can trigger Warp indexing that raises at dimension edges. - """ - if isinstance(frame, np.ndarray): - return frame - if isinstance(frame, torch.Tensor): - return frame.detach().cpu().numpy() - if isinstance(frame, wp.array): - return wp.to_torch(frame).detach().cpu().numpy() - return np.asarray(frame) - - -def _assert_non_black_frame_array(frame) -> None: - """Assert viewer-captured frame has visible, non-black content.""" - frame_arr = _frame_to_numpy(frame) - assert frame_arr.size > 0, "Viewer returned an empty frame." - if frame_arr.ndim == 2: - color = frame_arr - else: - assert frame_arr.shape[-1] >= 3, f"Expected at least 3 channels, got shape {frame_arr.shape}." - color = frame_arr[..., :3] - finite = np.where(np.isfinite(color), color, 0) - assert np.count_nonzero(finite) > 0, "Viewer frame appears fully black." - - -def _frame_rgb_255_space(frame) -> np.ndarray: - """Return HxWx3 float in ~0–255 space for per-channel differencing.""" - arr = _frame_to_numpy(frame) - if arr.ndim == 2: - rgb = np.stack([arr, arr, arr], axis=-1) - else: - rgb = arr[..., :3] - rgb = np.asarray(rgb, dtype=np.float64) - # Normalized HDR buffers: scale so threshold matches (0,255) semantics. - if rgb.size > 0 and float(np.nanmax(rgb)) <= 1.0 + 1e-6: - rgb = rgb * 255.0 - return rgb - - -def _count_significantly_differing_pixels( - frame_a, - frame_b, - *, - channel_diff_threshold: float = _FRAME_MOTION_CHANNEL_DIFF_THRESHOLD, -) -> int: - """Count pixels where max(|ΔR|, |ΔG|, |ΔB|) >= *channel_diff_threshold* (0–255 space).""" - a = _frame_rgb_255_space(frame_a) - b = _frame_rgb_255_space(frame_b) - assert a.shape == b.shape, f"Frame shape mismatch for motion check: {a.shape} vs {b.shape}." - per_pixel_max = np.max(np.abs(a - b), axis=-1) - return int(np.count_nonzero(per_pixel_max >= channel_diff_threshold)) - - -def _assert_early_and_late_motion_frames_differ( - frames: list, - *, - channel_diff_threshold: float = _FRAME_MOTION_CHANNEL_DIFF_THRESHOLD, - min_differing_pixels: int = _FRAME_MOTION_MIN_DIFFERING_PIXELS, -) -> None: - """Fail if early vs late frames lack enough strongly differing pixels (stale/frozen bodies). - - Compares :data:`_MOTION_FRAME_EARLY_IDX` vs :data:`_MOTION_FRAME_LATE_IDX` (e.g. 2nd vs 60th capture). - - Voids/background stay near-identical; we only count pixels that change by at least - *channel_diff_threshold* on some channel (0–255). - """ - assert len(frames) >= _VIS_FRAME_TEST_STEPS, ( - f"Need at least {_VIS_FRAME_TEST_STEPS} frames for motion check, got {len(frames)}." - ) - i_early = _MOTION_FRAME_EARLY_IDX - i_late = _MOTION_FRAME_LATE_IDX - early_1 = i_early + 1 - late_1 = i_late + 1 - n_diff = _count_significantly_differing_pixels( - frames[i_early], frames[i_late], channel_diff_threshold=channel_diff_threshold - ) - assert n_diff >= min_differing_pixels, ( - f"Viewport captures #{early_1} and #{late_1} have too few strongly differing pixels " - f"({n_diff} < {min_differing_pixels}; threshold per channel={channel_diff_threshold} in 0–255 space). " - "Possible frozen or stale robot visualization." - ) - - -def _step_until_non_black_camera(env, actions: torch.Tensor, *, max_steps: int = _MAX_NON_BLACK_STEPS) -> None: - """Step env until the env's tiled camera RGB tensor is non-black, bounded by *max_steps*.""" - last_rgb = None - for _ in range(max_steps): - env.step(action=actions) - rgb = env._tiled_camera.data.output.get("rgb") - if rgb is None: - rgb = env._tiled_camera.data.output[env.cfg.tiled_camera.data_types[0]] - last_rgb = rgb.torch - try: - _assert_non_black_tensor(rgb.torch) - return - except AssertionError: - continue - _assert_non_black_tensor(last_rgb) - - -def _run_newton_viewer_frame_motion_test( - viewer, - *, - step_hook, - physics_kind: str, - viz_kind: str = "newton", -) -> None: - """Exactly ``_VIS_FRAME_TEST_STEPS`` sim steps; last frame non-black; early vs late motion check.""" - frames: list = [] - for _ in range(_VIS_FRAME_TEST_STEPS): - step_hook() - frames.append(viewer.get_frame()) - _assert_non_black_frame_array(frames[-1]) - _assert_early_and_late_motion_frames_differ(frames) - - -def _step_env_without_frame_check(env, actions: torch.Tensor, *, max_steps: int = _MAX_NON_BLACK_STEPS) -> None: - """Step the env to exercise visualizers that do not implement ``get_frame`` (e.g. Rerun, Viser).""" - for _ in range(max_steps): - env.step(action=actions) - - -def _build_rgb_annotator_for_camera( - camera_path: str, - *, - resolution: tuple[int, int] | None = None, -): - """Create CPU RGB annotator attached to a camera render product.""" - import omni.replicator.core as rep - - if resolution is None: - resolution = _CARTPOLE_KIT_INTEGRATION_RENDER_RESOLUTION - render_product = rep.create.render_product(camera_path, resolution=resolution) - annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu") - annotator.attach([render_product]) - return annotator, render_product - - -def _annotator_rgb_to_numpy(rgb_data) -> np.ndarray: - """Convert replicator annotator output to HxWx3 uint8 numpy array.""" - rgb_array = np.frombuffer(rgb_data, dtype=np.uint8).reshape(*rgb_data.shape) - if rgb_array.size == 0: - return np.zeros((1, 1, 3), dtype=np.uint8) - return rgb_array[:, :, :3] - - -def _run_kit_viewport_frame_motion_test( - env, - kit_visualizer: KitVisualizer, - *, - physics_kind: str, - viz_kind: str = "kit", -) -> None: - """Exactly ``_VIS_FRAME_TEST_STEPS`` env steps; last Replicator frame non-black; early vs late motion check.""" - camera_path = getattr(kit_visualizer, "_controlled_camera_path", None) - assert camera_path, "Kit visualizer does not expose a controlled viewport camera path." - - annotator = None - render_product = None - try: - annotator, render_product = _build_rgb_annotator_for_camera(camera_path) - actions = torch.zeros((env.num_envs, env.action_space.shape[-1]), device=env.device) - frames: list = [] - for _ in range(_VIS_FRAME_TEST_STEPS): - env.step(action=actions) - rgb_data = annotator.get_data() - frames.append(_annotator_rgb_to_numpy(rgb_data)) - _assert_non_black_frame_array(frames[-1]) - _assert_early_and_late_motion_frames_differ(frames) - finally: - if annotator is not None and render_product is not None: - with contextlib.suppress(Exception): - annotator.detach([render_product]) - - -def _make_cartpole_camera_env(visualizer_kind: str, backend_kind: str) -> CartpoleCameraEnv: - """Create cartpole camera env configured with selected visualizer and physics backend.""" - env_cfg_root = CartpoleCameraPresetsEnvCfg() - env_cfg = getattr(env_cfg_root, "default", None) - if env_cfg is None: - env_cfg = getattr(type(env_cfg_root), "default", None) - if env_cfg is None: - raise RuntimeError( - "CartpoleCameraPresetsEnvCfg does not expose a 'default' preset config. " - f"Available attributes: {sorted(vars(env_cfg_root).keys())}" - ) - env_cfg = copy.deepcopy(env_cfg) - env_cfg.scene.num_envs = _CARTPOLE_INTEGRATION_NUM_ENVS - env_cfg.viewer.eye = _CARTPOLE_INTEGRATION_VISUALIZER_EYE - env_cfg.viewer.lookat = _CARTPOLE_INTEGRATION_VISUALIZER_LOOKAT - tw, th = _CARTPOLE_TILED_CAMERA_INTEGRATION_WH - env_cfg.tiled_camera.width = tw - env_cfg.tiled_camera.height = th - if isinstance(env_cfg.observation_space, list) and len(env_cfg.observation_space) >= 3: - env_cfg.observation_space = [th, tw, env_cfg.observation_space[2]] - env_cfg.seed = None - env_cfg.sim.physics, _ = _get_physics_cfg(backend_kind) - visualizer_cfg, _ = _get_visualizer_cfg(visualizer_kind) - env_cfg.sim.visualizer_cfgs = visualizer_cfg - return CartpoleCameraEnv(env_cfg) - - -@pytest.mark.isaacsim_ci -@pytest.mark.parametrize( - "backend_kind", - [ - # xfail: Kit visualizer + PhysX only (Newton backend uses skip below — separate CUDA issue). - pytest.param( - "physx", - marks=pytest.mark.xfail( - reason=("Kit visualizer + PhysX: TODO remove xfail when stale Fabric transforms bug in Kit is fixed"), - strict=False, - ), - ), - pytest.param( - "newton", - marks=pytest.mark.skip( - reason=( - "TODO: Kit visualizer + Newton physics + Isaac RTX tiled camera can hit CUDA illegal access " - "or bad GPU state. Repro: rl_games train Isaac-Cartpole-Camera-Presets-Direct-v0 " - "--enable_cameras presets=newton_mjwarp --viz kit. Re-enable when fixed." - ) - ), - ), - ], -) -def test_cartpole_kit_visualizer_replicator_viewport_rgb_motion( - backend_kind: str, caplog: pytest.LogCaptureFixture -) -> None: - """Kit + cartpole: Replicator RGB on viewport camera; last frame non-black; early vs late frame differ; logs.""" - env = None - try: - sim_utils.create_new_stage() - env = _make_cartpole_camera_env(visualizer_kind="kit", backend_kind=backend_kind) - _configure_sim_for_visualizer_test(env) - with caplog.at_level(logging.WARNING): - env.reset() - kit_visualizers = [viz for viz in env.sim.visualizers if isinstance(viz, KitVisualizer)] - assert kit_visualizers, "Expected an initialized Kit visualizer." - _run_kit_viewport_frame_motion_test(env, kit_visualizers[0], physics_kind=backend_kind) - _assert_no_visualizer_log_issues(caplog) - finally: - if env is not None: - env.close() - else: - SimulationContext.clear_instance() - - -@pytest.mark.isaacsim_ci -@pytest.mark.parametrize("backend_kind", ["physx", "newton"]) -def test_cartpole_newton_visualizer_tiled_camera_rgb_non_black( - backend_kind: str, caplog: pytest.LogCaptureFixture -) -> None: - """Newton visualizer + cartpole: env tiled-camera RGB becomes non-black within a few steps; clean logs.""" - env = None - try: - sim_utils.create_new_stage() - env = _make_cartpole_camera_env(visualizer_kind="newton", backend_kind=backend_kind) - _configure_sim_for_visualizer_test(env) - with caplog.at_level(logging.WARNING): - env.reset() - actions = torch.zeros((env.num_envs, env.action_space.shape[-1]), device=env.device) - _step_until_non_black_camera(env, actions, max_steps=_MAX_NON_BLACK_STEPS) - _assert_no_visualizer_log_issues(caplog) - finally: - if env is not None: - env.close() - else: - SimulationContext.clear_instance() - - -@pytest.mark.isaacsim_ci -@pytest.mark.skip(reason="ViewerGL frame motion is flaky on the current pinned Isaac Sim CI image.") -@pytest.mark.parametrize("backend_kind", ["physx", "newton"]) -def test_cartpole_newton_visualizer_viewergl_rgb_motion(backend_kind: str, caplog: pytest.LogCaptureFixture) -> None: - """Newton GL (``ViewerGL.get_frame``): full motion steps, last frame non-black; early vs late differ; logs.""" - env = None - try: - sim_utils.create_new_stage() - env = _make_cartpole_camera_env(visualizer_kind="newton", backend_kind=backend_kind) - _configure_sim_for_visualizer_test(env) - with caplog.at_level(logging.WARNING): - env.reset() - actions = torch.zeros((env.num_envs, env.action_space.shape[-1]), device=env.device) - newton_visualizers = [viz for viz in env.sim.visualizers if isinstance(viz, NewtonVisualizer)] - assert newton_visualizers, "Expected an initialized Newton visualizer." - viewer = getattr(newton_visualizers[0], "_viewer", None) - assert viewer is not None, "Newton viewer was not created." - - def _step_env() -> None: - env.step(action=actions) - - _run_newton_viewer_frame_motion_test(viewer, step_hook=_step_env, physics_kind=backend_kind) - _assert_no_visualizer_log_issues(caplog) - finally: - if env is not None: - env.close() - else: - SimulationContext.clear_instance() - - -@pytest.mark.isaacsim_ci -@pytest.mark.parametrize("backend_kind", ["physx", "newton"]) -def test_cartpole_rerun_visualizer_smoke_steps_and_logs(backend_kind: str, caplog: pytest.LogCaptureFixture) -> None: - """Rerun + cartpole: visualizer and viewer initialize; env steps exercise the pipeline; clean logs. - - Rerun does not expose a per-frame RGB API like ``get_frame``, so we do not assert pixel content. - """ - env = None - try: - sim_utils.create_new_stage() - env = _make_cartpole_camera_env(visualizer_kind="rerun", backend_kind=backend_kind) - _configure_sim_for_visualizer_test(env) - with caplog.at_level(logging.WARNING): - env.reset() - actions = torch.zeros((env.num_envs, env.action_space.shape[-1]), device=env.device) - rerun_visualizers = [viz for viz in env.sim.visualizers if isinstance(viz, RerunVisualizer)] - assert rerun_visualizers, "Expected an initialized Rerun visualizer." - assert getattr(rerun_visualizers[0], "_viewer", None) is not None, "Rerun viewer was not created." - _step_env_without_frame_check(env, actions, max_steps=_MAX_NON_BLACK_STEPS) - _assert_no_visualizer_log_issues(caplog) - finally: - if env is not None: - env.close() - else: - SimulationContext.clear_instance() - - -@pytest.mark.isaacsim_ci -@pytest.mark.parametrize("backend_kind", ["physx", "newton"]) -def test_cartpole_viser_visualizer_smoke_steps_and_logs(backend_kind: str, caplog: pytest.LogCaptureFixture) -> None: - """Viser + cartpole: visualizer and viewer initialize; env steps exercise the pipeline; clean logs. - - No per-frame RGB assertion (Viser does not mirror the Newton ``get_frame`` path used elsewhere). - """ - env = None - try: - sim_utils.create_new_stage() - env = _make_cartpole_camera_env(visualizer_kind="viser", backend_kind=backend_kind) - _configure_sim_for_visualizer_test(env) - with caplog.at_level(logging.WARNING): - env.reset() - actions = torch.zeros((env.num_envs, env.action_space.shape[-1]), device=env.device) - viser_visualizers = [viz for viz in env.sim.visualizers if isinstance(viz, ViserVisualizer)] - assert viser_visualizers, "Expected an initialized Viser visualizer." - assert getattr(viser_visualizers[0], "_viewer", None) is not None, "Viser viewer was not created." - _step_env_without_frame_check(env, actions, max_steps=_MAX_NON_BLACK_STEPS) - _assert_no_visualizer_log_issues(caplog) - finally: - if env is not None: - env.close() - else: - SimulationContext.clear_instance() - - -if __name__ == "__main__": - pytest.main([__file__, "-v", "--maxfail=1"]) diff --git a/source/isaaclab_visualizers/test/test_visualizer_integration_newton.py b/source/isaaclab_visualizers/test/test_visualizer_integration_newton.py new file mode 100644 index 000000000000..358b8953e5c2 --- /dev/null +++ b/source/isaaclab_visualizers/test/test_visualizer_integration_newton.py @@ -0,0 +1,37 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Cartpole env + all non-tiled visualizers on Newton MJWarp.""" + +import sys +from pathlib import Path + +from isaaclab.app import AppLauncher + +# launch Kit app +simulation_app = AppLauncher(headless=True, enable_cameras=True).app + +import pytest # noqa: E402 + +_TEST_DIR = Path(__file__).resolve().parent +if str(_TEST_DIR) not in sys.path: + sys.path.insert(0, str(_TEST_DIR)) + +import visualizer_integration_utils as _viz_utils # noqa: E402 + +_viz_utils.set_visualizer_integration_simulation_app(simulation_app) + +run_cartpole_env_visualizers_motion_with_play_pause = _viz_utils.run_cartpole_env_visualizers_motion_with_play_pause + +pytestmark = [pytest.mark.isaacsim_ci, pytest.mark.flaky(max_runs=5, min_passes=1)] + + +def test_cartpole_env_visualizers_motion_with_play_pause_newton(caplog: pytest.LogCaptureFixture) -> None: + """Cartpole env + all non-tiled visualizers on Newton MJWarp.""" + run_cartpole_env_visualizers_motion_with_play_pause("newton", caplog) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/source/isaaclab_visualizers/test/test_visualizer_integration_physx.py b/source/isaaclab_visualizers/test/test_visualizer_integration_physx.py new file mode 100644 index 000000000000..0c668782de1d --- /dev/null +++ b/source/isaaclab_visualizers/test/test_visualizer_integration_physx.py @@ -0,0 +1,37 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Cartpole env + all non-tiled visualizers on PhysX.""" + +import sys +from pathlib import Path + +from isaaclab.app import AppLauncher + +# launch Kit app +simulation_app = AppLauncher(headless=True, enable_cameras=True).app + +import pytest # noqa: E402 + +_TEST_DIR = Path(__file__).resolve().parent +if str(_TEST_DIR) not in sys.path: + sys.path.insert(0, str(_TEST_DIR)) + +import visualizer_integration_utils as _viz_utils # noqa: E402 + +_viz_utils.set_visualizer_integration_simulation_app(simulation_app) + +run_cartpole_env_visualizers_motion_with_play_pause = _viz_utils.run_cartpole_env_visualizers_motion_with_play_pause + +pytestmark = [pytest.mark.isaacsim_ci, pytest.mark.flaky(max_runs=5, min_passes=1)] + + +def test_cartpole_env_visualizers_motion_with_play_pause_physx(caplog: pytest.LogCaptureFixture) -> None: + """Cartpole env + all non-tiled visualizers on PhysX.""" + run_cartpole_env_visualizers_motion_with_play_pause("physx", caplog) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/source/isaaclab_visualizers/test/test_visualizer_tiled_integration_newton.py b/source/isaaclab_visualizers/test/test_visualizer_tiled_integration_newton.py new file mode 100644 index 000000000000..45983dfb9065 --- /dev/null +++ b/source/isaaclab_visualizers/test/test_visualizer_tiled_integration_newton.py @@ -0,0 +1,37 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Cartpole env + tiled Kit/Newton visualizers on Newton MJWarp.""" + +import sys +from pathlib import Path + +from isaaclab.app import AppLauncher + +# launch Kit app +simulation_app = AppLauncher(headless=True, enable_cameras=True).app + +import pytest # noqa: E402 + +_TEST_DIR = Path(__file__).resolve().parent +if str(_TEST_DIR) not in sys.path: + sys.path.insert(0, str(_TEST_DIR)) + +import visualizer_integration_utils as _viz_utils # noqa: E402 + +_viz_utils.set_visualizer_integration_simulation_app(simulation_app) + +run_cartpole_env_visualizers_tiled_camera_motion = _viz_utils.run_cartpole_env_visualizers_tiled_camera_motion + +pytestmark = [pytest.mark.isaacsim_ci, pytest.mark.flaky(max_runs=5, min_passes=1)] + + +def test_visualizer_tiled_integration_newton(caplog: pytest.LogCaptureFixture) -> None: + """Cartpole env + tiled Kit/Newton visualizers on Newton MJWarp.""" + run_cartpole_env_visualizers_tiled_camera_motion("newton", caplog) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/source/isaaclab_visualizers/test/test_visualizer_tiled_integration_physx.py b/source/isaaclab_visualizers/test/test_visualizer_tiled_integration_physx.py new file mode 100644 index 000000000000..ee94c5c7be5e --- /dev/null +++ b/source/isaaclab_visualizers/test/test_visualizer_tiled_integration_physx.py @@ -0,0 +1,37 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Cartpole env + tiled Kit/Newton visualizers on PhysX.""" + +import sys +from pathlib import Path + +from isaaclab.app import AppLauncher + +# launch Kit app +simulation_app = AppLauncher(headless=True, enable_cameras=True).app + +import pytest # noqa: E402 + +_TEST_DIR = Path(__file__).resolve().parent +if str(_TEST_DIR) not in sys.path: + sys.path.insert(0, str(_TEST_DIR)) + +import visualizer_integration_utils as _viz_utils # noqa: E402 + +_viz_utils.set_visualizer_integration_simulation_app(simulation_app) + +run_cartpole_env_visualizers_tiled_camera_motion = _viz_utils.run_cartpole_env_visualizers_tiled_camera_motion + +pytestmark = [pytest.mark.isaacsim_ci, pytest.mark.flaky(max_runs=5, min_passes=1)] + + +def test_visualizer_tiled_integration_physx(caplog: pytest.LogCaptureFixture) -> None: + """Cartpole env + tiled Kit/Newton visualizers on PhysX.""" + run_cartpole_env_visualizers_tiled_camera_motion("physx", caplog) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/source/isaaclab_visualizers/test/visualizer_integration_utils.py b/source/isaaclab_visualizers/test/visualizer_integration_utils.py new file mode 100644 index 000000000000..8d1328a43f6b --- /dev/null +++ b/source/isaaclab_visualizers/test/visualizer_integration_utils.py @@ -0,0 +1,1571 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared helpers for Cartpole visualizer integration tests. + +The suite covers four visualizers: Kit, Newton, Rerun, and Viser. All visualizers +must initialize and step without visualizer-scoped log errors on both physics backends. + +Kit and Newton also expose image-producing paths, so they get stronger checks: +- frames are non-flat +- frames change while simulation is playing +- frames remain stable while rendering or simulation is paused +- frames change again after play resumes + +Newton has separate rendering-pause and simulation-pause controls, so those tests +also verify that physics continues during rendering pause and stays frozen during +simulation pause. +""" + +from __future__ import annotations + +import contextlib +import copy +import logging +import math +import os +import re +import socket +from pathlib import Path + +import numpy as np +import pytest +import torch +import warp as wp +from isaaclab_visualizers.kit import KitVisualizer, KitVisualizerCfg +from isaaclab_visualizers.newton import NewtonVisualizer, NewtonVisualizerCfg + +from pxr import UsdGeom + +import isaaclab.sim as sim_utils +from isaaclab.app import AppLauncher +from isaaclab.envs.utils.camera_view import camera_rgb_batch, compose_rgb_grid_tensor, prim_world_positions +from isaaclab.sim import SimulationContext + +from isaaclab_tasks.direct.cartpole.cartpole_camera_env import CartpoleCameraEnv +from isaaclab_tasks.direct.cartpole.cartpole_camera_presets_env_cfg import CartpoleCameraPresetsEnvCfg +from isaaclab_tasks.manager_based.classic.cartpole.cartpole_env_cfg import CartpolePhysicsCfg + +# TODO: Several test cases currently show flakiness with frozen bodies. Remove the test-level retry once fixed. + +# Debugging mode configs. + +_WRITE_VIS_DEBUG_FRAMES = False +"""Whether to emit visualizer debug PNGs during integration tests. Disabled by default.""" + +_VIS_DEBUG_IMAGE_DIR = Path("logs/viz_integration_captures") +"""Directory for opt-in visualizer debug images emitted by integration tests.""" + + +# When True, tests also fail on WARNING-level records from visualizer-related loggers. +ASSERT_VISUALIZER_WARNINGS = False + +_MAX_FRAME_CHECK_STEPS = 5 +"""Steps for Rerun / Viser smoke tests.""" + +_CARTPOLE_INTEGRATION_NUM_ENVS = 1 +"""Vectorized env count for cartpole + visualizer integration tests.""" + +_CARTPOLE_TILED_CAMERA_INTEGRATION_NUM_ENVS = 4 +"""Vectorized env count for generated visualizer tiled-camera integration tests.""" + +_CARTPOLE_INTEGRATION_VISUALIZER_EYE: tuple[float, float, float] = (2.25, 0.0, 3.5) +"""Passed to :class:`~isaaclab.visualizers.visualizer_cfg.VisualizerCfg` subclasses (``eye``).""" + +_CARTPOLE_INTEGRATION_VISUALIZER_LOOKAT: tuple[float, float, float] = (0.0, 0.0, 2.25) +"""Passed to visualizer cfgs (``lookat``); also applied to :class:`~isaaclab.envs.common.ViewerCfg` for the env.""" + +_CARTPOLE_INTEGRATION_TILED_CAMERA_EYE_OFFSET: tuple[float, float, float] = tuple( + eye - lookat for eye, lookat in zip(_CARTPOLE_INTEGRATION_VISUALIZER_EYE, _CARTPOLE_INTEGRATION_VISUALIZER_LOOKAT) +) +"""Generated tiled-camera target-relative eye offset matching the shared visualizer viewing direction.""" + +# Resolution overrides for this test module (cartpole preset defaults: tiled camera 100×100; Kit helper was 320×240). +_CARTPOLE_KIT_INTEGRATION_RENDER_RESOLUTION: tuple[int, int] = (400, 400) +"""Kit: Replicator ``render_product`` (width, height) for viewport RGB in the motion check.""" + +_CARTPOLE_NEWTON_INTEGRATION_WINDOW_SIZE: tuple[int, int] = (400, 400) +"""Newton: ``NewtonVisualizerCfg`` framebuffer (window_width × window_height) for ``get_frame()``.""" + +_CARTPOLE_TILED_CAMERA_INTEGRATION_WH: tuple[int, int] = (400, 400) +"""Tiled camera per-env tile width/height (preset default is 100×100); keeps ``observation_space`` consistent.""" + +_CARTPOLE_VISUALIZER_TILED_CAMERA_NUM_TILES = 4 +"""Number of generated visualizer camera tiles exercised by tiled-camera integration tests.""" + +_CARTPOLE_VISUALIZER_TILED_CAMERA_TARGET_PRIM_PATH = "/World/envs/*/Robot" +"""Cartpole articulation root prim followed by generated visualizer tiled cameras.""" + +_START_BUFFER_STEPS = 5 +"""Warmup physics steps before capturing the first debug frame.""" + +PLAY_VIZ_N_STEP = 20 +"""Steps to run for each motion or resumed-play segment.""" + +PAUSE_VIZ_N_STEP = 5 +"""Steps to run for each paused visualization segment.""" + +# Early vs late frame motion: void background stays similar; only count *strongly* differing pixels. +_FRAME_MOTION_CHANNEL_DIFF_THRESHOLD = 50 +"""A pixel counts as differing if max(|ΔR|, |ΔG|, |ΔB|) >= this (0–255 space).""" + +_FRAME_MOTION_MIN_DIFFERING_PIXELS = 100 +"""Minimum number of such pixels between early and late frames (stale/frozen viz should be near zero).""" + +_TILED_CAMERA_MOTION_CHANNEL_DIFF_THRESHOLD = 5 +"""Lower per-channel threshold for Cartpole's fixed tiled camera view, where motion is more subtle.""" + +_TILED_CAMERA_MOTION_MIN_DIFFERING_PIXELS = 25 +"""Minimum differing pixels for tiled camera motion checks.""" + +_FRAME_MIN_CHANNEL_RANGE = 10 +"""Minimum per-frame channel range to reject all-one-color images.""" + +_BODY_STATE_STABLE_MAX_DELTA = 1.0e-6 +"""Maximum body-state delta allowed while simulation is paused.""" + +_BODY_STATE_MOTION_MIN_DELTA = 1.0e-5 +"""Minimum body-state delta expected while physics continues to advance.""" + +_VIS_LOGGER_PREFIXES = ( + "isaaclab.visualizers", + "isaaclab_visualizers", + "isaaclab.sim.simulation_context", +) + +_PYTEST_CURRENT_TEST_SUFFIX_PATTERN = re.compile(r"\s+\((setup|call|teardown)\)$") +_VIS_DEBUG_TEST_ID_OVERRIDE_ENV = "ISAACLAB_VISUALIZER_DEBUG_TEST_ID" + +_DEBUG_TEST_DIR_PREFIXES = { + "test_cartpole_env_visualizers_motion_with_play_pause_physx": "visualizers_physx", + "test_cartpole_env_visualizers_motion_with_play_pause_newton": "visualizers_newton", + "test_visualizer_tiled_integration_physx": "visualizers_physx", + "test_visualizer_tiled_integration_newton": "visualizers_newton", +} + +_DEBUG_TEST_TILED_SUFFIXES = { + "test_visualizer_tiled_integration_physx", + "test_visualizer_tiled_integration_newton", +} + + +_BACKEND_DISPLAY_NAMES = { + "physx": "PhysX", + "newton": "Newton MJWarp", +} + +_VISUALIZER_DISPLAY_NAMES = { + "kit": "Kit Visualizer", + "newton": "Newton Visualizer", + "rerun": "Rerun Visualizer", + "viser": "Viser Visualizer", +} + +_SIMULATION_APP = None + + +def set_visualizer_integration_simulation_app(simulation_app) -> None: + """Register the Kit app launched by a backend-specific test module.""" + global _SIMULATION_APP + _SIMULATION_APP = simulation_app + + +def _visualizer_case_label(viz_kind: str, physics_kind: str) -> str: + visualizer = _VISUALIZER_DISPLAY_NAMES.get(viz_kind, f"{viz_kind.title()} Visualizer") + backend = _BACKEND_DISPLAY_NAMES.get(physics_kind, physics_kind) + return f"{visualizer} on {backend}" + + +def _logger_name_matches_visualizer_scope(logger_name: str) -> bool: + """Return True if *logger_name* is a visualizer / SimulationContext visualizer path.""" + return any(logger_name.startswith(prefix) for prefix in _VIS_LOGGER_PREFIXES) + + +def _assert_no_visualizer_log_issues(caplog: pytest.LogCaptureFixture, *, fail_on_warnings: bool | None = None) -> None: + """Fail if captured records include ERROR/CRITICAL (always) or WARNING (if *fail_on_warnings*). + + *fail_on_warnings* defaults to :data:`ASSERT_VISUALIZER_WARNINGS`. + """ + if fail_on_warnings is None: + fail_on_warnings = ASSERT_VISUALIZER_WARNINGS + + error_logs = [ + r for r in caplog.records if r.levelno >= logging.ERROR and _logger_name_matches_visualizer_scope(r.name) + ] + assert not error_logs, "Visualizer-related error logs: " + "; ".join( + f"{r.name}: {r.getMessage()}" for r in error_logs + ) + + if fail_on_warnings: + warning_logs = [ + r for r in caplog.records if r.levelno == logging.WARNING and _logger_name_matches_visualizer_scope(r.name) + ] + assert not warning_logs, "Visualizer-related warning logs: " + "; ".join( + f"{r.name}: {r.getMessage()}" for r in warning_logs + ) + + +def _configure_sim_for_visualizer_test(env: CartpoleCameraEnv) -> None: + """Settings used by the previous smoke tests; keep RTX sensors enabled for camera paths.""" + AppLauncher.apply_rtx_determinism_settings() + env.sim.set_setting("/isaaclab/render/rtx_sensors", True) + env.sim._app_control_on_stop_handle = None # type: ignore[attr-defined] + + +def _find_free_tcp_port(host: str = "127.0.0.1") -> int: + """Ask OS for a currently free local TCP port.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind((host, 0)) + return int(sock.getsockname()[1]) + + +def _allocate_rerun_test_ports(host: str = "127.0.0.1") -> tuple[int, int]: + """Allocate distinct free ports for rerun web and gRPC endpoints.""" + grpc_port = _find_free_tcp_port(host) + web_port = _find_free_tcp_port(host) + while web_port == grpc_port: + web_port = _find_free_tcp_port(host) + return web_port, grpc_port + + +def _cartpole_integration_visualizer_camera_kwargs() -> dict[str, tuple[float, float, float]]: + """Eye/lookat for all :class:`~isaaclab.visualizers.visualizer_cfg.VisualizerCfg` subclasses in these tests.""" + return { + "eye": _CARTPOLE_INTEGRATION_VISUALIZER_EYE, + "lookat": _CARTPOLE_INTEGRATION_VISUALIZER_LOOKAT, + } + + +def _get_visualizer_cfg(visualizer_kind: str, *, tiled_camera: bool = False): + """Return (visualizer_cfg, expected_visualizer_cls) for the given visualizer kind.""" + cam = _cartpole_integration_visualizer_camera_kwargs() + tiled_cam = ( + { + "tiled_cam_view": True, + "tiled_cam_num": _CARTPOLE_VISUALIZER_TILED_CAMERA_NUM_TILES, + "tiled_cam_prim_path": None, + "tiled_cam_eye": _CARTPOLE_INTEGRATION_TILED_CAMERA_EYE_OFFSET, + "tiled_cam_target_prim_path": _CARTPOLE_VISUALIZER_TILED_CAMERA_TARGET_PRIM_PATH, + } + if tiled_camera + else {} + ) + if visualizer_kind == "newton": + __import__("newton") + nw, nh = _CARTPOLE_NEWTON_INTEGRATION_WINDOW_SIZE + return ( + NewtonVisualizerCfg( + headless=True, + window_width=nw, + window_height=nh, + randomly_sample_visible_envs=False, + **tiled_cam, + **cam, + ), + NewtonVisualizer, + ) + if visualizer_kind == "viser": + __import__("newton") + __import__("viser") + from isaaclab_visualizers.viser import ViserVisualizer, ViserVisualizerCfg + + port = _find_free_tcp_port(host="127.0.0.1") + return ( + ViserVisualizerCfg(open_browser=False, port=port, randomly_sample_visible_envs=False, **cam), + ViserVisualizer, + ) + if visualizer_kind == "rerun": + __import__("newton") + from isaaclab_visualizers.rerun import RerunVisualizer, RerunVisualizerCfg + + web_port, grpc_port = _allocate_rerun_test_ports(host="127.0.0.1") + return ( + RerunVisualizerCfg( + bind_address="127.0.0.1", + open_browser=False, + web_port=web_port, + grpc_port=grpc_port, + randomly_sample_visible_envs=False, + **cam, + ), + RerunVisualizer, + ) + return ( + KitVisualizerCfg( + window_width=_CARTPOLE_KIT_INTEGRATION_RENDER_RESOLUTION[0], + window_height=_CARTPOLE_KIT_INTEGRATION_RENDER_RESOLUTION[1], + randomly_sample_visible_envs=False, + **tiled_cam, + **cam, + ), + KitVisualizer, + ) + + +def _get_physics_cfg(backend_kind: str): + """Return physics config and expected backend substring for the given backend kind.""" + if backend_kind == "physx": + __import__("isaaclab_physx") + preset = CartpolePhysicsCfg() + physics_cfg = getattr(preset, "physx", None) + if physics_cfg is None: + from isaaclab_physx.physics import PhysxCfg + + physics_cfg = PhysxCfg() + return physics_cfg, "physx" + if backend_kind == "newton": + __import__("newton") + __import__("isaaclab_newton") + preset = CartpolePhysicsCfg() + physics_cfg = getattr(preset, "newton_mjwarp", None) + if physics_cfg is None: + from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg + + physics_cfg = NewtonCfg( + solver_cfg=MJWarpSolverCfg( + njmax=5, + nconmax=3, + cone="pyramidal", + impratio=1, + integrator="implicitfast", + ), + num_substeps=1, + debug_mode=False, + use_cuda_graph=True, + ) + return physics_cfg, "newton" + raise ValueError(f"Unknown backend: {backend_kind!r}") + + +def _frame_to_numpy(frame) -> np.ndarray: + """Convert viewer ``get_frame()`` output (numpy, torch, or Warp array) to host ``numpy.ndarray``. + + ``np.asarray(wp.array)`` is unsafe: NumPy can trigger Warp indexing that raises at dimension edges. + """ + if isinstance(frame, np.ndarray): + return frame + if torch.is_tensor(frame): + return frame.detach().cpu().numpy() + if isinstance(frame, wp.array): + return wp.to_torch(frame).detach().cpu().numpy() + return np.asarray(frame) + + +def _assert_non_flat_frame_array(frame) -> None: + """Assert viewer-captured frame has non-flat content.""" + frame_arr = _frame_to_numpy(frame) + assert frame_arr.size > 0, "Viewer returned an empty frame." + if frame_arr.ndim != 2: + assert frame_arr.shape[-1] >= 3, f"Expected at least 3 channels, got shape {frame_arr.shape}." + rgb = _frame_rgb_255_space(frame) + channel_range = float(np.max(rgb) - np.min(rgb)) + assert channel_range >= _FRAME_MIN_CHANNEL_RANGE, ( + f"Viewer frame appears flat / single-color (channel range {channel_range:.3f} < {_FRAME_MIN_CHANNEL_RANGE})." + ) + + +def _frame_rgb_255_space(frame) -> np.ndarray: + """Return HxWx3 float in ~0–255 space for per-channel differencing.""" + arr = _frame_to_numpy(frame) + if arr.ndim == 2: + rgb = np.stack([arr, arr, arr], axis=-1) + else: + rgb = arr[..., :3] + rgb = np.asarray(rgb, dtype=np.float64) + # Normalized HDR buffers: scale so threshold matches (0,255) semantics. + if rgb.size > 0 and float(np.nanmax(rgb)) <= 1.0 + 1e-6: + rgb = rgb * 255.0 + return rgb + + +def _current_visualizer_debug_dir() -> Path: + override_test_id = os.environ.get(_VIS_DEBUG_TEST_ID_OVERRIDE_ENV) + if override_test_id: + safe_override_id = re.sub(r"[^A-Za-z0-9_.-]+", "_", override_test_id).strip("_").lower() + return _VIS_DEBUG_IMAGE_DIR / (safe_override_id or "manual_run") + current_test = os.environ.get("PYTEST_CURRENT_TEST", "manual_run") + test_id = _PYTEST_CURRENT_TEST_SUFFIX_PATTERN.sub("", current_test).split("::")[-1] + is_tiled_test = False + match = re.fullmatch(r"(?P[^\[]+)(?:\[(?P[^\]]+)\])?", test_id) + if match: + test_name = match.group("test_name") + prefix = _DEBUG_TEST_DIR_PREFIXES.get(test_name, test_name) + backend = match.group("backend") + if backend: + test_id = f"{prefix}_{backend}" + else: + test_id = prefix + is_tiled_test = test_name in _DEBUG_TEST_TILED_SUFFIXES + if is_tiled_test: + test_id = f"{test_id}_tiled" + safe_test_id = re.sub(r"[^A-Za-z0-9_.-]+", "_", test_id).strip("_").lower() or "manual_run" + return _VIS_DEBUG_IMAGE_DIR / safe_test_id + + +@contextlib.contextmanager +def _visualizer_debug_case(viz_kind: str, physics_kind: str, *, tiled: bool = False): + """Route debug PNGs to the same per-visualizer folders even in combined tests.""" + previous = os.environ.get(_VIS_DEBUG_TEST_ID_OVERRIDE_ENV) + test_id = f"{viz_kind}_viz_{physics_kind}" + if tiled: + test_id = f"{test_id}_tiled" + os.environ[_VIS_DEBUG_TEST_ID_OVERRIDE_ENV] = test_id + try: + yield + finally: + if previous is None: + os.environ.pop(_VIS_DEBUG_TEST_ID_OVERRIDE_ENV, None) + else: + os.environ[_VIS_DEBUG_TEST_ID_OVERRIDE_ENV] = previous + + +def _save_visualizer_debug_image(frame, file_name: str, *, tiled: bool = False) -> None: + """Save a visualizer frame to a clearly named PNG for pause/motion debugging.""" + if not _WRITE_VIS_DEBUG_FRAMES: + return + from PIL import Image + + rgb = np.clip(_frame_rgb_255_space(frame), 0, 255).astype(np.uint8) + debug_dir = _current_visualizer_debug_dir() + debug_dir.mkdir(parents=True, exist_ok=True) + Image.fromarray(rgb).save(debug_dir / file_name) + + +def _log_camera_debug(message: str) -> None: + """Print camera debug information when visualizer debug capture is enabled.""" + if _WRITE_VIS_DEBUG_FRAMES: + print(f"[visualizer-debug] {message}", flush=True) + + +def _matrix_to_xyz_euler_degrees(rotation_matrix: np.ndarray) -> tuple[float, float, float]: + """Convert a 3x3 rotation matrix to XYZ Euler angles in degrees for debug output.""" + sy = math.sqrt(rotation_matrix[0, 0] * rotation_matrix[0, 0] + rotation_matrix[1, 0] * rotation_matrix[1, 0]) + if sy > 1.0e-6: + x = math.atan2(rotation_matrix[2, 1], rotation_matrix[2, 2]) + y = math.atan2(-rotation_matrix[2, 0], sy) + z = math.atan2(rotation_matrix[1, 0], rotation_matrix[0, 0]) + else: + x = math.atan2(-rotation_matrix[1, 2], rotation_matrix[1, 1]) + y = math.atan2(-rotation_matrix[2, 0], sy) + z = 0.0 + return tuple(round(math.degrees(v), 4) for v in (x, y, z)) + + +def _log_usd_camera_pose(stage, camera_path: str, *, viz_kind: str, physics_kind: str, label: str) -> None: + """Print USD camera world transform when visualizer debug capture is enabled.""" + if not _WRITE_VIS_DEBUG_FRAMES: + return + if stage is None or not stage: + import omni.usd + + stage = omni.usd.get_context().get_stage() + if stage is None or not stage: + _log_camera_debug(f"{viz_kind}/{physics_kind}: {label} camera stage unavailable") + return + prim = stage.GetPrimAtPath(camera_path) + if not prim.IsValid(): + _log_camera_debug(f"{viz_kind}/{physics_kind}: {label} camera prim invalid at {camera_path}") + return + matrix = UsdGeom.Xformable(prim).ComputeLocalToWorldTransform(0.0) + translation = matrix.ExtractTranslation() + rotation_quat = matrix.ExtractRotationQuat() + rotation_matrix = np.array(matrix.ExtractRotationMatrix(), dtype=np.float64) + rotation_rows = tuple(tuple(round(float(v), 4) for v in row) for row in rotation_matrix) + _log_camera_debug( + f"{viz_kind}/{physics_kind}: {label} camera world translation={tuple(round(float(v), 4) for v in translation)}" + ) + _log_camera_debug( + f"{viz_kind}/{physics_kind}: {label} camera world rotation=" + f"(real={round(float(rotation_quat.GetReal()), 4)}, " + f"imaginary={tuple(round(float(v), 4) for v in rotation_quat.GetImaginary())})" + ) + _log_camera_debug(f"{viz_kind}/{physics_kind}: {label} camera world rotation matrix={rotation_rows}") + _log_camera_debug( + f"{viz_kind}/{physics_kind}: {label} camera world rotation xyz_euler_deg=" + f"{_matrix_to_xyz_euler_degrees(rotation_matrix)}" + ) + + +def _log_frame_stats(frame, *, label: str) -> None: + """Print basic image stats for a captured viewport frame.""" + if not _WRITE_VIS_DEBUG_FRAMES: + return + rgb = _frame_rgb_255_space(frame) + per_bg_delta = np.max(np.abs(rgb - 238.0), axis=-1) + _log_camera_debug( + f"{label}: frame shape={rgb.shape} min={float(np.min(rgb)):.3f} max={float(np.max(rgb)):.3f} " + f"mean={float(np.mean(rgb)):.3f} range={float(np.max(rgb) - np.min(rgb)):.3f} " + f"std={float(np.std(rgb)):.3f} non_bg>=5={int(np.count_nonzero(per_bg_delta >= 5))}" + ) + + +def _log_frame_pair_delta(frame_a, frame_b, *, label: str) -> None: + """Print per-frame-pair motion stats for debug captures.""" + if not _WRITE_VIS_DEBUG_FRAMES: + return + a = _frame_rgb_255_space(frame_a) + b = _frame_rgb_255_space(frame_b) + if a.shape != b.shape: + _log_camera_debug(f"{label}: frame shape mismatch {a.shape} vs {b.shape}") + return + delta = np.max(np.abs(a - b), axis=-1) + _log_camera_debug( + f"{label}: diff_pixels>=5={int(np.count_nonzero(delta >= 5))} " + f"diff_pixels>=50={int(np.count_nonzero(delta >= 50))} max_delta={float(np.max(delta)):.3f} " + f"mean_delta={float(np.mean(delta)):.3f}" + ) + + +def _log_kit_viewport_state(env, kit_visualizer: KitVisualizer, camera_path: str, *, label: str) -> None: + """Print Kit viewport camera/path state around render-product capture.""" + if not _WRITE_VIS_DEBUG_FRAMES: + return + active_camera = None + with contextlib.suppress(Exception): + if getattr(kit_visualizer, "_viewport_api", None) is not None: + active_camera = kit_visualizer._viewport_api.get_active_camera() + play_flag = None + with contextlib.suppress(Exception): + from isaaclab.app.settings_manager import get_settings_manager + + play_flag = get_settings_manager().get("/app/player/playSimulations") + _log_camera_debug( + f"kit/{label}: requested_camera_path={camera_path} active_camera={active_camera} " + f"playSimulations={play_flag} physics_step_count={getattr(env.sim, '_physics_step_count', None)}" + ) + _log_usd_camera_pose( + kit_visualizer._scene_data_provider.usd_stage, + camera_path, + viz_kind="kit", + physics_kind=label, + label="viewport", + ) + + +def _log_usd_prim_pose(stage, prim_path: str, *, label: str) -> None: + """Print one USD prim transform if it exists.""" + if not _WRITE_VIS_DEBUG_FRAMES: + return + prim = stage.GetPrimAtPath(prim_path) if stage is not None else None + if prim is None or not prim.IsValid(): + _log_camera_debug(f"{label}: prim invalid at {prim_path}") + return + matrix = UsdGeom.Xformable(prim).ComputeLocalToWorldTransform(0.0) + translation = matrix.ExtractTranslation() + _log_camera_debug(f"{label}: {prim_path} usd_translation={tuple(round(float(v), 4) for v in translation)}") + + +def _log_cartpole_runtime_state(env, *, label: str) -> None: + """Print cartpole sim data and USD transforms to diagnose render staleness.""" + if not _WRITE_VIS_DEBUG_FRAMES: + return + cartpole = env.scene.articulations["cartpole"] + try: + joint_pos = cartpole.data.joint_pos.torch[0].detach().cpu().tolist() + joint_vel = cartpole.data.joint_vel.torch[0].detach().cpu().tolist() + body_pos = cartpole.data.body_pos_w.torch[0].detach().cpu() + body_names = list(cartpole.body_names) + body_summary = { + name: tuple(round(float(v), 4) for v in body_pos[idx].tolist()) for idx, name in enumerate(body_names[:4]) + } + _log_camera_debug( + f"{label}: joint_pos={tuple(round(float(v), 5) for v in joint_pos)} " + f"joint_vel={tuple(round(float(v), 5) for v in joint_vel)} body_pos={body_summary}" + ) + except Exception as exc: + _log_camera_debug(f"{label}: failed to read cartpole runtime state: {exc}") + + stage = sim_utils.get_current_stage() + for prim_path in ( + "/World/envs/env_0/Robot", + "/World/envs/env_0/Robot/cart", + "/World/envs/env_0/Robot/pole", + ): + _log_usd_prim_pose(stage, prim_path, label=label) + + +def _save_visualizer_debug_delta(frame_a, frame_b, file_name: str, *, tiled: bool = False) -> None: + """Save an amplified absolute-difference image for a start/end frame pair.""" + if not _WRITE_VIS_DEBUG_FRAMES: + return + from PIL import Image + + a = _frame_rgb_255_space(frame_a) + b = _frame_rgb_255_space(frame_b) + assert a.shape == b.shape, f"Frame shape mismatch for delta image: {a.shape} vs {b.shape}." + delta = np.clip(np.abs(a - b) * 4.0, 0, 255).astype(np.uint8) + debug_dir = _current_visualizer_debug_dir() + debug_dir.mkdir(parents=True, exist_ok=True) + Image.fromarray(delta).save(debug_dir / file_name) + + +def _save_visualizer_debug_phase_images( + frame_a, + frame_b, + *, + prefix: str, + phase: str, + frame_start_idx: int, + frame_end_idx: int, + tiled: bool = False, +) -> None: + """Save start/end/delta PNGs for one visualizer test phase.""" + _save_visualizer_debug_image(frame_a, f"{prefix}a_{phase}_frame_{frame_start_idx:02d}.png", tiled=tiled) + _save_visualizer_debug_image(frame_b, f"{prefix}b_{phase}_frame_{frame_end_idx:02d}.png", tiled=tiled) + _save_visualizer_debug_delta( + frame_a, + frame_b, + f"{prefix}c_{phase}_frame_{frame_start_idx:02d}_{frame_end_idx:02d}_delta.png", + tiled=tiled, + ) + + +def _clear_visualizer_debug_frames() -> None: + if not _WRITE_VIS_DEBUG_FRAMES: + return + debug_dir = _current_visualizer_debug_dir() + debug_dir.mkdir(parents=True, exist_ok=True) + for path in debug_dir.glob("*.png"): + path.unlink() + + +def _count_significantly_differing_pixels( + frame_a, + frame_b, + *, + channel_diff_threshold: float = _FRAME_MOTION_CHANNEL_DIFF_THRESHOLD, +) -> int: + """Count pixels where max(|ΔR|, |ΔG|, |ΔB|) >= *channel_diff_threshold* (0–255 space).""" + a = _frame_rgb_255_space(frame_a) + b = _frame_rgb_255_space(frame_b) + assert a.shape == b.shape, f"Frame shape mismatch for motion check: {a.shape} vs {b.shape}." + per_pixel_max = np.max(np.abs(a - b), axis=-1) + return int(np.count_nonzero(per_pixel_max >= channel_diff_threshold)) + + +def _frame_shape_for_message(frame) -> tuple[int, ...]: + return tuple(_frame_rgb_255_space(frame).shape) + + +def _assert_frames_remain_stable( + frame_a, + frame_b, + *, + case_label: str, + phase: str, + debug_phase: str, + max_differing_pixels: int = 100, +) -> None: + """Assert two viewport frames are effectively unchanged while simulation is paused.""" + n_diff = _count_significantly_differing_pixels(frame_a, frame_b) + assert n_diff <= max_differing_pixels, ( + f"{case_label} failed to pause during {phase}: {n_diff} pixels differed, expected at most " + f"{max_differing_pixels}. Frame shape={_frame_shape_for_message(frame_a)}. " + f"Debug frames: {_current_visualizer_debug_dir()}/*{debug_phase}*.png." + ) + + +def _assert_frames_differ( + frame_a, + frame_b, + *, + case_label: str, + phase: str, + debug_phase: str, + channel_diff_threshold: float = _FRAME_MOTION_CHANNEL_DIFF_THRESHOLD, + min_differing_pixels: int = _FRAME_MOTION_MIN_DIFFERING_PIXELS, +) -> None: + """Fail if two frames lack enough strongly differing pixels (stale/frozen bodies).""" + n_diff = _count_significantly_differing_pixels(frame_a, frame_b, channel_diff_threshold=channel_diff_threshold) + assert n_diff >= min_differing_pixels, ( + f"{case_label} is frozen during {phase}: {n_diff} pixels differed, expected at least " + f"{min_differing_pixels} with per-channel threshold {channel_diff_threshold} in 0-255 space. " + ) + + +def _assert_tiled_camera_frames_differ(frame_a, frame_b, *, case_label: str, phase: str, debug_phase: str) -> None: + """Fail if tiled camera frames lack enough motion for the fixed Cartpole camera view.""" + _assert_frames_differ( + frame_a, + frame_b, + case_label=case_label, + phase=phase, + debug_phase=debug_phase, + channel_diff_threshold=_TILED_CAMERA_MOTION_CHANNEL_DIFF_THRESHOLD, + min_differing_pixels=_TILED_CAMERA_MOTION_MIN_DIFFERING_PIXELS, + ) + + +def _assert_tiled_camera_frame_non_flat(frame) -> None: + """Assert the tiled camera frame has visible content.""" + _assert_non_flat_frame_array(frame) + + +def _assert_tiled_camera_frames_remain_stable( + frame_a, frame_b, *, case_label: str, phase: str, debug_phase: str +) -> None: + """Assert tiled camera frames are stable.""" + _assert_frames_remain_stable(frame_a, frame_b, case_label=case_label, phase=phase, debug_phase=debug_phase) + + +def _cartpole_body_state(env) -> torch.Tensor: + """Return a compact body transform state for cartpole motion/stability checks.""" + cartpole = env.scene.articulations["cartpole"] + pos = cartpole.data.body_pos_w.torch + quat = cartpole.data.body_quat_w.torch + return torch.cat((pos.reshape(-1), quat.reshape(-1))).detach().clone() + + +def _body_state_delta(state_a: torch.Tensor, state_b: torch.Tensor) -> float: + """Return max absolute body-state delta.""" + assert state_a.shape == state_b.shape, f"Body state shape mismatch: {state_a.shape} vs {state_b.shape}." + return float(torch.max(torch.abs(state_a - state_b)).item()) + + +def _assert_body_state_changed( + state_a: torch.Tensor, + state_b: torch.Tensor, + *, + case_label: str, + phase: str, + min_delta: float = _BODY_STATE_MOTION_MIN_DELTA, +) -> None: + delta = _body_state_delta(state_a, state_b) + assert delta >= min_delta, ( + f"{case_label} physics/body state did not advance during {phase}: max body-state delta {delta:.6g}, " + f"expected at least {min_delta:.6g}." + ) + + +def _assert_body_state_stable( + state_a: torch.Tensor, + state_b: torch.Tensor, + *, + case_label: str, + phase: str, + max_delta: float = _BODY_STATE_STABLE_MAX_DELTA, +) -> None: + delta = _body_state_delta(state_a, state_b) + assert delta <= max_delta, ( + f"{case_label} physics/body state changed during {phase}: max body-state delta {delta:.6g}, " + f"expected at most {max_delta:.6g}." + ) + + +def _select_newton_training_control_button(viewer, target_label: str) -> None: + """Trigger one Newton visualizer training-control button by label.""" + + class _FakeImgui: + def separator(self): + pass + + def text(self, _text): + pass + + def button(self, label): + return label == target_label + + def slider_int(self, _label, value, _min_value, _max_value, _format): + return False, value + + def is_item_hovered(self): + return False + + def set_tooltip(self, _text): + pass + + viewer._render_training_controls(_FakeImgui()) + + +def _select_newton_pause_simulation_button(viewer) -> None: + """Trigger the Newton visualizer's Pause/Resume Simulation UI button.""" + label = "Resume Simulation" if viewer.is_training_paused() else "Pause Simulation" + _select_newton_training_control_button(viewer, label) + + +def _set_newton_simulation_paused(viewer, paused: bool) -> None: + """Put Newton visualizer simulation pause control into a desired state.""" + if viewer.is_training_paused() != paused: + _select_newton_pause_simulation_button(viewer) + + +def _select_newton_pause_rendering_button(viewer) -> None: + """Trigger the Newton visualizer's Pause/Resume Rendering UI button.""" + label = "Resume Rendering" if viewer.is_rendering_paused() else "Pause Rendering" + _select_newton_training_control_button(viewer, label) + + +def _set_newton_rendering_paused(viewer, paused: bool) -> None: + """Put Newton visualizer rendering pause control into a desired state.""" + if viewer.is_rendering_paused() != paused: + _select_newton_pause_rendering_button(viewer) + + +def _newton_camera_front(camera) -> tuple[float, float, float]: + """Return Newton camera front vector.""" + front = camera.get_front() + return tuple(float(v) for v in front) + + +def _run_newton_viewer_frame_motion_test( + env, + viewer, + *, + visualizer: NewtonVisualizer, + step_hook, + get_physics_step_count, + physics_kind: str, + viz_kind: str = "newton", +) -> None: + """Check Newton viewer motion, rendering pause, simulation pause, and resumed motion.""" + _clear_visualizer_debug_frames() + case_label = _visualizer_case_label(viz_kind, physics_kind) + _log_camera_debug( + f"{viz_kind}/{physics_kind}: viewer camera pos={tuple(float(v) for v in viewer.camera.pos)} " + f"dir={_newton_camera_front(viewer.camera)}" + ) + for _ in range(_START_BUFFER_STEPS): + step_hook() + + motion_start_frame = viewer.get_frame() + for _ in range(PLAY_VIZ_N_STEP): + step_hook() + play_end_idx = PLAY_VIZ_N_STEP + motion_end_frame = viewer.get_frame() + _save_visualizer_debug_phase_images( + motion_start_frame, + motion_end_frame, + prefix="1", + phase="playing", + frame_start_idx=0, + frame_end_idx=play_end_idx, + ) + _assert_non_flat_frame_array(motion_end_frame) + _assert_frames_differ( + motion_start_frame, + motion_end_frame, + case_label=case_label, + phase="playing", + debug_phase="playing", + ) + + rendering_pause_start_idx = play_end_idx + rendering_pause_end_idx = rendering_pause_start_idx + PAUSE_VIZ_N_STEP + + def _attempt_rendering_pause(): + _set_newton_rendering_paused(viewer, True) + rendering_paused_start_frame = viewer.get_frame() + rendering_pause_start_state = _cartpole_body_state(env) + physics_step_before_render_pause = get_physics_step_count() + for _ in range(PAUSE_VIZ_N_STEP): + step_hook() + rendering_pause_end_state = _cartpole_body_state(env) + rendering_paused_end_frame = viewer.get_frame() + _save_visualizer_debug_phase_images( + rendering_paused_start_frame, + rendering_paused_end_frame, + prefix="2", + phase="pausing_rendering", + frame_start_idx=rendering_pause_start_idx, + frame_end_idx=rendering_pause_end_idx, + ) + _assert_frames_remain_stable( + rendering_paused_start_frame, + rendering_paused_end_frame, + case_label=case_label, + phase="pausing_rendering", + debug_phase="pausing_rendering", + ) + return physics_step_before_render_pause, rendering_pause_start_state, rendering_pause_end_state + + physics_step_before_render_pause, rendering_pause_start_state, rendering_pause_end_state = ( + _attempt_rendering_pause() + ) + assert get_physics_step_count() > physics_step_before_render_pause, ( + f"{case_label} physics step count did not advance during pausing_rendering." + ) + _assert_body_state_changed( + rendering_pause_start_state, + rendering_pause_end_state, + case_label=case_label, + phase="pausing_rendering", + ) + + rendering_play_start_idx = rendering_pause_end_idx + rendering_play_end_idx = rendering_play_start_idx + PLAY_VIZ_N_STEP + + def _attempt_rendering_play(): + _set_newton_rendering_paused(viewer, False) + rendering_play_start_frame = viewer.get_frame() + for _ in range(PLAY_VIZ_N_STEP): + step_hook() + rendering_play_end_frame = viewer.get_frame() + _save_visualizer_debug_phase_images( + rendering_play_start_frame, + rendering_play_end_frame, + prefix="3", + phase="playing", + frame_start_idx=rendering_play_start_idx, + frame_end_idx=rendering_play_end_idx, + ) + _assert_non_flat_frame_array(rendering_play_end_frame) + _assert_frames_differ( + rendering_play_start_frame, + rendering_play_end_frame, + case_label=case_label, + phase="playing after rendering pause", + debug_phase="playing", + ) + + _attempt_rendering_play() + + simulation_pause_start_idx = rendering_play_end_idx + simulation_pause_end_idx = simulation_pause_start_idx + PAUSE_VIZ_N_STEP + + def _attempt_simulation_pause(): + _set_newton_simulation_paused(viewer, True) + simulation_paused_start_frame = viewer.get_frame() + simulation_pause_start_state = _cartpole_body_state(env) + physics_step_before_simulation_pause = get_physics_step_count() + for _ in range(PAUSE_VIZ_N_STEP): + visualizer.step(0.0) + simulation_pause_end_state = _cartpole_body_state(env) + simulation_paused_end_frame = viewer.get_frame() + _save_visualizer_debug_phase_images( + simulation_paused_start_frame, + simulation_paused_end_frame, + prefix="4", + phase="pausing_simulation", + frame_start_idx=simulation_pause_start_idx, + frame_end_idx=simulation_pause_end_idx, + ) + _assert_frames_remain_stable( + simulation_paused_start_frame, + simulation_paused_end_frame, + case_label=case_label, + phase="pausing_simulation", + debug_phase="pausing_simulation", + ) + return physics_step_before_simulation_pause, simulation_pause_start_state, simulation_pause_end_state + + physics_step_before_simulation_pause, simulation_pause_start_state, simulation_pause_end_state = ( + _attempt_simulation_pause() + ) + assert get_physics_step_count() == physics_step_before_simulation_pause, ( + f"{case_label} physics step count advanced during pausing_simulation." + ) + _assert_body_state_stable( + simulation_pause_start_state, + simulation_pause_end_state, + case_label=case_label, + phase="pausing_simulation", + ) + + simulation_play_start_idx = simulation_pause_end_idx + simulation_play_end_idx = simulation_play_start_idx + PLAY_VIZ_N_STEP + + def _attempt_simulation_play(): + _set_newton_simulation_paused(viewer, False) + simulation_play_start_frame = viewer.get_frame() + for _ in range(PLAY_VIZ_N_STEP): + step_hook() + simulation_play_end_frame = viewer.get_frame() + _save_visualizer_debug_phase_images( + simulation_play_start_frame, + simulation_play_end_frame, + prefix="5", + phase="playing", + frame_start_idx=simulation_play_start_idx, + frame_end_idx=simulation_play_end_idx, + ) + _assert_non_flat_frame_array(simulation_play_end_frame) + _assert_frames_differ( + simulation_play_start_frame, + simulation_play_end_frame, + case_label=case_label, + phase="playing after simulation pause", + debug_phase="playing", + ) + + _attempt_simulation_play() + + +def _step_env_without_frame_check(env, actions: torch.Tensor, *, max_steps: int = _MAX_FRAME_CHECK_STEPS) -> None: + """Step the env to exercise visualizers that do not implement ``get_frame`` (e.g. Rerun, Viser).""" + for _ in range(max_steps): + env.step(action=actions) + + +def _set_kit_simulation_paused(env, paused: bool) -> None: + """Put Kit simulation play/pause state into a desired state.""" + if paused: + env.sim.pause() + else: + env.sim.play() + + +def _build_rgb_annotator_for_camera( + camera_path: str, + *, + resolution: tuple[int, int] | None = None, +): + """Create CPU RGB annotator attached to a camera render product.""" + import omni.replicator.core as rep + + if resolution is None: + resolution = _CARTPOLE_KIT_INTEGRATION_RENDER_RESOLUTION + render_product = rep.create.render_product(camera_path, resolution=resolution) + annotator = rep.AnnotatorRegistry.get_annotator("rgb", device="cpu") + annotator.attach([render_product]) + return annotator, render_product + + +def _annotator_rgb_to_numpy(rgb_data) -> np.ndarray: + """Convert replicator annotator output to HxWx3 uint8 numpy array.""" + rgb_array = np.frombuffer(rgb_data, dtype=np.uint8).reshape(*rgb_data.shape) + if rgb_array.size == 0: + return np.zeros((1, 1, 3), dtype=np.uint8) + return rgb_array[:, :, :3] + + +def _update_active_simulation_app() -> None: + """Pump the active Kit app launched by the backend test module.""" + if _SIMULATION_APP is not None: + _SIMULATION_APP.update() + return + + from isaacsim import SimulationApp + + sim_app = None + if hasattr(SimulationApp, "_instance") and SimulationApp._instance is not None: + sim_app = SimulationApp._instance + elif hasattr(SimulationApp, "instance") and callable(SimulationApp.instance): + sim_app = SimulationApp.instance() + assert sim_app is not None, "Isaac Sim app is not running." + sim_app.update() + + +def _reapply_kit_camera_pose(env, kit_visualizer: KitVisualizer) -> None: + """Re-apply Kit camera pose after Newton MJWarp stage/render-product setup settles.""" + _log_camera_debug( + f"kit/reapply: cfg eye={tuple(float(v) for v in kit_visualizer.cfg.eye)} " + f"lookat={tuple(float(v) for v in kit_visualizer.cfg.lookat)}" + ) + kit_visualizer.set_camera_view(kit_visualizer.cfg.eye, kit_visualizer.cfg.lookat) + env.sim.render() + _update_active_simulation_app() + + +def _run_kit_viewport_frame_motion_test( + env, + kit_visualizer: KitVisualizer, + *, + physics_kind: str, + viz_kind: str = "kit", +) -> None: + """Check Kit viewport motion, SimulationContext pause freeze, then resumed motion.""" + _clear_visualizer_debug_frames() + case_label = _visualizer_case_label(viz_kind, physics_kind) + camera_path = getattr(kit_visualizer, "_controlled_camera_path", None) + assert camera_path, "Kit visualizer does not expose a controlled viewport camera path." + _log_camera_debug(f"{viz_kind}/{physics_kind}: controlled camera path={camera_path}") + _log_camera_debug(f"{viz_kind}/{physics_kind}: cfg eye={kit_visualizer.cfg.eye} lookat={kit_visualizer.cfg.lookat}") + _log_usd_camera_pose( + kit_visualizer._scene_data_provider.usd_stage, + camera_path, + viz_kind=viz_kind, + physics_kind=physics_kind, + label="initial", + ) + + annotator = None + render_product = None + try: + _log_kit_viewport_state(env, kit_visualizer, camera_path, label=f"{physics_kind}/before_render_product") + annotator, render_product = _build_rgb_annotator_for_camera(camera_path) + _log_kit_viewport_state(env, kit_visualizer, camera_path, label=f"{physics_kind}/after_render_product") + # TODO: Remove this workaround step during the Visualizer class refactor + if viz_kind == "kit" and physics_kind == "newton": + _reapply_kit_camera_pose(env, kit_visualizer) + _log_kit_viewport_state(env, kit_visualizer, camera_path, label=f"{physics_kind}/after_reapply") + actions = torch.zeros((env.num_envs, env.action_space.shape[-1]), device=env.device) + for _ in range(_START_BUFFER_STEPS): + env.step(action=actions) + _log_kit_viewport_state(env, kit_visualizer, camera_path, label=f"{physics_kind}/after_warmup") + warmup_body_state = _cartpole_body_state(env) + _log_cartpole_runtime_state(env, label=f"kit/{physics_kind}/after_warmup") + motion_start_frame = _capture_kit_viewport_rgb(annotator) + _log_frame_stats(motion_start_frame, label=f"kit/{physics_kind}/1a_playing_frame_00") + for _ in range(PLAY_VIZ_N_STEP): + env.step(action=actions) + play_end_idx = PLAY_VIZ_N_STEP + _log_kit_viewport_state(env, kit_visualizer, camera_path, label=f"{physics_kind}/after_play") + play_body_state = _cartpole_body_state(env) + _log_cartpole_runtime_state(env, label=f"kit/{physics_kind}/after_play") + _log_camera_debug( + f"kit/{physics_kind}/playing_body_delta={_body_state_delta(warmup_body_state, play_body_state):.6g}" + ) + motion_end_frame = _capture_kit_viewport_rgb(annotator) + _log_frame_stats(motion_end_frame, label=f"kit/{physics_kind}/1b_playing_frame_20") + _log_frame_pair_delta( + motion_start_frame, + motion_end_frame, + label=f"kit/{physics_kind}/1a_to_1b", + ) + _save_visualizer_debug_phase_images( + motion_start_frame, + motion_end_frame, + prefix="1", + phase="playing", + frame_start_idx=0, + frame_end_idx=play_end_idx, + ) + _assert_non_flat_frame_array(motion_end_frame) + _assert_frames_differ( + motion_start_frame, + motion_end_frame, + case_label=case_label, + phase="playing", + debug_phase="playing", + ) + + pause_start_idx = play_end_idx + pause_end_idx = pause_start_idx + PAUSE_VIZ_N_STEP + + def _attempt_kit_pause(): + _set_kit_simulation_paused(env, True) + paused_start_frame = _capture_kit_viewport_rgb(annotator) + _log_frame_stats(paused_start_frame, label=f"kit/{physics_kind}/2a_pausing_frame_20") + for _ in range(PAUSE_VIZ_N_STEP): + env.sim.render() + paused_end_frame = _capture_kit_viewport_rgb(annotator) + _log_frame_stats(paused_end_frame, label=f"kit/{physics_kind}/2b_pausing_frame_25") + _save_visualizer_debug_phase_images( + paused_start_frame, + paused_end_frame, + prefix="2", + phase="pausing", + frame_start_idx=pause_start_idx, + frame_end_idx=pause_end_idx, + ) + _assert_frames_remain_stable( + paused_start_frame, + paused_end_frame, + case_label=case_label, + phase="pausing", + debug_phase="pausing", + ) + + try: + _attempt_kit_pause() + finally: + _set_kit_simulation_paused(env, False) + + replay_start_idx = pause_end_idx + replay_end_idx = replay_start_idx + PLAY_VIZ_N_STEP + + def _attempt_kit_replay(): + _set_kit_simulation_paused(env, False) + play_start_frame = _capture_kit_viewport_rgb(annotator) + _log_frame_stats(play_start_frame, label=f"kit/{physics_kind}/3a_playing_frame_25") + for _ in range(PLAY_VIZ_N_STEP): + env.step(action=actions) + play_end_frame = _capture_kit_viewport_rgb(annotator) + _log_frame_stats(play_end_frame, label=f"kit/{physics_kind}/3b_playing_frame_45") + _log_usd_camera_pose( + kit_visualizer._scene_data_provider.usd_stage, + camera_path, + viz_kind=viz_kind, + physics_kind=physics_kind, + label="final", + ) + _save_visualizer_debug_phase_images( + play_start_frame, + play_end_frame, + prefix="3", + phase="playing", + frame_start_idx=replay_start_idx, + frame_end_idx=replay_end_idx, + ) + _assert_non_flat_frame_array(play_end_frame) + _assert_frames_differ( + play_start_frame, + play_end_frame, + case_label=case_label, + phase="playing after pause", + debug_phase="playing", + ) + + _attempt_kit_replay() + finally: + if annotator is not None and render_product is not None: + with contextlib.suppress(Exception): + annotator.detach([render_product]) + + +def _capture_kit_viewport_rgb(annotator) -> np.ndarray: + frame = _annotator_rgb_to_numpy(annotator.get_data()) + for _ in range(5): + if frame.shape[:2] != (1, 1) or np.count_nonzero(frame) > 0: + return frame + _update_active_simulation_app() + frame = _annotator_rgb_to_numpy(annotator.get_data()) + return frame + + +def _capture_visualizer_tiled_camera_rgb(visualizer, *, label: str = "capture") -> np.ndarray: + """Return the visualizer-owned/generated tiled camera RGB frame as an HxWx3 array.""" + camera_sensor = visualizer._camera_sensor + assert camera_sensor is not None, "Visualizer did not create a tiled camera sensor." + camera_indices = [int(index) for index in (visualizer._camera_sensor_indices or [0])] + if getattr(visualizer, "_camera_is_owned", False): + visualizer._update_owned_camera_poses() + if isinstance(visualizer, KitVisualizer): + visualizer._sync_camera_pose_updates_to_kit() + _update_active_simulation_app() + camera_sensor.update(dt=0.0, force_recompute=True) + rgb_batch = camera_rgb_batch(camera_sensor, camera_indices) + if _WRITE_VIS_DEBUG_FRAMES: + _log_visualizer_tiled_camera_state(visualizer, camera_indices, rgb_batch, label=label) + frame = compose_rgb_grid_tensor(rgb_batch).detach().cpu().numpy() + assert frame.ndim == 3, f"Expected tiled camera RGB frame to be HxWxC, got shape {frame.shape}." + assert frame.shape[-1] >= 3, f"Expected tiled camera RGB frame to have at least 3 channels, got {frame.shape}." + return frame[..., :3] + + +def _log_visualizer_tiled_camera_state( + visualizer, camera_indices: list[int], rgb_batch: torch.Tensor, *, label: str +) -> None: + """Print generated tiled-camera pose and image-buffer diagnostics.""" + cfg = visualizer.cfg + _log_camera_debug( + f"{visualizer.__class__.__name__}/{label}: tiled cfg env_indices={camera_indices} " + f"target={cfg.tiled_cam_target_prim_path!r} eye_offset={cfg.tiled_cam_eye}" + ) + _log_camera_debug( + f"{visualizer.__class__.__name__}/{label}: " + f"camera_sensor_indices={visualizer._camera_sensor_indices} " + f"camera_env_indices={visualizer._camera_env_indices} " + f"generated_paths={visualizer._generated_camera_prim_paths}" + ) + + try: + stage = visualizer._scene_data_provider.get_usd_stage() + scene = visualizer._scene_data_provider.get_interactive_scene() + except Exception as exc: + _log_camera_debug(f"{visualizer.__class__.__name__}/{label}: scene data provider unavailable: {exc}") + stage = None + scene = None + try: + target_positions = prim_world_positions(stage, cfg.tiled_cam_target_prim_path, camera_indices, scene=scene) + rounded_targets = [ + tuple(round(float(value), 4) for value in row) for row in target_positions.detach().cpu().tolist() + ] + _log_camera_debug(f"{visualizer.__class__.__name__}/{label}: resolved target positions={rounded_targets}") + except Exception as exc: + _log_camera_debug(f"{visualizer.__class__.__name__}/{label}: failed to resolve target positions: {exc}") + + _log_camera_sensor_pose_buffers(visualizer, camera_indices, label=label) + _log_scene_tiled_camera_rgb(visualizer, camera_indices, label=label) + + for local_idx, env_id in enumerate(camera_indices): + if 0 <= local_idx < len(visualizer._generated_camera_prim_paths): + camera_path = visualizer._generated_camera_prim_paths[local_idx] + else: + camera_path = f"/World/envs/env_{int(env_id)}/VisualizerCamera" + _log_usd_camera_pose( + stage, + camera_path, + viz_kind=visualizer.__class__.__name__, + physics_kind=label, + label=f"env_{int(env_id)} tiled", + ) + + rgb_cpu = rgb_batch.detach().cpu() + _log_camera_debug( + f"{visualizer.__class__.__name__}/{label}: rgb_batch shape={tuple(rgb_cpu.shape)} " + f"dtype={rgb_cpu.dtype} min={float(rgb_cpu.min()):.3f} max={float(rgb_cpu.max()):.3f} " + f"mean={float(rgb_cpu.float().mean()):.3f}" + ) + per_tile_rgb = rgb_cpu[..., :3].float() + if per_tile_rgb.ndim == 3: + per_tile_rgb = per_tile_rgb.unsqueeze(0) + per_tile_max_delta = torch.max(torch.abs(per_tile_rgb - 238.0), dim=-1).values + logged_indices = camera_indices[: per_tile_rgb.shape[0]] + if per_tile_rgb.shape[0] != len(camera_indices): + _log_camera_debug( + f"{visualizer.__class__.__name__}/{label}: tile count mismatch " + f"rgb_tiles={per_tile_rgb.shape[0]} camera_indices={camera_indices}" + ) + for tile_idx, env_id in enumerate(logged_indices): + tile = per_tile_rgb[tile_idx] + _log_camera_debug( + f"{visualizer.__class__.__name__}/{label}: tile[{tile_idx}] env={int(env_id)} " + f"range={float(tile.max() - tile.min()):.3f} " + f"std={float(tile.std()):.3f} " + f"non_bg>=5={int(torch.count_nonzero(per_tile_max_delta[tile_idx] >= 5).item())}" + ) + + +def _camera_debug_tensor(value) -> torch.Tensor: + """Convert Camera/ProxyArray diagnostics to a CPU tensor for logging.""" + if isinstance(value, wp.array): + return wp.to_torch(value).detach().cpu() + if hasattr(value, "torch"): + return value.torch.detach().cpu() + if isinstance(value, torch.Tensor): + return value.detach().cpu() + return torch.as_tensor(value).detach().cpu() + + +def _log_camera_sensor_pose_buffers(visualizer, camera_indices: list[int], *, label: str) -> None: + """Print the Camera sensor's own pose buffers for the selected tiles.""" + camera_sensor = visualizer._camera_sensor + try: + pos_w = _camera_debug_tensor(camera_sensor.data.pos_w)[camera_indices] + quat_w_world = _camera_debug_tensor(camera_sensor.data.quat_w_world)[camera_indices] + quat_w_opengl = _camera_debug_tensor(camera_sensor.data.quat_w_opengl)[camera_indices] + except Exception as exc: + _log_camera_debug(f"{visualizer.__class__.__name__}/{label}: failed to read camera data poses: {exc}") + return + rounded_positions = [tuple(round(float(value), 4) for value in row) for row in pos_w.tolist()] + rounded_world_quats = [tuple(round(float(value), 4) for value in row) for row in quat_w_world.tolist()] + rounded_opengl_quats = [tuple(round(float(value), 4) for value in row) for row in quat_w_opengl.tolist()] + _log_camera_debug(f"{visualizer.__class__.__name__}/{label}: sensor data pos_w={rounded_positions}") + _log_camera_debug(f"{visualizer.__class__.__name__}/{label}: sensor data quat_w_world={rounded_world_quats}") + _log_camera_debug(f"{visualizer.__class__.__name__}/{label}: sensor data quat_w_opengl={rounded_opengl_quats}") + + +def _log_scene_tiled_camera_rgb(visualizer, camera_indices: list[int], *, label: str) -> None: + """Print stats for the env-owned tiled camera, when present, as a renderer sanity check.""" + try: + scene = visualizer._scene_data_provider.get_interactive_scene() + scene_camera = scene.sensors.get("tiled_camera") + if scene_camera is None: + return + scene_camera.update(dt=0.0, force_recompute=True) + rgb = camera_rgb_batch(scene_camera, camera_indices).detach().cpu() + except Exception as exc: + _log_camera_debug(f"{visualizer.__class__.__name__}/{label}: failed to sample scene tiled camera: {exc}") + return + _log_camera_debug( + f"{visualizer.__class__.__name__}/{label}: scene tiled_camera rgb shape={tuple(rgb.shape)} " + f"min={float(rgb.min()):.3f} max={float(rgb.max()):.3f} mean={float(rgb.float().mean()):.3f}" + ) + + +def _run_visualizer_tiled_camera_motion_test(env, visualizer, *, physics_kind: str, viz_kind: str) -> None: + """Check generated visualizer tiled-camera RGB moves, pauses, and resumes.""" + _clear_visualizer_debug_frames() + case_label = f"{_visualizer_case_label(viz_kind, physics_kind)} tiled camera" + actions = torch.zeros((env.num_envs, env.action_space.shape[-1]), device=env.device) + for _ in range(_START_BUFFER_STEPS): + env.step(action=actions) + + motion_start_frame = _capture_visualizer_tiled_camera_rgb(visualizer, label="1a_playing_frame_00") + for _ in range(PLAY_VIZ_N_STEP): + env.step(action=actions) + play_end_idx = PLAY_VIZ_N_STEP + motion_end_frame = _capture_visualizer_tiled_camera_rgb(visualizer, label="1b_playing_frame_20") + _save_visualizer_debug_phase_images( + motion_start_frame, + motion_end_frame, + prefix="1", + phase="playing", + frame_start_idx=0, + frame_end_idx=play_end_idx, + tiled=True, + ) + _assert_tiled_camera_frame_non_flat(motion_end_frame) + _assert_tiled_camera_frames_differ( + motion_start_frame, + motion_end_frame, + case_label=case_label, + phase="playing", + debug_phase="playing_tiled", + ) + + pause_start_idx = play_end_idx + pause_end_idx = pause_start_idx + PAUSE_VIZ_N_STEP + + def _attempt_pause(): + _set_kit_simulation_paused(env, True) + paused_start_frame = _capture_visualizer_tiled_camera_rgb(visualizer, label="2a_pausing_frame_20") + for _ in range(PAUSE_VIZ_N_STEP): + env.sim.render() + paused_end_frame = _capture_visualizer_tiled_camera_rgb(visualizer, label="2b_pausing_frame_25") + _save_visualizer_debug_phase_images( + paused_start_frame, + paused_end_frame, + prefix="2", + phase="pausing", + frame_start_idx=pause_start_idx, + frame_end_idx=pause_end_idx, + tiled=True, + ) + _assert_tiled_camera_frame_non_flat(paused_end_frame) + _assert_tiled_camera_frames_remain_stable( + paused_start_frame, + paused_end_frame, + case_label=case_label, + phase="pausing", + debug_phase="pausing_tiled", + ) + + try: + _attempt_pause() + finally: + _set_kit_simulation_paused(env, False) + + replay_start_idx = pause_end_idx + replay_end_idx = replay_start_idx + PLAY_VIZ_N_STEP + + def _attempt_replay(): + _set_kit_simulation_paused(env, False) + play_start_frame = _capture_visualizer_tiled_camera_rgb(visualizer, label="3a_playing_frame_25") + for _ in range(PLAY_VIZ_N_STEP): + env.step(action=actions) + play_end_frame = _capture_visualizer_tiled_camera_rgb(visualizer, label="3b_playing_frame_45") + _save_visualizer_debug_phase_images( + play_start_frame, + play_end_frame, + prefix="3", + phase="playing", + frame_start_idx=replay_start_idx, + frame_end_idx=replay_end_idx, + tiled=True, + ) + _assert_tiled_camera_frame_non_flat(play_end_frame) + _assert_tiled_camera_frames_differ( + play_start_frame, + play_end_frame, + case_label=case_label, + phase="playing after pause", + debug_phase="playing_tiled", + ) + + _attempt_replay() + + +def _make_cartpole_camera_env( + visualizer_kind: str | tuple[str, ...], backend_kind: str, *, tiled_camera: bool = False +) -> CartpoleCameraEnv: + """Create cartpole camera env configured with selected visualizer and physics backend.""" + env_cfg_root = CartpoleCameraPresetsEnvCfg() + env_cfg = getattr(env_cfg_root, "default", None) + if env_cfg is None: + env_cfg = getattr(type(env_cfg_root), "default", None) + if env_cfg is None: + raise RuntimeError( + "CartpoleCameraPresetsEnvCfg does not expose a 'default' preset config. " + f"Available attributes: {sorted(vars(env_cfg_root).keys())}" + ) + env_cfg = copy.deepcopy(env_cfg) + env_cfg.scene.num_envs = ( + _CARTPOLE_TILED_CAMERA_INTEGRATION_NUM_ENVS if tiled_camera else _CARTPOLE_INTEGRATION_NUM_ENVS + ) + env_cfg.viewer.eye = _CARTPOLE_INTEGRATION_VISUALIZER_EYE + env_cfg.viewer.lookat = _CARTPOLE_INTEGRATION_VISUALIZER_LOOKAT + tw, th = _CARTPOLE_TILED_CAMERA_INTEGRATION_WH + env_cfg.tiled_camera.width = tw + env_cfg.tiled_camera.height = th + if isinstance(env_cfg.observation_space, list) and len(env_cfg.observation_space) >= 3: + env_cfg.observation_space = [th, tw, env_cfg.observation_space[2]] + env_cfg.seed = None + env_cfg.sim.physics, _ = _get_physics_cfg(backend_kind) + visualizer_kinds = (visualizer_kind,) if isinstance(visualizer_kind, str) else tuple(visualizer_kind) + visualizer_cfgs = [_get_visualizer_cfg(kind, tiled_camera=tiled_camera)[0] for kind in visualizer_kinds] + env_cfg.sim.visualizer_cfgs = visualizer_cfgs[0] if len(visualizer_cfgs) == 1 else visualizer_cfgs + return CartpoleCameraEnv(env_cfg) + + +def run_cartpole_env_visualizers_motion_with_play_pause(backend_kind: str, caplog: pytest.LogCaptureFixture) -> None: + """Cartpole env + all non-tiled visualizers: frame checks and no visualizer log errors.""" + env = None + try: + sim_utils.create_new_stage() + env = _make_cartpole_camera_env( + visualizer_kind=("kit", "newton", "rerun", "viser"), + backend_kind=backend_kind, + ) + _configure_sim_for_visualizer_test(env) + with caplog.at_level(logging.WARNING): + env.reset() + kit_visualizers = [viz for viz in env.sim.visualizers if isinstance(viz, KitVisualizer)] + assert kit_visualizers, "Expected an initialized Kit visualizer." + with _visualizer_debug_case("kit", backend_kind): + _run_kit_viewport_frame_motion_test(env, kit_visualizers[0], physics_kind=backend_kind) + + actions = torch.zeros((env.num_envs, env.action_space.shape[-1]), device=env.device) + newton_visualizers = [viz for viz in env.sim.visualizers if isinstance(viz, NewtonVisualizer)] + assert newton_visualizers, "Expected an initialized Newton visualizer." + viewer = getattr(newton_visualizers[0], "_viewer", None) + assert viewer is not None, "Newton viewer was not created." + + def _step_env() -> None: + env.step(action=actions) + + with _visualizer_debug_case("newton", backend_kind): + _run_newton_viewer_frame_motion_test( + env, + viewer, + visualizer=newton_visualizers[0], + step_hook=_step_env, + get_physics_step_count=lambda: env.sim._physics_step_count, + physics_kind=backend_kind, + ) + + from isaaclab_visualizers.rerun import RerunVisualizer + from isaaclab_visualizers.viser import ViserVisualizer + + rerun_visualizers = [viz for viz in env.sim.visualizers if isinstance(viz, RerunVisualizer)] + assert rerun_visualizers, "Expected an initialized Rerun visualizer." + assert getattr(rerun_visualizers[0], "_viewer", None) is not None, "Rerun viewer was not created." + _step_env_without_frame_check(env, actions, max_steps=_MAX_FRAME_CHECK_STEPS) + + viser_visualizers = [viz for viz in env.sim.visualizers if isinstance(viz, ViserVisualizer)] + assert viser_visualizers, "Expected an initialized Viser visualizer." + assert getattr(viser_visualizers[0], "_viewer", None) is not None, "Viser viewer was not created." + _step_env_without_frame_check(env, actions, max_steps=_MAX_FRAME_CHECK_STEPS) + _assert_no_visualizer_log_issues(caplog) + finally: + if env is not None: + env.close() + else: + SimulationContext.clear_instance() + + +def run_cartpole_env_visualizers_tiled_camera_motion(backend_kind: str, caplog: pytest.LogCaptureFixture) -> None: + """Cartpole env + tiled Kit/Newton visualizers: RGB moves, pauses, and resumes without log errors.""" + env = None + try: + sim_utils.create_new_stage() + env = _make_cartpole_camera_env( + visualizer_kind=("kit", "newton"), + backend_kind=backend_kind, + tiled_camera=True, + ) + _configure_sim_for_visualizer_test(env) + with caplog.at_level(logging.WARNING): + env.reset() + kit_visualizers = [viz for viz in env.sim.visualizers if isinstance(viz, KitVisualizer)] + assert kit_visualizers, "Expected an initialized Kit visualizer." + with _visualizer_debug_case("kit", backend_kind, tiled=True): + _run_visualizer_tiled_camera_motion_test( + env, kit_visualizers[0], physics_kind=backend_kind, viz_kind="kit" + ) + + newton_visualizers = [viz for viz in env.sim.visualizers if isinstance(viz, NewtonVisualizer)] + assert newton_visualizers, "Expected an initialized Newton visualizer." + with _visualizer_debug_case("newton", backend_kind, tiled=True): + _run_visualizer_tiled_camera_motion_test( + env, newton_visualizers[0], physics_kind=backend_kind, viz_kind="newton" + ) + _assert_no_visualizer_log_issues(caplog) + finally: + if env is not None: + env.close() + else: + SimulationContext.clear_instance() diff --git a/tools/template/templates/agents/skrl_amp_cfg b/tools/template/templates/agents/skrl_amp_cfg index 0946e4c6e6fa..be95d6447ade 100644 --- a/tools/template/templates/agents/skrl_amp_cfg +++ b/tools/template/templates/agents/skrl_amp_cfg @@ -70,12 +70,14 @@ agent: learning_rate: 5.0e-05 learning_rate_scheduler: null learning_rate_scheduler_kwargs: null - state_preprocessor: RunningStandardScaler + observation_preprocessor: RunningStandardScaler + observation_preprocessor_kwargs: null + state_preprocessor: null state_preprocessor_kwargs: null value_preprocessor: RunningStandardScaler value_preprocessor_kwargs: null - amp_state_preprocessor: RunningStandardScaler - amp_state_preprocessor_kwargs: null + amp_observation_preprocessor: RunningStandardScaler + amp_observation_preprocessor_kwargs: null random_timesteps: 0 learning_starts: 0 grad_norm_clip: 0.0 @@ -86,10 +88,9 @@ agent: value_loss_scale: 2.5 discriminator_loss_scale: 5.0 amp_batch_size: 512 - task_reward_weight: 0.0 - style_reward_weight: 1.0 + task_reward_scale: 0.0 + style_reward_scale: 2.0 discriminator_batch_size: 4096 - discriminator_reward_scale: 2.0 discriminator_logit_regularization_scale: 0.05 discriminator_gradient_penalty_scale: 5.0 discriminator_weight_decay_scale: 1.0e-04 diff --git a/tools/template/templates/agents/skrl_ippo_cfg b/tools/template/templates/agents/skrl_ippo_cfg index a89939f95543..b9a699bb9616 100644 --- a/tools/template/templates/agents/skrl_ippo_cfg +++ b/tools/template/templates/agents/skrl_ippo_cfg @@ -49,7 +49,9 @@ agent: learning_rate_scheduler: KLAdaptiveLR learning_rate_scheduler_kwargs: kl_threshold: 0.008 - state_preprocessor: RunningStandardScaler + observation_preprocessor: RunningStandardScaler + observation_preprocessor_kwargs: null + state_preprocessor: null state_preprocessor_kwargs: null value_preprocessor: RunningStandardScaler value_preprocessor_kwargs: null diff --git a/tools/template/templates/agents/skrl_mappo_cfg b/tools/template/templates/agents/skrl_mappo_cfg index 255b30eac810..968c1a9ba5df 100644 --- a/tools/template/templates/agents/skrl_mappo_cfg +++ b/tools/template/templates/agents/skrl_mappo_cfg @@ -23,7 +23,7 @@ models: clip_actions: False network: - name: net - input: OBSERVATIONS + input: STATES layers: [32, 32] activations: elu output: ONE @@ -49,10 +49,10 @@ agent: learning_rate_scheduler: KLAdaptiveLR learning_rate_scheduler_kwargs: kl_threshold: 0.008 + observation_preprocessor: RunningStandardScaler + observation_preprocessor_kwargs: null state_preprocessor: RunningStandardScaler state_preprocessor_kwargs: null - shared_state_preprocessor: RunningStandardScaler - shared_state_preprocessor_kwargs: null value_preprocessor: RunningStandardScaler value_preprocessor_kwargs: null random_timesteps: 0 diff --git a/tools/wheel_builder/res/python_packages.toml b/tools/wheel_builder/res/python_packages.toml index e0a207f77e38..9c35e993cab7 100644 --- a/tools/wheel_builder/res/python_packages.toml +++ b/tools/wheel_builder/res/python_packages.toml @@ -41,6 +41,7 @@ pyproject.dependencies.all = [ # visualizers "imgui-bundle==1.92.4", "rerun-sdk>=0.29.0", + "pyarrow==22.0.0", # viser is intentionally not a base dep: viser>=1.0.16 pulls websockets>=13.1, # but isaacsim-kernel==6.0.0.0 pins websockets==12.0. Users who want the viser # visualizer install isaaclab[viser] explicitly (see the optional-dependencies