From e6e607ad94eae7c042151fb89734b8e394dfd3e1 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Tue, 26 May 2026 14:34:32 +0200 Subject: [PATCH 01/10] Install ovphysx wheel by default in `--install` The ovphysx wheel is now publicly available on PyPI, so promote it from the opt-in `ov[ovphysx]` extra into the default `./isaaclab.sh --install` flow, mirroring how the newton wheel is treated. The OVRTX renderer wheel stays opt-in via `--install 'ov[ovrtx]'` or `--install 'ov[all]'`. Concretely, drop `ov` from MANUAL_EXTRA_FEATURES and make an empty `ov` selector install just the ovphysx extra (full `ov[all]` and `ov[ovrtx]` behavior is unchanged). With the wheel guaranteed in every standard install, also remove the `pytest.importorskip("ovphysx.types")` guards from the isaaclab_ovphysx test suite -- a missing wheel should now be a hard failure, not a silent skip. --- .../antoiner-ovphysx-default-install.rst | 8 ++++ source/isaaclab/isaaclab/cli/__init__.py | 5 ++- .../isaaclab/isaaclab/cli/commands/install.py | 39 ++++++++++--------- .../test_install_command_parsing.py | 17 ++++++-- .../antoiner-ovphysx-default-install.rst | 7 ++++ .../test/assets/test_articulation.py | 34 +++++++--------- .../test/assets/test_articulation_helpers.py | 14 ++----- .../test/assets/test_rigid_object.py | 22 ++++------- .../assets/test_rigid_object_collection.py | 22 ++++------- .../test/assets/test_rigid_object_helpers.py | 13 +------ .../test_ovphysx_scene_data_backend.py | 7 ---- .../test/sensors/test_contact_sensor.py | 28 ++++++------- .../test/sim/test_views_xform_prim_ovphysx.py | 14 ++----- 13 files changed, 101 insertions(+), 129 deletions(-) create mode 100644 source/isaaclab/changelog.d/antoiner-ovphysx-default-install.rst create mode 100644 source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-default-install.rst diff --git a/source/isaaclab/changelog.d/antoiner-ovphysx-default-install.rst b/source/isaaclab/changelog.d/antoiner-ovphysx-default-install.rst new file mode 100644 index 000000000000..696be8f314d7 --- /dev/null +++ b/source/isaaclab/changelog.d/antoiner-ovphysx-default-install.rst @@ -0,0 +1,8 @@ +Changed +^^^^^^^ + +* Changed ``./isaaclab.sh --install`` (and ``--install all``) to automatically + install the publicly available :mod:`ovphysx` wheel alongside the Newton + extras, so the OVPhysX backend is usable out of the box. The OVRTX renderer + wheel remains opt-in via ``--install 'ov[ovrtx]'`` or ``--install 'ov[all]'``. + Use ``--install none`` to opt out of all extras. diff --git a/source/isaaclab/isaaclab/cli/__init__.py b/source/isaaclab/isaaclab/cli/__init__.py index 0d71d4e48bc0..7df2fc3f19b0 100644 --- a/source/isaaclab/isaaclab/cli/__init__.py +++ b/source/isaaclab/isaaclab/cli/__init__.py @@ -98,8 +98,9 @@ def cli() -> None: "\n" "* Special values:\n" " all - Core + optional submodules (mimic, teleop) + auto extra\n" - " features (newton, rl, visualizer). Does not install contrib/ov\n" - " dependency extras (default).\n" + " features (newton, ov, rl, visualizer). 'ov' defaults to\n" + " ovphysx only; use ov[ovrtx] or ov[all] for OVRTX. Does not\n" + " install contrib dependency extras (default).\n" " none - Core submodules only; no optional submodules, no extra features.\n" " (-i with no value) - Same as 'all'.\n" "\n" diff --git a/source/isaaclab/isaaclab/cli/commands/install.py b/source/isaaclab/isaaclab/cli/commands/install.py index 2dfc1f3d4a58..423295e6b46a 100644 --- a/source/isaaclab/isaaclab/cli/commands/install.py +++ b/source/isaaclab/isaaclab/cli/commands/install.py @@ -471,7 +471,7 @@ def _install_isaacsim() -> None: } # Extra features excluded from the automatic ``-i all`` / ``-i`` install. -MANUAL_EXTRA_FEATURES: set[str] = {"contrib", "ov"} +MANUAL_EXTRA_FEATURES: set[str] = {"contrib"} def _split_install_items(install_type: str) -> list[str]: @@ -565,31 +565,32 @@ def _install_contrib_extra_dependencies(selector: str) -> None: def _install_ov_extra_dependencies(selector: str) -> None: """Install optional OV runtime dependencies. + The default ``ov`` invocation (empty selector) installs the publicly + available :mod:`ovphysx` wheel so the OVPhysX backend is usable out of + the box; the OVRTX renderer wheel remains opt-in via ``ov[ovrtx]`` or + ``ov[all]``. + Args: selector: One or more OV selectors from ``ov[ovrtx]``, - ``ov[ovphysx]``, or ``ov[all]``. + ``ov[ovphysx]``, or ``ov[all]``. Empty defaults to ``ovphysx``. """ - if not selector: - print_info( - "OV source packages are installed with the core submodules. " - "Use 'ov[ovrtx]', 'ov[ovphysx]', or 'ov[all]' to install OV runtime dependencies." - ) - return - python_exe = extract_python_exe() pip_cmd = get_pip_command(python_exe) source_dir = ISAACLAB_ROOT / "source" - selectors = {item.strip().lower() for item in selector.split(",") if item.strip()} - valid_selectors = {"all", "ovrtx", "ovphysx"} - unknown_selectors = selectors - valid_selectors - if unknown_selectors: - print_warning( - f"Unknown ov selector(s): {', '.join(sorted(unknown_selectors))}. " - f"Valid selectors: {', '.join(sorted(valid_selectors))}." - ) - if "all" in selectors: - selectors.update({"ovrtx", "ovphysx"}) + if not selector: + selectors = {"ovphysx"} + else: + selectors = {item.strip().lower() for item in selector.split(",") if item.strip()} + valid_selectors = {"all", "ovrtx", "ovphysx"} + unknown_selectors = selectors - valid_selectors + if unknown_selectors: + print_warning( + f"Unknown ov selector(s): {', '.join(sorted(unknown_selectors))}. " + f"Valid selectors: {', '.join(sorted(valid_selectors))}." + ) + if "all" in selectors: + selectors.update({"ovrtx", "ovphysx"}) if "ovrtx" in selectors: print_info("Installing OVRTX optional dependency...") run_command(pip_cmd + ["install", "--editable", f"{source_dir}/isaaclab_ov[ovrtx]"]) diff --git a/source/isaaclab/test/install_ci/test_install_command_parsing.py b/source/isaaclab/test/install_ci/test_install_command_parsing.py index 0313843b59a0..8d8ec4c7d266 100644 --- a/source/isaaclab/test/install_ci/test_install_command_parsing.py +++ b/source/isaaclab/test/install_ci/test_install_command_parsing.py @@ -139,7 +139,7 @@ def test_manual_extra_features_subset_of_valid(self): assert MANUAL_EXTRA_FEATURES <= VALID_EXTRA_FEATURES def test_manual_extra_features(self): - assert {"contrib", "ov"} == MANUAL_EXTRA_FEATURES + assert {"contrib"} == MANUAL_EXTRA_FEATURES def test_no_overlap_between_optional_submodules_and_extra_features(self): assert not (set(OPTIONAL_ISAACLAB_SUBMODULES.keys()) & VALID_EXTRA_FEATURES) @@ -237,7 +237,16 @@ def test_all_does_not_install_manual_extra_dependencies(self): mocks = self._run("all") called_features = {c.args[0] for c in mocks["_install_extra_feature"].call_args_list} assert "contrib" not in called_features - assert "ov" not in called_features + + def test_all_installs_ov_with_default_ovphysx_selector(self): + # `ov` is in the auto-installed set so the publicly available + # OVPhysX wheel is pulled in by default; OVRTX remains opt-in via + # `ov[ovrtx]` / `ov[all]`. + mocks = self._run("all") + calls = mocks["_install_extra_feature"].call_args_list + ov_calls = [c for c in calls if c.args[0] == "ov"] + assert ov_calls, "`ov` should be auto-installed by `-i all`" + assert ov_calls[0].args[1] == "", f"Expected empty selector, got {ov_calls[0].args[1]!r}" def test_all_does_not_call_install_isaacsim(self): mocks = self._run("all") @@ -303,12 +312,12 @@ def test_mimic_adds_mimic_to_submodules(self): mocks["_install_extra_feature"].assert_not_called() mocks["_install_optional_submodule_extra_dependencies"].assert_not_called() - def test_ov_without_selector_dispatches_manual_extra_feature(self): + def test_ov_without_selector_dispatches_extra_feature(self): mocks = self._run("ov") mocks["_install_extra_feature"].assert_called_once_with("ov", "") mocks["_install_optional_submodule_extra_dependencies"].assert_not_called() - def test_ov_with_selector_dispatches_manual_extra_feature(self): + def test_ov_with_selector_dispatches_extra_feature(self): mocks = self._run("ov[ovrtx]") mocks["_install_extra_feature"].assert_called_once_with("ov", "ovrtx") mocks["_install_optional_submodule_extra_dependencies"].assert_not_called() diff --git a/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-default-install.rst b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-default-install.rst new file mode 100644 index 000000000000..c91c85828cd1 --- /dev/null +++ b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-default-install.rst @@ -0,0 +1,7 @@ +Changed +^^^^^^^ + +* Removed ``pytest.importorskip("ovphysx.types")`` guards from the + ``isaaclab_ovphysx`` test suite now that the ``ovphysx`` wheel is publicly + available on PyPI and installed by default via ``./isaaclab.sh --install``. + Tests now fail loudly if the wheel is missing instead of silently skipping. diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation.py b/source/isaaclab_ovphysx/test/assets/test_articulation.py index e4b052065127..64a0dea2567f 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation.py @@ -55,26 +55,20 @@ import pytest import torch import warp as wp - -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. -pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") - -from isaaclab_ovphysx import tensor_types as TT # noqa: E402 -from isaaclab_ovphysx.assets import Articulation # noqa: E402 -from isaaclab_ovphysx.physics import OvPhysxCfg # noqa: E402 - -import isaaclab.sim as sim_utils # noqa: E402 -import isaaclab.utils.math as math_utils # noqa: E402 -import isaaclab.utils.string as string_utils # noqa: E402 -from isaaclab.actuators import ActuatorBase, IdealPDActuatorCfg, ImplicitActuatorCfg # noqa: E402 -from isaaclab.assets import ArticulationCfg # noqa: E402 -from isaaclab.envs.mdp.terminations import joint_effort_out_of_limit # noqa: E402 -from isaaclab.managers import SceneEntityCfg # noqa: E402 -from isaaclab.sim import SimulationCfg, build_simulation_context # noqa: E402 -from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR # noqa: E402 -from isaaclab.utils.version import get_isaac_sim_version, has_kit # noqa: E402 +from isaaclab_ovphysx import tensor_types as TT +from isaaclab_ovphysx.assets import Articulation +from isaaclab_ovphysx.physics import OvPhysxCfg + +import isaaclab.sim as sim_utils +import isaaclab.utils.math as math_utils +import isaaclab.utils.string as string_utils +from isaaclab.actuators import ActuatorBase, IdealPDActuatorCfg, ImplicitActuatorCfg +from isaaclab.assets import ArticulationCfg +from isaaclab.envs.mdp.terminations import joint_effort_out_of_limit +from isaaclab.managers import SceneEntityCfg +from isaaclab.sim import SimulationCfg, build_simulation_context +from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR +from isaaclab.utils.version import get_isaac_sim_version, has_kit ## # Pre-defined configs diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation_helpers.py b/source/isaaclab_ovphysx/test/assets/test_articulation_helpers.py index db0a2cc89403..50b847384837 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation_helpers.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation_helpers.py @@ -14,20 +14,13 @@ from types import SimpleNamespace -import pytest import warp as wp +from isaaclab_ovphysx.assets.articulation.articulation import Articulation +from isaaclab_ovphysx.physics import OvPhysxManager +from isaaclab_ovphysx.test.mock_interfaces.views import MockOvPhysxBindingSet from pxr import Sdf, Usd, UsdPhysics -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. -pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") - -from isaaclab_ovphysx.assets.articulation.articulation import Articulation # noqa: E402 -from isaaclab_ovphysx.physics import OvPhysxManager # noqa: E402 -from isaaclab_ovphysx.test.mock_interfaces.views import MockOvPhysxBindingSet # noqa: E402 - wp.init() @@ -120,7 +113,6 @@ def test_process_tendons_scopes_to_articulation_root(tmp_path): def test_mock_binding_set_rigid_object_shapes(): - pytest.importorskip("isaaclab_ovphysx.tensor_types").RIGID_BODY_POSE # gates on wheel from isaaclab_ovphysx import tensor_types as TT from isaaclab_ovphysx.test.mock_interfaces.views import MockOvPhysxBindingSet diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py index 407cf4b41e22..65943b53fed8 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py @@ -31,20 +31,14 @@ import torch import warp as wp from flaky import flaky - -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. -pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") - -from isaaclab_ovphysx.assets import RigidObject # noqa: E402 -from isaaclab_ovphysx.physics import OvPhysxCfg, OvPhysxManager # noqa: E402 - -import isaaclab.sim as sim_utils # noqa: E402 -from isaaclab.assets import RigidObjectCfg # noqa: E402 -from isaaclab.sim import SimulationCfg, build_simulation_context # noqa: E402 -from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR # noqa: E402 -from isaaclab.utils.math import ( # noqa: E402 +from isaaclab_ovphysx.assets import RigidObject +from isaaclab_ovphysx.physics import OvPhysxCfg, OvPhysxManager + +import isaaclab.sim as sim_utils +from isaaclab.assets import RigidObjectCfg +from isaaclab.sim import SimulationCfg, build_simulation_context +from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR +from isaaclab.utils.math import ( combine_frame_transforms, default_orientation, quat_apply_inverse, diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py index 07ec860d6ec6..d2163989d32b 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py @@ -27,20 +27,14 @@ import pytest import torch import warp as wp - -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. -pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") - -from isaaclab_ovphysx.assets import RigidObjectCollection # noqa: E402 -from isaaclab_ovphysx.physics import OvPhysxCfg # noqa: E402 - -import isaaclab.sim as sim_utils # noqa: E402 -from isaaclab.assets import RigidObjectCfg, RigidObjectCollectionCfg # noqa: E402 -from isaaclab.sim import SimulationCfg, build_simulation_context # noqa: E402 -from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR # noqa: E402 -from isaaclab.utils.math import ( # noqa: E402 +from isaaclab_ovphysx.assets import RigidObjectCollection +from isaaclab_ovphysx.physics import OvPhysxCfg + +import isaaclab.sim as sim_utils +from isaaclab.assets import RigidObjectCfg, RigidObjectCollectionCfg +from isaaclab.sim import SimulationCfg, build_simulation_context +from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR +from isaaclab.utils.math import ( combine_frame_transforms, default_orientation, quat_apply_inverse, diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_helpers.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_helpers.py index e57d8d2651d7..a30b6b81ae68 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_helpers.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_helpers.py @@ -12,23 +12,14 @@ from __future__ import annotations -import pytest import warp as wp - -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. -pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") - -from isaaclab_ovphysx import tensor_types as TT # noqa: E402 -from isaaclab_ovphysx.test.mock_interfaces.views import MockOvPhysxBindingSet # noqa: E402 +from isaaclab_ovphysx import tensor_types as TT +from isaaclab_ovphysx.test.mock_interfaces.views import MockOvPhysxBindingSet wp.init() def test_mock_binding_set_rigid_object_shapes(): - pytest.importorskip("isaaclab_ovphysx.tensor_types").RIGID_BODY_POSE # gates on wheel - bindings = MockOvPhysxBindingSet( num_instances=4, num_joints=0, diff --git a/source/isaaclab_ovphysx/test/physics/test_ovphysx_scene_data_backend.py b/source/isaaclab_ovphysx/test/physics/test_ovphysx_scene_data_backend.py index 013ef8bdc092..cce2c7453a5d 100644 --- a/source/isaaclab_ovphysx/test/physics/test_ovphysx_scene_data_backend.py +++ b/source/isaaclab_ovphysx/test/physics/test_ovphysx_scene_data_backend.py @@ -9,13 +9,6 @@ from types import SimpleNamespace -import pytest - -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. -pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") - def _make_stub_binding(prim_paths: list[str]) -> SimpleNamespace: """Stub an ovphysx ``TensorBinding`` exposing ``shape``, ``count``, ``prim_paths``, and ``read(dst)``.""" diff --git a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py index 38b3bd9e579d..08d470f833f2 100644 --- a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py @@ -44,23 +44,17 @@ import torch import warp as wp from flaky import flaky - -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. -pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") - -from isaaclab_ovphysx.assets import RigidObject # noqa: E402 -from isaaclab_ovphysx.physics import OvPhysxCfg # noqa: E402 -from isaaclab_ovphysx.sensors import ContactSensor, ContactSensorCfg # noqa: E402 - -import isaaclab.sim as sim_utils # noqa: E402 -from isaaclab.assets import RigidObjectCfg # noqa: E402 -from isaaclab.scene import InteractiveScene, InteractiveSceneCfg # noqa: E402 -from isaaclab.sim import SimulationCfg, SimulationContext, build_simulation_context # noqa: E402 -from isaaclab.sim.utils.stage import get_current_stage # noqa: E402 -from isaaclab.terrains import HfRandomUniformTerrainCfg, TerrainGeneratorCfg, TerrainImporterCfg # noqa: E402 -from isaaclab.utils.configclass import configclass # noqa: E402 +from isaaclab_ovphysx.assets import RigidObject +from isaaclab_ovphysx.physics import OvPhysxCfg +from isaaclab_ovphysx.sensors import ContactSensor, ContactSensorCfg + +import isaaclab.sim as sim_utils +from isaaclab.assets import RigidObjectCfg +from isaaclab.scene import InteractiveScene, InteractiveSceneCfg +from isaaclab.sim import SimulationCfg, SimulationContext, build_simulation_context +from isaaclab.sim.utils.stage import get_current_stage +from isaaclab.terrains import HfRandomUniformTerrainCfg, TerrainGeneratorCfg, TerrainImporterCfg +from isaaclab.utils.configclass import configclass wp.init() diff --git a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py index bebd8aaa460d..241af40af715 100644 --- a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py +++ b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py @@ -11,17 +11,11 @@ from __future__ import annotations import pytest +from isaaclab_ovphysx.physics import OvPhysxCfg -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. -pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") - -from isaaclab_ovphysx.physics import OvPhysxCfg # noqa: E402 - -import isaaclab.sim as sim_utils # noqa: E402 -from isaaclab.sim import SimulationCfg, build_simulation_context # noqa: E402 -from isaaclab.sim.views import FrameView # noqa: E402 +import isaaclab.sim as sim_utils +from isaaclab.sim import SimulationCfg, build_simulation_context +from isaaclab.sim.views import FrameView OVPHYSX_SIM_CFG = SimulationCfg(physics=OvPhysxCfg()) From fec15c8c3702cb70008a411f0429b2b3d4da5353 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 10:23:54 +0200 Subject: [PATCH 02/10] Register device_split pytest marker --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 86ab12b38ceb..53a67ba2796f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -194,6 +194,7 @@ ignore-words-list = "haa,slq,collapsable,buss,reacher,thirdparty" markers = [ "isaacsim_ci: mark test to run in isaacsim ci", + "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5; see scripts/run_ovphysx.sh)", ] # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url. From c878a8aaf5579485bf34fb237d9c89f6cb36854d Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 10:35:33 +0200 Subject: [PATCH 03/10] Add is_device_split_file predicate with unit tests --- tools/_device_split.py | 63 ++++++++++++++++++++++++ tools/test_device_split.py | 99 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 tools/_device_split.py create mode 100644 tools/test_device_split.py diff --git a/tools/_device_split.py b/tools/_device_split.py new file mode 100644 index 000000000000..acc6bf4b1ae9 --- /dev/null +++ b/tools/_device_split.py @@ -0,0 +1,63 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Helpers for detecting and driving the ``device_split`` pytest marker. + +Test files that declare ``pytestmark = pytest.mark.device_split`` at module +scope must be re-invoked once per device (CPU and GPU) in separate processes +to work around process-global device locks such as ``ovphysx<=0.3.7`` gap G5. +The :func:`is_device_split_file` predicate lets the per-file CI runner in +``tools/conftest.py`` detect this without importing the test module. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +_DEVICE_SPLIT_MARK_RE = re.compile(r"^\s*pytestmark\b.*\bdevice_split\b", re.MULTILINE) +"""Match a module-level ``pytestmark`` assignment that mentions ``device_split``. + +Recognises both single-mark and single-line list forms: + +* ``pytestmark = pytest.mark.device_split`` +* ``pytestmark = [pytest.mark.device_split, pytest.mark.foo]`` + +Multi-line list forms are not supported (currently no test file uses one); if +a future test needs that, expand the parsing rule alongside the marker +contract documented in ``scripts/run_ovphysx.sh``. +""" + +# Per-pass pytest ``-k`` selectors used by ``tools/conftest.py`` when a file +# declares the ``device_split`` marker. Each entry is ``(suffix, k_expr)``: +# - ``suffix`` is appended to the JUnit report filename to keep both passes' XML. +# - ``k_expr`` is the ``-k`` keyword expression. ``"cpu or not cuda"`` keeps +# non-parametrized tests in the CPU pass; ``"cuda"`` catches GPU-parametrized +# tests only. +DEVICE_SPLIT_PASSES: list[tuple[str, str]] = [ + ("-cpu", "cpu or not cuda"), + ("-cuda", "cuda"), +] + + +def is_device_split_file(path: Path | str) -> bool: + """Return whether the test file at ``path`` declares the ``device_split`` marker. + + Reads the file source once and matches :data:`_DEVICE_SPLIT_MARK_RE`. A + missing or unreadable file returns ``False`` so the caller falls back to + the default single-pass invocation. + + Args: + path: Filesystem path to a candidate test file. + + Returns: + ``True`` when the file's module-level ``pytestmark`` mentions + ``device_split``; ``False`` otherwise. + """ + try: + source = Path(path).read_text(encoding="utf-8", errors="replace") + except OSError: + return False + return bool(_DEVICE_SPLIT_MARK_RE.search(source)) diff --git a/tools/test_device_split.py b/tools/test_device_split.py new file mode 100644 index 000000000000..d16f1de55d67 --- /dev/null +++ b/tools/test_device_split.py @@ -0,0 +1,99 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for ``tools/_device_split.py``.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from _device_split import is_device_split_file + + +def _write(tmp_path: Path, content: str) -> Path: + p = tmp_path / "test_foo.py" + p.write_text(textwrap.dedent(content)) + return p + + +def test_single_mark(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + pytestmark = pytest.mark.device_split + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is True + + +def test_list_form_single_line(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + pytestmark = [pytest.mark.device_split, pytest.mark.foo] + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is True + + +def test_no_mark(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is False + + +def test_word_in_comment_does_not_match(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + # This file mentions device_split in a comment but is not marked. + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is False + + +def test_unrelated_pytestmark_does_not_match(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + pytestmark = pytest.mark.skipif(False, reason="x") + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is False + + +def test_missing_file(tmp_path): + # A path that does not exist must not raise; treat as not-marked. + assert is_device_split_file(tmp_path / "does_not_exist.py") is False From 7258f44a65f85d4d3637911e5528583e40772c27 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 10:41:03 +0200 Subject: [PATCH 04/10] Drop dead scripts/run_ovphysx.sh references from device_split docs --- pyproject.toml | 2 +- tools/_device_split.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 53a67ba2796f..45017dd0adb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -194,7 +194,7 @@ ignore-words-list = "haa,slq,collapsable,buss,reacher,thirdparty" markers = [ "isaacsim_ci: mark test to run in isaacsim ci", - "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5; see scripts/run_ovphysx.sh)", + "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5)", ] # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url. diff --git a/tools/_device_split.py b/tools/_device_split.py index acc6bf4b1ae9..1c802eca489c 100644 --- a/tools/_device_split.py +++ b/tools/_device_split.py @@ -26,8 +26,7 @@ * ``pytestmark = [pytest.mark.device_split, pytest.mark.foo]`` Multi-line list forms are not supported (currently no test file uses one); if -a future test needs that, expand the parsing rule alongside the marker -contract documented in ``scripts/run_ovphysx.sh``. +a future test needs that, expand the parsing rule. """ # Per-pass pytest ``-k`` selectors used by ``tools/conftest.py`` when a file From c81140cf90dcbd545ade303e7e357d061ec0703b Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 10:47:24 +0200 Subject: [PATCH 05/10] Factor per-file pytest invocation in conftest into _run_one_pass helper Pure refactor; behavior unchanged. Enables the follow-up commit to invoke the helper twice per file for the device_split marker without duplicating the ~200-line retry/timeout/parse pipeline. --- tools/conftest.py | 469 +++++++++++++++++++++++++++------------------- 1 file changed, 271 insertions(+), 198 deletions(-) diff --git a/tools/conftest.py b/tools/conftest.py index 15aaa2323647..3037ae9c4f10 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -10,6 +10,7 @@ import subprocess import sys import time +from dataclasses import dataclass import pytest from junitparser import Error, JUnitXml, TestCase, TestSuite @@ -304,131 +305,145 @@ def _capture_system_diagnostics(): return "\n\n".join(sections) -def run_individual_tests(test_files, workspace_root, isaacsim_ci): - """Run each test file separately, ensuring one finishes before starting the next.""" - failed_tests = [] - test_status = {} - xml_reports = [] - cold_cache_applied = False - - for test_file in test_files: - print(f"\n\n🚀 Running {test_file} independently...\n") - file_name = os.path.basename(test_file) - env = os.environ.copy() - env["PYTHONFAULTHANDLER"] = "1" - - timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) - - # Read the test file once for cold-cache check. - try: - with open(test_file) as fh: - test_content = fh.read() - except OSError: - test_content = "" +@dataclass +class _PassContext: + """Inputs shared across all pytest invocations for a single test file. + + Attributes: + test_file: Absolute path to the test file being driven. + file_name: Basename of ``test_file`` (used for JUnit naming). + workspace_root: Repository root; passed to pytest's ``--config-file``. + isaacsim_ci: Whether ``ISAACSIM_CI_SHORT`` is active; toggles the + ``-m isaacsim_ci`` selector. + timeout: Per-pass hard timeout in seconds. + startup_deadline: Per-pass startup-hang deadline in seconds. + env: Environment passed to the pytest subprocess. + is_cold_cache_test: Used by callers to log the cold-cache buffer once. + """ - # The first camera-enabled test in a fresh container compiles shaders - # (~600 s). Give it extra time so that doesn't look like a test timeout. - is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content - if is_cold_cache_test: - timeout += COLD_CACHE_BUFFER - cold_cache_applied = True - print(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") + test_file: str + file_name: str + workspace_root: str + isaacsim_ci: bool + timeout: int + startup_deadline: int + env: dict + is_cold_cache_test: bool - extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 - startup_deadline = min(timeout, STARTUP_DEADLINE + extra) - cmd = [ - sys.executable, - "-m", - "pytest", - "-s", - "--no-header", - f"--config-file={workspace_root}/pyproject.toml", - f"--junitxml=tests/test-reports-{str(file_name)}.xml", - "--tb=short", - ] - - if isaacsim_ci: - cmd.append("-m") - cmd.append("isaacsim_ci") - - cmd.append(str(test_file)) - - report_file = f"tests/test-reports-{str(file_name)}.xml" - - # -- Run with retry on startup hang or hard timeout ----------------- - returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" - wall_time, pre_kill_diag = 0.0, "" - startup_hang_attempts = 0 - timeout_attempts = 0 - while True: - with contextlib.suppress(FileNotFoundError): - os.remove(report_file) - - returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = ( - capture_test_output_with_timeout( - cmd, timeout, env, startup_deadline=startup_deadline, report_file=report_file - ) - ) +def _run_one_pass( + ctx: _PassContext, + k_expr: str | None, + suffix: str, +) -> tuple[JUnitXml | None, dict, bool]: + """Drive one pytest subprocess for ``ctx.test_file`` and return its results. - has_report = os.path.exists(report_file) - - if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: - startup_hang_attempts += 1 - print( - f"⚠️ {test_file}: startup hang detected after {startup_deadline}s" - f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." - ) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue + Args: + ctx: Static per-file context (paths, timeouts, env). + k_expr: Optional ``-k`` selector. ``None`` means no selector (default + single-pass invocation). + suffix: Suffix appended to the JUnit report filename, e.g. ``"-cpu"`` + or ``""`` for the unsplit default. - if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: - timeout_attempts += 1 - print( - f"⚠️ {test_file}: timeout detected after {timeout}s" - f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." - ) - if stdout_data: - print("=== STDOUT (last 5000 chars) ===") - print(stdout_data.decode("utf-8", errors="replace")[-5000:]) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue - break + Returns: + A 3-tuple ``(xml_report, status_dict, was_failure)``: + * ``xml_report``: parsed JUnit XML, or ``None`` if the pass produced + no report (e.g. startup hang). + * ``status_dict``: per-pass counters compatible with the entries + currently appended to ``test_status``. + * ``was_failure``: whether the pass should add ``ctx.test_file`` to + the ``failed_tests`` list. + """ + pass_file_label = f"{ctx.file_name}{suffix}" + report_file = f"tests/test-reports-{pass_file_label}.xml" + + cmd = [ + sys.executable, + "-m", + "pytest", + "-s", + "--no-header", + f"--config-file={ctx.workspace_root}/pyproject.toml", + f"--junitxml={report_file}", + "--tb=short", + ] + if ctx.isaacsim_ci: + cmd += ["-m", "isaacsim_ci"] + if k_expr is not None: + cmd += ["-k", k_expr] + cmd.append(str(ctx.test_file)) + + # -- Run with retry on startup hang or hard timeout ----------------- + returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" + wall_time, pre_kill_diag = 0.0, "" + startup_hang_attempts = 0 + timeout_attempts = 0 + while True: + with contextlib.suppress(FileNotFoundError): + os.remove(report_file) + + returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( + cmd, ctx.timeout, ctx.env, startup_deadline=ctx.startup_deadline, report_file=report_file + ) - # -- Resolve result from kill_reason and report file ---------------- has_report = os.path.exists(report_file) - if kill_reason == "startup_hang": - diag = _get_diagnostics(pre_kill_diag) - print(f"⚠️ {test_file}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") + if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: + startup_hang_attempts += 1 + print( + f"⚠️ {ctx.test_file}{suffix}: startup hang detected after {ctx.startup_deadline}s" + f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." + ) + if stderr_data: + print("=== STDERR (last 5000 chars) ===") + print(stderr_data.decode("utf-8", errors="replace")[-5000:]) + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" print(diag) + continue - msg = f"Startup hang after {startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: + timeout_attempts += 1 + print( + f"⚠️ {ctx.test_file}{suffix}: timeout detected after {ctx.timeout}s" + f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." + ) if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("startup_hang", file_name, msg, details) - error_report.write(report_file) - xml_reports.append(error_report) - failed_tests.append(test_file) - test_status[test_file] = { + print("=== STDOUT (last 5000 chars) ===") + print(stdout_data.decode("utf-8", errors="replace")[-5000:]) + if stderr_data: + print("=== STDERR (last 5000 chars) ===") + print(stderr_data.decode("utf-8", errors="replace")[-5000:]) + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" + print(diag) + continue + break + + # -- Resolve result from kill_reason and report file ---------------- + has_report = os.path.exists(report_file) + + if kill_reason == "startup_hang": + diag = _get_diagnostics(pre_kill_diag) + print(f"⚠️ {ctx.test_file}{suffix}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") + print(diag) + + msg = f"Startup hang after {ctx.startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stderr_data: + details += "=== STDERR (last 5000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if stdout_data: + details += "=== STDOUT (last 2000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" + + error_report = _create_error_report("startup_hang", pass_file_label, msg, details) + error_report.write(report_file) + return ( + error_report, + { "errors": 1, "failures": 0, "skipped": 0, @@ -436,61 +451,63 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": "STARTUP_HANG", "time_elapsed": 0.0, "wall_time": wall_time, - } - continue - - if kill_reason == "timeout" and not has_report: - diag = _get_diagnostics(pre_kill_diag) - print(f"Test {test_file} timed out after {timeout} seconds...") - print(diag) - - msg = f"Timeout after {timeout} seconds (retried {timeout_attempts} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 5000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + }, + True, + ) - error_report = _create_error_report("timeout", file_name, msg, details) - error_report.write(report_file) - xml_reports.append(error_report) - failed_tests.append(test_file) - test_status[test_file] = { + if kill_reason == "timeout" and not has_report: + diag = _get_diagnostics(pre_kill_diag) + print(f"Test {ctx.test_file}{suffix} timed out after {ctx.timeout} seconds...") + print(diag) + + msg = f"Timeout after {ctx.timeout} seconds (retried {timeout_attempts} time(s))" + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout_data: + details += "=== STDOUT (last 5000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if stderr_data: + details += "=== STDERR (last 5000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + + error_report = _create_error_report("timeout", pass_file_label, msg, details) + error_report.write(report_file) + return ( + error_report, + { "errors": 1, "failures": 0, "skipped": 0, "tests": 1, "result": "TIMEOUT", - "time_elapsed": timeout, + "time_elapsed": ctx.timeout, "wall_time": wall_time, - } - continue - - if not has_report: - reason = ( - _signal_description(-returncode) - if returncode < 0 - else f"Process exited with code {returncode} but produced no report" - ) - diag = _get_diagnostics() - print(f"⚠️ {test_file}: {reason}") - print(diag) - - details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - if stderr_data: - details += "=== STDERR (last 2000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" + }, + True, + ) - error_report = _create_error_report("crash", file_name, reason, details) - error_report.write(report_file) - xml_reports.append(error_report) - failed_tests.append(test_file) - test_status[test_file] = { + if not has_report: + reason = ( + _signal_description(-returncode) + if returncode < 0 + else f"Process exited with code {returncode} but produced no report" + ) + diag = _get_diagnostics() + print(f"⚠️ {ctx.test_file}{suffix}: {reason}") + print(diag) + + details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout_data: + details += "=== STDOUT (last 2000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" + if stderr_data: + details += "=== STDERR (last 2000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" + + error_report = _create_error_report("crash", pass_file_label, reason, details) + error_report.write(report_file) + return ( + error_report, + { "errors": 1, "failures": 0, "skipped": 0, @@ -498,30 +515,31 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": "CRASHED", "time_elapsed": 0.0, "wall_time": wall_time, - } - continue - - # -- Report file exists: parse actual test results ----------------- - if kill_reason in ("shutdown_hang", "timeout"): - print(f"⚠️ {test_file}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") + }, + True, + ) - try: - report = JUnitXml.fromfile(report_file) - for suite in report: - if suite.name == "pytest": - suite.name = os.path.splitext(file_name)[0] - report.write(report_file) - xml_reports.append(report) + # -- Report file exists: parse actual test results ----------------- + if kill_reason in ("shutdown_hang", "timeout"): + print(f"⚠️ {ctx.test_file}{suffix}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") - errors = int(report.errors) if report.errors is not None else 0 - failures = int(report.failures) if report.failures is not None else 0 - skipped = int(report.skipped) if report.skipped is not None else 0 - tests = int(report.tests) if report.tests is not None else 0 - time_elapsed = float(report.time) if report.time is not None else 0.0 - except Exception as e: - print(f"Error reading test report {report_file}: {e}") - failed_tests.append(test_file) - test_status[test_file] = { + try: + report = JUnitXml.fromfile(report_file) + for suite in report: + if suite.name == "pytest": + suite.name = os.path.splitext(pass_file_label)[0] + report.write(report_file) + + errors = int(report.errors) if report.errors is not None else 0 + failures = int(report.failures) if report.failures is not None else 0 + skipped = int(report.skipped) if report.skipped is not None else 0 + tests = int(report.tests) if report.tests is not None else 0 + time_elapsed = float(report.time) if report.time is not None else 0.0 + except Exception as e: + print(f"Error reading test report {report_file}: {e}") + return ( + None, + { "errors": 1, "failures": 0, "skipped": 0, @@ -529,23 +547,24 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": "FAILED", "time_elapsed": 0.0, "wall_time": wall_time, - } - continue - - has_test_failures = errors > 0 or failures > 0 - shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures + }, + True, + ) - if has_test_failures or (returncode != 0 and not shutdown_hanged): - failed_tests.append(test_file) + has_test_failures = errors > 0 or failures > 0 + shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures + was_failure = has_test_failures or (returncode != 0 and not shutdown_hanged) - if shutdown_hanged: - result = "passed (shutdown hanged)" - elif has_test_failures: - result = "FAILED" - else: - result = "passed" + if shutdown_hanged: + result = "passed (shutdown hanged)" + elif has_test_failures: + result = "FAILED" + else: + result = "passed" - test_status[test_file] = { + return ( + report, + { "errors": errors, "failures": failures, "skipped": skipped, @@ -553,7 +572,61 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": result, "time_elapsed": time_elapsed, "wall_time": wall_time, - } + }, + was_failure, + ) + + +def run_individual_tests(test_files, workspace_root, isaacsim_ci): + """Run each test file separately, ensuring one finishes before starting the next.""" + failed_tests = [] + test_status = {} + xml_reports = [] + cold_cache_applied = False + + for test_file in test_files: + print(f"\n\n🚀 Running {test_file} independently...\n") + file_name = os.path.basename(test_file) + env = os.environ.copy() + env["PYTHONFAULTHANDLER"] = "1" + + timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) + + # Read the test file once for cold-cache and device-split detection. + try: + with open(test_file) as fh: + test_content = fh.read() + except OSError: + test_content = "" + + # The first camera-enabled test in a fresh container compiles shaders + # (~600 s). Give it extra time so that doesn't look like a test timeout. + is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content + if is_cold_cache_test: + timeout += COLD_CACHE_BUFFER + cold_cache_applied = True + print(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") + + extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 + startup_deadline = min(timeout, STARTUP_DEADLINE + extra) + + ctx = _PassContext( + test_file=test_file, + file_name=file_name, + workspace_root=workspace_root, + isaacsim_ci=isaacsim_ci, + timeout=timeout, + startup_deadline=startup_deadline, + env=env, + is_cold_cache_test=is_cold_cache_test, + ) + + report, status, was_failure = _run_one_pass(ctx, k_expr=None, suffix="") + if report is not None: + xml_reports.append(report) + if was_failure: + failed_tests.append(test_file) + test_status[test_file] = status print("~~~~~~~~~~~~ Finished running all tests") From 2ead23c7fdfc24a54e3d0a5f41f0951aa60fac62 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 13:32:01 +0200 Subject: [PATCH 06/10] Run device_split-marked test files twice in conftest (CPU then GPU) Resolves the ovphysx<=0.3.7 gap G5 device lock for any CI test file that opts in via 'pytestmark = pytest.mark.device_split'. Each pass gets its own subprocess, its own JUnit XML, and the merged counts are written under the original file key so the summary table is unchanged. --- tools/conftest.py | 56 ++++++++++++++++++++++++++++++++++---- tools/test_device_split.py | 2 -- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/tools/conftest.py b/tools/conftest.py index 3037ae9c4f10..0c0700038bb2 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -18,6 +18,7 @@ # Local imports import test_settings as test_settings # isort: skip +from _device_split import DEVICE_SPLIT_PASSES, is_device_split_file # isort: skip def pytest_ignore_collect(collection_path, config): @@ -331,6 +332,38 @@ class _PassContext: is_cold_cache_test: bool +_RESULT_PRIORITY = { + "STARTUP_HANG": 5, + "CRASHED": 4, + "TIMEOUT": 3, + "FAILED": 2, + "passed (shutdown hanged)": 1, + "passed": 0, +} + + +def _merge_pass_status(prev: dict | None, new: dict) -> dict: + """Merge per-pass status dicts into a single per-file entry. + + Counters (``errors``, ``failures``, ``skipped``, ``tests``, + ``time_elapsed``, ``wall_time``) are summed. ``result`` becomes the more + severe of the two via :data:`_RESULT_PRIORITY`. + """ + if prev is None: + return new + return { + "errors": prev["errors"] + new["errors"], + "failures": prev["failures"] + new["failures"], + "skipped": prev["skipped"] + new["skipped"], + "tests": prev["tests"] + new["tests"], + "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], + "wall_time": prev["wall_time"] + new["wall_time"], + "result": prev["result"] + if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) + else new["result"], + } + + def _run_one_pass( ctx: _PassContext, k_expr: str | None, @@ -621,12 +654,23 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): is_cold_cache_test=is_cold_cache_test, ) - report, status, was_failure = _run_one_pass(ctx, k_expr=None, suffix="") - if report is not None: - xml_reports.append(report) - if was_failure: - failed_tests.append(test_file) - test_status[test_file] = status + if is_device_split_file(test_file): + print(f"⚙️ device_split detected — invoking {file_name} once per device (CPU then GPU)") + passes = DEVICE_SPLIT_PASSES + else: + passes = [("", None)] + + merged_status: dict | None = None + for suffix, k_expr in passes: + report, status, was_failure = _run_one_pass(ctx, k_expr=k_expr, suffix=suffix) + if report is not None: + xml_reports.append(report) + if was_failure: + failed_tests.append(test_file) + merged_status = _merge_pass_status(merged_status, status) + + assert merged_status is not None # the pass list is never empty + test_status[test_file] = merged_status print("~~~~~~~~~~~~ Finished running all tests") diff --git a/tools/test_device_split.py b/tools/test_device_split.py index d16f1de55d67..c3f7ea67a72e 100644 --- a/tools/test_device_split.py +++ b/tools/test_device_split.py @@ -10,8 +10,6 @@ import textwrap from pathlib import Path -import pytest - from _device_split import is_device_split_file From ae0be2e24ffd92b33cddc5e9339a47a599f48c50 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 13:36:45 +0200 Subject: [PATCH 07/10] Mark test_views_xform_prim_ovphysx as device_split Avoids the ovphysx<=0.3.7 gap G5 device-lock failure surfaced by PR 5780 in the isaaclab_ov CI job. The conftest device_split runner splits this file into separate CPU and GPU subprocesses. --- .../isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py index 241af40af715..1ed2e7013f0b 100644 --- a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py +++ b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py @@ -19,6 +19,8 @@ OVPHYSX_SIM_CFG = SimulationCfg(physics=OvPhysxCfg()) +pytestmark = pytest.mark.device_split + @pytest.mark.parametrize("device", ["cpu", "cuda:0"]) def test_factory_dispatches_to_ovphysx_frame_view(device): From d8a57ac3196d575695a6a66393fdd8268518210c Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 13:39:13 +0200 Subject: [PATCH 08/10] Mark test_contact_sensor (ovphysx) as device_split Avoids the ovphysx<=0.3.7 gap G5 device-lock failure on the test_no_contact_reporting case that runs after GPU-parametrized cases in the isaaclab_ov CI job. --- source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py index 08d470f833f2..9cd72e90fe4a 100644 --- a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py @@ -58,6 +58,8 @@ wp.init() +pytestmark = pytest.mark.device_split + # --------------------------------------------------------------------------- # Device-lock autouse fixture # --------------------------------------------------------------------------- From 61f2adfbfc402b7cdde60900233a86f6e17d4263 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 13:41:12 +0200 Subject: [PATCH 09/10] Add changelog fragment for ovphysx device_split CI fix --- .../changelog.d/antoiner-ovphysx-device-split-ci.rst | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst diff --git a/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst new file mode 100644 index 000000000000..8f8b515af6e9 --- /dev/null +++ b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst @@ -0,0 +1,9 @@ +Fixed +^^^^^ + +* Re-enabled both CPU and GPU coverage in CI for + :file:`test/sim/test_views_xform_prim_ovphysx.py` and + :file:`test/sensors/test_contact_sensor.py` by tagging them with the new + ``device_split`` pytest marker, which causes the CI driver to invoke each + file once per device in separate subprocesses. Works around the + ``ovphysx<=0.3.7`` process-global device lock (gap G5). From e5411f7259f06bc3a8675ab47b7057e2a33d5a8e Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 17:31:27 +0200 Subject: [PATCH 10/10] Drop dead is_cold_cache_test field; dedup failed_tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two tiny cleanups on the device_split CI machinery added in this PR: * Remove _PassContext.is_cold_cache_test — set by the caller but never read by _run_one_pass; the cold-cache buffer is applied to timeout and startup_deadline before the dataclass is built, so the field was purely informational. * Add a presence check before appending to failed_tests so a device_split file failing on both CPU and GPU passes only appears once in the 'failed tests:' debug print (exit code is unaffected; it derives from test_status keys). --- tools/conftest.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/conftest.py b/tools/conftest.py index 0c0700038bb2..514778fe9930 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -319,7 +319,6 @@ class _PassContext: timeout: Per-pass hard timeout in seconds. startup_deadline: Per-pass startup-hang deadline in seconds. env: Environment passed to the pytest subprocess. - is_cold_cache_test: Used by callers to log the cold-cache buffer once. """ test_file: str @@ -329,7 +328,6 @@ class _PassContext: timeout: int startup_deadline: int env: dict - is_cold_cache_test: bool _RESULT_PRIORITY = { @@ -651,7 +649,6 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): timeout=timeout, startup_deadline=startup_deadline, env=env, - is_cold_cache_test=is_cold_cache_test, ) if is_device_split_file(test_file): @@ -665,7 +662,7 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): report, status, was_failure = _run_one_pass(ctx, k_expr=k_expr, suffix=suffix) if report is not None: xml_reports.append(report) - if was_failure: + if was_failure and test_file not in failed_tests: failed_tests.append(test_file) merged_status = _merge_pass_status(merged_status, status)