From 2df667fd71de507703975c4b82bbf8eeefff6049 Mon Sep 17 00:00:00 2001 From: yuecideng Date: Tue, 11 Aug 2026 13:20:24 +0800 Subject: [PATCH] feat(agents): add strict expert program frontend --- ...mbodichain.lab.gym.envs.expert_program.rst | 11 + embodichain/agents/__init__.py | 21 + embodichain/agents/mllm/__init__.py | 29 ++ embodichain/agents/mllm/expert_program.py | 260 +++++++++++ tests/agents/mllm/test_expert_program.py | 433 ++++++++++++++++++ .../test_simulation_environment.py | 200 +++++--- 6 files changed, 890 insertions(+), 64 deletions(-) create mode 100644 embodichain/agents/__init__.py create mode 100644 embodichain/agents/mllm/__init__.py create mode 100644 embodichain/agents/mllm/expert_program.py create mode 100644 tests/agents/mllm/test_expert_program.py diff --git a/docs/source/api_reference/embodichain/embodichain.lab.gym.envs.expert_program.rst b/docs/source/api_reference/embodichain/embodichain.lab.gym.envs.expert_program.rst index c4d7d1f4d..f94f74cd8 100644 --- a/docs/source/api_reference/embodichain/embodichain.lab.gym.envs.expert_program.rst +++ b/docs/source/api_reference/embodichain/embodichain.lab.gym.envs.expert_program.rst @@ -48,6 +48,17 @@ and 2. Version 2 adds deterministic parallel blocks with explicit barriers. .. autofunction:: decode_expert_program +MLLM frontend +------------- + +The MLLM frontend intentionally accepts only the constrained schema version 1 +surface. Trusted host code remains responsible for authoring version 2 +parallel structure and the integration selection. + +.. autofunction:: embodichain.agents.mllm.decode_mllm_expert_program + +.. autofunction:: embodichain.agents.mllm.compile_mllm_expert_program + Compilation and environment integration --------------------------------------- diff --git a/embodichain/agents/__init__.py b/embodichain/agents/__init__.py new file mode 100644 index 000000000..071c9ac48 --- /dev/null +++ b/embodichain/agents/__init__.py @@ -0,0 +1,21 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Agent-facing frontends built on EmbodiChain's typed runtime contracts.""" + +from __future__ import annotations + +__all__: list[str] = [] diff --git a/embodichain/agents/mllm/__init__.py b/embodichain/agents/mllm/__init__.py new file mode 100644 index 000000000..607c022d9 --- /dev/null +++ b/embodichain/agents/mllm/__init__.py @@ -0,0 +1,29 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Multimodal-model frontends for typed EmbodiChain agent contracts.""" + +from __future__ import annotations + +from .expert_program import ( + compile_mllm_expert_program, + decode_mllm_expert_program, +) + +__all__ = [ + "compile_mllm_expert_program", + "decode_mllm_expert_program", +] diff --git a/embodichain/agents/mllm/expert_program.py b/embodichain/agents/mllm/expert_program.py new file mode 100644 index 000000000..a54304d78 --- /dev/null +++ b/embodichain/agents/mllm/expert_program.py @@ -0,0 +1,260 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Strict MLLM frontend for declarative Expert Program JSON responses.""" + +from __future__ import annotations + +from collections.abc import Iterator + +from embodichain.lab.gym.envs.expert_program.cfg import ( + EXPERT_PROGRAM_SCHEMA_VERSION, + ExpertProgramCfg, + ExpertProgramIntegrationCfg, + HandOverCfg, + InvokeCfg, + OperateArticulationCfg, + PickCfg, + PlaceCfg, + ProgramNodeCfg, + RepeatCfg, + SegmentCfg, + SemanticCallCfg, + SequenceCfg, +) +from embodichain.lab.gym.envs.expert_program.compiler import CompiledProgram +from embodichain.lab.gym.envs.expert_program.decoder import ( + ConfigPath, + ExpertProgramDecodeError, + ExpertProgramValidationContext, + decode_expert_program, + validate_expert_program, +) +from embodichain.lab.gym.envs.expert_program.environment import ( + ExpertProgramEnvironmentAdapter, +) +from embodichain.lab.gym.envs.expert_program.loader import ( + MAX_EXPERT_PROGRAM_BYTES, + parse_expert_program_json, +) + +__all__ = [ + "compile_mllm_expert_program", + "decode_mllm_expert_program", +] + +_CURATED_CALL_TYPES = ( + PickCfg, + PlaceCfg, + HandOverCfg, + OperateArticulationCfg, +) + + +def _iter_calls( + node: ProgramNodeCfg, + *, + path: ConfigPath, +) -> Iterator[tuple[SemanticCallCfg, ConfigPath]]: + """Yield every semantic call and its decoder-compatible source path.""" + if type(node) is InvokeCfg: + yield node.call, (*path, "call") + return + if type(node) is SequenceCfg: + for index, child in enumerate(node.items): + yield from _iter_calls(child, path=(*path, "items", index)) + return + if type(node) is RepeatCfg: + yield from _iter_calls(node.body, path=(*path, "body")) + return + if type(node) is SegmentCfg: + yield from _iter_calls(node.steps, path=(*path, "steps")) + return + raise ExpertProgramDecodeError( + "mllm_program_node_not_allowed", + (*path, "kind"), + "The MLLM frontend permits only Version 1 sequential program nodes.", + ) + + +def _value_at_path(value: object, path: ConfigPath) -> object: + """Return a raw decoded JSON value at one already validated config path.""" + current = value + for part in path: + if type(part) is int: + if type(current) is not list or not 0 <= part < len(current): + raise ExpertProgramDecodeError( + "mllm_payload_mismatch", + path, + "Decoded model payload no longer matches the canonical program.", + ) + current = current[part] + else: + if type(current) is not dict or part not in current: + raise ExpertProgramDecodeError( + "mllm_payload_mismatch", + path, + "Decoded model payload no longer matches the canonical program.", + ) + current = current[part] + return current + + +def _validate_mllm_policy( + config: ExpertProgramCfg, + *, + raw_payload: dict[str, object], +) -> None: + """Apply the narrow agent-facing policy after canonical decoding.""" + if config.schema_version != EXPERT_PROGRAM_SCHEMA_VERSION: + raise ExpertProgramDecodeError( + "mllm_schema_version_not_allowed", + ("schema_version",), + "The MLLM frontend permits only Expert Program schema Version 1.", + ) + for call, path in _iter_calls(config.program, path=("program",)): + if type(call) not in _CURATED_CALL_TYPES: + raise ExpertProgramDecodeError( + "mllm_call_not_allowed", + (*path, "kind"), + "The MLLM frontend permits only curated pick, place, hand_over, " + "and operate_articulation calls.", + ) + raw_call = _value_at_path(raw_payload, path) + if type(raw_call) is not dict: + raise ExpertProgramDecodeError( + "mllm_payload_mismatch", + path, + "Decoded model payload no longer matches the canonical program.", + ) + raw_resources = raw_call.get("resources", {}) + if type(raw_resources) is dict and raw_resources: + raise ExpertProgramDecodeError( + "mllm_resource_override_not_allowed", + (*path, "resources"), + "MLLM responses cannot override robot resource bindings.", + ) + if type(call) is HandOverCfg and call.receiver is not None: + raise ExpertProgramDecodeError( + "mllm_resource_override_not_allowed", + (*path, "receiver"), + "MLLM responses cannot select a hand-over receiver resource.", + ) + if type(call) is OperateArticulationCfg and call.target is None: + raise ExpertProgramDecodeError( + "mllm_articulation_target_not_allowed", + (*path, "target_position"), + "MLLM articulation calls must select a host-declared named target.", + ) + if call.resources: + raise ExpertProgramDecodeError( + "mllm_resource_override_not_allowed", + (*path, "resources"), + "MLLM responses cannot override robot resource bindings.", + ) + + +def decode_mllm_expert_program( + response: str, + *, + integration: ExpertProgramIntegrationCfg, + validation_context: ExpertProgramValidationContext | None = None, + max_bytes: int = MAX_EXPERT_PROGRAM_BYTES, +) -> ExpertProgramCfg: + """Decode one untrusted model response into the canonical program config. + + The model response is a single plain JSON object containing + ``schema_version``, ``program_id``, ``targets``, and ``program``. The trusted + host supplies ``integration``; a response attempting to select its own + integration is rejected rather than silently overwritten. Version 1 curated + calls are the only admitted semantic surface, and robot resource overrides + are forbidden. + + Args: + response: Untrusted model response containing one plain JSON document. + integration: Host-owned scene, robot-profile, and runtime-preset choice. + validation_context: Optional provider-free static reference validator. + max_bytes: Maximum UTF-8 encoded response size. + + Returns: + An owned canonical :class:`ExpertProgramCfg`. + + Raises: + TypeError: If ``integration`` is not an exact integration config. + ExpertProgramDecodeError: If JSON, schema, or MLLM policy validation + fails. + """ + if type(integration) is not ExpertProgramIntegrationCfg: + raise TypeError("integration must be exactly ExpertProgramIntegrationCfg.") + data = parse_expert_program_json(response, max_bytes=max_bytes) + if "integration" in data: + raise ExpertProgramDecodeError( + "model_controlled_integration", + ("integration",), + "MLLM responses cannot select an integration; the host injects it.", + ) + payload = dict(data) + payload["integration"] = { + "robot_profile": integration.robot_profile, + "scene_registry": integration.scene_registry, + "runtime_preset": integration.runtime_preset, + } + config = decode_expert_program(payload) + _validate_mllm_policy(config, raw_payload=payload) + if validation_context is not None: + validate_expert_program(config, validation_context) + return config + + +def compile_mllm_expert_program( + response: str, + *, + adapter: ExpertProgramEnvironmentAdapter, + integration: ExpertProgramIntegrationCfg, + validation_context: ExpertProgramValidationContext | None = None, + max_bytes: int = MAX_EXPERT_PROGRAM_BYTES, +) -> CompiledProgram: + """Decode and compile a model response through the existing environment path. + + This function introduces no MLLM-specific compiler. It delegates the owned + config to :meth:`ExpertProgramEnvironmentAdapter.compile`, which performs the + canonical scene resolution and Expert Program lowering used by every other + frontend. + + Args: + response: Untrusted model response containing one plain JSON document. + adapter: Existing trusted Expert Program environment adapter. + integration: Host-owned scene, robot-profile, and runtime-preset choice. + validation_context: Optional provider-free static reference validator. + max_bytes: Maximum UTF-8 encoded response size. + + Returns: + Provider-free program produced by the existing Expert Program compiler. + + Raises: + TypeError: If ``adapter`` or ``integration`` has the wrong exact type. + ExpertProgramDecodeError: If JSON, schema, or MLLM policy validation + fails. + """ + if type(adapter) is not ExpertProgramEnvironmentAdapter: + raise TypeError("adapter must be exactly ExpertProgramEnvironmentAdapter.") + config = decode_mllm_expert_program( + response, + integration=integration, + validation_context=validation_context, + max_bytes=max_bytes, + ) + return adapter.compile(config) diff --git a/tests/agents/mllm/test_expert_program.py b/tests/agents/mllm/test_expert_program.py new file mode 100644 index 000000000..12542e6eb --- /dev/null +++ b/tests/agents/mllm/test_expert_program.py @@ -0,0 +1,433 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Tests for the strict MLLM Expert Program frontend.""" + +from __future__ import annotations + +from collections.abc import Iterable +import json + +import pytest + +from embodichain.agents.mllm import ( + compile_mllm_expert_program, + decode_mllm_expert_program, +) +from embodichain.lab.gym.envs.expert_program import ( + CompiledProgram, + EnvironmentStepClock, + ExpertProgramCompileError, + ExpertProgramDecodeError, + ExpertProgramEnvironmentAdapter, + ExpertProgramIntegrationCfg, + PlanningObservationPort, + decode_expert_program, +) +from embodichain.lab.sim.atomic_actions import AtomicActionEngine, EntityState +from embodichain.lab.sim.skills import ( + EffectEvidenceProvider, + Pick, + RobotSkillProfile, + SceneEntityRegistration, + SceneObjectRef, + SceneRegistry, +) + + +def _integration() -> ExpertProgramIntegrationCfg: + """Return the exact trusted integration selected by the host.""" + return ExpertProgramIntegrationCfg( + robot_profile="test_robot", + scene_registry="test_scene", + runtime_preset="safe", + ) + + +def _invoke(call: dict[str, object]) -> dict[str, object]: + """Wrap one semantic call in an Expert Program invoke node.""" + return {"kind": "invoke", "call": call} + + +def _model_data( + call: dict[str, object] | None = None, + *, + schema_version: int = 1, + program: dict[str, object] | None = None, +) -> dict[str, object]: + """Build the integration-free JSON envelope exposed to the model.""" + if call is None: + call = {"kind": "pick", "object": "cube"} + return { + "schema_version": schema_version, + "program_id": "model_program", + "targets": {}, + "program": _invoke(call) if program is None else program, + } + + +def _model_json( + call: dict[str, object] | None = None, + *, + schema_version: int = 1, + program: dict[str, object] | None = None, +) -> str: + """Serialize one integration-free model response.""" + return json.dumps( + _model_data( + call, + schema_version=schema_version, + program=program, + ) + ) + + +class _UnusedStateProvider: + """Satisfy the static scene contract without allowing live observation.""" + + def observe( + self, + *, + timestamp: float, + env_ids: object, + ) -> EntityState: + """Fail if provider-free compilation accidentally observes the scene.""" + del timestamp, env_ids + raise AssertionError("Provider-free compilation must not observe the scene.") + + +class _CompileOnlyFactory: + """Expose only the scene snapshot needed by adapter compilation.""" + + scene_registry_id = "test_scene" + robot_profile_id = "test_robot" + + def __init__(self) -> None: + self.scene_registry_calls = 0 + + def create_scene_registry(self) -> SceneRegistry: + """Return one canonical object registration and count compilation.""" + self.scene_registry_calls += 1 + return SceneRegistry( + ( + SceneEntityRegistration( + ref=SceneObjectRef("cube"), + state_provider=_UnusedStateProvider(), + semantic_type="cube", + ), + ) + ) + + def create_robot_skill_profile(self) -> RobotSkillProfile: + """Reject runtime assembly in this compile-only test factory.""" + raise AssertionError("MLLM frontend compilation must not assemble a runtime.") + + def create_atomic_action_engine( + self, + profile: RobotSkillProfile, + ) -> AtomicActionEngine: + """Reject engine creation in this compile-only test factory.""" + del profile + raise AssertionError("MLLM frontend compilation must not create an engine.") + + def create_planning_observation_provider( + self, + *, + scene_registry: SceneRegistry, + engine: AtomicActionEngine, + clock: EnvironmentStepClock, + ) -> PlanningObservationPort: + """Reject observation-port creation during provider-free compilation.""" + del scene_registry, engine, clock + raise AssertionError("MLLM frontend compilation must not create live ports.") + + def create_effect_evidence_providers( + self, + *, + scene_registry: SceneRegistry, + engine: AtomicActionEngine, + observation_provider: PlanningObservationPort, + ) -> Iterable[EffectEvidenceProvider]: + """Reject evidence-provider creation during provider-free compilation.""" + del scene_registry, engine, observation_provider + raise AssertionError("MLLM frontend compilation must not create live ports.") + + +def _adapter(factory: _CompileOnlyFactory) -> ExpertProgramEnvironmentAdapter: + """Create the existing production adapter around the compile-only factory.""" + return ExpertProgramEnvironmentAdapter(factory, step_dt=0.02) + + +def test_decoder_injects_exact_host_integration() -> None: + config = decode_mllm_expert_program( + _model_json(), + integration=_integration(), + ) + + assert config.integration.robot_profile == "test_robot" + assert config.integration.scene_registry == "test_scene" + assert config.integration.runtime_preset == "safe" + + +def test_decoder_rejects_model_controlled_integration() -> None: + response = _model_data() + response["integration"] = { + "robot_profile": "attacker_robot", + "scene_registry": "attacker_scene", + "runtime_preset": "unsafe", + } + + with pytest.raises(ExpertProgramDecodeError) as error: + decode_mllm_expert_program( + json.dumps(response), + integration=_integration(), + ) + + assert error.value.code == "model_controlled_integration" + assert error.value.path == ("integration",) + + +def test_decoder_rejects_version_two_parallel_program() -> None: + parallel = { + "kind": "parallel", + "branches": [ + _invoke({"kind": "pick", "object": "cube"}), + _invoke({"kind": "pick", "object": "cube"}), + ], + "barrier": {"kind": "barrier", "name": "join"}, + } + + with pytest.raises(ExpertProgramDecodeError) as error: + decode_mllm_expert_program( + _model_json(schema_version=2, program=parallel), + integration=_integration(), + ) + + assert error.value.code == "mllm_schema_version_not_allowed" + assert error.value.path == ("schema_version",) + + +def test_decoder_rejects_registered_semantic_calls() -> None: + registered = { + "kind": "registered", + "call_id": "vendor.inspect", + "schema_version": 1, + "arguments": {}, + } + + with pytest.raises(ExpertProgramDecodeError) as error: + decode_mllm_expert_program( + _model_json(registered), + integration=_integration(), + ) + + assert error.value.code == "mllm_call_not_allowed" + assert error.value.path == ("program", "call", "kind") + + +@pytest.mark.parametrize( + "call", + [ + {"kind": "pick", "object": "cube", "resources": {"primary": "left"}}, + { + "kind": "place", + "object": "cube", + "on": "tray", + "resources": {"primary": "left"}, + }, + { + "kind": "hand_over", + "object": "cube", + "receiver": "right", + "resources": {"destination": "right"}, + }, + { + "kind": "operate_articulation", + "articulation": "drawer", + "target": "open", + "resources": {"primary": "left"}, + }, + ], +) +def test_decoder_rejects_explicit_nonempty_resource_overrides( + call: dict[str, object], +) -> None: + with pytest.raises(ExpertProgramDecodeError) as error: + decode_mllm_expert_program( + _model_json(call), + integration=_integration(), + ) + + assert error.value.code == "mllm_resource_override_not_allowed" + assert error.value.path == ("program", "call", "resources") + + +def test_decoder_allows_explicit_empty_resources() -> None: + config = decode_mllm_expert_program( + _model_json({"kind": "pick", "object": "cube", "resources": {}}), + integration=_integration(), + ) + + assert config.program.call.resources == {} # type: ignore[union-attr] + + +def test_decoder_rejects_handover_receiver_resource_selection() -> None: + with pytest.raises(ExpertProgramDecodeError) as error: + decode_mllm_expert_program( + _model_json( + { + "kind": "hand_over", + "object": "cube", + "receiver": "right", + } + ), + integration=_integration(), + ) + + assert error.value.code == "mllm_resource_override_not_allowed" + assert error.value.path == ("program", "call", "receiver") + + +def test_decoder_rejects_explicit_articulation_motion_target() -> None: + with pytest.raises(ExpertProgramDecodeError) as error: + decode_mllm_expert_program( + _model_json( + { + "kind": "operate_articulation", + "articulation": "drawer", + "target_position": 1_000_000.0, + "target_displacement": 1_000_000.0, + } + ), + integration=_integration(), + ) + + assert error.value.code == "mllm_articulation_target_not_allowed" + assert error.value.path == ("program", "call", "target_position") + + +def test_decoder_allows_named_articulation_target() -> None: + config = decode_mllm_expert_program( + _model_json( + { + "kind": "operate_articulation", + "articulation": "drawer", + "target": "open", + } + ), + integration=_integration(), + ) + + assert config.program.call.target == "open" # type: ignore[union-attr] + + +@pytest.mark.parametrize( + ("call", "code"), + [ + ( + {"kind": "pick", "object": "env.robot.control_parts"}, + "environment_traversal", + ), + ({"kind": "pick", "object": "eval(1 + 1)"}, "executable_expression"), + ], +) +def test_decoder_reuses_executable_free_value_validation( + call: dict[str, object], + code: str, +) -> None: + with pytest.raises(ExpertProgramDecodeError) as error: + decode_mllm_expert_program( + _model_json(call), + integration=_integration(), + ) + + assert error.value.code == code + + +@pytest.mark.parametrize( + ("response", "code"), + [ + ("```json\n{}\n```", "invalid_json"), + ('{"schema_version": 1, "schema_version": 1}', "duplicate_json_key"), + ('{"schema_version": NaN}', "non_finite_number"), + ('{"schema_version": 1e400}', "non_finite_number"), + ], +) +def test_decoder_propagates_strict_json_failures(response: str, code: str) -> None: + with pytest.raises(ExpertProgramDecodeError) as error: + decode_mllm_expert_program(response, integration=_integration()) + + assert error.value.code == code + + +def test_compile_frontend_reuses_existing_adapter_and_compiler() -> None: + factory = _CompileOnlyFactory() + adapter = _adapter(factory) + response = _model_json() + + model_compiled = compile_mllm_expert_program( + response, + adapter=adapter, + integration=_integration(), + ) + direct_data = _model_data() + direct_data["integration"] = { + "robot_profile": "test_robot", + "scene_registry": "test_scene", + "runtime_preset": "safe", + } + direct_compiled = adapter.compile(decode_expert_program(direct_data)) + + model_call = list(model_compiled)[0].calls[0].call + direct_call = list(direct_compiled)[0].calls[0].call + assert type(model_compiled) is CompiledProgram + assert type(model_call) is Pick + assert type(direct_call) is Pick + assert model_call.object.entity_id == direct_call.object.entity_id == "cube" + assert factory.scene_registry_calls == 2 + + +def test_policy_failure_does_not_touch_adapter_or_runtime() -> None: + factory = _CompileOnlyFactory() + adapter = _adapter(factory) + registered = { + "kind": "registered", + "call_id": "vendor.inspect", + "schema_version": 1, + } + + with pytest.raises(ExpertProgramDecodeError): + compile_mllm_expert_program( + _model_json(registered), + adapter=adapter, + integration=_integration(), + ) + + assert factory.scene_registry_calls == 0 + + +def test_compile_frontend_rejects_unknown_scene_reference() -> None: + factory = _CompileOnlyFactory() + + with pytest.raises(ExpertProgramCompileError) as error: + compile_mllm_expert_program( + _model_json({"kind": "pick", "object": "missing"}), + adapter=_adapter(factory), + integration=_integration(), + ) + + assert error.value.code == "unknown_scene_reference" + assert error.value.path == ("program", "call", "object") diff --git a/tests/gym/envs/expert_program/test_simulation_environment.py b/tests/gym/envs/expert_program/test_simulation_environment.py index 0b8c5c186..119ecca39 100644 --- a/tests/gym/envs/expert_program/test_simulation_environment.py +++ b/tests/gym/envs/expert_program/test_simulation_environment.py @@ -22,6 +22,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, fields, is_dataclass import inspect +import json import textwrap from types import MethodType, SimpleNamespace from typing import Any, ClassVar @@ -30,8 +31,10 @@ import pytest import torch +from embodichain.agents.mllm import compile_mllm_expert_program from embodichain.lab.gym.envs.expert_program import ( AntipodalGraspAffordanceBinding, + CompiledProgram, ControlCommandStateEvidenceTracker, ControlPartCommandPreset, ControlPartEndpointBinding, @@ -1071,12 +1074,22 @@ def _place_evidence_plan(action: Any, request: Any, context: Any) -> Any: ) -def _evidence_runtime() -> tuple[ +def _evidence_integration() -> ExpertProgramIntegrationCfg: + """Return the host-owned integration shared by all frontend paths.""" + return ExpertProgramIntegrationCfg( + robot_profile="evidence_profile", + scene_registry="evidence_scene", + runtime_preset="evidence", + ) + + +def _evidence_adapter_runtime() -> tuple[ + ExpertProgramEnvironmentAdapter, ExpertProgramRuntimeAssembly, _EvidenceRobot, _RigidObject, ]: - """Assemble the production Pick/Place evidence chain on CPU fixtures.""" + """Assemble the production adapter and Pick/Place evidence chain.""" robot = _EvidenceRobot() cube = _RigidObject() simulation = _Simulation(robot, {"cube_native": cube}) @@ -1112,17 +1125,21 @@ def _evidence_runtime() -> tuple[ hold_on_completion=False, ) ) - assembly = adapter.assemble_runtime( - ExpertProgramIntegrationCfg( - robot_profile="evidence_profile", - scene_registry="evidence_scene", - runtime_preset="evidence", - ) - ) + assembly = adapter.assemble_runtime(_evidence_integration()) pick_action = assembly.engine.actions["pick_up"] place_action = assembly.engine.actions["place"] pick_action._plan = MethodType(_pick_evidence_plan, pick_action) place_action._plan = MethodType(_place_evidence_plan, place_action) + return adapter, assembly, robot, cube + + +def _evidence_runtime() -> tuple[ + ExpertProgramRuntimeAssembly, + _EvidenceRobot, + _RigidObject, +]: + """Assemble the production Pick/Place evidence chain on CPU fixtures.""" + _, assembly, robot, cube = _evidence_adapter_runtime() return assembly, robot, cube @@ -1305,60 +1322,79 @@ def _python_pick_place_calls() -> tuple[SemanticCallSpec, ...]: ) -def _decoded_pick_place_calls( - registry: SceneRegistry, -) -> tuple[SemanticCallSpec, ...]: - """Decode and provider-free compile the config equivalent of Python calls.""" - config = decode_expert_program( - { - "schema_version": 1, - "program_id": "pick_place_equivalence", - "integration": { - "robot_profile": "evidence_profile", - "scene_registry": "evidence_scene", - "runtime_preset": "evidence", - }, - "targets": { - "place_target": { - "kind": "cyclic_pose", - "values": [ - { - "position": _DIRECT_PLACE_TARGET.position.tolist(), - "quaternion_wxyz": ( - _DIRECT_PLACE_TARGET.quaternion_wxyz.tolist() - ), - } - ], - } - }, - "program": { - "kind": "sequence", - "items": [ - { - "kind": "invoke", - "call": {"kind": "pick", "object": "cube"}, - }, +def _pick_place_program_data() -> dict[str, object]: + """Return the integration-free program shared with the MLLM frontend.""" + return { + "schema_version": 1, + "program_id": "pick_place_equivalence", + "targets": { + "place_target": { + "kind": "cyclic_pose", + "values": [ { - "kind": "invoke", - "call": { - "kind": "place", - "object": "cube", - "at": { - "kind": "target_ref", - "target": "place_target", - }, + "position": _DIRECT_PLACE_TARGET.position.tolist(), + "quaternion_wxyz": ( + _DIRECT_PLACE_TARGET.quaternion_wxyz.tolist() + ), + } + ], + } + }, + "program": { + "kind": "sequence", + "items": [ + { + "kind": "invoke", + "call": {"kind": "pick", "object": "cube"}, + }, + { + "kind": "invoke", + "call": { + "kind": "place", + "object": "cube", + "at": { + "kind": "target_ref", + "target": "place_target", }, }, - ], - }, - } - ) - program = ExpertProgramCompiler.from_scene_registry(registry).compile(config) + }, + ], + }, + } + + +def _compiled_program_calls(program: CompiledProgram) -> tuple[SemanticCallSpec, ...]: + """Flatten one provider-free compiled program into semantic calls.""" return tuple( compiled_call.call for segment in program for compiled_call in segment.calls ) +def _decoded_pick_place_calls( + adapter: ExpertProgramEnvironmentAdapter, +) -> tuple[SemanticCallSpec, ...]: + """Decode and compile the config equivalent of the Python calls.""" + data = _pick_place_program_data() + data["integration"] = { + "robot_profile": "evidence_profile", + "scene_registry": "evidence_scene", + "runtime_preset": "evidence", + } + return _compiled_program_calls(adapter.compile(decode_expert_program(data))) + + +def _mllm_pick_place_calls( + adapter: ExpertProgramEnvironmentAdapter, +) -> tuple[SemanticCallSpec, ...]: + """Compile the same program through the strict MLLM frontend.""" + program = compile_mllm_expert_program( + json.dumps(_pick_place_program_data()), + adapter=adapter, + integration=_evidence_integration(), + ) + return _compiled_program_calls(program) + + def _capture_grounded_invocations( monkeypatch: pytest.MonkeyPatch, assembly: ExpertProgramRuntimeAssembly, @@ -1381,9 +1417,12 @@ def _run_evidence_pick_place( robot: _EvidenceRobot, cube: _RigidObject, calls: tuple[SemanticCallSpec, ...], + *, + skills: AtomicSkills | None = None, ) -> tuple[SkillResult, HeldObjectState]: """Drive one happy-path workflow through accepted commands and live evidence.""" - result = assembly.runtime.start(calls, workflow_id="pick_place_equivalence") + entry = assembly.runtime if skills is None else skills + result = entry.start(calls, workflow_id="pick_place_equivalence") verified_pick: HeldObjectState | None = None for _ in range(32): while assembly.command_sink.pending_count: @@ -1395,7 +1434,7 @@ def _run_evidence_pick_place( verified_pick = result.task_state.get_held_object("manipulator") cube.pose[:, 0, 3] = _RELEASE_SEPARATION assembly.clock.advance_after_env_step() - result = assembly.runtime.step() + result = entry.step() assert result.status is SkillStatus.COMPLETED assert verified_pick is not None @@ -1494,37 +1533,70 @@ def test_simulation_factory_aligns_every_motion_policy_to_gym_step() -> None: assert profile.presets["safe"].motion_policy.control_dt == pytest.approx(_STEP_DT) -def test_decoded_program_and_python_calls_share_invocations_and_results( +def test_mllm_config_and_atomic_skills_share_invocations_and_verified_results( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Both frontends reach equivalent core invocations and verified state.""" - python_assembly, python_robot, python_cube = _evidence_runtime() - config_assembly, config_robot, config_cube = _evidence_runtime() + """All public frontends reach equivalent invocations and verified state.""" + ( + _, + python_assembly, + python_robot, + python_cube, + ) = _evidence_adapter_runtime() + ( + config_adapter, + config_assembly, + config_robot, + config_cube, + ) = _evidence_adapter_runtime() + ( + mllm_adapter, + mllm_assembly, + mllm_robot, + mllm_cube, + ) = _evidence_adapter_runtime() python_invocations = _capture_grounded_invocations(monkeypatch, python_assembly) config_invocations = _capture_grounded_invocations(monkeypatch, config_assembly) + mllm_invocations = _capture_grounded_invocations(monkeypatch, mllm_assembly) + runtime_provider = _QuickstartRuntimeProvider(python_assembly.runtime) + python_skills = AtomicSkills.from_env(runtime_provider, preset="evidence") python_result, python_held = _run_evidence_pick_place( python_assembly, python_robot, python_cube, _python_pick_place_calls(), + skills=python_skills, ) config_result, config_held = _run_evidence_pick_place( config_assembly, config_robot, config_cube, - _decoded_pick_place_calls(config_assembly.scene_registry), + _decoded_pick_place_calls(config_adapter), + ) + mllm_result, mllm_held = _run_evidence_pick_place( + mllm_assembly, + mllm_robot, + mllm_cube, + _mllm_pick_place_calls(mllm_adapter), ) - assert len(python_invocations) == len(config_invocations) == 2 - for python_invocation, config_invocation in zip( + assert runtime_provider.presets == ["evidence"] + assert ( + len(python_invocations) == len(config_invocations) == len(mllm_invocations) == 2 + ) + for python_invocation, config_invocation, mllm_invocation in zip( python_invocations, config_invocations, + mllm_invocations, strict=True, ): _assert_invocation_equivalent(python_invocation, config_invocation) + _assert_invocation_equivalent(python_invocation, mllm_invocation) _assert_typed_equivalent(python_held, config_held) + _assert_typed_equivalent(python_held, mllm_held) _assert_typed_equivalent(python_result, config_result) + _assert_typed_equivalent(python_result, mllm_result) def test_atomic_skills_from_env_runs_documented_pick_place_quickstart() -> None: