From ab8e7d4a1562dcae1637e114d3629573484aa7a6 Mon Sep 17 00:00:00 2001 From: skywhite1024 <129768272+skywhite1024@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:19:23 +0800 Subject: [PATCH] feat(task-engine): add semantic task interpretation and grounding --- .../generation/config_builder.py | 814 +++++++++++ .../action_engine/generation/models.py | 82 ++ .../generation/templates/robot_profiles.json | 45 + .../gen_sim/action_engine/tasks/__init__.py | 48 + .../gen_sim/action_engine/tasks/assembly.py | 425 ++++++ .../gen_sim/action_engine/tasks/grounding.py | 513 +++++++ .../action_engine/tasks/interpretation.py | 404 ++++++ .../gen_sim/action_engine/tasks/scene.py | 177 +++ embodichain/gen_sim/task_engine/__init__.py | 95 ++ embodichain/gen_sim/task_engine/agent.py | 289 ++++ embodichain/gen_sim/task_engine/contracts.py | 410 ++++++ .../gen_sim/task_engine/interpretation.py | 1240 +++++++++++++++++ tests/gen_sim/action_engine/task_fixtures.py | 229 +++ tests/gen_sim/action_engine/tasks/__init__.py | 19 + .../action_engine/tasks/test_grounding.py | 339 +++++ tests/gen_sim/task_engine/__init__.py | 19 + .../task_engine/test_interpretation.py | 96 ++ 17 files changed, 5244 insertions(+) create mode 100644 embodichain/gen_sim/action_engine/generation/config_builder.py create mode 100644 embodichain/gen_sim/action_engine/generation/models.py create mode 100644 embodichain/gen_sim/action_engine/generation/templates/robot_profiles.json create mode 100644 embodichain/gen_sim/action_engine/tasks/__init__.py create mode 100644 embodichain/gen_sim/action_engine/tasks/assembly.py create mode 100644 embodichain/gen_sim/action_engine/tasks/grounding.py create mode 100644 embodichain/gen_sim/action_engine/tasks/interpretation.py create mode 100644 embodichain/gen_sim/action_engine/tasks/scene.py create mode 100644 embodichain/gen_sim/task_engine/__init__.py create mode 100644 embodichain/gen_sim/task_engine/agent.py create mode 100644 embodichain/gen_sim/task_engine/contracts.py create mode 100644 embodichain/gen_sim/task_engine/interpretation.py create mode 100644 tests/gen_sim/action_engine/task_fixtures.py create mode 100644 tests/gen_sim/action_engine/tasks/__init__.py create mode 100644 tests/gen_sim/action_engine/tasks/test_grounding.py create mode 100644 tests/gen_sim/task_engine/__init__.py create mode 100644 tests/gen_sim/task_engine/test_interpretation.py diff --git a/embodichain/gen_sim/action_engine/generation/config_builder.py b/embodichain/gen_sim/action_engine/generation/config_builder.py new file mode 100644 index 000000000..51e53648e --- /dev/null +++ b/embodichain/gen_sim/action_engine/generation/config_builder.py @@ -0,0 +1,814 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Build the simulator and Action Engine artifact manifests.""" + +from __future__ import annotations + +from collections.abc import Sequence +from copy import deepcopy +from functools import lru_cache +import json +import math +from pathlib import Path +from typing import Any + +from embodichain.gen_sim.action_engine.config import ( + ACTION_ENGINE_DEFAULTS_SCHEMA, + RuntimePolicyCfg, + default_runtime_policy, + generation_defaults, + runtime_policy_hash, +) +from embodichain.gen_sim.action_engine.protocol import ( + ACTION_ENGINE_CONFIG_SCHEMA, + ACTION_ENGINE_ENV_ID, + EXECUTION_PROGRAM_FILENAME, + SCENE_REQUIREMENTS_FILENAME, + TASK_SPEC_FILENAME, +) + +from .models import PreparedScene + +__all__ = [ + "build_agent_config", + "build_fast_gym_config", + "canonical_robot_profile", + "VLM_CAMERA_UIDS", + "validate_fast_gym_config", +] + +_TEMPLATE_DIR = Path(__file__).resolve().parent / "templates" +_GENERATION_DEFAULTS = generation_defaults() +_DEFAULT_TABLETOP_Z = float(_GENERATION_DEFAULTS["scene"]["default_tabletop_z"]) + +_ARM_SLOTS = { + "left": {"arm": "left_arm", "eef": "left_eef"}, + "right": {"arm": "right_arm", "eef": "right_eef"}, +} + +# These IDs are part of the A/B runtime contract. Keep the order stable so +# visual-fact payloads and comparison reports are reproducible across runs. +VLM_CAMERA_UIDS = ( + "vlm_front", + "vlm_left", + "vlm_rear", + "vlm_right", +) + + +def canonical_robot_profile(profile: str) -> str: + """Normalize the supported CLI aliases to one runtime profile ID.""" + normalized = str(profile).strip().lower().replace("-", "_") + profiles = _robot_profiles() + if normalized in profiles: + return normalized + for profile_id, value in profiles.items(): + if normalized in value["aliases"]: + return profile_id + raise ValueError( + f"Unsupported robot profile {profile!r}; expected one of: " + f"{', '.join(sorted(profiles))}" + ) + + +def build_agent_config( + *, + task_name: str, + robot_profile: str, + execution_program_hash: str, + source_config_path: Path, + uid_map: dict[str, str], + static_obstacle_uids: Sequence[str] | None = None, + dynamic_obstacle_uids: Sequence[str] | None = None, + table_top_z: float | None = None, + planning_mode: str = "offline", + seed_task_graph_path: str | Path | None = EXECUTION_PROGRAM_FILENAME, + vlm_model: str | None = None, + vlm_camera_uids: Sequence[str] | None = None, +) -> dict[str, Any]: + """Build the small manifest consumed by ``run_agent``.""" + profile = canonical_robot_profile(robot_profile) + runtime_policy = default_runtime_policy(profile) + if ( + static_obstacle_uids is not None + or dynamic_obstacle_uids is not None + or table_top_z is not None + ): + policy = runtime_policy.as_mapping() + planner = policy["planner"] + if static_obstacle_uids is not None: + planner["static_obstacle_uids"] = [str(uid) for uid in static_obstacle_uids] + if dynamic_obstacle_uids is not None: + planner["dynamic_obstacle_uids"] = [ + str(uid) for uid in dynamic_obstacle_uids + ] + planner["dynamic_collision"] = bool(dynamic_obstacle_uids) + if table_top_z is not None: + tabletop = float(table_top_z) + if not math.isfinite(tabletop): + raise ValueError("table_top_z must be finite when provided.") + height_offset = tabletop - _DEFAULT_TABLETOP_Z + height_policies = ( + policy["grounding"]["semantic_defaults"], + policy["grounding"]["handover"], + policy["motion_defaults"]["MoveEndEffector"], + policy["motion_modifiers"]["orientation"]["upright"]["MoveEndEffector"], + ) + for height_policy in height_policies: + height_policy["maximum_eef_height"] = round( + float(height_policy["maximum_eef_height"]) + height_offset, + 6, + ) + runtime_policy = RuntimePolicyCfg.from_mapping(policy) + _validate_planning_mode(planning_mode) + graph_path = _validate_seed_graph_path(seed_task_graph_path) + if planning_mode == "ab" and graph_path == EXECUTION_PROGRAM_FILENAME: + graph_path = f"offline/{EXECUTION_PROGRAM_FILENAME}" + result = { + "schema_version": ACTION_ENGINE_CONFIG_SCHEMA, + "task_name": task_name, + "robot_profile": profile, + "planning_mode": planning_mode, + "task_spec": TASK_SPEC_FILENAME, + "scene_requirements": SCENE_REQUIREMENTS_FILENAME, + "seed_task_graph": graph_path, + "seed_task_graph_hash": execution_program_hash, + "runtime_policy": runtime_policy.as_mapping(), + "runtime_policy_hash": runtime_policy_hash(runtime_policy), + "source": { + "gym_config": source_config_path.as_posix(), + "uid_map": dict(sorted(uid_map.items())), + }, + } + if planning_mode == "ab": + camera_uids = _normalize_vlm_camera_uids(vlm_camera_uids) + configured_model = _optional_model(vlm_model) + # Retain concise top-level aliases for early A/B bundles while keeping + # the nested section as the canonical runtime namespace. + result["offline_seed_task_graph"] = graph_path + result["vlm_model"] = configured_model + result["vlm_camera_uids"] = list(camera_uids) + result["online_planning"] = { + # Model names are deliberately persisted only when explicitly + # supplied by the generator. Runtime resolution can then apply + # the documented ACTION_ENGINE_VLM_MODEL/OPENAI_MODEL fallback. + "vlm_model": configured_model, + "camera_uids": camera_uids, + } + return result + + +def build_fast_gym_config( + scene: PreparedScene, + *, + task_name: str, + task_description: str, + robot_profile: str, + execution_program_hash: str, + max_episodes: int, + max_episode_steps: int, + randomize_scene: bool = False, + randomize_table_material: bool = False, + planning_mode: str = "offline", + seed_task_graph_path: str | Path | None = EXECUTION_PROGRAM_FILENAME, +) -> dict[str, Any]: + """Build a runnable EmbodiChain gym config from a prepared source scene.""" + if max_episodes < 1: + raise ValueError("max_episodes must be at least 1.") + if max_episode_steps < 1: + raise ValueError("max_episode_steps must be at least 1.") + _validate_planning_mode(planning_mode) + graph_path = _validate_seed_graph_path(seed_task_graph_path) + if planning_mode == "ab" and graph_path == EXECUTION_PROGRAM_FILENAME: + graph_path = f"offline/{EXECUTION_PROGRAM_FILENAME}" + profile = canonical_robot_profile(robot_profile) + + profile_config = _profile(profile) + robot = _make_robot(profile, profile_config, scene.table_top_z) + observations = _make_observations(robot) + # These two template fields describe serialization order to generation, not + # RobotCfg. Remove them after deriving observation IDs to avoid parser noise. + robot.pop("observation_joint_parts", None) + robot.pop("qpos_control_part_order", None) + sensors = _load_template("default_sensors.json") + if not isinstance(sensors, list) or not sensors: + raise ValueError("Default sensor template must define at least one camera.") + environment_policy = _GENERATION_DEFAULTS["environment"] + viewer_camera_uid = str(environment_policy["viewer_camera_uid"]) + sensors[0]["uid"] = viewer_camera_uid + if planning_mode == "ab": + vlm_sensors = _load_template("vlm_sensors.json") + if not isinstance(vlm_sensors, list) or len(vlm_sensors) != len( + VLM_CAMERA_UIDS + ): + raise ValueError("A/B planning requires exactly four VLM cameras.") + _validate_vlm_sensors(vlm_sensors) + _anchor_vlm_sensors(vlm_sensors, scene) + sensors.extend(vlm_sensors) + light = _load_template("default_lights.json") + + rigid_uids = [str(config["uid"]) for config in scene.rigid_objects] + background_uids = [str(config["uid"]) for config in scene.background] + engine_extension = { + "schema_version": "action_engine_runtime_v2", + "defaults_schema_version": ACTION_ENGINE_DEFAULTS_SCHEMA, + "task_name": task_name, + "robot_profile": profile, + "planning_mode": planning_mode, + "task_spec": TASK_SPEC_FILENAME, + "scene_requirements": SCENE_REQUIREMENTS_FILENAME, + "seed_task_graph": graph_path, + "seed_task_graph_hash": execution_program_hash, + "source_gym_config": scene.source_config_path.as_posix(), + "source_scene_z_rotation_degrees": scene.z_rotation_degrees, + "body_scale_policy": scene.body_scale_policy, + "body_scale": list(scene.body_scale), + "asset_hashes": dict(sorted(scene.asset_hashes.items())), + "asset_provenance": [deepcopy(value) for value in scene.asset_provenance], + "uid_map": dict(sorted(scene.uid_map.items())), + } + extensions = { + "action_engine": engine_extension, + "agent_robot_profile": profile, + "agent_arm_slots": deepcopy(_ARM_SLOTS), + "agent_static_obstacle_uids": background_uids, + "agent_dynamic_obstacle_uids": rigid_uids, + "gripper_open_state": list(profile_config["gripper_open_state"]), + "gripper_close_state": list(profile_config["gripper_close_state"]), + "arm_aim_yaw_offset": deepcopy(environment_policy["arm_aim_yaw_offset"]), + "ignore_terminations_during_agent": bool( + environment_policy["ignore_terminations_during_agent"] + ), + "viewer_camera_uid": viewer_camera_uid, + } + + config: dict[str, Any] = { + "id": ACTION_ENGINE_ENV_ID, + "max_episodes": int(max_episodes), + "max_episode_steps": int(max_episode_steps), + "env": { + "extensions": extensions, + "events": _make_events( + sensors[0], + rigid_uids, + planning_mode=planning_mode, + randomize_scene=randomize_scene, + randomize_table_material=randomize_table_material, + ), + "observations": observations, + "dataset": _make_dataset( + task_name=task_name, + task_description=task_description, + source_config_path=scene.source_config_path, + robot_type=str(robot["uid"]), + ), + }, + "robot": robot, + "sensor": sensors, + "light": light, + "background": [deepcopy(obj_config) for obj_config in scene.background], + "rigid_object": [deepcopy(obj_config) for obj_config in scene.rigid_objects], + } + if scene.articulations: + config["articulation"] = [ + deepcopy(articulation) for articulation in scene.articulations + ] + validate_fast_gym_config(config) + return config + + +def validate_fast_gym_config(config: dict[str, Any]) -> None: + """Check the cross-file and simulator-facing invariants generation owns.""" + if config.get("id") != ACTION_ENGINE_ENV_ID: + raise ValueError(f"Gym config id must be {ACTION_ENGINE_ENV_ID!r}.") + if not isinstance(config.get("robot"), dict) or not config["robot"].get("uid"): + raise ValueError("Gym config requires a concrete robot template.") + if not config.get("sensor"): + raise ValueError("Gym config requires at least one sensor.") + if not all(isinstance(sensor, dict) for sensor in config["sensor"]): + raise ValueError("Generated sensors must be object mappings.") + sensor_uids = [str(sensor.get("uid", "")) for sensor in config["sensor"]] + if not all(sensor_uids) or len(sensor_uids) != len(set(sensor_uids)): + raise ValueError("Generated sensor UIDs must be non-empty and unique.") + if not config.get("background"): + raise ValueError("Gym config requires at least one background object.") + + objects = [ + *config.get("background", []), + *config.get("rigid_object", []), + *config.get("articulation", []), + ] + uids = [str(obj.get("uid", "")) for obj in objects] + if not all(uids) or len(uids) != len(set(uids)): + raise ValueError("Generated scene object UIDs must be non-empty and unique.") + if "table" not in uids: + raise ValueError("Generated tabletop scene must expose runtime UID 'table'.") + + for obj in objects: + shape = obj.get("shape") + fpath = shape.get("fpath") if isinstance(shape, dict) else obj.get("fpath") + if fpath is None: + continue + path = Path(str(fpath)) + if not path.is_absolute() or not path.is_file(): + raise ValueError( + f"Generated asset path for {obj.get('uid')!r} is not an " + f"existing absolute file: {path}" + ) + + action_engine = config.get("env", {}).get("extensions", {}).get("action_engine", {}) + if action_engine.get("defaults_schema_version") != ACTION_ENGINE_DEFAULTS_SCHEMA: + raise ValueError("Gym config has an unexpected defaults schema version.") + if action_engine.get("task_spec") != TASK_SPEC_FILENAME: + raise ValueError("Gym config points to an unexpected TaskSpec artifact.") + if action_engine.get("scene_requirements") != SCENE_REQUIREMENTS_FILENAME: + raise ValueError("Gym config points to unexpected SceneRequirements.") + graph_path = action_engine.get("seed_task_graph") + if ( + not isinstance(graph_path, str) + or Path(graph_path).name != EXECUTION_PROGRAM_FILENAME + ): + raise ValueError("Gym config points to an unexpected SeedGraph artifact.") + + planning_mode = action_engine.get("planning_mode", "offline") + _validate_planning_mode(planning_mode) + if planning_mode == "ab": + sensors = config["sensor"] + vlm_sensors = [ + sensor + for sensor in sensors + if isinstance(sensor, dict) + and str(sensor.get("uid", "")).startswith("vlm_") + ] + _validate_vlm_sensors(vlm_sensors) + + registered = { + entry.get("entity_cfg", {}).get("uid") + for entry in ( + config.get("env", {}) + .get("events", {}) + .get("register_info_to_env", {}) + .get("params", {}) + .get("registry", []) + ) + } + rigid_uids = {obj["uid"] for obj in config.get("rigid_object", [])} + if registered != rigid_uids: + raise ValueError("Every rigid object must have one live-pose registry entry.") + + +def _make_robot( + profile_id: str, + profile: dict[str, Any], + table_top_z: float | None, +) -> dict[str, Any]: + robot = _load_template(str(profile["template"])) + tabletop_z = _DEFAULT_TABLETOP_Z if table_top_z is None else float(table_top_z) + robot["init_pos"][2] = round( + tabletop_z + + float(profile["tabletop_clearance"]) + - float(profile["arm_component_z"]), + 6, + ) + family = str(profile["robot_family"]) + if family.startswith("ur"): + display = family.upper() + urdf_dir = display + robot["uid"] = f"Dual{display}" + robot["urdf_cfg"]["fname"] = f"dual_{family}_robotiq_arg2f_140_basket" + for component in robot["urdf_cfg"]["components"]: + if str(component.get("component_type", "")).endswith("_arm"): + component["urdf_path"] = f"UniversalRobots/{urdf_dir}/{urdf_dir}.urdf" + component["transform"][0][3] = float(profile["arm_base_x"]) + component["transform"][2][3] = float(profile["arm_component_z"]) + for arm in ("left_arm", "right_arm"): + robot["solver_cfg"][arm]["ur_type"] = family + robot["drive_pros"]["max_effort"][arm] = float(profile["max_effort"]) + robot["qpos_control_part_order"] = [ + "left_arm", + "right_arm", + "left_eef", + "right_eef", + ] + robot["observation_joint_parts"] = ["left_eef", "right_eef"] + if profile_id != canonical_robot_profile(profile_id): + raise ValueError(f"Invalid canonical robot profile {profile_id!r}.") + return robot + + +@lru_cache(maxsize=1) +def _robot_profiles() -> dict[str, dict[str, Any]]: + value = _read_template("robot_profiles.json") + if not isinstance(value, dict) or not value: + raise ValueError("robot_profiles.json must contain a non-empty object.") + return value + + +def _profile(profile_id: str) -> dict[str, Any]: + profile = deepcopy(_robot_profiles()[profile_id]) + required = { + "aliases", + "template", + "robot_family", + "tabletop_clearance", + "arm_component_z", + "gripper_open_state", + "gripper_close_state", + } + missing = sorted(required - set(profile)) + if missing: + raise ValueError(f"Robot profile {profile_id!r} is missing fields: {missing}.") + return profile + + +def _make_events( + camera: dict[str, Any], + rigid_uids: list[str], + *, + planning_mode: str, + randomize_scene: bool = False, + randomize_table_material: bool = False, +) -> dict[str, Any]: + extrinsics = camera["extrinsics"] + eye = list(extrinsics["eye"]) + target = list(extrinsics["target"]) + # The recording view mirrors the interactive viewer around its target. + audience_eye = [ + 2.0 * float(target[0]) - float(eye[0]), + 2.0 * float(target[1]) - float(eye[1]), + float(eye[2]), + ] + recording_enabled, recording_resolution, recording_interval = _recording_policy( + planning_mode + ) + source_width = int(camera["width"]) + source_height = int(camera["height"]) + if source_width <= 0 or source_height <= 0: + raise ValueError("Recording source camera resolution must be positive.") + intrinsics = camera.get("intrinsics") + if ( + not isinstance(intrinsics, Sequence) + or isinstance(intrinsics, (str, bytes, bytearray)) + or len(intrinsics) != 4 + ): + raise ValueError("Recording source camera intrinsics must be a 4-vector.") + scale_x = recording_resolution[0] / source_width + scale_y = recording_resolution[1] / source_height + recording_intrinsics = [ + float(intrinsics[0]) * scale_x, + float(intrinsics[1]) * scale_y, + float(intrinsics[2]) * scale_x, + float(intrinsics[3]) * scale_y, + ] + events = { + "record_camera": { + "func": "record_camera_data", + "mode": "interval", + "interval_step": recording_interval, + "params": { + "name": "record_cam_audience_view", + "resolution": list(recording_resolution), + "intrinsics": recording_intrinsics, + "eye": audience_eye, + "target": target, + "up": [ + -float(extrinsics["up"][0]), + -float(extrinsics["up"][1]), + float(extrinsics["up"][2]), + ], + }, + }, + "validation_cameras": { + "func": "validation_cameras", + "mode": "trigger", + "params": {}, + }, + "prepare_extra_attr": { + "func": "prepare_extra_attr", + "mode": "reset", + "params": { + "attrs": [ + { + "name": "object_lengths", + "mode": "callable", + "entity_uids": "all_objects", + "func_name": "compute_object_length", + "func_kwargs": { + "is_svd_frame": True, + "sample_points": int( + _GENERATION_DEFAULTS["scene"][ + "object_length_sample_points" + ] + ), + }, + } + ] + }, + }, + "register_info_to_env": { + "func": "register_info_to_env", + "mode": "reset", + "params": { + "registry": [ + { + "entity_cfg": {"uid": uid}, + "pose_register_params": { + "compute_relative": False, + "compute_pose_object_to_arena": True, + "to_matrix": True, + }, + } + for uid in sorted(rigid_uids) + ], + "registration": "affordance_datas", + "sim_update": True, + }, + }, + } + if not recording_enabled: + events.pop("record_camera") + if randomize_table_material: + material = _GENERATION_DEFAULTS["randomization"]["table_material"] + events["randomize_table_material"] = { + "func": "randomize_visual_material", + "mode": "reset", + "params": { + "entity_cfg": {"uid": "table"}, + "random_texture_prob": float(material["random_texture_prob"]), + "base_color_range": deepcopy(material["base_color_range"]), + "metallic_range": list(material["metallic_range"]), + "roughness_range": list(material["roughness_range"]), + }, + } + if randomize_scene: + randomization = _GENERATION_DEFAULTS["randomization"] + for uid in sorted(rigid_uids): + events[f"randomize_{uid}_pose"] = { + "func": "randomize_rigid_object_pose", + "mode": "reset", + "params": { + "entity_cfg": {"uid": uid}, + "position_range": deepcopy( + randomization["rigid_object_position_range"] + ), + "rotation_range": deepcopy( + randomization["rigid_object_rotation_range"] + ), + "relative_position": True, + "relative_rotation": True, + }, + } + events["randomize_table_height"] = { + "func": "randomize_anchor_height", + "mode": "reset", + "params": { + "anchor_uid": "table", + "height_delta_range": deepcopy( + randomization["table_height_delta_range"] + ), + }, + } + return events + + +def _recording_policy(planning_mode: str) -> tuple[bool, tuple[int, int], int]: + """Resolve the bounded GenSim audience-recording policy.""" + value = _GENERATION_DEFAULTS["environment"].get("recording") + required = {"enabled", "resolution", "interval_step"} + if not isinstance(value, dict) or set(value) != required: + raise ValueError( + "generation.environment.recording must define enabled, resolution, " + "and interval_step." + ) + enabled = value["enabled"] + if not isinstance(enabled, bool): + raise ValueError("generation.environment.recording.enabled must be a boolean.") + resolution = value["resolution"] + if ( + not isinstance(resolution, Sequence) + or isinstance(resolution, (str, bytes, bytearray)) + or len(resolution) != 2 + or any( + isinstance(item, bool) or not isinstance(item, int) for item in resolution + ) + or any(int(item) <= 0 for item in resolution) + ): + raise ValueError( + "generation.environment.recording.resolution must contain two " + "positive integers." + ) + interval_step = value["interval_step"] + if ( + isinstance(interval_step, bool) + or not isinstance(interval_step, int) + or interval_step <= 0 + ): + raise ValueError( + "generation.environment.recording.interval_step must be positive." + ) + return ( + bool(enabled or planning_mode == "ab"), + (int(resolution[0]), int(resolution[1])), + int(interval_step), + ) + + +def _make_observations(robot: dict[str, Any]) -> dict[str, Any]: + control_parts = robot["control_parts"] + qpos_order = robot["qpos_control_part_order"] + observed_parts = set(robot["observation_joint_parts"]) + offset = 0 + joint_ids: list[int] = [] + for part in qpos_order: + count = len(control_parts[part]) + if part in observed_parts: + joint_ids.extend(range(offset, offset + count)) + offset += count + return { + "norm_robot_eef_joint": { + "func": "normalize_robot_joint_data", + "mode": "modify", + "name": "robot/qpos", + "params": {"joint_ids": joint_ids}, + } + } + + +def _make_dataset( + *, + task_name: str, + task_description: str, + source_config_path: Path, + robot_type: str, +) -> dict[str, Any]: + dataset_policy = _GENERATION_DEFAULTS["dataset"] + return { + "lerobot": { + "func": "LeRobotRecorder", + "mode": "save", + "save_failed_episodes": bool(dataset_policy["save_failed_episodes"]), + "params": { + "robot_meta": { + "robot_type": robot_type, + "control_freq": int(dataset_policy["control_frequency"]), + }, + "instruction": {"lang": task_description}, + "extra": { + "scene_type": source_config_path.parent.name, + "task_name": task_name, + # LeRobotRecorder uses this legacy field as a directory label. + "task_description": task_name, + "data_type": "sim", + }, + "use_videos": bool(dataset_policy["use_videos"]), + }, + } + } + + +def _load_template(name: str) -> Any: + return deepcopy(_read_template(name)) + + +@lru_cache(maxsize=None) +def _read_template(name: str) -> Any: + path = _TEMPLATE_DIR / name + if not path.is_file(): + raise FileNotFoundError(f"Action Engine template not found: {path}") + return json.loads(path.read_text(encoding="utf-8")) + + +def _validate_planning_mode(value: Any) -> str: + """Validate and return the two supported generation/runtime modes.""" + if value not in {"offline", "ab"}: + raise ValueError("planning_mode must be 'offline' or 'ab'.") + return str(value) + + +def _validate_seed_graph_path(value: str | Path | None) -> str: + """Validate a relative or absolute path while preserving caller spelling.""" + if value is None: + return EXECUTION_PROGRAM_FILENAME + if not isinstance(value, (str, Path)): + raise ValueError("seed_task_graph_path must be a non-empty path string.") + path = str(value).strip() + if not path: + raise ValueError("seed_task_graph_path must be a non-empty path string.") + if Path(path).name != EXECUTION_PROGRAM_FILENAME: + raise ValueError("seed_task_graph_path must point to seed_task_graph.json.") + return path + + +def _optional_model(value: Any) -> str | None: + """Normalize optional model names without serializing blank strings.""" + if value is None: + return None + if not isinstance(value, str): + raise TypeError("Model name must be a string or None.") + normalized = value.strip() + return normalized or None + + +def _normalize_vlm_camera_uids(value: Sequence[str] | None) -> list[str]: + """Return the canonical four-camera list used by A/B execution.""" + if value is None: + return list(VLM_CAMERA_UIDS) + if isinstance(value, (str, bytes, bytearray)) or not isinstance(value, Sequence): + raise TypeError("vlm_camera_uids must be a list of strings.") + if not all(isinstance(item, str) for item in value): + raise TypeError("vlm_camera_uids must be a list of strings.") + normalized = [item.strip() for item in value] + if normalized != list(VLM_CAMERA_UIDS): + raise ValueError( + "A/B planning requires VLM cameras in canonical order: " + f"{list(VLM_CAMERA_UIDS)}." + ) + return normalized + + +def _validate_vlm_sensors(value: list[dict[str, Any]]) -> None: + """Validate camera template fields needed by visual fact extraction.""" + if len(value) != len(VLM_CAMERA_UIDS): + raise ValueError("A/B planning requires exactly four VLM cameras.") + if not all(isinstance(sensor, dict) for sensor in value): + raise ValueError("VLM sensors must be object mappings.") + uids = [str(sensor.get("uid", "")) for sensor in value] + if uids != list(VLM_CAMERA_UIDS): + raise ValueError("VLM camera UIDs must be exactly " f"{list(VLM_CAMERA_UIDS)}.") + for sensor in value: + if sensor.get("sensor_type", "Camera") != "Camera": + raise ValueError(f"VLM sensor {sensor.get('uid')!r} must be a Camera.") + if int(sensor.get("width", 0)) != 640 or int(sensor.get("height", 0)) != 480: + raise ValueError("VLM cameras must use 640x480 resolution.") + if not bool(sensor.get("enable_color")) or not bool(sensor.get("enable_depth")): + raise ValueError("VLM cameras must enable RGB and depth.") + extrinsics = sensor.get("extrinsics") + if not isinstance(extrinsics, dict) or not all( + key in extrinsics for key in ("eye", "target", "up") + ): + raise ValueError( + f"VLM sensor {sensor.get('uid')!r} requires eye/target/up extrinsics." + ) + for name in ("eye", "target", "up"): + vector = extrinsics[name] + if ( + not isinstance(vector, Sequence) + or isinstance(vector, (str, bytes, bytearray)) + or len(vector) != 3 + ): + raise ValueError( + f"VLM sensor {sensor.get('uid')!r} {name} must be a 3-vector." + ) + try: + values = [float(item) for item in vector] + except (TypeError, ValueError) as exc: + raise ValueError( + f"VLM sensor {sensor.get('uid')!r} {name} must be numeric." + ) from exc + if not all(math.isfinite(item) for item in values): + raise ValueError( + f"VLM sensor {sensor.get('uid')!r} {name} must be finite." + ) + + +def _anchor_vlm_sensors(sensors: list[dict[str, Any]], scene: PreparedScene) -> None: + """Aim the fixed high views at the normalized tabletop center.""" + table = next( + ( + item + for item in scene.background + if isinstance(item, dict) and str(item.get("uid")) == "table" + ), + None, + ) + init_pos = table.get("init_pos", [0.0, 0.0, 0.0]) if table else [0.0, 0.0, 0.0] + if not isinstance(init_pos, Sequence) or len(init_pos) != 3: + init_pos = [0.0, 0.0, 0.0] + center = [ + float(init_pos[0]), + float(init_pos[1]), + float(scene.table_top_z if scene.table_top_z is not None else 0.75), + ] + for sensor in sensors: + extrinsics = sensor["extrinsics"] + eye = [float(value) for value in extrinsics["eye"]] + target = [float(value) for value in extrinsics["target"]] + offset = [target[index] - 0.0 for index in range(3)] + extrinsics["target"] = list(center) + extrinsics["eye"] = [ + center[index] + eye[index] - offset[index] for index in range(3) + ] diff --git a/embodichain/gen_sim/action_engine/generation/models.py b/embodichain/gen_sim/action_engine/generation/models.py new file mode 100644 index 000000000..ba35566f2 --- /dev/null +++ b/embodichain/gen_sim/action_engine/generation/models.py @@ -0,0 +1,82 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Small value objects used by Action Engine config generation.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +__all__ = ["GeneratedConfigPaths", "PreparedScene"] + + +@dataclass(frozen=True) +class GeneratedConfigPaths: + """Paths written by one successful generation transaction.""" + + gym_config: Path + agent_config: Path + task_spec: Path + scene_requirements: Path + seed_task_graph: Path + seed_task_graph_png: Path + planning_mode: str = "offline" + + @property + def execution_program(self) -> Path: + """Retain the Python API alias for callers migrating to SeedGraph v3.""" + return self.seed_task_graph + + @property + def offline_seed_task_graph(self) -> Path: + """Explicit A/B alias for the immutable offline SeedGraph artifact.""" + return self.seed_task_graph + + @property + def seed_task_graph_path(self) -> Path: + """Path-style alias used by runtime config loaders.""" + return self.seed_task_graph + + @property + def offline_seed_task_graph_path(self) -> Path: + """Verbose alias for callers that distinguish A/B graph branches.""" + return self.seed_task_graph + + @property + def offline_seed_task_graph_png(self) -> Path: + """Explicit A/B alias for the review rendering of the offline graph.""" + return self.seed_task_graph_png + + +@dataclass(frozen=True) +class PreparedScene: + """A source scene normalized for both planning and simulator loading.""" + + source_config_path: Path + scene_dir: Path + planner_objects: tuple[dict[str, Any], ...] + background: tuple[dict[str, Any], ...] + rigid_objects: tuple[dict[str, Any], ...] + articulations: tuple[dict[str, Any], ...] + uid_map: dict[str, str] + table_top_z: float | None + z_rotation_degrees: float + body_scale_policy: str + body_scale: tuple[float, float, float] + asset_hashes: dict[str, str] + asset_provenance: tuple[dict[str, Any], ...] = () diff --git a/embodichain/gen_sim/action_engine/generation/templates/robot_profiles.json b/embodichain/gen_sim/action_engine/generation/templates/robot_profiles.json new file mode 100644 index 000000000..31084a497 --- /dev/null +++ b/embodichain/gen_sim/action_engine/generation/templates/robot_profiles.json @@ -0,0 +1,45 @@ +{ + "dual_franka": { + "aliases": ["franka", "panda", "dual_panda", "dual_franka_panda"], + "template": "dual_franka_robot.json", + "robot_family": "franka", + "tabletop_clearance": 0.05, + "arm_component_z": 0.4, + "arm_base_x": -1.45, + "gripper_open_state": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + "gripper_close_state": [0.7, -0.7, 0.7, -0.7, -0.7, 0.7] + }, + "dual_ur3": { + "aliases": ["ur3", "dual_ur3_dh_pgi", "dual_ur3_robotiq", "dual_ur3_robotiq_arg2f_140"], + "template": "dual_ur_robot.json", + "robot_family": "ur3", + "tabletop_clearance": 0.05, + "arm_component_z": 0.4, + "arm_base_x": -1.45, + "max_effort": 56.0, + "gripper_open_state": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + "gripper_close_state": [0.7, -0.7, 0.7, -0.7, -0.7, 0.7] + }, + "dual_ur5": { + "aliases": ["ur5", "dual_ur5_dh_pgi", "dual_ur5_robotiq", "dual_ur5_robotiq_arg2f_140"], + "template": "dual_ur_robot.json", + "robot_family": "ur5", + "tabletop_clearance": 0.05, + "arm_component_z": 0.4, + "arm_base_x": -1.45, + "max_effort": 10000.0, + "gripper_open_state": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + "gripper_close_state": [0.7, -0.7, 0.7, -0.7, -0.7, 0.7] + }, + "dual_ur10": { + "aliases": ["ur10", "dual_ur10_dh_pgi", "dual_ur10_robotiq", "dual_ur10_robotiq_arg2f_140"], + "template": "dual_ur_robot.json", + "robot_family": "ur10", + "tabletop_clearance": 0.05, + "arm_component_z": 0.3, + "arm_base_x": -1.1, + "max_effort": 330.0, + "gripper_open_state": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + "gripper_close_state": [0.7, -0.7, 0.7, -0.7, -0.7, 0.7] + } +} diff --git a/embodichain/gen_sim/action_engine/tasks/__init__.py b/embodichain/gen_sim/action_engine/tasks/__init__.py new file mode 100644 index 000000000..536b3eca2 --- /dev/null +++ b/embodichain/gen_sim/action_engine/tasks/__init__.py @@ -0,0 +1,48 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Task-first generation and scene hand-off for Action Engine v2.""" + +from __future__ import annotations + +from .assembly import GroundedTaskSpec +from .interpretation import ( + GroundingCaller, + INSTRUCTION_INTENT_SCHEMA, + InstructionDraftResult, + InstructionCaller, + InstructionIntent, + ground_instruction_draft, + interpret_instruction_draft, + interpret_and_ground_task_spec, + validate_instruction_intent, +) +from .scene import SceneHandoff, validate_scene_handoff + +__all__ = [ + "GroundedTaskSpec", + "GroundingCaller", + "INSTRUCTION_INTENT_SCHEMA", + "InstructionDraftResult", + "InstructionCaller", + "InstructionIntent", + "SceneHandoff", + "ground_instruction_draft", + "interpret_instruction_draft", + "interpret_and_ground_task_spec", + "validate_instruction_intent", + "validate_scene_handoff", +] diff --git a/embodichain/gen_sim/action_engine/tasks/assembly.py b/embodichain/gen_sim/action_engine/tasks/assembly.py new file mode 100644 index 000000000..7fdfe7246 --- /dev/null +++ b/embodichain/gen_sim/action_engine/tasks/assembly.py @@ -0,0 +1,425 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Language-neutral scene inventory and grounded TaskSpec assembly.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from copy import deepcopy +from dataclasses import dataclass, field +from typing import Any + +from embodichain.gen_sim.action_engine.domain import ( + TASK_CONTRACTS, + task_contract, + task_success_type, + validate_scene_requirements, + validate_task_spec, +) +from embodichain.gen_sim.action_engine.generation.config_builder import ( + canonical_robot_profile, +) +from embodichain.gen_sim.action_engine.protocol import ( + SCENE_REQUIREMENTS_SCHEMA, + TASK_SPEC_SCHEMA, +) + +__all__ = [ + "GroundedTaskBuilder", + "GroundedTaskSpec", + "SceneEntity", + "SceneInventory", + "validate_source_compatibility", + "validate_target_compatibility", +] + + +@dataclass(frozen=True) +class GroundedTaskSpec: + """One explicit TaskSpec plus verified scene role bindings.""" + + task_spec: dict[str, Any] + scene_requirements: dict[str, Any] + role_bindings: dict[str, str] + + +@dataclass(frozen=True) +class SceneEntity: + """One scene entity with source semantics preserved verbatim.""" + + uid: str + role: str + name: str + description: str + category: str + color: str | None + position: tuple[float, float, float] + affordances: frozenset[str] = frozenset() + initial_state: Mapping[str, Any] = field(default_factory=dict) + attributes: Mapping[str, Any] = field(default_factory=dict) + source_uid: str = "" + + +class SceneInventory: + """Structural scene index without natural-language matching rules.""" + + _PASSIVE_ROLES = frozenset( + { + "background", + "camera", + "light", + "robot", + "sensor", + "support_surface", + "table", + } + ) + + def __init__( + self, + scene_objects: Sequence[Mapping[str, Any]], + *, + robot_profile: str, + ) -> None: + self.profile = canonical_robot_profile(robot_profile) + self.entities = tuple(_scene_entity(item) for item in scene_objects) + self.by_uid = {entity.uid: entity for entity in self.entities} + if len(self.by_uid) != len(self.entities): + raise ValueError("Scene inventory contains duplicate runtime UIDs.") + self.support = tuple( + entity + for entity in self.entities + if entity.uid == "table" or entity.role in {"table", "support_surface"} + ) + self.passive = tuple( + entity + for entity in self.entities + if entity in self.support or entity.role in self._PASSIVE_ROLES + ) + self.interactive = tuple( + entity for entity in self.entities if entity not in self.passive + ) + if not self.interactive: + raise ValueError("Task planning requires at least one interaction object.") + + @property + def movable(self) -> tuple[SceneEntity, ...]: + """Compatibility alias for callers that mean source candidates.""" + return self.interactive + + def left_score(self, entity: SceneEntity) -> float: + """Return robot-relative lateral score; positive values are left. + + Generated dual-arm profiles share one final world layout: the semantic + left arm is on world ``-Y`` after all robot-level transforms. + """ + return -entity.position[1] + + +class GroundedTaskBuilder: + """Assemble grounded E1-E9 instances without parsing instruction text.""" + + def __init__( + self, + task_id: str, + instruction: str, + inventory: SceneInventory, + *, + planner: str = "structured_llm_v2", + ) -> None: + self.task_id = task_id + self.instruction = instruction + self.inventory = inventory + self.planner = planner + self.instances: list[dict[str, Any]] = [] + self.role_by_uid: dict[str, str] = {} + self.requirements: dict[str, dict[str, Any]] = {} + self.previous_object_uid: str | None = None + self.previous_arm: str | None = None + self.last_task_by_object_uid: dict[str, tuple[str, str]] = {} + + def add( + self, + task_type: str, + object_entity: SceneEntity, + *, + target: SceneEntity | None = None, + params: Mapping[str, Any] | None = None, + depends_on: Sequence[str] | None = None, + ) -> str: + values = deepcopy(dict(params or {})) + relation = str(values.get("relation", "none")) + validate_source_compatibility(task_type, (object_entity,)) + validate_target_compatibility(task_type, target, relation=relation) + + instance_id = f"task_{len(self.instances) + 1:02d}" + object_role = self._role( + object_entity, + required_affordances=task_contract(task_type).required_affordances, + initial_state={"orientation": "fallen"} if task_type == "E2" else {}, + ) + values = {"object_role": object_role, **values} + if task_type == "E3": + values["source_role"] = values.pop("object_role") + if target is not None: + values["target_role"] = self._role( + target, + required_affordances=_target_affordances(task_type, relation), + ) + if depends_on is None: + dependencies = [self.instances[-1]["id"]] if self.instances else [] + else: + dependencies = list(depends_on) + previous_for_object = self.last_task_by_object_uid.get(object_entity.uid) + if ( + task_type == "E4" + and previous_for_object is not None + and previous_for_object[1] == "E2" + and previous_for_object[0] not in dependencies + ): + dependencies.append(previous_for_object[0]) + self.instances.append( + { + "id": instance_id, + "task_type": task_type, + "params": values, + "depends_on": dependencies, + "role": "primary", + } + ) + self.last_task_by_object_uid[object_entity.uid] = (instance_id, task_type) + self.previous_object_uid = object_entity.uid + if task_type == "E4": + receive_arm = str(values.get("receive_arm", "")) + self.previous_arm = ( + receive_arm if receive_arm in {"left_arm", "right_arm"} else None + ) + elif str(values.get("required_arm", "")) in {"left_arm", "right_arm"}: + self.previous_arm = str(values["required_arm"]) + return instance_id + + def build(self) -> GroundedTaskSpec: + types = {item["task_type"] for item in self.instances} + if len(self.instances) == 1: + level = "L1" + elif len(types) == 1: + level = "L2" + else: + level = "L3" + success_terms = [ + { + "type": task_success_type(item["task_type"], item.get("params")), + "task_instance_id": item["id"], + } + for item in self.instances + ] + task = validate_task_spec( + { + "schema_version": TASK_SPEC_SCHEMA, + "task_id": self.task_id, + "level": level, + "instruction": self.instruction, + "reasoning_type": "none", + "task_instances": self.instances, + "success": {"op": "all", "terms": success_terms}, + "oracle": { + "task_order": [item["id"] for item in self.instances], + "role_bindings": dict(sorted(self.role_bindings().items())), + }, + "metadata": {"planner": self.planner}, + } + ) + requirements = validate_scene_requirements( + { + "schema_version": SCENE_REQUIREMENTS_SCHEMA, + "task_id": self.task_id, + "objects": list(self.requirements.values()), + "cameras": [], + "spatial_constraints": [{"type": "preserve_source_scene"}], + "distractor_count": max( + 0, + len(self.inventory.interactive) - len(self.role_by_uid), + ), + "metadata": {"source": "existing_gym_project"}, + } + ) + return GroundedTaskSpec(task, requirements, self.role_bindings()) + + def role_bindings(self) -> dict[str, str]: + return {role: uid for uid, role in self.role_by_uid.items()} + + def _role( + self, + entity: SceneEntity, + task_type: str | None = None, + *, + required_affordances: Sequence[str] = (), + initial_state: Mapping[str, Any] | None = None, + ) -> str: + if task_type in TASK_CONTRACTS: + required_affordances = tuple( + set(required_affordances) + | set(task_contract(str(task_type)).required_affordances) + ) + if task_type == "E2": + initial_state = {"orientation": "fallen", **dict(initial_state or {})} + existing = self.role_by_uid.get(entity.uid) + if existing is not None: + requirement = self.requirements[existing] + requirement["affordances"] = sorted( + set(requirement["affordances"]) | set(required_affordances) + ) + requirement["initial_state"].update(dict(initial_state or {})) + return existing + role = f"object_{len(self.role_by_uid) + 1:02d}" + self.role_by_uid[entity.uid] = role + attributes = deepcopy(dict(entity.attributes)) + if entity.color is not None: + attributes.setdefault("color", entity.color) + self.requirements[role] = { + "role_id": role, + "category": entity.category or entity.role, + "count": 1, + "affordances": sorted(set(required_affordances)), + "initial_state": dict(initial_state or {}), + "attributes": attributes, + } + return role + + +def validate_source_compatibility( + task_type: str, + objects: Sequence[SceneEntity], +) -> None: + """Apply structural/explicit-affordance checks without a category taxonomy.""" + contract = task_contract(task_type) + if contract.source_structure == "articulation": + invalid = [entity.uid for entity in objects if entity.role != "articulation"] + else: + invalid = [ + entity.uid + for entity in objects + if entity.role not in {"object", "rigid_object"} + ] + if invalid: + structure_label = ( + "articulation" + if contract.source_structure == "articulation" + else "movable rigid-object" + ) + raise ValueError( + f"{task_type} requires {structure_label} structure; " + f"incompatible scene objects are {invalid}." + ) + required = set(contract.required_affordances) + for entity in objects: + if entity.affordances: + missing = required - set(entity.affordances) + if missing: + raise ValueError( + f"{task_type} is incompatible with scene object {entity.uid!r}; " + f"missing affordances {sorted(missing)}." + ) + + +def validate_target_compatibility( + task_type: str, + target: SceneEntity | None, + *, + relation: str, +) -> None: + """Reject only structural or explicitly declared target contradictions.""" + if task_type == "E1" and relation == "on" and target is not None: + # Support is a relation between two concrete bodies at a candidate + # pose. A positive affordance list is not a closed-world inventory, so + # omission of ``support_surface`` cannot prove incompatibility here. + return + requires_container = task_type == "E3" or ( + task_type == "E1" and relation == "inside" + ) + if requires_container and target is None: + raise ValueError( + f"{task_type} {relation} relation requires a target container." + ) + if not requires_container or target is None: + return + if target.role in SceneInventory._PASSIVE_ROLES: + raise ValueError( + f"{task_type} target {target.uid!r} is structurally incompatible " + "with containment." + ) + if target.affordances: + compatible = {"container", "fillable", "liquid_container", "receptacle"} + if set(target.affordances).isdisjoint(compatible): + raise ValueError( + f"{task_type} target {target.uid!r} has explicit affordances but " + f"none support containment; expected one of {sorted(compatible)}." + ) + + +def _target_affordances(task_type: str, relation: str) -> tuple[str, ...]: + if task_type == "E3" or (task_type == "E1" and relation == "inside"): + return ("container",) + return () + + +def _scene_entity(raw: Mapping[str, Any]) -> SceneEntity: + uid = str(raw.get("runtime_uid", raw.get("uid", ""))).strip() + if not uid: + raise ValueError("Every scene object requires a runtime UID.") + role = str(raw.get("role", raw.get("source_role", "object"))).strip().lower() + raw_category = raw.get("category", raw.get("object_category", "")) + category = "" if raw_category is None else str(raw_category).strip() + raw_color = raw.get("color") + attributes = raw.get("attributes", {}) + if not isinstance(attributes, Mapping): + raise ValueError(f"Scene object {uid!r} attributes must be a mapping.") + if raw_color is None: + raw_color = attributes.get("color") + color = str(raw_color).strip() if raw_color not in (None, "") else None + position = raw.get("init_pos", raw.get("position", (0.0, 0.0, 0.0))) + if ( + not isinstance(position, Sequence) + or isinstance(position, (str, bytes)) + or len(position) != 3 + ): + raise ValueError(f"Scene object {uid!r} requires a three-value init_pos.") + raw_affordances = raw.get("affordances", raw.get("capabilities", ())) + affordances = ( + frozenset( + str(item).strip().lower() for item in raw_affordances if str(item).strip() + ) + if isinstance(raw_affordances, Sequence) + and not isinstance(raw_affordances, (str, bytes)) + else frozenset() + ) + initial_state = raw.get("initial_state", raw.get("state", {})) + if not isinstance(initial_state, Mapping): + raise ValueError(f"Scene object {uid!r} initial_state must be a mapping.") + return SceneEntity( + uid=uid, + role=role, + name=str(raw.get("name", "")).strip(), + description=str(raw.get("description", "")).strip(), + category=category, + color=color, + position=tuple(float(value) for value in position), + affordances=affordances, + initial_state=dict(initial_state), + attributes=dict(attributes), + source_uid=str(raw.get("source_uid", "")).strip(), + ) diff --git a/embodichain/gen_sim/action_engine/tasks/grounding.py b/embodichain/gen_sim/action_engine/tasks/grounding.py new file mode 100644 index 000000000..de412e987 --- /dev/null +++ b/embodichain/gen_sim/action_engine/tasks/grounding.py @@ -0,0 +1,513 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Task-conditioned scene-UID grounding for structured instruction intents.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from copy import deepcopy +from dataclasses import dataclass +import json +import math +from time import perf_counter +from typing import Any + +from .assembly import SceneInventory + +__all__ = ["GroundingCaller", "GroundingResult", "ground_scene_references"] + +GroundingCaller = Callable[..., Mapping[str, Any]] + +_BINDING_KEYS = frozenset({"reference_id", "status", "uids", "confidence"}) +_QUANTIFIERS = frozenset({"one", "all", "count"}) +_REDACTED_KEYS = frozenset( + { + "absolute_position", + "bbox", + "bboxes", + "bounding_box", + "camera_matrix", + "center", + "centroid", + "coordinates", + "depth", + "dimensions", + "extrinsics", + "grasp_pose", + "init_local_pose", + "init_pos", + "init_rot", + "intrinsics", + "joint_positions", + "joints", + "keypoint", + "keypoints", + "location", + "matrix", + "pose", + "position", + "position_xyz", + "qpos", + "quaternion", + "rotation", + "scale", + "target_pose", + "trajectory", + "transform", + "translation", + "waypoints", + "world_x", + "world_y", + "world_z", + "x", + "y", + "z", + } +) + +_GROUNDING_SCHEMA: dict[str, Any] = { + "title": "ActionEngineSceneGrounding", + "type": "object", + "additionalProperties": False, + "required": ["bindings"], + "properties": { + "bindings": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "required": sorted(_BINDING_KEYS), + "properties": { + "reference_id": {"type": "string"}, + "status": { + "type": "string", + "enum": ["resolved", "ambiguous", "not_found"], + }, + "uids": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": True, + }, + "confidence": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + }, + }, + }, + } + }, +} + + +@dataclass(frozen=True) +class GroundingResult: + """Validated scene bindings and aggregate call statistics. + + Attributes: + bindings: Mapping from ``.`` to scene UIDs. + attempts: Number of grounding-model calls, including one repair call. + latency_seconds: Total elapsed wall-clock time across the grounding stage. + """ + + bindings: dict[str, tuple[str, ...]] + attempts: int + latency_seconds: float + + +def ground_scene_references( + instruction: str, + intent: Mapping[str, Any], + inventory: SceneInventory, + scene_objects: Sequence[Mapping[str, Any]], + model: str | None, + caller: GroundingCaller, +) -> GroundingResult: + """Resolve every ``scene_ref`` selector in one task-conditioned batch. + + The grounding model can only select stable UIDs from a redacted inventory. + Its output does not add affordances, physical state, coordinates, or poses. + One failed local validation is repaired with one additional model call. + + Args: + instruction: Original user instruction for task-level context. + intent: Validated structured instruction intent. + inventory: Structural scene inventory defining authoritative candidates. + scene_objects: Original semantic inventory used to retain open labels. + model: Model name forwarded unchanged to the injected caller. + caller: Structured model transport accepting ``prompt``, ``schema``, and + ``model`` keyword arguments. + + Returns: + Validated UID bindings together with call-count and latency statistics. + + Raises: + TypeError: If the intent or response has an invalid container type. + ValueError: If requests are malformed or grounding remains invalid after + one repair attempt. + """ + if not isinstance(instruction, str) or not instruction.strip(): + raise ValueError("Grounding instruction must be a non-empty string.") + if not callable(caller): + raise TypeError("Grounding caller must be callable.") + + requests = _collect_requests(intent) + prompt_inventory = _grounding_inventory(inventory, scene_objects) + prompt = _grounding_prompt(instruction.strip(), requests, prompt_inventory) + started = perf_counter() + first_error: Exception | None = None + + for attempt in range(2): + current_prompt = prompt + if first_error is not None: + current_prompt += ( + "\n\nREPAIR OVERRIDE: the previous grounding JSON failed local " + "validation. Return one corrected JSON object only. Preserve the " + "exact output fields bindings/reference_id/status/uids/confidence, " + "cover every requested reference exactly once, and select only " + "UIDs from the supplied candidate inventory. Validation error: " + f"{first_error}" + ) + try: + response = caller( + prompt=current_prompt, + schema=deepcopy(_GROUNDING_SCHEMA), + model=model, + ) + bindings = _validate_response( + response, + requests=requests, + inventory=inventory, + ) + return GroundingResult( + bindings=bindings, + attempts=attempt + 1, + latency_seconds=perf_counter() - started, + ) + except (TypeError, ValueError) as error: + if attempt: + raise ValueError( + "Scene grounding failed validation after one repair: " f"{error}" + ) from error + first_error = error + raise AssertionError("unreachable") + + +def _collect_requests(intent: Mapping[str, Any]) -> list[dict[str, Any]]: + if not isinstance(intent, Mapping): + raise TypeError("Instruction intent must be a mapping.") + steps = intent.get("steps") + if not isinstance(steps, Sequence) or isinstance(steps, (str, bytes)): + raise ValueError("Instruction intent steps must be a list.") + + requests: list[dict[str, Any]] = [] + request_ids: set[str] = set() + for step_index, step in enumerate(steps): + context = f"InstructionIntent.steps[{step_index}]" + if not isinstance(step, Mapping): + raise ValueError(f"{context} must be a mapping.") + step_id = step.get("id") + if not isinstance(step_id, str) or not step_id.strip(): + raise ValueError(f"{context}.id must be a non-empty string.") + task_type = step.get("task_type") + if not isinstance(task_type, str) or not task_type.strip(): + raise ValueError(f"{context}.task_type must be a non-empty string.") + relation = step.get("relation", "none") + if not isinstance(relation, str): + raise ValueError(f"{context}.relation must be a string.") + + for slot in ("object", "target"): + selector = step.get(slot) + if not isinstance(selector, Mapping): + raise ValueError(f"{context}.{slot} must be a mapping.") + if selector.get("kind") != "scene_ref": + continue + reference = selector.get("reference") + if not isinstance(reference, str) or not reference.strip(): + raise ValueError( + f"{context}.{slot}.reference must be a non-empty string." + ) + quantifier = selector.get("quantifier") + if quantifier not in _QUANTIFIERS: + raise ValueError( + f"{context}.{slot}.quantifier must be one of " + f"{sorted(_QUANTIFIERS)}." + ) + count = selector.get("count") + if not isinstance(count, int) or isinstance(count, bool) or count < 0: + raise ValueError(f"{context}.{slot}.count must be an integer >= 0.") + if quantifier == "count" and count < 1: + raise ValueError( + f"{context}.{slot} quantifier=count requires count>=1." + ) + if quantifier != "count" and count != 0: + raise ValueError( + f"{context}.{slot} quantifier={quantifier} requires count=0." + ) + + request_id = f"{step_id}.{slot}" + if request_id in request_ids: + raise ValueError(f"Duplicate grounding request ID {request_id!r}.") + request_ids.add(request_id) + requests.append( + { + "reference_id": request_id, + "step_id": step_id, + "slot": slot, + "task_type": task_type, + "relation": relation, + "reference": reference.strip(), + "quantifier": quantifier, + "count": count, + } + ) + return requests + + +def _grounding_inventory( + inventory: SceneInventory, + scene_objects: Sequence[Mapping[str, Any]], +) -> list[dict[str, Any]]: + raw_by_uid: dict[str, Mapping[str, Any]] = {} + for item_index, raw in enumerate(scene_objects): + if not isinstance(raw, Mapping): + raise ValueError(f"Scene inventory item {item_index} must be a mapping.") + uid = str(raw.get("runtime_uid", raw.get("uid", ""))).strip() + if uid: + raw_by_uid[uid] = raw + + ranked = sorted( + inventory.entities, + key=lambda entity: (-inventory.left_score(entity), entity.uid), + ) + rank_by_uid = {entity.uid: rank for rank, entity in enumerate(ranked, start=1)} + payload = [] + for entity in sorted(inventory.entities, key=lambda item: item.uid): + raw = raw_by_uid.get(entity.uid, {}) + score = inventory.left_score(entity) + side = "left" if score > 0.0 else "right" if score < 0.0 else "center" + raw_category = raw.get( + "category", + raw.get("object_category", entity.category), + ) + attributes = _redact_semantic_mapping(entity.attributes) + if entity.color is not None: + attributes.setdefault("color", entity.color) + payload.append( + { + "uid": entity.uid, + "role": entity.role, + "name": str(raw.get("name", entity.name)).strip(), + "category": str(raw_category).strip() or entity.category, + "description": entity.description, + "affordances": sorted(entity.affordances), + "attributes": attributes, + "initial_state": _redact_semantic_mapping(entity.initial_state), + "side": side, + "rank": rank_by_uid[entity.uid], + } + ) + return payload + + +def _grounding_prompt( + instruction: str, + requests: Sequence[Mapping[str, Any]], + inventory: Sequence[Mapping[str, Any]], +) -> str: + return ( + "Ground the requested natural-language scene references to the supplied " + "scene inventory. Resolve all requests together using the original task, " + "step type, relation, quantifier, and reference text as context. Select " + "only exact inventory UIDs. The inventory's affordances and states are " + "source evidence only: never infer, add, authorize, or return an " + "affordance, capability, physical state, coordinate, pose, orientation, " + "path, or action. The side and rank fields are discrete robot-relative " + "labels; rank 1 is leftmost. Object requests may select only movable " + "inventory entities. Target requests may also select support surfaces. " + "Use status=ambiguous or status=not_found instead of guessing when the " + "evidence is insufficient. Return exactly one binding per reference_id " + "with only reference_id, status, uids, and confidence.\n\n" + f"Instruction:\n{instruction}\n\n" + "Grounding requests:\n" + f"{json.dumps(list(requests), ensure_ascii=False, sort_keys=True)}\n\n" + "Redacted scene inventory:\n" + f"{json.dumps(list(inventory), ensure_ascii=False, sort_keys=True)}" + ) + + +def _validate_response( + value: Mapping[str, Any], + *, + requests: Sequence[Mapping[str, Any]], + inventory: SceneInventory, +) -> dict[str, tuple[str, ...]]: + if not isinstance(value, Mapping): + raise TypeError("Scene grounding output must be a mapping.") + if set(value) != {"bindings"}: + raise ValueError( + "Scene grounding output must contain exactly the 'bindings' field." + ) + raw_bindings = value["bindings"] + if not isinstance(raw_bindings, Sequence) or isinstance(raw_bindings, (str, bytes)): + raise ValueError("Scene grounding bindings must be a list.") + + request_by_id = {str(request["reference_id"]): request for request in requests} + bindings: dict[str, tuple[str, ...]] = {} + for binding_index, raw in enumerate(raw_bindings): + context = f"SceneGrounding.bindings[{binding_index}]" + if not isinstance(raw, Mapping): + raise ValueError(f"{context} must be a mapping.") + if set(raw) != _BINDING_KEYS: + missing = sorted(_BINDING_KEYS - set(raw)) + extra = sorted(set(raw) - _BINDING_KEYS) + raise ValueError( + f"{context} fields must be exactly {sorted(_BINDING_KEYS)}; " + f"missing={missing}, unsupported={extra}." + ) + reference_id = raw["reference_id"] + if not isinstance(reference_id, str) or not reference_id: + raise ValueError(f"{context}.reference_id must be a non-empty string.") + if reference_id not in request_by_id: + raise ValueError(f"{context} references unknown request {reference_id!r}.") + if reference_id in bindings: + raise ValueError(f"Duplicate grounding binding for {reference_id!r}.") + + status = raw["status"] + if status not in {"resolved", "ambiguous", "not_found"}: + raise ValueError( + f"{context}.status must be resolved, ambiguous, or not_found." + ) + if status != "resolved": + raise ValueError( + f"Grounding request {reference_id!r} was not resolved: {status}." + ) + confidence = raw["confidence"] + if ( + not isinstance(confidence, (int, float)) + or isinstance(confidence, bool) + or not math.isfinite(float(confidence)) + or not 0.0 <= float(confidence) <= 1.0 + ): + raise ValueError(f"{context}.confidence must be a number in [0, 1].") + if float(confidence) < 0.5: + raise ValueError( + f"Grounding request {reference_id!r} confidence is below 0.5." + ) + + raw_uids = raw["uids"] + if not isinstance(raw_uids, Sequence) or isinstance(raw_uids, (str, bytes)): + raise ValueError(f"{context}.uids must be a list.") + uids = tuple(raw_uids) + if any(not isinstance(uid, str) or not uid for uid in uids): + raise ValueError(f"{context}.uids must contain non-empty strings.") + if len(set(uids)) != len(uids): + raise ValueError( + f"Grounding request {reference_id!r} contains duplicate UIDs." + ) + unknown = sorted(set(uids) - set(inventory.by_uid)) + if unknown: + raise ValueError( + f"Grounding request {reference_id!r} selected unknown UIDs {unknown}." + ) + + request = request_by_id[reference_id] + allowed = ( + {entity.uid for entity in inventory.interactive} + if request["slot"] == "object" + else {entity.uid for entity in (*inventory.interactive, *inventory.support)} + ) + disallowed = sorted(set(uids) - allowed) + if disallowed: + raise ValueError( + f"Grounding request {reference_id!r} selected UIDs outside its " + f"{request['slot']} candidate range: {disallowed}." + ) + _validate_cardinality(request, uids) + bindings[reference_id] = uids + + missing = sorted(set(request_by_id) - set(bindings)) + if missing: + raise ValueError(f"Scene grounding omitted requests {missing}.") + _reject_self_references(requests, bindings) + return bindings + + +def _validate_cardinality( + request: Mapping[str, Any], + uids: Sequence[str], +) -> None: + request_id = str(request["reference_id"]) + quantifier = str(request["quantifier"]) + if quantifier == "one" and len(uids) != 1: + raise ValueError( + f"Grounding request {request_id!r} quantifier=one requires exactly one UID." + ) + if quantifier == "count" and len(uids) != int(request["count"]): + raise ValueError( + f"Grounding request {request_id!r} requires exactly " + f"{request['count']} UIDs." + ) + if quantifier == "all" and not uids: + raise ValueError( + f"Grounding request {request_id!r} quantifier=all requires at " + "least one UID." + ) + + +def _reject_self_references( + requests: Sequence[Mapping[str, Any]], + bindings: Mapping[str, tuple[str, ...]], +) -> None: + slots_by_step: dict[str, dict[str, str]] = {} + for request in requests: + slots_by_step.setdefault(str(request["step_id"]), {})[str(request["slot"])] = ( + str(request["reference_id"]) + ) + for step_id, slots in slots_by_step.items(): + object_id = slots.get("object") + target_id = slots.get("target") + if object_id is None or target_id is None: + continue + overlap = sorted(set(bindings[object_id]) & set(bindings[target_id])) + if overlap: + raise ValueError( + f"Grounding step {step_id!r} uses the same UID as object and " + f"target: {overlap}." + ) + + +def _redact_semantic_mapping(value: Mapping[str, Any]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, child in value.items(): + name = str(key) + normalized = name.strip().lower().replace("-", "_") + if normalized in _REDACTED_KEYS: + continue + if isinstance(child, Mapping): + nested = _redact_semantic_mapping(child) + if nested: + result[name] = nested + elif isinstance(child, (str, int, float, bool)) and not isinstance( + child, complex + ): + result[name] = child + elif isinstance(child, Sequence) and not isinstance(child, (str, bytes)): + semantic_values = [item for item in child if isinstance(item, (str, bool))] + if semantic_values and len(semantic_values) == len(child): + result[name] = semantic_values + return result diff --git a/embodichain/gen_sim/action_engine/tasks/interpretation.py b/embodichain/gen_sim/action_engine/tasks/interpretation.py new file mode 100644 index 000000000..f84f6d2bd --- /dev/null +++ b/embodichain/gen_sim/action_engine/tasks/interpretation.py @@ -0,0 +1,404 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Compatibility bridge from Task Engine drafts to Action Engine TaskSpec v2.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +from embodichain.gen_sim.task_engine.interpretation import ( + INSTRUCTION_INTENT_SCHEMA, + InstructionCaller, + InstructionDraftResult, + InstructionIntent, + _default_instruction_caller, + _instruction_prompt, + _instruction_selector_rules, + interpret_instruction_draft, + validate_instruction_intent, +) + +from .assembly import ( + GroundedTaskBuilder, + GroundedTaskSpec, + SceneEntity, + SceneInventory, + validate_source_compatibility, + validate_target_compatibility, +) +from .grounding import GroundingCaller, ground_scene_references + +__all__ = [ + "GroundingCaller", + "INSTRUCTION_INTENT_SCHEMA", + "InstructionCaller", + "InstructionDraftResult", + "InstructionIntent", + "ground_instruction_draft", + "interpret_and_ground_task_spec", + "interpret_instruction_draft", + "validate_instruction_intent", +] + + +def interpret_and_ground_task_spec( + task_name: str, + task_description: str, + scene_objects: Sequence[Mapping[str, Any]], + *, + robot_profile: str, + model: str | None = None, + caller: InstructionCaller | None = None, + grounding_caller: GroundingCaller | None = None, +) -> GroundedTaskSpec: + """Interpret through Task Engine, then ground through Action Engine.""" + task_id = str(task_name).strip() + instruction = str(task_description).strip() + if not task_id or not instruction: + raise ValueError("task_name and task_description must be non-empty.") + inventory = SceneInventory(scene_objects, robot_profile=robot_profile) + draft = interpret_instruction_draft(instruction, model=model, caller=caller) + invoke = caller or _default_instruction_caller + selected_model = None if draft.model == "injected_caller" else draft.model + grounding = ground_scene_references( + instruction=instruction, + intent=draft.intent, + inventory=inventory, + scene_objects=scene_objects, + model=selected_model, + caller=grounding_caller or invoke, + ) + grounded = _ground_intent( + task_id, + instruction, + draft.intent, + inventory, + grounding.bindings, + ) + grounded.task_spec["metadata"].update( + { + "instruction_interpreter": "structured_llm_v2", + "instruction_model": draft.model, + "instruction_call_count": draft.attempts, + "instruction_latency_seconds": draft.latency_seconds, + "scene_grounding_model": selected_model or "injected_caller", + "scene_grounding_call_count": grounding.attempts, + "scene_grounding_latency_seconds": grounding.latency_seconds, + } + ) + if draft.normalizations: + grounded.task_spec["metadata"]["instruction_intent_normalizations"] = list( + draft.normalizations + ) + return grounded + + +def ground_instruction_draft( + task_id: str, + instruction: str, + intent: Mapping[str, Any], + scene_objects: Sequence[Mapping[str, Any]], + *, + robot_profile: str, + reference_bindings: Mapping[str, Sequence[str]], +) -> GroundedTaskSpec: + """Lower a Task Engine draft using verified scene bindings.""" + normalized_task_id = str(task_id).strip() + normalized_instruction = str(instruction).strip() + if not normalized_task_id or not normalized_instruction: + raise ValueError("task_id and instruction must be non-empty.") + inventory = SceneInventory(scene_objects, robot_profile=robot_profile) + return _ground_intent( + normalized_task_id, + normalized_instruction, + validate_instruction_intent(intent), + inventory, + reference_bindings, + ) + + +def _ground_intent( + task_id: str, + instruction: str, + intent: Mapping[str, Any], + inventory: SceneInventory, + scene_bindings: Mapping[str, Sequence[str]], +) -> GroundedTaskSpec: + builder = GroundedTaskBuilder( + task_id, + instruction, + inventory, + planner="structured_llm_v2", + ) + objects_by_step: dict[str, list[SceneEntity]] = {} + task_ids_by_step: dict[str, list[str]] = {} + for step in _topological_steps(intent["steps"]): + step_id = str(step["id"]) + objects = _resolve_reference( + step["object"], + inventory, + objects_by_step, + context=f"instruction step {step_id!r} object", + reference_id=f"{step_id}.object", + scene_bindings=scene_bindings, + ) + validate_source_compatibility(str(step["task_type"]), objects) + target_objects = _resolve_reference( + step["target"], + inventory, + objects_by_step, + context=f"instruction step {step_id!r} target", + reference_id=f"{step_id}.target", + scene_bindings=scene_bindings, + allow_none=True, + exclude={item.uid for item in objects}, + allow_support=True, + ) + if len(target_objects) > 1: + raise ValueError(f"Instruction step {step_id!r} target is ambiguous.") + validate_target_compatibility( + str(step["task_type"]), + target_objects[0] if target_objects else None, + relation=str(step["relation"]), + ) + dependencies_by_step = list(step["depends_on"]) + for selector in (step["object"], step["target"]): + if selector["kind"] == "step_result": + reference = str(selector["step_id"]) + if reference not in dependencies_by_step: + dependencies_by_step.append(reference) + dependencies = [ + emitted_id + for dependency in dependencies_by_step + for emitted_id in task_ids_by_step[str(dependency)] + ] + emitted = _emit_step( + builder, + step, + objects, + target_objects[0] if target_objects else None, + dependencies, + ) + objects_by_step[step_id] = objects + task_ids_by_step[step_id] = emitted + return builder.build() + + +def _emit_step( + builder: GroundedTaskBuilder, + step: Mapping[str, Any], + objects: Sequence[SceneEntity], + target: SceneEntity | None, + dependencies: Sequence[str], +) -> list[str]: + task_type = str(step["task_type"]) + if step["layout"] == "line": + roles = [builder._role(entity, "E1") for entity in objects] + parent = str(step["id"]) + return [ + builder.add( + "E1", + entity, + params={ + "target_role": "table", + "relation": "on", + "layout": "line", + "objects_roles": roles, + "axis": "world_y" if step["axis"] == "none" else step["axis"], + "order_by": "explicit", + "order_direction": "given", + "order_constraint": "free", + "orientation_goal": step["orientation_goal"], + "orientation_axis": "none", + "nominal_slot_index": slot, + "slot_constraint": "free_reassignable", + "parent_task_instance_id": parent, + }, + depends_on=dependencies, + ) + for slot, entity in enumerate(objects) + ] + + emitted = [] + for entity in objects: + params: dict[str, Any] = {} + required_arm = str(step["required_arm"]) + if required_arm in {"left_arm", "right_arm"}: + params["required_arm"] = required_arm + if task_type == "E1": + relation = str(step["relation"]) + if relation == "none": + if target is None or target not in builder.inventory.support: + raise ValueError( + "E1 omitted relation is only valid for a unique table " + "support target." + ) + relation = "on" + params.update( + { + "relation": relation, + "relation_frame": "robot", + "orientation_goal": step["orientation_goal"], + "orientation_axis": "none", + } + ) + elif task_type == "E2": + params.update( + { + "orientation_goal": "upright", + "support_role": "table", + "upright_local_axis": "long_axis", + } + ) + elif task_type == "E3": + params.update({"relation": "above", "relation_frame": "robot"}) + elif task_type == "E4": + params.update( + { + "transfer_arm": step["transfer_arm"], + "receive_arm": step["receive_arm"], + "orientation_goal": step["orientation_goal"], + } + ) + elif task_type == "E5": + params.update( + { + "direction": step["direction"], + "terminal_behavior": step["terminal_behavior"], + "relation": step["relation"], + "relation_frame": "robot", + } + ) + elif task_type in {"E6", "E7"}: + params["target_state"] = step["target_state"] + elif task_type == "E8": + params["target_setting"] = int(step["target_setting"]) + elif task_type == "E9": + params["terminal_state"] = step["target_state"] + emitted.append( + builder.add( + task_type, + entity, + target=target, + params=params, + depends_on=dependencies, + ) + ) + return emitted + + +def _resolve_reference( + selector: Mapping[str, Any], + inventory: SceneInventory, + objects_by_step: Mapping[str, Sequence[SceneEntity]], + *, + context: str, + reference_id: str, + scene_bindings: Mapping[str, Sequence[str]], + allow_none: bool = False, + exclude: set[str] | None = None, + allow_support: bool = False, +) -> list[SceneEntity]: + kind = str(selector["kind"]) + if kind == "none": + if allow_none: + return [] + raise ValueError(f"{context} is required.") + if kind == "step_result": + step_id = str(selector["step_id"]) + if step_id not in objects_by_step: + raise ValueError(f"{context} references unavailable step {step_id!r}.") + objects = list(objects_by_step[step_id]) + if len(objects) != 1: + raise ValueError( + f"{context} references step {step_id!r}, which has {len(objects)} objects." + ) + if exclude and objects[0].uid in exclude: + raise ValueError( + f"{context} references the same object as its source; " + "self-referential placement is not allowed." + ) + return objects + + if reference_id not in scene_bindings: + raise ValueError(f"{context} has no verified scene-grounding binding.") + excluded = exclude or set() + source_uids = ( + {entity.uid for entity in inventory.entities} + if allow_support + else {entity.uid for entity in inventory.interactive} + ) + resolved_uids = tuple(str(uid) for uid in scene_bindings[reference_id]) + pool = [ + inventory.by_uid[uid] + for uid in resolved_uids + if uid in source_uids and uid not in excluded + ] + pool = sorted(pool, key=lambda item: item.uid) + if not pool: + raise ValueError(f"{context} did not bind an eligible scene object.") + quantifier = str(selector["quantifier"]) + count = int(selector["count"]) + if quantifier == "one" and len(pool) != 1: + raise ValueError( + f"{context} is ambiguous; matched scene UIDs {[item.uid for item in pool]}." + ) + if quantifier == "count" and (count < 1 or len(pool) != count): + raise ValueError( + f"{context} requested exactly {count} objects but matched {len(pool)}." + ) + if quantifier == "all" and count not in {0, len(pool)}: + raise ValueError( + f"{context} quantifier=all cannot carry count={count}; use count for an exact quantity." + ) + return pool + + +def _topological_steps( + steps: Sequence[Mapping[str, Any]], +) -> list[dict[str, Any]]: + by_id = {str(step["id"]): dict(step) for step in steps} + effective_dependencies: dict[str, tuple[str, ...]] = {} + for step_id, step in by_id.items(): + deps = [str(dep) for dep in step["depends_on"]] + for selector in (step["object"], step["target"]): + if selector["kind"] == "step_result": + reference = str(selector["step_id"]) + if reference not in deps: + deps.append(reference) + effective_dependencies[step_id] = tuple(deps) + pending = set(by_id) + ordered: list[dict[str, Any]] = [] + original = [str(step["id"]) for step in steps] + while pending: + ready = [ + step_id + for step_id in original + if step_id in pending + and all(str(dep) not in pending for dep in effective_dependencies[step_id]) + ] + if not ready: + raise ValueError("Instruction intent dependencies contain a cycle.") + # Select one earliest-ready step at a time. Emitting the whole ready + # frontier lets a later independent step leapfrog an earlier step that + # becomes ready after its predecessor, changing the instruction's + # resource-order tie break without any causal reason. + step_id = ready[0] + ordered.append(by_id[step_id]) + pending.remove(step_id) + return ordered diff --git a/embodichain/gen_sim/action_engine/tasks/scene.py b/embodichain/gen_sim/action_engine/tasks/scene.py new file mode 100644 index 000000000..242b15dd4 --- /dev/null +++ b/embodichain/gen_sim/action_engine/tasks/scene.py @@ -0,0 +1,177 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Validate a Scene Engine result against task-first requirements.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +from embodichain.gen_sim.action_engine.domain import validate_scene_requirements + +__all__ = ["SceneHandoff", "validate_scene_handoff"] + + +@dataclass(frozen=True) +class SceneHandoff: + """Validated role-to-UID resolution returned by an external Scene Engine.""" + + task_id: str + role_bindings: dict[str, Any] + object_uids: tuple[str, ...] + camera_uids: tuple[str, ...] + + +def validate_scene_handoff( + requirements: Mapping[str, Any], + scene: Mapping[str, Any], + role_bindings: Mapping[str, str], +) -> SceneHandoff: + """Reject scenes that do not satisfy roles, affordances, state, or cameras.""" + required = validate_scene_requirements(requirements) + objects = scene.get("objects") + if not isinstance(objects, Sequence) or isinstance(objects, (str, bytes)): + raise ValueError("Scene hand-off requires an objects list.") + object_by_uid: dict[str, Mapping[str, Any]] = {} + for index, item in enumerate(objects): + if not isinstance(item, Mapping): + raise ValueError(f"Scene objects[{index}] must be a mapping.") + uid = item.get("uid") + if not isinstance(uid, str) or not uid: + raise ValueError(f"Scene objects[{index}] requires a UID.") + if uid in object_by_uid: + raise ValueError(f"Scene contains duplicate object UID {uid!r}.") + object_by_uid[uid] = item + + bindings = dict(role_bindings) + required_roles = {item["role_id"] for item in required["objects"]} + if set(bindings) != required_roles: + missing = sorted(required_roles - set(bindings)) + extra = sorted(set(bindings) - required_roles) + raise ValueError( + f"Scene role bindings mismatch; missing={missing}, extra={extra}." + ) + normalized_bindings: dict[str, str | tuple[str, ...]] = {} + assigned_uids: list[str] = [] + for requirement in required["objects"]: + role = requirement["role_id"] + count = int(requirement["count"]) + binding = bindings[role] + if isinstance(binding, str): + uids = [binding] + elif isinstance(binding, Sequence) and not isinstance(binding, (str, bytes)): + uids = [str(uid) for uid in binding] + else: + raise ValueError(f"Scene role {role!r} has an invalid UID binding.") + if len(uids) != count or any(not uid for uid in uids): + raise ValueError( + f"Scene role {role!r} requires exactly {count} UID binding(s)." + ) + normalized_bindings[role] = uids[0] if count == 1 else tuple(uids) + assigned_uids.extend(uids) + for uid in uids: + _validate_bound_object(object_by_uid, uid, role, requirement) + if len(assigned_uids) != len(set(assigned_uids)): + raise ValueError("Each scene requirement role must resolve to unique UIDs.") + + cameras = scene.get("cameras", []) + if not isinstance(cameras, Sequence) or isinstance(cameras, (str, bytes)): + raise ValueError("Scene cameras must be a list.") + camera_uids = [] + normalized_cameras = [] + for camera in cameras: + if not isinstance(camera, Mapping) or not isinstance(camera.get("uid"), str): + raise ValueError("Every scene camera requires a UID.") + camera_uids.append(str(camera["uid"])) + normalized_cameras.append(camera) + for camera_requirement in required["cameras"]: + modalities = set(camera_requirement.get("modalities", ())) + coverage = camera_requirement.get("coverage") + if not any( + modalities <= set(camera.get("modalities", ())) + and (coverage is None or camera.get("coverage") == coverage) + for camera in normalized_cameras + ): + raise ValueError( + "Scene cameras do not satisfy requirement " + f"{dict(camera_requirement)!r}." + ) + reported_constraints = scene.get("satisfied_spatial_constraints", []) + if not isinstance(reported_constraints, Sequence) or isinstance( + reported_constraints, (str, bytes) + ): + raise ValueError("Scene satisfied_spatial_constraints must be a list.") + reported = {_canonical(item) for item in reported_constraints} + missing_constraints = [ + constraint + for constraint in required["spatial_constraints"] + if _canonical(constraint) not in reported + ] + if missing_constraints: + raise ValueError( + "Scene does not satisfy spatial constraints: " f"{missing_constraints}." + ) + return SceneHandoff( + task_id=required["task_id"], + role_bindings=normalized_bindings, + object_uids=tuple(sorted(object_by_uid)), + camera_uids=tuple(sorted(camera_uids)), + ) + + +def _validate_bound_object( + object_by_uid: Mapping[str, Mapping[str, Any]], + uid: str, + role: str, + requirement: Mapping[str, Any], +) -> None: + if uid not in object_by_uid: + raise ValueError(f"Scene role {role!r} references unknown UID {uid!r}.") + actual = object_by_uid[uid] + if actual.get("category") != requirement["category"]: + raise ValueError( + f"Scene object {uid!r} category does not satisfy role {role!r}." + ) + missing_affordances = set(requirement["affordances"]) - set( + actual.get("affordances", ()) + ) + if missing_affordances: + raise ValueError( + f"Scene object {uid!r} lacks affordances {sorted(missing_affordances)}." + ) + for field in ("initial_state", "attributes"): + actual_values = actual.get(field, {}) + if not isinstance(actual_values, Mapping): + raise ValueError(f"Scene object {uid!r} {field} must be a mapping.") + mismatched = { + key: expected + for key, expected in requirement[field].items() + if actual_values.get(key) != expected + } + if mismatched: + raise ValueError( + f"Scene object {uid!r} does not satisfy {field} {mismatched}." + ) + + +def _canonical(value: Any) -> str: + import json + + if not isinstance(value, Mapping): + raise ValueError("Every satisfied spatial constraint must be a mapping.") + return json.dumps(dict(value), sort_keys=True, separators=(",", ":")) diff --git a/embodichain/gen_sim/task_engine/__init__.py b/embodichain/gen_sim/task_engine/__init__.py new file mode 100644 index 000000000..669fa2216 --- /dev/null +++ b/embodichain/gen_sim/task_engine/__init__.py @@ -0,0 +1,95 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Scene-independent task interpretation and protocol ownership.""" + +from __future__ import annotations + +from .agent import ( + TaskAgent, + TaskGenerationError, + derive_scene_request, + derive_success_spec, +) +from .contracts import ( + SCENE_REQUEST_SCHEMA, + SUCCESS_SPEC_SCHEMA, + TASK_CANDIDATE_SET_SCHEMA, + TASK_DRAFT_SCHEMA, + SceneRequest, + SuccessSpec, + TaskCandidate, + TaskCandidateSet, + TaskDraft, + canonical_hash, + validate_scene_request, + validate_success_spec, + validate_task_candidate, + validate_task_candidate_set, + validate_task_draft, +) +from .interpretation import ( + INSTRUCTION_INTENT_SCHEMA, + InstructionCaller, + InstructionDraftResult, + InstructionIntent, + interpret_instruction_draft, + validate_instruction_intent, +) +from .ontology import ( + RELATIONS, + TASK_CONTRACTS, + TERMINAL_BEHAVIORS, + TRANSPORT_DIRECTIONS, + TaskContract, + task_contract, + task_success_type, +) + +__all__ = [ + "INSTRUCTION_INTENT_SCHEMA", + "InstructionCaller", + "InstructionDraftResult", + "InstructionIntent", + "RELATIONS", + "SCENE_REQUEST_SCHEMA", + "SUCCESS_SPEC_SCHEMA", + "SceneRequest", + "SuccessSpec", + "TASK_CANDIDATE_SET_SCHEMA", + "TASK_CONTRACTS", + "TASK_DRAFT_SCHEMA", + "TERMINAL_BEHAVIORS", + "TRANSPORT_DIRECTIONS", + "TaskAgent", + "TaskCandidate", + "TaskCandidateSet", + "TaskContract", + "TaskDraft", + "TaskGenerationError", + "canonical_hash", + "derive_scene_request", + "derive_success_spec", + "interpret_instruction_draft", + "task_contract", + "task_success_type", + "validate_instruction_intent", + "validate_scene_request", + "validate_success_spec", + "validate_task_candidate", + "validate_task_candidate_set", + "validate_task_draft", +] diff --git a/embodichain/gen_sim/task_engine/agent.py b/embodichain/gen_sim/task_engine/agent.py new file mode 100644 index 000000000..f5ab773f0 --- /dev/null +++ b/embodichain/gen_sim/task_engine/agent.py @@ -0,0 +1,289 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Scene-independent semantic candidate generation for Task Engine.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from concurrent.futures import ThreadPoolExecutor, as_completed +from copy import deepcopy +from dataclasses import dataclass +from typing import Any + +from .contracts import ( + SCENE_REQUEST_SCHEMA, + SUCCESS_SPEC_SCHEMA, + TASK_CANDIDATE_SET_SCHEMA, + TASK_DRAFT_SCHEMA, + TaskCandidate, + TaskCandidateSet, + canonical_hash, + validate_task_candidate, + validate_task_candidate_set, +) +from .interpretation import ( + InstructionCaller, + InstructionDraftResult, + interpret_instruction_draft, + validate_instruction_intent, +) +from .ontology import TASK_CONTRACTS, task_success_type + +__all__ = [ + "TaskAgent", + "TaskGenerationError", + "derive_scene_request", + "derive_success_spec", +] + +DraftInterpreter = Callable[..., InstructionDraftResult] + + +class TaskGenerationError(ValueError): + """Raised when every independently generated candidate fails validation.""" + + +@dataclass(frozen=True) +class _CandidateAttempt: + index: int + result: InstructionDraftResult | None = None + error: str = "" + + +class TaskAgent: + """Generate, validate, normalize, and vote on independent task drafts.""" + + def __init__( + self, + *, + caller: InstructionCaller | None = None, + interpreter: DraftInterpreter = interpret_instruction_draft, + ) -> None: + self._caller = caller + self._interpreter = interpreter + + def generate( + self, + task_id: str, + instruction: str, + model: str | None = None, + candidate_count: int = 3, + ) -> TaskCandidateSet: + """Generate candidates concurrently and retain votes after deduplication.""" + normalized_task_id = str(task_id).strip() + normalized_instruction = str(instruction).strip() + if not normalized_task_id or not normalized_instruction: + raise ValueError("task_id and instruction must be non-empty.") + if ( + isinstance(candidate_count, bool) + or not isinstance(candidate_count, int) + or candidate_count < 1 + ): + raise ValueError("candidate_count must be a positive integer.") + + attempts: list[_CandidateAttempt] = [] + with ThreadPoolExecutor( + max_workers=candidate_count, + thread_name_prefix="task-agent", + ) as executor: + futures = { + executor.submit( + self._interpreter, + normalized_instruction, + model=model, + caller=self._caller, + ): index + for index in range(candidate_count) + } + for future in as_completed(futures): + index = futures[future] + try: + attempts.append( + _CandidateAttempt(index=index, result=future.result()) + ) + except Exception as error: # Each candidate is an isolated vote. + attempts.append( + _CandidateAttempt( + index=index, + error=f"candidate_{index + 1:02d}: {type(error).__name__}: {error}", + ) + ) + attempts.sort(key=lambda item: item.index) + errors = [item.error for item in attempts if item.result is None] + unique: dict[str, TaskCandidate] = {} + valid_response_count = 0 + for attempt in attempts: + if attempt.result is None: + continue + assert attempt.result is not None + try: + canonical_intent = _canonicalize_intent(attempt.result.intent) + draft = { + "schema_version": TASK_DRAFT_SCHEMA, + "task_id": normalized_task_id, + "instruction": normalized_instruction, + "steps": canonical_intent["steps"], + } + semantic_hash = canonical_hash(draft["steps"]) + candidate_id = f"candidate_{len(unique) + 1:02d}" + candidate = validate_task_candidate( + { + "candidate_id": candidate_id, + "draft": draft, + "scene_request": derive_scene_request(draft), + "success_spec": derive_success_spec(draft), + "semantic_hash": semantic_hash, + "vote_count": 1, + "attempts": attempt.result.attempts, + "normalizations": deepcopy(list(attempt.result.normalizations)), + } + ) + existing = unique.get(semantic_hash) + if existing is not None: + existing["vote_count"] += 1 + existing["attempts"] = max( + existing["attempts"], attempt.result.attempts + ) + existing["normalizations"].extend(candidate["normalizations"]) + else: + unique[semantic_hash] = candidate + valid_response_count += 1 + except Exception as error: # Post-processing failures stay candidate-local. + errors.append( + f"candidate_{attempt.index + 1:02d}: " + f"{type(error).__name__}: {error}" + ) + + if not unique: + raise TaskGenerationError( + "All Task Agent candidates failed validation: " + "; ".join(errors) + ) + + return validate_task_candidate_set( + { + "schema_version": TASK_CANDIDATE_SET_SCHEMA, + "task_id": normalized_task_id, + "instruction": normalized_instruction, + "candidates": list(unique.values()), + "requested_candidate_count": candidate_count, + "valid_response_count": valid_response_count, + "errors": errors, + } + ) + + +def derive_scene_request(draft: Mapping[str, Any]) -> dict[str, Any]: + """Derive structural scene constraints without classifying reference text.""" + from .contracts import validate_scene_request, validate_task_draft + + normalized = validate_task_draft(draft) + references: list[dict[str, Any]] = [] + for step in normalized["steps"]: + task_type = str(step["task_type"]) + contract = TASK_CONTRACTS[task_type] + for role in ("object", "target"): + selector = step[role] + if selector["kind"] != "scene_ref": + continue + if role == "object": + structure = contract.source_structure + affordances = sorted(contract.scene_affordances) + initial_state = {"orientation": "fallen"} if task_type == "E2" else {} + attributes: dict[str, Any] = {} + else: + structure = _target_structure(task_type, str(step["relation"])) + affordances = _target_affordances(task_type, str(step["relation"])) + initial_state = {} + attributes = {} + references.append( + { + "reference_id": f"{step['id']}.{role}", + "step_id": step["id"], + "role": role, + "reference": selector["reference"], + "quantifier": selector["quantifier"], + "count": selector["count"], + "source_structure": structure, + "affordances": affordances, + "initial_state": initial_state, + "attributes": attributes, + } + ) + if not references: + raise ValueError("A TaskDraft must contain at least one scene_ref selector.") + return validate_scene_request( + { + "schema_version": SCENE_REQUEST_SCHEMA, + "task_id": normalized["task_id"], + "references": references, + } + ) + + +def derive_success_spec(draft: Mapping[str, Any]) -> dict[str, Any]: + """Derive every success term exclusively from the E-task ontology.""" + from .contracts import validate_success_spec, validate_task_draft + + normalized = validate_task_draft(draft) + return validate_success_spec( + { + "schema_version": SUCCESS_SPEC_SCHEMA, + "task_id": normalized["task_id"], + "op": "all", + "terms": [ + { + "step_id": step["id"], + "type": task_success_type(step["task_type"], step), + } + for step in normalized["steps"] + ], + }, + draft=normalized, + ) + + +def _canonicalize_intent(intent: Mapping[str, Any]) -> dict[str, Any]: + """Remove arbitrary model step IDs while preserving the explicit DAG order.""" + normalized = validate_instruction_intent(intent) + id_map = { + step["id"]: f"step_{index + 1:02d}" + for index, step in enumerate(normalized["steps"]) + } + steps = deepcopy(normalized["steps"]) + for step in steps: + old_id = step["id"] + step["id"] = id_map[old_id] + step["depends_on"] = [id_map[item] for item in step["depends_on"]] + for selector_name in ("object", "target"): + selector = step[selector_name] + if selector["kind"] == "step_result": + selector["step_id"] = id_map[selector["step_id"]] + return validate_instruction_intent({"steps": steps}) + + +def _target_affordances(task_type: str, relation: str) -> list[str]: + if task_type == "E3" or (task_type == "E1" and relation == "inside"): + return ["container"] + return [] + + +def _target_structure(task_type: str, relation: str) -> str: + if task_type == "E1" and relation == "on": + return "physical_entity" + if task_type == "E3" or (task_type == "E1" and relation == "inside"): + return "rigid_object" + return "spatial_reference" diff --git a/embodichain/gen_sim/task_engine/contracts.py b/embodichain/gen_sim/task_engine/contracts.py new file mode 100644 index 000000000..237423e55 --- /dev/null +++ b/embodichain/gen_sim/task_engine/contracts.py @@ -0,0 +1,410 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Strict, JSON-safe public contracts owned by Task Engine.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from copy import deepcopy +import hashlib +import json +from typing import Any, TypeAlias + +from .interpretation import validate_instruction_intent +from .ontology import TASK_CONTRACTS, task_success_type + +__all__ = [ + "SCENE_REQUEST_SCHEMA", + "SUCCESS_SPEC_SCHEMA", + "TASK_CANDIDATE_SET_SCHEMA", + "TASK_DRAFT_SCHEMA", + "SceneRequest", + "SuccessSpec", + "TaskCandidate", + "TaskCandidateSet", + "TaskDraft", + "canonical_hash", + "validate_scene_request", + "validate_success_spec", + "validate_task_candidate", + "validate_task_candidate_set", + "validate_task_draft", +] + +TASK_DRAFT_SCHEMA = "action_engine_task_draft_v1" +SCENE_REQUEST_SCHEMA = "action_engine_scene_request_v1" +SUCCESS_SPEC_SCHEMA = "action_engine_success_spec_v1" +TASK_CANDIDATE_SET_SCHEMA = "action_engine_task_candidate_set_v1" + +TaskDraft: TypeAlias = dict[str, Any] +SceneRequest: TypeAlias = dict[str, Any] +SuccessSpec: TypeAlias = dict[str, Any] +TaskCandidate: TypeAlias = dict[str, Any] +TaskCandidateSet: TypeAlias = dict[str, Any] + +_SUCCESS_TYPES = frozenset( + {contract.success_type for contract in TASK_CONTRACTS.values()} | {"semantic_goal"} +) +_DRAFT_KEYS = frozenset({"schema_version", "task_id", "instruction", "steps"}) +_SCENE_REQUEST_KEYS = frozenset({"schema_version", "task_id", "references"}) +_REFERENCE_KEYS = frozenset( + { + "reference_id", + "step_id", + "role", + "reference", + "quantifier", + "count", + "source_structure", + "affordances", + "initial_state", + "attributes", + } +) +_SUCCESS_KEYS = frozenset({"schema_version", "task_id", "op", "terms"}) +_SUCCESS_TERM_KEYS = frozenset({"step_id", "type"}) +_CANDIDATE_KEYS = frozenset( + { + "candidate_id", + "draft", + "scene_request", + "success_spec", + "semantic_hash", + "vote_count", + "attempts", + "normalizations", + } +) +_CANDIDATE_SET_KEYS = frozenset( + { + "schema_version", + "task_id", + "instruction", + "candidates", + "requested_candidate_count", + "valid_response_count", + "errors", + } +) + + +def canonical_hash(value: Any) -> str: + """Return the stable SHA-256 of one JSON-safe protocol value.""" + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def validate_task_draft(value: Mapping[str, Any]) -> TaskDraft: + result = _mapping(value, "TaskDraft") + _keys(result, _DRAFT_KEYS, "TaskDraft") + _schema(result, TASK_DRAFT_SCHEMA, "TaskDraft") + result["task_id"] = _nonempty(result.get("task_id"), "TaskDraft.task_id") + result["instruction"] = _nonempty( + result.get("instruction"), "TaskDraft.instruction" + ) + intent = validate_instruction_intent({"steps": result.get("steps")}) + result["steps"] = intent["steps"] + return result + + +def validate_scene_request(value: Mapping[str, Any]) -> SceneRequest: + result = _mapping(value, "SceneRequest") + _keys(result, _SCENE_REQUEST_KEYS, "SceneRequest") + _schema(result, SCENE_REQUEST_SCHEMA, "SceneRequest") + task_id = _nonempty(result.get("task_id"), "SceneRequest.task_id") + references: list[dict[str, Any]] = [] + for index, raw in enumerate( + _sequence(result.get("references"), "SceneRequest.references") + ): + context = f"SceneRequest.references[{index}]" + reference = _mapping(raw, context) + _keys(reference, _REFERENCE_KEYS, context) + for key in ("reference_id", "step_id", "role", "reference", "source_structure"): + reference[key] = _nonempty(reference.get(key), f"{context}.{key}") + reference["role"] = _enum( + reference["role"], {"object", "target"}, f"{context}.role" + ) + reference["quantifier"] = _enum( + reference.get("quantifier"), + {"one", "all", "count"}, + f"{context}.quantifier", + ) + reference["count"] = _integer( + reference.get("count"), f"{context}.count", minimum=0 + ) + if reference["quantifier"] in {"one", "all"} and reference["count"] != 0: + raise ValueError( + f"{context} quantifier={reference['quantifier']} requires count=0." + ) + if reference["quantifier"] == "count" and reference["count"] < 1: + raise ValueError(f"{context} quantifier=count requires count>=1.") + reference["affordances"] = _strings( + reference.get("affordances"), f"{context}.affordances" + ) + reference["initial_state"] = _mapping( + reference.get("initial_state"), f"{context}.initial_state" + ) + reference["attributes"] = _mapping( + reference.get("attributes"), f"{context}.attributes" + ) + references.append(reference) + _unique([item["reference_id"] for item in references], "SceneRequest reference IDs") + result["task_id"] = task_id + result["references"] = references + _json_safe(result, "SceneRequest") + return result + + +def validate_success_spec( + value: Mapping[str, Any], + *, + draft: Mapping[str, Any] | None = None, +) -> SuccessSpec: + result = _mapping(value, "SuccessSpec") + _keys(result, _SUCCESS_KEYS, "SuccessSpec") + _schema(result, SUCCESS_SPEC_SCHEMA, "SuccessSpec") + task_id = _nonempty(result.get("task_id"), "SuccessSpec.task_id") + if result.get("op") != "all": + raise ValueError("SuccessSpec.op must be 'all'.") + terms: list[dict[str, str]] = [] + for index, raw in enumerate(_sequence(result.get("terms"), "SuccessSpec.terms")): + context = f"SuccessSpec.terms[{index}]" + term = _mapping(raw, context) + _keys(term, _SUCCESS_TERM_KEYS, context) + terms.append( + { + "step_id": _nonempty(term.get("step_id"), f"{context}.step_id"), + "type": _enum(term.get("type"), set(_SUCCESS_TYPES), f"{context}.type"), + } + ) + if not terms: + raise ValueError("SuccessSpec.terms must not be empty.") + _unique([term["step_id"] for term in terms], "SuccessSpec step IDs") + if draft is not None: + normalized_draft = validate_task_draft(draft) + if normalized_draft["task_id"] != task_id: + raise ValueError("SuccessSpec.task_id must match TaskDraft.task_id.") + expected = [ + { + "step_id": step["id"], + "type": task_success_type(step["task_type"], step), + } + for step in normalized_draft["steps"] + ] + if terms != expected: + raise ValueError( + "SuccessSpec terms must be ordered, complete, and derived from " + "task_success_type." + ) + result["task_id"] = task_id + result["terms"] = terms + return result + + +def validate_task_candidate(value: Mapping[str, Any]) -> TaskCandidate: + result = _mapping(value, "TaskCandidate") + _keys(result, _CANDIDATE_KEYS, "TaskCandidate") + result["candidate_id"] = _nonempty( + result.get("candidate_id"), "TaskCandidate.candidate_id" + ) + result["draft"] = validate_task_draft(result.get("draft")) + result["scene_request"] = validate_scene_request(result.get("scene_request")) + result["success_spec"] = validate_success_spec( + result.get("success_spec"), draft=result["draft"] + ) + for name in ("scene_request", "success_spec"): + if result[name]["task_id"] != result["draft"]["task_id"]: + raise ValueError(f"TaskCandidate {name}.task_id must match its draft.") + from .agent import derive_scene_request + + if result["scene_request"] != derive_scene_request(result["draft"]): + raise ValueError( + "TaskCandidate.scene_request must be derived exactly from its draft." + ) + result["semantic_hash"] = _digest( + result.get("semantic_hash"), "TaskCandidate.semantic_hash" + ) + if result["semantic_hash"] != canonical_hash(result["draft"]["steps"]): + raise ValueError( + "TaskCandidate.semantic_hash does not match its canonical steps." + ) + result["vote_count"] = _integer( + result.get("vote_count"), "TaskCandidate.vote_count", minimum=1 + ) + result["attempts"] = _integer( + result.get("attempts"), "TaskCandidate.attempts", minimum=1, maximum=2 + ) + result["normalizations"] = _mapping_sequence( + result.get("normalizations"), "TaskCandidate.normalizations" + ) + return result + + +def validate_task_candidate_set(value: Mapping[str, Any]) -> TaskCandidateSet: + result = _mapping(value, "TaskCandidateSet") + _keys(result, _CANDIDATE_SET_KEYS, "TaskCandidateSet") + _schema(result, TASK_CANDIDATE_SET_SCHEMA, "TaskCandidateSet") + task_id = _nonempty(result.get("task_id"), "TaskCandidateSet.task_id") + instruction = _nonempty(result.get("instruction"), "TaskCandidateSet.instruction") + requested = _integer( + result.get("requested_candidate_count"), + "TaskCandidateSet.requested_candidate_count", + minimum=1, + ) + valid = _integer( + result.get("valid_response_count"), + "TaskCandidateSet.valid_response_count", + minimum=1, + maximum=requested, + ) + candidates = [ + validate_task_candidate(item) + for item in _sequence(result.get("candidates"), "TaskCandidateSet.candidates") + ] + if not candidates: + raise ValueError("TaskCandidateSet.candidates must not be empty.") + _unique([item["candidate_id"] for item in candidates], "TaskCandidate IDs") + _unique( + [item["semantic_hash"] for item in candidates], "TaskCandidate semantic hashes" + ) + if sum(item["vote_count"] for item in candidates) != valid: + raise ValueError( + "TaskCandidate vote_count values must sum to valid_response_count." + ) + for candidate in candidates: + if ( + candidate["draft"]["task_id"] != task_id + or candidate["draft"]["instruction"] != instruction + ): + raise ValueError("Every TaskCandidate draft must match its candidate set.") + errors = _strings(result.get("errors"), "TaskCandidateSet.errors", allow_empty=True) + if valid + len(errors) != requested: + raise ValueError( + "Valid responses plus errors must equal requested_candidate_count." + ) + result.update( + { + "task_id": task_id, + "instruction": instruction, + "requested_candidate_count": requested, + "valid_response_count": valid, + "candidates": candidates, + "errors": errors, + } + ) + return result + + +def _mapping(value: Any, context: str) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{context} must be a mapping.") + return deepcopy(dict(value)) + + +def _sequence(value: Any, context: str) -> list[Any]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError(f"{context} must be a list.") + return list(value) + + +def _keys( + value: Mapping[str, Any], expected: set[str] | frozenset[str], context: str +) -> None: + if set(value) != set(expected): + raise ValueError( + f"{context} requires exactly fields {sorted(expected)}; received {sorted(value)}." + ) + + +def _schema(value: Mapping[str, Any], expected: str, context: str) -> None: + if value.get("schema_version") != expected: + raise ValueError(f"{context}.schema_version must be {expected!r}.") + + +def _string(value: Any, context: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{context} must be a string.") + return value.strip() + + +def _nonempty(value: Any, context: str) -> str: + result = _string(value, context) + if not result: + raise ValueError(f"{context} must not be empty.") + return result + + +def _enum(value: Any, choices: set[str], context: str) -> str: + result = _string(value, context) + if result not in choices: + raise ValueError(f"{context} must be one of {sorted(choices)}.") + return result + + +def _integer( + value: Any, context: str, *, minimum: int, maximum: int | None = None +) -> int: + if ( + not isinstance(value, int) + or isinstance(value, bool) + or value < minimum + or (maximum is not None and value > maximum) + ): + raise ValueError(f"{context} must be an integer in the allowed range.") + return value + + +def _strings(value: Any, context: str, *, allow_empty: bool = False) -> list[str]: + result = [_string(item, context) for item in _sequence(value, context)] + if not allow_empty and any(not item for item in result): + raise ValueError(f"{context} values must not be empty.") + if len(result) != len(set(result)): + raise ValueError(f"{context} values must be unique.") + return result + + +def _mapping_sequence(value: Any, context: str) -> list[dict[str, Any]]: + result = [_mapping(item, context) for item in _sequence(value, context)] + _json_safe(result, context) + return result + + +def _digest(value: Any, context: str) -> str: + result = _string(value, context) + if len(result) != 64 or any( + character not in "0123456789abcdef" for character in result + ): + raise ValueError(f"{context} must be a lowercase SHA-256 digest.") + return result + + +def _unique(values: Sequence[str], context: str) -> None: + if len(values) != len(set(values)): + raise ValueError(f"{context} must be unique.") + + +def _json_safe(value: Any, context: str) -> None: + try: + json.dumps(value, allow_nan=False) + except (TypeError, ValueError) as error: + raise ValueError(f"{context} must be finite and JSON serializable.") from error diff --git a/embodichain/gen_sim/task_engine/interpretation.py b/embodichain/gen_sim/task_engine/interpretation.py new file mode 100644 index 000000000..68ee25c7c --- /dev/null +++ b/embodichain/gen_sim/task_engine/interpretation.py @@ -0,0 +1,1240 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Scene-independent structured interpretation for Task Engine.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from copy import deepcopy +from dataclasses import dataclass +import json +import os +from pathlib import Path +import re +from time import perf_counter +from typing import Any, TypeAlias + +from .ontology import ( + RELATIONS, + TASK_CONTRACTS, + TERMINAL_BEHAVIORS, + TRANSPORT_DIRECTIONS, +) + +__all__ = [ + "INSTRUCTION_INTENT_SCHEMA", + "InstructionDraftResult", + "InstructionIntent", + "InstructionCaller", + "interpret_instruction_draft", + "validate_instruction_intent", +] + +InstructionCaller = Callable[..., Mapping[str, Any]] +InstructionIntent: TypeAlias = dict[str, Any] +TASK_TYPES = frozenset(TASK_CONTRACTS) + +_RELATIONS = RELATIONS +_ARMS = frozenset({"none", "auto", "left_arm", "right_arm"}) +_ORIENTATIONS = frozenset({"none", "preserve", "upright"}) +_TARGET_STATES = frozenset({"none", "open", "closed", "activated"}) +_LAYOUTS = frozenset({"none", "line"}) +_AXES = frozenset({"none", "world_x", "world_y"}) +_DIRECTIONS = TRANSPORT_DIRECTIONS +_TERMINAL_BEHAVIORS = TERMINAL_BEHAVIORS +_SELECTOR_KINDS = frozenset({"none", "scene_ref", "step_result"}) +_QUANTIFIERS = frozenset({"one", "all", "count"}) +_STEP_KEYS = frozenset( + { + "id", + "task_type", + "object", + "target", + "relation", + "required_arm", + "transfer_arm", + "receive_arm", + "orientation_goal", + "target_state", + "target_setting", + "layout", + "axis", + "direction", + "terminal_behavior", + "depends_on", + } +) +_INTENT_TASK_FIELD_REGISTRY = { + task_type: contract.applicable_intent_fields + for task_type, contract in TASK_CONTRACTS.items() +} +_INTENT_FIELD_DEFAULTS: dict[str, Any] = { + "target": None, + "relation": "none", + "required_arm": "none", + "transfer_arm": "none", + "receive_arm": "none", + "orientation_goal": "none", + "target_state": "none", + "target_setting": 0, + "layout": "none", + "axis": "none", + "direction": "none", + "terminal_behavior": "none", +} +_SELECTOR_KEYS = frozenset( + { + "kind", + "step_id", + "reference", + "quantifier", + "count", + } +) +_FORBIDDEN_FIELDS = frozenset( + { + "atomic_action", + "atomic_actions", + "atomicaction", + "coordinates", + "bbox", + "bboxes", + "grasp_pose", + "keypoint", + "keypoints", + "joint_positions", + "joints", + "pose", + "position", + "qpos", + "rotation", + "target_pose", + "translation", + "trajectory", + "waypoints", + } +) +# MiMo's OpenAI-compatible endpoint can spend the whole completion budget in +# hidden reasoning when the request leaves thinking enabled. A sparse final +# JSON object then looks like a schema failure to the deterministic verifier. +# Keep the budget bounded and turn reasoning off for the text interpretation +# call; the parser must return an auditable object rather than a thought trace. +_MIMO_MAX_COMPLETION_TOKENS = 4096 +_GEN_SIM_DIR = Path(__file__).resolve().parents[1] +_GEN_SIM_ENV_PATH = _GEN_SIM_DIR / ".env" +_GEN_CONFIG_PATH = _GEN_SIM_DIR / "simready_pipeline" / "configs" / "gen_config.json" + + +class _MissingRequiredTargetError(ValueError): + """Identify a validation failure that receives targeted repair guidance.""" + + +@dataclass(frozen=True) +class InstructionDraftResult: + """One validated, scene-independent interpretation and its audit metadata.""" + + intent: InstructionIntent + model: str + attempts: int + latency_seconds: float + normalizations: tuple[dict[str, Any], ...] + + +# Object semantics remain open natural-language references until the dedicated +# scene-grounding phase resolves them. All other values are strict protocol +# enums; non-canonical model output is repaired by the model, never guessed by +# a local language alias table. + +_SELECTOR_SCHEMA = { + "type": "object", + "additionalProperties": False, + "required": sorted(_SELECTOR_KEYS), + "properties": { + "kind": {"type": "string", "enum": sorted(_SELECTOR_KINDS)}, + "step_id": {"type": "string"}, + "reference": {"type": "string"}, + "quantifier": {"type": "string", "enum": sorted(_QUANTIFIERS)}, + "count": {"type": "integer", "minimum": 0}, + }, +} + +_INTENT_OUTPUT_SCHEMA = { + "title": "ActionEngineInstructionIntent", + "type": "object", + "additionalProperties": False, + "required": ["steps"], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": False, + "required": sorted(_STEP_KEYS), + "properties": { + "id": {"type": "string"}, + "task_type": {"type": "string", "enum": sorted(TASK_TYPES)}, + "object": _SELECTOR_SCHEMA, + "target": _SELECTOR_SCHEMA, + "relation": {"type": "string", "enum": sorted(_RELATIONS)}, + "required_arm": {"type": "string", "enum": sorted(_ARMS)}, + "transfer_arm": {"type": "string", "enum": sorted(_ARMS)}, + "receive_arm": {"type": "string", "enum": sorted(_ARMS)}, + "orientation_goal": { + "type": "string", + "enum": sorted(_ORIENTATIONS), + }, + "target_state": { + "type": "string", + "enum": sorted(_TARGET_STATES), + }, + "target_setting": {"type": "integer"}, + "layout": {"type": "string", "enum": sorted(_LAYOUTS)}, + "axis": {"type": "string", "enum": sorted(_AXES)}, + "direction": { + "type": "string", + "enum": sorted(_DIRECTIONS), + }, + "terminal_behavior": { + "type": "string", + "enum": sorted(_TERMINAL_BEHAVIORS), + }, + "depends_on": { + "type": "array", + "items": {"type": "string"}, + }, + }, + }, + } + }, +} + +# Keep a read-only-by-convention public copy for callers that need to configure +# a structured client. The schema is an input contract, not a persisted task +# graph; ``validate_instruction_intent`` remains the authoritative verifier. +INSTRUCTION_INTENT_SCHEMA = deepcopy(_INTENT_OUTPUT_SCHEMA) + + +def interpret_instruction_draft( + instruction: str, + *, + model: str | None = None, + caller: InstructionCaller | None = None, +) -> InstructionDraftResult: + """Interpret one instruction without reading or grounding a scene.""" + instruction_text = str(instruction).strip() + if not instruction_text: + raise ValueError("instruction must be non-empty.") + prompt = _instruction_prompt(instruction_text) + invoke = caller or _default_instruction_caller + # An injected caller owns its transport and does not need provider config. + selected_model = model if caller is not None else _instruction_model(model) + if caller is None and selected_model is None: + raise ValueError( + "A text LLM model is required through --llm-model, " + "ACTION_ENGINE_LLM_MODEL, or OPENAI_MODEL." + ) + started = perf_counter() + first_error: Exception | None = None + for attempt in range(2): + current_prompt = prompt + if first_error is not None: + current_prompt += ( + "\n\nREPAIR OVERRIDE: the previous JSON was invalid. Return a corrected " + "JSON object only; do not repeat the sparse response. Every step " + "must contain all 16 step keys and every selector all 5 selector " + "keys. Keep semantic fields explicit: E4 requires transfer_arm " + "and receive_arm, and E1/E3 require target plus relation (unless " + "E1 layout=line). Use canonical defaults only for fields that do " + "not apply. Validation error: " + f"{first_error}\n" + "Copy this complete shape before filling values (shape only; do " + "not copy its values or step count):\n" + f"{json.dumps(_instruction_shape_example(), ensure_ascii=False, sort_keys=True)}\n" + "Selector kind rules:\n" + f"{_instruction_selector_rules()}" + f"{_instruction_repair_guidance(first_error)}" + ) + try: + response = invoke( + prompt=current_prompt, + schema=deepcopy(INSTRUCTION_INTENT_SCHEMA), + model=selected_model, + ) + normalized, normalizations = _normalize_instruction_intent_fields( + _coerce_instruction_response(response) + ) + intent = validate_instruction_intent(normalized) + return InstructionDraftResult( + intent=intent, + model=selected_model or "injected_caller", + attempts=attempt + 1, + latency_seconds=perf_counter() - started, + normalizations=tuple(normalizations), + ) + except (TypeError, ValueError) as error: + if attempt: + raise ValueError( + "Instruction intent failed validation after one repair: " f"{error}" + ) from error + first_error = error + raise AssertionError("unreachable") + + +def _normalize_instruction_intent_fields( + value: Mapping[str, Any], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Canonicalize defaults and uniquely constrained cross-step continuity. + + The strict public validator deliberately remains unchanged. This pass is + confined to the LLM boundary, where weak JSON-mode providers sometimes + copy a meaningful value into an inapplicable slot such as E4.required_arm. + Required scene facts and ambiguous arm assignments are never inferred here + and still fail closed. + """ + result = deepcopy(dict(value)) + raw_steps = result.get("steps") + if not isinstance(raw_steps, list): + return result, [] + changes: list[dict[str, Any]] = [] + for index, raw_step in enumerate(raw_steps): + if not isinstance(raw_step, dict) or set(raw_step) != _STEP_KEYS: + continue + task_type = raw_step.get("task_type") + applicable = _INTENT_TASK_FIELD_REGISTRY.get(task_type) + if applicable is None: + continue + for field, configured_default in _INTENT_FIELD_DEFAULTS.items(): + field_applies = field in applicable + if task_type == "E1" and field in {"target", "relation"}: + field_applies = raw_step.get("layout") != "line" + if task_type == "E1" and field == "axis": + field_applies = raw_step.get("layout") == "line" + if field_applies: + continue + default = ( + _empty_selector() + if field == "target" and configured_default is None + else deepcopy(configured_default) + ) + if raw_step[field] == default: + continue + previous = deepcopy(raw_step[field]) + raw_step[field] = default + changes.append( + { + "path": f"steps[{index}].{field}", + "from": previous, + "to": deepcopy(default), + "reason": f"inapplicable_for_{task_type}", + } + ) + target = raw_step.get("target") + if ( + task_type == "E5" + and isinstance(target, Mapping) + and target.get("kind") == "none" + and raw_step.get("relation") == "none" + and raw_step.get("direction") == "none" + and raw_step.get("terminal_behavior") == "hold" + ): + raw_step["direction"] = "up" + changes.append( + { + "path": f"steps[{index}].direction", + "from": "none", + "to": "up", + "reason": "e5_hold_defaults_to_lift", + } + ) + _normalize_handover_arm_continuity(raw_steps, changes) + return result, changes + + +def _normalize_handover_arm_continuity( + steps: Sequence[Any], + changes: list[dict[str, Any]], +) -> None: + """Repair a same-arm E4 only when adjacent ownership fixes both roles.""" + by_id: dict[str, Mapping[str, Any]] = {} + for step in steps: + if not isinstance(step, dict) or set(step) != _STEP_KEYS: + return + step_id = step.get("id") + if not isinstance(step_id, str) or not step_id or step_id in by_id: + return + by_id[step_id] = step + + explicit_arms = {"left_arm", "right_arm"} + for index, step in enumerate(steps): + assert isinstance(step, dict) + transfer = step.get("transfer_arm") + receive = step.get("receive_arm") + if ( + step.get("task_type") != "E4" + or transfer not in explicit_arms + or transfer != receive + ): + continue + + object_key = _object_lineage_key(step, by_id) + upstream_arm: str | None = None + for producer in reversed(steps[:index]): + assert isinstance(producer, Mapping) + if object_key is None or _object_lineage_key(producer, by_id) != object_key: + continue + candidate = ( + producer.get("receive_arm") + if producer.get("task_type") == "E4" + else producer.get("required_arm") + ) + if candidate in explicit_arms: + upstream_arm = str(candidate) + break + + downstream_arm: str | None = None + for consumer in steps[index + 1 :]: + assert isinstance(consumer, Mapping) + if object_key is None or _object_lineage_key(consumer, by_id) != object_key: + continue + candidate = ( + consumer.get("transfer_arm") + if consumer.get("task_type") == "E4" + else consumer.get("required_arm") + ) + if candidate in explicit_arms: + downstream_arm = str(candidate) + break + + desired_transfer = upstream_arm or str(transfer) + desired_receive = downstream_arm or str(receive) + if desired_transfer == desired_receive: + continue + for field, desired in ( + ("transfer_arm", desired_transfer), + ("receive_arm", desired_receive), + ): + if step[field] == desired: + continue + previous = step[field] + step[field] = desired + changes.append( + { + "path": f"steps[{index}].{field}", + "from": previous, + "to": desired, + "reason": "handover_arm_continuity", + } + ) + + +def _object_lineage_key( + step: Mapping[str, Any], + by_id: Mapping[str, Mapping[str, Any]], + seen: frozenset[str] = frozenset(), +) -> tuple[str, str] | None: + """Resolve object identity only through explicit step-result lineage.""" + selector = step.get("object") + if not isinstance(selector, Mapping): + return None + kind = selector.get("kind") + if kind == "scene_ref": + step_id = step.get("id") + if not isinstance(step_id, str) or not step_id: + return None + return ("step_result", step_id) + if kind != "step_result": + return None + producer_id = selector.get("step_id") + if not isinstance(producer_id, str) or producer_id in seen: + return None + producer = by_id.get(producer_id) + if producer is None: + return None + return _object_lineage_key(producer, by_id, seen | {producer_id}) + + +def _empty_selector() -> dict[str, Any]: + """Return the canonical selector value for an inapplicable target.""" + return { + "kind": "none", + "step_id": "", + "reference": "", + "quantifier": "one", + "count": 0, + } + + +def validate_instruction_intent(value: Mapping[str, Any]) -> dict[str, Any]: + """Validate the private, non-graph instruction interpretation contract.""" + if not isinstance(value, Mapping): + raise TypeError("Instruction intent must be a mapping.") + _reject_forbidden_fields(value) + if set(value) != {"steps"}: + raise ValueError("Instruction intent may contain only 'steps'.") + raw_steps = value.get("steps") + if not isinstance(raw_steps, Sequence) or isinstance(raw_steps, (str, bytes)): + raise ValueError("Instruction intent steps must be a list.") + if not raw_steps: + raise ValueError("Instruction intent steps must not be empty.") + steps = [] + ids: set[str] = set() + dependencies: dict[str, list[str]] = {} + for index, raw in enumerate(raw_steps): + context = f"InstructionIntent.steps[{index}]" + if not isinstance(raw, Mapping): + raise ValueError(f"{context} must be a mapping.") + if set(raw) != _STEP_KEYS: + raise ValueError( + f"{context} requires exactly fields {sorted(_STEP_KEYS)}; " + f"received {sorted(raw)}." + ) + step = deepcopy(dict(raw)) + step_id = _nonempty(step["id"], f"{context}.id") + if step_id in ids: + raise ValueError(f"Duplicate instruction step ID {step_id!r}.") + ids.add(step_id) + step["id"] = step_id + step["task_type"] = _choice( + step["task_type"], TASK_TYPES, f"{context}.task_type" + ) + step["object"] = _validate_selector(step["object"], f"{context}.object") + step["target"] = _validate_selector(step["target"], f"{context}.target") + step["relation"] = _canonical_relation(step["relation"], f"{context}.relation") + for key in ("required_arm", "transfer_arm", "receive_arm"): + step[key] = _canonical_arm(step[key], f"{context}.{key}") + step["orientation_goal"] = _canonical_orientation( + step["orientation_goal"], f"{context}.orientation_goal" + ) + step["target_state"] = _choice( + step["target_state"], _TARGET_STATES, f"{context}.target_state" + ) + if isinstance(step["target_setting"], bool) or not isinstance( + step["target_setting"], int + ): + raise ValueError(f"{context}.target_setting must be an integer.") + step["layout"] = _choice(step["layout"], _LAYOUTS, f"{context}.layout") + step["axis"] = _choice(step["axis"], _AXES, f"{context}.axis") + step["direction"] = _choice( + step["direction"], _DIRECTIONS, f"{context}.direction" + ) + step["terminal_behavior"] = _choice( + step["terminal_behavior"], + _TERMINAL_BEHAVIORS, + f"{context}.terminal_behavior", + ) + raw_depends = step["depends_on"] + if not isinstance(raw_depends, Sequence) or isinstance( + raw_depends, (str, bytes) + ): + raise ValueError(f"{context}.depends_on must be a list.") + step["depends_on"] = [ + _nonempty(item, f"{context}.depends_on") for item in raw_depends + ] + if step_id in step["depends_on"]: + raise ValueError(f"{context}.depends_on cannot contain its own ID.") + dependencies[step_id] = step["depends_on"] + _validate_task_fields(step, context) + steps.append(step) + positions = {str(step["id"]): index for index, step in enumerate(steps)} + for index, step in enumerate(steps): + for selector_name in ("object", "target"): + selector = step[selector_name] + if selector["kind"] != "step_result": + continue + reference = str(selector["step_id"]) + if reference not in positions: + raise ValueError( + f"Instruction step {step['id']!r} {selector_name} references " + f"unknown step {reference!r}." + ) + if positions[reference] >= index: + raise ValueError( + f"Instruction step {step['id']!r} {selector_name} must reference " + f"a preceding step, not {reference!r}." + ) + for step_id, depends_on in dependencies.items(): + unknown = set(depends_on) - ids + if unknown: + raise ValueError( + f"Instruction step {step_id!r} has unknown dependencies " + f"{sorted(unknown)}." + ) + _validate_dag(dependencies) + return {"steps": steps} + + +def _validate_selector(value: Any, context: str) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{context} must be a mapping.") + if set(value) != _SELECTOR_KEYS: + raise ValueError( + f"{context} requires exactly fields {sorted(_SELECTOR_KEYS)}; " + f"received {sorted(value)}." + ) + selector = deepcopy(dict(value)) + selector["kind"] = _choice(selector["kind"], _SELECTOR_KINDS, f"{context}.kind") + selector["step_id"] = _selector_string(selector["step_id"], f"{context}.step_id") + selector["reference"] = _selector_string( + selector["reference"], f"{context}.reference" + ) + selector["quantifier"] = _canonical_quantifier( + selector["quantifier"], f"{context}.quantifier" + ) + if isinstance(selector["count"], bool) or not isinstance(selector["count"], int): + raise ValueError(f"{context}.count must be an integer.") + if selector["count"] < 0: + raise ValueError(f"{context}.count must be non-negative.") + kind = selector["kind"] + if kind == "scene_ref" and not selector["reference"]: + raise ValueError(f"{context} scene_ref requires a reference.") + if kind == "step_result": + if not selector["step_id"]: + raise ValueError(f"{context} step_result requires step_id.") + if selector["reference"]: + raise ValueError( + f"{context} step_result may identify only a prior step_id." + ) + if selector["quantifier"] != "one" or selector["count"] != 0: + raise ValueError( + f"{context} step_result requires quantifier=one and count=0." + ) + if kind == "scene_ref" and selector["step_id"]: + raise ValueError(f"{context} scene_ref cannot carry step_id.") + if kind == "none" and (selector["step_id"] or selector["reference"]): + raise ValueError(f"{context} kind=none cannot carry constraints.") + if kind == "none" and (selector["quantifier"] != "one" or selector["count"] != 0): + raise ValueError(f"{context} kind=none requires quantifier=one and count=0.") + if selector["quantifier"] == "one" and selector["count"] != 0: + raise ValueError(f"{context} quantifier=one requires count=0.") + if selector["quantifier"] == "all" and selector["count"] != 0: + raise ValueError(f"{context} quantifier=all requires count=0.") + if selector["quantifier"] == "count" and selector["count"] < 1: + raise ValueError(f"{context} quantifier=count requires count>=1.") + return selector + + +def _validate_task_fields(step: Mapping[str, Any], context: str) -> None: + task_type = str(step["task_type"]) + target_kind = str(step["target"]["kind"]) + if task_type not in {"E1", "E3", "E5"} and step["relation"] != "none": + raise ValueError(f"{context} {task_type} does not accept relation.") + if task_type == "E3" and step["relation"] != "above": + raise ValueError(f"{context} E3 relation must be above.") + target_setting = int(step["target_setting"]) + if task_type != "E8" and target_setting != 0: + raise ValueError(f"{context} target_setting is only valid for E8.") + if task_type != "E1" and step["axis"] != "none": + raise ValueError(f"{context} axis is only valid for E1 line arrangement.") + if task_type == "E1" and step["layout"] != "line" and step["axis"] != "none": + raise ValueError(f"{context} axis is only valid for E1 line arrangement.") + if task_type not in {"E6", "E7", "E9"} and step["target_state"] != "none": + raise ValueError(f"{context} target_state is not valid for {task_type}.") + if task_type != "E4" and step["transfer_arm"] != "none": + raise ValueError(f"{context} transfer_arm is only valid for E4.") + if task_type != "E4" and step["receive_arm"] != "none": + raise ValueError(f"{context} receive_arm is only valid for E4.") + orientation_goal = str(step["orientation_goal"]) + if task_type == "E2" and orientation_goal != "upright": + raise ValueError(f"{context} E2 orientation_goal must be upright.") + if task_type not in {"E1", "E2", "E4"} and orientation_goal != "none": + raise ValueError( + f"{context} orientation_goal is only valid for E1, E2, and E4." + ) + if task_type == "E1" and step["layout"] == "line": + if target_kind != "none": + raise ValueError(f"{context} E1 line arrangement cannot carry a target.") + if step["relation"] != "none": + raise ValueError(f"{context} E1 line arrangement cannot carry a relation.") + elif task_type in {"E1", "E3"}: + if target_kind == "none": + raise _MissingRequiredTargetError( + f"{context} {task_type} requires a target selector." + ) + if step["relation"] == "none" and task_type == "E3": + raise ValueError(f"{context} {task_type} requires a symbolic relation.") + elif task_type == "E5": + direction = str(step["direction"]) + terminal = str(step["terminal_behavior"]) + if terminal not in _TERMINAL_BEHAVIORS - {"none"}: + raise ValueError(f"{context} E5 requires terminal_behavior hold/place.") + if target_kind == "none": + if step["relation"] != "none": + raise ValueError(f"{context} E5 relation requires a target selector.") + if direction == "none" and terminal != "place": + raise ValueError( + f"{context} E5 requires a direction or target relation." + ) + else: + if step["relation"] == "none": + raise ValueError(f"{context} E5 target requires a relation.") + if direction != "none": + raise ValueError( + f"{context} E5 target relation cannot also carry direction." + ) + elif target_kind != "none": + raise ValueError(f"{context} {task_type} does not accept a target selector.") + if task_type != "E5": + if step["direction"] != "none": + raise ValueError(f"{context} direction is only valid for E5.") + if step["terminal_behavior"] != "none": + raise ValueError(f"{context} terminal_behavior is only valid for E5.") + if task_type == "E4": + transfer = str(step["transfer_arm"]) + receive = str(step["receive_arm"]) + if transfer not in {"left_arm", "right_arm"} or receive not in { + "left_arm", + "right_arm", + }: + raise ValueError(f"{context} E4 requires two explicit arms.") + if transfer == receive: + raise ValueError(f"{context} E4 transfer and receive arms must differ.") + if step["required_arm"] not in {"none", "auto"}: + raise ValueError( + f"{context} E4 uses transfer_arm/receive_arm, not required_arm." + ) + if task_type == "E5" and step["required_arm"] not in {"none", "auto"}: + raise ValueError(f"{context} E5 always uses both arms, not required_arm.") + if task_type == "E6" and step["target_state"] != "open": + raise ValueError(f"{context} E6 target_state must be open.") + if task_type == "E7" and step["target_state"] != "closed": + raise ValueError(f"{context} E7 target_state must be closed.") + if task_type == "E9" and step["target_state"] != "activated": + raise ValueError(f"{context} E9 target_state must be activated.") + if step["layout"] == "line" and task_type != "E1": + raise ValueError(f"{context} only E1 supports layout=line.") + + +def _instruction_prompt(instruction: str) -> str: + return ( + "Convert the user's explicit L1-L3 instruction into typed E1-E9 task " + "intent. Understand synonyms, ellipsis, and pronouns, but " + "do not invent missing objects. Use step_result for cross-step pronouns " + "and explicit references to the result of an earlier manipulation. Keep " + "an independently selected repeated noun as scene_ref; identical text " + "alone does not prove object identity. " + "Object directions are robot-relative; arm names are robot body sides. " + "Preserve each concrete object or target phrase from the instruction as " + "an open scene_ref.reference. Do not classify it or emit a scene UID. " + "Emit no AtomicAction, category label, affordance, coordinates, poses, " + "paths, or reasoning. Encode explicit ordering with depends_on; same-action set " + "members may remain independent. Use empty strings and 'none' for " + "inapplicable required fields. A request to retract the transfer arm " + "immediately after an E4 handover is a mandatory runtime retreat/home " + "barrier for that E4; do " + "not emit a separate task step for it. The exact output keys are steps -> id, " + "task_type, object, target, relation, required_arm, transfer_arm, " + "receive_arm, orientation_goal, target_state, target_setting, layout, " + "axis, direction, terminal_behavior, depends_on; each selector has kind, " + "step_id, reference, quantifier, count.\n\n" + "Use orientation_goal=none unless the instruction explicitly requests " + "upright orientation or preserving the original orientation. Spatial " + "placement and handover alone do not imply preserve. " + f"Instruction:\n{instruction}\n\n" + f"E1-E9 catalog:\n{json.dumps(_intent_capability_catalog(), ensure_ascii=False, sort_keys=True)}\n\n" + "Shape-only complete JSON example (do not copy its step count or values; " + "copy every key, including keys whose value is none/empty/0):\n" + f"{json.dumps(_instruction_shape_example(), ensure_ascii=False, sort_keys=True)}\n\n" + "Selector kind rules (these are not extra output fields):\n" + f"{_instruction_selector_rules()}\n\n" + "For E5, use target+relation for moving an object relative to another " + "object, or direction for a small robot-relative move. A dual-arm pick, " + "lift, raise, or hold request without another target uses direction=up " + "and terminal_behavior=hold. Use hold unless the instruction explicitly " + "says to put/release the object. For pick " + "and release at the original location, use direction=none and place. A dual-arm " + "pick/move/transport request is E5, not E1. Final checklist: every step " + "has all 16 step keys; every object and target " + "has all 5 selector keys. For an inapplicable field use the canonical " + "default shown in the example, never omit the field. E4 must explicitly " + "state transfer_arm and receive_arm. E1/E3 must explicitly state target " + "and relation (except E1 layout=line)." + ) + + +def _instruction_shape_example() -> dict[str, Any]: + """Return a compact field-complete example for providers with weak schemas.""" + selector = { + "kind": "scene_ref", + "step_id": "", + "reference": "example object A", + "quantifier": "one", + "count": 0, + } + empty_selector = { + "kind": "none", + "step_id": "", + "reference": "", + "quantifier": "one", + "count": 0, + } + return { + "steps": [ + { + "id": "step_1", + "task_type": "E2", + "object": selector, + "target": empty_selector, + "relation": "none", + "required_arm": "auto", + "transfer_arm": "none", + "receive_arm": "none", + "orientation_goal": "upright", + "target_state": "none", + "target_setting": 0, + "layout": "none", + "axis": "none", + "direction": "none", + "terminal_behavior": "none", + "depends_on": [], + } + ] + } + + +def _instruction_selector_rules() -> str: + """Return the mutually exclusive selector encodings for model prompts.""" + step_result = { + "kind": "step_result", + "step_id": "step_1", + "reference": "", + "quantifier": "one", + "count": 0, + } + return ( + "- kind=none: step_id and reference are empty strings; " + "quantifier='one'; count=0.\n" + "- kind=scene_ref: step_id is empty and reference preserves the concrete " + "object phrase from the user's instruction. Repeated scene_ref text does " + "not establish cross-step identity.\n" + "- kind=step_result: use it only for a pronoun that means exactly one " + "object, or an explicit continuation of the result of an earlier " + "instruction step. Set step_id to that prior " + "step ID and set reference='', quantifier='one', count=0. Do not copy " + "the prior object's phrase into this selector. Replace step_1 in this " + f"complete shape with the actual prior step ID: {json.dumps(step_result, sort_keys=True)}\n" + "A step_result may identify only a prior step_id; it cannot carry any " + "other object constraint." + ) + + +def _instruction_repair_guidance(error: Exception) -> str: + """Add narrow semantic guidance for errors weak JSON-mode models repeat.""" + if "E4 transfer and receive arms must differ" in str(error): + return ( + "\nSame-arm handover repair rule: transfer_arm and receive_arm must " + "name different arms. Preserve the explicitly stated transfer arm. " + "When a later clause clearly continues with the handed object using " + "the other arm, use that arm as receive_arm. Resolve coreference from " + "the instruction semantics; identical scene_ref text alone does not " + "prove that two independently selected objects are the same.\n" + ) + if not isinstance(error, _MissingRequiredTargetError): + return "" + return ( + "\nMissing-target repair rule: for a non-line E1 placement, object is " + "the item being moved and target is the explicit reference object " + "after the spatial relation in the original instruction. For example, " + "in 'place it to the left of the striped pedestal', object is the earlier " + "step_result for 'it', while target selects the striped pedestal; target " + "must not use kind=none. Use target kind=step_result only when the " + "reference object itself is exactly the result of a prior step.\n" + ) + + +def _intent_capability_catalog() -> dict[str, dict[str, Any]]: + """Return the LLM's thin, import-safe E1-E9 capability view. + + Action Engine's online planning catalog also reports runtime availability + and therefore imports simulator action classes. Text interpretation only + needs symbolic E semantics and must remain testable before a simulator + backend is installed. + """ + return { + task_type: { + "semantics": contract.semantics, + "applicable_fields": sorted(_INTENT_TASK_FIELD_REGISTRY[task_type]), + } + for task_type, contract in TASK_CONTRACTS.items() + } + + +def _default_instruction_caller( + *, + prompt: str, + schema: Mapping[str, Any], + model: str | None, +) -> Mapping[str, Any]: + from langchain_core.messages import HumanMessage, SystemMessage + from langchain_openai import ChatOpenAI + + settings = _load_llm_settings(model=model) + kwargs: dict[str, Any] = { + "api_key": settings["api_key"], + "model": settings["model"], + "temperature": 0, + "http_socket_options": (), + } + for key in ("base_url", "default_query"): + if settings[key]: + kwargs[key] = settings[key] + if _is_mimo_compatible(settings): + # MiMo documents ``thinking`` as a provider extension carried in the + # OpenAI client's extra body. Disabling it is important here: hidden + # reasoning can consume the completion and leave only id/object/type. + kwargs.update( + { + "max_completion_tokens": _MIMO_MAX_COMPLETION_TOKENS, + "extra_body": {"thinking": {"type": "disabled"}}, + } + ) + client = ChatOpenAI(**kwargs) + # The full schema remains in the prompt and the local validator is still + # authoritative even when the provider only offers JSON mode. + structured = _structured_output_runnable( + client, + schema, + settings=settings, + ) + schema_prompt = ( + f"{prompt}\n\nReturn one JSON object conforming exactly to this JSON " + f"Schema:\n{json.dumps(schema, ensure_ascii=False, sort_keys=True)}" + ) + response = structured.invoke( + [ + SystemMessage( + content=( + "Return only the requested structured JSON response. Never " + "return reasoning, coordinates, or AtomicAction nodes." + ) + ), + HumanMessage(content=schema_prompt), + ] + ) + return _coerce_instruction_response(response) + + +def _instruction_model(explicit: str | None) -> str | None: + if isinstance(explicit, str) and explicit.strip(): + return explicit.strip() + # Keep model selection separate from credential loading. Reading the local + # dotenv file is side-effect free and gives generation the documented + # priority without leaking credentials into TaskSpec metadata. + for name in ("TASK_ENGINE_LLM_MODEL", "ACTION_ENGINE_LLM_MODEL", "OPENAI_MODEL"): + for source in ( + os.environ, + _load_local_env(), + ): + value = source.get(name) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _load_local_env() -> dict[str, str]: + """Read Task Engine model configuration without mutating the environment.""" + return _load_env_file(_GEN_SIM_ENV_PATH) + + +def _is_mimo_compatible(settings: Mapping[str, Any]) -> bool: + model = str(settings.get("model", "")).casefold() + base_url = str(settings.get("base_url", "")).casefold() + return "mimo" in model or "xiaomimimo.com" in base_url + + +def _structured_output_runnable( + client: Any, + schema: Mapping[str, Any], + *, + settings: Mapping[str, Any], +) -> Any: + if not hasattr(client, "with_structured_output"): + return client + method = "json_mode" if _is_mimo_compatible(settings) else "json_schema" + try: + return client.with_structured_output(schema, method=method) + except (TypeError, ValueError): + if method == "json_mode" and hasattr(client, "bind"): + from langchain_core.output_parsers import JsonOutputParser + + return ( + client.bind(response_format={"type": "json_object"}) + | JsonOutputParser() + ) + return client.with_structured_output(schema) + + +def _load_llm_settings(*, model: str | None) -> dict[str, Any]: + local_env = _load_local_env() + config: dict[str, Any] = {} + if _GEN_CONFIG_PATH.is_file(): + raw = json.loads(_GEN_CONFIG_PATH.read_text(encoding="utf-8")) + if isinstance(raw, Mapping): + llm = raw.get("llm", {}) + if isinstance(llm, Mapping): + configured = llm.get("openai_compatible", {}) + if isinstance(configured, Mapping): + config = dict(configured) + api_key, base_url = _resolve_transport_settings(local_env, config) + selected_model = ( + (model.strip() if isinstance(model, str) else "") + or _first_env_value( + local_env, + "TASK_ENGINE_LLM_MODEL", + "ACTION_ENGINE_LLM_MODEL", + "OPENAI_MODEL", + "LLM_MODEL", + ) + or str(config.get("model", "")).strip() + ) + default_query = config.get("default_query", {}) or {} + if not api_key: + raise ValueError( + "OPENAI_API_KEY is required for Task Engine interpretation. Set it " + f"in the process environment or {_GEN_SIM_ENV_PATH}." + ) + if not selected_model: + raise ValueError( + "A text LLM model is required through model=, TASK_ENGINE_LLM_MODEL, " + f"OPENAI_MODEL, or {_GEN_CONFIG_PATH}." + ) + if not isinstance(default_query, Mapping): + raise ValueError("LLM default_query must be a mapping.") + return { + "api_key": api_key, + "model": selected_model, + "base_url": base_url, + "default_query": dict(default_query), + } + + +def _resolve_transport_settings( + local_env: Mapping[str, str], + config: Mapping[str, Any], +) -> tuple[str, str]: + """Resolve an API key and endpoint from one configuration source.""" + transports = ( + ( + _mapping_value(os.environ, "OPENAI_API_KEY"), + _mapping_value( + os.environ, + "OPENAI_BASE_URL", + "OPENAI_API_BASE", + "LLM_URL", + ), + ), + ( + _mapping_value(local_env, "OPENAI_API_KEY"), + _mapping_value( + local_env, + "OPENAI_BASE_URL", + "OPENAI_API_BASE", + "LLM_URL", + ), + ), + ( + _mapping_value(config, "api_key"), + _mapping_value(config, "base_url"), + ), + ) + for api_key, base_url in transports: + if api_key and base_url: + return api_key, base_url.rstrip("/") + for api_key, base_url in transports: + if api_key: + return api_key, base_url.rstrip("/") + return "", "" + + +def _mapping_value(source: Mapping[str, Any], *names: str) -> str: + for name in names: + value = source.get(name) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + +def _load_env_file(path: Path) -> dict[str, str]: + if not path.is_file(): + return {} + values: dict[str, str] = {} + for line_number, raw_line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export "): + line = line[len("export ") :].lstrip() + if "=" not in line: + continue + key, raw_value = line.split("=", 1) + key = key.strip() + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + raise ValueError(f"Invalid dotenv key at {path}:{line_number}.") + value = raw_value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + elif " #" in value: + value = value.split(" #", 1)[0].rstrip() + values[key] = value + return values + + +def _first_env_value(local_env: Mapping[str, str], *names: str) -> str | None: + for source in (os.environ, local_env): + for name in names: + value = source.get(name) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _choice(value: Any, allowed: set[str] | frozenset[str], context: str) -> str: + if not isinstance(value, str) or value not in allowed: + raise ValueError(f"{context} must be one of {sorted(allowed)}.") + return value + + +def _selector_string(value: Any, context: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{context} must be a string.") + return value.strip() + + +def _canonical_quantifier(value: Any, context: str) -> str: + return _choice(value, _QUANTIFIERS, context) + + +def _canonical_arm(value: Any, context: str) -> str: + return _choice(value, _ARMS, context) + + +def _canonical_relation(value: Any, context: str) -> str: + return _choice(value, _RELATIONS, context) + + +def _canonical_orientation(value: Any, context: str) -> str: + return _choice(value, _ORIENTATIONS, context) + + +def _nonempty(value: Any, context: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{context} must be a non-empty string.") + return value.strip() + + +def _topological_steps(steps: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + """Return a stable topological ordering for validated intent steps.""" + by_id = {str(step["id"]): dict(step) for step in steps} + effective_dependencies: dict[str, tuple[str, ...]] = {} + for step_id, step in by_id.items(): + deps = list(str(dep) for dep in step["depends_on"]) + for selector in (step["object"], step["target"]): + if selector["kind"] == "step_result": + reference = str(selector["step_id"]) + if reference not in deps: + deps.append(reference) + effective_dependencies[step_id] = tuple(deps) + pending = set(by_id) + ordered: list[dict[str, Any]] = [] + original = [str(step["id"]) for step in steps] + while pending: + ready = [ + step_id + for step_id in original + if step_id in pending + and all(str(dep) not in pending for dep in effective_dependencies[step_id]) + ] + if not ready: + raise ValueError("Instruction intent dependencies contain a cycle.") + for step_id in ready: + ordered.append(by_id[step_id]) + pending.remove(step_id) + return ordered + + +def _coerce_instruction_response(response: Any) -> Mapping[str, Any]: + """Coerce common structured-client response wrappers without accepting prose.""" + if isinstance(response, Mapping): + return dict(response) + model_dump = getattr(response, "model_dump", None) + if callable(model_dump): + dumped = model_dump() + if isinstance(dumped, Mapping): + return dict(dumped) + content = getattr(response, "content", response) + if isinstance(content, Mapping): + return dict(content) + if isinstance(content, list): + content = "\n".join( + str(item.get("text", "")) + for item in content + if isinstance(item, Mapping) and item.get("type") == "text" + ) + if not isinstance(content, str): + raise ValueError( + f"Instruction model output has unsupported type {type(content).__name__}." + ) + text = content.strip() + if text.startswith("```"): + lines = text.splitlines() + if lines: + lines = lines[1:] + if lines and lines[-1].strip().startswith("```"): + lines = lines[:-1] + text = "\n".join(lines).strip() + try: + parsed = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError(f"Instruction model output is not valid JSON: {exc}") from exc + if not isinstance(parsed, Mapping): + raise ValueError("Instruction model output must decode to a JSON object.") + return dict(parsed) + + +def _validate_dag(dependencies: Mapping[str, Sequence[str]]) -> None: + visiting: set[str] = set() + visited: set[str] = set() + + def visit(node: str) -> None: + if node in visiting: + raise ValueError("Instruction intent dependencies contain a cycle.") + if node in visited: + return + visiting.add(node) + for dependency in dependencies[node]: + visit(str(dependency)) + visiting.remove(node) + visited.add(node) + + for node in dependencies: + visit(node) + + +def _reject_forbidden_fields(value: Any) -> None: + if isinstance(value, Mapping): + forbidden = _FORBIDDEN_FIELDS & {str(key).strip().lower() for key in value} + if forbidden: + raise ValueError( + f"Instruction intent contains forbidden fields {sorted(forbidden)}." + ) + for item in value.values(): + _reject_forbidden_fields(item) + elif isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + for item in value: + _reject_forbidden_fields(item) diff --git a/tests/gen_sim/action_engine/task_fixtures.py b/tests/gen_sim/action_engine/task_fixtures.py new file mode 100644 index 000000000..c845cdfab --- /dev/null +++ b/tests/gen_sim/action_engine/task_fixtures.py @@ -0,0 +1,229 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Language-neutral structured fixtures for Action Engine tests.""" + +from __future__ import annotations + +from typing import Any + +from embodichain.gen_sim.action_engine.domain import ( + task_success_type, + validate_scene_requirements, + validate_task_spec, +) +from embodichain.gen_sim.action_engine.protocol import ( + SCENE_REQUIREMENTS_SCHEMA, + TASK_SPEC_SCHEMA, +) + +__all__ = ["make_task_level", "make_task_spec"] + +_OBJECT_FIXTURES = { + "E1": ("can", ["graspable", "placeable"], {}), + "E2": ("can", ["graspable", "orientable"], {"orientation": "fallen"}), + "E3": ("container", ["graspable", "pourable"], {"held_by": "left_arm"}), + "E4": ("cup", ["graspable", "handover"], {}), + "E5": ("tray", ["dual_graspable", "rigid"], {}), + "E6": ("drawer", ["articulated", "pullable"], {"joint_state": "closed"}), + "E7": ("drawer", ["articulated", "pushable"], {"joint_state": "open"}), + "E8": ("knob", ["turnable"], {}), + "E9": ("button", ["pressable"], {"activation": "inactive"}), +} + + +def make_task_spec( + task_type: str = "E1", + *, + task_id: str | None = None, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Build one validated L1 TaskSpec and matching scene requirements.""" + if task_type not in _OBJECT_FIXTURES: + raise ValueError(f"Unsupported fixture task type {task_type!r}.") + category, affordances, initial_state = _OBJECT_FIXTURES[task_type] + object_role = "object_01" + params: dict[str, Any] = {"object_role": object_role} + objects = [ + { + "role_id": object_role, + "category": category, + "count": 1, + "affordances": affordances, + "initial_state": initial_state, + "attributes": {}, + } + ] + if task_type in {"E1", "E3"}: + target_role = "target_01" + params.update({"target_role": target_role, "relation": "inside"}) + if task_type == "E3": + params["source_role"] = params.pop("object_role") + objects.append( + { + "role_id": target_role, + "category": "container", + "count": 1, + "affordances": ["container", "support_surface"], + "initial_state": {}, + "attributes": {}, + } + ) + elif task_type == "E2": + params.update( + { + "orientation_goal": "upright", + "support_role": "table", + "upright_local_axis": "long_axis", + } + ) + elif task_type == "E4": + params.update( + { + "transfer_arm": "left_arm", + "receive_arm": "right_arm", + "orientation_goal": "none", + } + ) + elif task_type == "E5": + params.update({"direction": "up", "terminal_behavior": "hold"}) + elif task_type == "E6": + params["target_state"] = "open" + elif task_type == "E7": + params["target_state"] = "closed" + elif task_type == "E8": + params["target_setting"] = 2 + elif task_type == "E9": + params["target_state"] = "activated" + + effective_id = task_id or f"fixture-{task_type.lower()}" + task = validate_task_spec( + { + "schema_version": TASK_SPEC_SCHEMA, + "task_id": effective_id, + "level": "L1", + "instruction": "test-instruction", + "reasoning_type": "none", + "task_instances": [ + { + "id": "task_01", + "task_type": task_type, + "params": params, + "depends_on": [], + "role": "primary", + } + ], + "success": { + "type": task_success_type(task_type, params), + "task_instance_id": "task_01", + }, + "oracle": {}, + "metadata": {"fixture": True}, + } + ) + requirements = validate_scene_requirements( + { + "schema_version": SCENE_REQUIREMENTS_SCHEMA, + "task_id": effective_id, + "objects": objects, + "cameras": [], + "spatial_constraints": [ + {"type": "reachable", "roles": "all_interaction_objects"} + ], + "distractor_count": 0, + "metadata": {"fixture": True}, + } + ) + return task, requirements + + +def make_task_level( + level: str, + *, + reasoning: str | None = None, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Build a validated fixture for one public TaskSpec level.""" + if level == "L1": + return make_task_spec("E1") + first, requirements = make_task_spec("E1", task_id=f"fixture-{level.lower()}") + if level == "L2": + second = { + "id": "task_02", + "task_type": "E1", + "params": { + "object_role": "object_02", + "target_role": "target_02", + "relation": "inside", + }, + "depends_on": ["task_01"], + "role": "primary", + } + first["level"] = "L2" + first["task_instances"].append(second) + first["success"] = { + "op": "all", + "terms": [ + {"type": "semantic_goal", "task_instance_id": "task_01"}, + {"type": "semantic_goal", "task_instance_id": "task_02"}, + ], + } + requirements["objects"].extend( + [ + { + "role_id": "object_02", + "category": "can", + "count": 1, + "affordances": ["graspable", "placeable"], + "initial_state": {}, + "attributes": {}, + }, + { + "role_id": "target_02", + "category": "container", + "count": 1, + "affordances": ["container", "support_surface"], + "initial_state": {}, + "attributes": {}, + }, + ] + ) + return validate_task_spec(first), validate_scene_requirements(requirements) + if level == "L4": + first["level"] = "L4" + first["reasoning_type"] = reasoning or "visual_semantics" + first["success"] = { + "visual_semantics": { + "type": "visual_relation", + "relation": "mouth_completed", + }, + "pattern": { + "type": "visual_relation", + "relation": "pattern_completed", + }, + "logic": {"type": "sum_equals", "value": 5}, + "memory": {"type": "original_order_restored"}, + "common_sense": {"type": "functional_place_setting"}, + "constraint": {"type": "stable_unobstructed"}, + }[first["reasoning_type"]] + first["oracle"] = {"fixture": True} + requirements["cameras"] = [ + { + "role": "reasoning_view", + "modalities": ["rgb", "depth"], + "coverage": "all_interaction_objects", + } + ] + return validate_task_spec(first), validate_scene_requirements(requirements) + raise ValueError(f"Unsupported fixture task level {level!r}.") diff --git a/tests/gen_sim/action_engine/tasks/__init__.py b/tests/gen_sim/action_engine/tasks/__init__.py new file mode 100644 index 000000000..d9480994f --- /dev/null +++ b/tests/gen_sim/action_engine/tasks/__init__.py @@ -0,0 +1,19 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +from __future__ import annotations + +"""Action Engine task generation tests.""" diff --git a/tests/gen_sim/action_engine/tasks/test_grounding.py b/tests/gen_sim/action_engine/tasks/test_grounding.py new file mode 100644 index 000000000..e927c5fb6 --- /dev/null +++ b/tests/gen_sim/action_engine/tasks/test_grounding.py @@ -0,0 +1,339 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +from __future__ import annotations + +from copy import deepcopy +import json + +import pytest + +from embodichain.gen_sim.action_engine.tasks.grounding import ( + ground_scene_references, +) +from embodichain.gen_sim.action_engine.tasks.assembly import SceneInventory + + +def _selector( + reference: str, + *, + quantifier: str = "one", + count: int = 0, +) -> dict: + return { + "kind": "scene_ref", + "step_id": "", + "reference": reference, + "quantifier": quantifier, + "count": count, + } + + +def _scene() -> list[dict]: + return [ + { + "runtime_uid": "table", + "uid": "table", + "role": "background", + "category": "dining_table", + "name": "work table", + "description": "A rectangular work table.", + "init_pos": [0.0, 0.0, 0.0], + }, + { + "runtime_uid": "cutting_board", + "uid": "cutting_board", + "role": "rigid_object", + "category": "cutting_board", + "name": "wood board", + "description": "A large rectangular wooden cutting board.", + "attributes": { + "size": "large", + "geometry": {"position": [0.0, 0.2, 0.7], "note": "flat"}, + }, + "initial_state": {"orientation": "fallen"}, + "init_pos": [0.0, 0.2, 0.7], + }, + { + "runtime_uid": "salt_shaker", + "uid": "salt_shaker", + "role": "rigid_object", + "category": "salt_shaker", + "description": "A small glass salt shaker.", + "affordances": ["graspable"], + "init_pos": [0.0, -0.2, 0.7], + }, + ] + + +def _intent( + *, + object_selector: dict | None = None, + target_selector: dict | None = None, +) -> dict: + return { + "steps": [ + { + "id": "move", + "task_type": "E1", + "object": object_selector or _selector("object-alpha"), + "target": target_selector or _selector("target-alpha"), + "relation": "on", + } + ] + } + + +def _binding( + reference_id: str, + uids: list[str], + *, + status: str = "resolved", + confidence: float = 1.0, + **extra: object, +) -> dict: + return { + "reference_id": reference_id, + "status": status, + "uids": uids, + "confidence": confidence, + **extra, + } + + +def _run(intent: dict, caller) -> object: + scene = _scene() + return ground_scene_references( + instruction="test-instruction", + intent=intent, + inventory=SceneInventory(scene, robot_profile="franka"), + scene_objects=scene, + model="test-model", + caller=caller, + ) + + +def test_grounding_prompt_preserves_open_semantics_and_redacts_geometry() -> None: + captured: dict[str, object] = {} + + def caller(**kwargs): + captured.update(kwargs) + return { + "bindings": [ + _binding("move.object", ["cutting_board"]), + _binding("move.target", ["table"]), + ] + } + + result = _run(_intent(), caller) + + prompt = str(captured["prompt"]) + assert result.bindings == { + "move.object": ("cutting_board",), + "move.target": ("table",), + } + assert '"category": "cutting_board"' in prompt + assert '"category": "salt_shaker"' in prompt + assert '"name": "wood board"' in prompt + assert '"orientation": "fallen"' in prompt + assert '"size": "large"' in prompt + prompt_inventory = json.loads(prompt.split("Redacted scene inventory:\n", 1)[1]) + side_by_uid = {item["uid"]: item["side"] for item in prompt_inventory} + assert side_by_uid["cutting_board"] == "right" + assert side_by_uid["salt_shaker"] == "left" + assert '"position"' not in prompt + assert '"init_pos"' not in prompt + + +@pytest.mark.parametrize("robot_profile", ["ur5", "ur10", "franka"]) +def test_scene_inventory_uses_the_shared_final_world_lateral_axis( + robot_profile: str, +) -> None: + inventory = SceneInventory(_scene(), robot_profile=robot_profile) + + assert inventory.left_score(inventory.by_uid["salt_shaker"]) > 0.0 + assert inventory.left_score(inventory.by_uid["cutting_board"]) < 0.0 + + +def test_grounding_repairs_one_invalid_uid_in_the_same_batch() -> None: + responses = [ + { + "bindings": [ + _binding("move.object", ["invented"]), + _binding("move.target", ["table"]), + ] + }, + { + "bindings": [ + _binding("move.object", ["cutting_board"]), + _binding("move.target", ["table"]), + ] + }, + ] + prompts: list[str] = [] + + def caller(**kwargs): + prompts.append(kwargs["prompt"]) + return deepcopy(responses[len(prompts) - 1]) + + result = _run(_intent(), caller) + + assert result.attempts == 2 + assert "previous grounding JSON failed" in prompts[1] + assert result.bindings["move.object"] == ("cutting_board",) + + +@pytest.mark.parametrize( + "response,error", + [ + ( + { + "bindings": [ + _binding( + "move.object", + ["cutting_board"], + status="ambiguous", + ), + _binding("move.target", ["table"]), + ] + }, + "was not resolved", + ), + ( + { + "bindings": [ + _binding( + "move.object", + [], + status="not_found", + confidence=0.0, + ), + _binding("move.target", ["table"]), + ] + }, + "was not resolved", + ), + ( + { + "bindings": [ + _binding("move.object", ["cutting_board"], confidence=0.49), + _binding("move.target", ["table"]), + ] + }, + "confidence is below", + ), + ( + { + "bindings": [ + _binding("move.object", ["cutting_board", "cutting_board"]), + _binding("move.target", ["table"]), + ] + }, + "duplicate UIDs", + ), + ( + { + "bindings": [ + _binding("move.object", ["cutting_board"]), + _binding("move.object", ["salt_shaker"]), + _binding("move.target", ["table"]), + ] + }, + "Duplicate grounding binding", + ), + ( + { + "bindings": [ + _binding("move.object", ["cutting_board", "salt_shaker"]), + _binding("move.target", ["table"]), + ] + }, + "quantifier=one requires exactly one UID", + ), + ( + {"bindings": [_binding("move.object", ["cutting_board"])]}, + "omitted requests", + ), + ( + { + "bindings": [ + _binding("move.object", ["table"]), + _binding("move.target", ["cutting_board"]), + ] + }, + "candidate range", + ), + ( + { + "bindings": [ + _binding("move.object", ["cutting_board"]), + _binding("move.target", ["cutting_board"]), + ] + }, + "same UID", + ), + ( + { + "bindings": [ + _binding( + "move.object", + ["cutting_board"], + affordances=["graspable"], + ), + _binding("move.target", ["table"]), + ] + }, + "unsupported", + ), + ], +) +def test_grounding_fails_closed_after_one_repair(response: dict, error: str) -> None: + with pytest.raises(ValueError, match=f"after one repair.*{error}"): + _run(_intent(), lambda **_kwargs: deepcopy(response)) + + +def test_grounding_enforces_count_and_accepts_an_open_world_set() -> None: + intent = _intent( + object_selector=_selector("object-set", quantifier="count", count=2) + ) + response = { + "bindings": [ + _binding("move.object", ["cutting_board", "salt_shaker"]), + _binding("move.target", ["table"]), + ] + } + + result = _run(intent, lambda **_kwargs: response) + assert result.bindings["move.object"] == ("cutting_board", "salt_shaker") + + invalid = deepcopy(response) + invalid["bindings"][0]["uids"] = ["cutting_board"] + with pytest.raises(ValueError, match="requires exactly 2 UIDs"): + _run(intent, lambda **_kwargs: invalid) + + +def test_grounding_accepts_a_nonempty_all_binding() -> None: + intent = _intent(object_selector=_selector("object-set", quantifier="all")) + response = { + "bindings": [ + _binding("move.object", ["cutting_board", "salt_shaker"]), + _binding("move.target", ["table"]), + ] + } + + result = _run(intent, lambda **_kwargs: response) + + assert result.bindings["move.object"] == ("cutting_board", "salt_shaker") diff --git a/tests/gen_sim/task_engine/__init__.py b/tests/gen_sim/task_engine/__init__.py new file mode 100644 index 000000000..b201491d8 --- /dev/null +++ b/tests/gen_sim/task_engine/__init__.py @@ -0,0 +1,19 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +"""Tests for Task Engine semantics and orchestration.""" + +from __future__ import annotations diff --git a/tests/gen_sim/task_engine/test_interpretation.py b/tests/gen_sim/task_engine/test_interpretation.py new file mode 100644 index 000000000..009362e87 --- /dev/null +++ b/tests/gen_sim/task_engine/test_interpretation.py @@ -0,0 +1,96 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2021-2026 DexForce Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------- + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from embodichain.gen_sim.task_engine import interpretation as interpretation_module + + +def _write_dotenv(path: Path) -> None: + path.write_text( + "\n".join( + ( + "OPENAI_API_KEY=dotenv-key", + "OPENAI_BASE_URL=https://dotenv.example/v1", + "OPENAI_MODEL=dotenv-model", + ) + ), + encoding="utf-8", + ) + + +def _clear_process_provider(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_API_BASE", + "LLM_URL", + "TASK_ENGINE_LLM_MODEL", + "ACTION_ENGINE_LLM_MODEL", + "OPENAI_MODEL", + "LLM_MODEL", + ): + monkeypatch.delenv(name, raising=False) + + +def test_partial_process_transport_does_not_mix_with_dotenv( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + env_path = tmp_path / ".env" + _write_dotenv(env_path) + monkeypatch.setattr(interpretation_module, "_GEN_SIM_ENV_PATH", env_path) + monkeypatch.setattr( + interpretation_module, + "_GEN_CONFIG_PATH", + tmp_path / "missing.json", + ) + _clear_process_provider(monkeypatch) + monkeypatch.setenv("OPENAI_API_KEY", "unrelated-process-key") + + settings = interpretation_module._load_llm_settings(model=None) + + assert settings["api_key"] == "dotenv-key" + assert settings["base_url"] == "https://dotenv.example/v1" + assert settings["model"] == "dotenv-model" + + +def test_complete_process_transport_overrides_dotenv( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + env_path = tmp_path / ".env" + _write_dotenv(env_path) + monkeypatch.setattr(interpretation_module, "_GEN_SIM_ENV_PATH", env_path) + monkeypatch.setattr( + interpretation_module, + "_GEN_CONFIG_PATH", + tmp_path / "missing.json", + ) + _clear_process_provider(monkeypatch) + monkeypatch.setenv("OPENAI_API_KEY", "process-key") + monkeypatch.setenv("OPENAI_BASE_URL", "https://process.example/v1/") + monkeypatch.setenv("TASK_ENGINE_LLM_MODEL", "process-model") + + settings = interpretation_module._load_llm_settings(model=None) + + assert settings["api_key"] == "process-key" + assert settings["base_url"] == "https://process.example/v1" + assert settings["model"] == "process-model"