Inheritance and abstract class fixes for DepthCamera (#963)

* make inheritance and abstract class fixes

* CI code cleanup

* renamed stereo to depth camera since most depth cameras are not stereo cameras

* bug fix

* fix mypy

Former-commit-id: 35ea42c [formerly 888999f]
Former-commit-id: 528c6d6

Author: alexlin2

dimos/agents/temp/webcam_agent.py

@@ -115,7 +115,7 @@ def main() -> None:
         ),
         hardware=lambda: Webcam(
             camera_index=0,
-            frequency=15,
+            fps=15,
             stereo_slice="left",
             camera_info=zed.CameraInfo.SingleWebcam,
         ),

dimos/hardware/sensors/camera/realsense/camera.py

@@ -28,14 +28,15 @@
 from dimos.core.transport import LCMTransport
 from dimos.hardware.sensors.camera.spec import (
     OPTICAL_ROTATION,
-    StereoCamera,
-    StereoCameraConfig,
+    DepthCameraConfig,
+    DepthCameraHardware,
 )
 from dimos.msgs.geometry_msgs import Quaternion, Transform, Vector3
 from dimos.msgs.sensor_msgs import CameraInfo
 from dimos.msgs.sensor_msgs.Image import Image, ImageFormat
 from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2
 from dimos.robot.foxglove_bridge import FoxgloveBridge
+from dimos.spec import perception
 from dimos.utils.reactive import backpressure
 
 
@@ -48,7 +49,7 @@ def default_base_transform() -> Transform:
 
 
 @dataclass
-class RealSenseCameraConfig(ModuleConfig, StereoCameraConfig):
+class RealSenseCameraConfig(ModuleConfig, DepthCameraConfig):
     width: int = 848
     height: int = 480
     fps: int = 15
@@ -63,7 +64,7 @@ class RealSenseCameraConfig(ModuleConfig, StereoCameraConfig):
     serial_number: str | None = None
 
 
-class RealSenseCamera(StereoCamera, Module):
+class RealSenseCamera(DepthCameraHardware, Module, perception.DepthCamera):
     color_image: Out[Image]
     depth_image: Out[Image]
     pointcloud: Out[PointCloud2]

dimos/hardware/sensors/camera/spec.py

@@ -27,6 +27,9 @@
 
 class CameraConfig(Protocol):
     frame_id_prefix: str | None
+    width: int
+    height: int
+    fps: int | float
 
 
 CameraConfigT = TypeVar("CameraConfigT", bound=CameraConfig)
@@ -43,12 +46,9 @@ def camera_info(self) -> CameraInfo:
         pass
 
 
-class StereoCameraConfig(Protocol):
-    """Protocol for stereo camera configuration."""
+class DepthCameraConfig(CameraConfig):
+    """Protocol for depth camera configuration."""
 
-    width: int
-    height: int
-    fps: int
     camera_name: str
     base_frame_id: str
     base_transform: Transform | None
@@ -59,8 +59,10 @@ class StereoCameraConfig(Protocol):
     camera_info_fps: float
 
 
-class StereoCamera(ABC):
-    """Abstract class for stereo camera modules (RealSense, ZED, etc.)."""
+class DepthCameraHardware(ABC):
+    """Abstract class for depth camera modules (RealSense, ZED, etc.)."""
+
+    config: DepthCameraConfig
 
     @abstractmethod
     def get_color_camera_info(self) -> CameraInfo | None:

dimos/hardware/sensors/camera/test_webcam.py

@@ -43,7 +43,7 @@ def test_streaming_single(dimos) -> None:
         ),
         hardware=lambda: Webcam(
             camera_index=0,
-            frequency=0.0,  # full speed but set something to test sharpness barrier
+            fps=0.0,  # full speed but set something to test sharpness barrier
             camera_info=zed.CameraInfo.SingleWebcam,
         ),
     )

dimos/hardware/sensors/camera/webcam.py

@@ -32,9 +32,9 @@
 @dataclass
 class WebcamConfig(CameraConfig):
     camera_index: int = 0  # /dev/videoN
-    frame_width: int = 640
-    frame_height: int = 480
-    frequency: int = 15
+    width: int = 640
+    height: int = 480
+    fps: float = 15.0
     camera_info: CameraInfo = field(default_factory=CameraInfo)
     frame_id_prefix: str | None = None
     stereo_slice: Literal["left", "right"] | None = None  # For stereo cameras
@@ -84,8 +84,8 @@ def start(self):  # type: ignore[no-untyped-def]
             raise RuntimeError(f"Failed to open camera {self.config.camera_index}")
 
         # Set camera properties
-        self._capture.set(cv2.CAP_PROP_FRAME_WIDTH, self.config.frame_width)  # type: ignore[attr-defined]
-        self._capture.set(cv2.CAP_PROP_FRAME_HEIGHT, self.config.frame_height)  # type: ignore[attr-defined]
+        self._capture.set(cv2.CAP_PROP_FRAME_WIDTH, self.config.width)  # type: ignore[attr-defined]
+        self._capture.set(cv2.CAP_PROP_FRAME_HEIGHT, self.config.height)  # type: ignore[attr-defined]
 
         # Clear stop event and start the capture thread
         self._stop_event.clear()
@@ -99,7 +99,8 @@ def stop(self) -> None:
 
         # Wait for thread to finish
         if self._capture_thread and self._capture_thread.is_alive():
-            self._capture_thread.join(timeout=(1.0 / self.config.frequency) + 0.1)
+            timeout = 0.1 if self.config.fps <= 0 else (1.0 / self.config.fps) + 0.1
+            self._capture_thread.join(timeout=timeout)
 
         # Release the capture
         if self._capture:
@@ -142,7 +143,7 @@ def capture_frame(self) -> Image:
 
     def _capture_loop(self) -> None:
         """Capture frames at the configured frequency"""
-        frame_interval = 1.0 / self.config.frequency
+        frame_interval = 0.0 if self.config.fps <= 0 else 1.0 / self.config.fps
         next_frame_time = time.time()
 
         while self._capture and not self._stop_event.is_set():
@@ -153,6 +154,8 @@ def _capture_loop(self) -> None:
                 self._observer.on_next(image)
 
             # Wait for next frame time or until stopped
+            if frame_interval <= 0:
+                continue
             next_frame_time += frame_interval
             sleep_time = next_frame_time - time.time()
             if sleep_time > 0:

dimos/hardware/sensors/camera/zed/camera.py

@@ -28,14 +28,15 @@
 from dimos.core.transport import LCMTransport
 from dimos.hardware.sensors.camera.spec import (
     OPTICAL_ROTATION,
-    StereoCamera,
-    StereoCameraConfig,
+    DepthCameraConfig,
+    DepthCameraHardware,
 )
 from dimos.msgs.geometry_msgs import Quaternion, Transform, Vector3
 from dimos.msgs.sensor_msgs import CameraInfo
 from dimos.msgs.sensor_msgs.Image import Image, ImageFormat
 from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2
 from dimos.robot.foxglove_bridge import FoxgloveBridge
+from dimos.spec import perception
 from dimos.utils.reactive import backpressure
 
 
@@ -48,7 +49,7 @@ def default_base_transform() -> Transform:
 
 
 @dataclass
-class ZEDCameraConfig(ModuleConfig, StereoCameraConfig):
+class ZEDCameraConfig(ModuleConfig, DepthCameraConfig):
     width: int = 1280
     height: int = 720
     fps: int = 15
@@ -73,7 +74,7 @@ class ZEDCameraConfig(ModuleConfig, StereoCameraConfig):
     world_frame: str = "world"
 
 
-class ZEDCamera(StereoCamera, Module):
+class ZEDCamera(DepthCameraHardware, Module, perception.DepthCamera):
     color_image: Out[Image]
     depth_image: Out[Image]
     pointcloud: Out[PointCloud2]
@@ -123,7 +124,7 @@ def __init__(self, *args, **kwargs) -> None:  # type: ignore[no-untyped-def]
         self._tracking_enabled = False
         self._stream_width = self.config.width
         self._stream_height = self.config.height
-        self._camera_info: sl.CameraInformation | None = None
+        self._sl_camera_info: sl.CameraInformation | None = None
 
     def _publish_camera_info(self) -> None:
         ts = time.time()
@@ -165,10 +166,10 @@ def start(self) -> None:
         self._depth_map = sl.Mat()
         self._pose = sl.Pose()
 
-        self._camera_info = self._zed.get_camera_information()
-        if self._camera_info is not None:
-            self._stream_width = self._camera_info.camera_configuration.resolution.width
-            self._stream_height = self._camera_info.camera_configuration.resolution.height
+        self._sl_camera_info = self._zed.get_camera_information()
+        if self._sl_camera_info is not None:
+            self._stream_width = self._sl_camera_info.camera_configuration.resolution.width
+            self._stream_height = self._sl_camera_info.camera_configuration.resolution.height
 
         self._build_camera_info()
         self._get_extrinsics()
@@ -198,9 +199,9 @@ def start(self) -> None:
             )
 
     def _build_camera_info(self) -> None:
-        if self._camera_info is None:
+        if self._sl_camera_info is None:
             return
-        calib = self._camera_info.camera_configuration.calibration_parameters
+        calib = self._sl_camera_info.camera_configuration.calibration_parameters
         left_cam = calib.left_cam
 
         self._color_camera_info = self._intrinsics_to_camera_info(
@@ -236,9 +237,9 @@ def _intrinsics_to_camera_info(
         )
 
     def _get_extrinsics(self) -> None:
-        if self._camera_info is None:
+        if self._sl_camera_info is None:
             return
-        sensors_config = self._camera_info.sensors_configuration
+        sensors_config = self._sl_camera_info.sensors_configuration
         # camera_imu_transform gives the transform from IMU (body center) to left camera
         self._camera_link_to_color_extrinsics = sensors_config.camera_imu_transform
 
@@ -470,7 +471,7 @@ def stop(self) -> None:
         self._image_left = None
         self._depth_map = None
         self._pose = None
-        self._camera_info = None
+        self._sl_camera_info = None
         self._tracking_enabled = False
         super().stop()
 

dimos/robot/unitree/g1/g1zed.py

@@ -56,7 +56,7 @@ def deploy_g1_monozed(dimos: DimosCluster) -> CameraModule:
             ),
             hardware=lambda: Webcam(
                 camera_index=0,
-                frequency=5,
+                fps=5,
                 stereo_slice="left",
                 camera_info=zed.CameraInfo.SingleWebcam,
             ),

dimos/robot/unitree_webrtc/unitree_g1_blueprints.py

@@ -75,7 +75,7 @@
             ),
             hardware=lambda: Webcam(
                 camera_index=0,
-                frequency=15,
+                fps=15,
                 stereo_slice="left",
                 camera_info=zed.CameraInfo.SingleWebcam,
             ),

dimos/spec/perception.py

@@ -27,5 +27,10 @@ class Camera(Image):
     _camera_info: CameraInfo
 
 
+class DepthCamera(Camera):
+    depth_image: Out[ImageMsg]
+    depth_camera_info: Out[CameraInfo]
+
+
 class Pointcloud(Protocol):
     pointcloud: Out[PointCloud2]