Python: recursively serialize nested container values

python/packages/core/agent_framework/_serialization.py

@@ -9,7 +9,7 @@
 import re
 from collections.abc import Mapping, MutableMapping
 from dataclasses import asdict, is_dataclass
-from datetime import date, datetime
+from datetime import date, datetime, time
 from functools import lru_cache
 from typing import Any, ClassVar, Protocol, TypeGuard, TypeVar, cast, runtime_checkable
 
@@ -19,6 +19,7 @@
 ProtocolT = TypeVar("ProtocolT", bound="SerializationProtocol")
 _JSON_SCALAR_TYPES = (str, int, float, bool, type(None))
 _DIRECT_JSON_TYPES = (*_JSON_SCALAR_TYPES, list, dict)
+_SKIP_SERIALIZATION = object()
 
 # Regex pattern for converting CamelCase to snake_case
 _CAMEL_TO_SNAKE_PATTERN = re.compile(r"(?<!^)(?=[A-Z])")
@@ -151,6 +152,81 @@ def _is_serialization_protocol(value: Any) -> TypeGuard[SerializationProtocol]:
     return callable(getattr(value, "to_dict", None)) and callable(getattr(value, "from_dict", None))
 
 
+def _serialize_value(
+    value: Any,
+    *,
+    exclude: set[str] | None,
+    exclude_none: bool,
+    attribute_name: str,
+    active_container_ids: set[int] | None = None,
+    stringify_dict_keys: bool = False,
+) -> Any:
+    """Recursively serialize a value while preserving skip semantics."""
+    if active_container_ids is None:
+        active_container_ids = set()
+    if type(value) in _JSON_SCALAR_TYPES:
+        return value
+    if _is_serialization_protocol(value):
+        serialized = value.to_dict(exclude=exclude, exclude_none=exclude_none)
+        return _serialize_value(
+            serialized,
+            exclude=exclude,
+            exclude_none=exclude_none,
+            attribute_name=attribute_name,
+            active_container_ids=active_container_ids,
+        )
+    if isinstance(value, list):
+        value_as_list = cast(list[Any], value)
+        container_id = id(value_as_list)
+        if container_id in active_container_ids:
+            raise ValueError("Circular reference detected")
+        active_container_ids.add(container_id)
+        try:
+            serialized_list: list[Any] = []
+            for item in value_as_list:
+                serialized = _serialize_value(
+                    item,
+                    exclude=exclude,
+                    exclude_none=exclude_none,
+                    attribute_name=attribute_name,
+                    active_container_ids=active_container_ids,
+                )
+                if serialized is not _SKIP_SERIALIZATION:
+                    serialized_list.append(serialized)
+            return serialized_list
+        finally:
+            active_container_ids.remove(container_id)
+    if isinstance(value, dict):
+        value_as_dict = cast(dict[Any, Any], value)
+        container_id = id(value_as_dict)
+        if container_id in active_container_ids:
+            raise ValueError("Circular reference detected")
+        active_container_ids.add(container_id)
+        try:
+            serialized_dict: dict[Any, Any] = {}
+            for raw_key, item in value_as_dict.items():
+                dict_key = str(raw_key) if stringify_dict_keys else raw_key
+                if isinstance(item, (datetime, date, time)):
+                    serialized_dict[dict_key] = str(item)
+                    continue
+                serialized = _serialize_value(
+                    item,
+                    exclude=exclude,
+                    exclude_none=exclude_none,
+                    attribute_name=attribute_name,
+                    active_container_ids=active_container_ids,
+                )
+                if serialized is not _SKIP_SERIALIZATION:
+                    serialized_dict[dict_key] = serialized
+            return serialized_dict
+        finally:
+            active_container_ids.remove(container_id)
+    if is_serializable(value):
+        return value
+    logger.debug(f"Skipping non-serializable value in attribute '{attribute_name}' of type {type(value).__name__}")
+    return _SKIP_SERIALIZATION
+
+
 class SerializationMixin:
     """Mixin class providing comprehensive serialization and deserialization capabilities.
 
@@ -335,63 +411,15 @@ def to_dict(self, *, exclude: set[str] | None = None, exclude_none: bool = True)
             if key not in combined_exclude and not key.startswith("_"):
                 if exclude_none and value is None:
                     continue
-                if type(value) in _JSON_SCALAR_TYPES:
-                    result[key] = value
-                    continue
-                # Recursively serialize SerializationProtocol objects
-                if _is_serialization_protocol(value):
-                    result[key] = value.to_dict(exclude=exclude, exclude_none=exclude_none)
-                    continue
-                # Handle lists containing SerializationProtocol objects
-                if isinstance(value, list):
-                    value_as_list: list[Any] = []
-                    for item in cast(list[Any], value):
-                        if type(item) in _JSON_SCALAR_TYPES:
-                            value_as_list.append(item)
-                            continue
-                        if _is_serialization_protocol(item):
-                            value_as_list.append(item.to_dict(exclude=exclude, exclude_none=exclude_none))
-                            continue
-                        if is_serializable(item):
-                            value_as_list.append(item)
-                            continue
-                        logger.debug(
-                            f"Skipping non-serializable item in list attribute '{key}' of type {type(item).__name__}"
-                        )
-                    result[key] = value_as_list
-                    continue
-                # Handle dicts containing SerializationProtocol values
-                if isinstance(value, dict):
-                    from datetime import date, datetime, time
-
-                    serialized_dict: dict[str, Any] = {}
-                    for raw_key, v in cast(dict[Any, Any], value).items():
-                        dict_key = str(raw_key)
-                        # Convert datetime objects to strings
-                        if isinstance(v, (datetime, date, time)):
-                            serialized_dict[dict_key] = str(v)
-                            continue
-                        if type(v) in _JSON_SCALAR_TYPES:
-                            serialized_dict[dict_key] = v
-                            continue
-                        if _is_serialization_protocol(v):
-                            serialized_dict[dict_key] = v.to_dict(exclude=exclude, exclude_none=exclude_none)
-                            continue
-                        # Check if the value is JSON serializable
-                        if is_serializable(v):
-                            serialized_dict[dict_key] = v
-                            continue
-                        logger.debug(
-                            f"Skipping non-serializable value for key '{dict_key}' in dict attribute '{key}' "
-                            f"of type {type(v).__name__}"
-                        )
-                    result[key] = serialized_dict
-                    continue
-                # Directly include JSON serializable values
-                if is_serializable(value):
-                    result[key] = value
-                    continue
-                logger.debug(f"Skipping non-serializable attribute '{key}' of type {type(value).__name__}")
+                serialized = _serialize_value(
+                    value,
+                    exclude=exclude,
+                    exclude_none=exclude_none,
+                    attribute_name=key,
+                    stringify_dict_keys=isinstance(value, dict),
+                )
+                if serialized is not _SKIP_SERIALIZATION:
+                    result[key] = serialized
 
         return result
 

python/packages/core/tests/core/test_serializable_mixin.py

@@ -2,7 +2,9 @@
 
 """Tests for SerializationMixin functionality."""
 
+import json
 import logging
+from datetime import date, datetime, time
 from typing import Any
 
 import pytest
@@ -303,19 +305,96 @@ def __init__(self, items_dict: dict):
         assert data["items_dict"]["a"]["name"] == "item1"
         assert data["items_dict"]["b"]["name"] == "item2"
 
-    def test_to_dict_with_datetime_in_dict(self):
-        """Test to_dict converts datetime objects in dicts to strings."""
-        from datetime import datetime
+    def test_to_dict_recursively_serializes_nested_containers(self):
+        """Test to_dict serializes protocol objects nested in containers."""
+
+        class ItemClass(SerializationMixin):
+            def __init__(self, name: str):
+                self.name = name
+
+        class ContainerClass(SerializationMixin):
+            def __init__(self, payload: dict):
+                self.payload = payload
+
+        container = ContainerClass(payload={"groups": [{"items": [ItemClass(name="item1")]}]})
+
+        data = container.to_dict()
+
+        assert data["payload"]["groups"][0]["items"][0]["name"] == "item1"
+        assert json.loads(container.to_json()) == data
+
+    def test_to_dict_preserves_nested_non_string_dict_keys(self):
+        """Test recursive serialization preserves non-string keys below the attribute dictionary."""
+
+        class ContainerClass(SerializationMixin):
+            def __init__(self, payload: dict):
+                self.payload = payload
+
+        container = ContainerClass(payload={7: "direct", "nested": {True: "enabled", None: "missing"}})
+
+        data = container.to_dict()
+
+        assert data["payload"]["7"] == "direct"
+        assert data["payload"]["nested"] == {True: "enabled", None: "missing"}
+        json_payload = json.loads(container.to_json())["payload"]
+        assert json_payload["7"] == "direct"
+        assert json_payload["nested"] == {"true": "enabled", "null": "missing"}
+
+    def test_to_dict_rejects_circular_list(self):
+        """Test recursive serialization reports a controlled error for a circular list."""
+
+        class ContainerClass(SerializationMixin):
+            def __init__(self, payload: list[Any]):
+                self.payload = payload
+
+        payload: list[Any] = []
+        payload.append(payload)
+
+        with pytest.raises(ValueError, match="Circular reference detected"):
+            ContainerClass(payload).to_dict()
+
+    def test_to_dict_rejects_circular_dict(self):
+        """Test recursive serialization reports a controlled error for a circular dictionary."""
+
+        class ContainerClass(SerializationMixin):
+            def __init__(self, payload: dict[str, Any]):
+                self.payload = payload
+
+        payload: dict[str, Any] = {}
+        payload["self"] = payload
+
+        with pytest.raises(ValueError, match="Circular reference detected"):
+            ContainerClass(payload).to_dict()
+
+    def test_to_dict_skips_non_serializable_values_in_nested_containers(self):
+        """Test to_dict skips unsupported values at every nesting level."""
+
+        class NonSerializable:
+            pass
+
+        class ContainerClass(SerializationMixin):
+            def __init__(self, payload: dict):
+                self.payload = payload
+
+        container = ContainerClass(payload={"groups": [{"items": ["valid", NonSerializable()]}]})
+
+        assert container.to_dict()["payload"] == {"groups": [{"items": ["valid"]}]}
+
+    @pytest.mark.parametrize("value", [datetime(2025, 1, 27, 12), date(2025, 1, 27), time(12)])
+    def test_to_dict_only_converts_date_time_in_dict_values(self, value):
+        """Test to_dict preserves the existing date/time conversion contexts."""
 
         class TestClass(SerializationMixin):
-            def __init__(self, metadata: dict):
-                self.metadata = metadata
+            def __init__(self):
+                self.top_level = value
+                self.items = [value]
+                self.metadata = {"created_at": value}
 
-        now = datetime(2025, 1, 27, 12, 0, 0)
-        obj = TestClass(metadata={"created_at": now})
-        data = obj.to_dict()
+        data = TestClass().to_dict()
 
-        assert isinstance(data["metadata"]["created_at"], str)
+        assert "top_level" not in data
+        assert data["items"] == []
+        assert data["metadata"]["created_at"] == str(value)
 
     def test_to_dict_skips_non_serializable_in_dict(self, caplog):
         """Test to_dict skips non-serializable values in dicts with debug logging."""