From 1f32952d0066a9dc1ff1482cef48c3cbe0acb663 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 17 Dec 2025 10:45:45 +0100 Subject: [PATCH 1/8] fix(ai): redact message parts content of type blob --- sentry_sdk/ai/utils.py | 51 +++++++++++++++++ tests/test_ai_monitoring.py | 106 +++++++++++++++++++++++++++++++++++- 2 files changed, 156 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index 1d2b4483c9..73155b0305 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -5,6 +5,8 @@ from sys import getsizeof from typing import TYPE_CHECKING +from sentry_sdk._types import SENSITIVE_DATA_SUBSTITUTE + if TYPE_CHECKING: from typing import Any, Callable, Dict, List, Optional, Tuple @@ -141,6 +143,53 @@ def _find_truncation_index(messages: "List[Dict[str, Any]]", max_bytes: int) -> return 0 +def redact_blob_message_parts(messages): + # type: (List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], int] + """ + Redact blob message parts from the messages, by removing the "content" key. + e.g: + { + "role": "user", + "content": [ + { + "text": "How many ponies do you see in the image?", + "type": "text" + }, + { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "data:image/jpeg;base64,..." + } + ] + } + becomes: + { + "role": "user", + "content": [ + { + "text": "How many ponies do you see in the image?", + "type": "text" + }, + { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "[Filtered]" + } + ] + } + """ + + for message in messages: + content = message.get("content") + if isinstance(content, list): + for item in content: + if item.get("type") == "blob": + item["content"] = SENSITIVE_DATA_SUBSTITUTE + return messages + + def truncate_messages_by_size( messages: "List[Dict[str, Any]]", max_bytes: int = MAX_GEN_AI_MESSAGE_BYTES, @@ -186,6 +235,8 @@ def truncate_and_annotate_messages( if not messages: return None + messages = redact_blob_message_parts(messages) + truncated_messages, removed_count = truncate_messages_by_size(messages, max_bytes) if removed_count > 0: scope._gen_ai_original_message_count[span.span_id] = len(messages) diff --git a/tests/test_ai_monitoring.py b/tests/test_ai_monitoring.py index 8d3d4ba204..e9f3712cd3 100644 --- a/tests/test_ai_monitoring.py +++ b/tests/test_ai_monitoring.py @@ -4,7 +4,7 @@ import pytest import sentry_sdk -from sentry_sdk._types import AnnotatedValue +from sentry_sdk._types import AnnotatedValue, SENSITIVE_DATA_SUBSTITUTE from sentry_sdk.ai.monitoring import ai_track from sentry_sdk.ai.utils import ( MAX_GEN_AI_MESSAGE_BYTES, @@ -13,6 +13,7 @@ truncate_and_annotate_messages, truncate_messages_by_size, _find_truncation_index, + redact_blob_message_parts, ) from sentry_sdk.serializer import serialize from sentry_sdk.utils import safe_serialize @@ -542,3 +543,106 @@ def __init__(self): assert isinstance(messages_value, AnnotatedValue) assert messages_value.metadata["len"] == stored_original_length assert len(messages_value.value) == len(truncated_messages) + + +class TestRedactBlobMessageParts: + def test_redacts_single_blob_content(self): + """Test that blob content is redacted in a message with single blob part""" + messages = [ + { + "role": "user", + "content": [ + { + "text": "How many ponies do you see in the image?", + "type": "text", + }, + { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "data:image/jpeg;base64,/9j/4AAQSkZJRg==", + }, + ], + } + ] + + result = redact_blob_message_parts(messages) + + assert result == messages # Returns the same list + assert ( + messages[0]["content"][0]["text"] + == "How many ponies do you see in the image?" + ) + assert messages[0]["content"][0]["type"] == "text" + assert messages[0]["content"][1]["type"] == "blob" + assert messages[0]["content"][1]["modality"] == "image" + assert messages[0]["content"][1]["mime_type"] == "image/jpeg" + assert messages[0]["content"][1]["content"] == SENSITIVE_DATA_SUBSTITUTE + + def test_redacts_multiple_blob_parts(self): + """Test that multiple blob parts in a single message are all redacted""" + messages = [ + { + "role": "user", + "content": [ + {"text": "Compare these images", "type": "text"}, + { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "data:image/jpeg;base64,first_image", + }, + { + "type": "blob", + "modality": "image", + "mime_type": "image/png", + "content": "data:image/png;base64,second_image", + }, + ], + } + ] + + result = redact_blob_message_parts(messages) + + assert result == messages + assert messages[0]["content"][0]["text"] == "Compare these images" + assert messages[0]["content"][1]["content"] == SENSITIVE_DATA_SUBSTITUTE + assert messages[0]["content"][2]["content"] == SENSITIVE_DATA_SUBSTITUTE + + def test_redacts_blobs_in_multiple_messages(self): + """Test that blob parts are redacted across multiple messages""" + messages = [ + { + "role": "user", + "content": [ + {"text": "First message", "type": "text"}, + { + "type": "blob", + "modality": "image", + "content": "data:image/jpeg;base64,first", + }, + ], + }, + { + "role": "assistant", + "content": "I see the image.", + }, + { + "role": "user", + "content": [ + {"text": "Second message", "type": "text"}, + { + "type": "blob", + "modality": "image", + "content": "data:image/jpeg;base64,second", + }, + ], + }, + ] + + result = redact_blob_message_parts(messages) + + assert result == messages + assert messages[0]["content"][1]["content"] == SENSITIVE_DATA_SUBSTITUTE + assert messages[1]["content"] == "I see the image." # Unchanged + assert messages[2]["content"][1]["content"] == SENSITIVE_DATA_SUBSTITUTE From 795bcea241f7777e646a4da14c870a3049bdbe90 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 17 Dec 2025 11:05:04 +0100 Subject: [PATCH 2/8] fix(ai): skip non dict messages --- sentry_sdk/ai/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index 73155b0305..ae507e898b 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -182,6 +182,9 @@ def redact_blob_message_parts(messages): """ for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content") if isinstance(content, list): for item in content: From a623e137d26e982c0d85258256c0ba013f9ecb24 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 17 Dec 2025 11:21:43 +0100 Subject: [PATCH 3/8] fix(ai): typing --- sentry_sdk/ai/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index ae507e898b..1b61c7a113 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -143,8 +143,9 @@ def _find_truncation_index(messages: "List[Dict[str, Any]]", max_bytes: int) -> return 0 -def redact_blob_message_parts(messages): - # type: (List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], int] +def redact_blob_message_parts( + messages: "List[Dict[str, Any]]", +) -> "List[Dict[str, Any]]": """ Redact blob message parts from the messages, by removing the "content" key. e.g: From 3d3ce5bbdca43f14194edbbbee11d3b6dcd6d8a3 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 17 Dec 2025 11:37:12 +0100 Subject: [PATCH 4/8] fix(ai): content items may not be dicts --- sentry_sdk/ai/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index 1b61c7a113..78a64ab737 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -189,7 +189,7 @@ def redact_blob_message_parts( content = message.get("content") if isinstance(content, list): for item in content: - if item.get("type") == "blob": + if isinstance(item, dict) and item.get("type") == "blob": item["content"] = SENSITIVE_DATA_SUBSTITUTE return messages From 7fa0b372a88e72e7684f74f29c09984a43788a2e Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 17 Dec 2025 15:11:48 +0100 Subject: [PATCH 5/8] fix(integrations): pydantic-ai: properly format binary input message parts to be conformant with the `gen_ai.request.messages` structure --- .../pydantic_ai/spans/ai_client.py | 26 +++++++++++++++++-- .../pydantic_ai/spans/invoke_agent.py | 14 ++++++++-- .../integrations/pydantic_ai/spans/utils.py | 2 +- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/integrations/pydantic_ai/spans/ai_client.py b/sentry_sdk/integrations/pydantic_ai/spans/ai_client.py index cb34f36e4f..56952a2189 100644 --- a/sentry_sdk/integrations/pydantic_ai/spans/ai_client.py +++ b/sentry_sdk/integrations/pydantic_ai/spans/ai_client.py @@ -1,5 +1,10 @@ +import base64 import sentry_sdk -from sentry_sdk.ai.utils import set_data_normalized +from sentry_sdk.ai.utils import ( + normalize_message_roles, + set_data_normalized, + truncate_and_annotate_messages, +) from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.utils import safe_serialize @@ -29,6 +34,7 @@ UserPromptPart, TextPart, ThinkingPart, + BinaryContent, ) except ImportError: # Fallback if these classes are not available @@ -38,6 +44,7 @@ UserPromptPart = None TextPart = None ThinkingPart = None + BinaryContent = None def _set_input_messages(span: "sentry_sdk.tracing.Span", messages: "Any") -> None: @@ -107,6 +114,16 @@ def _set_input_messages(span: "sentry_sdk.tracing.Span", messages: "Any") -> Non for item in part.content: if isinstance(item, str): content.append({"type": "text", "text": item}) + elif BinaryContent and isinstance(item, BinaryContent): + breakpoint() + content.append( + { + "type": "blob", + "modality": item.media_type.split("/")[0], + "mime_type": item.media_type, + "content": f"data:{item.media_type};base64,{base64.b64encode(item.data).decode('utf-8')}", + } + ) else: content.append(safe_serialize(item)) else: @@ -124,8 +141,13 @@ def _set_input_messages(span: "sentry_sdk.tracing.Span", messages: "Any") -> Non formatted_messages.append(message) if formatted_messages: + normalized_messages = normalize_message_roles(formatted_messages) + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages( + normalized_messages, span, scope + ) set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, formatted_messages, unpack=False + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) except Exception: # If we fail to format messages, just skip it diff --git a/sentry_sdk/integrations/pydantic_ai/spans/invoke_agent.py b/sentry_sdk/integrations/pydantic_ai/spans/invoke_agent.py index 629b3d1206..a22125e33d 100644 --- a/sentry_sdk/integrations/pydantic_ai/spans/invoke_agent.py +++ b/sentry_sdk/integrations/pydantic_ai/spans/invoke_agent.py @@ -1,5 +1,10 @@ import sentry_sdk -from sentry_sdk.ai.utils import get_start_span_function, set_data_normalized +from sentry_sdk.ai.utils import ( + get_start_span_function, + normalize_message_roles, + set_data_normalized, + truncate_and_annotate_messages, +) from sentry_sdk.consts import OP, SPANDATA from ..consts import SPAN_ORIGIN @@ -102,8 +107,13 @@ def invoke_agent_span( ) if messages: + normalized_messages = normalize_message_roles(messages) + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages( + normalized_messages, span, scope + ) set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages, unpack=False + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) return span diff --git a/sentry_sdk/integrations/pydantic_ai/spans/utils.py b/sentry_sdk/integrations/pydantic_ai/spans/utils.py index c70afd5f31..89fef172e1 100644 --- a/sentry_sdk/integrations/pydantic_ai/spans/utils.py +++ b/sentry_sdk/integrations/pydantic_ai/spans/utils.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Union + from typing import Union, Dict, Any, List from pydantic_ai.usage import RequestUsage, RunUsage # type: ignore From 33675993b296398cfe08c3a8fb18bed3a84dfd5b Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 17 Dec 2025 15:28:19 +0100 Subject: [PATCH 6/8] fix: remove manual breakpoint() --- sentry_sdk/integrations/pydantic_ai/spans/ai_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sentry_sdk/integrations/pydantic_ai/spans/ai_client.py b/sentry_sdk/integrations/pydantic_ai/spans/ai_client.py index 56952a2189..6496bd3397 100644 --- a/sentry_sdk/integrations/pydantic_ai/spans/ai_client.py +++ b/sentry_sdk/integrations/pydantic_ai/spans/ai_client.py @@ -115,7 +115,6 @@ def _set_input_messages(span: "sentry_sdk.tracing.Span", messages: "Any") -> Non if isinstance(item, str): content.append({"type": "text", "text": item}) elif BinaryContent and isinstance(item, BinaryContent): - breakpoint() content.append( { "type": "blob", From 704414c02a087479f3b4ea7a95cba2b2e1770003 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Wed, 17 Dec 2025 15:41:14 +0100 Subject: [PATCH 7/8] tests: add tests for message formatting --- .../pydantic_ai/test_pydantic_ai.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/tests/integrations/pydantic_ai/test_pydantic_ai.py b/tests/integrations/pydantic_ai/test_pydantic_ai.py index 049bcde39c..a7587a6fcd 100644 --- a/tests/integrations/pydantic_ai/test_pydantic_ai.py +++ b/tests/integrations/pydantic_ai/test_pydantic_ai.py @@ -1,12 +1,17 @@ import asyncio +import json import pytest +from unittest.mock import MagicMock from typing import Annotated from pydantic import Field +import sentry_sdk from sentry_sdk.integrations.pydantic_ai import PydanticAIIntegration +from sentry_sdk.integrations.pydantic_ai.spans.ai_client import _set_input_messages from pydantic_ai import Agent +from pydantic_ai.messages import BinaryContent, UserPromptPart from pydantic_ai.models.test import TestModel from pydantic_ai.exceptions import ModelRetry, UnexpectedModelBehavior @@ -2604,3 +2609,128 @@ async def test_ai_client_span_gets_agent_from_scope(sentry_init, capture_events) # Should not crash assert transaction is not None + + +def _get_messages_from_span(span_data): + """Helper to extract and parse messages from span data.""" + messages_data = span_data["gen_ai.request.messages"] + return ( + json.loads(messages_data) if isinstance(messages_data, str) else messages_data + ) + + +def _find_binary_content(messages_data, expected_modality, expected_mime_type): + """Helper to find and verify binary content in messages.""" + for msg in messages_data: + if "content" not in msg: + continue + for content_item in msg["content"]: + if content_item.get("type") == "blob": + assert content_item["modality"] == expected_modality + assert content_item["mime_type"] == expected_mime_type + assert "content" in content_item + content_str = str(content_item["content"]) + assert ( + f"data:{expected_mime_type};base64," in content_str + or "[Filtered]" in content_str + ) + return True + return False + + +@pytest.mark.asyncio +async def test_binary_content_encoding_image(sentry_init, capture_events): + """Test that BinaryContent with image data is properly encoded in messages.""" + sentry_init( + integrations=[PydanticAIIntegration()], + traces_sample_rate=1.0, + send_default_pii=True, + ) + + events = capture_events() + + with sentry_sdk.start_transaction(op="test", name="test"): + span = sentry_sdk.start_span(op="test_span") + binary_content = BinaryContent( + data=b"fake_image_data_12345", media_type="image/png" + ) + user_part = UserPromptPart(content=["Look at this image:", binary_content]) + mock_msg = MagicMock() + mock_msg.parts = [user_part] + mock_msg.instructions = None + + _set_input_messages(span, [mock_msg]) + span.finish() + + (event,) = events + span_data = event["spans"][0]["data"] + messages_data = _get_messages_from_span(span_data) + assert _find_binary_content(messages_data, "image", "image/png") + + +@pytest.mark.asyncio +async def test_binary_content_encoding_mixed_content(sentry_init, capture_events): + """Test that BinaryContent mixed with text content is properly handled.""" + sentry_init( + integrations=[PydanticAIIntegration()], + traces_sample_rate=1.0, + send_default_pii=True, + ) + + events = capture_events() + + with sentry_sdk.start_transaction(op="test", name="test"): + span = sentry_sdk.start_span(op="test_span") + binary_content = BinaryContent( + data=b"fake_image_bytes", media_type="image/jpeg" + ) + user_part = UserPromptPart( + content=["Here is an image:", binary_content, "What do you see?"] + ) + mock_msg = MagicMock() + mock_msg.parts = [user_part] + mock_msg.instructions = None + + _set_input_messages(span, [mock_msg]) + span.finish() + + (event,) = events + span_data = event["spans"][0]["data"] + messages_data = _get_messages_from_span(span_data) + + # Verify both text and binary content are present + found_text = any( + content_item.get("type") == "text" + for msg in messages_data + if "content" in msg + for content_item in msg["content"] + ) + assert found_text, "Text content should be found" + assert _find_binary_content(messages_data, "image", "image/jpeg") + + +@pytest.mark.asyncio +async def test_binary_content_in_agent_run(sentry_init, capture_events): + """Test that BinaryContent in actual agent run is properly captured in spans.""" + agent = Agent("test", name="test_binary_agent") + + sentry_init( + integrations=[PydanticAIIntegration()], + traces_sample_rate=1.0, + send_default_pii=True, + ) + + events = capture_events() + binary_content = BinaryContent( + data=b"fake_image_data_for_testing", media_type="image/png" + ) + await agent.run(["Analyze this image:", binary_content]) + + (transaction,) = events + chat_spans = [s for s in transaction["spans"] if s["op"] == "gen_ai.chat"] + assert len(chat_spans) >= 1 + + chat_span = chat_spans[0] + if "gen_ai.request.messages" in chat_span["data"]: + messages_str = str(chat_span["data"]["gen_ai.request.messages"]) + assert any(keyword in messages_str for keyword in ["blob", "image", "base64"]) From a488747ac8dd40f4dd1e180d2640942091a39483 Mon Sep 17 00:00:00 2001 From: Fabian Schindler Date: Tue, 13 Jan 2026 14:14:13 +0100 Subject: [PATCH 8/8] test: fix testcase --- tests/integrations/pydantic_ai/test_pydantic_ai.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integrations/pydantic_ai/test_pydantic_ai.py b/tests/integrations/pydantic_ai/test_pydantic_ai.py index a7587a6fcd..3eec047194 100644 --- a/tests/integrations/pydantic_ai/test_pydantic_ai.py +++ b/tests/integrations/pydantic_ai/test_pydantic_ai.py @@ -7,6 +7,7 @@ from pydantic import Field import sentry_sdk +from sentry_sdk._types import BLOB_DATA_SUBSTITUTE from sentry_sdk.integrations.pydantic_ai import PydanticAIIntegration from sentry_sdk.integrations.pydantic_ai.spans.ai_client import _set_input_messages @@ -2632,7 +2633,7 @@ def _find_binary_content(messages_data, expected_modality, expected_mime_type): content_str = str(content_item["content"]) assert ( f"data:{expected_mime_type};base64," in content_str - or "[Filtered]" in content_str + or BLOB_DATA_SUBSTITUTE in content_str ) return True return False