diff --git a/astrbot/core/agent/context/compressor.py b/astrbot/core/agent/context/compressor.py index 759604dd93..685d417426 100644 --- a/astrbot/core/agent/context/compressor.py +++ b/astrbot/core/agent/context/compressor.py @@ -268,6 +268,7 @@ async def __call__(self, messages: list[Message]) -> list[Message]: sanitized_summary_contexts, sanitize_stats = sanitize_contexts_by_modalities( summary_contexts, self.provider.provider_config.get("modalities", None), + self.provider.provider_config.get("supported_image_mimes", None), ) log_context_sanitize_stats(sanitize_stats) diff --git a/astrbot/core/agent/runners/tool_loop_agent_runner.py b/astrbot/core/agent/runners/tool_loop_agent_runner.py index 98754f9b6a..44d31484f0 100644 --- a/astrbot/core/agent/runners/tool_loop_agent_runner.py +++ b/astrbot/core/agent/runners/tool_loop_agent_runner.py @@ -595,6 +595,7 @@ def _sanitize_contexts_for_provider( sanitized_contexts, stats = sanitize_contexts_by_modalities( contexts, self.provider.provider_config.get("modalities", None), + self.provider.provider_config.get("supported_image_mimes", None), ) log_context_sanitize_stats(stats) return sanitized_contexts diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index ae6a3e7883..9f43291390 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -2090,6 +2090,23 @@ "render_type": "checkbox", "hint": "模型支持的模态及能力。", }, + "supported_image_mimes": { + "description": "支持的图片 MIME 类型", + "type": "list", + "items": {"type": "string"}, + "options": [ + "image/jpeg", + "image/png", + "image/webp", + "image/gif", + ], + "labels": ["JPEG", "PNG", "WebP", "GIF"], + "render_type": "checkbox", + "hint": ( + "模型支持的图片格式。不勾选时保留全部（仅过滤已知不安全格式如 GIF）。" + "勾选后仅保留勾选的格式。" + ), + }, "custom_headers": { "description": "自定义请求头", "type": "dict", diff --git a/astrbot/core/provider/modalities.py b/astrbot/core/provider/modalities.py index 66ac74e9b7..b5739bed01 100644 --- a/astrbot/core/provider/modalities.py +++ b/astrbot/core/provider/modalities.py @@ -4,6 +4,7 @@ from collections.abc import Sequence from dataclasses import dataclass from typing import Any +from urllib.parse import urlsplit from astrbot import logger from astrbot.core.agent.message import Message @@ -34,10 +35,98 @@ def _message_to_dict(message: dict[str, Any] | Message) -> dict[str, Any] | None return None +# Image MIME types that some OpenAI-compatible gateways reject even when the +# model claims image support. Animated GIFs in particular are accepted by raw +# Gemini (which wants image/heif/video/mp4 for animations) but rejected by +# several Gemini-flavored OpenAI proxies with "mime type is not supported". +# Keeping this list narrow avoids over-stripping providers that do accept GIF +# (e.g. Anthropic Claude). See issue #9295. +_UNSUPPORTED_IMAGE_MIMES = frozenset({"image/gif"}) + +# Lazily-built extension → MIME mapping used only for http(s) URL fallback. +_IMAGE_EXT_TO_MIME: dict[str, str] = { + ".gif": "image/gif", +} + + +def _extract_image_mime(part: dict[str, Any]) -> str | None: + """Best-effort extraction of an image MIME type from a multimodal part. + + Handles the OpenAI-style ``{"image_url": {"url": ...}}`` and the Anthropic / + Gemini-style ``{"source": {"media_type": ...}}`` / ``{"mimeType": ...}`` + layouts, as well as a bare ``{"url": ...}`` or ``{"image_url": ""}``. + + Returns: + The normalized MIME type (e.g. ``image/gif``) if it can be determined, + otherwise ``None``. + """ + image_url = part.get("image_url") + if isinstance(image_url, dict): + url = image_url.get("url") + else: + url = image_url + if not isinstance(url, str): + url = part.get("url") if isinstance(part.get("url"), str) else None + + if isinstance(url, str): + url = url.strip() + # data URLs look like "data:image/gif;base64,...." + if url.lower().startswith("data:"): + head = url[5:].split(",", 1)[0] + # head is e.g. "image/gif;base64" + mime = head.split(";", 1)[0].strip().lower() + if mime: + return mime + else: + # Fall back to the URL path extension for http(s) URLs. urlsplit + # robustly separates the path from query/fragment even when those + # delimiters appear percent-encoded inside the path itself. + path = urlsplit(url).path.lower() + for ext, mime in _IMAGE_EXT_TO_MIME.items(): + if path.endswith(ext): + return mime + + source = part.get("source") + if isinstance(source, dict): + media_type = source.get("media_type") + if isinstance(media_type, str): + # Normalize by stripping parameters (e.g. "image/gif; charset=binary" + # -> "image/gif") so it matches _UNSUPPORTED_IMAGE_MIMES. + return media_type.split(";", 1)[0].strip().lower() + + mime_type = part.get("mimeType") or part.get("mime_type") + if isinstance(mime_type, str): + # Strip parameters (e.g. "image/gif;codec=xyz" -> "image/gif") to align + # with the data-URL and source.media_type handling above. + return mime_type.split(";", 1)[0].strip().lower() + + return None + + +def _is_unsupported_image_mime(mime: str | None) -> bool: + """Return True when the MIME is known to be rejected by some providers.""" + return bool(mime) and mime in _UNSUPPORTED_IMAGE_MIMES + + def sanitize_contexts_by_modalities( contexts: Sequence[dict[str, Any] | Message], modalities: list[str] | None, + supported_image_mimes: list[str] | None = None, ) -> tuple[list[dict[str, Any]], ContextSanitizeStats]: + """Sanitize message contexts based on provider capabilities. + + Args: + contexts: The message contexts to sanitize. + modalities: List of modalities the provider supports (e.g. ["text", "image"]). + supported_image_mimes: Whitelist of image MIME types the provider accepts. + When None or empty, a fallback blocklist (_UNSUPPORTED_IMAGE_MIMES) is used + to filter known-problematic formats (e.g. image/gif for some Gemini gateways). + When provided, images with MIME types not in this list are replaced with + "[Image]" placeholders. + + Returns: + Tuple of (sanitized contexts, statistics). + """ if not contexts: return [], ContextSanitizeStats() if not modalities or not isinstance(modalities, list): @@ -51,7 +140,13 @@ def sanitize_contexts_by_modalities( supports_image = "image" in modalities supports_audio = "audio" in modalities supports_tool_use = "tool_use" in modalities - if supports_image and supports_audio and supports_tool_use: + # Determine whether we need a MIME-filtering pass. When a whitelist is + # provided, we always filter; otherwise we fall back to the hardcoded + # blocklist for known-problematic MIME types (e.g. image/gif). + needs_mime_pass = supports_image and ( + bool(supported_image_mimes) or bool(_UNSUPPORTED_IMAGE_MIMES) + ) + if supports_image and supports_audio and supports_tool_use and not needs_mime_pass: copied_contexts = [] for msg in contexts: copied_msg = _message_to_dict(msg) @@ -83,7 +178,7 @@ def sanitize_contexts_by_modalities( msg.pop("tool_calls", None) msg.pop("tool_call_id", None) - if not supports_image or not supports_audio: + if not supports_image or not supports_audio or needs_mime_pass: content = msg.get("content") if isinstance(content, list): filtered_parts: list[Any] = [] @@ -91,7 +186,28 @@ def sanitize_contexts_by_modalities( for part in content: if isinstance(part, dict): part_type = str(part.get("type", "")).lower() - if not supports_image and part_type in {"image_url", "image"}: + # Determine whether this image part should be dropped. + should_drop_image = False + if part_type in {"image_url", "image"}: + if not supports_image: + # Provider declares no image support at all. + should_drop_image = True + else: + # Provider supports image, but may have MIME restrictions. + mime = _extract_image_mime(part) + if supported_image_mimes: + # Whitelist mode: drop if MIME not in the list. + if mime not in supported_image_mimes: + should_drop_image = True + else: + # Fallback blocklist mode: drop if MIME is known + # to be rejected by some gateways (e.g. image/gif). + if _is_unsupported_image_mime(mime): + should_drop_image = True + if should_drop_image: + # Replacing the block with a placeholder prevents unsupported + # bytes from being persisted into the session history and + # poisoning all subsequent requests. See issue #9295. removed_any_multimodal = True stats.fixed_image_blocks += 1 filtered_parts.append({"type": "text", "text": "[Image]"}) diff --git a/dashboard/src/i18n/locales/en-US/features/config-metadata.json b/dashboard/src/i18n/locales/en-US/features/config-metadata.json index 59f2c96944..25e76ba17f 100644 --- a/dashboard/src/i18n/locales/en-US/features/config-metadata.json +++ b/dashboard/src/i18n/locales/en-US/features/config-metadata.json @@ -1272,6 +1272,16 @@ "Tool use" ] }, + "supported_image_mimes": { + "description": "Supported image MIME types", + "hint": "Image formats the model accepts. When unchecked, all formats are kept (only known-unsafe formats like GIF are filtered on some gateways). When checked, only selected formats are preserved.", + "labels": [ + "JPEG", + "PNG", + "WebP", + "GIF" + ] + }, "custom_headers": { "description": "Custom request headers", "hint": "Key/value pairs added here are merged into the OpenAI SDK default_headers for custom HTTP headers. Values must be strings." diff --git a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json index 455587f308..d73feac355 100644 --- a/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json +++ b/dashboard/src/i18n/locales/zh-CN/features/config-metadata.json @@ -1274,6 +1274,16 @@ "工具使用" ] }, + "supported_image_mimes": { + "description": "支持的图片 MIME 类型", + "hint": "模型支持的图片格式。不勾选时保留全部（仅过滤已知不安全格式如 GIF）。勾选后仅保留勾选的格式。", + "labels": [ + "JPEG", + "PNG", + "WebP", + "GIF" + ] + }, "custom_headers": { "description": "自定义请求头", "hint": "此处添加的键值对将被合并到 OpenAI SDK 的 default_headers 中，用于自定义 HTTP 请求头。值必须为字符串。" diff --git a/tests/unit/test_modalities_sanitize.py b/tests/unit/test_modalities_sanitize.py new file mode 100644 index 0000000000..0f1c1cba60 --- /dev/null +++ b/tests/unit/test_modalities_sanitize.py @@ -0,0 +1,354 @@ +"""Tests for ``astrbot.core.provider.modalities.sanitize_contexts_by_modalities``. + +These tests focus on the image MIME handling added for issue #9295, where an +animated GIF referenced via a quote could poison the session history and make +subsequent requests to GIF-rejecting Gemini-compatible gateways fail forever. +""" + +from __future__ import annotations + +from astrbot.core.provider.modalities import ( + ContextSanitizeStats, + sanitize_contexts_by_modalities, +) + + +def _image_url_part(url: str) -> dict: + return {"type": "image_url", "image_url": {"url": url}} + + +def _user(*parts: dict) -> dict: + return {"role": "user", "content": list(parts)} + + +GIF_DATA_URL = "data:image/gif;base64,R0lGODlh8ADwAPcAAPxzxg==" +PNG_DATA_URL = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB" +JPEG_DATA_URL = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD" +WEBP_DATA_URL = "data:image/webp;base64,UklGRkBAAABXRUJQ" + + +# --------------------------------------------------------------------------- +# Issue #9295: GIF must be stripped even when the model claims image support +# --------------------------------------------------------------------------- + + +def test_gif_data_url_replaced_when_image_supported_but_gif_unsupported() -> None: + """Reproduces the exact reporter scenario. + + Provider declares ``[text, image, audio, tool_use]`` (which used to hit the + fast-path and skip sanitizing), and the context carries a ``data:image/gif`` + block. The GIF must be replaced with ``[Image]``. + """ + contexts = [_user(_image_url_part(GIF_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image", "audio", "tool_use"], + ) + + content = sanitized[0]["content"] + assert content == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + assert stats.changed + + +def test_gif_data_url_replaced_when_only_text_and_image_supported() -> None: + """Stripping also applies to the regular image-supported path.""" + contexts = [_user(_image_url_part(GIF_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image"], + ) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_gif_http_url_replaced_by_extension_fallback() -> None: + """http(s) URLs ending in ``.gif`` are also detected via extension.""" + contexts = [_user(_image_url_part("https://example.com/animation.gif"))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image", "tool_use"], + ) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_gif_http_url_with_query_string_still_detected() -> None: + """Query/fragment suffixes on the URL must not defeat detection.""" + contexts = [ + _user(_image_url_part("https://cdn.example.com/a.GIF?token=abc#frag")), + ] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_gif_http_url_path_strips_query_via_urlsplit() -> None: + """A real ``?`` inside the path (percent-encoded) must not be mis-split. + + ``urlsplit`` keeps the percent-encoded ``%3F`` inside the path, so a path + ending in ``.gif`` followed by an encoded delimiter is still detected. + """ + contexts = [ + _user( + _image_url_part( + "https://cdn.example.com/path%3Fextra/anim.gif?sig=1#x", + ), + ), + ] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_anthropic_media_type_with_params_still_detected() -> None: + """``media_type`` carrying parameters must normalize down to ``image/gif``.""" + contexts = [ + _user( + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/gif; charset=binary", + "data": "R0lGODlh8ADwAPcAAPxzxg==", + }, + }, + ), + ] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image", "audio", "tool_use"], + ) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_gemini_mimetype_field_with_params_still_detected() -> None: + """A bare ``mimeType``/``mime_type`` with params must also be normalized.""" + contexts = [ + _user( + { + "type": "image", + "mimeType": " image/gif;codec=xyz ", + }, + ), + ] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_non_gif_media_type_with_params_preserved() -> None: + """Non-unsupported MIME with parameters must be preserved, not stripped.""" + contexts = [ + _user( + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png; charset=utf-8", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB", + }, + }, + ), + ] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert sanitized[0]["content"] == contexts[0]["content"] + assert not stats.changed + + +# --------------------------------------------------------------------------- +# Non-GIF images must be preserved when image is supported +# --------------------------------------------------------------------------- + + +def test_png_preserved_when_image_supported() -> None: + contexts = [_user(_image_url_part(PNG_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image", "audio", "tool_use"], + ) + + assert sanitized[0]["content"] == [_image_url_part(PNG_DATA_URL)] + assert stats.fixed_image_blocks == 0 + assert not stats.changed + + +def test_jpeg_and_webp_preserved_when_image_supported() -> None: + contexts = [ + _user(_image_url_part(JPEG_DATA_URL), _image_url_part(WEBP_DATA_URL)), + ] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image"], + ) + + assert sanitized[0]["content"] == [ + _image_url_part(JPEG_DATA_URL), + _image_url_part(WEBP_DATA_URL), + ] + assert not stats.changed + + +def test_unknown_mime_image_preserved_when_image_supported() -> None: + """When no MIME can be determined we must not strip the image. + + Stripping on "unknown" would over-aggressively drop legitimate images and + break providers that accept arbitrary image formats. + """ + contexts = [_user({"type": "image_url", "image_url": {"url": "abc-not-a-url"}})] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert sanitized[0]["content"] == contexts[0]["content"] + assert not stats.changed + + +# --------------------------------------------------------------------------- +# Existing behavior: full removal when image modality is not declared +# --------------------------------------------------------------------------- + + +def test_all_images_removed_when_image_not_supported() -> None: + """Pre-existing behavior must remain: no image modality → strip everything.""" + contexts = [ + _user( + _image_url_part(GIF_DATA_URL), + _image_url_part(PNG_DATA_URL), + ) + ] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text"]) + + assert sanitized[0]["content"] == [ + {"type": "text", "text": "[Image]"}, + {"type": "text", "text": "[Image]"}, + ] + assert stats.fixed_image_blocks == 2 + + +def test_audio_branch_unchanged_by_gif_handling() -> None: + """The audio-stripping branch must keep working independently.""" + contexts = [ + _user( + {"type": "input_audio", "input_audio": {"data": "abc", "format": "mp3"}}, + ) + ] + sanitized, stats = sanitize_contexts_by_modalities(contexts, ["text", "image"]) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Audio]"}] + assert stats.fixed_audio_blocks == 1 + assert stats.fixed_image_blocks == 0 + + +def test_mixed_text_and_gif_only_gif_is_replaced() -> None: + """A GIF between text parts must only drop the GIF, preserving the text.""" + contexts = [ + _user( + {"type": "text", "text": "look at this"}, + _image_url_part(GIF_DATA_URL), + {"type": "text", "text": "isn't it cool"}, + ) + ] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image", "audio", "tool_use"], + ) + + assert sanitized[0]["content"] == [ + {"type": "text", "text": "look at this"}, + {"type": "text", "text": "[Image]"}, + {"type": "text", "text": "isn't it cool"}, + ] + assert stats.fixed_image_blocks == 1 + + +def test_empty_modalities_returns_contexts_unchanged() -> None: + contexts = [_user(_image_url_part(GIF_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities(contexts, None) + + assert sanitized[0]["content"] == contexts[0]["content"] + assert not stats.changed + + +def test_empty_contexts_returns_empty() -> None: + sanitized, stats = sanitize_contexts_by_modalities([], ["text", "image"]) + assert sanitized == [] + assert isinstance(stats, ContextSanitizeStats) + assert not stats.changed + + +# --------------------------------------------------------------------------- +# Whitelist mode: supported_image_mimes parameter +# --------------------------------------------------------------------------- + + +def test_whitelist_filters_gif_but_preserves_jpeg_and_png() -> None: + """When whitelist excludes GIF, GIF is replaced but JPEG/PNG are kept.""" + contexts = [ + _user( + _image_url_part(GIF_DATA_URL), + _image_url_part(JPEG_DATA_URL), + _image_url_part(PNG_DATA_URL), + ), + ] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image"], + supported_image_mimes=["image/jpeg", "image/png"], + ) + + assert sanitized[0]["content"] == [ + {"type": "text", "text": "[Image]"}, # GIF replaced + _image_url_part(JPEG_DATA_URL), # JPEG preserved + _image_url_part(PNG_DATA_URL), # PNG preserved + ] + assert stats.fixed_image_blocks == 1 + + +def test_whitelist_includes_gif_preserves_gif() -> None: + """When whitelist includes GIF, GIF is preserved.""" + contexts = [_user(_image_url_part(GIF_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image"], + supported_image_mimes=["image/jpeg", "image/png", "image/gif"], + ) + + assert sanitized[0]["content"] == [_image_url_part(GIF_DATA_URL)] + assert not stats.changed + + +def test_whitelist_none_falls_back_to_blocklist_gif_replaced() -> None: + """When whitelist is None, fallback blocklist applies (GIF replaced). + + This ensures backward compatibility: users who don't configure + supported_image_mimes still get the #9295 GIF fix. + """ + contexts = [_user(_image_url_part(GIF_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image"], + supported_image_mimes=None, + ) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1 + + +def test_whitelist_empty_list_same_as_none() -> None: + """Empty list for whitelist behaves the same as None (blocklist fallback).""" + contexts = [_user(_image_url_part(GIF_DATA_URL))] + sanitized, stats = sanitize_contexts_by_modalities( + contexts, + ["text", "image"], + supported_image_mimes=[], + ) + + assert sanitized[0]["content"] == [{"type": "text", "text": "[Image]"}] + assert stats.fixed_image_blocks == 1