From 513373e9b9008116c5b0bb611efa573d51c04465 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Fri, 10 Jul 2026 09:46:33 -0700 Subject: [PATCH 1/3] Warn instead of raising on reasoning-model token truncation When a reasoning model hits max_completion_tokens, the API returns finish_reason='length' with empty visible content. Previously this raised EmptyResponseException (misleadingly reported as 'Status Code: 204') and triggered the 10x retry storm. _validate_response now logs a warning explaining that reasoning models consume tokens on hidden reasoning, and returns a graceful empty response (error='empty') instead of raising, so runs continue. Non-length empty responses still raise as before. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../openai/openai_chat_target.py | 28 +++++++++++++++++-- .../target/test_openai_chat_target.py | 21 ++++++++++++-- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/pyrit/prompt_target/openai/openai_chat_target.py b/pyrit/prompt_target/openai/openai_chat_target.py index 9374ea73b8..cdcd28a480 100644 --- a/pyrit/prompt_target/openai/openai_chat_target.py +++ b/pyrit/prompt_target/openai/openai_chat_target.py @@ -296,11 +296,16 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | request: The original request MessagePiece. Returns: - None if valid, does not return Message for content filter (handled by _check_content_filter). + None when the response is valid and should be constructed normally. Returns an empty + response ``Message`` (with ``error="empty"``) when the response was truncated at the + token limit (``finish_reason="length"``) before producing any visible output, so the + run continues instead of raising. Content filter responses are handled separately by + ``_check_content_filter``. Raises: PyritException: For unexpected response structures or finish reasons. - EmptyResponseException: When the API returns an empty response. + EmptyResponseException: When the API returns an empty response that was not caused by + token-limit truncation. """ # Check for missing choices if not hasattr(response, "choices") or not response.choices: @@ -320,6 +325,25 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | # Check for at least one valid response type has_content, has_audio, has_tool_calls = self._detect_response_content(choice.message) + # A "length" finish_reason means the model hit max_completion_tokens. Reasoning models spend + # tokens on hidden reasoning before emitting visible output, so a low limit can truncate the + # answer or leave it empty. This may be a deliberate configuration, so warn instead of raising. + if finish_reason == "length": + logger.warning( + "The response was truncated because it reached the token limit (finish_reason='length'). " + "Reasoning models consume tokens on hidden reasoning in addition to the visible answer, so a " + "low max_completion_tokens can truncate or empty the response. Increase max_completion_tokens " + "if you expected complete content." + ) + if not (has_content or has_audio or has_tool_calls): + return construct_response_from_request( + request=request, + response_text_pieces=[""], + response_type="text", + error="empty", + ) + return None + if not (has_content or has_audio or has_tool_calls): logger.error("The chat returned an empty response (no content, audio, or tool_calls).") raise EmptyResponseException( diff --git a/tests/unit/prompt_target/target/test_openai_chat_target.py b/tests/unit/prompt_target/target/test_openai_chat_target.py index dbf1018a1b..39121d797f 100644 --- a/tests/unit/prompt_target/target/test_openai_chat_target.py +++ b/tests/unit/prompt_target/target/test_openai_chat_target.py @@ -1039,11 +1039,26 @@ def test_validate_response_success_stop(target: OpenAIChatTarget, dummy_text_mes assert result is None -def test_validate_response_success_length(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece): - """Test _validate_response passes for valid length response.""" +def test_validate_response_success_length(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece, caplog): + """Test _validate_response passes for a truncated response that still has content, and warns.""" mock_response = create_mock_completion(content="Hello", finish_reason="length") - result = target._validate_response(mock_response, dummy_text_message_piece) + with caplog.at_level(logging.WARNING): + result = target._validate_response(mock_response, dummy_text_message_piece) assert result is None + assert "finish_reason='length'" in caplog.text + + +def test_validate_response_length_empty_returns_empty_response( + target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece, caplog +): + """Test _validate_response returns a graceful empty response (no raise) when truncated before any output.""" + mock_response = create_mock_completion(content="", finish_reason="length") + with caplog.at_level(logging.WARNING): + result = target._validate_response(mock_response, dummy_text_message_piece) + assert isinstance(result, Message) + assert result.message_pieces[0].original_value == "" + assert result.message_pieces[0].response_error == "empty" + assert "finish_reason='length'" in caplog.text def test_validate_response_no_choices(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece): From dd1bb09af1bb67412c04aa571d155dd8827bcaf4 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:07:29 -0700 Subject: [PATCH 2/3] Warn instead of raising on reasoning-model token truncation in Response target Mirror the OpenAIChatTarget truncation fix in OpenAIResponseTarget. When the Responses API returns status='incomplete' with reason='max_output_tokens', warn and preserve any completed output (reasoning, partial text) instead of raising a PyritException that halts the run. Falls back to a graceful empty text piece when no visible text was produced, and skips partial tool/function-call sections so an incomplete call cannot re-enter the agentic loop. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 00b73356-ae65-4d83-8220-ebad37cc5850 --- .../openai/openai_response_target.py | 97 +++++++++++++- .../target/test_openai_response_target.py | 121 ++++++++++++++++++ 2 files changed, 214 insertions(+), 4 deletions(-) diff --git a/pyrit/prompt_target/openai/openai_response_target.py b/pyrit/prompt_target/openai/openai_response_target.py index 0b6b3b2acb..42ce949a44 100644 --- a/pyrit/prompt_target/openai/openai_response_target.py +++ b/pyrit/prompt_target/openai/openai_response_target.py @@ -25,6 +25,7 @@ MessagePiece, PromptDataType, PromptResponseError, + construct_response_from_request, ) from pyrit.prompt_target.common.json_response_config import _JsonResponseConfig from pyrit.prompt_target.common.target_capabilities import TargetCapabilities @@ -502,6 +503,7 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | Checks for: - Error responses (excluding content filtering which is checked separately) + - Truncation at the token limit (``max_output_tokens``), which is warned about, not raised - Invalid status - Empty output @@ -510,17 +512,35 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | request: The original request MessagePiece. Returns: - None if valid, does not return Message for content filter (handled by _check_content_filter). + None when the response is valid and should be constructed normally. Returns a Message + (built from whatever completed output exists, with a graceful empty text piece when no + visible text was produced) when the response was truncated at the token limit, so the + run continues instead of raising. Content filter responses are handled separately by + ``_check_content_filter``. Raises: PyritException: For unexpected response structures or errors. - EmptyResponseException: When the API returns no valid output. + EmptyResponseException: When the API returns no valid output (and was not truncated). """ # Check for error response - error is a ResponseError object or None # (content_filter is handled by _check_content_filter) if response.error is not None and response.error.code != "content_filter": raise PyritException(message=f"Response error: {response.error.code} - {response.error.message}") + # Truncation: the model hit max_output_tokens. Mirroring OpenAIChatTarget's handling of + # finish_reason == "length", warn instead of raising so the run continues -- reasoning models + # can spend the whole budget on hidden reasoning before emitting a visible answer, and a low + # limit may be a deliberate configuration. Preserve any completed output (reasoning, partial + # text) and fall back to a graceful empty response. + if self._is_truncated_response(response): + logger.warning( + "The response was truncated because it reached the token limit " + "(status='incomplete', reason='max_output_tokens'). Reasoning models consume tokens on " + "hidden reasoning in addition to the visible answer, so a low max_output_tokens can " + "truncate or empty the response. Increase max_output_tokens if you expected complete content." + ) + return self._build_truncated_message(response=response, request=request) + # Check status - should be "completed" for successful responses if response.status != "completed": raise PyritException(message=f"Unexpected status: {response.status}") @@ -532,6 +552,61 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | return None + def _is_truncated_response(self, response: Any) -> bool: + """ + Return True if the response was cut off by the ``max_output_tokens`` limit. + + The Responses API signals truncation via ``status == "incomplete"`` with + ``incomplete_details.reason == "max_output_tokens"`` (``content_filter`` is handled + separately by ``_check_content_filter``). + + Args: + response: A Response object from the OpenAI SDK. + + Returns: + bool: True if the response was truncated at the token limit, False otherwise. + """ + if getattr(response, "status", None) != "incomplete": + return False + incomplete_details = getattr(response, "incomplete_details", None) + reason = getattr(incomplete_details, "reason", None) if incomplete_details else None + return reason == "max_output_tokens" + + def _build_truncated_message(self, *, response: Any, request: MessagePiece) -> Message: + """ + Build a Message from a truncated response, tolerating empty output sections. + + Preserves the model's content pieces (reasoning and any partial visible text) and appends + a graceful empty text piece (``error="empty"``) when no visible text was produced, so the + run continues instead of raising. Partial tool/function-call sections are skipped because an + incomplete call must not re-enter the agentic loop. + + Args: + response: A Response object from the OpenAI SDK. + request: The original request MessagePiece. + + Returns: + Message: The preserved pieces plus, when no visible text exists, a graceful empty piece. + """ + pieces: list[MessagePiece] = [] + has_text = False + for section in getattr(response, "output", None) or []: + piece = self._parse_response_output_section( + section=section, message_piece=request, error=None, tolerate_empty=True + ) + if piece is None or piece.original_value_data_type not in ("text", "reasoning"): + continue + pieces.append(piece) + if piece.original_value_data_type == "text" and piece.original_value: + has_text = True + + if not has_text: + empty_piece = construct_response_from_request( + request=request, response_text_pieces=[""], response_type="text", error="empty" + ).message_pieces[0] + pieces.append(empty_piece) + return Message(message_pieces=pieces) + async def _construct_message_from_response_async(self, response: Any, request: MessagePiece) -> Message: """ Construct a Message from a Response API response. @@ -628,7 +703,12 @@ async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Me return responses_to_return def _parse_response_output_section( - self, *, section: Any, message_piece: MessagePiece, error: PromptResponseError | None + self, + *, + section: Any, + message_piece: MessagePiece, + error: PromptResponseError | None, + tolerate_empty: bool = False, ) -> MessagePiece | None: """ Parse model output sections, forwarding tool-calls for the agentic loop. @@ -637,12 +717,15 @@ def _parse_response_output_section( section: The section object from OpenAI SDK (Pydantic model). message_piece: The original message piece. error: Any error information from OpenAI. + tolerate_empty: When True, empty sections return None instead of raising + EmptyResponseException. Used when constructing a truncated response. Returns: A MessagePiece for this section, or None to skip. Raises: - EmptyResponseException: If the section content is empty or invalid. + EmptyResponseException: If the section content is empty or invalid (and tolerate_empty + is False). ValueError: If the section type is unsupported. """ section_type = section.type @@ -652,6 +735,8 @@ def _parse_response_output_section( if section_type == MessagePieceType.MESSAGE: section_content = section.content if len(section_content) == 0: + if tolerate_empty: + return None raise EmptyResponseException(message="The chat returned an empty message section.") piece_value = section_content[0].text @@ -706,6 +791,8 @@ def _parse_response_output_section( raise ValueError(msg) piece_value = section.input if len(piece_value) == 0: + if tolerate_empty: + return None raise EmptyResponseException(message="The chat returned an empty message section.") else: @@ -714,6 +801,8 @@ def _parse_response_output_section( # Handle empty response if not piece_value: + if tolerate_empty: + return None raise EmptyResponseException(message="The chat returned an empty response.") return MessagePiece( diff --git a/tests/unit/prompt_target/target/test_openai_response_target.py b/tests/unit/prompt_target/target/test_openai_response_target.py index d0fbd1e131..8a4a04fc59 100644 --- a/tests/unit/prompt_target/target/test_openai_response_target.py +++ b/tests/unit/prompt_target/target/test_openai_response_target.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. import json +import logging import os from collections.abc import MutableSequence from tempfile import NamedTemporaryFile @@ -1192,6 +1193,126 @@ def test_validate_response_empty_output(target: OpenAIResponseTarget, dummy_text target._validate_response(mock_response, dummy_text_message_piece) +def _make_reasoning_section() -> MagicMock: + section = MagicMock() + section.type = "reasoning" + section.model_dump.return_value = {"type": "reasoning", "summary": []} + return section + + +def _make_message_section(text: str) -> MagicMock: + section = MagicMock() + section.type = "message" + content_item = MagicMock() + content_item.text = text + section.content = [content_item] + return section + + +def _make_empty_message_section() -> MagicMock: + section = MagicMock() + section.type = "message" + section.content = [] + return section + + +def _make_truncated_response(output: list) -> MagicMock: + mock_response = MagicMock() + mock_response.error = None + mock_response.status = "incomplete" + incomplete_details = MagicMock() + incomplete_details.reason = "max_output_tokens" + mock_response.incomplete_details = incomplete_details + mock_response.output = output + return mock_response + + +def test_is_truncated_response_detects_max_output_tokens(target: OpenAIResponseTarget): + """_is_truncated_response is True only for incomplete status with a max_output_tokens reason.""" + truncated = _make_truncated_response(output=[]) + assert target._is_truncated_response(truncated) is True + + content_filtered = _make_truncated_response(output=[]) + content_filtered.incomplete_details.reason = "content_filter" + assert target._is_truncated_response(content_filtered) is False + + completed = MagicMock() + completed.status = "completed" + assert target._is_truncated_response(completed) is False + + +def test_validate_response_truncated_keeps_reasoning_and_empty_text( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece, caplog +): + """Truncated response with reasoning but empty text: keep reasoning, add graceful empty text, warn.""" + response = _make_truncated_response(output=[_make_reasoning_section(), _make_empty_message_section()]) + + with caplog.at_level(logging.WARNING): + result = target._validate_response(response, dummy_text_message_piece) + + assert isinstance(result, Message) + reasoning_pieces = [p for p in result.message_pieces if p.original_value_data_type == "reasoning"] + text_pieces = [p for p in result.message_pieces if p.original_value_data_type == "text"] + assert len(reasoning_pieces) == 1 + assert len(text_pieces) == 1 + assert text_pieces[0].original_value == "" + assert text_pieces[0].response_error == "empty" + assert "max_output_tokens" in caplog.text + + +def test_validate_response_truncated_keeps_partial_text( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece, caplog +): + """Truncated response with partial visible text keeps it (error=none), still warns, no empty piece added.""" + response = _make_truncated_response(output=[_make_reasoning_section(), _make_message_section("Partial answer")]) + + with caplog.at_level(logging.WARNING): + result = target._validate_response(response, dummy_text_message_piece) + + assert isinstance(result, Message) + text_pieces = [p for p in result.message_pieces if p.original_value_data_type == "text"] + assert len(text_pieces) == 1 + assert text_pieces[0].original_value == "Partial answer" + assert text_pieces[0].response_error == "none" + assert "max_output_tokens" in caplog.text + + +def test_validate_response_truncated_empty_output_returns_graceful_empty( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece, caplog +): + """Truncated response with no output returns a single graceful empty text piece, warns, does not raise.""" + response = _make_truncated_response(output=[]) + + with caplog.at_level(logging.WARNING): + result = target._validate_response(response, dummy_text_message_piece) + + assert isinstance(result, Message) + assert len(result.message_pieces) == 1 + assert result.message_pieces[0].original_value == "" + assert result.message_pieces[0].response_error == "empty" + assert "max_output_tokens" in caplog.text + + +def test_validate_response_truncated_skips_partial_tool_call( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece, caplog +): + """A partial function_call in a truncated response is skipped so it cannot re-enter the agentic loop.""" + func_section = MagicMock() + func_section.type = "function_call" + func_section.call_id = "call_1" + func_section.name = "do_thing" + func_section.arguments = "{}" + response = _make_truncated_response(output=[_make_reasoning_section(), func_section]) + + with caplog.at_level(logging.WARNING): + result = target._validate_response(response, dummy_text_message_piece) + + data_types = [p.original_value_data_type for p in result.message_pieces] + assert "function_call" not in data_types + assert "reasoning" in data_types + assert any(p.original_value_data_type == "text" and p.response_error == "empty" for p in result.message_pieces) + + async def test_construct_message_from_response(target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece): """Test _construct_message_from_response parses output sections.""" mock_response = MagicMock() From 5ec85b12ed00fbfc6989c767a497fda6712f04c5 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:58:27 -0700 Subject: [PATCH 3/3] Keep _validate_response validation-only; build truncated messages in construction Address review feedback: _validate_response no longer returns a Message. It now only validates (warns and returns None on token-limit truncation, raises on genuinely empty responses so retries still fire). Building the graceful-empty or partial truncated Message moves into _construct_message_from_response_async in both OpenAIChatTarget and OpenAIResponseTarget, and the shared _validate_response return type narrows to None. Behavior is unchanged; responsibilities are cleaner. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 00b73356-ae65-4d83-8220-ebad37cc5850 --- .../openai/openai_chat_target.py | 52 ++++++------ .../openai/openai_response_target.py | 84 ++++++++----------- pyrit/prompt_target/openai/openai_target.py | 19 ++--- .../target/test_openai_chat_target.py | 24 +++++- .../target/test_openai_response_target.py | 47 ++++++----- 5 files changed, 116 insertions(+), 110 deletions(-) diff --git a/pyrit/prompt_target/openai/openai_chat_target.py b/pyrit/prompt_target/openai/openai_chat_target.py index cdcd28a480..56153fca2a 100644 --- a/pyrit/prompt_target/openai/openai_chat_target.py +++ b/pyrit/prompt_target/openai/openai_chat_target.py @@ -282,7 +282,7 @@ def _extract_partial_content(self, response: Any) -> str | None: pass return None - def _validate_response(self, response: Any, request: MessagePiece) -> Message | None: + def _validate_response(self, response: Any, request: MessagePiece) -> None: """ Validate a Chat Completions API response for errors. @@ -291,17 +291,16 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | - Invalid finish_reason - At least one valid response type (text content, audio, or tool_calls) + A ``finish_reason == "length"`` (token-limit truncation) response is treated as valid, with a + warning, so that ``_construct_message_from_response_async`` can preserve any partial content + or fall back to a graceful empty response. Genuinely empty responses (no truncation) are + raised so the retry logic can attempt to get a complete response. Content filter responses + are handled separately by ``_check_content_filter``. + Args: response: The ChatCompletion response from OpenAI SDK. request: The original request MessagePiece. - Returns: - None when the response is valid and should be constructed normally. Returns an empty - response ``Message`` (with ``error="empty"``) when the response was truncated at the - token limit (``finish_reason="length"``) before producing any visible output, so the - run continues instead of raising. Content filter responses are handled separately by - ``_check_content_filter``. - Raises: PyritException: For unexpected response structures or finish reasons. EmptyResponseException: When the API returns an empty response that was not caused by @@ -322,12 +321,10 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | message=f"Unknown finish_reason {finish_reason} from response: {response.model_dump_json()}" ) - # Check for at least one valid response type - has_content, has_audio, has_tool_calls = self._detect_response_content(choice.message) - # A "length" finish_reason means the model hit max_completion_tokens. Reasoning models spend # tokens on hidden reasoning before emitting visible output, so a low limit can truncate the - # answer or leave it empty. This may be a deliberate configuration, so warn instead of raising. + # answer or leave it empty. This may be a deliberate configuration, so warn instead of raising + # and let construction preserve partial content or fall back to a graceful empty response. if finish_reason == "length": logger.warning( "The response was truncated because it reached the token limit (finish_reason='length'). " @@ -335,23 +332,17 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | "low max_completion_tokens can truncate or empty the response. Increase max_completion_tokens " "if you expected complete content." ) - if not (has_content or has_audio or has_tool_calls): - return construct_response_from_request( - request=request, - response_text_pieces=[""], - response_type="text", - error="empty", - ) - return None + return + # Genuinely empty responses (no truncation) are usually a transient server-side issue; raise + # so the retry logic can attempt to get a complete response. + has_content, has_audio, has_tool_calls = self._detect_response_content(choice.message) if not (has_content or has_audio or has_tool_calls): logger.error("The chat returned an empty response (no content, audio, or tool_calls).") raise EmptyResponseException( message="The chat returned an empty response (no content, audio, or tool_calls)." ) - return None - def _detect_response_content(self, message: Any) -> tuple[bool, bool, bool]: """ Detect what content types are present in a ChatCompletion message. @@ -421,9 +412,12 @@ async def _construct_message_from_response_async(self, response: Any, request: M Message: Constructed message with one or more MessagePiece entries. Raises: - EmptyResponseException: If the response contains no content, audio, or tool calls. + EmptyResponseException: If a non-truncated response contains no content, audio, or tool + calls. A truncated (``finish_reason == "length"``) response with no content instead + yields a graceful empty piece so the run continues. """ - message = response.choices[0].message + choice = response.choices[0] + message = choice.message has_content, has_audio, has_tool_calls = self._detect_response_content(message) pieces: list[MessagePiece] = [] @@ -483,6 +477,16 @@ async def _construct_message_from_response_async(self, response: Any, request: M pieces.append(tool_piece) if not pieces: + # A truncated (finish_reason == "length") response may legitimately produce no content; + # return a graceful empty piece so the run continues. Validation already raised for + # genuinely empty (non-truncated) responses. + if choice.finish_reason == "length": + return construct_response_from_request( + request=request, + response_text_pieces=[""], + response_type="text", + error="empty", + ) raise EmptyResponseException(message="Failed to extract any response content.") # Capture token usage from the API response and store in the first piece's metadata diff --git a/pyrit/prompt_target/openai/openai_response_target.py b/pyrit/prompt_target/openai/openai_response_target.py index 42ce949a44..65fa44aaf2 100644 --- a/pyrit/prompt_target/openai/openai_response_target.py +++ b/pyrit/prompt_target/openai/openai_response_target.py @@ -497,7 +497,7 @@ def _extract_partial_content(self, response: Any) -> str | None: except (AttributeError, IndexError, TypeError): return None - def _validate_response(self, response: Any, request: MessagePiece) -> Message | None: + def _validate_response(self, response: Any, request: MessagePiece) -> None: """ Validate a Response API response for errors. @@ -507,17 +507,16 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | - Invalid status - Empty output + Truncation is treated as valid, with a warning, so that + ``_construct_message_from_response_async`` can preserve any completed output (reasoning, + partial text) or fall back to a graceful empty response. Genuinely empty responses (no + truncation) are raised so the retry logic can attempt to get a complete response. Content + filter responses are handled separately by ``_check_content_filter``. + Args: response: The Response object from the OpenAI SDK. request: The original request MessagePiece. - Returns: - None when the response is valid and should be constructed normally. Returns a Message - (built from whatever completed output exists, with a graceful empty text piece when no - visible text was produced) when the response was truncated at the token limit, so the - run continues instead of raising. Content filter responses are handled separately by - ``_check_content_filter``. - Raises: PyritException: For unexpected response structures or errors. EmptyResponseException: When the API returns no valid output (and was not truncated). @@ -530,8 +529,8 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | # Truncation: the model hit max_output_tokens. Mirroring OpenAIChatTarget's handling of # finish_reason == "length", warn instead of raising so the run continues -- reasoning models # can spend the whole budget on hidden reasoning before emitting a visible answer, and a low - # limit may be a deliberate configuration. Preserve any completed output (reasoning, partial - # text) and fall back to a graceful empty response. + # limit may be a deliberate configuration. Construction preserves any completed output + # (reasoning, partial text) and falls back to a graceful empty response. if self._is_truncated_response(response): logger.warning( "The response was truncated because it reached the token limit " @@ -539,7 +538,7 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | "hidden reasoning in addition to the visible answer, so a low max_output_tokens can " "truncate or empty the response. Increase max_output_tokens if you expected complete content." ) - return self._build_truncated_message(response=response, request=request) + return # Check status - should be "completed" for successful responses if response.status != "completed": @@ -550,8 +549,6 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | logger.error("The response returned no valid output.") raise EmptyResponseException(message="The response returned an empty response.") - return None - def _is_truncated_response(self, response: Any) -> bool: """ Return True if the response was cut off by the ``max_output_tokens`` limit. @@ -572,45 +569,15 @@ def _is_truncated_response(self, response: Any) -> bool: reason = getattr(incomplete_details, "reason", None) if incomplete_details else None return reason == "max_output_tokens" - def _build_truncated_message(self, *, response: Any, request: MessagePiece) -> Message: - """ - Build a Message from a truncated response, tolerating empty output sections. - - Preserves the model's content pieces (reasoning and any partial visible text) and appends - a graceful empty text piece (``error="empty"``) when no visible text was produced, so the - run continues instead of raising. Partial tool/function-call sections are skipped because an - incomplete call must not re-enter the agentic loop. - - Args: - response: A Response object from the OpenAI SDK. - request: The original request MessagePiece. - - Returns: - Message: The preserved pieces plus, when no visible text exists, a graceful empty piece. - """ - pieces: list[MessagePiece] = [] - has_text = False - for section in getattr(response, "output", None) or []: - piece = self._parse_response_output_section( - section=section, message_piece=request, error=None, tolerate_empty=True - ) - if piece is None or piece.original_value_data_type not in ("text", "reasoning"): - continue - pieces.append(piece) - if piece.original_value_data_type == "text" and piece.original_value: - has_text = True - - if not has_text: - empty_piece = construct_response_from_request( - request=request, response_text_pieces=[""], response_type="text", error="empty" - ).message_pieces[0] - pieces.append(empty_piece) - return Message(message_pieces=pieces) - async def _construct_message_from_response_async(self, response: Any, request: MessagePiece) -> Message: """ Construct a Message from a Response API response. + For a truncated response (see ``_is_truncated_response``), empty output sections are + tolerated, partial tool/function calls are skipped so an incomplete call cannot re-enter the + agentic loop, and a graceful empty text piece is appended when no visible text was produced. + Reasoning and any partial text are always preserved. + Args: response: The Response object from OpenAI SDK. request: The original request MessagePiece. @@ -618,17 +585,36 @@ async def _construct_message_from_response_async(self, response: Any, request: M Returns: Message: Constructed message with extracted content from output sections. """ + truncated = self._is_truncated_response(response) + # Extract and parse message pieces from validated output sections extracted_response_pieces: list[MessagePiece] = [] - for section in response.output: + has_text = False + for section in getattr(response, "output", None) or []: piece = self._parse_response_output_section( section=section, message_piece=request, error=None, # error is already handled in validation + tolerate_empty=truncated, ) if piece is None: continue + # On truncation, keep only reasoning/text; a partial tool or function call must not + # re-enter the agentic loop. + if truncated and piece.original_value_data_type not in ("text", "reasoning"): + continue extracted_response_pieces.append(piece) + if piece.original_value_data_type == "text" and piece.original_value: + has_text = True + + if truncated and not has_text: + empty_piece = construct_response_from_request( + request=request, + response_text_pieces=[""], + response_type="text", + error="empty", + ).message_pieces[0] + extracted_response_pieces.append(empty_piece) return Message(message_pieces=extracted_response_pieces) diff --git a/pyrit/prompt_target/openai/openai_target.py b/pyrit/prompt_target/openai/openai_target.py index 3700ebc370..6117afe425 100644 --- a/pyrit/prompt_target/openai/openai_target.py +++ b/pyrit/prompt_target/openai/openai_target.py @@ -448,10 +448,8 @@ async def _handle_openai_request_async( if self._check_content_filter(response): return self._handle_content_filter_response(response, request_piece) - # Validate response via subclass implementation - error_message = self._validate_response(response, request_piece) - if error_message: - return error_message + # Validate response via subclass implementation (raises on invalid responses) + self._validate_response(response, request_piece) # Construct and return Message from validated response return await self._construct_message_from_response_async(response, request_piece) @@ -604,24 +602,21 @@ def _extract_partial_content(self, response: Any) -> str | None: """ return None - def _validate_response(self, response: Any, request: MessagePiece) -> Message | None: + def _validate_response(self, response: Any, request: MessagePiece) -> None: """ - Validate the response and return error Message if needed. + Validate the response, raising if it is invalid. - Override this method in subclasses that need custom response validation. - Default implementation returns None (no validation errors). + Override this method in subclasses that need custom response validation. Validation only + inspects the response; constructing the resulting Message is the responsibility of + ``_construct_message_from_response_async``. The default implementation is a no-op. Args: response: The response object from OpenAI SDK. request: The original request MessagePiece. - Returns: - Message | None: Error Message if validation fails, None otherwise. - Raises: Various exceptions for validation failures. """ - return None @abstractmethod def _set_openai_env_configuration_vars(self) -> None: diff --git a/tests/unit/prompt_target/target/test_openai_chat_target.py b/tests/unit/prompt_target/target/test_openai_chat_target.py index 70a78e7bd5..cccdccf4a4 100644 --- a/tests/unit/prompt_target/target/test_openai_chat_target.py +++ b/tests/unit/prompt_target/target/test_openai_chat_target.py @@ -1048,17 +1048,35 @@ def test_validate_response_success_length(target: OpenAIChatTarget, dummy_text_m assert "finish_reason='length'" in caplog.text -def test_validate_response_length_empty_returns_empty_response( +def test_validate_response_length_empty_does_not_raise( target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece, caplog ): - """Test _validate_response returns a graceful empty response (no raise) when truncated before any output.""" + """Test _validate_response treats a truncated-but-empty response as valid (warns, does not raise).""" mock_response = create_mock_completion(content="", finish_reason="length") with caplog.at_level(logging.WARNING): result = target._validate_response(mock_response, dummy_text_message_piece) + assert result is None + assert "finish_reason='length'" in caplog.text + + +async def test_construct_message_length_empty_returns_graceful_empty( + target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece +): + """Test construction returns a graceful empty piece when a truncated response produced no content.""" + mock_response = create_mock_completion(content="", finish_reason="length") + result = await target._construct_message_from_response_async(mock_response, dummy_text_message_piece) assert isinstance(result, Message) assert result.message_pieces[0].original_value == "" assert result.message_pieces[0].response_error == "empty" - assert "finish_reason='length'" in caplog.text + + +async def test_construct_message_empty_non_truncated_raises( + target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece +): + """Test construction raises for a genuinely empty (non-truncated) response so retries can kick in.""" + mock_response = create_mock_completion(content="", finish_reason="stop") + with pytest.raises(EmptyResponseException, match="Failed to extract any response content"): + await target._construct_message_from_response_async(mock_response, dummy_text_message_piece) def test_validate_response_no_choices(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece): diff --git a/tests/unit/prompt_target/target/test_openai_response_target.py b/tests/unit/prompt_target/target/test_openai_response_target.py index 8a4a04fc59..668f2b0b37 100644 --- a/tests/unit/prompt_target/target/test_openai_response_target.py +++ b/tests/unit/prompt_target/target/test_openai_response_target.py @@ -1241,60 +1241,64 @@ def test_is_truncated_response_detects_max_output_tokens(target: OpenAIResponseT assert target._is_truncated_response(completed) is False -def test_validate_response_truncated_keeps_reasoning_and_empty_text( +def test_validate_response_truncated_warns_and_does_not_raise( target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece, caplog ): - """Truncated response with reasoning but empty text: keep reasoning, add graceful empty text, warn.""" + """Truncation is treated as valid: _validate_response warns and returns None (does not raise).""" response = _make_truncated_response(output=[_make_reasoning_section(), _make_empty_message_section()]) with caplog.at_level(logging.WARNING): result = target._validate_response(response, dummy_text_message_piece) - assert isinstance(result, Message) + assert result is None + assert "max_output_tokens" in caplog.text + + +async def test_construct_message_truncated_keeps_reasoning_and_empty_text( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece +): + """Truncated response with reasoning but empty text: keep reasoning, add a graceful empty text piece.""" + response = _make_truncated_response(output=[_make_reasoning_section(), _make_empty_message_section()]) + + result = await target._construct_message_from_response_async(response, dummy_text_message_piece) + reasoning_pieces = [p for p in result.message_pieces if p.original_value_data_type == "reasoning"] text_pieces = [p for p in result.message_pieces if p.original_value_data_type == "text"] assert len(reasoning_pieces) == 1 assert len(text_pieces) == 1 assert text_pieces[0].original_value == "" assert text_pieces[0].response_error == "empty" - assert "max_output_tokens" in caplog.text -def test_validate_response_truncated_keeps_partial_text( - target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece, caplog +async def test_construct_message_truncated_keeps_partial_text( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece ): - """Truncated response with partial visible text keeps it (error=none), still warns, no empty piece added.""" + """Truncated response with partial visible text keeps it (error=none), no empty piece added.""" response = _make_truncated_response(output=[_make_reasoning_section(), _make_message_section("Partial answer")]) - with caplog.at_level(logging.WARNING): - result = target._validate_response(response, dummy_text_message_piece) + result = await target._construct_message_from_response_async(response, dummy_text_message_piece) - assert isinstance(result, Message) text_pieces = [p for p in result.message_pieces if p.original_value_data_type == "text"] assert len(text_pieces) == 1 assert text_pieces[0].original_value == "Partial answer" assert text_pieces[0].response_error == "none" - assert "max_output_tokens" in caplog.text -def test_validate_response_truncated_empty_output_returns_graceful_empty( - target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece, caplog +async def test_construct_message_truncated_empty_output_returns_graceful_empty( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece ): - """Truncated response with no output returns a single graceful empty text piece, warns, does not raise.""" + """Truncated response with no output yields a single graceful empty text piece (does not raise).""" response = _make_truncated_response(output=[]) - with caplog.at_level(logging.WARNING): - result = target._validate_response(response, dummy_text_message_piece) + result = await target._construct_message_from_response_async(response, dummy_text_message_piece) - assert isinstance(result, Message) assert len(result.message_pieces) == 1 assert result.message_pieces[0].original_value == "" assert result.message_pieces[0].response_error == "empty" - assert "max_output_tokens" in caplog.text -def test_validate_response_truncated_skips_partial_tool_call( - target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece, caplog +async def test_construct_message_truncated_skips_partial_tool_call( + target: OpenAIResponseTarget, dummy_text_message_piece: MessagePiece ): """A partial function_call in a truncated response is skipped so it cannot re-enter the agentic loop.""" func_section = MagicMock() @@ -1304,8 +1308,7 @@ def test_validate_response_truncated_skips_partial_tool_call( func_section.arguments = "{}" response = _make_truncated_response(output=[_make_reasoning_section(), func_section]) - with caplog.at_level(logging.WARNING): - result = target._validate_response(response, dummy_text_message_piece) + result = await target._construct_message_from_response_async(response, dummy_text_message_piece) data_types = [p.original_value_data_type for p in result.message_pieces] assert "function_call" not in data_types