From 472bdfb5e21b60da96e69dfcb7c9fcd03c9e5a53 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 6 May 2026 05:39:47 +0000 Subject: [PATCH 1/3] fix: case-insensitive UserError blame check in _get_single_run_results + unit tests Agent-Logs-Url: https://github.com/Azure/azure-sdk-for-python/sessions/85522f11-e665-4eb7-b1bc-05e48272e704 Co-authored-by: posaninagendra <2713981+posaninagendra@users.noreply.github.com> --- .../ai/evaluation/_evaluate/_evaluate_aoai.py | 4 +- .../tests/unittests/test_evaluate_aoai.py | 86 ++++++++++++++++++- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate_aoai.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate_aoai.py index b0d3fb405746..04eb1056ac1e 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate_aoai.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate_aoai.py @@ -307,10 +307,12 @@ def _get_single_run_results( LOGGER.info(f"AOAI: Eval run {run_info['eval_run_id']} completed with status: {run_results.status}") if run_results.status != "completed": + error_code = getattr(getattr(run_results, "error", None), "code", None) + blame = ErrorBlame.USER_ERROR if isinstance(error_code, str) and error_code.lower() == "usererror" else ErrorBlame.UNKNOWN raise EvaluationException( message=f"AOAI evaluation run {run_info['eval_group_id']}/{run_info['eval_run_id']}" + f" failed with status {run_results.status}.", - blame=ErrorBlame.UNKNOWN, + blame=blame, category=ErrorCategory.FAILED_EXECUTION, target=ErrorTarget.AOAI_GRADER, ) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_aoai.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_aoai.py index 6b1eab60dd13..dee7b45517b4 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_aoai.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_aoai.py @@ -1,6 +1,8 @@ import pytest import copy -from azure.ai.evaluation._evaluate._evaluate_aoai import _combine_item_schemas +from unittest.mock import MagicMock, patch +from azure.ai.evaluation._evaluate._evaluate_aoai import _combine_item_schemas, _get_single_run_results +from azure.ai.evaluation._exceptions import ErrorBlame, EvaluationException @pytest.fixture @@ -119,3 +121,85 @@ def test_combine_item_schemas_with_external_properties_without_required(self, de assert data_source_config["item_schema"]["properties"] == expected_properties assert data_source_config["item_schema"]["required"] == expected_required + + +class TestGetSingleRunResultsBlame: + """Unit tests for blame attribution in _get_single_run_results.""" + + def _make_run_info(self, client): + return { + "client": client, + "eval_group_id": "group-1", + "eval_run_id": "run-1", + "grader_name_map": {}, + } + + @patch("azure.ai.evaluation._evaluate._evaluate_aoai._wait_for_run_conclusion") + @pytest.mark.parametrize("code", ["UserError", "usererror", "USERERROR", "uSeReRrOr"]) + def test_user_error_code_sets_user_blame(self, mock_wait, code): + """When run fails with error.code matching 'usererror' (case-insensitive), blame should be USER_ERROR.""" + run_result = MagicMock() + run_result.status = "failed" + run_result.error.code = code + mock_wait.return_value = run_result + client = MagicMock() + + with pytest.raises(EvaluationException) as exc_info: + _get_single_run_results(self._make_run_info(client)) + + assert exc_info.value.blame == ErrorBlame.USER_ERROR + + @patch("azure.ai.evaluation._evaluate._evaluate_aoai._wait_for_run_conclusion") + def test_non_user_error_code_sets_unknown_blame(self, mock_wait): + """When run fails with a non-UserError code, blame should be UNKNOWN.""" + run_result = MagicMock() + run_result.status = "failed" + run_result.error.code = "SystemError" + mock_wait.return_value = run_result + client = MagicMock() + + with pytest.raises(EvaluationException) as exc_info: + _get_single_run_results(self._make_run_info(client)) + + assert exc_info.value.blame == ErrorBlame.UNKNOWN + + @patch("azure.ai.evaluation._evaluate._evaluate_aoai._wait_for_run_conclusion") + def test_missing_error_attribute_sets_unknown_blame(self, mock_wait): + """When run fails and error attribute is absent, blame should be UNKNOWN.""" + run_result = MagicMock(spec=["status"]) + run_result.status = "failed" + mock_wait.return_value = run_result + client = MagicMock() + + with pytest.raises(EvaluationException) as exc_info: + _get_single_run_results(self._make_run_info(client)) + + assert exc_info.value.blame == ErrorBlame.UNKNOWN + + @patch("azure.ai.evaluation._evaluate._evaluate_aoai._wait_for_run_conclusion") + def test_error_present_but_code_missing_sets_unknown_blame(self, mock_wait): + """When error object exists but has no code attribute, blame should be UNKNOWN.""" + run_result = MagicMock() + run_result.status = "failed" + run_result.error = MagicMock(spec=[]) # error object without 'code' + mock_wait.return_value = run_result + client = MagicMock() + + with pytest.raises(EvaluationException) as exc_info: + _get_single_run_results(self._make_run_info(client)) + + assert exc_info.value.blame == ErrorBlame.UNKNOWN + + @patch("azure.ai.evaluation._evaluate._evaluate_aoai._wait_for_run_conclusion") + def test_error_is_none_sets_unknown_blame(self, mock_wait): + """When error attribute is None, blame should be UNKNOWN.""" + run_result = MagicMock() + run_result.status = "failed" + run_result.error = None + mock_wait.return_value = run_result + client = MagicMock() + + with pytest.raises(EvaluationException) as exc_info: + _get_single_run_results(self._make_run_info(client)) + + assert exc_info.value.blame == ErrorBlame.UNKNOWN From ef2623e65030fc7aad16be8cc598b4ef4355da91 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 6 May 2026 17:09:22 +0000 Subject: [PATCH 2/3] eval: improve error blame attribution and fix black formatting in _evaluate_aoai Agent-Logs-Url: https://github.com/Azure/azure-sdk-for-python/sessions/f4a706f3-5992-4bba-8ca8-21d3af1b16cb Co-authored-by: posaninagendra <2713981+posaninagendra@users.noreply.github.com> --- .../azure/ai/evaluation/_evaluate/_evaluate_aoai.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate_aoai.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate_aoai.py index 04eb1056ac1e..6d44320f9f9a 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate_aoai.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate_aoai.py @@ -308,7 +308,11 @@ def _get_single_run_results( LOGGER.info(f"AOAI: Eval run {run_info['eval_run_id']} completed with status: {run_results.status}") if run_results.status != "completed": error_code = getattr(getattr(run_results, "error", None), "code", None) - blame = ErrorBlame.USER_ERROR if isinstance(error_code, str) and error_code.lower() == "usererror" else ErrorBlame.UNKNOWN + blame = ( + ErrorBlame.USER_ERROR + if isinstance(error_code, str) and error_code.lower() == "usererror" + else ErrorBlame.UNKNOWN + ) raise EvaluationException( message=f"AOAI evaluation run {run_info['eval_group_id']}/{run_info['eval_run_id']}" + f" failed with status {run_results.status}.", From 29f8ed3dd28dd4d3d3934b7e999fb504ed6f6c6d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 6 May 2026 19:32:40 +0000 Subject: [PATCH 3/3] eval: fix case-insensitive UserError blame attribution in AOAI evaluation + changelog Agent-Logs-Url: https://github.com/Azure/azure-sdk-for-python/sessions/e699106e-c9a8-428c-ac53-4d7a6729bb71 Co-authored-by: posaninagendra <2713981+posaninagendra@users.noreply.github.com> --- sdk/evaluation/azure-ai-evaluation/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 91da8edbb732..8df529389d1d 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -14,6 +14,7 @@ ### Bugs Fixed +- Fixed error blame attribution in `_get_single_run_results` to perform a case-insensitive comparison when checking the AOAI error code for `UserError`, ensuring failed evaluation runs are correctly classified as user errors regardless of server-side casing. - Fixed row classification double-counting in `_calculate_aoai_evaluation_summary` where errored rows were counted separately and could also be counted as passed/failed. Rows are now classified into mutually exclusive buckets with priority: passed > failed > errored > skipped. - Fixed row classification where rows with empty or missing results lists were incorrectly counted as "passed" (the condition `passed_count == len(results) - error_count` evaluated `0 == 0` as True). - Fixed `_get_metric_result` prefix matching where shorter metric names (e.g., `xpia`) could match before longer, more-specific ones (e.g., `xpia_manipulated_content`). Now sorts by length descending for correct longest-prefix matching.