Skip to content

Commit 9e5dbd6

Browse files
vertex-sdk-botcopybara-github
authored andcommitted
fix: GenAI Client(evals): Avoid required session_inputs column to eval dataset
PiperOrigin-RevId: 877610185
1 parent aa22f54 commit 9e5dbd6

2 files changed

Lines changed: 55 additions & 4 deletions

File tree

tests/unit/vertexai/genai/test_evals.py

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4439,6 +4439,49 @@ async def test_run_adk_user_simulation_missing_columns(
44394439
with pytest.raises(ValueError, match="User simulation requires"):
44404440
await _evals_common._run_adk_user_simulation(row, mock_agent)
44414441

4442+
@mock.patch("vertexai._genai._evals_common.ADK_SessionInput") # fmt: skip
4443+
@mock.patch("vertexai._genai._evals_common.EvaluationGenerator") # fmt: skip
4444+
@mock.patch("vertexai._genai._evals_common.LlmBackedUserSimulator") # fmt: skip
4445+
@mock.patch("vertexai._genai._evals_common.ConversationScenario") # fmt: skip
4446+
@mock.patch("vertexai._genai._evals_common.LlmBackedUserSimulatorConfig") # fmt: skip
4447+
@pytest.mark.asyncio
4448+
async def test_run_adk_user_simulation_missing_session_inputs(
4449+
self,
4450+
mock_config_cls,
4451+
mock_scenario_cls,
4452+
mock_simulator_cls,
4453+
mock_generator_cls,
4454+
mock_session_input_cls,
4455+
):
4456+
row = pd.Series(
4457+
{
4458+
"starting_prompt": "start",
4459+
"conversation_plan": "plan",
4460+
}
4461+
)
4462+
mock_agent = mock.Mock()
4463+
mock_invocation = mock.Mock()
4464+
mock_invocation.user_content.model_dump.return_value = {"text": "user msg"}
4465+
mock_invocation.final_response.model_dump.return_value = {"text": "agent msg"}
4466+
mock_invocation.intermediate_data = None
4467+
mock_invocation.creation_timestamp = 12345
4468+
mock_invocation.invocation_id = "turn1"
4469+
4470+
mock_generator_cls._generate_inferences_from_root_agent = mock.AsyncMock(
4471+
return_value=[mock_invocation]
4472+
)
4473+
4474+
await _evals_common._run_adk_user_simulation(row, mock_agent)
4475+
4476+
mock_scenario_cls.assert_called_once_with(
4477+
starting_prompt="start", conversation_plan="plan"
4478+
)
4479+
mock_session_input_cls.assert_called_once_with(
4480+
app_name="user_simulation_app",
4481+
user_id="user_simulation_default_user",
4482+
state={},
4483+
)
4484+
44424485

44434486
@pytest.mark.usefixtures("google_auth_mock")
44444487
class TestLLMMetricHandlerPayload:

vertexai/_genai/_evals_common.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -856,16 +856,24 @@ async def _run_adk_user_simulation(
856856
conversation_scenario=scenario, config=user_simulator_config
857857
)
858858

859-
initial_session = _get_session_inputs(row)
859+
try:
860+
initial_session = _get_session_inputs(row)
861+
app_name = initial_session.app_name or "user_simulation_app"
862+
user_id = initial_session.user_id or "user_simulation_default_user"
863+
state = initial_session.state or {}
864+
except (KeyError, TypeError, ValueError):
865+
app_name = "user_simulation_app"
866+
user_id = "user_simulation_default_user"
867+
state = {}
860868

861869
invocations = await EvaluationGenerator._generate_inferences_from_root_agent( # pylint: disable=protected-access
862870
root_agent=agent,
863871
user_simulator=user_simulator,
864872
reset_func=getattr(agent, "reset_data", None),
865873
initial_session=ADK_SessionInput(
866-
app_name=initial_session.app_name or "user_simulation_app",
867-
user_id=initial_session.user_id or "user_simulation_default_user",
868-
state=initial_session.state or {},
874+
app_name=app_name,
875+
user_id=user_id,
876+
state=state,
869877
),
870878
)
871879

0 commit comments

Comments
 (0)