diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index 11bec7fd9f..d91d352dc1 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -983,33 +983,6 @@ def _agent_data_response_text(agent_data: types.evals.AgentData) -> Optional[str return "".join(text_parts) or None -def _agent_resource_to_agent_info( - agent: str, api_client: BaseApiClient -) -> "types.evals.AgentInfo": - """Builds an `AgentInfo` from a Gemini Agents API agent resource name. - - Fetches the agent through the SDK's `api_client` (so replay recording is - preserved) via `_fetch_agent_config_dict` and derives a single-agent - `AgentInfo`: the agent's short name is the agents-map key and - `root_agent_id`. - - Args: - agent: The Gemini Agents API agent resource name - (`projects/{p}/locations/{l}/agents/{name}`). - api_client: The API client used to fetch the agent. - - Returns: - An `AgentInfo` describing the fetched agent. - """ - agent_config = _fetch_agent_config_dict(api_client, agent) - short_name = agent_config.agent_id - return types.evals.AgentInfo( # pytype: disable=missing-parameter - name=short_name, - agents={short_name: agent_config}, - root_agent_id=short_name, - ) - - _INTERACTION_TERMINAL_STATES = frozenset( ["completed", "failed", "cancelled", "incomplete", "budget_exceeded"] ) diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index 63fe6d1858..1f4aa7e1de 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -725,6 +725,11 @@ def _GenerateUserScenariosParameters_to_vertex( getv(from_object, ["allow_cross_region_model"]), ) + if getv(from_object, ["gemini_agent_config"]) is not None: + setv( + to_object, ["geminiAgentConfig"], getv(from_object, ["gemini_agent_config"]) + ) + return to_object @@ -1518,6 +1523,7 @@ def _generate_user_scenarios( ] = None, config: Optional[types.GenerateUserScenariosConfigOrDict] = None, allow_cross_region_model: Optional[bool] = None, + gemini_agent_config: Optional[types.GeminiAgentConfigOrDict] = None, ) -> types.GenerateUserScenariosResponse: """ Generates user scenarios for agent evaluation. @@ -1530,6 +1536,7 @@ def _generate_user_scenarios( user_scenario_generation_config=user_scenario_generation_config, config=config, allow_cross_region_model=allow_cross_region_model, + gemini_agent_config=gemini_agent_config, ) request_url_dict: Optional[dict[str, str]] @@ -2938,16 +2945,16 @@ def generate_conversation_scenarios( and the agent under test. Exactly one of `agent_info` or `agent` must be provided. When `agent` is - a Gemini Agents API agent resource name, the agent is fetched and an - `AgentInfo` is derived from it. + a Gemini Agents API agent resource name, the agent config is resolved + server-side via ``gemini_agent_config``. Args: agent_info: The agent info to generate user scenarios for. Mutually exclusive with `agent`. agent: A Gemini Agents API agent resource name (`projects/{p}/locations/{l}/agents/{name}`). When provided, the - agent is fetched and its configuration is used to build the agent - info. Mutually exclusive with `agent_info`. + server resolves the agent config directly. Mutually exclusive + with `agent_info`. config: Configuration for generating user scenarios. allow_cross_region_model: Opt-in flag to authorize cross-region routing for model inference. @@ -2967,8 +2974,10 @@ def generate_conversation_scenarios( "`agent` must be a Gemini Agents API agent resource name of the" " form projects/{project}/locations/{location}/agents/{agent}." ) - parsed_agent_info = _evals_common._agent_resource_to_agent_info( - agent, self._api_client + response = self._generate_user_scenarios( + gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent), + user_scenario_generation_config=config, + allow_cross_region_model=allow_cross_region_model, ) else: parsed_agent_info = ( @@ -2976,12 +2985,12 @@ def generate_conversation_scenarios( if isinstance(agent_info, dict) else agent_info ) - response = self._generate_user_scenarios( - agents=parsed_agent_info.agents, - root_agent_id=parsed_agent_info.root_agent_id, - user_scenario_generation_config=config, - allow_cross_region_model=allow_cross_region_model, - ) + response = self._generate_user_scenarios( + agents=parsed_agent_info.agents, + root_agent_id=parsed_agent_info.root_agent_id, + user_scenario_generation_config=config, + allow_cross_region_model=allow_cross_region_model, + ) return _evals_utils._postprocess_user_scenarios_response(response) def generate_loss_clusters( @@ -3683,6 +3692,7 @@ async def _generate_user_scenarios( ] = None, config: Optional[types.GenerateUserScenariosConfigOrDict] = None, allow_cross_region_model: Optional[bool] = None, + gemini_agent_config: Optional[types.GeminiAgentConfigOrDict] = None, ) -> types.GenerateUserScenariosResponse: """ Generates user scenarios for agent evaluation. @@ -3695,6 +3705,7 @@ async def _generate_user_scenarios( user_scenario_generation_config=user_scenario_generation_config, config=config, allow_cross_region_model=allow_cross_region_model, + gemini_agent_config=gemini_agent_config, ) request_url_dict: Optional[dict[str, str]] @@ -4729,16 +4740,16 @@ async def generate_conversation_scenarios( and the agent under test. Exactly one of `agent_info` or `agent` must be provided. When `agent` is - a Gemini Agents API agent resource name, the agent is fetched and an - `AgentInfo` is derived from it. + a Gemini Agents API agent resource name, the agent config is resolved + server-side via ``gemini_agent_config``. Args: agent_info: The agent info to generate user scenarios for. Mutually exclusive with `agent`. agent: A Gemini Agents API agent resource name (`projects/{p}/locations/{l}/agents/{name}`). When provided, the - agent is fetched and its configuration is used to build the agent - info. Mutually exclusive with `agent_info`. + server resolves the agent config directly. Mutually exclusive + with `agent_info`. config: Configuration for generating user scenarios. allow_cross_region_model: Opt-in flag to authorize cross-region routing for model inference. @@ -4758,8 +4769,10 @@ async def generate_conversation_scenarios( "`agent` must be a Gemini Agents API agent resource name of the" " form projects/{project}/locations/{location}/agents/{agent}." ) - parsed_agent_info = _evals_common._agent_resource_to_agent_info( - agent, self._api_client + response = await self._generate_user_scenarios( + gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent), + user_scenario_generation_config=config, + allow_cross_region_model=allow_cross_region_model, ) else: parsed_agent_info = ( @@ -4767,12 +4780,12 @@ async def generate_conversation_scenarios( if isinstance(agent_info, dict) else agent_info ) - response = await self._generate_user_scenarios( - agents=parsed_agent_info.agents, - root_agent_id=parsed_agent_info.root_agent_id, - user_scenario_generation_config=config, - allow_cross_region_model=allow_cross_region_model, - ) + response = await self._generate_user_scenarios( + agents=parsed_agent_info.agents, + root_agent_id=parsed_agent_info.root_agent_id, + user_scenario_generation_config=config, + allow_cross_region_model=allow_cross_region_model, + ) return _evals_utils._postprocess_user_scenarios_response(response) async def generate_loss_clusters( diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index 02820baf86..dd57c62b1e 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -5501,6 +5501,11 @@ class _GenerateUserScenariosParameters(_common.BaseModel): default=None, description="""Opt-in flag to authorize cross-region routing for LLM models.""", ) + gemini_agent_config: Optional[GeminiAgentConfig] = Field( + default=None, + description="""If set, the server derives the agents map and root_agent_id + from the referenced Gemini Agent server-side.""", + ) class _GenerateUserScenariosParametersDict(TypedDict, total=False): @@ -5524,6 +5529,10 @@ class _GenerateUserScenariosParametersDict(TypedDict, total=False): allow_cross_region_model: Optional[bool] """Opt-in flag to authorize cross-region routing for LLM models.""" + gemini_agent_config: Optional[GeminiAgentConfigDict] + """If set, the server derives the agents map and root_agent_id + from the referenced Gemini Agent server-side.""" + _GenerateUserScenariosParametersOrDict = Union[ _GenerateUserScenariosParameters, _GenerateUserScenariosParametersDict diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 962814a35d..e6dfa74c75 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -9431,19 +9431,9 @@ async def test_async_generate_conversation_scenarios(self): request_body = call_args[0][2] # Third positional arg is the request dict assert request_body.get("allowCrossRegionModel") is True - @mock.patch.object(_evals_common, "_fetch_agent_config_dict") - def test_generate_conversation_scenarios_from_gemini_agent( - self, mock_fetch_agent_config - ): - mock_fetch_agent_config.return_value = ( - agentplatform_genai_types.evals.AgentConfig( - agent_id="test-agent", - instruction="You are a helpful travel assistant.", - description="An agent that books flights.", - tools=[genai_types.Tool(google_search=genai_types.GoogleSearch())], - ) - ) - + def test_generate_conversation_scenarios_from_gemini_agent(self): + """When `agent` is a Gemini agent resource, gemini_agent_config is + forwarded to the server (no client-side synthesis).""" evals_module = evals.Evals(api_client_=self.mock_api_client) with mock.patch.object( @@ -9455,18 +9445,10 @@ def test_generate_conversation_scenarios_from_gemini_agent( config={"count": 2}, ) - mock_fetch_agent_config.assert_called_once_with( - self.mock_api_client, _TEST_GEMINI_AGENT - ) call_kwargs = mock_generate_user_scenarios.call_args.kwargs - assert call_kwargs["root_agent_id"] == "test-agent" - agents = call_kwargs["agents"] - assert "test-agent" in agents - derived_config = agents["test-agent"] - assert derived_config.instruction == "You are a helpful travel assistant." - assert derived_config.description == "An agent that books flights." - assert derived_config.tools is not None - assert derived_config.tools[0].google_search is not None + assert call_kwargs["gemini_agent_config"].gemini_agent == _TEST_GEMINI_AGENT + assert call_kwargs.get("agents") is None + assert call_kwargs.get("root_agent_id") is None def test_generate_conversation_scenarios_agent_and_agent_info_raises(self): evals_module = evals.Evals(api_client_=self.mock_api_client)