From eacc86c1fec17d48cf6b2ec93e8ef03560496f7d Mon Sep 17 00:00:00 2001 From: A Vertex SDK engineer Date: Mon, 9 Mar 2026 11:37:51 -0700 Subject: [PATCH] feat: GenAI Client(evals): onboarding agent_config (agents) in create_evaluation_run PiperOrigin-RevId: 880959518 --- vertexai/_genai/evals.py | 60 +++++++- vertexai/_genai/types/__init__.py | 18 +-- vertexai/_genai/types/common.py | 237 +++++++++++++++--------------- 3 files changed, 189 insertions(+), 126 deletions(-) diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index a045c6da79..ebff5e6366 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -94,7 +94,14 @@ def _CreateEvaluationRunParameters_to_vertex( setv(to_object, ["labels"], getv(from_object, ["labels"])) if getv(from_object, ["inference_configs"]) is not None: - setv(to_object, ["inferenceConfigs"], getv(from_object, ["inference_configs"])) + setv( + to_object, + ["inferenceConfigs"], + { + k: _EvaluationRunInferenceConfig_to_vertex(v, to_object) + for k, v in getv(from_object, ["inference_configs"]).items() + }, + ) if getv(from_object, ["config"]) is not None: setv(to_object, ["config"], getv(from_object, ["config"])) @@ -331,6 +338,52 @@ def _EvaluationRunConfig_to_vertex( return to_object +def _EvaluationRunInferenceConfig_from_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["agentConfig"]) is not None: + setv(to_object, ["agent_config"], getv(from_object, ["agentConfig"])) + + if getv(from_object, ["model"]) is not None: + setv(to_object, ["model"], getv(from_object, ["model"])) + + if getv(from_object, ["promptTemplate"]) is not None: + setv(to_object, ["prompt_template"], getv(from_object, ["promptTemplate"])) + + if getv(from_object, ["agentRunConfig"]) is not None: + setv(to_object, ["agent_run_config"], getv(from_object, ["agentRunConfig"])) + + if getv(from_object, ["agents"]) is not None: + setv(to_object, ["agent_configs"], getv(from_object, ["agents"])) + + return to_object + + +def _EvaluationRunInferenceConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["agent_config"]) is not None: + setv(to_object, ["agentConfig"], getv(from_object, ["agent_config"])) + + if getv(from_object, ["model"]) is not None: + setv(to_object, ["model"], getv(from_object, ["model"])) + + if getv(from_object, ["prompt_template"]) is not None: + setv(to_object, ["promptTemplate"], getv(from_object, ["prompt_template"])) + + if getv(from_object, ["agent_run_config"]) is not None: + setv(to_object, ["agentRunConfig"], getv(from_object, ["agent_run_config"])) + + if getv(from_object, ["agent_configs"]) is not None: + setv(to_object, ["agents"], getv(from_object, ["agent_configs"])) + + return to_object + + def _EvaluationRunMetric_from_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -423,7 +476,10 @@ def _EvaluationRun_from_vertex( setv( to_object, ["inference_configs"], - {k: v for k, v in getv(from_object, ["inferenceConfigs"]).items()}, + { + k: _EvaluationRunInferenceConfig_from_vertex(v, to_object) + for k, v in getv(from_object, ["inferenceConfigs"]).items() + }, ) if getv(from_object, ["labels"]) is not None: diff --git a/vertexai/_genai/types/__init__.py b/vertexai/_genai/types/__init__.py index e6bdc09cad..edbf8caec6 100644 --- a/vertexai/_genai/types/__init__.py +++ b/vertexai/_genai/types/__init__.py @@ -1346,6 +1346,15 @@ "EvaluationRunConfig", "EvaluationRunConfigDict", "EvaluationRunConfigOrDict", + "EvaluationRunAgentConfig", + "EvaluationRunAgentConfigDict", + "EvaluationRunAgentConfigOrDict", + "AgentRunConfig", + "AgentRunConfigDict", + "AgentRunConfigOrDict", + "EvaluationRunInferenceConfig", + "EvaluationRunInferenceConfigDict", + "EvaluationRunInferenceConfigOrDict", "CreateEvaluationRunConfig", "CreateEvaluationRunConfigDict", "CreateEvaluationRunConfigOrDict", @@ -1391,15 +1400,6 @@ "EvaluationResult", "EvaluationResultDict", "EvaluationResultOrDict", - "EvaluationRunAgentConfig", - "EvaluationRunAgentConfigDict", - "EvaluationRunAgentConfigOrDict", - "AgentRunConfig", - "AgentRunConfigDict", - "AgentRunConfigOrDict", - "EvaluationRunInferenceConfig", - "EvaluationRunInferenceConfigDict", - "EvaluationRunInferenceConfigOrDict", "EvaluationRun", "EvaluationRunDict", "EvaluationRunOrDict", diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index a8b765de01..4a7512d0fe 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -2576,6 +2576,126 @@ class EvaluationRunConfigDict(TypedDict, total=False): EvaluationRunConfigOrDict = Union[EvaluationRunConfig, EvaluationRunConfigDict] +class EvaluationRunAgentConfig(_common.BaseModel): + """This field is experimental and may change in future versions. + + Agent config for an evaluation run. + """ + + developer_instruction: Optional[genai_types.Content] = Field( + default=None, description="""The developer instruction for the agent.""" + ) + tools: Optional[list[genai_types.Tool]] = Field( + default=None, description="""The tools available to the agent.""" + ) + + +class EvaluationRunAgentConfigDict(TypedDict, total=False): + """This field is experimental and may change in future versions. + + Agent config for an evaluation run. + """ + + developer_instruction: Optional[genai_types.ContentDict] + """The developer instruction for the agent.""" + + tools: Optional[list[genai_types.ToolDict]] + """The tools available to the agent.""" + + +EvaluationRunAgentConfigOrDict = Union[ + EvaluationRunAgentConfig, EvaluationRunAgentConfigDict +] + + +class AgentRunConfig(_common.BaseModel): + """Configuration for an Agent Run.""" + + session_input: Optional[evals_types.SessionInput] = Field( + default=None, description="""The session input to get agent running results.""" + ) + agent_engine: Optional[str] = Field( + default=None, description="""The resource name of the Agent Engine.""" + ) + user_simulator_config: Optional[evals_types.UserSimulatorConfig] = Field( + default=None, + description="""Used for multi-turn agent run. + Contains configuration for a user simulator that + uses an LLM to generate messages on behalf of the user.""", + ) + + +class AgentRunConfigDict(TypedDict, total=False): + """Configuration for an Agent Run.""" + + session_input: Optional[evals_types.SessionInput] + """The session input to get agent running results.""" + + agent_engine: Optional[str] + """The resource name of the Agent Engine.""" + + user_simulator_config: Optional[evals_types.UserSimulatorConfig] + """Used for multi-turn agent run. + Contains configuration for a user simulator that + uses an LLM to generate messages on behalf of the user.""" + + +AgentRunConfigOrDict = Union[AgentRunConfig, AgentRunConfigDict] + + +class EvaluationRunInferenceConfig(_common.BaseModel): + """This field is experimental and may change in future versions. + + Configuration that describes an agent. + """ + + agent_config: Optional[EvaluationRunAgentConfig] = Field( + default=None, description="""The agent config.""" + ) + model: Optional[str] = Field( + default=None, + description="""The fully qualified name of the publisher model or endpoint to use for inference.""", + ) + prompt_template: Optional[EvaluationRunPromptTemplate] = Field( + default=None, description="""The prompt template used for inference.""" + ) + agent_run_config: Optional[AgentRunConfig] = Field( + default=None, + description="""Configuration for Agent Run in evaluation management service.""", + ) + agent_configs: Optional[dict[str, evals_types.AgentConfig]] = Field( + default=None, + description="""A map of agent IDs to their respective agent config.""", + ) + + +class EvaluationRunInferenceConfigDict(TypedDict, total=False): + """This field is experimental and may change in future versions. + + Configuration that describes an agent. + """ + + agent_config: Optional[EvaluationRunAgentConfigDict] + """The agent config.""" + + model: Optional[str] + """The fully qualified name of the publisher model or endpoint to use for inference.""" + + prompt_template: Optional[EvaluationRunPromptTemplateDict] + """The prompt template used for inference.""" + + agent_run_config: Optional[AgentRunConfigDict] + """Configuration for Agent Run in evaluation management service.""" + + agent_configs: Optional[dict[str, evals_types.AgentConfig]] + """A map of agent IDs to their respective agent config.""" + + +EvaluationRunInferenceConfigOrDict = Union[ + EvaluationRunInferenceConfig, EvaluationRunInferenceConfigDict +] + + class CreateEvaluationRunConfig(_common.BaseModel): """Config to create an evaluation run.""" @@ -2608,7 +2728,7 @@ class _CreateEvaluationRunParameters(_common.BaseModel): default=None, description="""""" ) labels: Optional[dict[str, str]] = Field(default=None, description="""""") - inference_configs: Optional[dict[str, "EvaluationRunInferenceConfig"]] = Field( + inference_configs: Optional[dict[str, EvaluationRunInferenceConfig]] = Field( default=None, description="""""" ) config: Optional[CreateEvaluationRunConfig] = Field( @@ -2634,7 +2754,7 @@ class _CreateEvaluationRunParametersDict(TypedDict, total=False): labels: Optional[dict[str, str]] """""" - inference_configs: Optional[dict[str, "EvaluationRunInferenceConfigDict"]] + inference_configs: Optional[dict[str, EvaluationRunInferenceConfigDict]] """""" config: Optional[CreateEvaluationRunConfigDict] @@ -3252,119 +3372,6 @@ class EvaluationResultDict(TypedDict, total=False): EvaluationResultOrDict = Union[EvaluationResult, EvaluationResultDict] -class EvaluationRunAgentConfig(_common.BaseModel): - """This field is experimental and may change in future versions. - - Agent config for an evaluation run. - """ - - developer_instruction: Optional[genai_types.Content] = Field( - default=None, description="""The developer instruction for the agent.""" - ) - tools: Optional[list[genai_types.Tool]] = Field( - default=None, description="""The tools available to the agent.""" - ) - - -class EvaluationRunAgentConfigDict(TypedDict, total=False): - """This field is experimental and may change in future versions. - - Agent config for an evaluation run. - """ - - developer_instruction: Optional[genai_types.ContentDict] - """The developer instruction for the agent.""" - - tools: Optional[list[genai_types.ToolDict]] - """The tools available to the agent.""" - - -EvaluationRunAgentConfigOrDict = Union[ - EvaluationRunAgentConfig, EvaluationRunAgentConfigDict -] - - -class AgentRunConfig(_common.BaseModel): - """Configuration for an Agent Run.""" - - session_input: Optional[evals_types.SessionInput] = Field( - default=None, description="""The session input to get agent running results.""" - ) - agent_engine: Optional[str] = Field( - default=None, description="""The resource name of the Agent Engine.""" - ) - user_simulator_config: Optional[evals_types.UserSimulatorConfig] = Field( - default=None, - description="""Used for multi-turn agent run. - Contains configuration for a user simulator that - uses an LLM to generate messages on behalf of the user.""", - ) - - -class AgentRunConfigDict(TypedDict, total=False): - """Configuration for an Agent Run.""" - - session_input: Optional[evals_types.SessionInput] - """The session input to get agent running results.""" - - agent_engine: Optional[str] - """The resource name of the Agent Engine.""" - - user_simulator_config: Optional[evals_types.UserSimulatorConfig] - """Used for multi-turn agent run. - Contains configuration for a user simulator that - uses an LLM to generate messages on behalf of the user.""" - - -AgentRunConfigOrDict = Union[AgentRunConfig, AgentRunConfigDict] - - -class EvaluationRunInferenceConfig(_common.BaseModel): - """This field is experimental and may change in future versions. - - Configuration that describes an agent. - """ - - agent_config: Optional[EvaluationRunAgentConfig] = Field( - default=None, description="""The agent config.""" - ) - model: Optional[str] = Field( - default=None, - description="""The fully qualified name of the publisher model or endpoint to use for inference.""", - ) - prompt_template: Optional[EvaluationRunPromptTemplate] = Field( - default=None, description="""The prompt template used for inference.""" - ) - agent_run_config: Optional[AgentRunConfig] = Field( - default=None, - description="""Configuration for Agent Run in evaluation management service.""", - ) - - -class EvaluationRunInferenceConfigDict(TypedDict, total=False): - """This field is experimental and may change in future versions. - - Configuration that describes an agent. - """ - - agent_config: Optional[EvaluationRunAgentConfigDict] - """The agent config.""" - - model: Optional[str] - """The fully qualified name of the publisher model or endpoint to use for inference.""" - - prompt_template: Optional[EvaluationRunPromptTemplateDict] - """The prompt template used for inference.""" - - agent_run_config: Optional[AgentRunConfigDict] - """Configuration for Agent Run in evaluation management service.""" - - -EvaluationRunInferenceConfigOrDict = Union[ - EvaluationRunInferenceConfig, EvaluationRunInferenceConfigDict -] - - class EvaluationRun(_common.BaseModel): """Represents an evaluation run."""