From d0f51732b4f70a11dad4cac9f1c2251f31d9038f Mon Sep 17 00:00:00 2001 From: A Vertex SDK engineer Date: Mon, 9 Mar 2026 05:32:30 -0700 Subject: [PATCH] feat: Add support for metric_resource_name in rubric generation PiperOrigin-RevId: 880805629 --- vertexai/_genai/_evals_common.py | 9 ++++++ vertexai/_genai/evals.py | 47 ++++++++++++++++++++++++++++---- vertexai/_genai/types/common.py | 19 +++++++++++++ 3 files changed, 69 insertions(+), 6 deletions(-) diff --git a/vertexai/_genai/_evals_common.py b/vertexai/_genai/_evals_common.py index 6bc0b2b728..de7deb801b 100644 --- a/vertexai/_genai/_evals_common.py +++ b/vertexai/_genai/_evals_common.py @@ -1329,6 +1329,15 @@ def _resolve_evaluation_run_metrics( for metric_instance in metrics: if isinstance(metric_instance, types.EvaluationRunMetric): resolved_metrics_list.append(metric_instance) + elif isinstance(metric_instance, str) and metric_instance.startswith( + "projects/" + ): + resolved_metrics_list.append( + types.EvaluationRunMetric( + metric=metric_instance.split("/")[-1], + metric_resource_name=metric_instance, + ) + ) elif isinstance( metric_instance, _evals_metric_loaders.LazyLoadedPrebuiltMetric ): diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index a045c6da79..c2088f111a 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -346,6 +346,13 @@ def _EvaluationRunMetric_from_vertex( _UnifiedMetric_from_vertex(getv(from_object, ["metricConfig"]), to_object), ) + if getv(from_object, ["metricResourceName"]) is not None: + setv( + to_object, + ["metric_resource_name"], + getv(from_object, ["metricResourceName"]), + ) + return to_object @@ -364,6 +371,13 @@ def _EvaluationRunMetric_to_vertex( _UnifiedMetric_to_vertex(getv(from_object, ["metric_config"]), to_object), ) + if getv(from_object, ["metric_resource_name"]) is not None: + setv( + to_object, + ["metricResourceName"], + getv(from_object, ["metric_resource_name"]), + ) + return to_object @@ -456,6 +470,13 @@ def _GenerateInstanceRubricsRequest_to_vertex( ), ) + if getv(from_object, ["metric_resource_name"]) is not None: + setv( + to_object, + ["metricResourceName"], + getv(from_object, ["metric_resource_name"]), + ) + if getv(from_object, ["config"]) is not None: setv(to_object, ["config"], getv(from_object, ["config"])) @@ -993,6 +1014,7 @@ def _generate_rubrics( types.PredefinedMetricSpecOrDict ] = None, rubric_generation_spec: Optional[types.RubricGenerationSpecOrDict] = None, + metric_resource_name: Optional[str] = None, config: Optional[types.RubricGenerationConfigOrDict] = None, ) -> types.GenerateInstanceRubricsResponse: """ @@ -1003,6 +1025,7 @@ def _generate_rubrics( contents=contents, predefined_rubric_generation_spec=predefined_rubric_generation_spec, rubric_generation_spec=rubric_generation_spec, + metric_resource_name=metric_resource_name, config=config, ) @@ -1505,17 +1528,21 @@ def generate_rubrics( rubric_type_ontology: Optional[list[str]] = None, predefined_spec_name: Optional[Union[str, "types.PrebuiltMetric"]] = None, metric_spec_parameters: Optional[dict[str, Any]] = None, + metric_resource_name: Optional[str] = None, config: Optional[types.RubricGenerationConfigOrDict] = None, ) -> types.EvaluationDataset: """Generates rubrics for each prompt in the source and adds them as a new column structured as a dictionary. You can generate rubrics by providing either: - 1. A `predefined_spec_name` to use a Vertex AI backend recipe. - 2. A `prompt_template` along with other configuration parameters + 1. A metric_resource_name` to use a pre-registered metric resource. + 2. A `predefined_spec_name` to use a Vertex AI backend recipe. + 3. A `prompt_template` along with other configuration parameters (`generator_model_config`, `rubric_content_type`, `rubric_type_ontology`) for custom rubric generation. - + with `metric_resource_name` taking + precedence over `predefined_spec_name`, and `predefined_spec_name` taking + precedence over `prompt_template` These two modes are mutually exclusive. Args: @@ -1543,6 +1570,9 @@ def generate_rubrics( Mutually exclusive with `prompt_template` and its related parameters. metric_spec_parameters: Optional. Parameters for the Predefined Metric, used to customize rubric generation. Only used if `predefined_spec_name` is set. + metric_resource_name: Optional. The resource name of a predefined metric to + use for rubric generation. If provided, this will override + `predefined_spec_name` and `prompt_template`. Example: {"guidelines": ["The response must be in Japanese."]} config: Optional. Configuration for the rubric generation process. @@ -1585,8 +1615,10 @@ def generate_rubrics( rubric_gen_spec = None predefined_spec = None - - if predefined_spec_name: + if metric_resource_name: + predefined_spec = None + rubric_gen_spec = None + elif predefined_spec_name: if prompt_template: logger.warning( "prompt_template is ignored when predefined_spec_name is provided." @@ -1643,7 +1675,7 @@ def generate_rubrics( rubric_gen_spec = types.RubricGenerationSpec.model_validate(spec_dict) else: raise ValueError( - "Either predefined_spec_name or prompt_template must be provided." + "Either metric_resource_name, predefined_spec_name or prompt_template must be provided." ) for _, row in prompts_df.iterrows(): @@ -1665,6 +1697,7 @@ def generate_rubrics( response = self._generate_rubrics( contents=contents, rubric_generation_spec=rubric_gen_spec, + metric_resource_name=metric_resource_name, predefined_rubric_generation_spec=predefined_spec, config=config, ) @@ -2249,6 +2282,7 @@ async def _generate_rubrics( types.PredefinedMetricSpecOrDict ] = None, rubric_generation_spec: Optional[types.RubricGenerationSpecOrDict] = None, + metric_resource_name: Optional[str] = None, config: Optional[types.RubricGenerationConfigOrDict] = None, ) -> types.GenerateInstanceRubricsResponse: """ @@ -2259,6 +2293,7 @@ async def _generate_rubrics( contents=contents, predefined_rubric_generation_spec=predefined_rubric_generation_spec, rubric_generation_spec=rubric_generation_spec, + metric_resource_name=metric_resource_name, config=config, ) diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index a8b765de01..d4139949f4 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -2326,6 +2326,9 @@ class LLMBasedMetricSpec(_common.BaseModel): default=None, description="""Dynamically generate rubrics using this specification.""", ) + metric_resource_name: Optional[str] = Field( + default=None, description="""The resource name of the metric definition.""" + ) class LLMBasedMetricSpecDict(TypedDict, total=False): @@ -2350,6 +2353,9 @@ class LLMBasedMetricSpecDict(TypedDict, total=False): rubric_generation_spec: Optional[RubricGenerationSpecDict] """Dynamically generate rubrics using this specification.""" + metric_resource_name: Optional[str] + """The resource name of the metric definition.""" + LLMBasedMetricSpecOrDict = Union[LLMBasedMetricSpec, LLMBasedMetricSpecDict] @@ -2482,6 +2488,9 @@ class EvaluationRunMetric(_common.BaseModel): metric_config: Optional[UnifiedMetric] = Field( default=None, description="""The unified metric used for evaluation run.""" ) + metric_resource_name: Optional[str] = Field( + default=None, description="""The resource name of the metric definition.""" + ) class EvaluationRunMetricDict(TypedDict, total=False): @@ -2493,6 +2502,9 @@ class EvaluationRunMetricDict(TypedDict, total=False): metric_config: Optional[UnifiedMetricDict] """The unified metric used for evaluation run.""" + metric_resource_name: Optional[str] + """The resource name of the metric definition.""" + EvaluationRunMetricOrDict = Union[EvaluationRunMetric, EvaluationRunMetricDict] @@ -5347,6 +5359,10 @@ class _GenerateInstanceRubricsRequest(_common.BaseModel): default=None, description="""Specification for how the rubrics should be generated.""", ) + metric_resource_name: Optional[str] = Field( + default=None, + description="""Registered metric resource name. If this field is set, the configuration provided in this field is used for rubric generation. The `predefined_rubric_generation_spec` and `rubric_generation_spec` fields will be ignored.""", + ) config: Optional[RubricGenerationConfig] = Field(default=None, description="""""") @@ -5367,6 +5383,9 @@ class _GenerateInstanceRubricsRequestDict(TypedDict, total=False): rubric_generation_spec: Optional[RubricGenerationSpecDict] """Specification for how the rubrics should be generated.""" + metric_resource_name: Optional[str] + """Registered metric resource name. If this field is set, the configuration provided in this field is used for rubric generation. The `predefined_rubric_generation_spec` and `rubric_generation_spec` fields will be ignored.""" + config: Optional[RubricGenerationConfigDict] """"""