feat: Add support for referencing registered metrics by resource name in evaluation APIs

PiperOrigin-RevId: 878604099

Author: vertex-sdk-bot

vertexai/_genai/_evals_common.py

@@ -1389,15 +1389,17 @@ def _resolve_evaluation_run_metrics(
 
 
 def _resolve_metrics(
-    metrics: list[types.Metric], api_client: Any
-) -> list[types.Metric]:
+    metrics: list[Any], api_client: Any
+) -> list[types.MetricSource]:
     """Resolves a list of metric instances, loading RubricMetric if necessary."""
     resolved_metrics_list = []
     for metric_instance in metrics:
         if isinstance(metric_instance, _evals_metric_loaders.LazyLoadedPrebuiltMetric):
             try:
                 resolved_metrics_list.append(
-                    metric_instance.resolve(api_client=api_client)
+                    types.MetricSource(
+                        metric=metric_instance.resolve(api_client=api_client)
+                    )
                 )
             except Exception as e:
                 logger.error(
@@ -1408,18 +1410,25 @@ def _resolve_metrics(
                 )
                 raise
         elif isinstance(metric_instance, types.Metric):
-            resolved_metrics_list.append(metric_instance)
+            resolved_metrics_list.append(types.MetricSource(metric=metric_instance))
         else:
             try:
                 metric_name_str = str(metric_instance)
+                if metric_name_str.lower().startswith("projects/"):
+                    resolved_metrics_list.append(
+                        types.MetricSource(metric_resource_name=metric_name_str)
+                    )
+                    continue
                 lazy_metric_instance = getattr(
                     _evals_metric_loaders.RubricMetric, metric_name_str.upper()
                 )
                 if isinstance(
                     lazy_metric_instance, _evals_metric_loaders.LazyLoadedPrebuiltMetric
                 ):
                     resolved_metrics_list.append(
-                        lazy_metric_instance.resolve(api_client=api_client)
+                        types.MetricSource(
+                            metric=lazy_metric_instance.resolve(api_client=api_client)
+                        )
                     )
                 else:
                     raise TypeError(

vertexai/_genai/_transformers.py

@@ -23,20 +23,28 @@
 
 
 def t_metrics(
-    metrics: list["types.MetricSubclass"],
+    metrics: list["types.MetricSource"],
     set_default_aggregation_metrics: bool = False,
 ) -> list[dict[str, Any]]:
     """Prepares the metric payload for the evaluation request.
 
     Args:
         metrics: A list of metrics used for evaluation.
         set_default_aggregation_metrics: Whether to set default aggregation metrics.
+
     Returns:
         A list of resolved metric payloads for the evaluation request.
     """
     metrics_payload = []
 
-    for metric in metrics:
+    for metric_source in metrics:
+        if metric_source.metric_resource_name:
+            metrics_payload.append(
+                {"metric_resource_name": metric_source.metric_resource_name}
+            )
+            continue
+
+        metric = metric_source.metric
         metric_payload_item: dict[str, Any] = {}
 
         metric_name = getv(metric, ["name"]).lower()

vertexai/_genai/evals.py

@@ -232,11 +232,7 @@ def _EvaluateInstancesRequestParameters_to_vertex(
         setv(to_object, ["autoraterConfig"], getv(from_object, ["autorater_config"]))
 
     if getv(from_object, ["metrics"]) is not None:
-        setv(
-            to_object,
-            ["metrics"],
-            [item for item in t.t_metrics(getv(from_object, ["metrics"]))],
-        )
+        setv(to_object, ["metricSources"], t.t_metrics(getv(from_object, ["metrics"])))
 
     if getv(from_object, ["instance"]) is not None:
         setv(
@@ -346,6 +342,13 @@ def _EvaluationRunMetric_from_vertex(
             _UnifiedMetric_from_vertex(getv(from_object, ["metricConfig"]), to_object),
         )
 
+    if getv(from_object, ["metricResourceName"]) is not None:
+        setv(
+            to_object,
+            ["metric_resource_name"],
+            getv(from_object, ["metricResourceName"]),
+        )
+
     return to_object
 
 
@@ -364,6 +367,13 @@ def _EvaluationRunMetric_to_vertex(
             _UnifiedMetric_to_vertex(getv(from_object, ["metric_config"]), to_object),
         )
 
+    if getv(from_object, ["metric_resource_name"]) is not None:
+        setv(
+            to_object,
+            ["metricResourceName"],
+            getv(from_object, ["metric_resource_name"]),
+        )
+
     return to_object
 
 
@@ -440,6 +450,13 @@ def _GenerateInstanceRubricsRequest_to_vertex(
     if getv(from_object, ["contents"]) is not None:
         setv(to_object, ["contents"], getv(from_object, ["contents"]))
 
+    if getv(from_object, ["metric_resource_name"]) is not None:
+        setv(
+            to_object,
+            ["metricResourceName"],
+            getv(from_object, ["metric_resource_name"]),
+        )
+
     if getv(from_object, ["predefined_rubric_generation_spec"]) is not None:
         setv(
             to_object,
@@ -989,6 +1006,7 @@ def _generate_rubrics(
         self,
         *,
         contents: list[genai_types.ContentOrDict],
+        metric_resource_name: Optional[str] = None,
         predefined_rubric_generation_spec: Optional[
             types.PredefinedMetricSpecOrDict
         ] = None,
@@ -1001,6 +1019,7 @@ def _generate_rubrics(
 
         parameter_model = types._GenerateInstanceRubricsRequest(
             contents=contents,
+            metric_resource_name=metric_resource_name,
             predefined_rubric_generation_spec=predefined_rubric_generation_spec,
             rubric_generation_spec=rubric_generation_spec,
             config=config,
@@ -2245,6 +2264,7 @@ async def _generate_rubrics(
         self,
         *,
         contents: list[genai_types.ContentOrDict],
+        metric_resource_name: Optional[str] = None,
         predefined_rubric_generation_spec: Optional[
             types.PredefinedMetricSpecOrDict
         ] = None,
@@ -2257,6 +2277,7 @@ async def _generate_rubrics(
 
         parameter_model = types._GenerateInstanceRubricsRequest(
             contents=contents,
+            metric_resource_name=metric_resource_name,
             predefined_rubric_generation_spec=predefined_rubric_generation_spec,
             rubric_generation_spec=rubric_generation_spec,
             config=config,

vertexai/_genai/types/__init__.py

@@ -690,6 +690,9 @@
 from .common import Metric
 from .common import MetricDict
 from .common import MetricOrDict
+from .common import MetricSource
+from .common import MetricSourceDict
+from .common import MetricSourceOrDict
 from .common import MetricPromptBuilder
 from .common import MetricResult
 from .common import MetricResultDict
@@ -1505,6 +1508,9 @@
     "Metric",
     "MetricDict",
     "MetricOrDict",
+    "MetricSource",
+    "MetricSourceDict",
+    "MetricSourceOrDict",
     "MetricResult",
     "MetricResultDict",
     "MetricResultOrDict",

vertexai/_genai/types/common.py

@@ -2326,6 +2326,9 @@ class LLMBasedMetricSpec(_common.BaseModel):
         default=None,
         description="""Dynamically generate rubrics using this specification.""",
     )
+    metric_resource_name: Optional[str] = Field(
+        default=None, description="""The resource name of the metric definition."""
+    )
 
 
 class LLMBasedMetricSpecDict(TypedDict, total=False):
@@ -2350,6 +2353,9 @@ class LLMBasedMetricSpecDict(TypedDict, total=False):
     rubric_generation_spec: Optional[RubricGenerationSpecDict]
     """Dynamically generate rubrics using this specification."""
 
+    metric_resource_name: Optional[str]
+    """The resource name of the metric definition."""
+
 
 LLMBasedMetricSpecOrDict = Union[LLMBasedMetricSpec, LLMBasedMetricSpecDict]
 
@@ -2482,6 +2488,9 @@ class EvaluationRunMetric(_common.BaseModel):
     metric_config: Optional[UnifiedMetric] = Field(
         default=None, description="""The unified metric used for evaluation run."""
     )
+    metric_resource_name: Optional[str] = Field(
+        default=None, description="""The resource name of the metric definition."""
+    )
 
 
 class EvaluationRunMetricDict(TypedDict, total=False):
@@ -2493,6 +2502,9 @@ class EvaluationRunMetricDict(TypedDict, total=False):
     metric_config: Optional[UnifiedMetricDict]
     """The unified metric used for evaluation run."""
 
+    metric_resource_name: Optional[str]
+    """The resource name of the metric definition."""
+
 
 EvaluationRunMetricOrDict = Union[EvaluationRunMetric, EvaluationRunMetricDict]
 
@@ -4640,6 +4652,29 @@ class MetricDict(TypedDict, total=False):
 MetricOrDict = Union[Metric, MetricDict]
 
 
+class MetricSource(_common.BaseModel):
+    """The metric source used for evaluation."""
+
+    metric: Optional[Metric] = Field(
+        default=None,
+        description="""Inline metric config.""",
+    )
+    metric_resource_name: Optional[str] = Field(
+        default=None,
+        description="""Resource name for registered metric.""",
+    )
+
+
+class MetricSourceDict(TypedDict, total=False):
+    """The metric source used for evaluation."""
+
+    metric: Optional[MetricDict]
+    metric_resource_name: Optional[str]
+
+
+MetricSourceOrDict = Union[MetricSource, MetricSourceDict]
+
+
 class _EvaluateInstancesRequestParameters(_common.BaseModel):
     """Parameters for evaluating instances."""
 
@@ -5334,6 +5369,14 @@ class _GenerateInstanceRubricsRequest(_common.BaseModel):
         default=None,
         description="""The prompt to generate rubrics from. For single-turn queries, this is a single instance. For multi-turn queries, this is a repeated field that contains conversation history + latest request.""",
     )
+    metric_resource_name: Optional[str] = Field(
+        default=None,
+        description="""The resource name of a registered metric. Rubric generation using
+      predefined metric spec or LLMBasedMetricSpec is supported. If this field is
+      set, the configuration provided in this field is used for rubric
+      generation. The `predefined_rubric_generation_spec` and
+      `rubric_generation_spec` fields will be ignored.""",
+    )
     predefined_rubric_generation_spec: Optional[PredefinedMetricSpec] = Field(
         default=None,
         description="""Specification for using the rubric generation configs of a pre-defined
@@ -5356,6 +5399,13 @@ class _GenerateInstanceRubricsRequestDict(TypedDict, total=False):
     contents: Optional[list[genai_types.ContentDict]]
     """The prompt to generate rubrics from. For single-turn queries, this is a single instance. For multi-turn queries, this is a repeated field that contains conversation history + latest request."""
 
+    metric_resource_name: Optional[str]
+    """The resource name of a registered metric. Rubric generation using
+      predefined metric spec or LLMBasedMetricSpec is supported. If this field is
+      set, the configuration provided in this field is used for rubric
+      generation. The `predefined_rubric_generation_spec` and
+      `rubric_generation_spec` fields will be ignored."""
+
     predefined_rubric_generation_spec: Optional[PredefinedMetricSpecDict]
     """Specification for using the rubric generation configs of a pre-defined
           metric, e.g. "generic_quality_v1" and "instruction_following_v1".