Skip to content

feat: Add support for metric_resource_name in rubric generation #6400

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
+69 −6
Open

feat: Add support for metric_resource_name in rubric generation #6400

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions vertexai/_genai/_evals_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1329,6 +1329,15 @@ def _resolve_evaluation_run_metrics(
for metric_instance in metrics:
if isinstance(metric_instance, types.EvaluationRunMetric):
resolved_metrics_list.append(metric_instance)
elif isinstance(metric_instance, str) and metric_instance.startswith(
"projects/"
):
resolved_metrics_list.append(
types.EvaluationRunMetric(
metric=metric_instance.split("/")[-1],
metric_resource_name=metric_instance,
)
)
elif isinstance(
metric_instance, _evals_metric_loaders.LazyLoadedPrebuiltMetric
):
Expand Down
47 changes: 41 additions & 6 deletions vertexai/_genai/evals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,13 @@ def _EvaluationRunMetric_from_vertex(
_UnifiedMetric_from_vertex(getv(from_object, ["metricConfig"]), to_object),
)

if getv(from_object, ["metricResourceName"]) is not None:
setv(
to_object,
["metric_resource_name"],
getv(from_object, ["metricResourceName"]),
)

return to_object


Expand All @@ -364,6 +371,13 @@ def _EvaluationRunMetric_to_vertex(
_UnifiedMetric_to_vertex(getv(from_object, ["metric_config"]), to_object),
)

if getv(from_object, ["metric_resource_name"]) is not None:
setv(
to_object,
["metricResourceName"],
getv(from_object, ["metric_resource_name"]),
)

return to_object


Expand Down Expand Up @@ -456,6 +470,13 @@ def _GenerateInstanceRubricsRequest_to_vertex(
),
)

if getv(from_object, ["metric_resource_name"]) is not None:
setv(
to_object,
["metricResourceName"],
getv(from_object, ["metric_resource_name"]),
)

if getv(from_object, ["config"]) is not None:
setv(to_object, ["config"], getv(from_object, ["config"]))

Expand Down Expand Up @@ -993,6 +1014,7 @@ def _generate_rubrics(
types.PredefinedMetricSpecOrDict
] = None,
rubric_generation_spec: Optional[types.RubricGenerationSpecOrDict] = None,
metric_resource_name: Optional[str] = None,
config: Optional[types.RubricGenerationConfigOrDict] = None,
) -> types.GenerateInstanceRubricsResponse:
"""
Expand All @@ -1003,6 +1025,7 @@ def _generate_rubrics(
contents=contents,
predefined_rubric_generation_spec=predefined_rubric_generation_spec,
rubric_generation_spec=rubric_generation_spec,
metric_resource_name=metric_resource_name,
config=config,
)

Expand Down Expand Up @@ -1505,17 +1528,21 @@ def generate_rubrics(
rubric_type_ontology: Optional[list[str]] = None,
predefined_spec_name: Optional[Union[str, "types.PrebuiltMetric"]] = None,
metric_spec_parameters: Optional[dict[str, Any]] = None,
metric_resource_name: Optional[str] = None,
config: Optional[types.RubricGenerationConfigOrDict] = None,
) -> types.EvaluationDataset:
"""Generates rubrics for each prompt in the source and adds them as a new column
structured as a dictionary.

You can generate rubrics by providing either:
1. A `predefined_spec_name` to use a Vertex AI backend recipe.
2. A `prompt_template` along with other configuration parameters
1. A metric_resource_name` to use a pre-registered metric resource.
2. A `predefined_spec_name` to use a Vertex AI backend recipe.
3. A `prompt_template` along with other configuration parameters
(`generator_model_config`, `rubric_content_type`, `rubric_type_ontology`)
for custom rubric generation.

with `metric_resource_name` taking
precedence over `predefined_spec_name`, and `predefined_spec_name` taking
precedence over `prompt_template`
These two modes are mutually exclusive.

Args:
Expand Down Expand Up @@ -1543,6 +1570,9 @@ def generate_rubrics(
Mutually exclusive with `prompt_template` and its related parameters.
metric_spec_parameters: Optional. Parameters for the Predefined Metric,
used to customize rubric generation. Only used if `predefined_spec_name` is set.
metric_resource_name: Optional. The resource name of a predefined metric to
use for rubric generation. If provided, this will override
`predefined_spec_name` and `prompt_template`.
Example: {"guidelines": ["The response must be in Japanese."]}
config: Optional. Configuration for the rubric generation process.

Expand Down Expand Up @@ -1585,8 +1615,10 @@ def generate_rubrics(

rubric_gen_spec = None
predefined_spec = None

if predefined_spec_name:
if metric_resource_name:
predefined_spec = None
rubric_gen_spec = None
elif predefined_spec_name:
if prompt_template:
logger.warning(
"prompt_template is ignored when predefined_spec_name is provided."
Expand Down Expand Up @@ -1643,7 +1675,7 @@ def generate_rubrics(
rubric_gen_spec = types.RubricGenerationSpec.model_validate(spec_dict)
else:
raise ValueError(
"Either predefined_spec_name or prompt_template must be provided."
"Either metric_resource_name, predefined_spec_name or prompt_template must be provided."
)

for _, row in prompts_df.iterrows():
Expand All @@ -1665,6 +1697,7 @@ def generate_rubrics(
response = self._generate_rubrics(
contents=contents,
rubric_generation_spec=rubric_gen_spec,
metric_resource_name=metric_resource_name,
predefined_rubric_generation_spec=predefined_spec,
config=config,
)
Expand Down Expand Up @@ -2249,6 +2282,7 @@ async def _generate_rubrics(
types.PredefinedMetricSpecOrDict
] = None,
rubric_generation_spec: Optional[types.RubricGenerationSpecOrDict] = None,
metric_resource_name: Optional[str] = None,
config: Optional[types.RubricGenerationConfigOrDict] = None,
) -> types.GenerateInstanceRubricsResponse:
"""
Expand All @@ -2259,6 +2293,7 @@ async def _generate_rubrics(
contents=contents,
predefined_rubric_generation_spec=predefined_rubric_generation_spec,
rubric_generation_spec=rubric_generation_spec,
metric_resource_name=metric_resource_name,
config=config,
)

Expand Down
19 changes: 19 additions & 0 deletions vertexai/_genai/types/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2326,6 +2326,9 @@ class LLMBasedMetricSpec(_common.BaseModel):
default=None,
description="""Dynamically generate rubrics using this specification.""",
)
metric_resource_name: Optional[str] = Field(
default=None, description="""The resource name of the metric definition."""
)


class LLMBasedMetricSpecDict(TypedDict, total=False):
Expand All @@ -2350,6 +2353,9 @@ class LLMBasedMetricSpecDict(TypedDict, total=False):
rubric_generation_spec: Optional[RubricGenerationSpecDict]
"""Dynamically generate rubrics using this specification."""

metric_resource_name: Optional[str]
"""The resource name of the metric definition."""


LLMBasedMetricSpecOrDict = Union[LLMBasedMetricSpec, LLMBasedMetricSpecDict]

Expand Down Expand Up @@ -2482,6 +2488,9 @@ class EvaluationRunMetric(_common.BaseModel):
metric_config: Optional[UnifiedMetric] = Field(
default=None, description="""The unified metric used for evaluation run."""
)
metric_resource_name: Optional[str] = Field(
default=None, description="""The resource name of the metric definition."""
)


class EvaluationRunMetricDict(TypedDict, total=False):
Expand All @@ -2493,6 +2502,9 @@ class EvaluationRunMetricDict(TypedDict, total=False):
metric_config: Optional[UnifiedMetricDict]
"""The unified metric used for evaluation run."""

metric_resource_name: Optional[str]
"""The resource name of the metric definition."""


EvaluationRunMetricOrDict = Union[EvaluationRunMetric, EvaluationRunMetricDict]

Expand Down Expand Up @@ -5347,6 +5359,10 @@ class _GenerateInstanceRubricsRequest(_common.BaseModel):
default=None,
description="""Specification for how the rubrics should be generated.""",
)
metric_resource_name: Optional[str] = Field(
default=None,
description="""Registered metric resource name. If this field is set, the configuration provided in this field is used for rubric generation. The `predefined_rubric_generation_spec` and `rubric_generation_spec` fields will be ignored.""",
)
config: Optional[RubricGenerationConfig] = Field(default=None, description="""""")


Expand All @@ -5367,6 +5383,9 @@ class _GenerateInstanceRubricsRequestDict(TypedDict, total=False):
rubric_generation_spec: Optional[RubricGenerationSpecDict]
"""Specification for how the rubrics should be generated."""

metric_resource_name: Optional[str]
"""Registered metric resource name. If this field is set, the configuration provided in this field is used for rubric generation. The `predefined_rubric_generation_spec` and `rubric_generation_spec` fields will be ignored."""

config: Optional[RubricGenerationConfigDict]
""""""

Expand Down
Loading