diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index d682d3be47..a3185e57df 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -224,6 +224,20 @@ def _CustomCodeExecutionSpec_to_vertex( return to_object +def _DeleteEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _DeleteEvaluationMetricParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1091,6 +1105,48 @@ def _UnifiedMetric_to_vertex( return to_object +def _UpdateEvaluationExperimentConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["update_mask"]) is not None: + setv( + parent_object, ["_query", "updateMask"], getv(from_object, ["update_mask"]) + ) + + if getv(from_object, ["display_name"]) is not None: + setv(parent_object, ["displayName"], getv(from_object, ["display_name"])) + + if getv(from_object, ["labels"]) is not None: + setv(parent_object, ["labels"], getv(from_object, ["labels"])) + + if getv(from_object, ["merge_strategy"]) is not None: + setv(parent_object, ["mergeStrategy"], getv(from_object, ["merge_strategy"])) + + if getv(from_object, ["metadata"]) is not None: + setv(parent_object, ["metadata"], getv(from_object, ["metadata"])) + + return to_object + + +def _UpdateEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + _UpdateEvaluationExperimentConfig_to_vertex( + getv(from_object, ["config"]), to_object + ) + + return to_object + + class Evals(_api_module.BaseModule): def create_evaluation_experiment( @@ -1509,6 +1565,89 @@ def _create_evaluation_set( self._api_client._verify_response(return_value) return return_value + def delete_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: + """ + Deletes an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to delete. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration for the delete operation. + + Returns: + The delete operation. + + """ + + parameter_model = types._DeleteEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _DeleteEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("delete", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationExperimentOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _delete_evaluation_metric( self, *, @@ -2459,6 +2598,90 @@ def _list_evaluation_metrics( self._api_client._verify_response(return_value) return return_value + def update_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Updates an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to update. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration specifying the fields to update (e.g. + display_name, labels, merge_strategy, metadata) and the update_mask. + + Returns: + The updated evaluation experiment. + + """ + + parameter_model = types._UpdateEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _UpdateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("patch", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def evaluate_instances( self, *, @@ -3970,6 +4193,91 @@ async def _create_evaluation_set( self._api_client._verify_response(return_value) return return_value + async def delete_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: + """ + Deletes an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to delete. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration for the delete operation. + + Returns: + The delete operation. + + """ + + parameter_model = types._DeleteEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _DeleteEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "delete", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationExperimentOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _delete_evaluation_metric( self, *, @@ -4944,6 +5252,92 @@ async def _list_evaluation_metrics( self._api_client._verify_response(return_value) return return_value + async def update_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Updates an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to update. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration specifying the fields to update (e.g. + display_name, labels, merge_strategy, metadata) and the update_mask. + + Returns: + The updated evaluation experiment. + + """ + + parameter_model = types._UpdateEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _UpdateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "patch", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def batch_evaluate( self, *, diff --git a/agentplatform/_genai/types/__init__.py b/agentplatform/_genai/types/__init__.py index e7612f28d1..6027cc3bc0 100644 --- a/agentplatform/_genai/types/__init__.py +++ b/agentplatform/_genai/types/__init__.py @@ -56,6 +56,7 @@ from .common import _DeleteAgentEngineSessionRequestParameters from .common import _DeleteAgentEngineTaskRequestParameters from .common import _DeleteDatasetRequestParameters +from .common import _DeleteEvaluationExperimentParameters from .common import _DeleteEvaluationMetricParameters from .common import _DeleteMultimodalDatasetRequestParameters from .common import _DeletePromptVersionRequestParameters @@ -159,6 +160,7 @@ from .common import _UpdateAgentEngineRequestParameters from .common import _UpdateAgentEngineSessionRequestParameters from .common import _UpdateDatasetParameters +from .common import _UpdateEvaluationExperimentParameters from .common import _UpdateMultimodalDatasetParameters from .common import _UpdateRagConfigRequestParameters from .common import _UpdateRagCorpusRequestParameters @@ -440,6 +442,12 @@ from .common import DeleteAgentEngineTaskConfig from .common import DeleteAgentEngineTaskConfigDict from .common import DeleteAgentEngineTaskConfigOrDict +from .common import DeleteEvaluationExperimentConfig +from .common import DeleteEvaluationExperimentConfigDict +from .common import DeleteEvaluationExperimentConfigOrDict +from .common import DeleteEvaluationExperimentOperation +from .common import DeleteEvaluationExperimentOperationDict +from .common import DeleteEvaluationExperimentOperationOrDict from .common import DeleteEvaluationMetricConfig from .common import DeleteEvaluationMetricConfigDict from .common import DeleteEvaluationMetricConfigOrDict @@ -1903,6 +1911,9 @@ from .common import UpdateAgentEngineSessionConfig from .common import UpdateAgentEngineSessionConfigDict from .common import UpdateAgentEngineSessionConfigOrDict +from .common import UpdateEvaluationExperimentConfig +from .common import UpdateEvaluationExperimentConfigDict +from .common import UpdateEvaluationExperimentConfigOrDict from .common import UpdatePromptConfig from .common import UpdatePromptConfigDict from .common import UpdatePromptConfigOrDict @@ -2173,6 +2184,12 @@ "EvaluationSet", "EvaluationSetDict", "EvaluationSetOrDict", + "DeleteEvaluationExperimentConfig", + "DeleteEvaluationExperimentConfigDict", + "DeleteEvaluationExperimentConfigOrDict", + "DeleteEvaluationExperimentOperation", + "DeleteEvaluationExperimentOperationDict", + "DeleteEvaluationExperimentOperationOrDict", "DeleteEvaluationMetricConfig", "DeleteEvaluationMetricConfigDict", "DeleteEvaluationMetricConfigOrDict", @@ -2371,6 +2388,9 @@ "ListEvaluationMetricsResponse", "ListEvaluationMetricsResponseDict", "ListEvaluationMetricsResponseOrDict", + "UpdateEvaluationExperimentConfig", + "UpdateEvaluationExperimentConfigDict", + "UpdateEvaluationExperimentConfigOrDict", "OptimizeConfig", "OptimizeConfigDict", "OptimizeConfigOrDict", @@ -3720,6 +3740,7 @@ "_CreateEvaluationMetricParameters", "_CreateEvaluationRunParameters", "_CreateEvaluationSetParameters", + "_DeleteEvaluationExperimentParameters", "_DeleteEvaluationMetricParameters", "_EvaluateInstancesRequestParameters", "_GenerateUserScenariosParameters", @@ -3732,6 +3753,7 @@ "_GetEvaluationItemParameters", "_ListEvaluationExperimentsParameters", "_ListEvaluationMetricsParameters", + "_UpdateEvaluationExperimentParameters", "_OptimizeRequestParameters", "_CustomJobParameters", "_GetCustomJobParameters", diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index cd6fb70b6c..b5da65ffb3 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -4137,6 +4137,92 @@ class EvaluationSetDict(TypedDict, total=False): EvaluationSetOrDict = Union[EvaluationSet, EvaluationSetDict] +class DeleteEvaluationExperimentConfig(_common.BaseModel): + """Config for deleting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class DeleteEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for deleting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + +DeleteEvaluationExperimentConfigOrDict = Union[ + DeleteEvaluationExperimentConfig, DeleteEvaluationExperimentConfigDict +] + + +class _DeleteEvaluationExperimentParameters(_common.BaseModel): + """Parameters for deleting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[DeleteEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _DeleteEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for deleting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[DeleteEvaluationExperimentConfigDict] + """""" + + +_DeleteEvaluationExperimentParametersOrDict = Union[ + _DeleteEvaluationExperimentParameters, _DeleteEvaluationExperimentParametersDict +] + + +class DeleteEvaluationExperimentOperation(_common.BaseModel): + """Operation for deleting an evaluation experiment.""" + + name: Optional[str] = Field( + default=None, + description="""The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""", + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""", + ) + done: Optional[bool] = Field( + default=None, + description="""If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""", + ) + error: Optional[dict[str, Any]] = Field( + default=None, + description="""The error result of the operation in case of failure or cancellation.""", + ) + + +class DeleteEvaluationExperimentOperationDict(TypedDict, total=False): + """Operation for deleting an evaluation experiment.""" + + name: Optional[str] + """The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""" + + metadata: Optional[dict[str, Any]] + """Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""" + + done: Optional[bool] + """If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""" + + error: Optional[dict[str, Any]] + """The error result of the operation in case of failure or cancellation.""" + + +DeleteEvaluationExperimentOperationOrDict = Union[ + DeleteEvaluationExperimentOperation, DeleteEvaluationExperimentOperationDict +] + + class DeleteEvaluationMetricConfig(_common.BaseModel): """Config for deleting an evaluation metric.""" @@ -6350,6 +6436,85 @@ class ListEvaluationMetricsResponseDict(TypedDict, total=False): ] +class UpdateEvaluationExperimentConfig(_common.BaseModel): + """Config for updating an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + update_mask: Optional[str] = Field( + default=None, + description="""The update mask to apply. For the `FieldMask` definition, see + https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, description="""Metadata about the evaluation experiment.""" + ) + + +class UpdateEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for updating an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + update_mask: Optional[str] + """The update mask to apply. For the `FieldMask` definition, see + https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment.""" + + +UpdateEvaluationExperimentConfigOrDict = Union[ + UpdateEvaluationExperimentConfig, UpdateEvaluationExperimentConfigDict +] + + +class _UpdateEvaluationExperimentParameters(_common.BaseModel): + """Parameters for updating an evaluation experiment.""" + + name: Optional[str] = Field( + default=None, description="""The resource name of the EvaluationExperiment.""" + ) + config: Optional[UpdateEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _UpdateEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for updating an evaluation experiment.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment.""" + + config: Optional[UpdateEvaluationExperimentConfigDict] + """""" + + +_UpdateEvaluationExperimentParametersOrDict = Union[ + _UpdateEvaluationExperimentParameters, _UpdateEvaluationExperimentParametersDict +] + + class OptimizeConfig(_common.BaseModel): """Config for Prompt Optimizer.""" diff --git a/tests/unit/agentplatform/genai/replays/test_delete_update_evaluation_experiment.py b/tests/unit/agentplatform/genai/replays/test_delete_update_evaluation_experiment.py new file mode 100644 index 0000000000..2ff500923e --- /dev/null +++ b/tests/unit/agentplatform/genai/replays/test_delete_update_evaluation_experiment.py @@ -0,0 +1,80 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=protected-access,bad-continuation,missing-function-docstring + +from agentplatform._genai import types +from tests.unit.agentplatform.genai.replays import pytest_helper +from google.genai._api_client import HttpOptions +import pytest + +pytestmark = pytest_helper.setup( + file=__file__, + globals_for_file=globals(), + test_method="evals.update_evaluation_experiment", + http_options=HttpOptions( + api_version="v1beta1", + ), +) + +pytest_plugins = ("pytest_asyncio",) + + +def test_update_and_delete(client): + experiment = client.evals.create_evaluation_experiment( + display_name="sdk-update-delete-test" + ) + assert isinstance(experiment, types.EvaluationExperiment) + + updated = client.evals.update_evaluation_experiment( + name=experiment.name, + config={ + "display_name": "sdk-update-delete-test-renamed", + "update_mask": "display_name", + }, + ) + assert isinstance(updated, types.EvaluationExperiment) + assert updated.display_name == "sdk-update-delete-test-renamed" + + delete_operation = client.evals.delete_evaluation_experiment( + name=experiment.name + ) + assert isinstance( + delete_operation, types.DeleteEvaluationExperimentOperation + ) + + +@pytest.mark.asyncio +async def test_update_and_delete_async(client): + experiment = await client.aio.evals.create_evaluation_experiment( + display_name="sdk-update-delete-test-async" + ) + assert isinstance(experiment, types.EvaluationExperiment) + + updated = await client.aio.evals.update_evaluation_experiment( + name=experiment.name, + config={ + "display_name": "sdk-update-delete-test-async-renamed", + "update_mask": "display_name", + }, + ) + assert isinstance(updated, types.EvaluationExperiment) + assert updated.display_name == "sdk-update-delete-test-async-renamed" + + delete_operation = await client.aio.evals.delete_evaluation_experiment( + name=experiment.name + ) + assert isinstance( + delete_operation, types.DeleteEvaluationExperimentOperation + ) diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index e8aa60efcc..998bbaaf29 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -12024,3 +12024,65 @@ def test_create_evaluation_experiment_passes_all_params(self): assert request_body.get("mergeStrategy") == "SHARED_RESULT_SET" assert request_body.get("labels") == {"team": "agents"} assert request_body.get("metadata") == {"owner": "test"} + + +class TestUpdateEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.experiment_name = ( + "projects/123/locations/us-central1/evaluationExperiments/456" + ) + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + {"name": self.experiment_name, "displayName": "updated_name"} + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_update_evaluation_experiment_returns_experiment(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + experiment = evals_module.update_evaluation_experiment( + name=self.experiment_name, + config={"display_name": "updated_name", "update_mask": "display_name"}, + ) + + assert isinstance(experiment, agentplatform_genai_types.EvaluationExperiment) + assert experiment.display_name == "updated_name" + + def test_update_evaluation_experiment_uses_patch_and_full_name(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.update_evaluation_experiment( + name=self.experiment_name, + config={"display_name": "updated_name", "update_mask": "display_name"}, + ) + + self.mock_api_client.request.assert_called_once() + call_args = self.mock_api_client.request.call_args + assert call_args[0][0] == "patch" + assert call_args[0][1].startswith(self.experiment_name) + + +class TestDeleteEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.experiment_name = ( + "projects/123/locations/us-central1/evaluationExperiments/456" + ) + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps({"name": "operations/789"}) + self.mock_api_client.request.return_value = self.mock_response + + def test_delete_evaluation_experiment_uses_delete_and_full_name(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.delete_evaluation_experiment(name=self.experiment_name) + + self.mock_api_client.request.assert_called_once() + call_args = self.mock_api_client.request.call_args + assert call_args[0][0] == "delete" + assert call_args[0][1] == self.experiment_name diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index 9c78c5f04b..625f64f5d6 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -224,6 +224,20 @@ def _CustomCodeExecutionSpec_to_vertex( return to_object +def _DeleteEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _DeleteEvaluationMetricParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1079,6 +1093,48 @@ def _UnifiedMetric_to_vertex( return to_object +def _UpdateEvaluationExperimentConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["update_mask"]) is not None: + setv( + parent_object, ["_query", "updateMask"], getv(from_object, ["update_mask"]) + ) + + if getv(from_object, ["display_name"]) is not None: + setv(parent_object, ["displayName"], getv(from_object, ["display_name"])) + + if getv(from_object, ["labels"]) is not None: + setv(parent_object, ["labels"], getv(from_object, ["labels"])) + + if getv(from_object, ["merge_strategy"]) is not None: + setv(parent_object, ["mergeStrategy"], getv(from_object, ["merge_strategy"])) + + if getv(from_object, ["metadata"]) is not None: + setv(parent_object, ["metadata"], getv(from_object, ["metadata"])) + + return to_object + + +def _UpdateEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + _UpdateEvaluationExperimentConfig_to_vertex( + getv(from_object, ["config"]), to_object + ) + + return to_object + + class Evals(_api_module.BaseModule): def _create_evaluation_item( @@ -3375,6 +3431,173 @@ def create_evaluation_experiment( self._api_client._verify_response(return_value) return return_value + def delete_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: + """ + Deletes an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to delete. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration for the delete operation. + + Returns: + The delete operation. + + """ + + parameter_model = types._DeleteEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _DeleteEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("delete", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationExperimentOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + def update_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Updates an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to update. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration specifying the fields to update (e.g. + display_name, labels, merge_strategy, metadata) and the update_mask. + + Returns: + The updated evaluation experiment. + + """ + + parameter_model = types._UpdateEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _UpdateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("patch", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def get_evaluation_metric( self, *, @@ -5397,6 +5620,177 @@ async def create_evaluation_experiment( self._api_client._verify_response(return_value) return return_value + async def delete_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: + """ + Deletes an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to delete. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration for the delete operation. + + Returns: + The delete operation. + + """ + + parameter_model = types._DeleteEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _DeleteEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "delete", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationExperimentOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + async def update_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Updates an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to update. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration specifying the fields to update (e.g. + display_name, labels, merge_strategy, metadata) and the update_mask. + + Returns: + The updated evaluation experiment. + + """ + + parameter_model = types._UpdateEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _UpdateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "patch", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def get_evaluation_metric( self, *, diff --git a/vertexai/_genai/types/__init__.py b/vertexai/_genai/types/__init__.py index f692f2351d..a75584f31b 100644 --- a/vertexai/_genai/types/__init__.py +++ b/vertexai/_genai/types/__init__.py @@ -374,6 +374,17 @@ from .common import DeleteAgentEngineTaskConfig from .common import DeleteAgentEngineTaskConfigDict from .common import DeleteAgentEngineTaskConfigOrDict +from .common import DeleteEvaluationExperimentConfig +from .common import DeleteEvaluationExperimentConfigDict +from .common import DeleteEvaluationExperimentConfigOrDict +from .common import _DeleteEvaluationExperimentParameters +from .common import DeleteEvaluationExperimentOperation +from .common import DeleteEvaluationExperimentOperationDict +from .common import DeleteEvaluationExperimentOperationOrDict +from .common import UpdateEvaluationExperimentConfig +from .common import UpdateEvaluationExperimentConfigDict +from .common import UpdateEvaluationExperimentConfigOrDict +from .common import _UpdateEvaluationExperimentParameters from .common import DeleteEvaluationMetricConfig from .common import DeleteEvaluationMetricConfigDict from .common import DeleteEvaluationMetricConfigOrDict @@ -1693,6 +1704,15 @@ "EvaluationSet", "EvaluationSetDict", "EvaluationSetOrDict", + "DeleteEvaluationExperimentConfig", + "DeleteEvaluationExperimentConfigDict", + "DeleteEvaluationExperimentConfigOrDict", + "DeleteEvaluationExperimentOperation", + "DeleteEvaluationExperimentOperationDict", + "DeleteEvaluationExperimentOperationOrDict", + "UpdateEvaluationExperimentConfig", + "UpdateEvaluationExperimentConfigDict", + "UpdateEvaluationExperimentConfigOrDict", "DeleteEvaluationMetricConfig", "DeleteEvaluationMetricConfigDict", "DeleteEvaluationMetricConfigOrDict", @@ -2810,6 +2830,7 @@ "_CreateEvaluationMetricParameters", "_CreateEvaluationRunParameters", "_CreateEvaluationSetParameters", + "_DeleteEvaluationExperimentParameters", "_DeleteEvaluationMetricParameters", "_EvaluateInstancesRequestParameters", "_GenerateUserScenariosParameters", @@ -2884,6 +2905,7 @@ "_GetMultimodalDatasetParameters", "_GetMultimodalDatasetOperationParameters", "_ListMultimodalDatasetsRequestParameters", + "_UpdateEvaluationExperimentParameters", "_UpdateMultimodalDatasetParameters", "_CreateDatasetParameters", "_CreateDatasetVersionParameters", diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 3791b6ad13..ac4947c9f2 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -3877,6 +3877,92 @@ class EvaluationSetDict(TypedDict, total=False): EvaluationSetOrDict = Union[EvaluationSet, EvaluationSetDict] +class DeleteEvaluationExperimentConfig(_common.BaseModel): + """Config for deleting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class DeleteEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for deleting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + +DeleteEvaluationExperimentConfigOrDict = Union[ + DeleteEvaluationExperimentConfig, DeleteEvaluationExperimentConfigDict +] + + +class _DeleteEvaluationExperimentParameters(_common.BaseModel): + """Parameters for deleting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[DeleteEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _DeleteEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for deleting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[DeleteEvaluationExperimentConfigDict] + """""" + + +_DeleteEvaluationExperimentParametersOrDict = Union[ + _DeleteEvaluationExperimentParameters, _DeleteEvaluationExperimentParametersDict +] + + +class DeleteEvaluationExperimentOperation(_common.BaseModel): + """Operation for deleting an evaluation experiment.""" + + name: Optional[str] = Field( + default=None, + description="""The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""", + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""", + ) + done: Optional[bool] = Field( + default=None, + description="""If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""", + ) + error: Optional[dict[str, Any]] = Field( + default=None, + description="""The error result of the operation in case of failure or cancellation.""", + ) + + +class DeleteEvaluationExperimentOperationDict(TypedDict, total=False): + """Operation for deleting an evaluation experiment.""" + + name: Optional[str] + """The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""" + + metadata: Optional[dict[str, Any]] + """Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""" + + done: Optional[bool] + """If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""" + + error: Optional[dict[str, Any]] + """The error result of the operation in case of failure or cancellation.""" + + +DeleteEvaluationExperimentOperationOrDict = Union[ + DeleteEvaluationExperimentOperation, DeleteEvaluationExperimentOperationDict +] + + class DeleteEvaluationMetricConfig(_common.BaseModel): """Config for deleting an evaluation metric.""" @@ -5919,6 +6005,85 @@ class ListEvaluationMetricsResponseDict(TypedDict, total=False): ] +class UpdateEvaluationExperimentConfig(_common.BaseModel): + """Config for updating an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + update_mask: Optional[str] = Field( + default=None, + description="""The update mask to apply. For the `FieldMask` definition, see + https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, description="""Metadata about the evaluation experiment.""" + ) + + +class UpdateEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for updating an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + update_mask: Optional[str] + """The update mask to apply. For the `FieldMask` definition, see + https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment.""" + + +UpdateEvaluationExperimentConfigOrDict = Union[ + UpdateEvaluationExperimentConfig, UpdateEvaluationExperimentConfigDict +] + + +class _UpdateEvaluationExperimentParameters(_common.BaseModel): + """Parameters for updating an evaluation experiment.""" + + name: Optional[str] = Field( + default=None, description="""The resource name of the EvaluationExperiment.""" + ) + config: Optional[UpdateEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _UpdateEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for updating an evaluation experiment.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment.""" + + config: Optional[UpdateEvaluationExperimentConfigDict] + """""" + + +_UpdateEvaluationExperimentParametersOrDict = Union[ + _UpdateEvaluationExperimentParameters, _UpdateEvaluationExperimentParametersDict +] + + class OptimizeConfig(_common.BaseModel): """Config for Prompt Optimizer."""