From 61d9dae2d0c0ae56dce664196b1908b7a0e17c57 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Fri, 27 Feb 2026 15:45:10 -0800 Subject: [PATCH] feat: Support include_server_side_tool_invocations for genai. Adds ToolConfig.include_server_side_tool_invocations Adds ExecutableCode.id Adds CodeExecutionResult.id Adds ToolCall and ToolResponse FUTURE_COPYBARA_INTEGRATE_REVIEW=https://github.com/googleapis/python-genai/pull/2125 from googleapis:release-please--branches--main 0061b63851c5f406312768a8505600b9d22769d2 PiperOrigin-RevId: 876445909 --- google/genai/_live_converters.py | 137 ++++++++++- google/genai/_tokens_converters.py | 18 +- google/genai/batches.py | 25 +- google/genai/caches.py | 148 ++++++++++- google/genai/models.py | 181 +++++++++++++- .../models/test_generate_content_tools.py | 78 ++++++ google/genai/types.py | 229 +++++++++++++----- 7 files changed, 719 insertions(+), 97 deletions(-) diff --git a/google/genai/_live_converters.py b/google/genai/_live_converters.py index a3e2e7ebf..a65bfbc85 100644 --- a/google/genai/_live_converters.py +++ b/google/genai/_live_converters.py @@ -106,6 +106,27 @@ def _Content_to_mldev( return to_object +def _Content_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['parts']) is not None: + setv( + to_object, + ['parts'], + [ + _Part_to_vertex(item, to_object) + for item in getv(from_object, ['parts']) + ], + ) + + if getv(from_object, ['role']) is not None: + setv(to_object, ['role'], getv(from_object, ['role'])) + + return to_object + + def _FileData_to_mldev( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -361,6 +382,27 @@ def _LiveClientContent_to_mldev( return to_object +def _LiveClientContent_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['turns']) is not None: + setv( + to_object, + ['turns'], + [ + _Content_to_vertex(item, to_object) + for item in getv(from_object, ['turns']) + ], + ) + + if getv(from_object, ['turn_complete']) is not None: + setv(to_object, ['turnComplete'], getv(from_object, ['turn_complete'])) + + return to_object + + def _LiveClientMessage_to_mldev( api_client: BaseApiClient, from_object: Union[dict[str, Any], object], @@ -416,7 +458,13 @@ def _LiveClientMessage_to_vertex( ) if getv(from_object, ['client_content']) is not None: - setv(to_object, ['clientContent'], getv(from_object, ['client_content'])) + setv( + to_object, + ['clientContent'], + _LiveClientContent_to_vertex( + getv(from_object, ['client_content']), to_object + ), + ) if getv(from_object, ['realtime_input']) is not None: setv( @@ -617,7 +665,9 @@ def _LiveClientSetup_to_vertex( setv( to_object, ['systemInstruction'], - t.t_content(getv(from_object, ['system_instruction'])), + _Content_to_vertex( + t.t_content(getv(from_object, ['system_instruction'])), to_object + ), ) if getv(from_object, ['tools']) is not None: @@ -930,7 +980,9 @@ def _LiveConnectConfig_to_vertex( setv( parent_object, ['setup', 'systemInstruction'], - t.t_content(getv(from_object, ['system_instruction'])), + _Content_to_vertex( + t.t_content(getv(from_object, ['system_instruction'])), to_object + ), ) if getv(from_object, ['tools']) is not None: @@ -1344,10 +1396,8 @@ def _Part_to_mldev( parent_object: Optional[dict[str, Any]] = None, ) -> dict[str, Any]: to_object: dict[str, Any] = {} - if getv(from_object, ['media_resolution']) is not None: - setv( - to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) - ) + if getv(from_object, ['executable_code']) is not None: + setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) if getv(from_object, ['code_execution_result']) is not None: setv( @@ -1356,8 +1406,16 @@ def _Part_to_mldev( getv(from_object, ['code_execution_result']), ) - if getv(from_object, ['executable_code']) is not None: - setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) + if getv(from_object, ['media_resolution']) is not None: + setv( + to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) + ) + + if getv(from_object, ['tool_call']) is not None: + setv(to_object, ['toolCall'], getv(from_object, ['tool_call'])) + + if getv(from_object, ['tool_response']) is not None: + setv(to_object, ['toolResponse'], getv(from_object, ['tool_response'])) if getv(from_object, ['file_data']) is not None: setv( @@ -1406,6 +1464,67 @@ def _Part_to_mldev( return to_object +def _Part_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['executable_code']) is not None: + setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) + + if getv(from_object, ['code_execution_result']) is not None: + setv( + to_object, + ['codeExecutionResult'], + getv(from_object, ['code_execution_result']), + ) + + if getv(from_object, ['media_resolution']) is not None: + setv( + to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) + ) + + if getv(from_object, ['tool_call']) is not None: + raise ValueError('tool_call parameter is not supported in Vertex AI.') + + if getv(from_object, ['tool_response']) is not None: + raise ValueError('tool_response parameter is not supported in Vertex AI.') + + if getv(from_object, ['file_data']) is not None: + setv(to_object, ['fileData'], getv(from_object, ['file_data'])) + + if getv(from_object, ['function_call']) is not None: + setv(to_object, ['functionCall'], getv(from_object, ['function_call'])) + + if getv(from_object, ['function_response']) is not None: + setv( + to_object, + ['functionResponse'], + getv(from_object, ['function_response']), + ) + + if getv(from_object, ['inline_data']) is not None: + setv(to_object, ['inlineData'], getv(from_object, ['inline_data'])) + + if getv(from_object, ['text']) is not None: + setv(to_object, ['text'], getv(from_object, ['text'])) + + if getv(from_object, ['thought']) is not None: + setv(to_object, ['thought'], getv(from_object, ['thought'])) + + if getv(from_object, ['thought_signature']) is not None: + setv( + to_object, + ['thoughtSignature'], + getv(from_object, ['thought_signature']), + ) + + if getv(from_object, ['video_metadata']) is not None: + setv(to_object, ['videoMetadata'], getv(from_object, ['video_metadata'])) + + return to_object + + def _SessionResumptionConfig_to_mldev( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, diff --git a/google/genai/_tokens_converters.py b/google/genai/_tokens_converters.py index 957458827..35b2e8829 100644 --- a/google/genai/_tokens_converters.py +++ b/google/genai/_tokens_converters.py @@ -456,10 +456,8 @@ def _Part_to_mldev( parent_object: Optional[dict[str, Any]] = None, ) -> dict[str, Any]: to_object: dict[str, Any] = {} - if getv(from_object, ['media_resolution']) is not None: - setv( - to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) - ) + if getv(from_object, ['executable_code']) is not None: + setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) if getv(from_object, ['code_execution_result']) is not None: setv( @@ -468,8 +466,16 @@ def _Part_to_mldev( getv(from_object, ['code_execution_result']), ) - if getv(from_object, ['executable_code']) is not None: - setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) + if getv(from_object, ['media_resolution']) is not None: + setv( + to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) + ) + + if getv(from_object, ['tool_call']) is not None: + setv(to_object, ['toolCall'], getv(from_object, ['tool_call'])) + + if getv(from_object, ['tool_response']) is not None: + setv(to_object, ['toolResponse'], getv(from_object, ['tool_response'])) if getv(from_object, ['file_data']) is not None: setv( diff --git a/google/genai/batches.py b/google/genai/batches.py index 7a43a9aec..c16bfb1e7 100644 --- a/google/genai/batches.py +++ b/google/genai/batches.py @@ -1395,10 +1395,8 @@ def _Part_to_mldev( parent_object: Optional[dict[str, Any]] = None, ) -> dict[str, Any]: to_object: dict[str, Any] = {} - if getv(from_object, ['media_resolution']) is not None: - setv( - to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) - ) + if getv(from_object, ['executable_code']) is not None: + setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) if getv(from_object, ['code_execution_result']) is not None: setv( @@ -1407,8 +1405,16 @@ def _Part_to_mldev( getv(from_object, ['code_execution_result']), ) - if getv(from_object, ['executable_code']) is not None: - setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) + if getv(from_object, ['media_resolution']) is not None: + setv( + to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) + ) + + if getv(from_object, ['tool_call']) is not None: + setv(to_object, ['toolCall'], getv(from_object, ['tool_call'])) + + if getv(from_object, ['tool_response']) is not None: + setv(to_object, ['toolResponse'], getv(from_object, ['tool_response'])) if getv(from_object, ['file_data']) is not None: setv( @@ -1484,6 +1490,13 @@ def _ToolConfig_to_mldev( to_object, ['retrievalConfig'], getv(from_object, ['retrieval_config']) ) + if getv(from_object, ['include_server_side_tool_invocations']) is not None: + setv( + to_object, + ['includeServerSideToolInvocations'], + getv(from_object, ['include_server_side_tool_invocations']), + ) + if getv(from_object, ['function_calling_config']) is not None: setv( to_object, diff --git a/google/genai/caches.py b/google/genai/caches.py index 010a820f3..127a59f9a 100644 --- a/google/genai/caches.py +++ b/google/genai/caches.py @@ -105,6 +105,27 @@ def _Content_to_mldev( return to_object +def _Content_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['parts']) is not None: + setv( + to_object, + ['parts'], + [ + _Part_to_vertex(item, to_object) + for item in getv(from_object, ['parts']) + ], + ) + + if getv(from_object, ['role']) is not None: + setv(to_object, ['role'], getv(from_object, ['role'])) + + return to_object + + def _CreateCachedContentConfig_to_mldev( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -181,14 +202,19 @@ def _CreateCachedContentConfig_to_vertex( setv( parent_object, ['contents'], - [item for item in t.t_contents(getv(from_object, ['contents']))], + [ + _Content_to_vertex(item, to_object) + for item in t.t_contents(getv(from_object, ['contents'])) + ], ) if getv(from_object, ['system_instruction']) is not None: setv( parent_object, ['systemInstruction'], - t.t_content(getv(from_object, ['system_instruction'])), + _Content_to_vertex( + t.t_content(getv(from_object, ['system_instruction'])), to_object + ), ) if getv(from_object, ['tools']) is not None: @@ -202,7 +228,11 @@ def _CreateCachedContentConfig_to_vertex( ) if getv(from_object, ['tool_config']) is not None: - setv(parent_object, ['toolConfig'], getv(from_object, ['tool_config'])) + setv( + parent_object, + ['toolConfig'], + _ToolConfig_to_vertex(getv(from_object, ['tool_config']), to_object), + ) if getv(from_object, ['kms_key_name']) is not None: setv( @@ -608,10 +638,8 @@ def _Part_to_mldev( parent_object: Optional[dict[str, Any]] = None, ) -> dict[str, Any]: to_object: dict[str, Any] = {} - if getv(from_object, ['media_resolution']) is not None: - setv( - to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) - ) + if getv(from_object, ['executable_code']) is not None: + setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) if getv(from_object, ['code_execution_result']) is not None: setv( @@ -620,8 +648,16 @@ def _Part_to_mldev( getv(from_object, ['code_execution_result']), ) - if getv(from_object, ['executable_code']) is not None: - setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) + if getv(from_object, ['media_resolution']) is not None: + setv( + to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) + ) + + if getv(from_object, ['tool_call']) is not None: + setv(to_object, ['toolCall'], getv(from_object, ['tool_call'])) + + if getv(from_object, ['tool_response']) is not None: + setv(to_object, ['toolResponse'], getv(from_object, ['tool_response'])) if getv(from_object, ['file_data']) is not None: setv( @@ -670,6 +706,67 @@ def _Part_to_mldev( return to_object +def _Part_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['executable_code']) is not None: + setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) + + if getv(from_object, ['code_execution_result']) is not None: + setv( + to_object, + ['codeExecutionResult'], + getv(from_object, ['code_execution_result']), + ) + + if getv(from_object, ['media_resolution']) is not None: + setv( + to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) + ) + + if getv(from_object, ['tool_call']) is not None: + raise ValueError('tool_call parameter is not supported in Vertex AI.') + + if getv(from_object, ['tool_response']) is not None: + raise ValueError('tool_response parameter is not supported in Vertex AI.') + + if getv(from_object, ['file_data']) is not None: + setv(to_object, ['fileData'], getv(from_object, ['file_data'])) + + if getv(from_object, ['function_call']) is not None: + setv(to_object, ['functionCall'], getv(from_object, ['function_call'])) + + if getv(from_object, ['function_response']) is not None: + setv( + to_object, + ['functionResponse'], + getv(from_object, ['function_response']), + ) + + if getv(from_object, ['inline_data']) is not None: + setv(to_object, ['inlineData'], getv(from_object, ['inline_data'])) + + if getv(from_object, ['text']) is not None: + setv(to_object, ['text'], getv(from_object, ['text'])) + + if getv(from_object, ['thought']) is not None: + setv(to_object, ['thought'], getv(from_object, ['thought'])) + + if getv(from_object, ['thought_signature']) is not None: + setv( + to_object, + ['thoughtSignature'], + getv(from_object, ['thought_signature']), + ) + + if getv(from_object, ['video_metadata']) is not None: + setv(to_object, ['videoMetadata'], getv(from_object, ['video_metadata'])) + + return to_object + + def _ToolConfig_to_mldev( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -680,6 +777,13 @@ def _ToolConfig_to_mldev( to_object, ['retrievalConfig'], getv(from_object, ['retrieval_config']) ) + if getv(from_object, ['include_server_side_tool_invocations']) is not None: + setv( + to_object, + ['includeServerSideToolInvocations'], + getv(from_object, ['include_server_side_tool_invocations']), + ) + if getv(from_object, ['function_calling_config']) is not None: setv( to_object, @@ -692,6 +796,32 @@ def _ToolConfig_to_mldev( return to_object +def _ToolConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['retrieval_config']) is not None: + setv( + to_object, ['retrievalConfig'], getv(from_object, ['retrieval_config']) + ) + + if getv(from_object, ['include_server_side_tool_invocations']) is not None: + raise ValueError( + 'include_server_side_tool_invocations parameter is not supported in' + ' Vertex AI.' + ) + + if getv(from_object, ['function_calling_config']) is not None: + setv( + to_object, + ['functionCallingConfig'], + getv(from_object, ['function_calling_config']), + ) + + return to_object + + def _Tool_to_mldev( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, diff --git a/google/genai/models.py b/google/genai/models.py index d2db40507..bb2264a6f 100644 --- a/google/genai/models.py +++ b/google/genai/models.py @@ -198,7 +198,10 @@ def _ComputeTokensParameters_to_vertex( setv( to_object, ['contents'], - [item for item in t.t_contents(getv(from_object, ['contents']))], + [ + _Content_to_vertex(item, to_object, root_object) + for item in t.t_contents(getv(from_object, ['contents'])) + ], ) return to_object @@ -283,6 +286,28 @@ def _Content_to_mldev( return to_object +def _Content_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, + root_object: Optional[Union[dict[str, Any], object]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['parts']) is not None: + setv( + to_object, + ['parts'], + [ + _Part_to_vertex(item, to_object, root_object) + for item in getv(from_object, ['parts']) + ], + ) + + if getv(from_object, ['role']) is not None: + setv(to_object, ['role'], getv(from_object, ['role'])) + + return to_object + + def _ControlReferenceConfig_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -336,7 +361,11 @@ def _CountTokensConfig_to_vertex( setv( parent_object, ['systemInstruction'], - t.t_content(getv(from_object, ['system_instruction'])), + _Content_to_vertex( + t.t_content(getv(from_object, ['system_instruction'])), + to_object, + root_object, + ), ) if getv(from_object, ['tools']) is not None: @@ -411,7 +440,10 @@ def _CountTokensParameters_to_vertex( setv( to_object, ['contents'], - [item for item in t.t_contents(getv(from_object, ['contents']))], + [ + _Content_to_vertex(item, to_object, root_object) + for item in t.t_contents(getv(from_object, ['contents'])) + ], ) if getv(from_object, ['config']) is not None: @@ -898,7 +930,15 @@ def _EmbedContentParametersPrivate_to_vertex( discriminator = 'PREDICT' if discriminator == 'EMBED_CONTENT': if getv(from_object, ['content']) is not None: - setv(to_object, ['content'], t.t_content(getv(from_object, ['content']))) + setv( + to_object, + ['content'], + _Content_to_vertex( + t.t_content(getv(from_object, ['content'])), + to_object, + root_object, + ), + ) if getv(from_object, ['config']) is not None: _EmbedContentConfig_to_vertex( @@ -1291,7 +1331,11 @@ def _GenerateContentConfig_to_vertex( setv( parent_object, ['systemInstruction'], - t.t_content(getv(from_object, ['system_instruction'])), + _Content_to_vertex( + t.t_content(getv(from_object, ['system_instruction'])), + to_object, + root_object, + ), ) if getv(from_object, ['temperature']) is not None: @@ -1388,7 +1432,13 @@ def _GenerateContentConfig_to_vertex( ) if getv(from_object, ['tool_config']) is not None: - setv(parent_object, ['toolConfig'], getv(from_object, ['tool_config'])) + setv( + parent_object, + ['toolConfig'], + _ToolConfig_to_vertex( + getv(from_object, ['tool_config']), to_object, root_object + ), + ) if getv(from_object, ['labels']) is not None: setv(parent_object, ['labels'], getv(from_object, ['labels'])) @@ -1505,7 +1555,10 @@ def _GenerateContentParameters_to_vertex( setv( to_object, ['contents'], - [item for item in t.t_contents(getv(from_object, ['contents']))], + [ + _Content_to_vertex(item, to_object, root_object) + for item in t.t_contents(getv(from_object, ['contents'])) + ], ) if getv(from_object, ['config']) is not None: @@ -3251,10 +3304,8 @@ def _Part_to_mldev( root_object: Optional[Union[dict[str, Any], object]] = None, ) -> dict[str, Any]: to_object: dict[str, Any] = {} - if getv(from_object, ['media_resolution']) is not None: - setv( - to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) - ) + if getv(from_object, ['executable_code']) is not None: + setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) if getv(from_object, ['code_execution_result']) is not None: setv( @@ -3263,8 +3314,16 @@ def _Part_to_mldev( getv(from_object, ['code_execution_result']), ) - if getv(from_object, ['executable_code']) is not None: - setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) + if getv(from_object, ['media_resolution']) is not None: + setv( + to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) + ) + + if getv(from_object, ['tool_call']) is not None: + setv(to_object, ['toolCall'], getv(from_object, ['tool_call'])) + + if getv(from_object, ['tool_response']) is not None: + setv(to_object, ['toolResponse'], getv(from_object, ['tool_response'])) if getv(from_object, ['file_data']) is not None: setv( @@ -3319,6 +3378,68 @@ def _Part_to_mldev( return to_object +def _Part_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, + root_object: Optional[Union[dict[str, Any], object]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['executable_code']) is not None: + setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) + + if getv(from_object, ['code_execution_result']) is not None: + setv( + to_object, + ['codeExecutionResult'], + getv(from_object, ['code_execution_result']), + ) + + if getv(from_object, ['media_resolution']) is not None: + setv( + to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) + ) + + if getv(from_object, ['tool_call']) is not None: + raise ValueError('tool_call parameter is not supported in Vertex AI.') + + if getv(from_object, ['tool_response']) is not None: + raise ValueError('tool_response parameter is not supported in Vertex AI.') + + if getv(from_object, ['file_data']) is not None: + setv(to_object, ['fileData'], getv(from_object, ['file_data'])) + + if getv(from_object, ['function_call']) is not None: + setv(to_object, ['functionCall'], getv(from_object, ['function_call'])) + + if getv(from_object, ['function_response']) is not None: + setv( + to_object, + ['functionResponse'], + getv(from_object, ['function_response']), + ) + + if getv(from_object, ['inline_data']) is not None: + setv(to_object, ['inlineData'], getv(from_object, ['inline_data'])) + + if getv(from_object, ['text']) is not None: + setv(to_object, ['text'], getv(from_object, ['text'])) + + if getv(from_object, ['thought']) is not None: + setv(to_object, ['thought'], getv(from_object, ['thought'])) + + if getv(from_object, ['thought_signature']) is not None: + setv( + to_object, + ['thoughtSignature'], + getv(from_object, ['thought_signature']), + ) + + if getv(from_object, ['video_metadata']) is not None: + setv(to_object, ['videoMetadata'], getv(from_object, ['video_metadata'])) + + return to_object + + def _ProductImage_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -3764,6 +3885,13 @@ def _ToolConfig_to_mldev( to_object, ['retrievalConfig'], getv(from_object, ['retrieval_config']) ) + if getv(from_object, ['include_server_side_tool_invocations']) is not None: + setv( + to_object, + ['includeServerSideToolInvocations'], + getv(from_object, ['include_server_side_tool_invocations']), + ) + if getv(from_object, ['function_calling_config']) is not None: setv( to_object, @@ -3778,6 +3906,33 @@ def _ToolConfig_to_mldev( return to_object +def _ToolConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, + root_object: Optional[Union[dict[str, Any], object]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['retrieval_config']) is not None: + setv( + to_object, ['retrievalConfig'], getv(from_object, ['retrieval_config']) + ) + + if getv(from_object, ['include_server_side_tool_invocations']) is not None: + raise ValueError( + 'include_server_side_tool_invocations parameter is not supported in' + ' Vertex AI.' + ) + + if getv(from_object, ['function_calling_config']) is not None: + setv( + to_object, + ['functionCallingConfig'], + getv(from_object, ['function_calling_config']), + ) + + return to_object + + def _Tool_to_mldev( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, diff --git a/google/genai/tests/models/test_generate_content_tools.py b/google/genai/tests/models/test_generate_content_tools.py index 142d9f980..b162d476b 100644 --- a/google/genai/tests/models/test_generate_content_tools.py +++ b/google/genai/tests/models/test_generate_content_tools.py @@ -603,6 +603,84 @@ def divide_floats(a: float, b: float) -> float: config={'tools': [{'google_maps': {'enable_widget': True}}]}, ), ), + pytest_helper.TestTableItem( + name='test_include_server_side_tool_invocations', + parameters=types._GenerateContentParameters( + model='gemini-3.1-pro-preview', + contents=t.t_contents('Why is the sky blue?'), + config=types.GenerateContentConfig( + tools=[ + types.Tool( + google_search=types.GoogleSearch(), + ), + ], + tool_config=types.ToolConfig( + include_server_side_tool_invocations=True, + ), + ), + ), + exception_if_vertex='parameter is not supported', + ), + pytest_helper.TestTableItem( + name='test_include_server_side_tool_invocations_with_tool_call_echo', + parameters=types._GenerateContentParameters( + model='gemini-3.1-pro-preview', + contents=[ + types.Content.model_validate(item) + for item in [ + { + 'role': 'user', + 'parts': [{'text': 'Why is the sky blue?'}], + }, + { + 'role': 'model', + 'parts': [ + { + 'tool_call': { + 'tool_type': 'GOOGLE_SEARCH', + 'args': { + 'query': 'why is the sky blue', + }, + }, + }, + { + 'tool_response': { + 'tool_type': 'GOOGLE_SEARCH', + 'response': { + 'result': ( + 'The sky is blue because of' + ' Rayleigh scattering.' + ), + }, + }, + }, + { + 'text': ( + 'The sky is blue due to a phenomenon called' + ' Rayleigh scattering.' + ), + }, + ], + }, + { + 'role': 'user', + 'parts': [{'text': 'What about Mars?'}], + }, + ] + ], + config=types.GenerateContentConfig( + tools=[ + types.Tool( + google_search=types.GoogleSearch(), + ), + ], + tool_config=types.ToolConfig( + include_server_side_tool_invocations=True, + ), + ), + ), + exception_if_vertex='parameter is not supported', + ), ] diff --git a/google/genai/types.py b/google/genai/types.py index d0cd822ac..eae0b7dbd 100644 --- a/google/genai/types.py +++ b/google/genai/types.py @@ -140,6 +140,15 @@ MetricSubclass = typing.TypeVar('MetricSubclass', bound='Metric') +class Language(_common.CaseInSensitiveEnum): + """Programming language of the `code`.""" + + LANGUAGE_UNSPECIFIED = 'LANGUAGE_UNSPECIFIED' + """Unspecified language. This value should not be used.""" + PYTHON = 'PYTHON' + """Python >= 3.10, with numpy and simpy available.""" + + class Outcome(_common.CaseInSensitiveEnum): """Outcome of the code execution.""" @@ -153,15 +162,6 @@ class Outcome(_common.CaseInSensitiveEnum): """Code execution ran for too long, and was cancelled. There may or may not be a partial output present.""" -class Language(_common.CaseInSensitiveEnum): - """Programming language of the `code`.""" - - LANGUAGE_UNSPECIFIED = 'LANGUAGE_UNSPECIFIED' - """Unspecified language. This value should not be used.""" - PYTHON = 'PYTHON' - """Python >= 3.10, with numpy and simpy available.""" - - class FunctionResponseScheduling(_common.CaseInSensitiveEnum): """Specifies how the response should be scheduled in the conversation.""" @@ -737,6 +737,23 @@ class PartMediaResolutionLevel(_common.CaseInSensitiveEnum): """Media resolution set to ultra high.""" +class ToolType(_common.CaseInSensitiveEnum): + """The type of tool.""" + + TOOL_TYPE_UNSPECIFIED = 'TOOL_TYPE_UNSPECIFIED' + """Unspecified tool type.""" + GOOGLE_SEARCH_WEB = 'GOOGLE_SEARCH_WEB' + """Google search tool, maps to Tool.google_search.""" + GOOGLE_SEARCH_IMAGE = 'GOOGLE_SEARCH_IMAGE' + """Google search tool, maps to Tool.google_search.""" + URL_CONTEXT = 'URL_CONTEXT' + """URL context tool, maps to Tool.url_context.""" + GOOGLE_MAPS = 'GOOGLE_MAPS' + """Google maps tool, maps to Tool.google_maps.""" + FILE_SEARCH = 'FILE_SEARCH' + """File search tool, maps to Tool.file_search.""" + + class ResourceScope(_common.CaseInSensitiveEnum): """Resource scope.""" @@ -1126,6 +1143,69 @@ class LiveMusicPlaybackControl(_common.CaseInSensitiveEnum): Retains the current prompts and config.""" +class ExecutableCode(_common.BaseModel): + """Code generated by the model that is meant to be executed.""" + + id: Optional[str] = Field( + default=None, description="""The identifier of the executable code.""" + ) + code: Optional[str] = Field( + default=None, description="""Required. The code to be executed.""" + ) + language: Optional[Language] = Field( + default=None, + description="""Required. Programming language of the `code`.""", + ) + + +class ExecutableCodeDict(TypedDict, total=False): + """Code generated by the model that is meant to be executed.""" + + id: Optional[str] + """The identifier of the executable code.""" + + code: Optional[str] + """Required. The code to be executed.""" + + language: Optional[Language] + """Required. Programming language of the `code`.""" + + +ExecutableCodeOrDict = Union[ExecutableCode, ExecutableCodeDict] + + +class CodeExecutionResult(_common.BaseModel): + """Result of executing the ExecutableCode.""" + + id: Optional[str] = Field( + default=None, + description="""The identifier of the code execution result.""", + ) + outcome: Optional[Outcome] = Field( + default=None, description="""Required. Outcome of the code execution.""" + ) + output: Optional[str] = Field( + default=None, + description="""Optional. Contains stdout when code execution is successful, stderr or other description otherwise.""", + ) + + +class CodeExecutionResultDict(TypedDict, total=False): + """Result of executing the ExecutableCode.""" + + id: Optional[str] + """The identifier of the code execution result.""" + + outcome: Optional[Outcome] + """Required. Outcome of the code execution.""" + + output: Optional[str] + """Optional. Contains stdout when code execution is successful, stderr or other description otherwise.""" + + +CodeExecutionResultOrDict = Union[CodeExecutionResult, CodeExecutionResultDict] + + class PartMediaResolution(_common.BaseModel): """Media resolution for the input media.""" @@ -1156,72 +1236,92 @@ class PartMediaResolutionDict(TypedDict, total=False): PartMediaResolutionOrDict = Union[PartMediaResolution, PartMediaResolutionDict] -class CodeExecutionResult(_common.BaseModel): - """Result of executing the [ExecutableCode]. +class ToolCall(_common.BaseModel): + """A predicted server-side tool call returned from the model. - Only generated when using the [CodeExecution] tool, and always follows a - `part` containing the [ExecutableCode]. + This message contains information about a tool that the model wants to invoke. + The client is NOT expected to execute this tool call. Instead, the client + should pass this back to the API in a subsequent turn within a Content + message, + along with the corresponding ToolResponse. """ - outcome: Optional[Outcome] = Field( - default=None, description="""Required. Outcome of the code execution.""" + id: Optional[str] = Field( + default=None, description="""Unique identifier of the tool call.""" ) - output: Optional[str] = Field( - default=None, - description="""Optional. Contains stdout when code execution is successful, stderr or other description otherwise.""", + tool_type: Optional[ToolType] = Field( + default=None, description="""The type of tool that was called.""" + ) + args: Optional[dict[str, Any]] = Field( + default=None, description="""The tool call arguments.""" ) -class CodeExecutionResultDict(TypedDict, total=False): - """Result of executing the [ExecutableCode]. +class ToolCallDict(TypedDict, total=False): + """A predicted server-side tool call returned from the model. - Only generated when using the [CodeExecution] tool, and always follows a - `part` containing the [ExecutableCode]. + This message contains information about a tool that the model wants to invoke. + The client is NOT expected to execute this tool call. Instead, the client + should pass this back to the API in a subsequent turn within a Content + message, + along with the corresponding ToolResponse. """ - outcome: Optional[Outcome] - """Required. Outcome of the code execution.""" + id: Optional[str] + """Unique identifier of the tool call.""" - output: Optional[str] - """Optional. Contains stdout when code execution is successful, stderr or other description otherwise.""" + tool_type: Optional[ToolType] + """The type of tool that was called.""" + args: Optional[dict[str, Any]] + """The tool call arguments.""" -CodeExecutionResultOrDict = Union[CodeExecutionResult, CodeExecutionResultDict] +ToolCallOrDict = Union[ToolCall, ToolCallDict] -class ExecutableCode(_common.BaseModel): - """Code generated by the model that is meant to be executed, and the result returned to the model. - Generated when using the [CodeExecution] tool, in which the code will be - automatically executed, and a corresponding [CodeExecutionResult] will also be - generated. +class ToolResponse(_common.BaseModel): + """The output from a server-side tool call execution. + + This message contains the results of a tool invocation that was initiated by a + ToolCall from the model. The client should pass this back to the API in a + subsequent turn within a Content message, along with the corresponding + ToolCall. """ - code: Optional[str] = Field( - default=None, description="""Required. The code to be executed.""" + id: Optional[str] = Field( + default=None, + description="""The identifier of the tool call this response is for.""", ) - language: Optional[Language] = Field( + tool_type: Optional[ToolType] = Field( default=None, - description="""Required. Programming language of the `code`.""", + description="""The type of tool that was called, matching the tool_type in the corresponding ToolCall.""", + ) + response: Optional[dict[str, Any]] = Field( + default=None, description="""The tool response.""" ) -class ExecutableCodeDict(TypedDict, total=False): - """Code generated by the model that is meant to be executed, and the result returned to the model. +class ToolResponseDict(TypedDict, total=False): + """The output from a server-side tool call execution. - Generated when using the [CodeExecution] tool, in which the code will be - automatically executed, and a corresponding [CodeExecutionResult] will also be - generated. + This message contains the results of a tool invocation that was initiated by a + ToolCall from the model. The client should pass this back to the API in a + subsequent turn within a Content message, along with the corresponding + ToolCall. """ - code: Optional[str] - """Required. The code to be executed.""" + id: Optional[str] + """The identifier of the tool call this response is for.""" - language: Optional[Language] - """Required. Programming language of the `code`.""" + tool_type: Optional[ToolType] + """The type of tool that was called, matching the tool_type in the corresponding ToolCall.""" + + response: Optional[dict[str, Any]] + """The tool response.""" -ExecutableCodeOrDict = Union[ExecutableCode, ExecutableCodeDict] +ToolResponseOrDict = Union[ToolResponse, ToolResponseDict] class FileData(_common.BaseModel): @@ -1693,18 +1793,26 @@ class Part(_common.BaseModel): instance is considered invalid. """ + executable_code: Optional[ExecutableCode] = Field( + default=None, + description="""Optional. Code generated by the model that is intended to be executed.""", + ) + code_execution_result: Optional[CodeExecutionResult] = Field( + default=None, + description="""Optional. The result of executing the ExecutableCode.""", + ) media_resolution: Optional[PartMediaResolution] = Field( default=None, description="""Media resolution for the input media. """, ) - code_execution_result: Optional[CodeExecutionResult] = Field( + tool_call: Optional[ToolCall] = Field( default=None, - description="""Optional. The result of executing the ExecutableCode.""", + description="""Server-side tool call. This field is populated when the model predicts a tool invocation that should be executed on the server. The client is expected to echo this message back to the API.""", ) - executable_code: Optional[ExecutableCode] = Field( + tool_response: Optional[ToolResponse] = Field( default=None, - description="""Optional. Code generated by the model that is intended to be executed.""", + description="""The output from a server-side ToolCall execution. This field is populated by the client with the results of executing the corresponding ToolCall.""", ) file_data: Optional[FileData] = Field( default=None, @@ -1926,15 +2034,21 @@ class PartDict(TypedDict, total=False): instance is considered invalid. """ + executable_code: Optional[ExecutableCodeDict] + """Optional. Code generated by the model that is intended to be executed.""" + + code_execution_result: Optional[CodeExecutionResultDict] + """Optional. The result of executing the ExecutableCode.""" + media_resolution: Optional[PartMediaResolutionDict] """Media resolution for the input media. """ - code_execution_result: Optional[CodeExecutionResultDict] - """Optional. The result of executing the ExecutableCode.""" + tool_call: Optional[ToolCallDict] + """Server-side tool call. This field is populated when the model predicts a tool invocation that should be executed on the server. The client is expected to echo this message back to the API.""" - executable_code: Optional[ExecutableCodeDict] - """Optional. Code generated by the model that is intended to be executed.""" + tool_response: Optional[ToolResponseDict] + """The output from a server-side ToolCall execution. This field is populated by the client with the results of executing the corresponding ToolCall.""" file_data: Optional[FileDataDict] """Optional. The URI-based data of the part. This can be used to include files from Google Cloud Storage.""" @@ -4742,6 +4856,10 @@ class ToolConfig(_common.BaseModel): retrieval_config: Optional[RetrievalConfig] = Field( default=None, description="""Optional. Retrieval config.""" ) + include_server_side_tool_invocations: Optional[bool] = Field( + default=None, + description="""If true, the API response will include the server-side tool calls and responses within the Content message. This allows clients to observe the server's tool invocations.""", + ) function_calling_config: Optional[FunctionCallingConfig] = Field( default=None, description="""Optional. Function calling config.""" ) @@ -4756,6 +4874,9 @@ class ToolConfigDict(TypedDict, total=False): retrieval_config: Optional[RetrievalConfigDict] """Optional. Retrieval config.""" + include_server_side_tool_invocations: Optional[bool] + """If true, the API response will include the server-side tool calls and responses within the Content message. This allows clients to observe the server's tool invocations.""" + function_calling_config: Optional[FunctionCallingConfigDict] """Optional. Function calling config."""