From 546440c9f56118c8d27005f2d5b935603e50454e Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Tue, 17 Mar 2026 16:50:15 -0700 Subject: [PATCH] feat: Support include_server_side_tool_invocations for genai. Adds ToolConfig.include_server_side_tool_invocations Adds ExecutableCode.id Adds CodeExecutionResult.id Adds ToolCall and ToolResponse PiperOrigin-RevId: 885278310 --- google/genai/_live_converters.py | 125 +++++++++- google/genai/_tokens_converters.py | 6 + google/genai/batches.py | 13 + google/genai/caches.py | 136 ++++++++++- google/genai/models.py | 169 ++++++++++++- .../models/test_generate_content_tools.py | 78 ++++++ google/genai/types.py | 228 ++++++++++++++---- 7 files changed, 697 insertions(+), 58 deletions(-) diff --git a/google/genai/_live_converters.py b/google/genai/_live_converters.py index 0f5efe982..4c2d7ec23 100644 --- a/google/genai/_live_converters.py +++ b/google/genai/_live_converters.py @@ -106,6 +106,27 @@ def _Content_to_mldev( return to_object +def _Content_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['parts']) is not None: + setv( + to_object, + ['parts'], + [ + _Part_to_vertex(item, to_object) + for item in getv(from_object, ['parts']) + ], + ) + + if getv(from_object, ['role']) is not None: + setv(to_object, ['role'], getv(from_object, ['role'])) + + return to_object + + def _FileData_to_mldev( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -361,6 +382,27 @@ def _LiveClientContent_to_mldev( return to_object +def _LiveClientContent_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['turns']) is not None: + setv( + to_object, + ['turns'], + [ + _Content_to_vertex(item, to_object) + for item in getv(from_object, ['turns']) + ], + ) + + if getv(from_object, ['turn_complete']) is not None: + setv(to_object, ['turnComplete'], getv(from_object, ['turn_complete'])) + + return to_object + + def _LiveClientMessage_to_mldev( api_client: BaseApiClient, from_object: Union[dict[str, Any], object], @@ -416,7 +458,13 @@ def _LiveClientMessage_to_vertex( ) if getv(from_object, ['client_content']) is not None: - setv(to_object, ['clientContent'], getv(from_object, ['client_content'])) + setv( + to_object, + ['clientContent'], + _LiveClientContent_to_vertex( + getv(from_object, ['client_content']), to_object + ), + ) if getv(from_object, ['realtime_input']) is not None: setv( @@ -617,7 +665,9 @@ def _LiveClientSetup_to_vertex( setv( to_object, ['systemInstruction'], - t.t_content(getv(from_object, ['system_instruction'])), + _Content_to_vertex( + t.t_content(getv(from_object, ['system_instruction'])), to_object + ), ) if getv(from_object, ['tools']) is not None: @@ -930,7 +980,9 @@ def _LiveConnectConfig_to_vertex( setv( parent_object, ['setup', 'systemInstruction'], - t.t_content(getv(from_object, ['system_instruction'])), + _Content_to_vertex( + t.t_content(getv(from_object, ['system_instruction'])), to_object + ), ) if getv(from_object, ['tools']) is not None: @@ -1403,6 +1455,73 @@ def _Part_to_mldev( if getv(from_object, ['video_metadata']) is not None: setv(to_object, ['videoMetadata'], getv(from_object, ['video_metadata'])) + if getv(from_object, ['tool_call']) is not None: + setv(to_object, ['toolCall'], getv(from_object, ['tool_call'])) + + if getv(from_object, ['tool_response']) is not None: + setv(to_object, ['toolResponse'], getv(from_object, ['tool_response'])) + + return to_object + + +def _Part_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['media_resolution']) is not None: + setv( + to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) + ) + + if getv(from_object, ['code_execution_result']) is not None: + setv( + to_object, + ['codeExecutionResult'], + getv(from_object, ['code_execution_result']), + ) + + if getv(from_object, ['executable_code']) is not None: + setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) + + if getv(from_object, ['file_data']) is not None: + setv(to_object, ['fileData'], getv(from_object, ['file_data'])) + + if getv(from_object, ['function_call']) is not None: + setv(to_object, ['functionCall'], getv(from_object, ['function_call'])) + + if getv(from_object, ['function_response']) is not None: + setv( + to_object, + ['functionResponse'], + getv(from_object, ['function_response']), + ) + + if getv(from_object, ['inline_data']) is not None: + setv(to_object, ['inlineData'], getv(from_object, ['inline_data'])) + + if getv(from_object, ['text']) is not None: + setv(to_object, ['text'], getv(from_object, ['text'])) + + if getv(from_object, ['thought']) is not None: + setv(to_object, ['thought'], getv(from_object, ['thought'])) + + if getv(from_object, ['thought_signature']) is not None: + setv( + to_object, + ['thoughtSignature'], + getv(from_object, ['thought_signature']), + ) + + if getv(from_object, ['video_metadata']) is not None: + setv(to_object, ['videoMetadata'], getv(from_object, ['video_metadata'])) + + if getv(from_object, ['tool_call']) is not None: + raise ValueError('tool_call parameter is not supported in Vertex AI.') + + if getv(from_object, ['tool_response']) is not None: + raise ValueError('tool_response parameter is not supported in Vertex AI.') + return to_object diff --git a/google/genai/_tokens_converters.py b/google/genai/_tokens_converters.py index 9fd9e95f2..6a0477bb9 100644 --- a/google/genai/_tokens_converters.py +++ b/google/genai/_tokens_converters.py @@ -515,6 +515,12 @@ def _Part_to_mldev( if getv(from_object, ['video_metadata']) is not None: setv(to_object, ['videoMetadata'], getv(from_object, ['video_metadata'])) + if getv(from_object, ['tool_call']) is not None: + setv(to_object, ['toolCall'], getv(from_object, ['tool_call'])) + + if getv(from_object, ['tool_response']) is not None: + setv(to_object, ['toolResponse'], getv(from_object, ['tool_response'])) + return to_object diff --git a/google/genai/batches.py b/google/genai/batches.py index 7a43a9aec..287101c4a 100644 --- a/google/genai/batches.py +++ b/google/genai/batches.py @@ -1454,6 +1454,12 @@ def _Part_to_mldev( if getv(from_object, ['video_metadata']) is not None: setv(to_object, ['videoMetadata'], getv(from_object, ['video_metadata'])) + if getv(from_object, ['tool_call']) is not None: + setv(to_object, ['toolCall'], getv(from_object, ['tool_call'])) + + if getv(from_object, ['tool_response']) is not None: + setv(to_object, ['toolResponse'], getv(from_object, ['tool_response'])) + return to_object @@ -1493,6 +1499,13 @@ def _ToolConfig_to_mldev( ), ) + if getv(from_object, ['include_server_side_tool_invocations']) is not None: + setv( + to_object, + ['includeServerSideToolInvocations'], + getv(from_object, ['include_server_side_tool_invocations']), + ) + return to_object diff --git a/google/genai/caches.py b/google/genai/caches.py index 010a820f3..e982b601b 100644 --- a/google/genai/caches.py +++ b/google/genai/caches.py @@ -105,6 +105,27 @@ def _Content_to_mldev( return to_object +def _Content_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['parts']) is not None: + setv( + to_object, + ['parts'], + [ + _Part_to_vertex(item, to_object) + for item in getv(from_object, ['parts']) + ], + ) + + if getv(from_object, ['role']) is not None: + setv(to_object, ['role'], getv(from_object, ['role'])) + + return to_object + + def _CreateCachedContentConfig_to_mldev( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -181,14 +202,19 @@ def _CreateCachedContentConfig_to_vertex( setv( parent_object, ['contents'], - [item for item in t.t_contents(getv(from_object, ['contents']))], + [ + _Content_to_vertex(item, to_object) + for item in t.t_contents(getv(from_object, ['contents'])) + ], ) if getv(from_object, ['system_instruction']) is not None: setv( parent_object, ['systemInstruction'], - t.t_content(getv(from_object, ['system_instruction'])), + _Content_to_vertex( + t.t_content(getv(from_object, ['system_instruction'])), to_object + ), ) if getv(from_object, ['tools']) is not None: @@ -202,7 +228,11 @@ def _CreateCachedContentConfig_to_vertex( ) if getv(from_object, ['tool_config']) is not None: - setv(parent_object, ['toolConfig'], getv(from_object, ['tool_config'])) + setv( + parent_object, + ['toolConfig'], + _ToolConfig_to_vertex(getv(from_object, ['tool_config']), to_object), + ) if getv(from_object, ['kms_key_name']) is not None: setv( @@ -667,6 +697,73 @@ def _Part_to_mldev( if getv(from_object, ['video_metadata']) is not None: setv(to_object, ['videoMetadata'], getv(from_object, ['video_metadata'])) + if getv(from_object, ['tool_call']) is not None: + setv(to_object, ['toolCall'], getv(from_object, ['tool_call'])) + + if getv(from_object, ['tool_response']) is not None: + setv(to_object, ['toolResponse'], getv(from_object, ['tool_response'])) + + return to_object + + +def _Part_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['media_resolution']) is not None: + setv( + to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) + ) + + if getv(from_object, ['code_execution_result']) is not None: + setv( + to_object, + ['codeExecutionResult'], + getv(from_object, ['code_execution_result']), + ) + + if getv(from_object, ['executable_code']) is not None: + setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) + + if getv(from_object, ['file_data']) is not None: + setv(to_object, ['fileData'], getv(from_object, ['file_data'])) + + if getv(from_object, ['function_call']) is not None: + setv(to_object, ['functionCall'], getv(from_object, ['function_call'])) + + if getv(from_object, ['function_response']) is not None: + setv( + to_object, + ['functionResponse'], + getv(from_object, ['function_response']), + ) + + if getv(from_object, ['inline_data']) is not None: + setv(to_object, ['inlineData'], getv(from_object, ['inline_data'])) + + if getv(from_object, ['text']) is not None: + setv(to_object, ['text'], getv(from_object, ['text'])) + + if getv(from_object, ['thought']) is not None: + setv(to_object, ['thought'], getv(from_object, ['thought'])) + + if getv(from_object, ['thought_signature']) is not None: + setv( + to_object, + ['thoughtSignature'], + getv(from_object, ['thought_signature']), + ) + + if getv(from_object, ['video_metadata']) is not None: + setv(to_object, ['videoMetadata'], getv(from_object, ['video_metadata'])) + + if getv(from_object, ['tool_call']) is not None: + raise ValueError('tool_call parameter is not supported in Vertex AI.') + + if getv(from_object, ['tool_response']) is not None: + raise ValueError('tool_response parameter is not supported in Vertex AI.') + return to_object @@ -689,6 +786,39 @@ def _ToolConfig_to_mldev( ), ) + if getv(from_object, ['include_server_side_tool_invocations']) is not None: + setv( + to_object, + ['includeServerSideToolInvocations'], + getv(from_object, ['include_server_side_tool_invocations']), + ) + + return to_object + + +def _ToolConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['retrieval_config']) is not None: + setv( + to_object, ['retrievalConfig'], getv(from_object, ['retrieval_config']) + ) + + if getv(from_object, ['function_calling_config']) is not None: + setv( + to_object, + ['functionCallingConfig'], + getv(from_object, ['function_calling_config']), + ) + + if getv(from_object, ['include_server_side_tool_invocations']) is not None: + raise ValueError( + 'include_server_side_tool_invocations parameter is not supported in' + ' Vertex AI.' + ) + return to_object diff --git a/google/genai/models.py b/google/genai/models.py index d2db40507..ddee88f2d 100644 --- a/google/genai/models.py +++ b/google/genai/models.py @@ -198,7 +198,10 @@ def _ComputeTokensParameters_to_vertex( setv( to_object, ['contents'], - [item for item in t.t_contents(getv(from_object, ['contents']))], + [ + _Content_to_vertex(item, to_object, root_object) + for item in t.t_contents(getv(from_object, ['contents'])) + ], ) return to_object @@ -283,6 +286,28 @@ def _Content_to_mldev( return to_object +def _Content_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, + root_object: Optional[Union[dict[str, Any], object]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['parts']) is not None: + setv( + to_object, + ['parts'], + [ + _Part_to_vertex(item, to_object, root_object) + for item in getv(from_object, ['parts']) + ], + ) + + if getv(from_object, ['role']) is not None: + setv(to_object, ['role'], getv(from_object, ['role'])) + + return to_object + + def _ControlReferenceConfig_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -336,7 +361,11 @@ def _CountTokensConfig_to_vertex( setv( parent_object, ['systemInstruction'], - t.t_content(getv(from_object, ['system_instruction'])), + _Content_to_vertex( + t.t_content(getv(from_object, ['system_instruction'])), + to_object, + root_object, + ), ) if getv(from_object, ['tools']) is not None: @@ -411,7 +440,10 @@ def _CountTokensParameters_to_vertex( setv( to_object, ['contents'], - [item for item in t.t_contents(getv(from_object, ['contents']))], + [ + _Content_to_vertex(item, to_object, root_object) + for item in t.t_contents(getv(from_object, ['contents'])) + ], ) if getv(from_object, ['config']) is not None: @@ -898,7 +930,15 @@ def _EmbedContentParametersPrivate_to_vertex( discriminator = 'PREDICT' if discriminator == 'EMBED_CONTENT': if getv(from_object, ['content']) is not None: - setv(to_object, ['content'], t.t_content(getv(from_object, ['content']))) + setv( + to_object, + ['content'], + _Content_to_vertex( + t.t_content(getv(from_object, ['content'])), + to_object, + root_object, + ), + ) if getv(from_object, ['config']) is not None: _EmbedContentConfig_to_vertex( @@ -1291,7 +1331,11 @@ def _GenerateContentConfig_to_vertex( setv( parent_object, ['systemInstruction'], - t.t_content(getv(from_object, ['system_instruction'])), + _Content_to_vertex( + t.t_content(getv(from_object, ['system_instruction'])), + to_object, + root_object, + ), ) if getv(from_object, ['temperature']) is not None: @@ -1388,7 +1432,13 @@ def _GenerateContentConfig_to_vertex( ) if getv(from_object, ['tool_config']) is not None: - setv(parent_object, ['toolConfig'], getv(from_object, ['tool_config'])) + setv( + parent_object, + ['toolConfig'], + _ToolConfig_to_vertex( + getv(from_object, ['tool_config']), to_object, root_object + ), + ) if getv(from_object, ['labels']) is not None: setv(parent_object, ['labels'], getv(from_object, ['labels'])) @@ -1505,7 +1555,10 @@ def _GenerateContentParameters_to_vertex( setv( to_object, ['contents'], - [item for item in t.t_contents(getv(from_object, ['contents']))], + [ + _Content_to_vertex(item, to_object, root_object) + for item in t.t_contents(getv(from_object, ['contents'])) + ], ) if getv(from_object, ['config']) is not None: @@ -3316,6 +3369,74 @@ def _Part_to_mldev( if getv(from_object, ['video_metadata']) is not None: setv(to_object, ['videoMetadata'], getv(from_object, ['video_metadata'])) + if getv(from_object, ['tool_call']) is not None: + setv(to_object, ['toolCall'], getv(from_object, ['tool_call'])) + + if getv(from_object, ['tool_response']) is not None: + setv(to_object, ['toolResponse'], getv(from_object, ['tool_response'])) + + return to_object + + +def _Part_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, + root_object: Optional[Union[dict[str, Any], object]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['media_resolution']) is not None: + setv( + to_object, ['mediaResolution'], getv(from_object, ['media_resolution']) + ) + + if getv(from_object, ['code_execution_result']) is not None: + setv( + to_object, + ['codeExecutionResult'], + getv(from_object, ['code_execution_result']), + ) + + if getv(from_object, ['executable_code']) is not None: + setv(to_object, ['executableCode'], getv(from_object, ['executable_code'])) + + if getv(from_object, ['file_data']) is not None: + setv(to_object, ['fileData'], getv(from_object, ['file_data'])) + + if getv(from_object, ['function_call']) is not None: + setv(to_object, ['functionCall'], getv(from_object, ['function_call'])) + + if getv(from_object, ['function_response']) is not None: + setv( + to_object, + ['functionResponse'], + getv(from_object, ['function_response']), + ) + + if getv(from_object, ['inline_data']) is not None: + setv(to_object, ['inlineData'], getv(from_object, ['inline_data'])) + + if getv(from_object, ['text']) is not None: + setv(to_object, ['text'], getv(from_object, ['text'])) + + if getv(from_object, ['thought']) is not None: + setv(to_object, ['thought'], getv(from_object, ['thought'])) + + if getv(from_object, ['thought_signature']) is not None: + setv( + to_object, + ['thoughtSignature'], + getv(from_object, ['thought_signature']), + ) + + if getv(from_object, ['video_metadata']) is not None: + setv(to_object, ['videoMetadata'], getv(from_object, ['video_metadata'])) + + if getv(from_object, ['tool_call']) is not None: + raise ValueError('tool_call parameter is not supported in Vertex AI.') + + if getv(from_object, ['tool_response']) is not None: + raise ValueError('tool_response parameter is not supported in Vertex AI.') + return to_object @@ -3775,6 +3896,40 @@ def _ToolConfig_to_mldev( ), ) + if getv(from_object, ['include_server_side_tool_invocations']) is not None: + setv( + to_object, + ['includeServerSideToolInvocations'], + getv(from_object, ['include_server_side_tool_invocations']), + ) + + return to_object + + +def _ToolConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, + root_object: Optional[Union[dict[str, Any], object]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ['retrieval_config']) is not None: + setv( + to_object, ['retrievalConfig'], getv(from_object, ['retrieval_config']) + ) + + if getv(from_object, ['function_calling_config']) is not None: + setv( + to_object, + ['functionCallingConfig'], + getv(from_object, ['function_calling_config']), + ) + + if getv(from_object, ['include_server_side_tool_invocations']) is not None: + raise ValueError( + 'include_server_side_tool_invocations parameter is not supported in' + ' Vertex AI.' + ) + return to_object diff --git a/google/genai/tests/models/test_generate_content_tools.py b/google/genai/tests/models/test_generate_content_tools.py index 142d9f980..b162d476b 100644 --- a/google/genai/tests/models/test_generate_content_tools.py +++ b/google/genai/tests/models/test_generate_content_tools.py @@ -603,6 +603,84 @@ def divide_floats(a: float, b: float) -> float: config={'tools': [{'google_maps': {'enable_widget': True}}]}, ), ), + pytest_helper.TestTableItem( + name='test_include_server_side_tool_invocations', + parameters=types._GenerateContentParameters( + model='gemini-3.1-pro-preview', + contents=t.t_contents('Why is the sky blue?'), + config=types.GenerateContentConfig( + tools=[ + types.Tool( + google_search=types.GoogleSearch(), + ), + ], + tool_config=types.ToolConfig( + include_server_side_tool_invocations=True, + ), + ), + ), + exception_if_vertex='parameter is not supported', + ), + pytest_helper.TestTableItem( + name='test_include_server_side_tool_invocations_with_tool_call_echo', + parameters=types._GenerateContentParameters( + model='gemini-3.1-pro-preview', + contents=[ + types.Content.model_validate(item) + for item in [ + { + 'role': 'user', + 'parts': [{'text': 'Why is the sky blue?'}], + }, + { + 'role': 'model', + 'parts': [ + { + 'tool_call': { + 'tool_type': 'GOOGLE_SEARCH', + 'args': { + 'query': 'why is the sky blue', + }, + }, + }, + { + 'tool_response': { + 'tool_type': 'GOOGLE_SEARCH', + 'response': { + 'result': ( + 'The sky is blue because of' + ' Rayleigh scattering.' + ), + }, + }, + }, + { + 'text': ( + 'The sky is blue due to a phenomenon called' + ' Rayleigh scattering.' + ), + }, + ], + }, + { + 'role': 'user', + 'parts': [{'text': 'What about Mars?'}], + }, + ] + ], + config=types.GenerateContentConfig( + tools=[ + types.Tool( + google_search=types.GoogleSearch(), + ), + ], + tool_config=types.ToolConfig( + include_server_side_tool_invocations=True, + ), + ), + ), + exception_if_vertex='parameter is not supported', + ), ] diff --git a/google/genai/types.py b/google/genai/types.py index ef6e87937..609134c58 100644 --- a/google/genai/types.py +++ b/google/genai/types.py @@ -140,6 +140,15 @@ MetricSubclass = typing.TypeVar('MetricSubclass', bound='Metric') +class Language(_common.CaseInSensitiveEnum): + """Programming language of the `code`.""" + + LANGUAGE_UNSPECIFIED = 'LANGUAGE_UNSPECIFIED' + """Unspecified language. This value should not be used.""" + PYTHON = 'PYTHON' + """Python >= 3.10, with numpy and simpy available.""" + + class Outcome(_common.CaseInSensitiveEnum): """Outcome of the code execution.""" @@ -153,15 +162,6 @@ class Outcome(_common.CaseInSensitiveEnum): """Code execution ran for too long, and was cancelled. There may or may not be a partial output present.""" -class Language(_common.CaseInSensitiveEnum): - """Programming language of the `code`.""" - - LANGUAGE_UNSPECIFIED = 'LANGUAGE_UNSPECIFIED' - """Unspecified language. This value should not be used.""" - PYTHON = 'PYTHON' - """Python >= 3.10, with numpy and simpy available.""" - - class FunctionResponseScheduling(_common.CaseInSensitiveEnum): """Specifies how the response should be scheduled in the conversation.""" @@ -737,6 +737,23 @@ class PartMediaResolutionLevel(_common.CaseInSensitiveEnum): """Media resolution set to ultra high.""" +class ToolType(_common.CaseInSensitiveEnum): + """The type of tool in the function call.""" + + TOOL_TYPE_UNSPECIFIED = 'TOOL_TYPE_UNSPECIFIED' + """Unspecified tool type.""" + GOOGLE_SEARCH_WEB = 'GOOGLE_SEARCH_WEB' + """Google search tool, maps to Tool.google_search.search_types.web_search.""" + GOOGLE_SEARCH_IMAGE = 'GOOGLE_SEARCH_IMAGE' + """Image search tool, maps to Tool.google_search.search_types.image_search.""" + URL_CONTEXT = 'URL_CONTEXT' + """URL context tool, maps to Tool.url_context.""" + GOOGLE_MAPS = 'GOOGLE_MAPS' + """Google maps tool, maps to Tool.google_maps.""" + FILE_SEARCH = 'FILE_SEARCH' + """File search tool, maps to Tool.file_search.""" + + class ResourceScope(_common.CaseInSensitiveEnum): """Resource scope.""" @@ -1126,6 +1143,86 @@ class LiveMusicPlaybackControl(_common.CaseInSensitiveEnum): Retains the current prompts and config.""" +class ExecutableCode(_common.BaseModel): + """Model-generated code executed server-side, results returned to the model. + + Only generated when using the `CodeExecution` tool, in which the code will + be automatically executed, and a corresponding `CodeExecutionResult` will + also be generated. + """ + + code: Optional[str] = Field( + default=None, description="""Required. The code to be executed.""" + ) + language: Optional[Language] = Field( + default=None, + description="""Required. Programming language of the `code`.""", + ) + id: Optional[str] = Field( + default=None, + description="""Unique identifier of the `ExecutableCode` part. The server returns the `CodeExecutionResult` with the matching `id`.""", + ) + + +class ExecutableCodeDict(TypedDict, total=False): + """Model-generated code executed server-side, results returned to the model. + + Only generated when using the `CodeExecution` tool, in which the code will + be automatically executed, and a corresponding `CodeExecutionResult` will + also be generated. + """ + + code: Optional[str] + """Required. The code to be executed.""" + + language: Optional[Language] + """Required. Programming language of the `code`.""" + + id: Optional[str] + """Unique identifier of the `ExecutableCode` part. The server returns the `CodeExecutionResult` with the matching `id`.""" + + +ExecutableCodeOrDict = Union[ExecutableCode, ExecutableCodeDict] + + +class CodeExecutionResult(_common.BaseModel): + """Result of executing the `ExecutableCode`. + + Generated only when the `CodeExecution` tool is used. + """ + + outcome: Optional[Outcome] = Field( + default=None, description="""Required. Outcome of the code execution.""" + ) + output: Optional[str] = Field( + default=None, + description="""Optional. Contains stdout when code execution is successful, stderr or other description otherwise.""", + ) + id: Optional[str] = Field( + default=None, + description="""The identifier of the `ExecutableCode` part this result is for. Only populated if the corresponding `ExecutableCode` has an id.""", + ) + + +class CodeExecutionResultDict(TypedDict, total=False): + """Result of executing the `ExecutableCode`. + + Generated only when the `CodeExecution` tool is used. + """ + + outcome: Optional[Outcome] + """Required. Outcome of the code execution.""" + + output: Optional[str] + """Optional. Contains stdout when code execution is successful, stderr or other description otherwise.""" + + id: Optional[str] + """The identifier of the `ExecutableCode` part this result is for. Only populated if the corresponding `ExecutableCode` has an id.""" + + +CodeExecutionResultOrDict = Union[CodeExecutionResult, CodeExecutionResultDict] + + class PartMediaResolution(_common.BaseModel): """Media resolution for the input media.""" @@ -1156,72 +1253,92 @@ class PartMediaResolutionDict(TypedDict, total=False): PartMediaResolutionOrDict = Union[PartMediaResolution, PartMediaResolutionDict] -class CodeExecutionResult(_common.BaseModel): - """Result of executing the [ExecutableCode]. +class ToolCall(_common.BaseModel): + """A predicted server-side `ToolCall` returned from the model. - Only generated when using the [CodeExecution] tool, and always follows a - `part` containing the [ExecutableCode]. + This message contains information about a tool that the model wants to invoke. + The client is NOT expected to execute this `ToolCall`. Instead, the + client should pass this `ToolCall` back to the API in a subsequent turn + within a `Content` message, along with the corresponding `ToolResponse`. """ - outcome: Optional[Outcome] = Field( - default=None, description="""Required. Outcome of the code execution.""" + id: Optional[str] = Field( + default=None, + description="""Unique identifier of the tool call. The server returns the tool response with the matching `id`.""", ) - output: Optional[str] = Field( + tool_type: Optional[ToolType] = Field( + default=None, description="""The type of tool that was called.""" + ) + args: Optional[dict[str, Any]] = Field( default=None, - description="""Optional. Contains stdout when code execution is successful, stderr or other description otherwise.""", + description="""The tool call arguments. Example: {"arg1": "value1", "arg2": "value2"}.""", ) -class CodeExecutionResultDict(TypedDict, total=False): - """Result of executing the [ExecutableCode]. +class ToolCallDict(TypedDict, total=False): + """A predicted server-side `ToolCall` returned from the model. - Only generated when using the [CodeExecution] tool, and always follows a - `part` containing the [ExecutableCode]. + This message contains information about a tool that the model wants to invoke. + The client is NOT expected to execute this `ToolCall`. Instead, the + client should pass this `ToolCall` back to the API in a subsequent turn + within a `Content` message, along with the corresponding `ToolResponse`. """ - outcome: Optional[Outcome] - """Required. Outcome of the code execution.""" + id: Optional[str] + """Unique identifier of the tool call. The server returns the tool response with the matching `id`.""" - output: Optional[str] - """Optional. Contains stdout when code execution is successful, stderr or other description otherwise.""" + tool_type: Optional[ToolType] + """The type of tool that was called.""" + args: Optional[dict[str, Any]] + """The tool call arguments. Example: {"arg1": "value1", "arg2": "value2"}.""" -CodeExecutionResultOrDict = Union[CodeExecutionResult, CodeExecutionResultDict] +ToolCallOrDict = Union[ToolCall, ToolCallDict] -class ExecutableCode(_common.BaseModel): - """Code generated by the model that is meant to be executed, and the result returned to the model. - Generated when using the [CodeExecution] tool, in which the code will be - automatically executed, and a corresponding [CodeExecutionResult] will also be - generated. +class ToolResponse(_common.BaseModel): + """The output from a server-side `ToolCall` execution. + + This message contains the results of a tool invocation that was initiated by a + `ToolCall` from the model. The client should pass this `ToolResponse` back to + the API in a subsequent turn within a `Content` message, along with the + corresponding `ToolCall`. """ - code: Optional[str] = Field( - default=None, description="""Required. The code to be executed.""" + id: Optional[str] = Field( + default=None, + description="""The identifier of the tool call this response is for.""", ) - language: Optional[Language] = Field( + tool_type: Optional[ToolType] = Field( default=None, - description="""Required. Programming language of the `code`.""", + description="""The type of tool that was called, matching the tool_type in the corresponding ToolCall.""", + ) + response: Optional[dict[str, Any]] = Field( + default=None, description="""The tool response.""" ) -class ExecutableCodeDict(TypedDict, total=False): - """Code generated by the model that is meant to be executed, and the result returned to the model. +class ToolResponseDict(TypedDict, total=False): + """The output from a server-side `ToolCall` execution. - Generated when using the [CodeExecution] tool, in which the code will be - automatically executed, and a corresponding [CodeExecutionResult] will also be - generated. + This message contains the results of a tool invocation that was initiated by a + `ToolCall` from the model. The client should pass this `ToolResponse` back to + the API in a subsequent turn within a `Content` message, along with the + corresponding `ToolCall`. """ - code: Optional[str] - """Required. The code to be executed.""" + id: Optional[str] + """The identifier of the tool call this response is for.""" - language: Optional[Language] - """Required. Programming language of the `code`.""" + tool_type: Optional[ToolType] + """The type of tool that was called, matching the tool_type in the corresponding ToolCall.""" + response: Optional[dict[str, Any]] + """The tool response.""" -ExecutableCodeOrDict = Union[ExecutableCode, ExecutableCodeDict] + +ToolResponseOrDict = Union[ToolResponse, ToolResponseDict] class FileData(_common.BaseModel): @@ -1738,6 +1855,14 @@ class Part(_common.BaseModel): default=None, description="""Optional. Video metadata. The metadata should only be specified while the video data is presented in inline_data or file_data.""", ) + tool_call: Optional[ToolCall] = Field( + default=None, + description="""Server-side tool call. This field is populated when the model predicts a tool invocation that should be executed on the server. The client is expected to echo this message back to the API.""", + ) + tool_response: Optional[ToolResponse] = Field( + default=None, + description="""The output from a server-side ToolCall execution. This field is populated by the client with the results of executing the corresponding ToolCall.""", + ) def __init__( self, @@ -1960,6 +2085,12 @@ class PartDict(TypedDict, total=False): video_metadata: Optional[VideoMetadataDict] """Optional. Video metadata. The metadata should only be specified while the video data is presented in inline_data or file_data.""" + tool_call: Optional[ToolCallDict] + """Server-side tool call. This field is populated when the model predicts a tool invocation that should be executed on the server. The client is expected to echo this message back to the API.""" + + tool_response: Optional[ToolResponseDict] + """The output from a server-side ToolCall execution. This field is populated by the client with the results of executing the corresponding ToolCall.""" + PartOrDict = Union[Part, PartDict] @@ -4745,6 +4876,10 @@ class ToolConfig(_common.BaseModel): function_calling_config: Optional[FunctionCallingConfig] = Field( default=None, description="""Optional. Function calling config.""" ) + include_server_side_tool_invocations: Optional[bool] = Field( + default=None, + description="""If true, the API response will include the server-side tool calls and responses within the `Content` message. This allows clients to observe the server's tool invocations.""", + ) class ToolConfigDict(TypedDict, total=False): @@ -4759,6 +4894,9 @@ class ToolConfigDict(TypedDict, total=False): function_calling_config: Optional[FunctionCallingConfigDict] """Optional. Function calling config.""" + include_server_side_tool_invocations: Optional[bool] + """If true, the API response will include the server-side tool calls and responses within the `Content` message. This allows clients to observe the server's tool invocations.""" + ToolConfigOrDict = Union[ToolConfig, ToolConfigDict]