diff --git a/docs/genai.html b/docs/genai.html index f01f209f3..59ab94315 100644 --- a/docs/genai.html +++ b/docs/genai.html @@ -2541,7 +2541,7 @@

genai.interactions module
-async edit_image(*, model, prompt, reference_images, config=None)
+edit_image(*, model, prompt, reference_images, config=None)

Edits an image based on a text description and configuration.

Return type:
@@ -2740,7 +2740,7 @@

genai.interactions module
-async generate_images(*, model, prompt, config=None)
+generate_images(*, model, prompt, config=None)

Generates images based on a text description and configuration.

Return type:
@@ -6432,6 +6432,32 @@

genai.interactions module "default": null, "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" } }, "$defs": { @@ -6476,9 +6502,11 @@

genai.interactions module

Validators:
@@ -6510,6 +6538,19 @@

genai.interactions module +
+field diarization: Optional[bool] = None
+

Configures speaker diarization.

+
+
Validated by:
+
    +

  • _check_field_type_mismatches

    • +
+
+
+

+
field language_auto: Optional[LanguageAuto] = None (alias 'languageAuto')
@@ -6549,6 +6590,19 @@

genai.interactions module +
+field word_timestamp: Optional[bool] = None (alias 'wordTimestamp')
+

Configures word-level timestamp generation.

+
+
Validated by:
+
    +

  • _check_field_type_mismatches

    • +
+
+
+

+
@@ -6568,6 +6622,12 @@

genai.interactions module +
+diarization: Optional[bool]
+

Configures speaker diarization.

+

+
language_auto: Optional[LanguageAutoDict]
@@ -6591,6 +6651,12 @@

genai.interactions module +
+word_timestamp: Optional[bool]
+

Configures word-level timestamp generation.

+

+
@@ -8074,6 +8140,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "Delivery": { "description": "Delivery mode for the generated content.", "enum": [ @@ -8423,6 +8595,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -8577,6 +8761,37 @@

genai.interactions module "title": "ImageSize", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MediaResolution": { "description": "The media resolution to use.", "enum": [ @@ -13121,6 +13336,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -13714,6 +13941,82 @@

genai.interactions module "title": "TrafficType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "UrlContextMetadata": { "additionalProperties": false, "description": "Metadata returned when the model uses the `url_context` tool to get information from a user-provided URL.", @@ -13863,6 +14166,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -17112,6 +17462,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -17705,6 +18067,82 @@

genai.interactions module "title": "TrafficType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "UrlContextMetadata": { "additionalProperties": false, "description": "Metadata returned when the model uses the `url_context` tool to get information from a user-provided URL.", @@ -17854,6 +18292,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -21907,6 +22392,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -22455,6 +22952,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "UrlContextMetadata": { "additionalProperties": false, "description": "Metadata returned when the model uses the `url_context` tool to get information from a user-provided URL.", @@ -22570,6 +23143,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -23993,6 +24613,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AutoraterConfig": { "additionalProperties": false, "description": "Autorater config used for evaluation.", @@ -24434,6 +25160,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -24588,6 +25326,37 @@

genai.interactions module "title": "ImageSize", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MatchOperation": { "description": "Match operation to use for evaluating rewards.\n\nThis enum is not supported in Gemini API.", "enum": [ @@ -26053,6 +26822,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AutoraterConfig": { "additionalProperties": false, "description": "Autorater config used for evaluation.", @@ -26461,6 +27336,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -26615,6 +27502,37 @@

genai.interactions module "title": "ImageSize", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MatchOperation": { "description": "Match operation to use for evaluating rewards.\n\nThis enum is not supported in Gemini API.", "enum": [ @@ -29465,6 +30383,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -29709,6 +30639,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -29755,6 +30761,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -31273,6 +32326,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AuthConfig": { "additionalProperties": false, "description": "The authentication config to access the API.", @@ -33072,6 +34231,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -33404,6 +34575,37 @@

genai.interactions module "title": "Language", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "McpServer": { "additionalProperties": false, "description": "A MCPServer is a server that can be called by the model to perform actions.\n\nIt is a server that implements the MCP protocol. Next ID: 6. This data type is\nnot supported in Vertex AI.", @@ -33692,6 +34894,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -35227,6 +36441,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "TranslationConfig": { "additionalProperties": false, "description": "Config for stream translation.", @@ -35685,6 +36975,53 @@

genai.interactions module "properties": {}, "title": "WebSearch", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -39432,6 +40769,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -40025,6 +41374,82 @@

genai.interactions module "title": "TrafficType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "UrlContextMetadata": { "additionalProperties": false, "description": "Metadata returned when the model uses the `url_context` tool to get information from a user-provided URL.", @@ -40212,6 +41637,53 @@

genai.interactions module }, "title": "WebhookConfig", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -42598,6 +44070,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -43905,6 +45389,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "Type": { "description": "Data type of the schema field.", "enum": [ @@ -44218,6 +45778,53 @@

genai.interactions module "properties": {}, "title": "WebSearch", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -45359,6 +46966,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AutoraterConfig": { "additionalProperties": false, "description": "Autorater config used for evaluation.", @@ -46075,6 +47788,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -46338,6 +48063,37 @@

genai.interactions module "title": "LLMBasedMetricSpec", "type": "object" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MatchOperation": { "description": "Match operation to use for evaluating rewards.\n\nThis enum is not supported in Gemini API.", "enum": [ @@ -51034,6 +52790,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -51354,6 +53122,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -51400,6 +53244,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -53811,6 +55702,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -54131,6 +56034,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -54177,6 +56156,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -54410,6 +56436,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "Delivery": { "description": "Delivery mode for the generated content.", "enum": [ @@ -54759,6 +56891,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -54913,6 +57057,37 @@

genai.interactions module "title": "ImageSize", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MediaResolution": { "description": "The media resolution to use.", "enum": [ @@ -56123,6 +58298,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "Delivery": { "description": "Delivery mode for the generated content.", "enum": [ @@ -56556,6 +58837,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -56710,6 +59003,37 @@

genai.interactions module "title": "ImageSize", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MediaResolution": { "description": "The media resolution to use.", "enum": [ @@ -57943,6 +60267,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "Delivery": { "description": "Delivery mode for the generated content.", "enum": [ @@ -58376,6 +60806,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -58530,6 +60972,37 @@

genai.interactions module "title": "ImageSize", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MediaResolution": { "description": "The media resolution to use.", "enum": [ @@ -62444,6 +64917,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -62688,6 +65173,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -62734,6 +65295,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -64271,6 +66879,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -64515,6 +67135,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -64561,6 +67257,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -67044,6 +69787,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AutoraterConfig": { "additionalProperties": false, "description": "Autorater config used for evaluation.", @@ -67622,6 +70471,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -67885,6 +70746,37 @@

genai.interactions module "title": "LLMBasedMetricSpec", "type": "object" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MediaResolution": { "description": "The media resolution to use.", "enum": [ @@ -75451,6 +78343,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -75695,6 +78599,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -75741,6 +78721,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -76553,6 +79580,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -76797,6 +79836,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -76843,6 +79958,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -77408,9 +80570,127 @@

genai.interactions module ], "default": null, "description": "The service tier to use for the request. For example, ServiceTier.FLEX." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "$defs": { + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AutomaticFunctionCallingConfig": { "additionalProperties": false, "description": "The configuration for automatic function calling.", @@ -78583,6 +81863,37 @@

genai.interactions module "title": "Language", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "LatLng": { "additionalProperties": false, "description": "An object that represents a latitude/longitude pair.\n\nThis is expressed as a pair of doubles to represent degrees latitude and\ndegrees longitude. Unless specified otherwise, this object must conform to the\nWGS84 standard. Values must be within normalized ranges.", @@ -78894,6 +82205,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -79842,6 +83165,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "Type": { "description": "Data type of the schema field.", "enum": [ @@ -79956,6 +83355,53 @@

genai.interactions module }, "title": "VoiceConsentSignature", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -79966,6 +83412,7 @@

genai.interactions moduleFields:

+
field automatic_function_calling: Optional[AutomaticFunctionCallingConfig] = None (alias 'automaticFunctionCalling')
@@ -80524,6 +83984,12 @@

genai.interactions module +
+audio_transcription_config: Optional[AudioTranscriptionConfigDict]
+

Optional. Configuration for audio transcription (speech recognition).

+

+
automatic_function_calling: Optional[AutomaticFunctionCallingConfigDict]
@@ -83491,6 +86957,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -84051,6 +87529,82 @@

genai.interactions module "title": "TrafficType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "UrlContextMetadata": { "additionalProperties": false, "description": "Metadata returned when the model uses the `url_context` tool to get information from a user-provided URL.", @@ -84166,6 +87720,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -88782,6 +92383,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "$defs": { @@ -88866,6 +92479,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "Delivery": { "description": "Delivery mode for the generated content.", "enum": [ @@ -89036,6 +92755,37 @@

genai.interactions module "title": "ImageSize", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MediaResolution": { "description": "The media resolution to use.", "enum": [ @@ -89921,6 +93671,7 @@

genai.interactions moduleFields:

+
field candidate_count: Optional[int] = None (alias 'candidateCount')
@@ -90290,6 +94054,12 @@

genai.interactions module +
+audio_transcription_config: Optional[AudioTranscriptionConfigDict]
+

Optional. Configuration for audio transcription (speech recognition).

+

+
candidate_count: Optional[int]
@@ -101856,6 +105626,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -102100,6 +105882,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -102146,6 +106004,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -105058,6 +108963,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -105618,6 +109535,82 @@

genai.interactions module "title": "TrafficType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "UrlContextMetadata": { "additionalProperties": false, "description": "Metadata returned when the model uses the `url_context` tool to get information from a user-provided URL.", @@ -105733,6 +109726,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -107440,6 +111480,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AutoraterConfig": { "additionalProperties": false, "description": "Autorater config used for evaluation.", @@ -107889,6 +112035,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -108043,6 +112201,37 @@

genai.interactions module "title": "ImageSize", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MediaResolution": { "description": "The media resolution to use.", "enum": [ @@ -112886,6 +117075,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AutoraterConfig": { "additionalProperties": false, "description": "Autorater config used for evaluation.", @@ -115154,6 +119449,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -115533,6 +119840,37 @@

genai.interactions module "title": "Language", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MatchOperation": { "description": "Match operation to use for evaluating rewards.\n\nThis enum is not supported in Gemini API.", "enum": [ @@ -115896,6 +120234,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -118617,6 +122967,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "TranslationConfig": { "additionalProperties": false, "description": "Config for stream translation.", @@ -119797,6 +124223,53 @@

genai.interactions module }, "title": "VoiceConsentSignature", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -120654,6 +125127,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -120898,6 +125383,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -120944,6 +125505,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -121255,6 +125863,32 @@

genai.interactions module "default": null, "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" } }, "title": "AudioTranscriptionConfig", @@ -122528,6 +127162,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -123396,6 +128042,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -124409,6 +129067,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "TranslationConfig": { "additionalProperties": false, "description": "Config for stream translation.", @@ -124615,6 +129349,53 @@

genai.interactions module }, "title": "VoiceConsentSignature", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -125450,6 +130231,32 @@

genai.interactions module "default": null, "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" } }, "title": "AudioTranscriptionConfig", @@ -126723,6 +131530,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -127234,6 +132053,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -128247,6 +133078,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "TranslationConfig": { "additionalProperties": false, "description": "Config for stream translation.", @@ -128453,6 +133360,53 @@

genai.interactions module }, "title": "VoiceConsentSignature", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -129595,6 +134549,32 @@

genai.interactions module "default": null, "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" } }, "title": "AudioTranscriptionConfig", @@ -130949,6 +135929,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -131460,6 +136452,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -132527,6 +137531,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "TranslationConfig": { "additionalProperties": false, "description": "Config for stream translation.", @@ -132744,6 +137824,53 @@

genai.interactions module }, "title": "VoiceConsentSignature", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -138696,6 +143823,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -139191,6 +144330,35 @@

genai.interactions module "default": null, "description": "The BCP-47 language code of the transcription.", "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" } }, "title": "Transcription", @@ -139346,6 +144514,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -141821,6 +147036,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -142339,6 +147566,35 @@

genai.interactions module "default": null, "description": "The BCP-47 language code of the transcription.", "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" } }, "title": "Transcription", @@ -142761,6 +148017,53 @@

genai.interactions module }, "title": "VoiceConsentSignature", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -146821,6 +152124,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -147065,6 +152380,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -147111,6 +152502,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false, @@ -148726,6 +154164,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "$defs": { @@ -149479,6 +154929,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -149525,6 +155051,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -149534,6 +155107,7 @@

genai.interactions module
Fields:

+
+
+field audio_transcription: Optional[Transcription] = None (alias 'audioTranscription')
+

Output only. The transcription of the audio part.

+
+
Validated by:
+
    +

  • _check_field_type_mismatches

    • +
+
+
+
+
field code_execution_result: Optional[CodeExecutionResult] = None (alias 'codeExecutionResult')
@@ -149834,6 +155421,12 @@

genai.interactions module +
+audio_transcription: Optional[TranscriptionDict]
+

Output only. The transcription of the audio part.

+

+
code_execution_result: Optional[CodeExecutionResultDict]
@@ -152313,6 +157906,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -152557,6 +158162,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -152603,6 +158284,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -156855,6 +162583,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AutoraterConfig": { "additionalProperties": false, "description": "Autorater config used for evaluation.", @@ -157263,6 +163097,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -157417,6 +163263,37 @@

genai.interactions module "title": "ImageSize", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MediaResolution": { "description": "The media resolution to use.", "enum": [ @@ -159720,6 +165597,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -159964,6 +165853,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -160010,6 +165975,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -160891,6 +166903,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AutoraterConfig": { "additionalProperties": false, "description": "Autorater config used for evaluation.", @@ -161356,6 +167474,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -161510,6 +167640,37 @@

genai.interactions module "title": "ImageSize", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MatchOperation": { "description": "Match operation to use for evaluating rewards.\n\nThis enum is not supported in Gemini API.", "enum": [ @@ -164396,6 +170557,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -164692,6 +170865,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -164738,6 +170987,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -171614,6 +177910,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AutoraterConfig": { "additionalProperties": false, "description": "Autorater config used for evaluation.", @@ -172022,6 +178424,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -172176,6 +178590,37 @@

genai.interactions module "title": "ImageSize", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MatchOperation": { "description": "Match operation to use for evaluating rewards.\n\nThis enum is not supported in Gemini API.", "enum": [ @@ -176413,6 +182858,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -176832,6 +183289,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -176878,6 +183411,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -182503,6 +189083,84 @@

genai.interactions module "default": null, "description": "The BCP-47 language code of the transcription.", "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "$defs": { + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -182514,7 +189172,9 @@

genai.interactions module
Validators:
@@ -182546,6 +189206,19 @@

genai.interactions module +
+field speaker_label: Optional[str] = None (alias 'speakerLabel')
+

A label identifying the speaker of this audio segment (e.g. “spk_1”, “spk_2”).

+
+
Validated by:
+
    +

  • _check_field_type_mismatches

    • +
+
+
+

+
field text: Optional[str] = None
@@ -182559,6 +189232,19 @@

genai.interactions module +
+field words: Optional[list[WordInfo]] = None
+

Detailed word-level transcriptions and timing details.

+
+
Validated by:
+
    +

  • _check_field_type_mismatches

    • +
+
+
+

+
@@ -182578,12 +189264,24 @@

genai.interactions module +
+speaker_label: Optional[str]
+

A label identifying the speaker of this audio segment (e.g. “spk_1”, “spk_2”).

+

+
text: Optional[str]

Optional. Transcription text.

+
+
+words: Optional[list[WordInfoDict]]
+

Detailed word-level transcriptions and timing details.

+
+
@@ -184440,6 +191138,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -185248,6 +191958,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -185294,6 +192080,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -186374,6 +193207,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AutoraterConfig": { "additionalProperties": false, "description": "Autorater config used for evaluation.", @@ -188642,6 +195581,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -189021,6 +195972,37 @@

genai.interactions module "title": "Language", "type": "string" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MatchOperation": { "description": "Match operation to use for evaluating rewards.\n\nThis enum is not supported in Gemini API.", "enum": [ @@ -189384,6 +196366,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -192105,6 +199099,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "TranslationConfig": { "additionalProperties": false, "description": "Config for stream translation.", @@ -192844,6 +199914,53 @@

genai.interactions module }, "title": "VoiceConsentSignature", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false @@ -194638,6 +201755,112 @@

genai.interactions module "title": "AudioResponseFormat", "type": "object" }, + "AudioTranscriptionConfig": { + "additionalProperties": false, + "description": "The audio transcription configuration in Setup.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated: use LanguageAuto or LanguageHints instead.", + "title": "Languagecodes" + }, + "languageAuto": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageAuto" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The model will detect the language automatically. Do not use together with LanguageHints." + }, + "languageHints": { + "anyOf": [ + { + "$ref": "#/$defs/LanguageHints" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specifies one or more languages in the audio. Do not use together with LanguageAuto." + }, + "customVocabulary": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom vocabulary phrases, which biases the ASR model to improve recognition of these specific terms.", + "title": "Customvocabulary" + }, + "adaptationPhrases": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Deprecated. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", + "title": "Adaptationphrases" + }, + "wordTimestamp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures word-level timestamp generation.\n ", + "title": "Wordtimestamp" + }, + "diarization": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configures speaker diarization.\n ", + "title": "Diarization" + } + }, + "title": "AudioTranscriptionConfig", + "type": "object" + }, "AutoraterConfig": { "additionalProperties": false, "description": "Autorater config used for evaluation.", @@ -195195,6 +202418,18 @@

genai.interactions module ], "default": null, "description": "Optional. Config for translation. This field is not supported in Vertex AI." + }, + "audioTranscriptionConfig": { + "anyOf": [ + { + "$ref": "#/$defs/AudioTranscriptionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Configuration for audio transcription (speech recognition).\n " } }, "title": "GenerationConfig", @@ -195458,6 +202693,37 @@

genai.interactions module "title": "LLMBasedMetricSpec", "type": "object" }, + "LanguageAuto": { + "additionalProperties": false, + "description": "Indicates the language of the audio should be automatically detected.", + "properties": {}, + "title": "LanguageAuto", + "type": "object" + }, + "LanguageHints": { + "additionalProperties": false, + "description": "Provides hints to the model about possible languages present in the audio.", + "properties": { + "languageCodes": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BCP-47 language codes. At least one must be specified.", + "title": "Languagecodes" + } + }, + "title": "LanguageHints", + "type": "object" + }, "MediaResolution": { "description": "The media resolution to use.", "enum": [ @@ -200457,6 +207723,18 @@

genai.interactions module "default": null, "description": "Custom metadata associated with the Part. Agents using genai.Part as content representation may need to keep track of the additional information. For example it can be name of a file/source from which the Part originates or a way to multiplex multiple Part streams. This field is not supported in Vertex AI.", "title": "Partmetadata" + }, + "audioTranscription": { + "anyOf": [ + { + "$ref": "#/$defs/Transcription" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output only. The transcription of the audio part." } }, "title": "Part", @@ -200701,6 +207979,82 @@

genai.interactions module "title": "ToolType", "type": "string" }, + "Transcription": { + "additionalProperties": false, + "description": "Audio transcription in Server Content.", + "properties": { + "text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Transcription text.", + "title": "Text" + }, + "finished": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. The bool indicates the end of the transcription.", + "title": "Finished" + }, + "languageCode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The BCP-47 language code of the transcription.", + "title": "Languagecode" + }, + "speakerLabel": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A label identifying the speaker of this audio segment (e.g. \"spk_1\", \"spk_2\").\n ", + "title": "Speakerlabel" + }, + "words": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/WordInfo" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Detailed word-level transcriptions and timing details.\n ", + "title": "Words" + } + }, + "title": "Transcription", + "type": "object" + }, "VideoMetadata": { "additionalProperties": false, "description": "Provides metadata for a video, including the start and end offsets for clipping and the frame rate.", @@ -200747,6 +208101,53 @@

genai.interactions module }, "title": "VideoMetadata", "type": "object" + }, + "WordInfo": { + "additionalProperties": false, + "description": "Information about a single recognized word.", + "properties": { + "word": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Transcript of the word.\n ", + "title": "Word" + }, + "startOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start offset in time of the word relative to the start of the audio.\n ", + "title": "Startoffset" + }, + "endOffset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End offset in time of the word relative to the start of the audio.\n ", + "title": "Endoffset" + } + }, + "title": "WordInfo", + "type": "object" } }, "additionalProperties": false, @@ -204855,6 +212256,142 @@

genai.interactions module +
+pydantic model genai.types.WordInfo
+

Bases: BaseModel

+

Information about a single recognized word.

+

Create a new model by parsing and validating input data from keyword arguments.

+

Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be +validated to form a valid model.

+

self is explicitly positional-only to allow self as a field name.

+

+Show JSON schema
{
+   "title": "WordInfo",
+   "description": "Information about a single recognized word.",
+   "type": "object",
+   "properties": {
+      "word": {
+         "anyOf": [
+            {
+               "type": "string"
+            },
+            {
+               "type": "null"
+            }
+         ],
+         "default": null,
+         "description": "Transcript of the word.\n      ",
+         "title": "Word"
+      },
+      "startOffset": {
+         "anyOf": [
+            {
+               "type": "string"
+            },
+            {
+               "type": "null"
+            }
+         ],
+         "default": null,
+         "description": "Start offset in time of the word relative to the start of the audio.\n      ",
+         "title": "Startoffset"
+      },
+      "endOffset": {
+         "anyOf": [
+            {
+               "type": "string"
+            },
+            {
+               "type": "null"
+            }
+         ],
+         "default": null,
+         "description": "End offset in time of the word relative to the start of the audio.\n      ",
+         "title": "Endoffset"
+      }
+   },
+   "additionalProperties": false
+}
+
+
+

+
Fields:
+
+
+
Validators:
+

+
+
+
+field end_offset: Optional[str] = None (alias 'endOffset')
+

End offset in time of the word relative to the start of the audio.

+
+
Validated by:
+
    +

  • _check_field_type_mismatches

    • +
+
+
+
+ +
+
+field start_offset: Optional[str] = None (alias 'startOffset')
+

Start offset in time of the word relative to the start of the audio.

+
+
Validated by:
+
    +

  • _check_field_type_mismatches

    • +
+
+
+
+ +
+
+field word: Optional[str] = None
+

Transcript of the word.

+
+
Validated by:
+
    +

  • _check_field_type_mismatches

    • +
+
+
+
+ +

+ +
+
+class genai.types.WordInfoDict
+

Bases: TypedDict

+

Information about a single recognized word.

+
+
+end_offset: Optional[str]
+

End offset in time of the word relative to the start of the audio.

+
+ +
+
+start_offset: Optional[str]
+

Start offset in time of the word relative to the start of the audio.

+
+ +
+
+word: Optional[str]
+

Transcript of the word.

+
+ +
+ diff --git a/docs/genindex.html b/docs/genindex.html index 0c06d6164..56fe5f6c2 100644 --- a/docs/genindex.html +++ b/docs/genindex.html @@ -734,6 +734,22 @@

A

+
  • audio_transcription (genai.types.Part attribute) + +
    • +
  • audio_transcription_config (genai.types.GenerateContentConfig attribute) + +
  • AudioChunkDict (class in genai.types) @@ -2232,6 +2248,12 @@

    D

  • (genai.types.JobError attribute)
  • (genai.types.JobErrorDict attribute) +
    • + +
  • diarization (genai.types.AudioTranscriptionConfig attribute) + +
  • disable (genai.types.AutomaticFunctionCallingConfig attribute) @@ -2700,6 +2722,10 @@

    E

  • END_SENSITIVITY_HIGH (genai.types.EndSensitivity attribute) @@ -2824,6 +2850,8 @@

    E

  • (genai.types.TunedModelCheckpointDict attribute)
    • + + - - + - +