From df33689835da4885d8d7a2acc137aed5e313ec2e Mon Sep 17 00:00:00 2001 From: timpratim Date: Wed, 19 Aug 2026 15:48:16 +0200 Subject: [PATCH 1/5] docs: add Gradium custom voice and custom transcriber guides --- .../custom-transcriber/gradium.mdx | 218 ++++++++++++++++++ fern/customization/custom-voices/gradium.mdx | 169 ++++++++++++++ fern/docs.yml | 7 +- 3 files changed, 393 insertions(+), 1 deletion(-) create mode 100644 fern/customization/custom-transcriber/gradium.mdx create mode 100644 fern/customization/custom-voices/gradium.mdx diff --git a/fern/customization/custom-transcriber/gradium.mdx b/fern/customization/custom-transcriber/gradium.mdx new file mode 100644 index 000000000..bd297b04e --- /dev/null +++ b/fern/customization/custom-transcriber/gradium.mdx @@ -0,0 +1,218 @@ +--- +title: Gradium +subtitle: Use Gradium speech-to-text as a custom transcriber in Vapi +description: Stream Vapi call audio to Gradium's realtime speech-to-text over WebSocket, and use Gradium's semantic VAD to decide when the caller has finished speaking. +slug: customization/custom-transcriber/gradium +--- + +[Gradium](https://gradium.ai) builds realtime audio language models. Gradium is not a built-in Vapi provider, so you connect it through a [custom transcriber](/customization/custom-transcriber): a small WebSocket bridge that receives audio from Vapi and forwards it to Gradium's realtime speech-to-text. + +Gradium's speech-to-text stream also emits a semantic voice activity detection (VAD) signal, which you can use as Vapi's end-of-turn authority instead of a silence timer. + +A Gradium account and API key are required. Install the SDK with `pip install gradium`. + +## How the bridge works + + + + Vapi connects to your endpoint and sends a JSON `start` frame describing the stream: + + ```json + { + "type": "start", + "encoding": "linear16", + "container": "raw", + "sampleRate": 16000, + "channels": 2 + } + ``` + + + Vapi sends interleaved 16-bit PCM. When `channels` is 2, channel 0 carries the customer and channel 1 carries the assistant. Forward **only channel 0** to Gradium, otherwise the agent transcribes its own speech. + + + Your bridge sends ~80 ms mono chunks to Gradium's realtime speech-to-text session. Gradium emits `text` events as words arrive and `step` events carrying VAD state. + + + Send partials as they arrive and a final when the turn ends: + + ```json + { + "type": "transcriber-response", + "transcription": "I'd like a large oat latte", + "channel": "customer", + "transcriptType": "final" + } + ``` + + Stream `"partial"` messages as well as the `"final"`. Vapi needs to see speech while it is happening for barge-in and turn tracking to work. + + + +## Detecting end of turn + +Gradium's `step` events include a `vad` horizon list, where each entry carries an `inactivity_prob`. Reading that probability is how you know the caller has stopped talking, rather than guessing from silence. + +Finalizing the moment the probability crosses your threshold truncates the last words of the sentence, because Gradium is still holding audio in its decoding window. Use the flush handshake instead: + + + + `inactivity_prob` stays above your threshold (0.8 works well) for a few consecutive steps. + + + Call `send_flush()` on the session and keep appending any `text` events that still arrive. + + + When Gradium returns `flushed`, emit the accumulated text as your `"final"` transcript. Keep a timeout fallback of about 2.5 seconds so a lost ack cannot stall the call. + + + +Gradium sessions are capped at roughly 300 seconds. Reconnect transparently when Gradium closes the session so longer calls continue uninterrupted. + +## Bridge implementation + +This FastAPI bridge forwards the customer channel to Gradium and returns transcripts to Vapi. + +```python title="transcriber.py" +import json +from fastapi import APIRouter, WebSocket, WebSocketDisconnect +from gradium.client import GradiumClient + +router = APIRouter() + +def customer_channel(pcm: bytes, channels: int) -> bytes: + """Keep channel 0 (the customer) from interleaved 16-bit PCM.""" + if channels == 1: + return pcm + stride = channels * 2 + return b"".join(pcm[i:i + 2] for i in range(0, len(pcm) - stride + 1, stride)) + +@router.websocket("/vapi/transcriber") +async def vapi_transcriber(ws: WebSocket) -> None: + await ws.accept() + client = GradiumClient(api_key=GRADIUM_API_KEY) + channels, stt = 2, None + + async def send(text: str, transcript_type: str) -> None: + await ws.send_text(json.dumps({ + "type": "transcriber-response", + "transcription": text, + "channel": "customer", + "transcriptType": transcript_type, + })) + + try: + while True: + message = await ws.receive() + if message.get("type") == "websocket.disconnect": + break + + if (payload := message.get("text")) is not None: + frame = json.loads(payload) + if frame.get("type") == "start": + channels = int(frame.get("channels", 2)) + rate = int(frame.get("sampleRate", 16000)) + # Open the Gradium session, then read text and VAD events + # in a background task that calls send(...) as they arrive. + stt = await open_gradium_session(client, rate, send) + + elif (audio := message.get("bytes")) is not None and stt is not None: + await stt.send_audio(customer_channel(audio, channels)) + except WebSocketDisconnect: + pass + finally: + if stt is not None: + await stt.close() +``` + +Open the Gradium session with the sample rate Vapi announced: + +```python title="gradium_session.py" +stt = client.stt_realtime( + model_name="default", + input_format="pcm_16000", # match the rate from the start frame + json_config={"language": "en", "delay_in_frames": 8}, +) +``` + +`delay_in_frames` trades latency for accuracy: each frame is 80 ms, and a larger value gives the model more context before it commits to text. + +## Configure your assistant + +Point `transcriber` at your bridge over `wss`: + +```bash +curl -X PATCH "https://api.vapi.ai/assistant/ASSISTANT_ID" \ + -H "Authorization: Bearer $VAPI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "transcriber": { + "provider": "custom-transcriber", + "server": { "url": "wss://your-server.com/vapi/transcriber" } + } + }' +``` + +## Let Gradium decide when the turn ends + +By default Vapi runs its own endpointing on your partial transcripts, which races Gradium's VAD and makes the assistant reply before the flushed final arrives. Hand the decision to Gradium with a custom endpointing model: + +```json +{ + "startSpeakingPlan": { + "waitSeconds": 0.25, + "smartEndpointingPlan": { + "provider": "custom-endpointing-model", + "server": { "url": "https://your-server.com/vapi/endpointing", "timeoutSeconds": 3 } + } + } +} +``` + +Vapi then sends a `call.endpointing.request` on every transcript update, and your server answers with how long to keep waiting, based on the live Gradium session: + +```python title="endpointing.py" +@router.post("/vapi/endpointing") +async def endpointing(body: dict) -> dict: + message = body.get("message") or {} + if message.get("type") != "call.endpointing.request": + return {"ok": True} + + state = gradium_turn_state() # your live STT session + if not state["active"]: + return {"timeoutSeconds": 1.0} # no session, do not stall the call + if state["seconds_since_final"] is not None and state["seconds_since_final"] < 2.5: + return {"timeoutSeconds": 0.05} # Gradium flushed a final, the turn is over + if state["inactivity_prob"] is not None and state["inactivity_prob"] < 0.5: + return {"timeoutSeconds": 5.0} # caller is audibly mid-sentence + return {"timeoutSeconds": 3.0} +``` + +For barge-in, keep `stopSpeakingPlan.numWords` at `0` so Vapi interrupts on audio VAD rather than waiting for transcript text: + +```json +{ + "stopSpeakingPlan": { "numWords": 0, "voiceSeconds": 0.2, "backoffSeconds": 0.5 } +} +``` + +## Troubleshooting + +| Symptom | Cause | Fix | +| --- | --- | --- | +| The agent transcribes its own speech | Both channels forwarded to Gradium | Send only channel 0 from the interleaved PCM | +| Final transcripts lose the last word or two | Finalizing on the VAD threshold alone | Finalize on the `flushed` ack, not the threshold crossing | +| The assistant replies before the caller finishes | Vapi's endpointing racing Gradium's VAD | Use the custom endpointing model above | +| Turns get swallowed, barge-in stops working | Only finals sent to Vapi | Stream `"partial"` transcripts as well | +| Transcription stops on long calls | Gradium's ~300 s session cap | Reconnect the Gradium session and keep Vapi's socket open | + +## Related + + + + Use Gradium voices as a custom voice. + + + The general custom transcriber protocol. + + diff --git a/fern/customization/custom-voices/gradium.mdx b/fern/customization/custom-voices/gradium.mdx new file mode 100644 index 000000000..33123c0bf --- /dev/null +++ b/fern/customization/custom-voices/gradium.mdx @@ -0,0 +1,169 @@ +--- +title: Gradium +subtitle: Use Gradium text-to-speech as a custom voice in Vapi +description: Serve Gradium voices to Vapi through a custom voice endpoint, including sample rates, connection pooling, and pronunciation dictionaries. +slug: customization/custom-voices/gradium +--- + +[Gradium](https://gradium.ai) builds realtime audio language models, including text-to-speech with instant voice cloning. Gradium is not a built-in Vapi provider, so you connect it through a [custom voice](/customization/custom-voices/custom-tts) endpoint: Vapi posts the text it wants spoken, and your server returns raw PCM audio. + +A Gradium account and API key are required. Install the SDK with `pip install gradium`. + +## How it works + + + + Vapi posts one request per sentence to your endpoint: + + ```json + { + "message": { + "type": "voice-request", + "text": "Your table is booked for seven o'clock.", + "sampleRate": 16000 + } + } + ``` + + + Stream the text to Gradium's realtime text-to-speech with `output_format` set to the PCM rate Vapi asked for. + + + Return 16-bit little-endian mono PCM as `application/octet-stream`, writing chunks as they arrive rather than buffering the whole utterance. Vapi plays the audio as it streams. + + + +## Sample rates + +Vapi requests one of `8000`, `16000`, `22050`, `24000`, or `44100` Hz. Gradium accepts `pcm_8000`, `pcm_16000`, `pcm_24000`, `pcm_44100`, and `pcm_48000`. + +Every rate matches except `22050`. If Vapi asks for a rate Gradium does not support, synthesize at the nearest supported rate and resample before responding, or configure your assistant to use a rate both sides share. + +## Voice endpoint implementation + +```python title="tts.py" +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import StreamingResponse +from gradium.client import GradiumClient + +router = APIRouter() +SUPPORTED_RATES = {8000, 16000, 24000, 44100, 48000} + +@router.post("/vapi/tts") +async def vapi_tts(request: Request) -> StreamingResponse: + body = await request.json() + message = body.get("message") or {} + if message.get("type") != "voice-request": + raise HTTPException(status_code=400, detail="expected message.type=voice-request") + + text = (message.get("text") or "").strip() + rate = message.get("sampleRate") + if not text or not isinstance(rate, int): + raise HTTPException(status_code=400, detail="missing text or sampleRate") + if rate not in SUPPORTED_RATES: + rate = 24000 # synthesize here, then resample to the requested rate + + async def pcm_stream(): + client = GradiumClient(api_key=GRADIUM_API_KEY) + async with client.tts_realtime( + model_name="default", + voice_id=GRADIUM_VOICE_ID, + output_format=f"pcm_{rate}", + ) as tts: + await tts.send_text(text) + await tts.send_eos() + async for msg in tts: + if msg["type"] == "audio": + yield msg["audio"] # SDK returns decoded PCM bytes + elif msg["type"] == "end_of_stream": + break + + return StreamingResponse(pcm_stream(), media_type="application/octet-stream") +``` + +## Configure your assistant + +```bash +curl -X PATCH "https://api.vapi.ai/assistant/ASSISTANT_ID" \ + -H "Authorization: Bearer $VAPI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "voice": { + "provider": "custom-voice", + "server": { "url": "https://your-server.com/vapi/tts", "timeoutSeconds": 45 } + } + }' +``` + +Vapi caches custom voice audio by default. While you are iterating on a voice or a pronunciation dictionary, set `"cachingEnabled": false` on the `voice` object so every request produces fresh audio. + +## Reduce latency with a pooled connection + +Vapi sends one request per sentence, so a naive implementation pays WebSocket setup on every sentence of every reply. Gradium supports multiplexing: keep one connection open per sample rate and route concurrent requests over it. + +Pass `close_ws_on_eos: false` in the setup message and stamp every message with a `client_req_id`. Gradium echoes that id on each response, so you can dispatch audio chunks back to the right request. + +```python title="pooled_setup.py" +setup = { + "model_name": "default", + "voice_id": GRADIUM_VOICE_ID, + "output_format": f"pcm_{rate}", + "close_ws_on_eos": False, # keep the socket open for reuse +} +stamp = {"client_req_id": "req-01"} +await stream.send_setup(setup | stamp) +await stream.send_text(text, **stamp) +await stream.send_eos(**stamp) +``` + +Recycle pooled connections before Gradium's ~300 second session cap, and fall back to a fresh single-use connection if a pooled socket fails before it produces audio. + +## Fix pronunciation of names and domain terms + +Voice agents often need to say brand names, menu items, or technical terms that an English model mispronounces. Create a [Gradium pronunciation dictionary](https://docs.gradium.ai) of phonetic respellings and reference it in the setup message: + +```python +setup = { + "voice_id": GRADIUM_VOICE_ID, + "output_format": f"pcm_{rate}", + "pronunciation_id": "bb1ckYhNHCcIJjdK", +} +``` + +Rules are plain text rewrites applied before synthesis, so `karaage` becomes `ka-ra-gay`. Use multi-word entries when a single word would collide with ordinary English, so rewrite `Pain Perdu` rather than `pain`. + +## Add a fallback voice + +If your endpoint is unreachable, Vapi can fall back to a built-in provider rather than dropping the call: + +```json +{ + "voice": { + "provider": "custom-voice", + "server": { "url": "https://your-server.com/vapi/tts" }, + "fallbackPlan": { + "voices": [{ "provider": "cartesia", "voiceId": "VOICE_ID" }] + } + } +} +``` + +## Troubleshooting + +| Symptom | Cause | Fix | +| --- | --- | --- | +| Audio plays at the wrong speed or pitch | Sample rate mismatch | Synthesize at the rate in `sampleRate`, or resample before responding | +| Long pause before the first word | New WebSocket per sentence | Pool one connection per sample rate with `close_ws_on_eos: false` | +| Voice edits do not take effect | Vapi's TTS cache | Set `"cachingEnabled": false` on the `voice` object | +| Speech cuts off mid-sentence | Response ended early | Stream until Gradium sends `end_of_stream`, and raise `timeoutSeconds` for long replies | + +## Related + + + + Use Gradium as a custom transcriber. + + + The general custom voice protocol. + + diff --git a/fern/docs.yml b/fern/docs.yml index d25ac50e4..7fea28018 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -287,9 +287,14 @@ navigation: path: customization/custom-voices/playht.mdx - page: Cartesia path: customization/custom-voices/cartesia.mdx - - page: Custom transcriber + - page: Gradium + path: customization/custom-voices/gradium.mdx + - section: Custom transcriber path: customization/custom-transcriber.mdx icon: fa-light fa-microphone + contents: + - page: Gradium + path: customization/custom-transcriber/gradium.mdx - page: Custom TTS path: customization/custom-tts.mdx icon: fa-light fa-volume-high From df2c4e6bcd6732245165f4eddd2f761d136413fa Mon Sep 17 00:00:00 2001 From: Pratim Date: Wed, 19 Aug 2026 16:02:01 +0200 Subject: [PATCH 2/5] Update gradium.mdx --- fern/customization/custom-voices/gradium.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fern/customization/custom-voices/gradium.mdx b/fern/customization/custom-voices/gradium.mdx index 33123c0bf..0a080dc89 100644 --- a/fern/customization/custom-voices/gradium.mdx +++ b/fern/customization/custom-voices/gradium.mdx @@ -5,7 +5,7 @@ description: Serve Gradium voices to Vapi through a custom voice endpoint, inclu slug: customization/custom-voices/gradium --- -[Gradium](https://gradium.ai) builds realtime audio language models, including text-to-speech with instant voice cloning. Gradium is not a built-in Vapi provider, so you connect it through a [custom voice](/customization/custom-voices/custom-tts) endpoint: Vapi posts the text it wants spoken, and your server returns raw PCM audio. +[Gradium](https://gradium.ai) builds real-time audio language models, including text-to-speech with instant voice cloning. You can connect to the Gradium Text-to-Speech API through a [custom voice](/customization/custom-voices/custom-tts) endpoint: Vapi posts the text it wants spoken, and your server returns raw PCM audio. A Gradium account and API key are required. Install the SDK with `pip install gradium`. @@ -26,7 +26,7 @@ slug: customization/custom-voices/gradium ``` - Stream the text to Gradium's realtime text-to-speech with `output_format` set to the PCM rate Vapi asked for. + Stream the text to Gradium's real-time text-to-speech with `output_format` set to the PCM rate Vapi asked for. Return 16-bit little-endian mono PCM as `application/octet-stream`, writing chunks as they arrive rather than buffering the whole utterance. Vapi plays the audio as it streams. @@ -130,7 +130,7 @@ setup = { } ``` -Rules are plain text rewrites applied before synthesis, so `karaage` becomes `ka-ra-gay`. Use multi-word entries when a single word would collide with ordinary English, so rewrite `Pain Perdu` rather than `pain`. +Rules are plain text rewrites applied before synthesis, so `karaage` the Japanese cooking style, becomes `ka-ra-gay`. Use multi-word entries when a single word would collide with ordinary English, so rewrite `Pain Perdu` rather than `pain`. ## Add a fallback voice From fdc8fac3454ed7db24f61cd18f419eb29aab2fd1 Mon Sep 17 00:00:00 2001 From: timpratim Date: Wed, 19 Aug 2026 16:24:08 +0200 Subject: [PATCH 3/5] docs(gradium): correct endpointing sample against the reference implementation --- .../custom-transcriber/gradium.mdx | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/fern/customization/custom-transcriber/gradium.mdx b/fern/customization/custom-transcriber/gradium.mdx index bd297b04e..3a98e0108 100644 --- a/fern/customization/custom-transcriber/gradium.mdx +++ b/fern/customization/custom-transcriber/gradium.mdx @@ -178,21 +178,32 @@ async def endpointing(body: dict) -> dict: if message.get("type") != "call.endpointing.request": return {"ok": True} - state = gradium_turn_state() # your live STT session - if not state["active"]: - return {"timeoutSeconds": 1.0} # no session, do not stall the call - if state["seconds_since_final"] is not None and state["seconds_since_final"] < 2.5: - return {"timeoutSeconds": 0.05} # Gradium flushed a final, the turn is over - if state["inactivity_prob"] is not None and state["inactivity_prob"] < 0.5: - return {"timeoutSeconds": 5.0} # caller is audibly mid-sentence - return {"timeoutSeconds": 3.0} + state = gradium_turn_state() # your live STT session + since_final = state.get("seconds_since_final") + inactivity = state.get("inactivity_prob") + + if not state.get("active"): + timeout = 1.0 # no live session, do not stall the call + elif since_final is not None and since_final < 2.5: + timeout = 0.05 # Gradium flushed a final, the turn is over + elif inactivity is not None and inactivity < 0.5: + timeout = 5.0 # caller is audibly mid-sentence + else: + timeout = 3.0 # a pause, the flush-final should arrive first + return {"timeoutSeconds": timeout} ``` -For barge-in, keep `stopSpeakingPlan.numWords` at `0` so Vapi interrupts on audio VAD rather than waiting for transcript text: +Extend the table with any state your bridge tracks. The agent this is drawn from adds one more branch: while it is holding a short acknowledgement like "yeah" to merge with the rest of the sentence, it answers 2 seconds so the caller can finish the thought. + +This endpoint does not have to be a separate route. Vapi posts to whatever URL you put in `smartEndpointingPlan.server`, so you can point it at the same server URL that already handles your tool calls and branch on `message.type`. + +Clear your reference to the Gradium session when a call ends. If a finished session stays registered, the first endpointing requests of the next call are answered from stale turn state before the new session takes over. + +For barge-in, keep `stopSpeakingPlan.numWords` at `0` so Vapi interrupts on audio VAD rather than waiting for transcript text. An empty `acknowledgementPhrases` list stops Vapi from treating answers that open with "yeah" or "okay" as backchannels and ignoring the turn: ```json { - "stopSpeakingPlan": { "numWords": 0, "voiceSeconds": 0.2, "backoffSeconds": 0.5 } + "stopSpeakingPlan": { "numWords": 0, "voiceSeconds": 0.2, "backoffSeconds": 0.5, "acknowledgementPhrases": [] } } ``` From 19b73b4445c3277bdee861b3bd03a1d2df0f24d9 Mon Sep 17 00:00:00 2001 From: timpratim Date: Wed, 19 Aug 2026 16:32:56 +0200 Subject: [PATCH 4/5] docs(gradium): frame both pages around what the integration supports --- .../custom-transcriber/gradium.mdx | 22 +++++++++---------- fern/customization/custom-voices/gradium.mdx | 10 ++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/fern/customization/custom-transcriber/gradium.mdx b/fern/customization/custom-transcriber/gradium.mdx index 3a98e0108..6859ab496 100644 --- a/fern/customization/custom-transcriber/gradium.mdx +++ b/fern/customization/custom-transcriber/gradium.mdx @@ -1,13 +1,13 @@ --- title: Gradium subtitle: Use Gradium speech-to-text as a custom transcriber in Vapi -description: Stream Vapi call audio to Gradium's realtime speech-to-text over WebSocket, and use Gradium's semantic VAD to decide when the caller has finished speaking. +description: Stream Vapi call audio to Gradium's real-time speech-to-text over WebSocket, and use Gradium's semantic VAD to decide when the caller has finished speaking. slug: customization/custom-transcriber/gradium --- -[Gradium](https://gradium.ai) builds realtime audio language models. Gradium is not a built-in Vapi provider, so you connect it through a [custom transcriber](/customization/custom-transcriber): a small WebSocket bridge that receives audio from Vapi and forwards it to Gradium's realtime speech-to-text. +[Gradium](https://gradium.ai) builds real-time audio language models. You can connect to the Gradium Speech-to-Text API through a [custom transcriber](/customization/custom-transcriber) endpoint: a small WebSocket bridge that receives audio from Vapi and forwards it to Gradium's real-time speech-to-text. -Gradium's speech-to-text stream also emits a semantic voice activity detection (VAD) signal, which you can use as Vapi's end-of-turn authority instead of a silence timer. +Gradium's speech-to-text stream also emits a semantic voice activity detection (VAD) signal, which you can use as Vapi's end-of-turn authority for turn detection that follows the meaning of what the caller is saying. A Gradium account and API key are required. Install the SDK with `pip install gradium`. @@ -28,10 +28,10 @@ Gradium's speech-to-text stream also emits a semantic voice activity detection ( ``` - Vapi sends interleaved 16-bit PCM. When `channels` is 2, channel 0 carries the customer and channel 1 carries the assistant. Forward **only channel 0** to Gradium, otherwise the agent transcribes its own speech. + Vapi sends interleaved 16-bit PCM. When `channels` is 2, channel 0 carries the customer and channel 1 carries the assistant. Forward **only channel 0** to Gradium so the agent transcribes the caller alone. - Your bridge sends ~80 ms mono chunks to Gradium's realtime speech-to-text session. Gradium emits `text` events as words arrive and `step` events carrying VAD state. + Your bridge sends ~80 ms mono chunks to Gradium's real-time speech-to-text session. Gradium emits `text` events as words arrive and `step` events carrying VAD state. Send partials as they arrive and a final when the turn ends: @@ -63,11 +63,11 @@ Finalizing the moment the probability crosses your threshold truncates the last Call `send_flush()` on the session and keep appending any `text` events that still arrive. - When Gradium returns `flushed`, emit the accumulated text as your `"final"` transcript. Keep a timeout fallback of about 2.5 seconds so a lost ack cannot stall the call. + When Gradium returns `flushed`, emit the accumulated text as your `"final"` transcript. Keep a timeout fallback of about 2.5 seconds so the call keeps moving if an ack is delayed. -Gradium sessions are capped at roughly 300 seconds. Reconnect transparently when Gradium closes the session so longer calls continue uninterrupted. +A Gradium session runs for up to about 300 seconds. Reconnect transparently when a session completes so calls of any length keep transcribing. ## Bridge implementation @@ -183,7 +183,7 @@ async def endpointing(body: dict) -> dict: inactivity = state.get("inactivity_prob") if not state.get("active"): - timeout = 1.0 # no live session, do not stall the call + timeout = 1.0 # no live session, keep the call moving elif since_final is not None and since_final < 2.5: timeout = 0.05 # Gradium flushed a final, the turn is over elif inactivity is not None and inactivity < 0.5: @@ -195,11 +195,11 @@ async def endpointing(body: dict) -> dict: Extend the table with any state your bridge tracks. The agent this is drawn from adds one more branch: while it is holding a short acknowledgement like "yeah" to merge with the rest of the sentence, it answers 2 seconds so the caller can finish the thought. -This endpoint does not have to be a separate route. Vapi posts to whatever URL you put in `smartEndpointingPlan.server`, so you can point it at the same server URL that already handles your tool calls and branch on `message.type`. +You can serve this from any route. Vapi posts to whatever URL you put in `smartEndpointingPlan.server`, so it can share the server URL that already handles your tool calls, branching on `message.type`. Clear your reference to the Gradium session when a call ends. If a finished session stays registered, the first endpointing requests of the next call are answered from stale turn state before the new session takes over. -For barge-in, keep `stopSpeakingPlan.numWords` at `0` so Vapi interrupts on audio VAD rather than waiting for transcript text. An empty `acknowledgementPhrases` list stops Vapi from treating answers that open with "yeah" or "okay" as backchannels and ignoring the turn: +For barge-in, keep `stopSpeakingPlan.numWords` at `0` so Vapi interrupts on audio VAD as soon as the caller speaks. An empty `acknowledgementPhrases` list keeps answers that open with "yeah" or "okay" as real turns: ```json { @@ -215,7 +215,7 @@ For barge-in, keep `stopSpeakingPlan.numWords` at `0` so Vapi interrupts on audi | Final transcripts lose the last word or two | Finalizing on the VAD threshold alone | Finalize on the `flushed` ack, not the threshold crossing | | The assistant replies before the caller finishes | Vapi's endpointing racing Gradium's VAD | Use the custom endpointing model above | | Turns get swallowed, barge-in stops working | Only finals sent to Vapi | Stream `"partial"` transcripts as well | -| Transcription stops on long calls | Gradium's ~300 s session cap | Reconnect the Gradium session and keep Vapi's socket open | +| Transcription stops on long calls | The ~300 s session length was reached | Reconnect the Gradium session and keep Vapi's socket open | ## Related diff --git a/fern/customization/custom-voices/gradium.mdx b/fern/customization/custom-voices/gradium.mdx index 0a080dc89..0ecefaee0 100644 --- a/fern/customization/custom-voices/gradium.mdx +++ b/fern/customization/custom-voices/gradium.mdx @@ -35,9 +35,9 @@ slug: customization/custom-voices/gradium ## Sample rates -Vapi requests one of `8000`, `16000`, `22050`, `24000`, or `44100` Hz. Gradium accepts `pcm_8000`, `pcm_16000`, `pcm_24000`, `pcm_44100`, and `pcm_48000`. +Gradium accepts `pcm_8000`, `pcm_16000`, `pcm_24000`, `pcm_44100`, and `pcm_48000`. Vapi requests one of `8000`, `16000`, `22050`, `24000`, or `44100` Hz, so `8000`, `16000`, `24000`, and `44100` map straight through: pass the requested rate to `output_format` and stream the result back. -Every rate matches except `22050`. If Vapi asks for a rate Gradium does not support, synthesize at the nearest supported rate and resample before responding, or configure your assistant to use a rate both sides share. +For `22050`, synthesize at the nearest Gradium rate and resample before responding, or set your assistant to one of the rates both sides share. ## Voice endpoint implementation @@ -116,7 +116,7 @@ await stream.send_text(text, **stamp) await stream.send_eos(**stamp) ``` -Recycle pooled connections before Gradium's ~300 second session cap, and fall back to a fresh single-use connection if a pooled socket fails before it produces audio. +Recycle pooled connections within Gradium's ~300 second session length, and open a fresh single-use connection if a pooled socket closes before it produces audio. ## Fix pronunciation of names and domain terms @@ -134,7 +134,7 @@ Rules are plain text rewrites applied before synthesis, so `karaage` the Japanes ## Add a fallback voice -If your endpoint is unreachable, Vapi can fall back to a built-in provider rather than dropping the call: +Vapi can fall back to a built-in provider so calls keep running while your endpoint recovers: ```json { @@ -154,7 +154,7 @@ If your endpoint is unreachable, Vapi can fall back to a built-in provider rathe | --- | --- | --- | | Audio plays at the wrong speed or pitch | Sample rate mismatch | Synthesize at the rate in `sampleRate`, or resample before responding | | Long pause before the first word | New WebSocket per sentence | Pool one connection per sample rate with `close_ws_on_eos: false` | -| Voice edits do not take effect | Vapi's TTS cache | Set `"cachingEnabled": false` on the `voice` object | +| Voice edits take effect on the next call | Vapi's TTS cache | Set `"cachingEnabled": false` on the `voice` object | | Speech cuts off mid-sentence | Response ended early | Stream until Gradium sends `end_of_stream`, and raise `timeoutSeconds` for long replies | ## Related From 7c0ba7f9bd81b2c03591fa495fa5fb463297671e Mon Sep 17 00:00:00 2001 From: timpratim Date: Wed, 19 Aug 2026 16:49:15 +0200 Subject: [PATCH 5/5] docs(gradium): every Vapi sample rate maps to a Gradium PCM format --- fern/customization/custom-voices/gradium.mdx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/fern/customization/custom-voices/gradium.mdx b/fern/customization/custom-voices/gradium.mdx index 0ecefaee0..e9cf6d00f 100644 --- a/fern/customization/custom-voices/gradium.mdx +++ b/fern/customization/custom-voices/gradium.mdx @@ -35,9 +35,13 @@ slug: customization/custom-voices/gradium ## Sample rates -Gradium accepts `pcm_8000`, `pcm_16000`, `pcm_24000`, `pcm_44100`, and `pcm_48000`. Vapi requests one of `8000`, `16000`, `22050`, `24000`, or `44100` Hz, so `8000`, `16000`, `24000`, and `44100` map straight through: pass the requested rate to `output_format` and stream the result back. +Vapi requests one of `8000`, `16000`, `22050`, `24000`, or `44100` Hz. Gradium's `output_format` accepts a matching PCM variant for every one of them, plus `pcm_48000`, so pass the requested rate straight through: -For `22050`, synthesize at the nearest Gradium rate and resample before responding, or set your assistant to one of the rates both sides share. +```python +output_format = f"pcm_{sample_rate}" +``` + +See [Gradium's limits page](https://docs.gradium.ai/guides/limits) for the full list of audio formats. ## Voice endpoint implementation @@ -47,7 +51,7 @@ from fastapi.responses import StreamingResponse from gradium.client import GradiumClient router = APIRouter() -SUPPORTED_RATES = {8000, 16000, 24000, 44100, 48000} +SUPPORTED_RATES = {8000, 16000, 22050, 24000, 44100, 48000} @router.post("/vapi/tts") async def vapi_tts(request: Request) -> StreamingResponse: