From 14b31ec222719881087ad2292f7ba372b81639e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Ctinalenguyen=E2=80=9D?= Date: Wed, 12 Aug 2026 01:34:28 -0400 Subject: [PATCH] Match Python starter defaults and enable expressive mode Port the Python starter's updated pipeline defaults to the Node starter: AssemblyAI Universal 3.5 Pro for STT (English), Fish Audio S2.1 Pro for TTS, and adaptive interruptions alongside the existing turn detector and preemptive generation. Enable expressive mode on the AgentSession, which injects the TTS provider's markup guide into the LLM prompt so the model emits inline delivery tags that Fish Audio renders and the transcript never shows. Requires @livekit/agents 1.6.3, which is now the dependency floor. --- README.md | 2 ++ package.json | 2 +- src/main.ts | 29 +++++++++++++++++++---------- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 3e56ef3..da97e44 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,10 @@ The starter project includes: - A simple voice AI assistant, ready for extension and customization - A voice AI pipeline built on [LiveKit Inference](https://docs.livekit.io/agents/models/inference), providing zero-configuration access to [models](https://docs.livekit.io/agents/models) from top labs - Uses the fast, open-weight Gemma 4 31B model, [hosted by LiveKit](https://docs.livekit.io/agents/models/llm/livekit/) and tuned for optimal performance in voice AI, as the default LLM + - Uses Fish Audio S2.1 Pro for TTS, which renders the inline delivery markup that expressive mode relies on - Supports more than 50 models from OpenAI, Cartesia, Deepgram, and other providers - Access to a wide range of other models, including [Realtime models](https://docs.livekit.io/agents/models/realtime), through extensive plugin ecosystem +- Expressive mode, enabled by default: the framework injects the TTS provider's markup guide into the LLM prompt, so the model emits inline delivery tags (emotion, pacing, non-verbal sounds) that the TTS renders and the transcript never shows - Eval suite based on the LiveKit Agents [testing & evaluation framework](https://docs.livekit.io/agents/start/testing) - [LiveKit Turn Detector](https://docs.livekit.io/agents/logic/turns/turn-detector/), an end-of-turn model that listens to the user's audio directly, combining semantic understanding with acoustic cues for state-of-the-art accuracy across 14 languages - [Background voice cancellation](https://docs.livekit.io/transport/media/noise-cancellation/) diff --git a/package.json b/package.json index 75b3539..5ee3f8d 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "vitest": "^4.1.4" }, "dependencies": { - "@livekit/agents": "^1.5.0", + "@livekit/agents": "^1.6.3", "@livekit/plugins-ai-coustics": "^0.2.14", "@livekit/rtc-node": "^0.13.31", "dotenv": "^17.4.1", diff --git a/src/main.ts b/src/main.ts index d2f0683..81e6d4c 100644 --- a/src/main.ts +++ b/src/main.ts @@ -11,32 +11,41 @@ dotenv.config({ path: '.env.local' }); export default defineAgent({ entry: async (ctx) => { - // Set up a voice AI pipeline using OpenAI, Cartesia, Deepgram, and the LiveKit turn detector + // Set up a voice AI pipeline using AssemblyAI, Fish Audio, and the LiveKit turn detector const session = new voice.AgentSession({ // Speech-to-text (STT) is your agent's ears, turning the user's speech into text that the LLM can understand // See all available models at https://docs.livekit.io/agents/models/stt/ stt: new inference.STT({ - model: 'deepgram/nova-3', - language: 'multi', + model: 'assemblyai/universal-3-5-pro', + language: 'en', }), // Text-to-speech (TTS) is your agent's voice, turning the LLM's text into speech that the user can hear // See all available models as well as voice selections at https://docs.livekit.io/agents/models/tts/ tts: new inference.TTS({ - model: 'cartesia/sonic-3', - voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc', + model: 'fishaudio/s2.1-pro', + voice: 'fa4c9eb3dccc4806b382b40d61c6b10a', }), - // Turn detection determines when the user is speaking and when the agent should respond. - // The LiveKit audio turn detector is a multimodal model that encodes the user's audio - // directly to predict end of turn. It's built into the SDK (no extra plugin) and - // AgentSession supplies the required VAD automatically. - // See more at https://docs.livekit.io/agents/logic/turns/turn-detector/ turnHandling: { + // Turn detection determines when the user is speaking and when the agent should respond. + // The LiveKit audio turn detector is a multimodal model that encodes the user's audio + // directly to predict end of turn. It's built into the SDK (no extra plugin) and + // AgentSession supplies the required VAD automatically. + // See more at https://docs.livekit.io/agents/logic/turns/turn-detector/ turnDetection: new inference.TurnDetector(), + // Adaptive interruptions use the turn detector to tell a real interruption from a + // backchannel like "mhm" or "right", so the agent keeps talking through the latter. + interruption: { mode: 'adaptive' }, // Allow the LLM to generate a response while waiting for the end of turn preemptiveGeneration: { enabled: true }, }, + + // Expressive mode injects the TTS provider's markup guide into the LLM prompt, so the model + // emits inline delivery tags (emotion, pacing, non-verbal sounds) that the TTS renders and + // the transcript never shows. Requires a TTS model that supports markup, such as the Fish + // Audio model above. + expressive: true, }); // Start the session, which initializes the voice pipeline and warms up the models