prewarm LLM to skip connection establishment

livekit-agents/livekit/agents/inference/llm.py

@@ -253,7 +253,11 @@ def __init__(
             ),
         )
 
+    async def _prewarm_impl(self) -> None:
+        await self._client.models.list()
+
     async def aclose(self) -> None:
+        await super().aclose()
         await self._client.close()
 
     @classmethod

livekit-agents/livekit/agents/llm/llm.py

@@ -102,6 +102,7 @@ class LLM(
     def __init__(self) -> None:
         super().__init__()
         self._label = f"{type(self).__module__}.{type(self).__name__}"
+        self._prewarm_task: asyncio.Task[None] | None = None
 
     @property
     def label(self) -> str:
@@ -143,11 +144,50 @@ def chat(
         extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
     ) -> LLMStream: ...
 
-    def prewarm(self) -> None:
-        """Pre-warm connection to the LLM service"""
-        pass
+    def prewarm(self, *, loop: asyncio.AbstractEventLoop | None = None) -> None:
+        """Pre-warm connection to the LLM service.
 
-    async def aclose(self) -> None: ...
+        Establishes DNS resolution and the TLS connection to the provider before the
+        first inference request, reducing time-to-first-token on the initial reply.
+        It is called automatically when an ``AgentSession`` is constructed and when an
+        agent activity starts, but can also be called directly.
+
+        Non-blocking (fire-and-forget) and idempotent. Providers enable it by
+        overriding ``_prewarm_impl``.
+
+        Args:
+            loop: Event loop to schedule the prewarm request on. Defaults to the
+                running event loop.
+        """
+        if type(self)._prewarm_impl is LLM._prewarm_impl:
+            return  # no provider-specific prewarm implemented
+
+        if self._prewarm_task is not None:
+            return
+
+        if loop is None:
+            loop = asyncio.get_event_loop()
+
+        async def _prewarm() -> None:
+            try:
+                await self._prewarm_impl()
+            except Exception:
+                pass
+
+        self._prewarm_task = loop.create_task(_prewarm())
+
+    async def _prewarm_impl(self) -> None:
+        """Provider-specific prewarm request, overriding it enables ``prewarm()``.
+
+        Implementations should perform a cheap, token-free request (e.g. listing
+        models) using the same client that serves chat requests, so DNS + TLS are
+        established and a keep-alive connection is left in the pool. Exceptions
+        are swallowed by ``prewarm()``; subclasses overriding ``aclose`` must call
+        ``await super().aclose()`` to cancel an in-flight prewarm."""
+
+    async def aclose(self) -> None:
+        if self._prewarm_task is not None:
+            await aio.cancel_and_wait(self._prewarm_task)
 
     async def __aenter__(self) -> LLM:
         return self

livekit-agents/livekit/agents/voice/agent_session.py

@@ -30,7 +30,7 @@
 from .. import cli, inference, llm, stt, tts, utils, vad
 from .._exceptions import APIError
 from ..job import get_job_context
-from ..llm import AgentHandoff, ChatContext, MetricsReport
+from ..llm import LLM, AgentHandoff, ChatContext, MetricsReport
 from ..llm.chat_context import Instructions
 from ..log import logger
 from ..metrics import AgentSessionUsage, ModelUsageCollector
@@ -477,6 +477,11 @@ def __init__(
         self._llm = llm or None
         self._tts = tts or None
 
+        # eagerly establish DNS/TLS to the LLM provider so the first inference
+        # request doesn't pay connection setup costs
+        if isinstance(self._llm, LLM):
+            self._llm.prewarm(loop=self._loop)
+
         self._keyterm_detector = KeytermDetector(
             static_keyterms=self._opts.stt_context_options["keyterms"],
             options=self._opts.stt_context_options["keyterm_detection"],

livekit-plugins/livekit-plugins-anthropic/livekit/plugins/anthropic/llm.py

@@ -135,6 +135,9 @@ def __init__(
             ),
         )
 
+    async def _prewarm_impl(self) -> None:
+        await self._client.models.list(limit=1)
+
     @property
     def model(self) -> str:
         return self._opts.model

livekit-plugins/livekit-plugins-google/livekit/plugins/google/llm.py

@@ -266,6 +266,10 @@ def __init__(
         # Store thought_signatures for Gemini 2.5+ multi-turn function calling
         self._thought_signatures: dict[str, bytes] = {}
 
+    async def _prewarm_impl(self) -> None:
+        # also fetches auth tokens ahead of time on vertexai
+        await self._client.aio.models.list(config={"page_size": 1})
+
     @property
     def model(self) -> str:
         return self._opts.model

livekit-plugins/livekit-plugins-mistralai/livekit/plugins/mistralai/llm.py

@@ -113,6 +113,9 @@ def __init__(
             raise ValueError("Mistral AI API key is required. Set MISTRAL_API_KEY or pass api_key")
         self._client = client or Mistral(api_key=mistral_api_key)
 
+    async def _prewarm_impl(self) -> None:
+        await self._client.models.list_async()
+
     @property
     def model(self) -> str:
         return self._opts.model

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/llm.py

@@ -175,7 +175,13 @@ def __init__(
             ),
         )
 
+    async def _prewarm_impl(self) -> None:
+        # token-free request supported by openai and openai-compatible servers
+        await self._client.models.list()
+
     async def aclose(self) -> None:
+        await super().aclose()
+
         if self._owns_client:
             await self._client.close()