From 4e8bd4745a7d50fcca93a5d71a606e275dce4931 Mon Sep 17 00:00:00 2001 From: Katrina Date: Thu, 4 Jun 2026 15:13:14 -0400 Subject: [PATCH 1/3] speed up script for regen metric signatures by removing imports we don't need --- scripts/regen_metric_signatures.py | 74 ++++++++++++++++++++++++++++- src/eva/assistant/agentic/system.py | 4 +- src/eva/metrics/processor.py | 2 +- 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/scripts/regen_metric_signatures.py b/scripts/regen_metric_signatures.py index 9c27e69e..dcc6721a 100644 --- a/scripts/regen_metric_signatures.py +++ b/scripts/regen_metric_signatures.py @@ -11,9 +11,81 @@ """ import json +import sys +from importlib.abc import Loader, MetaPathFinder +from importlib.machinery import ModuleSpec from pathlib import Path +from types import ModuleType +from typing import TypedDict -from eva.metrics.signatures import compute_all_metric_signatures + +def _stub_heavy_imports() -> None: + """Install a meta-path finder that stubs heavy packages before any eva imports. + + litellm and pipecat together account for ~4 s of import time, but + regen_metric_signatures only needs class definitions and source hashes — + it never calls into these libraries at runtime. + + Using a MetaPathFinder means every submodule import (no matter how deep) + is automatically intercepted and given a lightweight stub, with no manual + per-submodule registration required. + + The one exception is DeploymentTypedDict, which is used as a base class in + eva.models.config and must be an actual type, not a stub object. + """ + _STUB_PACKAGES = frozenset({"litellm"}) + + class _AutoStub(ModuleType): + """A module stub that satisfies arbitrary attribute and submodule access.""" + + def __getattr__(self, name: str) -> "_AutoStub": + child = _AutoStub(f"{self.__name__}.{name}") + object.__setattr__(self, name, child) + sys.modules[child.__name__] = child + return child + + def __call__(self, *args: object, **kwargs: object) -> "_AutoStub": + return self + + def __iter__(self): # type: ignore[override] + return iter([]) + + # Allow use in type union expressions at module level, e.g. `Router | None` + def __or__(self, other: object) -> object: + return object + + def __ror__(self, other: object) -> object: + return object + + class _StubLoader(Loader): + def create_module(self, spec: ModuleSpec) -> _AutoStub: + return _AutoStub(spec.name) + + def exec_module(self, module: ModuleType) -> None: + pass # _AutoStub handles everything via __getattr__ + + class _StubFinder(MetaPathFinder): + def find_spec(self, fullname: str, path: object, target: object = None) -> ModuleSpec | None: + if fullname.split(".")[0] in _STUB_PACKAGES: + return ModuleSpec(fullname, _StubLoader()) + return None + + sys.meta_path.insert(0, _StubFinder()) + + # DeploymentTypedDict is inherited by ModelDeployment in eva.models.config, + # so it must be a real class. Trigger the import so the stub is registered in + # sys.modules, then replace the attribute with a proper TypedDict. + import litellm.types.router # noqa: PLC0415, F401 + + class DeploymentTypedDict(TypedDict, total=False): + pass + + sys.modules["litellm.types.router"].DeploymentTypedDict = DeploymentTypedDict # type: ignore[attr-defined] + + +_stub_heavy_imports() + +from eva.metrics.signatures import compute_all_metric_signatures # noqa: E402 REPO_ROOT = Path(__file__).resolve().parent.parent FIXTURE_PATH = REPO_ROOT / "tests" / "fixtures" / "metric_signatures.json" diff --git a/src/eva/assistant/agentic/system.py b/src/eva/assistant/agentic/system.py index 7be55472..12575b45 100644 --- a/src/eva/assistant/agentic/system.py +++ b/src/eva/assistant/agentic/system.py @@ -17,6 +17,7 @@ ) from eva.assistant.tools.tool_executor import ToolExecutor from eva.models.agents import AgentConfig +from eva.utils.conversation_checks import LLM_GENERIC_ERROR_MESSAGE as GENERIC_ERROR from eva.utils.error_handler import categorize_error from eva.utils.log_processing import truncate_data_uris from eva.utils.logging import get_logger @@ -27,9 +28,6 @@ # Suppress LiteLLM's Pydantic serialization warnings (harmless internal warnings) warnings.filterwarnings("ignore", category=UserWarning, message=".*Pydantic serializer warnings.*") -# Response messages -GENERIC_ERROR = "I'm sorry, I encountered an error processing your request." - def _clean_tool_name(name: str) -> str: """Strip Harmony special tokens that leak into tool names due to a known vLLM bug. diff --git a/src/eva/metrics/processor.py b/src/eva/metrics/processor.py index aa4a34b9..2a6f83db 100644 --- a/src/eva/metrics/processor.py +++ b/src/eva/metrics/processor.py @@ -5,9 +5,9 @@ from dataclasses import dataclass, field from pathlib import Path -from eva.assistant.agentic.system import GENERIC_ERROR from eva.models.config import PipelineType from eva.models.results import ConversationResult +from eva.utils.conversation_checks import LLM_GENERIC_ERROR_MESSAGE as GENERIC_ERROR from eva.utils.log_processing import ( AnnotationLabel, aggregate_pipecat_logs_by_type, From ab807c336995e2b16ba6d89da275ae97752253b6 Mon Sep 17 00:00:00 2001 From: Katrina Date: Thu, 4 Jun 2026 15:19:38 -0400 Subject: [PATCH 2/3] update default for conversation timeout --- .env.example | 2 +- src/eva/models/config.py | 2 +- tests/unit/models/test_config_models.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index dd78e6ef..62b9f072 100644 --- a/.env.example +++ b/.env.example @@ -172,7 +172,7 @@ EVA_MODEL__LLM=gpt-5.2 #i Conversation timeout in seconds. #d int #r 30,10000,10 -#v EVA_CONVERSATION_TIMEOUT_SECONDS=360 +#v EVA_CONVERSATION_TIMEOUT_SECONDS=600 #i Maximum rerun attempts for failed records. #d int diff --git a/src/eva/models/config.py b/src/eva/models/config.py index 6752d54b..afaa7eb3 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -485,7 +485,7 @@ class ModelDeployment(DeploymentTypedDict): description="Maximum number of concurrent conversations", ) conversation_timeout_seconds: int = Field( - 360, + 600, ge=30, le=10000, description="Timeout for each conversation in seconds", diff --git a/tests/unit/models/test_config_models.py b/tests/unit/models/test_config_models.py index 0500240f..f60d3e64 100644 --- a/tests/unit/models/test_config_models.py +++ b/tests/unit/models/test_config_models.py @@ -93,7 +93,7 @@ def test_create_minimal_config(self): # run_id = timestamp + model suffix (e.g. "2024-01-15_14-30-45.123456_nova-2_gpt-5.2_sonic") assert config.run_id.endswith("nova-2_gpt-5.2_sonic") assert config.max_concurrent_conversations == 1 - assert config.conversation_timeout_seconds == 360 + assert config.conversation_timeout_seconds == 600 def test_create_full_config(self, temp_dir: Path): """Test creating a RunConfig with all options.""" @@ -519,7 +519,7 @@ def test_defaults(self): assert c.model.stt == "deepgram" assert c.model.tts == "cartesia" assert c.max_concurrent_conversations == 1 - assert c.conversation_timeout_seconds == 360 + assert c.conversation_timeout_seconds == 600 assert c.base_port == 10000 assert c.port_pool_size == 150 assert c.max_rerun_attempts == 3 From d0fe51cd35948ec5375abe5d1244b87e85c2899a Mon Sep 17 00:00:00 2001 From: Katrina Date: Thu, 4 Jun 2026 15:53:21 -0400 Subject: [PATCH 3/3] Revert "update default for conversation timeout" This reverts commit ab807c336995e2b16ba6d89da275ae97752253b6. --- .env.example | 2 +- src/eva/models/config.py | 2 +- tests/unit/models/test_config_models.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 62b9f072..dd78e6ef 100644 --- a/.env.example +++ b/.env.example @@ -172,7 +172,7 @@ EVA_MODEL__LLM=gpt-5.2 #i Conversation timeout in seconds. #d int #r 30,10000,10 -#v EVA_CONVERSATION_TIMEOUT_SECONDS=600 +#v EVA_CONVERSATION_TIMEOUT_SECONDS=360 #i Maximum rerun attempts for failed records. #d int diff --git a/src/eva/models/config.py b/src/eva/models/config.py index afaa7eb3..6752d54b 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -485,7 +485,7 @@ class ModelDeployment(DeploymentTypedDict): description="Maximum number of concurrent conversations", ) conversation_timeout_seconds: int = Field( - 600, + 360, ge=30, le=10000, description="Timeout for each conversation in seconds", diff --git a/tests/unit/models/test_config_models.py b/tests/unit/models/test_config_models.py index f60d3e64..0500240f 100644 --- a/tests/unit/models/test_config_models.py +++ b/tests/unit/models/test_config_models.py @@ -93,7 +93,7 @@ def test_create_minimal_config(self): # run_id = timestamp + model suffix (e.g. "2024-01-15_14-30-45.123456_nova-2_gpt-5.2_sonic") assert config.run_id.endswith("nova-2_gpt-5.2_sonic") assert config.max_concurrent_conversations == 1 - assert config.conversation_timeout_seconds == 600 + assert config.conversation_timeout_seconds == 360 def test_create_full_config(self, temp_dir: Path): """Test creating a RunConfig with all options.""" @@ -519,7 +519,7 @@ def test_defaults(self): assert c.model.stt == "deepgram" assert c.model.tts == "cartesia" assert c.max_concurrent_conversations == 1 - assert c.conversation_timeout_seconds == 600 + assert c.conversation_timeout_seconds == 360 assert c.base_port == 10000 assert c.port_pool_size == 150 assert c.max_rerun_attempts == 3