diff --git a/.env.example b/.env.example index 31093441..4e7dad0b 100644 --- a/.env.example +++ b/.env.example @@ -237,14 +237,25 @@ EVA_MODEL_LIST='[ #v EVA_MODEL__TURN_START_STRATEGY_PARAMS='{}' #i Turn stop strategy: when to consider the user has finished speaking. +#i krisp_viva_turn uses Krisp's VIVA SDK turn detection v3 (streaming, frame-by-frame). +#i It needs the proprietary krisp_audio SDK installed plus KRISP_VIVA_TURN_MODEL_PATH +#i (path to a .kef model file) and KRISP_VIVA_API_KEY set below. #d enum -#e turn_analyzer,speech_timeout,external +#e turn_analyzer,speech_timeout,krisp_viva_turn,external #v EVA_MODEL__TURN_STOP_STRATEGY=turn_analyzer #i Turn stop strategy parameters. For speech_timeout: {"user_speech_timeout": 0.8}. +#i For krisp_viva_turn: {"threshold": 0.5, "frame_duration_ms": 20} (model_path/api_key +#i can also be passed here to override the env vars). #d json_object #v EVA_MODEL__TURN_STOP_STRATEGY_PARAMS='{}' +#i Krisp VIVA SDK credentials (only used when TURN_STOP_STRATEGY=krisp_viva_turn). +#i The turn model ships in the Krisp VIVA "Turn-Taking models" download (krisp-viva-tt-models); +#i point at the krisp-viva-tp-v3.kef file. VAD is supplied by Silero, so no Krisp VAD model needed. +#v KRISP_VIVA_TURN_MODEL_PATH=/path/to/krisp-viva-tt-models/krisp-viva-tp-v3.kef +#v KRISP_VIVA_API_KEY=your_krisp_license_key + #i VAD (Voice Activity Detection) analyzer. #d enum #e silero,none diff --git a/.gitignore b/.gitignore index e9ef7669..bf3094b3 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,7 @@ scripts/extract_and_generate_leaderboard.py # Website data generation scripts (local only) website/scripts/ + +# Krisp VIVA SDK — licensed binaries, provisioned locally before Docker build (not public) +vendor/krisp/*.whl +vendor/krisp/*.kef diff --git a/Dockerfile b/Dockerfile index 8fe229a2..da5d4920 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,6 +71,28 @@ COPY --from=builder /opt/venv/lib/python3.11/site-packages/eva /opt/venv/lib/pyt COPY --from=builder /opt/venv/lib/python3.11/site-packages/eva-*.dist-info /opt/venv/lib/python3.11/site-packages/ COPY --from=builder /opt/venv/bin/eva /opt/venv/bin/eva +# Optionally bake in the licensed Krisp VIVA SDK + turn model for the +# krisp_viva_turn turn-stop strategy. The wheel/model are proprietary (not public, +# not on PyPI) so they are git-ignored and provisioned into vendor/krisp/ before the +# build — see vendor/krisp/README.md. If they are absent the build still succeeds and +# krisp_viva_turn is simply unavailable at runtime. KRISP_VIVA_API_KEY is NOT baked in; +# it is supplied as a runtime env var/secret. +# Install with uv into the venv explicitly: /opt/venv is a `uv venv` (no seeded pip), +# so a bare `pip install` would miss the venv. This mirrors the builder stage. +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +COPY vendor/krisp/ /tmp/krisp/ +RUN if ls /tmp/krisp/*.whl >/dev/null 2>&1; then \ + echo "Installing Krisp VIVA SDK..." && \ + uv pip install --python /opt/venv/bin/python --no-cache /tmp/krisp/*.whl && \ + mkdir -p /opt/krisp/models && \ + cp /tmp/krisp/*.kef /opt/krisp/models/ && \ + echo "Krisp VIVA SDK baked into image"; \ + else \ + echo "No Krisp SDK in vendor/krisp/ — krisp_viva_turn will be unavailable"; \ + fi && \ + rm -rf /tmp/krisp +ENV KRISP_VIVA_TURN_MODEL_PATH=/opt/krisp/models/krisp-viva-tp-v3.kef + # Copy application code COPY src/ ./src/ COPY scripts/ ./scripts/ diff --git a/scripts/docker_build.sh b/scripts/docker_build.sh new file mode 100755 index 00000000..5fe8bf11 --- /dev/null +++ b/scripts/docker_build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +IMAGE="${1:?Missing image tag. Usage: $0 [--push]}" +PUSH="${2:-}" +shift 2>/dev/null || true + +GIT_COMMIT_SHA=$(git rev-parse HEAD) +GIT_BRANCH=$(git branch --show-current) +GIT_DIRTY=$([[ -n $(git status --porcelain) ]] && echo true || echo false) +GIT_DIFF_HASH=$(git diff | shasum -a 256 | cut -c1-12) + +docker build \ + --platform linux/amd64 \ + --build-arg GIT_COMMIT_SHA="$GIT_COMMIT_SHA" \ + --build-arg GIT_BRANCH="$GIT_BRANCH" \ + --build-arg GIT_DIRTY="$GIT_DIRTY" \ + --build-arg GIT_DIFF_HASH="$GIT_DIFF_HASH" \ + -t "$IMAGE" \ + "$@" . + +if [[ "$PUSH" == "--push" ]]; then + docker push "$IMAGE" +fi diff --git a/src/eva/assistant/pipeline/turn_config.py b/src/eva/assistant/pipeline/turn_config.py index e4ec3083..7208f638 100644 --- a/src/eva/assistant/pipeline/turn_config.py +++ b/src/eva/assistant/pipeline/turn_config.py @@ -94,7 +94,7 @@ def create_turn_stop_strategy( """Create a user turn stop strategy from configuration. Args: - strategy_type: Strategy type ('speech_timeout', 'turn_analyzer', 'external') + strategy_type: Strategy type ('speech_timeout', 'turn_analyzer', 'krisp_viva_turn', 'external') strategy_params: Strategy-specific parameters smart_turn_stop_secs: stop_secs for SmartTurnParams (used with turn_analyzer strategy) @@ -117,11 +117,26 @@ def create_turn_stop_strategy( smart_params = SmartTurnParams(stop_secs=stop_secs) if stop_secs is not None else None turn_analyzer = LocalSmartTurnAnalyzerV3(params=smart_params) return TurnAnalyzerUserTurnStopStrategy(turn_analyzer=turn_analyzer, **params) + elif strategy_type_lower == "krisp_viva_turn": + # KrispVivaTurn uses Krisp's VIVA SDK turn detection v3 (Tt) API, processing audio + # frame-by-frame in real time with an external VAD flag. It depends on the proprietary + # krisp_audio SDK plus a .kef model file (KRISP_VIVA_TURN_MODEL_PATH) and license key + # (KRISP_VIVA_API_KEY), so import lazily to keep the SDK optional. + from pipecat.audio.turn.krisp_viva_turn import KrispTurnParams, KrispVivaTurn + + params = dict(strategy_params) + # Krisp analyzer tuning params; fall back to KrispTurnParams defaults when absent. + krisp_kwargs = {k: params.pop(k) for k in ("threshold", "frame_duration_ms") if k in params} + krisp_params = KrispTurnParams(**krisp_kwargs) if krisp_kwargs else None + # Constructor kwargs; model_path/api_key fall back to their env vars when omitted. + analyzer_kwargs = {k: params.pop(k) for k in ("model_path", "api_key", "sample_rate") if k in params} + krisp_analyzer = KrispVivaTurn(params=krisp_params, **analyzer_kwargs) + return TurnAnalyzerUserTurnStopStrategy(turn_analyzer=krisp_analyzer, **params) elif strategy_type_lower == "external": # ExternalUserTurnStopStrategy has no required parameters return ExternalUserTurnStopStrategy(**strategy_params) else: raise ValueError( f"Unsupported turn stop strategy: {strategy_type}. " - f"Supported types: 'speech_timeout', 'turn_analyzer', 'external'" + f"Supported types: 'speech_timeout', 'turn_analyzer', 'krisp_viva_turn', 'external'" ) diff --git a/src/eva/models/config.py b/src/eva/models/config.py index 8bd9130b..057ce240 100644 --- a/src/eva/models/config.py +++ b/src/eva/models/config.py @@ -168,8 +168,10 @@ class ModelConfig(BaseModel): turn_stop_strategy: str = Field( "turn_analyzer", description=( - "User turn stop strategy: 'speech_timeout', 'turn_analyzer', or 'external'. " + "User turn stop strategy: 'speech_timeout', 'turn_analyzer', 'krisp_viva_turn', or 'external'. " "Defaults to 'turn_analyzer' (TurnAnalyzerUserTurnStopStrategy with LocalSmartTurnAnalyzerV3). " + "'krisp_viva_turn' uses Krisp's VIVA SDK (requires the krisp_audio SDK, " + "KRISP_VIVA_TURN_MODEL_PATH (for local dev), and KRISP_VIVA_API_KEY). " "Set via EVA_MODEL__TURN_STOP_STRATEGY." ), ) diff --git a/tests/unit/assistant/test_turn_config.py b/tests/unit/assistant/test_turn_config.py index 72e135a6..a0b1370e 100644 --- a/tests/unit/assistant/test_turn_config.py +++ b/tests/unit/assistant/test_turn_config.py @@ -226,5 +226,71 @@ def test_unsupported_strategy_raises(self): def test_unsupported_strategy_error_lists_supported(self): """ValueError message lists supported strategies.""" - with pytest.raises(ValueError, match="speech_timeout.*turn_analyzer.*external"): + with pytest.raises(ValueError, match="speech_timeout.*turn_analyzer.*krisp_viva_turn.*external"): create_turn_stop_strategy("unknown", {}) + + +# --------------------------------------------------------------------------- +# create_turn_stop_strategy — krisp_viva_turn +# --------------------------------------------------------------------------- + + +class TestCreateTurnStopStrategyKrispViva: + """Tests for the 'krisp_viva_turn' branch of create_turn_stop_strategy. + + KrispVivaTurn depends on the proprietary ``krisp_audio`` SDK, which is not a + project dependency. The factory imports ``pipecat.audio.turn.krisp_viva_turn`` + lazily, so these tests inject a fake module into ``sys.modules``. + """ + + @staticmethod + def _install_fake_krisp(monkeypatch): + """Inject a fake pipecat.audio.turn.krisp_viva_turn module and return it.""" + import sys + + fake_module = MagicMock() + fake_module.KrispVivaTurn.return_value = MagicMock() + monkeypatch.setitem(sys.modules, "pipecat.audio.turn.krisp_viva_turn", fake_module) + return fake_module + + def test_krisp_viva_turn_strategy(self, monkeypatch): + """'krisp_viva_turn' returns a TurnAnalyzerUserTurnStopStrategy with a KrispVivaTurn.""" + fake_module = self._install_fake_krisp(monkeypatch) + + result = create_turn_stop_strategy("krisp_viva_turn", {}) + + assert isinstance(result, TurnAnalyzerUserTurnStopStrategy) + # No tuning params given -> KrispTurnParams left at its defaults (params=None). + fake_module.KrispVivaTurn.assert_called_once_with(params=None) + fake_module.KrispTurnParams.assert_not_called() + + def test_krisp_viva_turn_case_insensitive(self, monkeypatch): + """Strategy type is matched case-insensitively.""" + self._install_fake_krisp(monkeypatch) + assert isinstance(create_turn_stop_strategy("KRISP_VIVA_TURN", {}), TurnAnalyzerUserTurnStopStrategy) + + def test_krisp_viva_turn_tuning_params(self, monkeypatch): + """Tuning params threshold / frame_duration_ms are forwarded to KrispTurnParams.""" + fake_module = self._install_fake_krisp(monkeypatch) + + create_turn_stop_strategy("krisp_viva_turn", {"threshold": 0.7, "frame_duration_ms": 30}) + + fake_module.KrispTurnParams.assert_called_once_with(threshold=0.7, frame_duration_ms=30) + # The constructed params object is passed to the analyzer. + assert fake_module.KrispVivaTurn.call_args.kwargs["params"] is fake_module.KrispTurnParams.return_value + + def test_krisp_viva_turn_constructor_params(self, monkeypatch): + """model_path / api_key / sample_rate are forwarded to the KrispVivaTurn constructor.""" + fake_module = self._install_fake_krisp(monkeypatch) + + create_turn_stop_strategy( + "krisp_viva_turn", + {"model_path": "/models/turn.kef", "api_key": "secret", "sample_rate": 16000}, + ) + + call_kwargs = fake_module.KrispVivaTurn.call_args.kwargs + assert call_kwargs["model_path"] == "/models/turn.kef" + assert call_kwargs["api_key"] == "secret" + assert call_kwargs["sample_rate"] == 16000 + # None of these should leak into the strategy as tuning params. + fake_module.KrispTurnParams.assert_not_called() diff --git a/vendor/krisp/README.md b/vendor/krisp/README.md new file mode 100644 index 00000000..1cdfabab --- /dev/null +++ b/vendor/krisp/README.md @@ -0,0 +1,46 @@ +# Krisp VIVA SDK — build-time vendor files + +The `krisp_viva_turn` turn-stop strategy needs Krisp's **licensed** VIVA SDK, which +is **not** on PyPI and **not** public — it is downloaded per-account from the Krisp +developer portal. The binaries are therefore **git-ignored** (see `.gitignore`) and +must be placed here manually before building the Docker image. + +The image only publishes to our private Toolkit registry, so baking the licensed +files into the image is acceptable; committing them to git is not. + +## What to put here + +Download from https://developers.krisp.ai (log in with your Krisp account): + +- **VIVA UAR Python SDK** zip → unzip → copy the **Linux x86_64** wheel matching the + image's Python (3.11) into this directory: + `krisp_audio--cp311-cp311-linux_x86_64.whl` + (The image is `linux/amd64` — do **not** use the macOS wheel you may have installed + locally.) +- **Turn-Taking models** zip → unzip → copy the turn model here: + `krisp-viva-tp-v3.kef` + +Result: + +``` +vendor/krisp/ +├── README.md (tracked) +├── krisp_audio-1.10.0-cp311-cp311-linux_x86_64.whl (git-ignored) +└── krisp-viva-tp-v3.kef (git-ignored) +``` + +## How it's used + +The `Dockerfile` runtime stage installs the wheel and copies the `.kef` into the +image, setting `KRISP_VIVA_TURN_MODEL_PATH`. If this directory has no wheel, the build +still succeeds — Krisp is simply unavailable and `krisp_viva_turn` cannot be selected +at runtime. + +`KRISP_VIVA_API_KEY` is **not** baked in — it is supplied as a runtime env var/secret, +like the other provider API keys. + +## Runtime requirement + +The SDK validates its license against `sdkapi.krisp.ai` and reports usage to +`analytics.krisp.ai` at runtime. The container must have outbound HTTPS to both, or +license validation fails.