From 95fb7e74429a67d41a514eab63d1321542b9d925 Mon Sep 17 00:00:00 2001 From: joyathawale1628 Date: Mon, 29 Jun 2026 19:04:13 +0530 Subject: [PATCH 1/6] feat: stable PTT lifecycle, real-time typing, audio feedback, SendInput fallback PTT Lifecycle Rewrite: - Merged stop+commit into atomic stopAndCommit() to prevent race conditions - Added isCommittingRef to block new starts during commit wait - All stop paths (Hotkey, MicButton, SpaceBar, Widget, Tray) now commit text - Settings save uses commit=false to avoid unwanted typing Real-Time Typing: - Text committed on raw ASR events during recording (immediate feedback) - Text committed on processed LLM events (final version) - User sees text appear as they speak, not just on release Typing Compatibility: - Added SendInput with KEYEVENTF_UNICODE as fallback for clipboard+Ctrl+V - Works in terminals, Electron apps, and UWP apps that ignore Ctrl+V - Character-by-character Unicode typing with surrogate pair support UI Fixes: - Fixed duplicate React keys (live-1 collision) with session-scoped IDs - Lines cleared on each new PTT session start - Added audio feedback beeps on PTT press (880Hz) and release (440Hz) Build Fixes: - Fixed pre-existing TS errors (unlistenShortcut, loadHistory, totalWords) - Created tsconfig.app.json to exclude test files from build - Fixed pnpm build to only check app code Hotkey: - Default: Ctrl+Shift+Space (valid Tauri format) - Added Alt+K option to Settings - Hotkey re-registration on change without restart --- stt-ui/package.json | 2 +- stt-ui/src-tauri/src/lib.rs | 83 ++++++++++++++-- stt-ui/src/App.tsx | 126 ++++++++++++++++++------ stt-ui/src/api-tauri.ts | 12 ++- stt-ui/src/components/HistoryPage.tsx | 2 +- stt-ui/src/components/InsightsPage.tsx | 2 - stt-ui/src/components/SettingsPanel.tsx | 8 +- stt-ui/src/lib/audio.ts | 38 +++++++ stt-ui/tsconfig.app.json | 4 + stt/orchestrator.py | 19 +++- tests/test_sidecar_e2e.py | 1 + 11 files changed, 247 insertions(+), 50 deletions(-) create mode 100644 stt-ui/src/lib/audio.ts create mode 100644 stt-ui/tsconfig.app.json diff --git a/stt-ui/package.json b/stt-ui/package.json index 249b9ff..772b760 100644 --- a/stt-ui/package.json +++ b/stt-ui/package.json @@ -5,7 +5,7 @@ "type": "module", "scripts": { "dev": "vite", - "build": "tsc && vite build", + "build": "tsc --noEmit -p tsconfig.app.json && vite build", "preview": "vite preview", "tauri": "tauri", "test": "vitest run", diff --git a/stt-ui/src-tauri/src/lib.rs b/stt-ui/src-tauri/src/lib.rs index 0d96ea1..602cfeb 100644 --- a/stt-ui/src-tauri/src/lib.rs +++ b/stt-ui/src-tauri/src/lib.rs @@ -963,7 +963,30 @@ mod win32 { fn lstrcpyW(lpString1: *mut u16, lpString2: *const u16) -> *mut u16; } + // SendInput structures + #[repr(C)] + struct KeyBDInput { + wVk: u16, + wScan: u16, + dwFlags: u32, + time: u32, + dwExtraInfo: usize, + } + + #[repr(C)] + struct Input { + r#type: u32, + ki: KeyBDInput, + _pad: [u8; 8], // union padding + } + + extern "system" { + fn SendInput(cInputs: u32, pInputs: *const Input, cbSize: i32) -> u32; + } + + const INPUT_KEYBOARD: u32 = 1; const KEYEVENTF_KEYUP: u32 = 0x0002; + const KEYEVENTF_UNICODE: u32 = 0x0004; const VK_CONTROL: u8 = 0x11; const VK_V: u8 = 0x56; const CF_UNICODETEXT: u32 = 13; @@ -1034,6 +1057,52 @@ mod win32 { keybd_event(VK_CONTROL, 0, KEYEVENTF_KEYUP, 0); } } + + /// Type text character-by-character using SendInput with KEYEVENTF_UNICODE. + /// Works in apps that ignore clipboard+Ctrl+V ( terminals, Electron, some UWP apps). + pub fn send_text_unicode(text: &str) { + for ch in text.chars() { + let code = ch as u32; + let mut inputs = [ + Input { + r#type: INPUT_KEYBOARD, + ki: KeyBDInput { + wVk: 0, + wScan: code as u16, + dwFlags: KEYEVENTF_UNICODE, + time: 0, + dwExtraInfo: 0, + }, + _pad: [0; 8], + }, + Input { + r#type: INPUT_KEYBOARD, + ki: KeyBDInput { + wVk: 0, + wScan: code as u16, + dwFlags: KEYEVENTF_UNICODE | KEYEVENTF_KEYUP, + time: 0, + dwExtraInfo: 0, + }, + _pad: [0; 8], + }, + ]; + // Handle supplementary plane characters (surrogate pairs) + if code > 0xFFFF { + // Send as two UTF-16 surrogates + let hi = (0xD800 + ((code - 0x10000) >> 10)) as u16; + let lo = (0xDC00 + ((code - 0x10000) & 0x3FF)) as u16; + inputs[0].ki.wScan = hi; + inputs[1].ki.wScan = hi; + unsafe { SendInput(2, inputs.as_ptr(), std::mem::size_of::() as i32); } + inputs[0].ki.wScan = lo; + inputs[1].ki.wScan = lo; + unsafe { SendInput(2, inputs.as_ptr(), std::mem::size_of::() as i32); } + } else { + unsafe { SendInput(2, inputs.as_ptr(), std::mem::size_of::() as i32); } + } + } + } } #[cfg(not(target_os = "windows"))] @@ -1147,14 +1216,14 @@ fn type_text(text: String, restore_hwnd: Option) -> Result { win32::set_foreground_hwnd(hwnd); std::thread::sleep(std::time::Duration::from_millis(50)); } - // Set clipboard via Win32 API (no PowerShell overhead) - if !win32::set_clipboard(&text) { - return Err("Failed to set clipboard".into()); + // Try clipboard+Ctrl+V first (fast, preserves clipboard) + if win32::set_clipboard(&text) { + std::thread::sleep(std::time::Duration::from_millis(30)); + win32::send_ctrl_v(); + return Ok(true); } - // Small delay for clipboard to propagate - std::thread::sleep(std::time::Duration::from_millis(30)); - // Send Ctrl+V via keybd_event (runs in Tauri's GUI thread — has active message loop) - win32::send_ctrl_v(); + // Fallback: SendInput Unicode (works in apps that ignore Ctrl+V) + win32::send_text_unicode(&text); return Ok(true); } // Linux — clipboard approach (works regardless of which window has focus) diff --git a/stt-ui/src/App.tsx b/stt-ui/src/App.tsx index a438a92..c4aba2d 100644 --- a/stt-ui/src/App.tsx +++ b/stt-ui/src/App.tsx @@ -3,6 +3,7 @@ import { z } from "zod"; import type { STTApi, STTEvent } from "./api"; import { createTauriApi } from "./api-tauri"; import { createWebAudioApi } from "./api-web-audio"; +import { playStartBeep, playStopBeep } from "./lib/audio"; import { Mic, PlugZap, ShieldCheck, Mic2, Sparkles, Settings2, Activity, Terminal } from "lucide-react"; import OnboardingWizard from "./components/OnboardingWizard"; import MicButton from "./components/MicButton"; @@ -252,7 +253,7 @@ function FeedView({ hasMoreHistory: boolean; onFeedScroll: () => void; start: (overrideSettings?: RuntimeSettings, source?: string) => void; - stop: () => void; + stop: (source?: string) => void; copyLatest: () => void; copyLine: (line: TranscriptLine) => void; clearLines: () => void; @@ -267,7 +268,7 @@ function FeedView({ }) { const handleToggle = () => { if (connected) { - stop(); + stop("MicButton"); } else { const isTauri = typeof window !== "undefined" && "__TAURI_INTERNALS__" in window; if (isTauri) { @@ -787,6 +788,7 @@ function App() { const [resolvedModel, setResolvedModel] = useState<{ profile: string; model: string; backend: string; device: string } | null>(null); const pttHwndRef = useRef(null); // Target HWND captured on PTT press const pttTextRef = useRef(""); // Latest transcription text for PTT commit + const pttIdleResolveRef = useRef<(() => void) | null>(null); // Resolver for idle state after stop const [view, setView] = useState( localStorage.getItem("onboarding_completed") === "true" ? "main" : "onboarding" ); @@ -800,11 +802,13 @@ function App() { const runtimeRef = useRef(null); const nextLocalId = useRef(1); + const sessionCounter = useRef(0); const feedRef = useRef(null); const connectedRef = useRef(connected); const isStartingRef = useRef(false); + const isCommittingRef = useRef(false); const startRef = useRef<(overrideSettings?: RuntimeSettings, source?: string) => void>(() => {}); - const stopRef = useRef<() => void>(() => {}); + const stopRef = useRef<(source?: string, commit?: boolean) => void>(() => {}); const [settingsVersion, setSettingsVersion] = useState(0); const [hotkey, setHotkey] = useState(() => localStorage.getItem("stt-hotkey") || "CommandOrControl+Shift+Space"); @@ -828,10 +832,12 @@ function App() { : createTauriApi(buildCliArgs(settings)); api.onEvent(applyEvent); - runtimeRef.current = api; // Spawn backend — loads models, warms ASR, stays idle until PTT + // NOTE: runtimeRef is set ONLY after spawn resolves, so start() won't + // fire before the sidecar's stdin pipe is ready. api.spawn().then(() => { + runtimeRef.current = api; console.log("[Engine] Backend ready — waiting for PTT hotkey"); }).catch((err) => { const msg = err instanceof Error ? err.message : "Failed to start engine"; @@ -961,8 +967,9 @@ function App() { (target as HTMLInputElement).isContentEditable === true; if (e.code === "Space" && tag !== "INPUT" && tag !== "SELECT" && tag !== "TEXTAREA" && !isInteractive) { e.preventDefault(); - if (connectedRef.current) stopRef.current(); - else startRef.current(undefined, "SpaceBar"); + if (connectedRef.current || isCommittingRef.current) { + stopRef.current("SpaceBar"); + } else startRef.current(undefined, "SpaceBar"); } }; window.addEventListener("keydown", handler); @@ -987,6 +994,11 @@ function App() { const applyEvent = (event: STTEvent) => { if (event.type === "state") { setStatus(event.state); + // Resolve idle promise if waiting for PTT commit + if (event.state === "idle" && pttIdleResolveRef.current) { + pttIdleResolveRef.current(); + pttIdleResolveRef.current = null; + } return; } if (event.type === "mic") { @@ -1020,11 +1032,23 @@ function App() { return; } if (event.type === "raw") { - const id = event.utterance_id ?? nextLocalId.current++; + const rawId = event.utterance_id ?? nextLocalId.current++; + const id = sessionCounter.current * 100000 + rawId; setLines((prev) => [ ...prev, { id, raw: event.text, processed: "", status: "transcribing", createdAt: new Date().toISOString() }, ].slice(-500)); + // Real-time: commit ASR text to focused window during recording + pttTextRef.current = event.text; + if (connectedRef.current && pttHwndRef.current && event.text.trim()) { + const hwnd = pttHwndRef.current; + const text = event.text; + import("@tauri-apps/api/core").then(({ invoke }) => { + invoke("type_text", { text, restoreHwnd: hwnd }).then((ok) => { + console.log(`[PTT] Real-time commit (raw):`, ok); + }).catch(() => {}); + }).catch(() => {}); + } return; } if (event.type === "processed") { @@ -1034,6 +1058,16 @@ function App() { line.id === id ? { ...line, processed: event.text, status: "done" } : line )); pttTextRef.current = event.text; + // Real-time: commit LLM-rewritten text to focused window + if (connectedRef.current && pttHwndRef.current && event.text.trim()) { + const hwnd = pttHwndRef.current; + const text = event.text; + import("@tauri-apps/api/core").then(({ invoke }) => { + invoke("type_text", { text, restoreHwnd: hwnd }).then((ok) => { + console.log(`[PTT] Real-time commit (processed):`, ok); + }).catch(() => {}); + }).catch(() => {}); + } } if (event.type === "llm_partial") { const id = event.utterance_id; @@ -1051,8 +1085,8 @@ function App() { // --- PTT lifecycle: send commands to running backend --- const start = async (_overrideSettings?: RuntimeSettings, source: string = "Unknown") => { - if (connected || isStartingRef.current) { - console.log(`[PTT] Start rejected — already recording, source=${source}`); + if (connected || isStartingRef.current || isCommittingRef.current) { + console.log(`[PTT] Start rejected — busy, source=${source}`); return; } isStartingRef.current = true; @@ -1072,29 +1106,65 @@ function App() { pttHwndRef.current = null; } pttTextRef.current = ""; - console.log(`[PTT] Start requested — source=${source}`); + sessionCounter.current++; + nextLocalId.current = 1; + setLines([]); + console.log(`[PTT] Start requested — source=${source}, session=${sessionCounter.current}`); + playStartBeep(); runtimeRef.current.start(); // Sends start_recording to backend setConnected(true); setPttActive(true); dismissErrorsOfCategory("connection"); }; - const stop = async () => { + /** Atomic stop + wait-for-idle + type + cleanup. All callers use this. */ + const stopAndCommit = async (source: string = "Unknown", commit = true) => { if (!runtimeRef.current) return; + if (!connectedRef.current && !isStartingRef.current) return; isStartingRef.current = false; - // Backend handles typing directly — no need for frontend type_text - pttHwndRef.current = null; - pttTextRef.current = ""; - console.log("[PTT] Stop requested"); + isCommittingRef.current = true; + console.log(`[PTT] Stop requested — source=${source}`); + playStopBeep(); runtimeRef.current.stop(); // Sends stop_recording to backend setConnected(false); - setStatus("idle"); setPttActive(false); micLevelEmitter.emit(0); + + if (commit) { + // Wait for backend to finish transcriptions (state → idle) with timeout + await new Promise((resolve) => { + pttIdleResolveRef.current = resolve; + setTimeout(() => { resolve(); }, 10000); + }); + pttIdleResolveRef.current = null; + + // Read accumulated text and type it + const text = pttTextRef.current.trim(); + const hwnd = pttHwndRef.current; + if (text && hwnd) { + try { + const { invoke } = await import("@tauri-apps/api/core"); + const ok = await invoke("type_text", { text, restoreHwnd: hwnd }); + console.log(`[PTT] Committed text to HWND ${hwnd} (source=${source}):`, ok); + if (!ok) setToast("Failed to commit text"); + } catch (e) { + console.error("[PTT] type_text failed:", e); + setToast("Failed to commit text"); + } + } else { + console.log(`[PTT] Nothing to commit (source=${source}, text:`, !!text, "hwnd:", !!hwnd, ")"); + } + } + + // Clean up + pttTextRef.current = ""; + pttHwndRef.current = null; + setStatus("idle"); + isCommittingRef.current = false; }; startRef.current = start; - stopRef.current = stop; + stopRef.current = stopAndCommit; // --- Widget: emit status to widget window --- useEffect(() => { @@ -1114,8 +1184,9 @@ function App() { try { const { listen } = await import("@tauri-apps/api/event"); unlistenToggle = await listen("widget-toggle", () => { - if (connectedRef.current) stopRef.current(); - else startRef.current(undefined, "Widget"); + if (connectedRef.current || isCommittingRef.current) { + stopRef.current("Widget"); + } else startRef.current(undefined, "Widget"); }); unlistenShowMain = await listen("widget-show-main", async () => { try { @@ -1133,16 +1204,15 @@ function App() { useEffect(() => { let unlisten: (() => void) | undefined; - let unlistenShortcut: (() => void) | undefined; let registeredShortcut: string | null = null; (async () => { try { const { listen } = await import("@tauri-apps/api/event"); unlisten = await listen("tray-action", (event) => { - if (event.payload === "start" && !connectedRef.current) { + if (event.payload === "start" && !connectedRef.current && !isCommittingRef.current) { startRef.current(undefined, "Tray"); - } else if (event.payload === "stop" && connectedRef.current) { - stopRef.current(); + } else if (event.payload === "stop" && (connectedRef.current || isCommittingRef.current)) { + stopRef.current("Tray"); } }); } catch { /* not in Tauri */ } @@ -1164,14 +1234,11 @@ function App() { startRef.current(settings, "Hotkey"); } else if (event.state === "Released") { console.log("[PTT] Hotkey released — committing text"); - if (!connectedRef.current) { + if (!connectedRef.current && !isCommittingRef.current) { console.log("[PTT] Not recording — nothing to commit"); return; } - // Wait briefly for in-flight transcription to complete, then stop+commit - setTimeout(() => { - stopRef.current(); - }, 300); + stopRef.current("Hotkey"); } }); registeredShortcut = savedHotkey; @@ -1317,9 +1384,10 @@ function App() { setSettings(s); setSettingsVersion((v) => v + 1); // Trigger engine respawn with new CLI args if (connectedRef.current) { - stopRef.current(); + stopRef.current("SettingsSave", false); } }} + onHotkeyChange={(hk) => setHotkey(hk)} onClose={() => { setShowSettings(false); setActiveItem("Home"); diff --git a/stt-ui/src/api-tauri.ts b/stt-ui/src/api-tauri.ts index a94fa00..b7b821f 100644 --- a/stt-ui/src/api-tauri.ts +++ b/stt-ui/src/api-tauri.ts @@ -40,13 +40,17 @@ export function getWebAudioAnalyser(): AnalyserNode | null { export function createTauriApi(args: string[], sidecarName: string = "binaries/stt-engine"): STTApi { let listeners: Array<(e: STTEvent) => void> = []; let child: Child | null = null; + let lineBuffer = ""; // Buffer for partial JSON lines across chunk boundaries const notifyError = (msg: string) => { for (const cb of listeners) cb({ type: "error", message: msg }); }; const handleLine = (source: string, data: string) => { - const lines = data.split("\n"); + lineBuffer += data; + const lines = lineBuffer.split("\n"); + // Keep the last (potentially incomplete) chunk in the buffer + lineBuffer = lines.pop() ?? ""; for (const raw of lines) { const trimmed = raw.trim(); if (!trimmed) continue; @@ -102,11 +106,9 @@ export function createTauriApi(args: string[], sidecarName: string = "binaries/s sendCommand(cmd: Record) { if (child) { - try { - child.write(JSON.stringify(cmd) + "\n"); - } catch (e) { + child.write(JSON.stringify(cmd) + "\n").catch((e) => { console.warn("[Engine] Failed to write command:", e); - } + }); } }, }; diff --git a/stt-ui/src/components/HistoryPage.tsx b/stt-ui/src/components/HistoryPage.tsx index e909627..b666b77 100644 --- a/stt-ui/src/components/HistoryPage.tsx +++ b/stt-ui/src/components/HistoryPage.tsx @@ -254,7 +254,7 @@ export default function HistoryPage({ onBack }: Props) { - diff --git a/stt-ui/src/components/InsightsPage.tsx b/stt-ui/src/components/InsightsPage.tsx index f9bbc00..1fc47ef 100644 --- a/stt-ui/src/components/InsightsPage.tsx +++ b/stt-ui/src/components/InsightsPage.tsx @@ -61,7 +61,6 @@ export default function InsightsPage() { const [categories, setCategories] = useState([]); const [heatmap, setHeatmap] = useState([]); const [streak, setStreak] = useState({ current: 0, longest: 0 }); - const [totalWords, setTotalWords] = useState(0); const [weeklyWordsTotal, setWeeklyWordsTotal] = useState(0); const [wordsTrend, setWordsTrend] = useState(0); const [weeklyData, setWeeklyData] = useState<{ label: string; value: number }[]>([]); @@ -92,7 +91,6 @@ export default function InsightsPage() { }, []); const applyData = (data: InsightsData) => { - setTotalWords(data.totalWords || 0); setWeeklyWordsTotal(data.wordsThisWeek || 0); setWordsTrend(data.wordsTrend || 0); setCategories(data.categories || []); diff --git a/stt-ui/src/components/SettingsPanel.tsx b/stt-ui/src/components/SettingsPanel.tsx index 788f533..68e94ec 100644 --- a/stt-ui/src/components/SettingsPanel.tsx +++ b/stt-ui/src/components/SettingsPanel.tsx @@ -5,6 +5,7 @@ import type { RuntimeSettings } from "../App"; interface Props { settings: RuntimeSettings; onSave: (s: RuntimeSettings) => void; + onHotkeyChange?: (hotkey: string) => void; visible: boolean; onClose: () => void; } @@ -13,11 +14,12 @@ const HOTKEY_OPTIONS = [ { value: "CommandOrControl+Shift+Space", label: "Ctrl + Shift + Space" }, { value: "CommandOrControl+Alt+Space", label: "Ctrl + Alt + Space" }, { value: "Alt+Space", label: "Alt + Space" }, - { value: "Super+Space", label: "Super + Space" }, + { value: "Super+Space", label: "Win + Space" }, { value: "CommandOrControl+Shift+K", label: "Ctrl + Shift + K" }, + { value: "Alt+K", label: "Alt + K" }, ]; -export default function SettingsPanel({ settings, onSave, visible, onClose }: Props) { +export default function SettingsPanel({ settings, onSave, onHotkeyChange, visible, onClose }: Props) { const [local, setLocal] = useState({ ...settings }); const [showKeys, setShowKeys] = useState(false); const [hotkey, setHotkey] = useState(() => localStorage.getItem("stt-hotkey") || "CommandOrControl+Shift+Space"); @@ -215,7 +217,7 @@ export default function SettingsPanel({ settings, onSave, visible, onClose }: Pr "focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-accent/30", "disabled:pointer-events-none disabled:opacity-50", )} - onClick={() => { localStorage.setItem("stt-hotkey", hotkey); onSave(local); onClose(); }} + onClick={() => { localStorage.setItem("stt-hotkey", hotkey); onHotkeyChange?.(hotkey); onSave(local); onClose(); }} > Save & Apply diff --git a/stt-ui/src/lib/audio.ts b/stt-ui/src/lib/audio.ts new file mode 100644 index 0000000..d5b2251 --- /dev/null +++ b/stt-ui/src/lib/audio.ts @@ -0,0 +1,38 @@ +let ctx: AudioContext | null = null; + +function getCtx(): AudioContext { + if (!ctx) ctx = new AudioContext(); + return ctx; +} + +/** Short high-pitched beep for PTT start (press). */ +export function playStartBeep() { + try { + const ac = getCtx(); + const osc = ac.createOscillator(); + const gain = ac.createGain(); + osc.type = "sine"; + osc.frequency.value = 880; + gain.gain.setValueAtTime(0.15, ac.currentTime); + gain.gain.exponentialRampToValueAtTime(0.001, ac.currentTime + 0.08); + osc.connect(gain).connect(ac.destination); + osc.start(); + osc.stop(ac.currentTime + 0.08); + } catch { /* audio not available */ } +} + +/** Short low-pitched beep for PTT end (release). */ +export function playStopBeep() { + try { + const ac = getCtx(); + const osc = ac.createOscillator(); + const gain = ac.createGain(); + osc.type = "sine"; + osc.frequency.value = 440; + gain.gain.setValueAtTime(0.15, ac.currentTime); + gain.gain.exponentialRampToValueAtTime(0.001, ac.currentTime + 0.08); + osc.connect(gain).connect(ac.destination); + osc.start(); + osc.stop(ac.currentTime + 0.08); + } catch { /* audio not available */ } +} diff --git a/stt-ui/tsconfig.app.json b/stt-ui/tsconfig.app.json new file mode 100644 index 0000000..b30459b --- /dev/null +++ b/stt-ui/tsconfig.app.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["src/__tests__", "src/**/*.test.ts", "src/**/*.test.tsx"] +} diff --git a/stt/orchestrator.py b/stt/orchestrator.py index a067a08..7448de0 100644 --- a/stt/orchestrator.py +++ b/stt/orchestrator.py @@ -605,6 +605,9 @@ def _stop(_sig, _frame): _echo(msg) # --- PTT loop: wait for start → record → wait for stop → repeat --- + # Track transcription threads so we can wait for them before emitting idle. + _active_xcribe_threads: list[threading.Thread] = [] + while _stdin_alive or _is_tty: if not _is_tty: _start_event.wait() @@ -771,6 +774,7 @@ def _stop(_sig, _frame): args=(config, segment.copy(), sr, ring.total_samples() / sr, utterance_id, telemetry, hooks), daemon=True, ) + _active_xcribe_threads.append(thread) thread.start() except KeyboardInterrupt: @@ -778,6 +782,17 @@ def _stop(_sig, _frame): finally: pass # Don't close stream_iter — reuse across PTT sessions + # --- Wait for pending transcription threads before going idle --- + # This ensures pttTextRef has the latest transcription when the + # frontend reads it after receiving state:idle. + if _active_xcribe_threads: + _echo(f"Waiting for {_active_xcribe_threads.__len__()} pending transcription(s)...") + _deadline = time.monotonic() + 15.0 # max 15 s wait + for t in _active_xcribe_threads: + remaining = max(0.1, _deadline - time.monotonic()) + t.join(timeout=remaining) + _active_xcribe_threads.clear() + # Recording session ended — wait for next start or exit _stop_event.clear() _start_event.clear() @@ -1144,7 +1159,7 @@ def _on_partial(text: str) -> None: _json_emit(config, {"type": "processed", "text": raw, "utterance_id": utterance_id}) if hooks and hooks.on_processed: hooks.on_processed(raw) - if not hooks: + if not hooks and not config.json_mode: _output_text(raw, config) total_elapsed = time.monotonic() - ts_total telemetry.record("total", total_elapsed) @@ -1204,7 +1219,7 @@ def _on_partial(text: str) -> None: if hooks and hooks.on_processed: hooks.on_processed(processed) - if not hooks: + if not hooks and not config.json_mode: _output_text(processed, config) total_elapsed = time.monotonic() - ts_total telemetry.record("total", total_elapsed) diff --git a/tests/test_sidecar_e2e.py b/tests/test_sidecar_e2e.py index c790b37..c8f1ca0 100644 --- a/tests/test_sidecar_e2e.py +++ b/tests/test_sidecar_e2e.py @@ -279,6 +279,7 @@ def test_valid_hotkeys_have_main_key(self): "Alt+Space", "Super+Space", "CommandOrControl+Shift+K", + "Alt+K", ] for hotkey in valid_hotkeys: parts = hotkey.split("+") From 0777f13691b21f9fe52c4e89c05c54c09c3e04ce Mon Sep 17 00:00:00 2001 From: joyathawale1628 Date: Mon, 29 Jun 2026 19:09:00 +0530 Subject: [PATCH 2/6] chore: update uv.lock with new pytest dependencies (iniconfig, pluggy) --- uv.lock | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index c0f7326..018b3b7 100644 --- a/uv.lock +++ b/uv.lock @@ -751,6 +751,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1580,6 +1589,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, ] +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "pooch" version = "1.9.0" @@ -1766,6 +1784,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2042,6 +2076,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, ] +[[package]] +name = "ruff" +version = "0.15.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/dc/35b341fc554ba02f217fc10da57d1a75168cfbcf75b0ef2202176d4c4f2d/ruff-0.15.20.tar.gz", hash = "sha256:1416eb04349192646b54de98f146c4f59afe37d0decfc02c3cbbf396f3a28566", size = 4755489, upload-time = "2026-06-25T17:20:37.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/d9/2d5014f0253ba541d2061d9fa7193f48e941c8b21bb88a7ff9bbe0bd0596/ruff-0.15.20-py3-none-linux_armv6l.whl", hash = "sha256:00e188c53e499c3c1637f73c91dcf2fb56d576cab76ce1be50a27c4e80e37078", size = 10839665, upload-time = "2026-06-25T17:19:44.702Z" }, + { url = "https://files.pythonhosted.org/packages/c6/d3/ac1798ba64f670698867fcfc591d50e7e421bef137db564858f619a30fcf/ruff-0.15.20-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9ebd1fd9b9c95fc0bd7b2761aebec1f030013d2e193a2901b224af68fe47251b", size = 11208649, upload-time = "2026-06-25T17:19:48.787Z" }, + { url = "https://files.pythonhosted.org/packages/47/47/d3ac899991202095dfcf3d5176be4272642be3cf981a2f1a30f72a2afb95/ruff-0.15.20-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5b16cdd67ca108185cd36dce98c576350c03b1660a751de725fb049193a0632", size = 10622638, upload-time = "2026-06-25T17:19:51.354Z" }, + { url = "https://files.pythonhosted.org/packages/33/13/4e043fe30aa94d4ff5213a9881fc296d12960f5971b234a5263fdc225312/ruff-0.15.20-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3413bb3c3d2ca6a8208f1f4809cd2dca3c6de6d0b491c0e70847672bde6e6efd", size = 10984227, upload-time = "2026-06-25T17:19:54.044Z" }, + { url = "https://files.pythonhosted.org/packages/76/e6/92e7bf40388bc5800073b96564f56264f7e48bfd1a498f5ced6ae6d5a769/ruff-0.15.20-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd7ec42b3bb3da066488db093308a69c4ac5ee6d2af333a86ba6e2eb2e7dd44b", size = 10622882, upload-time = "2026-06-25T17:19:57.037Z" }, + { url = "https://files.pythonhosted.org/packages/13/7a/43460be3f24495a3aa46d4b16873e2c4941b3b5f0b00cf88c03b7b94b339/ruff-0.15.20-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1a36ad0eb77fba9aabfb69ede54de6f376d04ac18ebea022847046d340a8267", size = 11474808, upload-time = "2026-06-25T17:20:00.357Z" }, + { url = "https://files.pythonhosted.org/packages/27/a0/f37077884873221c6b33b4ab49eb18f9f88e54a16a25a5bca59bef46dd66/ruff-0.15.20-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b6df3b1e4610432f0386dba04d853b5f08cbbc903410c6fcc02f620f05aff53c", size = 12293094, upload-time = "2026-06-25T17:20:03.446Z" }, + { url = "https://files.pythonhosted.org/packages/a6/74/165545b60256a9704c21ac0ec4a0d07933b320812f9584836c9f4aca4292/ruff-0.15.20-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e89f198a1ea6ef0d727c1cf16088bc91a6cb0ab947dedc966715691647186eae", size = 11526176, upload-time = "2026-06-25T17:20:06.301Z" }, + { url = "https://files.pythonhosted.org/packages/86/b1/a976a136d40ade83ce743578399865f57001003a409acadc0ecbb3051082/ruff-0.15.20-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309809086c2acb67624950a3c8133e80f32d0d3e27106c0cd60ff26657c9f24b", size = 11520767, upload-time = "2026-06-25T17:20:09.191Z" }, + { url = "https://files.pythonhosted.org/packages/19/0f/f032696cb01c9b54c0263fa393474d7758f1cdc021a01b04e3cbc2500999/ruff-0.15.20-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2d2374caa2f2c2f9e2b7da0a50802cfb8b79f55a9b5e49379f564544fbf56487", size = 11500132, upload-time = "2026-06-25T17:20:13.602Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f4/51b1a14bc69e8c224b15dab9cce8e99b425e0455d462caa2b3c9be2b6a8e/ruff-0.15.20-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a1ed17b65293e0c2f22fc387bc13198a5de94bf4429589b0ff6946b0feaf21a3", size = 10943828, upload-time = "2026-06-25T17:20:16.635Z" }, + { url = "https://files.pythonhosted.org/packages/71/4b/fe267640783cd02bf6c5cc290b1df1051be2ec294c678b5c15fe19e52343/ruff-0.15.20-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f701305e66b38ea6c91882490eb73459796808e4c6362a1b765255e0cdcd4053", size = 10645418, upload-time = "2026-06-25T17:20:19.4Z" }, + { url = "https://files.pythonhosted.org/packages/b0/c0/a65aa4ec2f5e87a1df32dc3ec1fede434fe3dfd5cbcf3b503cafc676ab54/ruff-0.15.20-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5b9c0c367ad8e5d0d5b5b8537864c469a0a0e55417aadfbeca41fa61333be9f4", size = 11211770, upload-time = "2026-06-25T17:20:22.033Z" }, + { url = "https://files.pythonhosted.org/packages/5a/a4/0caa331d954ae2723d729d351c989cb4ca8b6077d5c6c2cb6de75e98c041/ruff-0.15.20-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:01cc00dd58f0df339d0e902219dd53990ea99996a0344e5d9cc8d45d5307e460", size = 11618698, upload-time = "2026-06-25T17:20:25.259Z" }, + { url = "https://files.pythonhosted.org/packages/10/9b/5f14927848d2fd4aa891fd88d883788c5a7baba561c7874732364045708c/ruff-0.15.20-py3-none-win32.whl", hash = "sha256:ed65ef510e43a137207e0f01cfcf998aeddb1aeeda5c9d35023e910284d7cf21", size = 10857322, upload-time = "2026-06-25T17:20:28.612Z" }, + { url = "https://files.pythonhosted.org/packages/fa/f0/fe47c501f9dea92a26d788ff98bb5d92ed4cb4c88792c5c88af6b697dc8e/ruff-0.15.20-py3-none-win_amd64.whl", hash = "sha256:a525c81c70fb0380344dd1d8745d8cc1c890b7fc94a58d5a07bd8eb9557b8415", size = 11993274, upload-time = "2026-06-25T17:20:31.871Z" }, + { url = "https://files.pythonhosted.org/packages/d7/2b/9555445e1201d92b3195f45cdb153a0b68f24e0a4273f6e3d5ab46e212bb/ruff-0.15.20-py3-none-win_arm64.whl", hash = "sha256:2f5b2a6d614e8700388806a14996c40fab2c47b819ef57d790a34878858ed9ca", size = 11343498, upload-time = "2026-06-25T17:20:35.03Z" }, +] + [[package]] name = "safetensors" version = "0.8.0" @@ -2362,6 +2421,8 @@ embeddings = [ [package.dev-dependencies] dev = [ { name = "pyinstaller" }, + { name = "pytest" }, + { name = "ruff" }, ] [package.metadata] @@ -2387,7 +2448,11 @@ requires-dist = [ provides-extras = ["llm", "clipboard", "embeddings", "diarization"] [package.metadata.requires-dev] -dev = [{ name = "pyinstaller", specifier = ">=6.20.0" }] +dev = [ + { name = "pyinstaller", specifier = ">=6.20.0" }, + { name = "pytest", specifier = ">=9.1.1" }, + { name = "ruff", specifier = ">=0.8.0" }, +] [[package]] name = "sympy" From 5bdeda272c65a2619068bf8300ad76df48807650 Mon Sep 17 00:00:00 2001 From: joyathawale1628 Date: Mon, 29 Jun 2026 19:11:53 +0530 Subject: [PATCH 3/6] chore: add .gitattributes to normalize line endings (fix Cargo.toml CRLF noise) --- .gitattributes | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1639419 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Auto-detect text files and normalize line endings to LF +* text=auto eol=lf + +# Windows-specific files keep CRLF +*.bat text eol=crlf +*.cmd text eol=crlf +*.ps1 text eol=crlf From 234229e56526bffc037cf2514f24b627fb67e7af Mon Sep 17 00:00:00 2001 From: joyathawale1628 Date: Tue, 30 Jun 2026 10:18:40 +0530 Subject: [PATCH 4/6] refactor: frontend-owned state machine, backend as pure execution engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Created commands.rs: begin_capture, end_capture, insert_text, show_overlay, hide_overlay - Removed Rust state machine (state.rs, typing.rs, overlay.rs) - Rewrote App.tsx with PttState: idle→listening→processing→inserting→success - Rewrote OverlayView.tsx to listen overlay:command events (pure rendering) - Added overlay window to tauri.conf.json (280x60, transparent, alwaysOnTop) - Added main.tsx routing for overlay window - Positioned overlay centered above taskbar via primary_monitor() - Backend now only executes commands, frontend owns all decisions --- stt-ui/src-tauri/Cargo.lock | 1 + stt-ui/src-tauri/Cargo.toml | 5 + stt-ui/src-tauri/src/commands.rs | 176 +++++++++++++++++++++ stt-ui/src-tauri/src/lib.rs | 81 +--------- stt-ui/src-tauri/tauri.conf.json | 15 ++ stt-ui/src/App.tsx | 201 +++++++++++++++--------- stt-ui/src/main.tsx | 20 ++- stt-ui/src/overlay/OverlayView.tsx | 241 +++++++++++++++++++++++++++++ stt-ui/src/overlay/spring.ts | 129 +++++++++++++++ stt-ui/src/overlay/waveform.ts | 198 ++++++++++++++++++++++++ 10 files changed, 914 insertions(+), 153 deletions(-) create mode 100644 stt-ui/src-tauri/src/commands.rs create mode 100644 stt-ui/src/overlay/OverlayView.tsx create mode 100644 stt-ui/src/overlay/spring.ts create mode 100644 stt-ui/src/overlay/waveform.ts diff --git a/stt-ui/src-tauri/Cargo.lock b/stt-ui/src-tauri/Cargo.lock index 4ab7ddb..5b3cbdd 100644 --- a/stt-ui/src-tauri/Cargo.lock +++ b/stt-ui/src-tauri/Cargo.lock @@ -4653,6 +4653,7 @@ dependencies = [ "tauri-plugin-updater", "tempfile", "thiserror 2.0.18", + "windows", ] [[package]] diff --git a/stt-ui/src-tauri/Cargo.toml b/stt-ui/src-tauri/Cargo.toml index 785e4ab..42aa3ca 100644 --- a/stt-ui/src-tauri/Cargo.toml +++ b/stt-ui/src-tauri/Cargo.toml @@ -33,6 +33,11 @@ rusqlite = { version = "0.31", features = ["bundled"] } csv = "1" dirs-next = "2" thiserror = "2" +[target.'cfg(target_os = "windows")'.dependencies] +windows = { version = "0.61", features = [ + "Win32_UI_WindowsAndMessaging", + "Win32_Foundation", +] } [dev-dependencies] tempfile = "3" diff --git a/stt-ui/src-tauri/src/commands.rs b/stt-ui/src-tauri/src/commands.rs new file mode 100644 index 0000000..8fc232f --- /dev/null +++ b/stt-ui/src-tauri/src/commands.rs @@ -0,0 +1,176 @@ +use tauri::{AppHandle, Emitter, Manager}; + +// --------------------------------------------------------------------------- +// Backend commands — the backend is a pure execution engine. +// It never decides when to start, stop, show, hide, or type. +// The frontend owns every decision. +// --------------------------------------------------------------------------- + +/// Tell the sidecar to begin capturing audio. +/// The frontend calls this only after transitioning to Listening. +#[tauri::command] +pub fn begin_capture(app: AppHandle) -> Result<(), String> { + // Emit to the main window so the STTApi can send "start_recording" to the sidecar + let _ = app.emit("backend:begin_capture", ()); + Ok(()) +} + +/// Tell the sidecar to stop capturing audio. +/// The frontend calls this when the user releases the hotkey. +#[tauri::command] +pub fn end_capture(app: AppHandle) -> Result<(), String> { + let _ = app.emit("backend:end_capture", ()); + Ok(()) +} + +/// Insert text into the focused window. +/// Uses Win32 clipboard + Ctrl+V (with SendInput Unicode fallback). +/// The frontend calls this after transcription completes. +#[tauri::command] +pub fn insert_text(text: String, restore_hwnd: Option) -> Result { + if text.trim().is_empty() { + return Ok(false); + } + let platform = std::env::consts::OS; + match platform { + "windows" => win32_insert(&text, restore_hwnd), + "linux" => linux_insert(&text, restore_hwnd), + "macos" => macos_insert(&text), + _ => false, + } + .pipe(Ok) +} + +/// Show the overlay window, positioned centered above the taskbar. +/// The frontend calls this when transitioning to Listening. +#[tauri::command] +pub fn show_overlay(app: AppHandle) -> Result<(), String> { + let win = app + .get_webview_window("overlay") + .ok_or_else(|| "Overlay window not found".to_string())?; + + // Position: centered horizontally, 80px above the bottom edge + if let Ok(Some(monitor)) = win.primary_monitor() { + let m_size = monitor.size(); + let m_pos = monitor.position(); + let pill_w = 280; + let pill_h = 60; + let margin_bottom = 80; + let x = (m_pos.x + (m_size.width as i32 - pill_w) / 2) as f64; + let y = (m_pos.y + m_size.height as i32 - pill_h - margin_bottom) as f64; + let _ = win.set_position(tauri::Position::Physical(tauri::PhysicalPosition { + x: x as i32, + y: y as i32, + })); + } + + // Show without stealing focus + #[cfg(target_os = "windows")] + { + use windows::Win32::UI::WindowsAndMessaging::ShowWindow; + use windows::Win32::UI::WindowsAndMessaging::SW_SHOWNOACTIVATE; + if let Some(hwnd) = win.hwnd().ok() { + let _ = unsafe { ShowWindow(hwnd, SW_SHOWNOACTIVATE) }; + } + } + #[cfg(not(target_os = "windows"))] + { + let _ = win.show(); + } + + Ok(()) +} + +/// Hide the overlay window. +/// The frontend calls this when transitioning back to Ready. +#[tauri::command] +pub fn hide_overlay(app: AppHandle) -> Result<(), String> { + let win = app + .get_webview_window("overlay") + .ok_or_else(|| "Overlay window not found".to_string())?; + let _ = win.hide(); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Platform-specific text insertion +// --------------------------------------------------------------------------- + +trait Pipe { + fn pipe(self, f: F) -> R + where + F: FnOnce(T) -> R; +} + +impl Pipe for T { + fn pipe(self, f: F) -> R + where + F: FnOnce(T) -> R, + { + f(self) + } +} + +#[cfg(target_os = "windows")] +fn win32_insert(text: &str, hwnd: Option) -> bool { + use crate::win32; + if let Some(h) = hwnd { + win32::set_foreground_hwnd(h); + std::thread::sleep(std::time::Duration::from_millis(50)); + } + if win32::set_clipboard(text) { + std::thread::sleep(std::time::Duration::from_millis(30)); + win32::send_ctrl_v(); + return true; + } + win32::send_text_unicode(text); + true +} + +#[cfg(not(target_os = "windows"))] +fn win32_insert(_text: &str, _hwnd: Option) -> bool { + false +} + +#[cfg(target_os = "linux")] +fn linux_insert(text: &str, hwnd: Option) -> bool { + use crate::win32; + if let Some(h) = hwnd { + if h != 0 { + win32::set_foreground_hwnd(h); + std::thread::sleep(std::time::Duration::from_millis(50)); + } + } + if !win32::set_clipboard(text) { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(30)); + let is_wayland = std::env::var("WAYLAND_DISPLAY").is_ok(); + if is_wayland { + let out = std::process::Command::new("wtype").arg(text).output(); + return out.map(|o| o.status.success()).unwrap_or(false); + } + win32::send_ctrl_v(); + true +} + +#[cfg(not(target_os = "linux"))] +fn linux_insert(_text: &str, _hwnd: Option) -> bool { + false +} + +#[cfg(target_os = "macos")] +fn macos_insert(text: &str) -> bool { + let escaped = text.replace('\\', "\\\\").replace('"', "\\\""); + let script = format!("tell application \"System Events\" to keystroke \"{escaped}\""); + std::process::Command::new("osascript") + .args(["-e", &script]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +#[cfg(not(target_os = "macos"))] +fn macos_insert(_text: &str) -> bool { + false +} diff --git a/stt-ui/src-tauri/src/lib.rs b/stt-ui/src-tauri/src/lib.rs index 602cfeb..393d029 100644 --- a/stt-ui/src-tauri/src/lib.rs +++ b/stt-ui/src-tauri/src/lib.rs @@ -1,4 +1,5 @@ mod widget; +mod commands; #[cfg(test)] mod tests; @@ -1200,80 +1201,6 @@ fn set_foreground_hwnd(hwnd: u64) -> bool { win32::set_foreground_hwnd(hwnd) } -/// Type text into the focused input using Win32 clipboard + Ctrl+V. -/// Pure Win32 API — no PowerShell needed for the critical path. -/// -/// Flow: restore previous window focus → set clipboard via Win32 → send Ctrl+V via keybd_event -#[tauri::command] -fn type_text(text: String, restore_hwnd: Option) -> Result { - if text.trim().is_empty() { - return Ok(false); - } - let platform = std::env::consts::OS; - if platform == "windows" { - // Restore focus to the previously-focused window FIRST - if let Some(hwnd) = restore_hwnd { - win32::set_foreground_hwnd(hwnd); - std::thread::sleep(std::time::Duration::from_millis(50)); - } - // Try clipboard+Ctrl+V first (fast, preserves clipboard) - if win32::set_clipboard(&text) { - std::thread::sleep(std::time::Duration::from_millis(30)); - win32::send_ctrl_v(); - return Ok(true); - } - // Fallback: SendInput Unicode (works in apps that ignore Ctrl+V) - win32::send_text_unicode(&text); - return Ok(true); - } - // Linux — clipboard approach (works regardless of which window has focus) - if platform == "linux" { - // Restore focus to the previously-captured window (X11 only — Wayland can't) - if let Some(hwnd) = restore_hwnd { - if hwnd != 0 { - win32::set_foreground_hwnd(hwnd); - std::thread::sleep(std::time::Duration::from_millis(50)); - } - } - // Set clipboard — wtype/xdotool will type from clipboard - if !win32::set_clipboard(&text) { - return Err("Failed to set clipboard on Linux".into()); - } - std::thread::sleep(std::time::Duration::from_millis(30)); - // On X11: send Ctrl+V to paste. On Wayland: wtype to paste from clipboard. - let is_wayland = std::env::var("WAYLAND_DISPLAY").is_ok(); - if is_wayland { - // wtype types directly into the focused Wayland window - let out = std::process::Command::new("wtype") - .arg(&text) - .output(); - if let Ok(o) = out { - if o.status.success() { - return Ok(true); - } else { - return Err("wtype failed to type text on Wayland".into()); - } - } else { - return Err("wtype command failed on Wayland".into()); - } - } - // X11: Ctrl+V via xdotool - win32::send_ctrl_v(); - return Ok(true); - } - if platform == "macos" { - let escaped = text.replace('\\', "\\\\").replace('"', "\\\""); - let script = format!("tell application \"System Events\" to keystroke \"{escaped}\""); - let out = std::process::Command::new("osascript") - .args(["-e", &script]) - .output(); - if let Ok(o) = out { - return Ok(o.status.success()); - } - } - Err("No typing backend available".into()) -} - #[tauri::command] fn get_backend_path() -> Result { let candidates = vec![ @@ -1487,9 +1414,13 @@ pub fn run() { toggle_dictionary_favorite, import_dictionary_csv, export_dictionary_csv, - type_text, get_foreground_hwnd, set_foreground_hwnd, + commands::begin_capture, + commands::end_capture, + commands::insert_text, + commands::show_overlay, + commands::hide_overlay, widget::show_widget, widget::hide_widget, widget::get_widget_visible, diff --git a/stt-ui/src-tauri/tauri.conf.json b/stt-ui/src-tauri/tauri.conf.json index 30c3794..2769d8c 100644 --- a/stt-ui/src-tauri/tauri.conf.json +++ b/stt-ui/src-tauri/tauri.conf.json @@ -35,6 +35,21 @@ "visible": false, "transparent": true, "shadow": false + }, + { + "label": "overlay", + "title": "", + "url": "/index.html?window=overlay", + "width": 280, + "height": 60, + "decorations": false, + "resizable": false, + "alwaysOnTop": true, + "skipTaskbar": true, + "visible": false, + "transparent": true, + "shadow": false, + "focusable": false } ], "security": { diff --git a/stt-ui/src/App.tsx b/stt-ui/src/App.tsx index c4aba2d..edbb504 100644 --- a/stt-ui/src/App.tsx +++ b/stt-ui/src/App.tsx @@ -8,7 +8,6 @@ import { Mic, PlugZap, ShieldCheck, Mic2, Sparkles, Settings2, Activity, Termina import OnboardingWizard from "./components/OnboardingWizard"; import MicButton from "./components/MicButton"; import MicPermissionModal from "./components/MicPermissionModal"; -import PttOverlay from "./components/PttOverlay"; import ModelBadge from "./components/ModelBadge"; import ErrorBanner from "./components/ErrorBanner"; import type { AppError } from "./components/ErrorBanner"; @@ -62,6 +61,9 @@ interface TranscriptLine { createdAt: string; } +/** Frontend-owned PTT state machine. The backend is a pure engine. */ +type PttState = "idle" | "listening" | "processing" | "inserting" | "success"; + export interface RuntimeSettings { wsPort: number; asrProfile: "auto" | "speed" | "balanced" | "accuracy" | "distil" | "turbo"; @@ -784,7 +786,6 @@ function App() { const [showErrors, setShowErrors] = useState(false); const [showMicModal, setShowMicModal] = useState(false); const [highlightPermissions, setHighlightPermissions] = useState(false); - const [pttActive, setPttActive] = useState(false); const [resolvedModel, setResolvedModel] = useState<{ profile: string; model: string; backend: string; device: string } | null>(null); const pttHwndRef = useRef(null); // Target HWND captured on PTT press const pttTextRef = useRef(""); // Latest transcription text for PTT commit @@ -805,7 +806,6 @@ function App() { const sessionCounter = useRef(0); const feedRef = useRef(null); const connectedRef = useRef(connected); - const isStartingRef = useRef(false); const isCommittingRef = useRef(false); const startRef = useRef<(overrideSettings?: RuntimeSettings, source?: string) => void>(() => {}); const stopRef = useRef<(source?: string, commit?: boolean) => void>(() => {}); @@ -967,9 +967,11 @@ function App() { (target as HTMLInputElement).isContentEditable === true; if (e.code === "Space" && tag !== "INPUT" && tag !== "SELECT" && tag !== "TEXTAREA" && !isInteractive) { e.preventDefault(); - if (connectedRef.current || isCommittingRef.current) { + if (pttStateRef.current === "listening") { stopRef.current("SpaceBar"); - } else startRef.current(undefined, "SpaceBar"); + } else if (pttStateRef.current === "idle") { + startRef.current(undefined, "SpaceBar"); + } } }; window.addEventListener("keydown", handler); @@ -1038,17 +1040,8 @@ function App() { ...prev, { id, raw: event.text, processed: "", status: "transcribing", createdAt: new Date().toISOString() }, ].slice(-500)); - // Real-time: commit ASR text to focused window during recording + // Store the latest text for PTT commit pttTextRef.current = event.text; - if (connectedRef.current && pttHwndRef.current && event.text.trim()) { - const hwnd = pttHwndRef.current; - const text = event.text; - import("@tauri-apps/api/core").then(({ invoke }) => { - invoke("type_text", { text, restoreHwnd: hwnd }).then((ok) => { - console.log(`[PTT] Real-time commit (raw):`, ok); - }).catch(() => {}); - }).catch(() => {}); - } return; } if (event.type === "processed") { @@ -1058,16 +1051,6 @@ function App() { line.id === id ? { ...line, processed: event.text, status: "done" } : line )); pttTextRef.current = event.text; - // Real-time: commit LLM-rewritten text to focused window - if (connectedRef.current && pttHwndRef.current && event.text.trim()) { - const hwnd = pttHwndRef.current; - const text = event.text; - import("@tauri-apps/api/core").then(({ invoke }) => { - invoke("type_text", { text, restoreHwnd: hwnd }).then((ok) => { - console.log(`[PTT] Real-time commit (processed):`, ok); - }).catch(() => {}); - }).catch(() => {}); - } } if (event.type === "llm_partial") { const id = event.utterance_id; @@ -1083,83 +1066,150 @@ function App() { setErrors((prev) => prev.map((e) => (e.category === category ? { ...e, dismissed: true } : e))); }; + // --- PTT state machine — frontend owns every decision --- + const [_pttState, setPttState] = useState("idle"); + const pttStateRef = useRef("idle"); + + const setPtt = (next: PttState) => { + pttStateRef.current = next; + setPttState(next); + }; + // --- PTT lifecycle: send commands to running backend --- const start = async (_overrideSettings?: RuntimeSettings, source: string = "Unknown") => { - if (connected || isStartingRef.current || isCommittingRef.current) { - console.log(`[PTT] Start rejected — busy, source=${source}`); + if (pttStateRef.current !== "idle") { + console.log(`[PTT] Start rejected — state=${pttStateRef.current}, source=${source}`); return; } - isStartingRef.current = true; if (!runtimeRef.current) { console.log(`[PTT] Start rejected — engine not ready, source=${source}`); - isStartingRef.current = false; setToast("Engine not ready — wait a moment and try again"); return; } - // Capture the foreground window BEFORE recording steals focus + + // Idle → Listening + setPtt("listening"); + console.log(`[PTT] Start — source=${source}`); + + // Tell overlay to show in listening mode + try { + const { emit } = await import("@tauri-apps/api/event"); + await emit("overlay:command", "listening" as const); + } catch { /* not in Tauri */ } + + // Capture foreground window try { const { invoke } = await import("@tauri-apps/api/core"); const hwnd = await invoke("get_foreground_hwnd"); pttHwndRef.current = hwnd; - console.log(`[PTT] Captured HWND: ${hwnd} (source=${source})`); } catch { pttHwndRef.current = null; } + pttTextRef.current = ""; sessionCounter.current++; nextLocalId.current = 1; setLines([]); - console.log(`[PTT] Start requested — source=${source}, session=${sessionCounter.current}`); playStartBeep(); - runtimeRef.current.start(); // Sends start_recording to backend + + // Show overlay (positioned above taskbar) + try { + const { invoke } = await import("@tauri-apps/api/core"); + await invoke("show_overlay"); + } catch { /* not in Tauri */ } + + // Tell backend to begin capturing + runtimeRef.current.start(); setConnected(true); - setPttActive(true); dismissErrorsOfCategory("connection"); }; - /** Atomic stop + wait-for-idle + type + cleanup. All callers use this. */ + /** Listening → Processing → (wait) → Inserting → Success → Idle */ const stopAndCommit = async (source: string = "Unknown", commit = true) => { + if (pttStateRef.current !== "listening") return; if (!runtimeRef.current) return; - if (!connectedRef.current && !isStartingRef.current) return; - isStartingRef.current = false; - isCommittingRef.current = true; - console.log(`[PTT] Stop requested — source=${source}`); + + // Listening → Processing + setPtt("processing"); + console.log(`[PTT] Stop — source=${source}`); playStopBeep(); - runtimeRef.current.stop(); // Sends stop_recording to backend + + // Tell overlay to show processing state + try { + const { emit } = await import("@tauri-apps/api/event"); + await emit("overlay:command", "processing" as const); + } catch { /* not in Tauri */ } + + // Tell backend to stop capturing + runtimeRef.current.stop(); setConnected(false); - setPttActive(false); micLevelEmitter.emit(0); - if (commit) { - // Wait for backend to finish transcriptions (state → idle) with timeout - await new Promise((resolve) => { - pttIdleResolveRef.current = resolve; - setTimeout(() => { resolve(); }, 10000); - }); - pttIdleResolveRef.current = null; - - // Read accumulated text and type it - const text = pttTextRef.current.trim(); - const hwnd = pttHwndRef.current; - if (text && hwnd) { - try { - const { invoke } = await import("@tauri-apps/api/core"); - const ok = await invoke("type_text", { text, restoreHwnd: hwnd }); - console.log(`[PTT] Committed text to HWND ${hwnd} (source=${source}):`, ok); - if (!ok) setToast("Failed to commit text"); - } catch (e) { - console.error("[PTT] type_text failed:", e); - setToast("Failed to commit text"); - } - } else { - console.log(`[PTT] Nothing to commit (source=${source}, text:`, !!text, "hwnd:", !!hwnd, ")"); + if (!commit) { + // No commit — go straight to Idle + setPtt("idle"); + setStatus("idle"); + try { + const { invoke } = await import("@tauri-apps/api/core"); + await invoke("hide_overlay"); + } catch { /* not in Tauri */ } + return; + } + + // Wait for backend to finish all transcriptions (state → idle) + await new Promise((resolve) => { + pttIdleResolveRef.current = resolve; + setTimeout(() => { resolve(); }, 10000); + }); + pttIdleResolveRef.current = null; + + // Read accumulated text + const text = pttTextRef.current.trim(); + const hwnd = pttHwndRef.current; + + if (text && hwnd) { + // Processing → Inserting + setPtt("inserting"); + // Tell overlay to show inserting state + try { + const { emit } = await import("@tauri-apps/api/event"); + await emit("overlay:command", "inserting" as const); + } catch { /* not in Tauri */ } + try { + const { invoke } = await import("@tauri-apps/api/core"); + const ok = await invoke("insert_text", { text, restoreHwnd: hwnd }); + console.log(`[PTT] Inserted text to HWND ${hwnd} (source=${source}):`, ok); + if (!ok) setToast("Failed to insert text"); + } catch (e) { + console.error("[PTT] insert_text failed:", e); + setToast("Failed to insert text"); } + } else { + console.log(`[PTT] Nothing to commit (source=${source})`); } + // Inserting → Success (brief confirmation) + setPtt("success"); + setStatus("idle"); + + // Tell overlay to show success state + try { + const { emit } = await import("@tauri-apps/api/event"); + await emit("overlay:command", "success" as const); + } catch { /* not in Tauri */ } + + // Success → Idle after 400ms + setTimeout(() => { + setPtt("idle"); + // Hide overlay + import("@tauri-apps/api/core").then(({ invoke }) => { + invoke("hide_overlay"); + }).catch(() => {}); + }, 400); + // Clean up pttTextRef.current = ""; pttHwndRef.current = null; - setStatus("idle"); isCommittingRef.current = false; }; @@ -1184,9 +1234,11 @@ function App() { try { const { listen } = await import("@tauri-apps/api/event"); unlistenToggle = await listen("widget-toggle", () => { - if (connectedRef.current || isCommittingRef.current) { + if (pttStateRef.current === "listening") { stopRef.current("Widget"); - } else startRef.current(undefined, "Widget"); + } else if (pttStateRef.current === "idle") { + startRef.current(undefined, "Widget"); + } }); unlistenShowMain = await listen("widget-show-main", async () => { try { @@ -1209,9 +1261,9 @@ function App() { try { const { listen } = await import("@tauri-apps/api/event"); unlisten = await listen("tray-action", (event) => { - if (event.payload === "start" && !connectedRef.current && !isCommittingRef.current) { + if (event.payload === "start" && pttStateRef.current === "idle") { startRef.current(undefined, "Tray"); - } else if (event.payload === "stop" && (connectedRef.current || isCommittingRef.current)) { + } else if (event.payload === "stop" && pttStateRef.current === "listening") { stopRef.current("Tray"); } }); @@ -1226,16 +1278,16 @@ function App() { const savedHotkey = localStorage.getItem("stt-hotkey") || "CommandOrControl+Shift+Space"; await register(savedHotkey, (event) => { if (event.state === "Pressed") { - if (connectedRef.current) { - console.log("[PTT] Ignored — already recording"); + if (pttStateRef.current !== "idle") { + console.log("[PTT] Ignored — not idle"); return; } console.log("[PTT] Hotkey pressed — starting recording"); startRef.current(settings, "Hotkey"); } else if (event.state === "Released") { console.log("[PTT] Hotkey released — committing text"); - if (!connectedRef.current && !isCommittingRef.current) { - console.log("[PTT] Not recording — nothing to commit"); + if (pttStateRef.current !== "listening") { + console.log("[PTT] Not listening — nothing to commit"); return; } stopRef.current("Hotkey"); @@ -1383,7 +1435,7 @@ function App() { onSave={async (s) => { setSettings(s); setSettingsVersion((v) => v + 1); // Trigger engine respawn with new CLI args - if (connectedRef.current) { + if (pttStateRef.current === "listening") { stopRef.current("SettingsSave", false); } }} @@ -1403,7 +1455,6 @@ function App() { }} onClose={() => setShowMicModal(false)} /> - ); } diff --git a/stt-ui/src/main.tsx b/stt-ui/src/main.tsx index cc2160a..ec85b30 100644 --- a/stt-ui/src/main.tsx +++ b/stt-ui/src/main.tsx @@ -2,18 +2,32 @@ import React from "react"; import ReactDOM from "react-dom/client"; import App from "./App"; import WidgetView from "./components/WidgetView"; +import OverlayView from "./overlay/OverlayView"; import "./styles/globals.css"; -const isWidget = new URLSearchParams(window.location.search).get("window") === "widget"; +const windowType = new URLSearchParams(window.location.search).get("window"); -if (isWidget) { +const isTransparent = windowType === "widget" || windowType === "overlay"; + +if (isTransparent) { document.documentElement.style.background = "transparent"; document.body.style.background = "transparent"; document.getElementById("root")!.style.background = "transparent"; } +function Root() { + switch (windowType) { + case "widget": + return ; + case "overlay": + return ; + default: + return ; + } +} + ReactDOM.createRoot(document.getElementById("root") as HTMLElement).render( - {isWidget ? : } + , ); diff --git a/stt-ui/src/overlay/OverlayView.tsx b/stt-ui/src/overlay/OverlayView.tsx new file mode 100644 index 0000000..17e6bfa --- /dev/null +++ b/stt-ui/src/overlay/OverlayView.tsx @@ -0,0 +1,241 @@ +import { useEffect, useRef, useState } from "react"; +import { Mic } from "lucide-react"; +import { createWaveform, type WaveformConfig } from "./waveform"; +import { createSpring, SPRING_SNAPPY } from "./spring"; + +type OverlayState = "idle" | "listening" | "processing" | "inserting" | "success"; + +const WAVEFORM_CONFIG: WaveformConfig = { + barCount: 14, + fftSize: 64, + riseSmoothing: 0.55, + fallSmoothing: 0.22, + barWidth: 3, + barGap: 2, + barRadius: 1.5, +}; + +function stateToLabel(state: OverlayState): string { + switch (state) { + case "idle": return ""; + case "listening": return "Listening..."; + case "processing": return "Transcribing..."; + case "inserting": return "Inserting..."; + case "success": return "Done"; + default: return ""; + } +} + +function stateToColor(state: OverlayState): string { + switch (state) { + case "success": return "#22c55e"; + case "processing": + case "inserting": return "#f59e0b"; + default: return "#FF3B56"; + } +} + +export default function OverlayView() { + const [state, setState] = useState("idle"); + + const svgRef = useRef(null); + const waveformRef = useRef | null>(null); + const rendererInitRef = useRef(false); + const scaleSpring = createSpring(SPRING_SNAPPY); + + // Listen for overlay commands from the frontend (main window) + useEffect(() => { + let unlisten: (() => void) | undefined; + + (async () => { + try { + const { listen } = await import("@tauri-apps/api/event"); + + unlisten = await listen("overlay:command", (event) => { + const newState = event.payload; + setState(newState); + + // Start/stop waveform based on listening state + const shouldCapture = newState === "listening"; + if (shouldCapture && waveformRef.current && !waveformRef.current.isActive()) { + waveformRef.current.start(); + } else if (!shouldCapture && waveformRef.current?.isActive()) { + waveformRef.current.stop(); + } + }); + } catch { + // Not in Tauri — show a static pill for development + setState("listening"); + } + })(); + + return () => { + unlisten?.(); + waveformRef.current?.stop(); + }; + }, []); + + // Initialize waveform and SVG renderer + useEffect(() => { + if (svgRef.current && !rendererInitRef.current) { + rendererInitRef.current = true; + const svg = svgRef.current; + while (svg.firstChild) svg.removeChild(svg.firstChild); + + const bars: SVGRectElement[] = []; + const { barCount, barWidth = 3, barGap = 2, barRadius = 1.5 } = WAVEFORM_CONFIG; + const totalWidth = barCount * (barWidth + barGap) - barGap; + const maxHeight = 24; + + for (let i = 0; i < barCount; i++) { + const rect = document.createElementNS("http://www.w3.org/2000/svg", "rect"); + rect.setAttribute("x", String(i * (barWidth + barGap))); + rect.setAttribute("width", String(barWidth)); + rect.setAttribute("rx", String(barRadius)); + rect.setAttribute("ry", String(barRadius)); + rect.setAttribute("fill", "currentColor"); + rect.setAttribute("y", String(maxHeight / 2 - 1)); + rect.setAttribute("height", "2"); + svg.appendChild(rect); + bars.push(rect); + } + svg.setAttribute("viewBox", `0 0 ${totalWidth} ${maxHeight}`); + + const wf = createWaveform(WAVEFORM_CONFIG); + waveformRef.current = wf; + + wf.onBars((values) => { + for (let i = 0; i < bars.length; i++) { + const v = values[i] ?? 0; + const h = Math.max(2, v * maxHeight); + const y = (maxHeight - h) / 2; + bars[i].setAttribute("y", String(y)); + bars[i].setAttribute("height", String(h)); + } + }); + } + }, []); + + // Spring animation for scale + useEffect(() => { + scaleSpring.onUpdate((v) => { + const el = document.getElementById("overlay-pill"); + if (el) { + el.style.transform = `scale(${0.92 + v * 0.08})`; + } + }); + scaleSpring.setValue(0); + }, []); + + useEffect(() => { + if (state !== "idle") { + scaleSpring.setTarget(1); + } else { + scaleSpring.setTarget(0); + } + }, [state]); + + if (state === "idle") return null; + + const color = stateToColor(state); + const label = stateToLabel(state); + const showWaveform = state === "listening"; + const showCheckmark = state === "success"; + const showTranscribing = state === "processing" || state === "inserting"; + + return ( +
+
+ {/* Mic icon with pulsing background */} +
+
+ {showCheckmark ? ( + + + + ) : ( + + )} +
+ + {/* Status label */} + + {label} + + + {/* Waveform bars */} + {showWaveform && ( + + )} + + {/* Transcribing dots animation */} + {showTranscribing && ( +
+ {[0, 1, 2].map((i) => ( +
+ ))} +
+ )} +
+ + +
+ ); +} diff --git a/stt-ui/src/overlay/spring.ts b/stt-ui/src/overlay/spring.ts new file mode 100644 index 0000000..f2988c3 --- /dev/null +++ b/stt-ui/src/overlay/spring.ts @@ -0,0 +1,129 @@ +/** + * Spring animation utility — creates natural, bouncy transitions. + * Based on the damped harmonic oscillator model used by Framer Motion. + * + * Usage: + * const spring = createSpring({ stiffness: 400, damping: 0.7 }); + * spring.setTarget(1.0); + * // In requestAnimationFrame loop: + * const value = spring.step(dt); + */ + +export interface SpringConfig { + stiffness: number; // Spring constant (higher = snappier) + damping: number; // Damping ratio (0 = no damping, 1 = critically damped) + mass?: number; // Mass (default 1) + precision?: number; // Stop threshold (default 0.001) +} + +interface SpringState { + value: number; + velocity: number; + target: number; +} + +export function createSpring(config: SpringConfig) { + const { stiffness, damping, mass = 1, precision = 0.001 } = config; + const state: SpringState = { value: 0, velocity: 0, target: 0 }; + let rafId: number | null = null; + let lastTime: number | null = null; + let onUpdate: ((value: number) => void) | null = null; + let onSettle: (() => void) | null = null; + + function step(dt: number) { + // Clamp dt to prevent huge jumps (e.g. when tab is backgrounded) + const clampedDt = Math.min(dt, 0.064); + + const displacement = state.value - state.target; + const springForce = -stiffness * displacement; + const dampingForce = -damping * 2 * Math.sqrt(stiffness) * state.velocity; + const acceleration = (springForce + dampingForce) / mass; + + state.velocity += acceleration * clampedDt; + state.value += state.velocity * clampedDt; + + // Check if settled + if ( + Math.abs(state.velocity) < precision && + Math.abs(state.value - state.target) < precision + ) { + state.value = state.target; + state.velocity = 0; + return true; // settled + } + return false; + } + + function tick(time: number) { + if (lastTime === null) { + lastTime = time; + rafId = requestAnimationFrame(tick); + return; + } + const dt = (time - lastTime) / 1000; + lastTime = time; + + const settled = step(dt); + onUpdate?.(state.value); + + if (settled) { + rafId = null; + lastTime = null; + onSettle?.(); + return; + } + rafId = requestAnimationFrame(tick); + } + + return { + /** Set the target value. Animation starts automatically. */ + setTarget(target: number) { + state.target = target; + if (rafId === null) { + lastTime = null; + rafId = requestAnimationFrame(tick); + } + }, + + /** Set the current value directly (no animation). */ + setValue(value: number) { + state.value = value; + state.velocity = 0; + onUpdate?.(value); + }, + + /** Get the current value. */ + getValue() { + return state.value; + }, + + /** Register update callback. */ + onUpdate(cb: (value: number) => void) { + onUpdate = cb; + }, + + /** Register settle callback (called when animation finishes). */ + onSettle(cb: () => void) { + onSettle = cb; + }, + + /** Stop animation immediately. */ + stop() { + if (rafId !== null) { + cancelAnimationFrame(rafId); + rafId = null; + lastTime = null; + } + }, + + /** Check if currently animating. */ + isAnimating() { + return rafId !== null; + }, + }; +} + +// Pre-configured spring presets +export const SPRING_SNAPPY = { stiffness: 400, damping: 0.7 }; +export const SPRING_SOFT = { stiffness: 200, damping: 0.8 }; +export const SPRING_BOUNCY = { stiffness: 300, damping: 0.5 }; diff --git a/stt-ui/src/overlay/waveform.ts b/stt-ui/src/overlay/waveform.ts new file mode 100644 index 0000000..d4bc075 --- /dev/null +++ b/stt-ui/src/overlay/waveform.ts @@ -0,0 +1,198 @@ +/** + * Real RMS waveform — drives SVG bars from live microphone audio via + * AudioContext + AnalyserNode + FFT + requestAnimationFrame. + * + * Asymmetric smoothing: rise=0.55 (fast), fall=0.22 (slow) makes the + * bars feel alive and responsive. + */ + +export interface WaveformConfig { + barCount: number; + fftSize?: number; // Default 64 (32 frequency bins) + minDecibels?: number; // Default -90 + maxDecibels?: number; // Default -10 + smoothingTimeConstant?: number; // Default 0.4 + riseSmoothing?: number; // Default 0.55 + fallSmoothing?: number; // Default 0.22 + barWidth?: number; // Default 3 + barGap?: number; // Default 2 + barRadius?: number; // Default 1.5 +} + +interface WaveformState { + audioContext: AudioContext | null; + analyser: AnalyserNode | null; + stream: MediaStream | null; + dataArray: Uint8Array | null; + smoothed: Float32Array | null; + rafId: number | null; + onBars: ((bars: number[]) => void) | null; + active: boolean; +} + +export function createWaveform(config: WaveformConfig) { + const { + barCount, + fftSize = 64, + minDecibels = -90, + maxDecibels = -10, + smoothingTimeConstant = 0.4, + riseSmoothing = 0.55, + fallSmoothing = 0.22, + } = config; + + const state: WaveformState = { + audioContext: null, + analyser: null, + stream: null, + dataArray: null, + smoothed: null, + rafId: null, + onBars: null, + active: false, + }; + + function computeBars(): number[] { + if (!state.analyser || !state.dataArray || !state.smoothed) { + return new Array(barCount).fill(0); + } + + state.analyser.getByteFrequencyData(state.dataArray); + + // Map FFT bins to bars (downsample if more bins than bars) + const binsPerBar = Math.floor(state.dataArray.length / barCount); + const bars: number[] = []; + + for (let i = 0; i < barCount; i++) { + // Average the frequency bins for this bar + let sum = 0; + const start = i * binsPerBar; + for (let j = start; j < start + binsPerBar && j < state.dataArray.length; j++) { + sum += state.dataArray[j]; + } + const raw = sum / binsPerBar / 255; // Normalize to 0-1 + + // Asymmetric smoothing — fast rise, slow fall + const prev = state.smoothed[i]; + const smoothing = raw > prev ? riseSmoothing : fallSmoothing; + const smoothed = prev + (raw - prev) * smoothing; + state.smoothed[i] = smoothed; + + bars.push(smoothed); + } + + return bars; + } + + function tick() { + if (!state.active) return; + const bars = computeBars(); + state.onBars?.(bars); + state.rafId = requestAnimationFrame(tick); + } + + return { + /** Start capturing audio and emitting bar values. */ + async start() { + if (state.active) return; + + try { + // Create AudioContext + state.audioContext = new AudioContext(); + state.analyser = state.audioContext.createAnalyser(); + state.analyser.fftSize = fftSize; + state.analyser.minDecibels = minDecibels; + state.analyser.maxDecibels = maxDecibels; + state.analyser.smoothingTimeConstant = smoothingTimeConstant; + + // Get microphone stream + state.stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + + // Connect: mic → analyser + const source = state.audioContext.createMediaStreamSource(state.stream); + source.connect(state.analyser); + + // Allocate buffers + state.dataArray = new Uint8Array(state.analyser.frequencyBinCount); + state.smoothed = new Float32Array(barCount); + + state.active = true; + state.rafId = requestAnimationFrame(tick); + } catch (err) { + console.error("[waveform] Failed to start:", err); + // Fallback: start fake waveform + state.active = true; + state.smoothed = new Float32Array(barCount); + state.rafId = requestAnimationFrame(tick); + } + }, + + /** Stop capturing audio. */ + stop() { + state.active = false; + if (state.rafId !== null) { + cancelAnimationFrame(state.rafId); + state.rafId = null; + } + if (state.stream) { + state.stream.getTracks().forEach((t) => t.stop()); + state.stream = null; + } + if (state.audioContext) { + state.audioContext.close().catch(() => {}); + state.audioContext = null; + } + state.analyser = null; + state.dataArray = null; + state.smoothed = null; + }, + + /** Set the callback that receives bar values. */ + onBars(cb: (bars: number[]) => void) { + state.onBars = cb; + }, + + /** Check if actively capturing. */ + isActive() { + return state.active; + }, + }; +} + +/** + * SVG bar renderer for the waveform. + * Returns a function that updates an SVG element's bar heights. + */ +export function createWaveformRenderer( + svgEl: SVGSVGElement, + config: WaveformConfig +) { + const { barCount, barWidth = 3, barGap = 2, barRadius = 1.5 } = config; + const totalWidth = barCount * (barWidth + barGap) - barGap; + const maxHeight = 24; + + // Create bar elements + const bars: SVGRectElement[] = []; + for (let i = 0; i < barCount; i++) { + const rect = document.createElementNS("http://www.w3.org/2000/svg", "rect"); + rect.setAttribute("x", String(i * (barWidth + barGap))); + rect.setAttribute("width", String(barWidth)); + rect.setAttribute("rx", String(barRadius)); + rect.setAttribute("ry", String(barRadius)); + rect.setAttribute("fill", "currentColor"); + svgEl.appendChild(rect); + bars.push(rect); + } + + svgEl.setAttribute("viewBox", `0 0 ${totalWidth} ${maxHeight}`); + + return function update(values: number[]) { + for (let i = 0; i < bars.length; i++) { + const v = values[i] ?? 0; + const h = Math.max(2, v * maxHeight); // Minimum height of 2px + const y = (maxHeight - h) / 2; + bars[i].setAttribute("y", String(y)); + bars[i].setAttribute("height", String(h)); + } + }; +} From 3bad50ee5ae6e69d58cfed574b88773627611482 Mon Sep 17 00:00:00 2001 From: joyathawale1628 Date: Sat, 18 Jul 2026 22:34:11 +0530 Subject: [PATCH 5/6] fix: PTT typing, stdout JSON channel, and idle model memory release - Stop writing raw LLM tokens to stdout in json_mode (corrupted the Tauri JSON event channel, dropped processed events, and emitted non-UTF-8 bytes causing 'invalid utf-8' sidecar errors) - Wait for in-flight transcription threads before emitting idle so the UI commits transcript text instead of 'Nothing to commit' - Release ASR model on idle (STT_IDLE_RELEASE_SEC, default 20s) to free several GB of RAM/VRAM; re-warm on next session. Add release_backend() - Gate few-shot embeddings behind STT_EMBEDDINGS=1 (was loading sentence-transformers on every utterance) - Redirect whisper.cpp subprocess stderr to devnull instead of capture_output=True to avoid buffering child output in RAM - Add stt-win11/ (1.8GB generated venv copy) to .gitignore --- .gitignore | 2 + stt-ui/src-tauri/capabilities/default.json | 2 +- stt-ui/src-tauri/src/commands.rs | 19 +- stt-ui/src-tauri/src/lib.rs | 2 - stt-ui/src/App.tsx | 271 +++++++++++-- stt-ui/src/api-tauri.ts | 55 ++- stt-ui/src/components/PttOverlay.tsx | 73 ---- stt-ui/src/components/WidgetView.tsx | 6 +- stt-ui/src/overlay/OverlayView.tsx | 59 ++- stt/orchestrator.py | 436 ++++++++++++++++----- stt/transcription.py | 39 ++ stt/vad.py | 53 ++- 12 files changed, 731 insertions(+), 286 deletions(-) delete mode 100644 stt-ui/src/components/PttOverlay.tsx diff --git a/.gitignore b/.gitignore index fa31f88..0b4036d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ Thumbs.db venv/ # Virtual environment .vscode/ +# Generated Windows build environment duplicate (1.8GB venv copy) +stt-win11/ # Ignore all node_modules folders everywhere in the project node_modules/ **/node_modules/ diff --git a/stt-ui/src-tauri/capabilities/default.json b/stt-ui/src-tauri/capabilities/default.json index 50a58c2..ba4e42b 100644 --- a/stt-ui/src-tauri/capabilities/default.json +++ b/stt-ui/src-tauri/capabilities/default.json @@ -2,7 +2,7 @@ "$schema": "../gen/schemas/desktop-schema.json", "identifier": "default", "description": "Capability for the main window", - "windows": ["main", "widget"], + "windows": ["main", "widget", "overlay"], "permissions": [ "core:default", "core:window:default", diff --git a/stt-ui/src-tauri/src/commands.rs b/stt-ui/src-tauri/src/commands.rs index 8fc232f..4da976a 100644 --- a/stt-ui/src-tauri/src/commands.rs +++ b/stt-ui/src-tauri/src/commands.rs @@ -1,4 +1,4 @@ -use tauri::{AppHandle, Emitter, Manager}; +use tauri::{AppHandle, Manager}; // --------------------------------------------------------------------------- // Backend commands — the backend is a pure execution engine. @@ -6,23 +6,6 @@ use tauri::{AppHandle, Emitter, Manager}; // The frontend owns every decision. // --------------------------------------------------------------------------- -/// Tell the sidecar to begin capturing audio. -/// The frontend calls this only after transitioning to Listening. -#[tauri::command] -pub fn begin_capture(app: AppHandle) -> Result<(), String> { - // Emit to the main window so the STTApi can send "start_recording" to the sidecar - let _ = app.emit("backend:begin_capture", ()); - Ok(()) -} - -/// Tell the sidecar to stop capturing audio. -/// The frontend calls this when the user releases the hotkey. -#[tauri::command] -pub fn end_capture(app: AppHandle) -> Result<(), String> { - let _ = app.emit("backend:end_capture", ()); - Ok(()) -} - /// Insert text into the focused window. /// Uses Win32 clipboard + Ctrl+V (with SendInput Unicode fallback). /// The frontend calls this after transcription completes. diff --git a/stt-ui/src-tauri/src/lib.rs b/stt-ui/src-tauri/src/lib.rs index 393d029..b2b6cce 100644 --- a/stt-ui/src-tauri/src/lib.rs +++ b/stt-ui/src-tauri/src/lib.rs @@ -1416,8 +1416,6 @@ pub fn run() { export_dictionary_csv, get_foreground_hwnd, set_foreground_hwnd, - commands::begin_capture, - commands::end_capture, commands::insert_text, commands::show_overlay, commands::hide_overlay, diff --git a/stt-ui/src/App.tsx b/stt-ui/src/App.tsx index edbb504..4581cf5 100644 --- a/stt-ui/src/App.tsx +++ b/stt-ui/src/App.tsx @@ -790,6 +790,11 @@ function App() { const pttHwndRef = useRef(null); // Target HWND captured on PTT press const pttTextRef = useRef(""); // Latest transcription text for PTT commit const pttIdleResolveRef = useRef<(() => void) | null>(null); // Resolver for idle state after stop + const overlayCleaningRef = useRef(false); // True while awaiting overlay cleanup + const overlayIdleResolveRef = useRef<(() => void) | null>(null); // Resolver for overlay:idle_ready + const hideOverlayTimeoutRef = useRef | null>(null); + const sessionIdRef = useRef(0); // Monotonic session counter for cleanup safety + const stopTimestampRef = useRef(0); // Timestamp when stop() was called (for timeout diagnostics) const [view, setView] = useState( localStorage.getItem("onboarding_completed") === "true" ? "main" : "onboarding" ); @@ -805,15 +810,12 @@ function App() { const nextLocalId = useRef(1); const sessionCounter = useRef(0); const feedRef = useRef(null); - const connectedRef = useRef(connected); - const isCommittingRef = useRef(false); const startRef = useRef<(overrideSettings?: RuntimeSettings, source?: string) => void>(() => {}); const stopRef = useRef<(source?: string, commit?: boolean) => void>(() => {}); const [settingsVersion, setSettingsVersion] = useState(0); const [hotkey, setHotkey] = useState(() => localStorage.getItem("stt-hotkey") || "CommandOrControl+Shift+Space"); - connectedRef.current = connected; const { permissions, requestClipboard, requestMic, isCapturingMic, stopMic } = usePermissions(); useEffect(() => { @@ -823,33 +825,69 @@ function App() { }, []); // --- Engine lifecycle: spawn once on mount, keep alive permanently --- - useEffect(() => { - // StrictMode guard: prevent double-spawn in development - if (runtimeRef.current) return; + // spawnedRef tracks the LATEST api instance created by this effect. In + // React StrictMode the effect fires twice (mount → cleanup → mount). + // The old code's guard (`if (runtimeRef.current) return`) failed because + // cleanup runs BEFORE the first spawn resolves, so `child` is null and + // `api.kill()` is a no-op — leaving two sidecar processes alive. + // + // Fix: track the latest API in spawnedRef. When a spawn resolves, check + // whether it is still the latest; if not, kill it immediately. Cleanup + // only kills the API if it is the current one (preventing a stale cleanup + // from killing a newer API). + const spawnedRef = useRef(null); + useEffect(() => { const api: STTApi = mode === "ws" ? createWebAudioApi(settings.wsPort) : createTauriApi(buildCliArgs(settings)); + // Register event listener before spawn so we never miss early events api.onEvent(applyEvent); + // Mark this as the latest API — any older API that resolves later is stale + spawnedRef.current = api; + // Spawn backend — loads models, warms ASR, stays idle until PTT // NOTE: runtimeRef is set ONLY after spawn resolves, so start() won't // fire before the sidecar's stdin pipe is ready. api.spawn().then(() => { + // If a newer API was created (StrictMode re-run or settings change), + // kill this stale instance immediately. + if (spawnedRef.current !== api) { + console.warn("[Engine] Stale sidecar killed — newer instance active"); + api.kill(); + return; + } + // Kill any previously-active sidecar (shouldn't happen with this + // guard, but defensive) + if (runtimeRef.current && runtimeRef.current !== api) { + runtimeRef.current.kill(); + } runtimeRef.current = api; console.log("[Engine] Backend ready — waiting for PTT hotkey"); }).catch((err) => { const msg = err instanceof Error ? err.message : "Failed to start engine"; setToast(msg); addError("connection", msg, true, "Check if stt-engine is installed"); - runtimeRef.current = null; + if (spawnedRef.current === api) { + runtimeRef.current = null; + } }); - // Cleanup: kill backend on app unmount + // Cleanup: kill backend on app unmount or when deps change return () => { - api.kill(); - runtimeRef.current = null; + // Clear spawnedRef if this is still the latest API + if (spawnedRef.current === api) { + spawnedRef.current = null; + } + // Kill only if this API is the active one + if (runtimeRef.current === api) { + api.kill(); + runtimeRef.current = null; + } + // If runtimeRef points to a DIFFERENT (older) API, leave it alone — + // its own spawn handler or cleanup will deal with it. }; }, [mode, settingsVersion]); // Re-spawn when mode changes OR settings saved @@ -995,9 +1033,11 @@ function App() { const applyEvent = (event: STTEvent) => { if (event.type === "state") { + console.log(`[PTT] Backend state event: ${event.state} (resolver=${pttIdleResolveRef.current !== null})`); setStatus(event.state); // Resolve idle promise if waiting for PTT commit if (event.state === "idle" && pttIdleResolveRef.current) { + console.log("[PTT] Resolving idle promise from backend event"); pttIdleResolveRef.current(); pttIdleResolveRef.current = null; } @@ -1008,8 +1048,8 @@ function App() { // Forward mic level to widget (async () => { try { - const { emit } = await import("@tauri-apps/api/event"); - await emit("widget-mic-level", event.level); + const { emitTo } = await import("@tauri-apps/api/event"); + await emitTo("widget", "widget-mic-level", event.level); } catch { /* not in Tauri */ } })(); return; @@ -1087,14 +1127,29 @@ function App() { return; } + // Cancel any pending cleanup from a previous session + if (hideOverlayTimeoutRef.current) { + clearTimeout(hideOverlayTimeoutRef.current); + hideOverlayTimeoutRef.current = null; + } + if (overlayIdleResolveRef.current) { + overlayIdleResolveRef.current = null; + } + overlayCleaningRef.current = false; + + // New session — increment counter so old cleanup timeouts are invalidated + sessionIdRef.current++; + const sessionId = sessionIdRef.current; + console.log(`[PTT] Start — source=${source} — session=${sessionId}`); + // Idle → Listening setPtt("listening"); - console.log(`[PTT] Start — source=${source}`); // Tell overlay to show in listening mode try { - const { emit } = await import("@tauri-apps/api/event"); - await emit("overlay:command", "listening" as const); + const { emitTo } = await import("@tauri-apps/api/event"); + await emitTo("overlay", "overlay:command", "listening" as const); + console.log("[Overlay] Command: listening"); } catch { /* not in Tauri */ } // Capture foreground window @@ -1116,6 +1171,7 @@ function App() { try { const { invoke } = await import("@tauri-apps/api/core"); await invoke("show_overlay"); + console.log("[Overlay] Shown"); } catch { /* not in Tauri */ } // Tell backend to begin capturing @@ -1129,38 +1185,108 @@ function App() { if (pttStateRef.current !== "listening") return; if (!runtimeRef.current) return; + // Capture session ID — used to invalidate old cleanup timeouts + const sessionId = sessionIdRef.current; + // Listening → Processing setPtt("processing"); - console.log(`[PTT] Stop — source=${source}`); + console.log(`[PTT] Stop — source=${source} — session=${sessionId}`); playStopBeep(); // Tell overlay to show processing state try { - const { emit } = await import("@tauri-apps/api/event"); - await emit("overlay:command", "processing" as const); + const { emitTo } = await import("@tauri-apps/api/event"); + await emitTo("overlay", "overlay:command", "processing" as const); } catch { /* not in Tauri */ } - // Tell backend to stop capturing - runtimeRef.current.stop(); - setConnected(false); - micLevelEmitter.emit(0); - if (!commit) { - // No commit — go straight to Idle - setPtt("idle"); - setStatus("idle"); + // No commit — stop backend and clean up overlay + console.log("[PTT] No commit — cleaning up overlay"); + runtimeRef.current.stop(); + setConnected(false); + micLevelEmitter.emit(0); + + // Verify session is still active before cleaning up + if (sessionIdRef.current !== sessionId) { + console.log(`[PTT] No-commit cleanup aborted — session ${sessionId} superseded by ${sessionIdRef.current}`); + return; + } + + // Tell overlay to clean up, wait for acknowledgement, then hide + overlayCleaningRef.current = true; + try { + const { emitTo } = await import("@tauri-apps/api/event"); + await emitTo("overlay", "overlay:command", "idle" as const); + } catch { /* not in Tauri */ } + await new Promise((resolve) => { + overlayIdleResolveRef.current = resolve; + hideOverlayTimeoutRef.current = setTimeout(() => { + console.warn("[Overlay] idle_ready timeout — hiding anyway"); + resolve(); + }, 2000); + }); + overlayIdleResolveRef.current = null; + if (hideOverlayTimeoutRef.current) { + clearTimeout(hideOverlayTimeoutRef.current); + hideOverlayTimeoutRef.current = null; + } + + // Final session check before hiding + if (sessionIdRef.current !== sessionId) { + console.log(`[PTT] No-commit hide aborted — session ${sessionId} superseded`); + overlayCleaningRef.current = false; + return; + } + try { const { invoke } = await import("@tauri-apps/api/core"); await invoke("hide_overlay"); + console.log("[Overlay] Hidden"); } catch { /* not in Tauri */ } + setPtt("idle"); + setStatus("idle"); + overlayCleaningRef.current = false; return; } - // Wait for backend to finish all transcriptions (state → idle) - await new Promise((resolve) => { + // IMPORTANT: Set up idle resolver BEFORE sending stop_recording to backend. + // This closes the race window where backend emits "state: idle" before we're + // ready to listen for it. + // + // The backend waits up to 15s for transcription threads (orchestrator.py:790). + // This timeout must exceed that deadline. If it fires first, the frontend + // reads stale text before transcription completes. + // Backend emits idle immediately after recording loop ends (no thread join). + // Transcription runs in background and emits events through the pipeline. + const IDLE_TIMEOUT_MS = 20000; + stopTimestampRef.current = Date.now(); + let idleTimeout: ReturnType | null = null; + const idlePromise = new Promise((resolve) => { pttIdleResolveRef.current = resolve; - setTimeout(() => { resolve(); }, 10000); + console.log(`[PTT] Idle resolver set — session=${sessionId}`); + idleTimeout = setTimeout(() => { + const elapsed = Date.now() - stopTimestampRef.current; + console.error("[PTT] Idle timeout triggered"); + console.error(`[PTT] Session: ${sessionId}`); + console.error(`[PTT] Backend state: ${status}`); + console.error(`[PTT] Frontend state: ${pttStateRef.current}`); + console.error(`[PTT] Elapsed: ${elapsed}ms (timeout: ${IDLE_TIMEOUT_MS}ms)`); + console.error(`[PTT] Backend alive: ${runtimeRef.current !== null}`); + console.error(`[PTT] Resolver still set: ${pttIdleResolveRef.current !== null}`); + resolve(); + }, IDLE_TIMEOUT_MS); }); + + // Tell backend to stop capturing + runtimeRef.current.stop(); + setConnected(false); + micLevelEmitter.emit(0); + + // Now wait for backend to finish all transcriptions (state → idle) + await idlePromise; + if (idleTimeout) clearTimeout(idleTimeout); + const resolvedByEvent = pttIdleResolveRef.current === null; + console.log(`[PTT] Idle promise resolved (byEvent=${resolvedByEvent}) — session=${sessionId}`); pttIdleResolveRef.current = null; // Read accumulated text @@ -1170,10 +1296,11 @@ function App() { if (text && hwnd) { // Processing → Inserting setPtt("inserting"); + console.log("[PTT] Inserting text"); // Tell overlay to show inserting state try { - const { emit } = await import("@tauri-apps/api/event"); - await emit("overlay:command", "inserting" as const); + const { emitTo } = await import("@tauri-apps/api/event"); + await emitTo("overlay", "overlay:command", "inserting" as const); } catch { /* not in Tauri */ } try { const { invoke } = await import("@tauri-apps/api/core"); @@ -1191,26 +1318,66 @@ function App() { // Inserting → Success (brief confirmation) setPtt("success"); setStatus("idle"); + console.log("[PTT] Success — showing confirmation"); // Tell overlay to show success state try { - const { emit } = await import("@tauri-apps/api/event"); - await emit("overlay:command", "success" as const); + const { emitTo } = await import("@tauri-apps/api/event"); + await emitTo("overlay", "overlay:command", "success" as const); } catch { /* not in Tauri */ } - // Success → Idle after 400ms - setTimeout(() => { + // After 400ms: clean up overlay, then transition to idle + // NOTE: setPtt("idle") is called AFTER cleanup, not before. + // This prevents a new session from starting while old cleanup is running. + setTimeout(async () => { + console.log("[PTT] Success timeout — cleaning up overlay"); + + // Verify session is still active before cleaning up + if (sessionIdRef.current !== sessionId) { + console.log(`[PTT] Cleanup aborted — session ${sessionId} superseded by ${sessionIdRef.current}`); + return; + } + + // Tell overlay to clean up, wait for acknowledgement, then hide + overlayCleaningRef.current = true; + try { + const { emitTo } = await import("@tauri-apps/api/event"); + await emitTo("overlay", "overlay:command", "idle" as const); + } catch { /* not in Tauri */ } + await new Promise((resolve) => { + overlayIdleResolveRef.current = resolve; + hideOverlayTimeoutRef.current = setTimeout(() => { + console.warn("[Overlay] idle_ready timeout — hiding anyway"); + resolve(); + }, 2000); + }); + overlayIdleResolveRef.current = null; + if (hideOverlayTimeoutRef.current) { + clearTimeout(hideOverlayTimeoutRef.current); + hideOverlayTimeoutRef.current = null; + } + + // Final session check before hiding + if (sessionIdRef.current !== sessionId) { + console.log(`[PTT] Hide aborted — session ${sessionId} superseded`); + overlayCleaningRef.current = false; + return; + } + + try { + const { invoke } = await import("@tauri-apps/api/core"); + await invoke("hide_overlay"); + console.log("[Overlay] Hidden — session finished"); + } catch { /* not in Tauri */ } + + // NOW transition to idle — new sessions can start after this setPtt("idle"); - // Hide overlay - import("@tauri-apps/api/core").then(({ invoke }) => { - invoke("hide_overlay"); - }).catch(() => {}); + overlayCleaningRef.current = false; }, 400); - // Clean up + // Clean up refs (async — happens while overlay is showing success) pttTextRef.current = ""; pttHwndRef.current = null; - isCommittingRef.current = false; }; startRef.current = start; @@ -1220,12 +1387,30 @@ function App() { useEffect(() => { (async () => { try { - const { emit } = await import("@tauri-apps/api/event"); - await emit("widget-status", status); + const { emitTo } = await import("@tauri-apps/api/event"); + await emitTo("widget", "widget-status", status); } catch { /* not in Tauri */ } })(); }, [status]); + // --- Overlay: listen for cleanup acknowledgement --- + useEffect(() => { + let unlisten: (() => void) | undefined; + (async () => { + try { + const { listen } = await import("@tauri-apps/api/event"); + unlisten = await listen("overlay:idle_ready", () => { + console.log("[Overlay] Received idle_ready — cleanup complete"); + if (overlayIdleResolveRef.current) { + overlayIdleResolveRef.current(); + overlayIdleResolveRef.current = null; + } + }); + } catch { /* not in Tauri */ } + })(); + return () => { unlisten?.(); }; + }, []); + // --- Widget: listen for toggle and show-main events --- useEffect(() => { let unlistenToggle: (() => void) | undefined; diff --git a/stt-ui/src/api-tauri.ts b/stt-ui/src/api-tauri.ts index b7b821f..9a3599e 100644 --- a/stt-ui/src/api-tauri.ts +++ b/stt-ui/src/api-tauri.ts @@ -40,30 +40,53 @@ export function getWebAudioAnalyser(): AnalyserNode | null { export function createTauriApi(args: string[], sidecarName: string = "binaries/stt-engine"): STTApi { let listeners: Array<(e: STTEvent) => void> = []; let child: Child | null = null; - let lineBuffer = ""; // Buffer for partial JSON lines across chunk boundaries + // IMPORTANT: stdout and stderr are two independent, asynchronously-arriving + // streams — stdout carries the line-delimited JSON event protocol + // (`_json_emit` in the Python backend), stderr carries plain-text log + // lines from the `logging` module. They MUST use separate buffers. A + // previous version of this code buffered both streams through a single + // shared `lineBuffer`, so if a stderr "data" event fired in between two + // stdout chunks (or vice versa), fragments from the two unrelated streams + // got concatenated together before the next newline — corrupting the + // JSON line so it silently failed to parse and the event was dropped + // (with only a garbled fallback console.log/warn, no error surfaced). + // This caused state/transcript events (e.g. "transcribing", the final + // transcript, or "idle") to intermittently vanish depending on exact + // timing — the exact kind of "works most of the time, randomly fails" + // behavior reported for PTT. + let stdoutBuffer = ""; + let stderrBuffer = ""; const notifyError = (msg: string) => { for (const cb of listeners) cb({ type: "error", message: msg }); }; - const handleLine = (source: string, data: string) => { - lineBuffer += data; - const lines = lineBuffer.split("\n"); - // Keep the last (potentially incomplete) chunk in the buffer - lineBuffer = lines.pop() ?? ""; - for (const raw of lines) { - const trimmed = raw.trim(); - if (!trimmed) continue; - try { - const event: STTEvent = JSON.parse(trimmed); - for (const cb of listeners) cb(event); - } catch { - if (source === "stderr") { - console.warn(`[Sidecar stderr] ${trimmed}`); - } else { + const handleLine = (source: "stdout" | "stderr", data: string) => { + if (source === "stdout") { + stdoutBuffer += data; + const lines = stdoutBuffer.split("\n"); + // Keep the last (potentially incomplete) chunk in the buffer + stdoutBuffer = lines.pop() ?? ""; + for (const raw of lines) { + const trimmed = raw.trim(); + if (!trimmed) continue; + try { + const event: STTEvent = JSON.parse(trimmed); + for (const cb of listeners) cb(event); + } catch { console.log(`[Sidecar stdout] ${trimmed}`); } } + } else { + stderrBuffer += data; + const lines = stderrBuffer.split("\n"); + stderrBuffer = lines.pop() ?? ""; + for (const raw of lines) { + const trimmed = raw.trim(); + if (!trimmed) continue; + // stderr is plain-text logging, never JSON — no parse attempt needed. + console.warn(`[Sidecar stderr] ${trimmed}`); + } } }; diff --git a/stt-ui/src/components/PttOverlay.tsx b/stt-ui/src/components/PttOverlay.tsx deleted file mode 100644 index 9aa1075..0000000 --- a/stt-ui/src/components/PttOverlay.tsx +++ /dev/null @@ -1,73 +0,0 @@ -import { useEffect, useState } from "react"; -import { Mic } from "lucide-react"; - -interface PttOverlayProps { - visible: boolean; -} - -export default function PttOverlay({ visible }: PttOverlayProps) { - const [show, setShow] = useState(false); - - useEffect(() => { - if (visible) { - setShow(true); - } else { - // Small delay before hiding for smooth exit - const timer = setTimeout(() => setShow(false), 200); - return () => clearTimeout(timer); - } - }, [visible]); - - if (!show) return null; - - return ( -
-
- {/* Pulsing mic icon */} -
-
- -
- - Listening... - - {/* Waveform dots */} -
- {[0, 1, 2].map((i) => ( -
- ))} -
-
- - -
- ); -} diff --git a/stt-ui/src/components/WidgetView.tsx b/stt-ui/src/components/WidgetView.tsx index 0b15714..146799b 100644 --- a/stt-ui/src/components/WidgetView.tsx +++ b/stt-ui/src/components/WidgetView.tsx @@ -3,7 +3,7 @@ import { Maximize2, X } from "lucide-react"; import { MicIcon } from "./icons/MicIcon"; import { MicOffIcon } from "./icons/MicOffIcon"; import { LanguagesIcon } from "./icons/LanguagesIcon"; -import { listen, emit } from "@tauri-apps/api/event"; +import { listen, emitTo } from "@tauri-apps/api/event"; import { invoke } from "@tauri-apps/api/core"; type WidgetStatus = "idle" | "listening" | "transcribing" | "rewriting" | "error"; @@ -147,12 +147,12 @@ export default function WidgetView() { const handleToggle = useCallback(async () => { if (!isTauri()) return; - await emit("widget-toggle"); + await emitTo("main", "widget-toggle"); }, []); const handleShowMain = useCallback(async () => { if (!isTauri()) return; - await emit("widget-show-main"); + await emitTo("main", "widget-show-main"); setShowMenu(false); }, []); diff --git a/stt-ui/src/overlay/OverlayView.tsx b/stt-ui/src/overlay/OverlayView.tsx index 17e6bfa..4a84c94 100644 --- a/stt-ui/src/overlay/OverlayView.tsx +++ b/stt-ui/src/overlay/OverlayView.tsx @@ -46,27 +46,72 @@ export default function OverlayView() { // Listen for overlay commands from the frontend (main window) useEffect(() => { let unlisten: (() => void) | undefined; + let retries = 0; + const MAX_RETRIES = 5; + const RETRY_DELAY = 200; - (async () => { + const tryListen = async (): Promise => { try { const { listen } = await import("@tauri-apps/api/event"); unlisten = await listen("overlay:command", (event) => { const newState = event.payload; + console.log(`[Overlay] Received command: ${newState}`); setState(newState); - // Start/stop waveform based on listening state - const shouldCapture = newState === "listening"; - if (shouldCapture && waveformRef.current && !waveformRef.current.isActive()) { + // Start waveform + if (newState === "listening" && waveformRef.current && !waveformRef.current.isActive()) { + console.log("[Overlay] Starting waveform"); waveformRef.current.start(); - } else if (!shouldCapture && waveformRef.current?.isActive()) { + } + + // Stop waveform + if (newState !== "listening" && waveformRef.current?.isActive()) { + console.log("[Overlay] Stopping waveform"); waveformRef.current.stop(); } + + // Full cleanup — reset to initial state, then signal ready + if (newState === "idle") { + console.log("[Overlay] Cleaning up"); + waveformRef.current?.stop(); + scaleSpring.setValue(0); + + // Reset SVG bars to minimum height + const svg = svgRef.current; + if (svg) { + const bars = svg.querySelectorAll("rect"); + bars.forEach((bar) => { + bar.setAttribute("y", "11"); + bar.setAttribute("height", "2"); + }); + } + + rendererInitRef.current = false; + console.log("[Overlay] Cleanup complete"); + + // Acknowledge cleanup — App.tsx waits for this before hiding + import("@tauri-apps/api/event").then(({ emitTo }) => { + emitTo("main", "overlay:idle_ready", {}); + console.log("[Overlay] Sent idle_ready"); + }).catch(() => {}); + } }); + return true; } catch { - // Not in Tauri — show a static pill for development - setState("listening"); + return false; + } + }; + + (async () => { + while (retries < MAX_RETRIES) { + if (await tryListen()) return; + retries++; + if (retries < MAX_RETRIES) { + await new Promise((r) => setTimeout(r, RETRY_DELAY)); + } } + console.warn("[Overlay] Failed to register listener after retries — staying idle"); })(); return () => { diff --git a/stt/orchestrator.py b/stt/orchestrator.py index 7448de0..f8317fe 100644 --- a/stt/orchestrator.py +++ b/stt/orchestrator.py @@ -36,7 +36,7 @@ from stt.audio_capture import mic_stream, find_default_microphone, find_best_microphone from stt.speaker import SpeakerVerifier from stt.vad import ( - compute_rms, StreamingEndpointDetector, + compute_rms, StreamingEndpointDetector, VADEvent, compute_spectral_centroid, compute_spectral_flux, compute_zero_crossing_rate, compute_band_energy_ratio, ) @@ -79,6 +79,52 @@ def _debug(config: AppConfig, *args, **kwargs) -> None: _ws_clients: list = [] _ws_loop = None # set by start_ws_server +# Pending idle model-release timer (see _schedule_idle_release / _cancel_idle_release). +# The ASR model is the dominant memory user; we free it a short grace period +# after the user stops dictating, then re-warm on the next session. +_idle_release_timer: threading.Timer | None = None +_IDLE_RELEASE_GRACE_SEC = float(os.environ.get("STT_IDLE_RELEASE_SEC", "20")) + + +def _cancel_idle_release() -> None: + """Cancel a pending idle model-release (called when a new session starts).""" + global _idle_release_timer + timer = _idle_release_timer + if timer is not None: + timer.cancel() + _idle_release_timer = None + + +def _schedule_idle_release() -> None: + """Free the ASR model after a grace period of continued idleness. + + Releasing the model reclaims several GB of RAM/VRAM that would otherwise + stay pinned for the entire app lifetime under push-to-talk. If a new + session starts before the timer fires, _cancel_idle_release() aborts it + and the warm model is reused (no latency cost). + """ + global _idle_release_timer + _cancel_idle_release() + if _IDLE_RELEASE_GRACE_SEC <= 0: + return + + def _release(): + global _idle_release_timer + _idle_release_timer = None + try: + from stt.transcription import release_backend + release_backend() + logger.info("idle: released ASR model to free memory") + except Exception: + pass + + try: + _idle_release_timer = threading.Timer(_IDLE_RELEASE_GRACE_SEC, _release) + _idle_release_timer.daemon = True + _idle_release_timer.start() + except Exception: + pass + async def _ws_broadcast(payload: str) -> None: """Send payload to all connected WS clients (async).""" @@ -117,7 +163,7 @@ def _json_emit(config: AppConfig, event: dict) -> None: logger.warning("dropped", reason=event.get("reason"), utterance_id=event.get("utterance_id")) else: logger.debug("event", type=event_type, payload=payload[:200]) - # Print to stdout for frontend consumption + # Print to stdout for frontend consumption. if config.json_mode: print(payload, flush=True) # Try legacy WebSocket broadcast @@ -361,20 +407,30 @@ def __init__(self, max_samples: int): self._total = 0 def extend(self, chunk: np.ndarray) -> None: - n = len(chunk) - if n >= self._max: - self._buf[:] = chunk[-self._max:] - self._total += n - return - pos = self._total % self._max - self._total += n - end = pos + n + total_n = len(chunk) + write_n = min(total_n, self._max) + if write_n < total_n: + # Only the tail fits; the rest is older than the buffer's + # capacity and would be evicted immediately anyway. + chunk = chunk[-write_n:] + # `_total` must track the true cumulative sample count (other code + # uses total_samples() as an absolute sample-index counter), even + # when only the tail of an oversized chunk is physically written. + # The write position is derived from the absolute index of the + # first *written* sample, not from the old total directly — writing + # at `old_total % max` would misalign the circular pointer relative + # to slice_range()'s `sample_index % max` invariant. + old_total = self._total + self._total += total_n + write_start_abs = old_total + (total_n - write_n) + pos = write_start_abs % self._max + end = pos + write_n if end <= self._max: self._buf[pos:end] = chunk else: first = self._max - pos self._buf[pos:] = chunk[:first] - self._buf[:n - first] = chunk[first:] + self._buf[:write_n - first] = chunk[first:] def slice_range(self, start_sample: int, end_sample: int) -> np.ndarray: """Return a copy of samples [start_sample, end_sample).""" @@ -548,29 +604,80 @@ def run( _is_tty = sys.stdin.isatty() def _stdin_reader(): - """Read JSON commands from stdin (one per line).""" + """Read JSON commands from stdin (one per line). + + Uses os.read() on the raw file descriptor (fd 0) instead of the + sys.stdin iterator — the Python text-mode wrapper on Windows pipes + can buffer incomplete lines and never flush, causing the reader + thread to block indefinitely on partial reads. + """ nonlocal _stdin_alive + _fd = 0 # stdin file descriptor + _buf = b"" + logger.info("stdin_reader: thread started (fd=%d, tty=%s)", _fd, _is_tty) try: - for line in sys.stdin: - line = line.strip() - if not line: - continue - try: - cmd = _json.loads(line) - if cmd.get("type") == "start_recording": - _start_event.set() - elif cmd.get("type") == "stop_recording": - _stop_event.set() - _start_event.set() # Also unblocks if waiting - except _json.JSONDecodeError: - pass - except (EOFError, OSError): - pass + while True: + chunk = os.read(_fd, 4096) + if not chunk: + logger.info("stdin_reader: EOF received (pipe closed)") + break + _buf += chunk + while b"\n" in _buf: + line_bytes, _buf = _buf.split(b"\n", 1) + line = line_bytes.decode("utf-8", errors="replace").strip() + if not line: + continue + logger.info("stdin_reader: received %d bytes: %s", len(line_bytes), line[:200]) + try: + cmd = _json.loads(line) + cmd_type = cmd.get("type", "") + if cmd_type == "start_recording": + # Clear any stale stop from a previous session. + # If stop_recording was queued in the stdin pipe + # (e.g. user released hotkey while the backend was + # still processing the previous session), it would + # set _stop_event. The new start_recording must + # clear it so calibration doesn't see a phantom + # stop and bail with 0 chunks. + if _stop_event.is_set(): + logger.info("stdin_reader: clearing stale _stop_event before start") + _stop_event.clear() + logger.info("stdin_reader: → setting _start_event") + _start_event.set() + elif cmd_type == "stop_recording": + logger.info("stdin_reader: → setting _stop_event") + _stop_event.set() + # NOTE: intentionally NOT setting _start_event here. + # _start_event.wait() (top of the PTT loop) should + # only ever be unblocked by a genuine start_recording + # command (or stdin EOF). If a stray/duplicate + # stop_recording arrives while the backend is + # legitimately idle and waiting for the next + # start_recording, setting _start_event here would + # spuriously kick off a phantom recording session + # that the frontend never asked for and will never + # send a matching stop for — it opens the mic, + # calibrates, sits in "listening" forever, and the + # frontend eventually times out waiting for an + # "idle" event that was never coming for the real + # session. This was the root cause of PTT sessions + # getting stuck / producing no transcript after a + # quick or duplicate release. + else: + logger.info("stdin_reader: unknown command type=%s", cmd_type) + except _json.JSONDecodeError as exc: + logger.warning("stdin_reader: JSON decode error: %s — raw=%s", exc, line[:200]) + except OSError as exc: + logger.error("stdin_reader: OSError: %s", exc) + except Exception as exc: + logger.error("stdin_reader: unexpected error: %s", exc) _stdin_alive = False + logger.info("stdin_reader: thread exiting (setting _start_event)") _start_event.set() # Unblock main thread if waiting on start stdin_thread = threading.Thread(target=_stdin_reader, daemon=True) stdin_thread.start() + logger.info("stdin_reader: thread launched (daemon=%s, alive=%s)", stdin_thread.daemon, stdin_thread.is_alive()) if _is_tty: # Interactive terminal — auto-start recording immediately @@ -580,9 +687,16 @@ def _stdin_reader(): sr = config.audio.sample_rate block_size = config.audio.blocksize - # Warm the selected ASR backend in parallel with calibration. - warmup_thread = threading.Thread(target=warm_up_backend, args=(config.transcription,), daemon=True) + # Warm the selected ASR backend (model loading can take 10-20s on first use). + # Run in a thread so frontend can connect in parallel. + def _warmup_target(cfg): + try: + warm_up_backend(cfg) + finally: + pass + warmup_thread = threading.Thread(target=_warmup_target, args=(config.transcription,), daemon=True) warmup_thread.start() + _echo("ASR warm-up started (model loading)...") # --- Speaker enrollment (deferred for non-TTY: done after start_recording) --- speaker_verifier: SpeakerVerifier | None = None @@ -609,8 +723,12 @@ def _stop(_sig, _frame): _active_xcribe_threads: list[threading.Thread] = [] while _stdin_alive or _is_tty: + logger.info("PTT loop: _stdin_alive=%s, _is_tty=%s, _start_event=%s, _stop_event=%s", + _stdin_alive, _is_tty, _start_event.is_set(), _stop_event.is_set()) if not _is_tty: _start_event.wait() + logger.info("PTT loop: _start_event.wait() returned — _stdin_alive=%s, _stop_event=%s", + _stdin_alive, _stop_event.is_set()) if not _stdin_alive: _echo("Received stop before start — exiting.") return @@ -618,6 +736,24 @@ def _stop(_sig, _frame): if not running: break + # A new session is starting — keep the (possibly still warm) model. + _cancel_idle_release() + + # If the model was released during the previous idle period, re-warm it + # in the background. The 1.5s calibration window covers most of the + # load time; if it isn't ready, the first transcription reloads cold. + try: + if _idle_release_timer is None: + _warmup_thread = threading.Thread( + target=_warmup_target, args=(config.transcription,), daemon=True + ) + _warmup_thread.start() + except Exception: + pass + + # NOTE: Warmup runs in background. If not done yet, first transcription + # will be slow but the mic opens immediately — no blocking here. + # --- Open mic + calibrate (only after start_recording) --- ring = _RingBuffer(int(30 * sr)) detector = StreamingEndpointDetector(config.vad, sr, block_size) @@ -632,10 +768,21 @@ def _stop(_sig, _frame): calib_flux: list[float] = [] calib_zcr: list[float] = [] calib_ber: list[float] = [] + session_start_sample = ring.total_samples() try: stream_iter = mic_stream(config.audio, debug=False) + logger.info("Mic stream opened successfully") deadline = time.monotonic() + 1.5 while time.monotonic() < deadline: + if _stop_event.is_set(): + # PTT was already released before calibration finished + # (a fast, natural press-speak-release cycle). Stop + # collecting immediately instead of blocking the user + # for the rest of the 1.5s window — the audio captured + # so far is still in `ring` and will be handled by the + # quick-tap fallback below. + logger.info("Calibration interrupted by stop_recording (%d chunks collected)", len(calib_rms)) + break chunk = next(stream_iter) ring.extend(chunk) calib_rms.append(compute_rms(chunk)) @@ -662,7 +809,7 @@ def _stop(_sig, _frame): p10 = sorted_r[len(sorted_r) // 10] detector.set_noise_floor(p10) st, et = detector.thresholds() - _debug(config, f"calibration: p10={p10:.4f}, start_th={st:.4f}, end_th={et:.4f}") + logger.info(f"VAD calibration: p10={p10:.4f}, start_th={st:.4f}, end_th={et:.4f}, calib_chunks={len(calib_rms)}") if calib_centroid and config.vad.use_spectral_vad: sorted_c = sorted(calib_centroid) @@ -675,12 +822,12 @@ def _stop(_sig, _frame): avg_zcr = float(sorted_z[p10_idx]) avg_ber = float(sorted_b[p10_idx]) detector.set_spectral_baselines(avg_centroid, avg_flux, avg_zcr, avg_ber) - _debug(config, f"calibration: centroid={avg_centroid:.0f}Hz, flux={avg_flux:.4f}, " + logger.info(f"VAD calibration spectral: centroid={avg_centroid:.0f}Hz, flux={avg_flux:.4f}, " f"zcr={avg_zcr:.4f}, ber={avg_ber:.4f}") # --- Speaker enrollment (on first start only) --- if speaker_verifier is None and config.diarization.enabled: - from numpy import NDArray + from numpy.typing import NDArray speaker_verifier = SpeakerVerifier(method=config.diarization.method) enrollment_embs: list[NDArray[np.float32]] = [] n_chunks = config.diarization.enrollment_chunks @@ -704,16 +851,90 @@ def _stop(_sig, _frame): _debug(config, f"speaker enrollment failed: {exc}") speaker_profile = None + chunk_count = 0 + utterance_id = 0 + + def _dispatch_vad_event(event: VADEvent) -> None: + """Turn a VAD 'end' event into a transcription thread, if it + passes the minimum-duration and speaker-gate checks.""" + nonlocal utterance_id + segment = ring.slice_range(event.start_sample, event.end_sample) + if len(segment) == 0: + return + + dur = len(segment) / sr + rms_seg = compute_rms(segment) + logger.info(f"VAD utterance: {dur:.1f}s, rms={rms_seg:.4f}" + + (" [forced split]" if event.forced_split else "")) + + if dur < config.vad.min_recording_sec: + return + + # Speaker gate: reject segments that don't match enrolled speaker + if config.diarization.enabled and speaker_verifier is not None and speaker_profile is not None: + accepted, score = speaker_verifier.verify(segment, sr, speaker_profile, threshold=config.diarization.similarity_threshold) + _json_emit(config, {"type": "speaker", "accepted": accepted, "similarity": round(score, 4)}) + if not accepted: + _debug(config, f"speaker rejected: sim={score:.3f}") + return + + utterance_id += 1 + thread = threading.Thread( + target=_transcribe_and_print, + args=(config, segment.copy(), sr, ring.total_samples() / sr, utterance_id, telemetry, hooks), + daemon=True, + ) + _active_xcribe_threads.append(thread) + thread.start() + + # --- Quick-tap fallback --- + # Calibration (and speaker enrollment, if enabled) can eat up to + # ~1.5s+ of audio before the VAD state machine ever runs a single + # `detector.update()` call on it. A fast, completely normal + # press-speak-release PTT cycle can finish entirely inside that + # window: the audio is captured into `ring`, but since VAD never + # saw it, `_in_speech` stays False and the later force_end() flush + # (see the `finally` block below) has nothing to finalize — the + # utterance would otherwise be silently discarded with zero + # transcription. If stop_recording already arrived by the time we + # get here, treat everything captured this session as one + # candidate utterance and dispatch it directly. + if _stop_event.is_set(): + fallback_end = ring.total_samples() + if fallback_end > session_start_sample: + logger.info( + "PTT released during calibration/enrollment — dispatching " + "%.2fs as fallback utterance", + (fallback_end - session_start_sample) / sr, + ) + _dispatch_vad_event(VADEvent( + kind="end", + start_sample=session_start_sample, + end_sample=fallback_end, + forced_split=False, + )) + + # If stop arrived during calibration, close the mic stream NOW so the + # recording-loop's `next(stream_iter)` immediately raises StopIteration + # instead of blocking forever waiting for audio. The fallback utterance + # (if any) was already dispatched above. + if _stop_event.is_set(): + logger.info("stop_recording arrived during calibration — closing mic before recording loop") + try: + stream_iter.close() + except Exception: + pass + _echo("Recording started by frontend.") _json_emit(config, {"type": "state", "state": "listening"}) if hooks and hooks.on_state: hooks.on_state("listening") - chunk_count = 0 - utterance_id = 0 try: for chunk in stream_iter: if not running or (stop_event is not None and stop_event.is_set()) or _stop_event.is_set(): + logger.info("Recording loop: breaking — running=%s, stop_event=%s, _stop_event=%s", + running, stop_event.is_set() if stop_event else "N/A", _stop_event.is_set()) break chunk_start = ring.total_samples() ring.extend(chunk) @@ -725,15 +946,16 @@ def _stop(_sig, _frame): _json_emit(config, {"type": "mic", "level": round(rms, 6)}) chunk_count += 1 - if config.debug and chunk_count % 8 == 0: + if chunk_count % 8 == 0: st, et = detector.thresholds() noise = detector.noise_floor snr_db = 20 * np.log10(rms / max(noise, 1e-10)) if noise > 0 else 0 state = detector.vad_state.name - _debug(config, f"rms={rms:.6f} noise={noise:.4f} snr={snr_db:.1f}dB state={state}") + in_speech = detector._in_speech + logger.info(f"VAD: rms={rms:.6f} noise={noise:.4f} snr={snr_db:.1f}dB state={state} in_speech={in_speech} onset_th={st:.4f}") if config.vad.use_spectral_vad: score = detector._compute_speech_score(chunk) - _debug(config, f" spectral: score={score:.4f}") + logger.info(f" spectral: score={score:.4f}") event = detector.update( rms=rms, @@ -743,57 +965,56 @@ def _stop(_sig, _frame): ) if event is None or event.kind == "start": if event: - _debug(config, f"speech start at {event.start_sample/sr:.2f}s") + logger.info(f"VAD: speech START at {event.start_sample/sr:.2f}s") continue if event.end_sample is None: continue - segment = ring.slice_range(event.start_sample, event.end_sample) - if len(segment) == 0: - continue - - dur = len(segment) / sr - rms_seg = compute_rms(segment) - _debug(config, f"utterance: {dur:.1f}s, rms={rms_seg:.4f}" - + (" [forced split]" if event.forced_split else "")) - - if dur < config.vad.min_recording_sec: - continue - - # Speaker gate: reject segments that don't match enrolled speaker - if config.diarization.enabled and speaker_verifier is not None and speaker_profile is not None: - accepted, score = speaker_verifier.verify(segment, sr, speaker_profile, threshold=config.diarization.similarity_threshold) - _json_emit(config, {"type": "speaker", "accepted": accepted, "similarity": round(score, 4)}) - if not accepted: - _debug(config, f"speaker rejected: sim={score:.3f}") - continue - - utterance_id += 1 - thread = threading.Thread( - target=_transcribe_and_print, - args=(config, segment.copy(), sr, ring.total_samples() / sr, utterance_id, telemetry, hooks), - daemon=True, - ) - _active_xcribe_threads.append(thread) - thread.start() + _dispatch_vad_event(event) except KeyboardInterrupt: break finally: - pass # Don't close stream_iter — reuse across PTT sessions + # Flush any utterance still in progress (e.g. the user released + # PTT right after finishing their sentence, before the VAD's + # ~500ms trailing-silence window elapsed). Without this, the + # last thing the user said is silently discarded instead of + # transcribed — normal endpointing never fires because the mic + # stream is about to be torn down. + try: + flush_event = detector.force_end(ring.total_samples()) + if flush_event is not None: + logger.info( + "VAD: force-flushing in-progress speech on stop (%.2fs)", + (flush_event.end_sample - flush_event.start_sample) / sr, + ) + _dispatch_vad_event(flush_event) + except Exception: + logger.exception("VAD flush-on-stop failed") - # --- Wait for pending transcription threads before going idle --- - # This ensures pttTextRef has the latest transcription when the - # frontend reads it after receiving state:idle. + # Close mic stream to stop audio capture and prevent input overflow. + # A new stream is opened at the start of each PTT session. + try: + if stream_iter is not None: + stream_iter.close() + except Exception: + pass + + # --- Wait for in-flight transcription threads before emitting idle --- + # The `processed` event (which carries the final text the frontend + # types) is emitted from inside the transcription thread. If we emit + # `idle` immediately, the frontend commits text before the transcript + # exists (pttTextRef still empty) → "Nothing to commit", nothing typed. + # Join with a bounded timeout so a hung ASR can't stall the loop + # forever, but a normal utterance finishes well within the window. if _active_xcribe_threads: - _echo(f"Waiting for {_active_xcribe_threads.__len__()} pending transcription(s)...") - _deadline = time.monotonic() + 15.0 # max 15 s wait - for t in _active_xcribe_threads: - remaining = max(0.1, _deadline - time.monotonic()) - t.join(timeout=remaining) - _active_xcribe_threads.clear() + _echo(f"{len(_active_xcribe_threads)} transcription(s) running in background — waiting to finish") + for _t in _active_xcribe_threads: + _t.join(timeout=180) + _active_xcribe_threads.clear() # Recording session ended — wait for next start or exit + logger.info("Session ended: clearing _stop_event and _start_event") _stop_event.clear() _start_event.clear() _json_emit(config, {"type": "state", "state": "idle"}) @@ -801,6 +1022,10 @@ def _stop(_sig, _frame): hooks.on_state("idle") _echo("Recording stopped. Waiting for start_recording...") + # Free the ASR model after a short grace period of idleness so it + # doesn't pin several GB of RAM/VRAM for the whole app lifetime. + _schedule_idle_release() + # --- Cleanup: close stream and print telemetry --- if stream_iter is not None: try: @@ -929,7 +1154,7 @@ def _stop(_sig, _frame): ) if event is None or event.kind == "start": if event: - _debug(config, f"speech start at {event.start_sample/sr:.2f}s") + logger.info(f"VAD: speech START at {event.start_sample/sr:.2f}s") continue if event.end_sample is None: continue @@ -1025,8 +1250,10 @@ def _on_partial(text: str) -> None: ts_total = time.monotonic() - # Drop overlap instead of queueing many decode jobs (keeps tail latency low). - if not _asr_semaphore.acquire(blocking=False): + # Acquire the ASR semaphore (warmup holds it while loading the model). + # Use a blocking acquire with timeout so we wait for warmup instead of + # silently dropping the utterance. + if not _asr_semaphore.acquire(blocking=True, timeout=120): _json_emit( config, { @@ -1171,18 +1398,23 @@ def _on_partial(text: str) -> None: if hooks and hooks.on_state: hooks.on_state("rewriting") - # Build few-shot context from past corrected transcripts (latency-gated) + # Build few-shot context from past corrected transcripts (latency-gated). + # Only when embeddings are explicitly enabled (STT_EMBEDDINGS=1) — otherwise + # _build_few_shot_ctx's availability probe would lazily import and pin the + # sentence-transformers model (~hundreds of MB) into RAM on every utterance. few_shot_context = "" dict_llm_context = "" + _embeddings_enabled = os.environ.get("STT_EMBEDDINGS", "0") == "1" try: store = get_store() - candidates = store.recent_cleanups(limit=20) - if candidates: - before_ctx = time.monotonic() - few_shot_context = _build_few_shot_ctx(raw, candidates, top_k=3, max_tokens=400) - ctx_ms = (time.monotonic() - before_ctx) * 1000 - if few_shot_context: - _debug(config, f"few-shot: {ctx_ms:.0f}ms embedding latency") + if _embeddings_enabled: + candidates = store.recent_cleanups(limit=20) + if candidates: + before_ctx = time.monotonic() + few_shot_context = _build_few_shot_ctx(raw, candidates, top_k=3, max_tokens=400) + ctx_ms = (time.monotonic() - before_ctx) * 1000 + if few_shot_context: + _debug(config, f"few-shot: {ctx_ms:.0f}ms embedding latency") # Build dictionary context for LLM (Layer 3) dict_llm_context = store.build_dict_llm_context() if dict_llm_context: @@ -1197,8 +1429,15 @@ def _on_partial(text: str) -> None: for token in rewrite_stream(raw, config.llm, few_shot_context=few_shot_context, dictionary_context=dict_llm_context): if token: collected.append(token) - sys.stdout.write(token) - sys.stdout.flush() + # In json_mode (Tauri/browser sidecar), stdout is the + # line-delimited JSON event channel. Writing raw tokens + # here corrupts that channel (breaks JSON.parse on the + # client, drops the `processed` event, prevents typing) + # and can emit non-UTF-8 bytes. Raw token echo is only + # for interactive terminal mode. + if not config.json_mode: + sys.stdout.write(token) + sys.stdout.flush() # Stream partial LLM result to browser in real-time _json_emit(config, {"type": "llm_partial", "text": "".join(collected), "utterance_id": utterance_id}) processed = _clean_response("".join(collected)) @@ -1252,6 +1491,7 @@ def _transcribe_with_partials( ) from stt.types import TranscriptionResult, TranscriptionSegment from stt.config import TranscriptionBackend + import time as _time audio = preprocess_audio(audio, sr, tcfg) if audio is None: @@ -1265,6 +1505,7 @@ def _transcribe_with_partials( import os import sys as _sys import tempfile as _tempfile + import time as _time audio_path = None try: @@ -1291,11 +1532,18 @@ def _transcribe_with_partials( result = run_worker(worker_cfg) else: import subprocess as _subprocess - proc = _subprocess.run( - [_sys.executable, "-u", "-m", "stt._cpp_worker"], - input=_json.dumps(worker_cfg), - capture_output=True, - text=True, + # Redirect the child's stderr to DEVNULL instead of + # capture_output=True: whisper.cpp is verbose and + # capture_output buffers the entire child stream in the + # parent's RAM until the process exits, spiking memory for + # long utterances. We only need the JSON on stdout. + with open(os.devnull, "w") as _devnull: + proc = _subprocess.run( + [_sys.executable, "-u", "-m", "stt._cpp_worker"], + input=_json.dumps(worker_cfg), + stdout=_subprocess.PIPE, + stderr=_devnull, + text=True, timeout=120, ) diff --git a/stt/transcription.py b/stt/transcription.py index 27ea87f..24e6d3a 100644 --- a/stt/transcription.py +++ b/stt/transcription.py @@ -27,6 +27,39 @@ _batched_cache: dict[str, "object"] = {} # BatchedInferencePipeline wrappers +def release_backend() -> None: + """Drop all cached ASR models to free GPU/CPU memory. + + The ASR model (e.g. large-v3-turbo) is the single largest memory + consumer in the process. Under push-to-talk the model is idle the vast + majority of the time yet stays resident, pinning several GB of RAM/VRAM + for the whole app lifetime. Releasing it on idle and re-warming on the + next session reclaims that memory without affecting an active session. + """ + global _whisper_cpp_cache, _faster_whisper_cache, _batched_cache + _whisper_cpp_cache.clear() + _faster_whisper_cache.clear() + _batched_cache.clear() + # Release any framework-level allocator caches (ctranslate2 / torch). + try: + import gc + gc.collect() + except Exception: + pass + try: + import ctranslate2 + if hasattr(ctranslate2, "clear_cache"): + ctranslate2.clear_cache() + except Exception: + pass + try: + import torch + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + + def _get_cpp_model(config: TranscriptionConfig): """Return cached whisper.cpp model.""" from pywhispercpp.model import Model as CppModel @@ -166,6 +199,8 @@ def transcribe( def warm_up_backend(config: TranscriptionConfig) -> None: """Preload model weights to avoid first-utterance cold-start latency.""" + import time + t0 = time.monotonic() try: if config.backend is TranscriptionBackend.WHISPER_CPP: _get_cpp_model(config) @@ -175,8 +210,12 @@ def warm_up_backend(config: TranscriptionConfig) -> None: batch_size = config.batch_size if config.batch_size > 0 else (8 if config.device == "cuda" else 0) if batch_size > 0: _get_batched_model(config) + # Pre-import noisereduce so first transcription isn't blocked by lazy import + if config.noise_reduce: + import noisereduce # noqa: F401 except Exception as exc: logger.warning("Warm-up failed (non-fatal): %s", exc) + logger.info("Warm-up complete (%.1fs)", time.monotonic() - t0) # --------------------------------------------------------------------------- diff --git a/stt/vad.py b/stt/vad.py index c988c68..51277a7 100644 --- a/stt/vad.py +++ b/stt/vad.py @@ -190,7 +190,7 @@ def __init__(self, config: VADConfig, sample_rate: int, block_size: int): # --- Thresholds (SNR-based, adaptive) --- self._speech_threshold_db = 6.0 # SNR threshold for speech - self._hysteresis_up_db = 4.0 # Onset margin (harder to start) + self._hysteresis_up_db = 2.0 # Onset margin (lower = more sensitive) self._hysteresis_down_db = 3.0 # Offset margin (easier to stay) self._endpoint_timeout_ms = 500 # Silence before speech end self._min_speech_ms = 50 # Minimum speech duration @@ -484,33 +484,28 @@ def _finish_segment(self, end_sample: int, forced_split: bool) -> VADEvent: self._silence_duration_ms = 0 return event + def force_end(self, end_sample: int) -> VADEvent | None: + """Immediately finalize an in-progress speech segment, without waiting + for trailing silence. -# --------------------------------------------------------------------------- -# Legacy factory (kept for one-shot mode compatibility) -# --------------------------------------------------------------------------- - -def make_speech_detector(config: VADConfig): - thresh = config.silence_threshold_rms - silence_samples = int(config.silence_duration_sec * 16000) - min_samples = int(config.min_recording_sec * 16000) - _cell: list[int] = [-1] - - def is_speech(chunk, sr): - return compute_rms(chunk) > thresh + Used when the caller is stopping the mic stream externally (e.g. the + user released a push-to-talk key). Normal endpointing requires ~500ms + of silence after speech before `update()` emits an "end" event; if the + stream is torn down before that silence arrives, the in-progress + utterance would otherwise be silently discarded. Call this right + before closing the mic stream so the last thing the user said is + still transcribed. - def should_stop(accumulated): - n = len(accumulated) - if n == 0: return False - tail = accumulated[-4096:] - rms = compute_rms(tail) - if rms > thresh: _cell[0] = n; return False - if _cell[0] < 0: _cell[0] = 0 - if (n - _cell[0]) >= silence_samples and n >= min_samples: return True - return False - - def reset(): _cell[0] = -1 - return is_speech, should_stop, reset - - -def is_silent(segment: AudioSegment, threshold: float) -> bool: - return compute_rms(segment.data) < threshold + Returns None if no speech was in progress, or if the in-progress + speech is shorter than the configured minimum. + """ + if not self._in_speech: + return None + speech_samples = end_sample - self._speech_start_sample + if speech_samples < self._min_speech_samples: + self._in_speech = False + self._speech_start_sample = 0 + self._speech_duration_ms = 0 + self._silence_duration_ms = 0 + return None + return self._finish_segment(end_sample, forced_split=False) From 9963f2941c9e172cd3289a661e77ba5dd9961a48 Mon Sep 17 00:00:00 2001 From: joyathawale1628 Date: Tue, 21 Jul 2026 16:29:04 +0530 Subject: [PATCH 6/6] fix: accumulate all utterance text for PTT commit instead of overwriting Long sentences split into multiple utterances by the VAD were only committing the last segment because pttTextRef was overwritten on each new utterance. Changed to a Map that accumulates all segments and joins them at commit time. Also fixed processed/llm_partial line-matching: they used event.utterance_id directly as the line id, but raw events create lines with sessionCounter*100000+utterance_id, so processed events never matched and feed lines were never marked done. --- stt-ui/src/App.tsx | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/stt-ui/src/App.tsx b/stt-ui/src/App.tsx index 4581cf5..fefe7ee 100644 --- a/stt-ui/src/App.tsx +++ b/stt-ui/src/App.tsx @@ -788,7 +788,7 @@ function App() { const [highlightPermissions, setHighlightPermissions] = useState(false); const [resolvedModel, setResolvedModel] = useState<{ profile: string; model: string; backend: string; device: string } | null>(null); const pttHwndRef = useRef(null); // Target HWND captured on PTT press - const pttTextRef = useRef(""); // Latest transcription text for PTT commit + const pttPartsRef = useRef>(new Map()); // utterance_id → text for PTT commit const pttIdleResolveRef = useRef<(() => void) | null>(null); // Resolver for idle state after stop const overlayCleaningRef = useRef(false); // True while awaiting overlay cleanup const overlayIdleResolveRef = useRef<(() => void) | null>(null); // Resolver for overlay:idle_ready @@ -1080,25 +1080,30 @@ function App() { ...prev, { id, raw: event.text, processed: "", status: "transcribing", createdAt: new Date().toISOString() }, ].slice(-500)); - // Store the latest text for PTT commit - pttTextRef.current = event.text; + // Store this utterance's text for PTT commit (accumulates across utterances) + pttPartsRef.current.set(event.utterance_id ?? rawId, event.text); return; } if (event.type === "processed") { - const id = event.utterance_id; - if (!id) return; + const utteranceId = event.utterance_id; + if (!utteranceId) return; + // Match the line id scheme from the raw handler: sessionCounter * 100000 + utterance_id + const lineId = sessionCounter.current * 100000 + utteranceId; setLines((prev) => prev.map((line) => - line.id === id ? { ...line, processed: event.text, status: "done" } : line + line.id === lineId ? { ...line, processed: event.text, status: "done" } : line )); - pttTextRef.current = event.text; + // Accumulate this utterance's final text for PTT commit + pttPartsRef.current.set(utteranceId, event.text); } if (event.type === "llm_partial") { - const id = event.utterance_id; - if (!id) return; + const utteranceId = event.utterance_id; + if (!utteranceId) return; + const lineId = sessionCounter.current * 100000 + utteranceId; setLines((prev) => prev.map((line) => - line.id === id ? { ...line, processed: event.text, status: "rewriting" } : line + line.id === lineId ? { ...line, processed: event.text, status: "rewriting" } : line )); - pttTextRef.current = event.text; + // Update this utterance's text (partial, will be overwritten by processed) + pttPartsRef.current.set(utteranceId, event.text); } }; @@ -1161,7 +1166,7 @@ function App() { pttHwndRef.current = null; } - pttTextRef.current = ""; + pttPartsRef.current.clear(); sessionCounter.current++; nextLocalId.current = 1; setLines([]); @@ -1289,8 +1294,8 @@ function App() { console.log(`[PTT] Idle promise resolved (byEvent=${resolvedByEvent}) — session=${sessionId}`); pttIdleResolveRef.current = null; - // Read accumulated text - const text = pttTextRef.current.trim(); + // Join all utterances' text in order for the PTT commit + const text = Array.from(pttPartsRef.current.values()).join(" ").trim(); const hwnd = pttHwndRef.current; if (text && hwnd) { @@ -1376,7 +1381,7 @@ function App() { }, 400); // Clean up refs (async — happens while overlay is showing success) - pttTextRef.current = ""; + pttPartsRef.current.clear(); pttHwndRef.current = null; };