-
Notifications
You must be signed in to change notification settings - Fork 1
Windows dev #13
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Windows dev #13
Changes from all commits
95fb7e7
0777f13
5bdeda2
234229e
3bad50e
9963f29
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| # Auto-detect text files and normalize line endings to LF | ||
| * text=auto eol=lf | ||
|
|
||
| # Windows-specific files keep CRLF | ||
| *.bat text eol=crlf | ||
| *.cmd text eol=crlf | ||
| *.ps1 text eol=crlf |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,159 @@ | ||
| use tauri::{AppHandle, Manager}; | ||
|
|
||
| // --------------------------------------------------------------------------- | ||
| // Backend commands — the backend is a pure execution engine. | ||
| // It never decides when to start, stop, show, hide, or type. | ||
| // The frontend owns every decision. | ||
| // --------------------------------------------------------------------------- | ||
|
|
||
| /// Insert text into the focused window. | ||
| /// Uses Win32 clipboard + Ctrl+V (with SendInput Unicode fallback). | ||
| /// The frontend calls this after transcription completes. | ||
| #[tauri::command] | ||
| pub fn insert_text(text: String, restore_hwnd: Option<u64>) -> Result<bool, String> { | ||
| if text.trim().is_empty() { | ||
| return Ok(false); | ||
| } | ||
| let platform = std::env::consts::OS; | ||
| match platform { | ||
| "windows" => win32_insert(&text, restore_hwnd), | ||
| "linux" => linux_insert(&text, restore_hwnd), | ||
| "macos" => macos_insert(&text), | ||
| _ => false, | ||
| } | ||
| .pipe(Ok) | ||
| } | ||
|
|
||
| /// Show the overlay window, positioned centered above the taskbar. | ||
| /// The frontend calls this when transitioning to Listening. | ||
| #[tauri::command] | ||
| pub fn show_overlay(app: AppHandle) -> Result<(), String> { | ||
| let win = app | ||
| .get_webview_window("overlay") | ||
| .ok_or_else(|| "Overlay window not found".to_string())?; | ||
|
|
||
| // Position: centered horizontally, 80px above the bottom edge | ||
| if let Ok(Some(monitor)) = win.primary_monitor() { | ||
| let m_size = monitor.size(); | ||
| let m_pos = monitor.position(); | ||
| let pill_w = 280; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 Medium
🤖 Copy this AI Prompt to have your agent fix this: |
||
| let pill_h = 60; | ||
| let margin_bottom = 80; | ||
| let x = (m_pos.x + (m_size.width as i32 - pill_w) / 2) as f64; | ||
| let y = (m_pos.y + m_size.height as i32 - pill_h - margin_bottom) as f64; | ||
| let _ = win.set_position(tauri::Position::Physical(tauri::PhysicalPosition { | ||
| x: x as i32, | ||
| y: y as i32, | ||
| })); | ||
| } | ||
|
|
||
| // Show without stealing focus | ||
| #[cfg(target_os = "windows")] | ||
| { | ||
| use windows::Win32::UI::WindowsAndMessaging::ShowWindow; | ||
| use windows::Win32::UI::WindowsAndMessaging::SW_SHOWNOACTIVATE; | ||
| if let Some(hwnd) = win.hwnd().ok() { | ||
| let _ = unsafe { ShowWindow(hwnd, SW_SHOWNOACTIVATE) }; | ||
| } | ||
| } | ||
| #[cfg(not(target_os = "windows"))] | ||
| { | ||
| let _ = win.show(); | ||
| } | ||
|
|
||
| Ok(()) | ||
| } | ||
|
|
||
| /// Hide the overlay window. | ||
| /// The frontend calls this when transitioning back to Ready. | ||
| #[tauri::command] | ||
| pub fn hide_overlay(app: AppHandle) -> Result<(), String> { | ||
| let win = app | ||
| .get_webview_window("overlay") | ||
| .ok_or_else(|| "Overlay window not found".to_string())?; | ||
| let _ = win.hide(); | ||
| Ok(()) | ||
| } | ||
|
|
||
| // --------------------------------------------------------------------------- | ||
| // Platform-specific text insertion | ||
| // --------------------------------------------------------------------------- | ||
|
|
||
| trait Pipe<T> { | ||
| fn pipe<F, R>(self, f: F) -> R | ||
| where | ||
| F: FnOnce(T) -> R; | ||
| } | ||
|
|
||
| impl<T> Pipe<T> for T { | ||
| fn pipe<F, R>(self, f: F) -> R | ||
| where | ||
| F: FnOnce(T) -> R, | ||
| { | ||
| f(self) | ||
| } | ||
| } | ||
|
|
||
| #[cfg(target_os = "windows")] | ||
| fn win32_insert(text: &str, hwnd: Option<u64>) -> bool { | ||
| use crate::win32; | ||
| if let Some(h) = hwnd { | ||
| win32::set_foreground_hwnd(h); | ||
| std::thread::sleep(std::time::Duration::from_millis(50)); | ||
| } | ||
| if win32::set_clipboard(text) { | ||
| std::thread::sleep(std::time::Duration::from_millis(30)); | ||
| win32::send_ctrl_v(); | ||
| return true; | ||
| } | ||
| win32::send_text_unicode(text); | ||
| true | ||
| } | ||
|
|
||
| #[cfg(not(target_os = "windows"))] | ||
| fn win32_insert(_text: &str, _hwnd: Option<u64>) -> bool { | ||
| false | ||
| } | ||
|
|
||
| #[cfg(target_os = "linux")] | ||
| fn linux_insert(text: &str, hwnd: Option<u64>) -> bool { | ||
| use crate::win32; | ||
| if let Some(h) = hwnd { | ||
| if h != 0 { | ||
| win32::set_foreground_hwnd(h); | ||
| std::thread::sleep(std::time::Duration::from_millis(50)); | ||
| } | ||
| } | ||
| if !win32::set_clipboard(text) { | ||
| return false; | ||
| } | ||
| std::thread::sleep(std::time::Duration::from_millis(30)); | ||
| let is_wayland = std::env::var("WAYLAND_DISPLAY").is_ok(); | ||
| if is_wayland { | ||
| let out = std::process::Command::new("wtype").arg(text).output(); | ||
| return out.map(|o| o.status.success()).unwrap_or(false); | ||
| } | ||
| win32::send_ctrl_v(); | ||
| true | ||
| } | ||
|
|
||
| #[cfg(not(target_os = "linux"))] | ||
| fn linux_insert(_text: &str, _hwnd: Option<u64>) -> bool { | ||
| false | ||
| } | ||
|
|
||
| #[cfg(target_os = "macos")] | ||
| fn macos_insert(text: &str) -> bool { | ||
| let escaped = text.replace('\\', "\\\\").replace('"', "\\\""); | ||
| let script = format!("tell application \"System Events\" to keystroke \"{escaped}\""); | ||
| std::process::Command::new("osascript") | ||
| .args(["-e", &script]) | ||
| .output() | ||
| .map(|o| o.status.success()) | ||
| .unwrap_or(false) | ||
| } | ||
|
|
||
| #[cfg(not(target_os = "macos"))] | ||
| fn macos_insert(_text: &str) -> bool { | ||
| false | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,5 @@ | ||
| mod widget; | ||
| mod commands; | ||
| #[cfg(test)] | ||
| mod tests; | ||
|
|
||
|
|
@@ -963,7 +964,30 @@ mod win32 { | |
| fn lstrcpyW(lpString1: *mut u16, lpString2: *const u16) -> *mut u16; | ||
| } | ||
|
|
||
| // SendInput structures | ||
| #[repr(C)] | ||
| struct KeyBDInput { | ||
| wVk: u16, | ||
| wScan: u16, | ||
| dwFlags: u32, | ||
| time: u32, | ||
| dwExtraInfo: usize, | ||
| } | ||
|
|
||
| #[repr(C)] | ||
| struct Input { | ||
| r#type: u32, | ||
| ki: KeyBDInput, | ||
| _pad: [u8; 8], // union padding | ||
| } | ||
|
|
||
| extern "system" { | ||
| fn SendInput(cInputs: u32, pInputs: *const Input, cbSize: i32) -> u32; | ||
| } | ||
|
|
||
| const INPUT_KEYBOARD: u32 = 1; | ||
| const KEYEVENTF_KEYUP: u32 = 0x0002; | ||
| const KEYEVENTF_UNICODE: u32 = 0x0004; | ||
| const VK_CONTROL: u8 = 0x11; | ||
| const VK_V: u8 = 0x56; | ||
| const CF_UNICODETEXT: u32 = 13; | ||
|
|
@@ -1034,6 +1058,52 @@ mod win32 { | |
| keybd_event(VK_CONTROL, 0, KEYEVENTF_KEYUP, 0); | ||
| } | ||
| } | ||
|
|
||
| /// Type text character-by-character using SendInput with KEYEVENTF_UNICODE. | ||
| /// Works in apps that ignore clipboard+Ctrl+V ( terminals, Electron, some UWP apps). | ||
| pub fn send_text_unicode(text: &str) { | ||
| for ch in text.chars() { | ||
| let code = ch as u32; | ||
| let mut inputs = [ | ||
| Input { | ||
| r#type: INPUT_KEYBOARD, | ||
| ki: KeyBDInput { | ||
| wVk: 0, | ||
| wScan: code as u16, | ||
| dwFlags: KEYEVENTF_UNICODE, | ||
| time: 0, | ||
| dwExtraInfo: 0, | ||
| }, | ||
| _pad: [0; 8], | ||
| }, | ||
| Input { | ||
| r#type: INPUT_KEYBOARD, | ||
| ki: KeyBDInput { | ||
| wVk: 0, | ||
| wScan: code as u16, | ||
| dwFlags: KEYEVENTF_UNICODE | KEYEVENTF_KEYUP, | ||
| time: 0, | ||
| dwExtraInfo: 0, | ||
| }, | ||
| _pad: [0; 8], | ||
| }, | ||
| ]; | ||
| // Handle supplementary plane characters (surrogate pairs) | ||
| if code > 0xFFFF { | ||
| // Send as two UTF-16 surrogates | ||
| let hi = (0xD800 + ((code - 0x10000) >> 10)) as u16; | ||
| let lo = (0xDC00 + ((code - 0x10000) & 0x3FF)) as u16; | ||
| inputs[0].ki.wScan = hi; | ||
| inputs[1].ki.wScan = hi; | ||
| unsafe { SendInput(2, inputs.as_ptr(), std::mem::size_of::<Input>() as i32); } | ||
| inputs[0].ki.wScan = lo; | ||
| inputs[1].ki.wScan = lo; | ||
| unsafe { SendInput(2, inputs.as_ptr(), std::mem::size_of::<Input>() as i32); } | ||
| } else { | ||
| unsafe { SendInput(2, inputs.as_ptr(), std::mem::size_of::<Input>() as i32); } | ||
| } | ||
|
Comment on lines
+1064
to
+1104
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🎯 Functional Correctness | 🟠 Major | ⚡ Quick win 🧩 Analysis chain🌐 Web query:
💡 Result: The Win32 SendInput function returns the number of events that were successfully inserted into the keyboard or mouse input stream [1][2]. If the function returns zero, it indicates that the input was blocked by another thread [1][3]. When SendInput is blocked by User Interface Privilege Isolation (UIPI), the behavior is distinct in the following ways: 1. Return Value: If UIPI blocks the input, SendInput may return zero [4][5]. However, it is important to note that the return value is not a dedicated error code and does not always explicitly identify UIPI as the cause of failure [3][4]. 2. GetLastError: Calling GetLastError after a failure typically returns 5 (ERROR_ACCESS_DENIED) [4], but neither the return value nor GetLastError is guaranteed to indicate that UIPI blocking was the specific cause of the failure [3][4]. Because UIPI restricts input based on integrity levels—where an application can generally only send input to processes of equal or lower integrity—the discrepancy between the number of requested events and the successful return value is often used by developers as an indirect indicator that UIPI has blocked the input [4][6][5]. To bypass these restrictions, applications typically require the 'uiAccess="true"' attribute in their manifest, must be digitally signed, and often need to be installed in a protected location like Program Files [6][7]. Citations:
🏁 Script executed: #!/bin/bash
set -euo pipefail
# Map the relevant file and inspect the cited region plus the caller.
FILE="stt-ui/src-tauri/src/lib.rs"
echo "== file exists =="
git ls-files "$FILE"
echo
echo "== line count =="
wc -l "$FILE"
echo
echo "== outline around relevant symbols =="
ast-grep outline "$FILE" --view expanded || true
echo
echo "== cited region around send_text_unicode =="
sed -n '1040,1135p' "$FILE" | cat -n
echo
echo "== caller region around type_text =="
sed -n '1200,1255p' "$FILE" | cat -n
echo
echo "== search for send_text_unicode usages =="
rg -n "send_text_unicode|type_text\(" "$FILE"Repository: IntegerAlex/floure-core Length of output: 11192 Propagate 🤖 Prompt for AI Agents |
||
| } | ||
| } | ||
| } | ||
|
|
||
| #[cfg(not(target_os = "windows"))] | ||
|
|
@@ -1131,80 +1201,6 @@ fn set_foreground_hwnd(hwnd: u64) -> bool { | |
| win32::set_foreground_hwnd(hwnd) | ||
| } | ||
|
|
||
| /// Type text into the focused input using Win32 clipboard + Ctrl+V. | ||
| /// Pure Win32 API — no PowerShell needed for the critical path. | ||
| /// | ||
| /// Flow: restore previous window focus → set clipboard via Win32 → send Ctrl+V via keybd_event | ||
| #[tauri::command] | ||
| fn type_text(text: String, restore_hwnd: Option<u64>) -> Result<bool, String> { | ||
| if text.trim().is_empty() { | ||
| return Ok(false); | ||
| } | ||
| let platform = std::env::consts::OS; | ||
| if platform == "windows" { | ||
| // Restore focus to the previously-focused window FIRST | ||
| if let Some(hwnd) = restore_hwnd { | ||
| win32::set_foreground_hwnd(hwnd); | ||
| std::thread::sleep(std::time::Duration::from_millis(50)); | ||
| } | ||
| // Set clipboard via Win32 API (no PowerShell overhead) | ||
| if !win32::set_clipboard(&text) { | ||
| return Err("Failed to set clipboard".into()); | ||
| } | ||
| // Small delay for clipboard to propagate | ||
| std::thread::sleep(std::time::Duration::from_millis(30)); | ||
| // Send Ctrl+V via keybd_event (runs in Tauri's GUI thread — has active message loop) | ||
| win32::send_ctrl_v(); | ||
| return Ok(true); | ||
| } | ||
| // Linux — clipboard approach (works regardless of which window has focus) | ||
| if platform == "linux" { | ||
| // Restore focus to the previously-captured window (X11 only — Wayland can't) | ||
| if let Some(hwnd) = restore_hwnd { | ||
| if hwnd != 0 { | ||
| win32::set_foreground_hwnd(hwnd); | ||
| std::thread::sleep(std::time::Duration::from_millis(50)); | ||
| } | ||
| } | ||
| // Set clipboard — wtype/xdotool will type from clipboard | ||
| if !win32::set_clipboard(&text) { | ||
| return Err("Failed to set clipboard on Linux".into()); | ||
| } | ||
| std::thread::sleep(std::time::Duration::from_millis(30)); | ||
| // On X11: send Ctrl+V to paste. On Wayland: wtype to paste from clipboard. | ||
| let is_wayland = std::env::var("WAYLAND_DISPLAY").is_ok(); | ||
| if is_wayland { | ||
| // wtype types directly into the focused Wayland window | ||
| let out = std::process::Command::new("wtype") | ||
| .arg(&text) | ||
| .output(); | ||
| if let Ok(o) = out { | ||
| if o.status.success() { | ||
| return Ok(true); | ||
| } else { | ||
| return Err("wtype failed to type text on Wayland".into()); | ||
| } | ||
| } else { | ||
| return Err("wtype command failed on Wayland".into()); | ||
| } | ||
| } | ||
| // X11: Ctrl+V via xdotool | ||
| win32::send_ctrl_v(); | ||
| return Ok(true); | ||
| } | ||
| if platform == "macos" { | ||
| let escaped = text.replace('\\', "\\\\").replace('"', "\\\""); | ||
| let script = format!("tell application \"System Events\" to keystroke \"{escaped}\""); | ||
| let out = std::process::Command::new("osascript") | ||
| .args(["-e", &script]) | ||
| .output(); | ||
| if let Ok(o) = out { | ||
| return Ok(o.status.success()); | ||
| } | ||
| } | ||
| Err("No typing backend available".into()) | ||
| } | ||
|
|
||
| #[tauri::command] | ||
| fn get_backend_path() -> Result<String, AppError> { | ||
| let candidates = vec![ | ||
|
|
@@ -1418,9 +1414,11 @@ pub fn run() { | |
| toggle_dictionary_favorite, | ||
| import_dictionary_csv, | ||
| export_dictionary_csv, | ||
| type_text, | ||
| get_foreground_hwnd, | ||
| set_foreground_hwnd, | ||
| commands::insert_text, | ||
| commands::show_overlay, | ||
| commands::hide_overlay, | ||
| widget::show_widget, | ||
| widget::hide_widget, | ||
| widget::get_widget_visible, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🔒 Security & Privacy | 🟠 Major | ⚡ Quick win
🧩 Analysis chain
🏁 Script executed:
Repository: IntegerAlex/floure-core
Length of output: 2747
🏁 Script executed:
Repository: IntegerAlex/floure-core
Length of output: 2833
🏁 Script executed:
Repository: IntegerAlex/floure-core
Length of output: 4477
Split the overlay into a least-privilege capability.
stt-ui/src-tauri/capabilities/default.jsoncurrently givesoverlayshell, clipboard, global-shortcut, and updater permissions even thoughstt-ui/src/overlay/OverlayView.tsxonly uses Tauri events (listen/emitTo). Move it to a dedicated overlay capability with just the window/event permissions it needs.🤖 Prompt for AI Agents