Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Auto-detect text files and normalize line endings to LF
* text=auto eol=lf

# Windows-specific files keep CRLF
*.bat text eol=crlf
*.cmd text eol=crlf
*.ps1 text eol=crlf
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ Thumbs.db
venv/
# Virtual environment
.vscode/
# Generated Windows build environment duplicate (1.8GB venv copy)
stt-win11/
# Ignore all node_modules folders everywhere in the project
node_modules/
**/node_modules/
Expand Down
2 changes: 1 addition & 1 deletion stt-ui/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"type": "module",
"scripts": {
"dev": "vite",
"build": "tsc && vite build",
"build": "tsc --noEmit -p tsconfig.app.json && vite build",
"preview": "vite preview",
"tauri": "tauri",
"test": "vitest run",
Expand Down
1 change: 1 addition & 0 deletions stt-ui/src-tauri/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions stt-ui/src-tauri/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ rusqlite = { version = "0.31", features = ["bundled"] }
csv = "1"
dirs-next = "2"
thiserror = "2"
[target.'cfg(target_os = "windows")'.dependencies]
windows = { version = "0.61", features = [
"Win32_UI_WindowsAndMessaging",
"Win32_Foundation",
] }

[dev-dependencies]
tempfile = "3"
Expand Down
2 changes: 1 addition & 1 deletion stt-ui/src-tauri/capabilities/default.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"$schema": "../gen/schemas/desktop-schema.json",
"identifier": "default",
"description": "Capability for the main window",
"windows": ["main", "widget"],
"windows": ["main", "widget", "overlay"],

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔒 Security & Privacy | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Verify which privileged APIs the overlay actually uses.
rg -n 'invoke|listen|emit|clipboard|shell|globalShortcut|updater' \
  stt-ui/src/overlay/OverlayView.tsx stt-ui/src/main.tsx

# Inspect existing capability assignments.
rg -n '"windows"|shell:|clipboard-manager:|global-shortcut:|updater:' \
  stt-ui/src-tauri/capabilities

Repository: IntegerAlex/floure-core

Length of output: 2747


🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Map capability files and overlay references.
printf '\n## capabilities files\n'
git ls-files 'stt-ui/src-tauri/capabilities/*' | sort

printf '\n## overlay references in source\n'
rg -n 'overlay:|OverlayView|window.*overlay|label.*overlay|windows": \["main", "widget", "overlay"\]' stt-ui/src-tauri stt-ui/src/overlay

printf '\n## default capability context\n'
sed -n '1,120p' stt-ui/src-tauri/capabilities/default.json

Repository: IntegerAlex/floure-core

Length of output: 2833


🏁 Script executed:

#!/bin/bash
set -euo pipefail

sed -n '1,140p' stt-ui/src-tauri/src/commands.rs

Repository: IntegerAlex/floure-core

Length of output: 4477


Split the overlay into a least-privilege capability. stt-ui/src-tauri/capabilities/default.json currently gives overlay shell, clipboard, global-shortcut, and updater permissions even though stt-ui/src/overlay/OverlayView.tsx only uses Tauri events (listen/emitTo). Move it to a dedicated overlay capability with just the window/event permissions it needs.

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@stt-ui/src-tauri/capabilities/default.json` at line 5, Remove overlay from
the windows list in the default capability and create a dedicated overlay
capability configuration for the OverlayView component, granting only the
required window and event permissions used by its Tauri listen/emitTo calls.
Preserve the existing permissions for main and widget windows and ensure the
overlay remains associated with its dedicated capability.

"permissions": [
"core:default",
"core:window:default",
Expand Down
159 changes: 159 additions & 0 deletions stt-ui/src-tauri/src/commands.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
use tauri::{AppHandle, Manager};

// ---------------------------------------------------------------------------
// Backend commands — the backend is a pure execution engine.
// It never decides when to start, stop, show, hide, or type.
// The frontend owns every decision.
// ---------------------------------------------------------------------------

/// Insert text into the focused window.
/// Uses Win32 clipboard + Ctrl+V (with SendInput Unicode fallback).
/// The frontend calls this after transcription completes.
#[tauri::command]
pub fn insert_text(text: String, restore_hwnd: Option<u64>) -> Result<bool, String> {
if text.trim().is_empty() {
return Ok(false);
}
let platform = std::env::consts::OS;
match platform {
"windows" => win32_insert(&text, restore_hwnd),
"linux" => linux_insert(&text, restore_hwnd),
"macos" => macos_insert(&text),
_ => false,
}
.pipe(Ok)
}

/// Show the overlay window, positioned centered above the taskbar.
/// The frontend calls this when transitioning to Listening.
#[tauri::command]
pub fn show_overlay(app: AppHandle) -> Result<(), String> {
let win = app
.get_webview_window("overlay")
.ok_or_else(|| "Overlay window not found".to_string())?;

// Position: centered horizontally, 80px above the bottom edge
if let Ok(Some(monitor)) = win.primary_monitor() {
let m_size = monitor.size();
let m_pos = monitor.position();
let pill_w = 280;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Medium src/commands.rs:39

show_overlay positions the overlay using hard-coded 280x60 dimensions, but these are logical-pixel values. On a scaled display (e.g. 150% or 200% DPI), the window's physical size is larger, so the computed PhysicalPosition is visibly off-center and the bottom margin is wrong. The position is derived from pill_w and pill_h constants instead of the window's actual physical size, so set_position receives coordinates that don't match the real window dimensions. Consider scaling the logical dimensions by monitor.scale_factor() (or using LogicalPosition so the framework converts for you).

🤖 Copy this AI Prompt to have your agent fix this:
In file @stt-ui/src-tauri/src/commands.rs around line 39:

`show_overlay` positions the overlay using hard-coded `280x60` dimensions, but these are logical-pixel values. On a scaled display (e.g. 150% or 200% DPI), the window's physical size is larger, so the computed `PhysicalPosition` is visibly off-center and the bottom margin is wrong. The position is derived from `pill_w` and `pill_h` constants instead of the window's actual physical size, so `set_position` receives coordinates that don't match the real window dimensions. Consider scaling the logical dimensions by `monitor.scale_factor()` (or using `LogicalPosition` so the framework converts for you).

let pill_h = 60;
let margin_bottom = 80;
let x = (m_pos.x + (m_size.width as i32 - pill_w) / 2) as f64;
let y = (m_pos.y + m_size.height as i32 - pill_h - margin_bottom) as f64;
let _ = win.set_position(tauri::Position::Physical(tauri::PhysicalPosition {
x: x as i32,
y: y as i32,
}));
}

// Show without stealing focus
#[cfg(target_os = "windows")]
{
use windows::Win32::UI::WindowsAndMessaging::ShowWindow;
use windows::Win32::UI::WindowsAndMessaging::SW_SHOWNOACTIVATE;
if let Some(hwnd) = win.hwnd().ok() {
let _ = unsafe { ShowWindow(hwnd, SW_SHOWNOACTIVATE) };
}
}
#[cfg(not(target_os = "windows"))]
{
let _ = win.show();
}

Ok(())
}

/// Hide the overlay window.
/// The frontend calls this when transitioning back to Ready.
#[tauri::command]
pub fn hide_overlay(app: AppHandle) -> Result<(), String> {
let win = app
.get_webview_window("overlay")
.ok_or_else(|| "Overlay window not found".to_string())?;
let _ = win.hide();
Ok(())
}

// ---------------------------------------------------------------------------
// Platform-specific text insertion
// ---------------------------------------------------------------------------

trait Pipe<T> {
fn pipe<F, R>(self, f: F) -> R
where
F: FnOnce(T) -> R;
}

impl<T> Pipe<T> for T {
fn pipe<F, R>(self, f: F) -> R
where
F: FnOnce(T) -> R,
{
f(self)
}
}

#[cfg(target_os = "windows")]
fn win32_insert(text: &str, hwnd: Option<u64>) -> bool {
use crate::win32;
if let Some(h) = hwnd {
win32::set_foreground_hwnd(h);
std::thread::sleep(std::time::Duration::from_millis(50));
}
if win32::set_clipboard(text) {
std::thread::sleep(std::time::Duration::from_millis(30));
win32::send_ctrl_v();
return true;
}
win32::send_text_unicode(text);
true
}

#[cfg(not(target_os = "windows"))]
fn win32_insert(_text: &str, _hwnd: Option<u64>) -> bool {
false
}

#[cfg(target_os = "linux")]
fn linux_insert(text: &str, hwnd: Option<u64>) -> bool {
use crate::win32;
if let Some(h) = hwnd {
if h != 0 {
win32::set_foreground_hwnd(h);
std::thread::sleep(std::time::Duration::from_millis(50));
}
}
if !win32::set_clipboard(text) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(30));
let is_wayland = std::env::var("WAYLAND_DISPLAY").is_ok();
if is_wayland {
let out = std::process::Command::new("wtype").arg(text).output();
return out.map(|o| o.status.success()).unwrap_or(false);
}
win32::send_ctrl_v();
true
}

#[cfg(not(target_os = "linux"))]
fn linux_insert(_text: &str, _hwnd: Option<u64>) -> bool {
false
}

#[cfg(target_os = "macos")]
fn macos_insert(text: &str) -> bool {
let escaped = text.replace('\\', "\\\\").replace('"', "\\\"");
let script = format!("tell application \"System Events\" to keystroke \"{escaped}\"");
std::process::Command::new("osascript")
.args(["-e", &script])
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}

#[cfg(not(target_os = "macos"))]
fn macos_insert(_text: &str) -> bool {
false
}
148 changes: 73 additions & 75 deletions stt-ui/src-tauri/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
mod widget;
mod commands;
#[cfg(test)]
mod tests;

Expand Down Expand Up @@ -963,7 +964,30 @@ mod win32 {
fn lstrcpyW(lpString1: *mut u16, lpString2: *const u16) -> *mut u16;
}

// SendInput structures
#[repr(C)]
struct KeyBDInput {
wVk: u16,
wScan: u16,
dwFlags: u32,
time: u32,
dwExtraInfo: usize,
}

#[repr(C)]
struct Input {
r#type: u32,
ki: KeyBDInput,
_pad: [u8; 8], // union padding
}

extern "system" {
fn SendInput(cInputs: u32, pInputs: *const Input, cbSize: i32) -> u32;
}

const INPUT_KEYBOARD: u32 = 1;
const KEYEVENTF_KEYUP: u32 = 0x0002;
const KEYEVENTF_UNICODE: u32 = 0x0004;
const VK_CONTROL: u8 = 0x11;
const VK_V: u8 = 0x56;
const CF_UNICODETEXT: u32 = 13;
Expand Down Expand Up @@ -1034,6 +1058,52 @@ mod win32 {
keybd_event(VK_CONTROL, 0, KEYEVENTF_KEYUP, 0);
}
}

/// Type text character-by-character using SendInput with KEYEVENTF_UNICODE.
/// Works in apps that ignore clipboard+Ctrl+V ( terminals, Electron, some UWP apps).
pub fn send_text_unicode(text: &str) {
for ch in text.chars() {
let code = ch as u32;
let mut inputs = [
Input {
r#type: INPUT_KEYBOARD,
ki: KeyBDInput {
wVk: 0,
wScan: code as u16,
dwFlags: KEYEVENTF_UNICODE,
time: 0,
dwExtraInfo: 0,
},
_pad: [0; 8],
},
Input {
r#type: INPUT_KEYBOARD,
ki: KeyBDInput {
wVk: 0,
wScan: code as u16,
dwFlags: KEYEVENTF_UNICODE | KEYEVENTF_KEYUP,
time: 0,
dwExtraInfo: 0,
},
_pad: [0; 8],
},
];
// Handle supplementary plane characters (surrogate pairs)
if code > 0xFFFF {
// Send as two UTF-16 surrogates
let hi = (0xD800 + ((code - 0x10000) >> 10)) as u16;
let lo = (0xDC00 + ((code - 0x10000) & 0x3FF)) as u16;
inputs[0].ki.wScan = hi;
inputs[1].ki.wScan = hi;
unsafe { SendInput(2, inputs.as_ptr(), std::mem::size_of::<Input>() as i32); }
inputs[0].ki.wScan = lo;
inputs[1].ki.wScan = lo;
unsafe { SendInput(2, inputs.as_ptr(), std::mem::size_of::<Input>() as i32); }
} else {
unsafe { SendInput(2, inputs.as_ptr(), std::mem::size_of::<Input>() as i32); }
}
Comment on lines +1064 to +1104

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🌐 Web query:

Win32 SendInput return value number of INPUT events successfully inserted zero failure UIPI

💡 Result:

The Win32 SendInput function returns the number of events that were successfully inserted into the keyboard or mouse input stream [1][2]. If the function returns zero, it indicates that the input was blocked by another thread [1][3]. When SendInput is blocked by User Interface Privilege Isolation (UIPI), the behavior is distinct in the following ways: 1. Return Value: If UIPI blocks the input, SendInput may return zero [4][5]. However, it is important to note that the return value is not a dedicated error code and does not always explicitly identify UIPI as the cause of failure [3][4]. 2. GetLastError: Calling GetLastError after a failure typically returns 5 (ERROR_ACCESS_DENIED) [4], but neither the return value nor GetLastError is guaranteed to indicate that UIPI blocking was the specific cause of the failure [3][4]. Because UIPI restricts input based on integrity levels—where an application can generally only send input to processes of equal or lower integrity—the discrepancy between the number of requested events and the successful return value is often used by developers as an indirect indicator that UIPI has blocked the input [4][6][5]. To bypass these restrictions, applications typically require the 'uiAccess="true"' attribute in their manifest, must be digitally signed, and often need to be installed in a protected location like Program Files [6][7].

Citations:


🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Map the relevant file and inspect the cited region plus the caller.
FILE="stt-ui/src-tauri/src/lib.rs"

echo "== file exists =="
git ls-files "$FILE"

echo
echo "== line count =="
wc -l "$FILE"

echo
echo "== outline around relevant symbols =="
ast-grep outline "$FILE" --view expanded || true

echo
echo "== cited region around send_text_unicode =="
sed -n '1040,1135p' "$FILE" | cat -n

echo
echo "== caller region around type_text =="
sed -n '1200,1255p' "$FILE" | cat -n

echo
echo "== search for send_text_unicode usages =="
rg -n "send_text_unicode|type_text\(" "$FILE"

Repository: IntegerAlex/floure-core

Length of output: 11192


Propagate SendInput failures in the Windows Unicode fallback. send_text_unicode ignores SendInput’s return value, so type_text can still return Ok(true) even when no input was injected. Bubble the status back up at stt-ui/src-tauri/src/lib.rs:1063-1103 and 1226-1227.

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@stt-ui/src-tauri/src/lib.rs` around lines 1063 - 1103, The Windows Unicode
input path in send_text_unicode is ignoring SendInput failures, which can make
type_text report success even when nothing was injected. Update
send_text_unicode to return a status/result from each SendInput call, including
the surrogate-pair branch, and have type_text check and propagate that failure
instead of always returning Ok(true). Use the existing send_text_unicode and
type_text symbols to wire the error/status back to the caller.

}
}
}

#[cfg(not(target_os = "windows"))]
Expand Down Expand Up @@ -1131,80 +1201,6 @@ fn set_foreground_hwnd(hwnd: u64) -> bool {
win32::set_foreground_hwnd(hwnd)
}

/// Type text into the focused input using Win32 clipboard + Ctrl+V.
/// Pure Win32 API — no PowerShell needed for the critical path.
///
/// Flow: restore previous window focus → set clipboard via Win32 → send Ctrl+V via keybd_event
#[tauri::command]
fn type_text(text: String, restore_hwnd: Option<u64>) -> Result<bool, String> {
if text.trim().is_empty() {
return Ok(false);
}
let platform = std::env::consts::OS;
if platform == "windows" {
// Restore focus to the previously-focused window FIRST
if let Some(hwnd) = restore_hwnd {
win32::set_foreground_hwnd(hwnd);
std::thread::sleep(std::time::Duration::from_millis(50));
}
// Set clipboard via Win32 API (no PowerShell overhead)
if !win32::set_clipboard(&text) {
return Err("Failed to set clipboard".into());
}
// Small delay for clipboard to propagate
std::thread::sleep(std::time::Duration::from_millis(30));
// Send Ctrl+V via keybd_event (runs in Tauri's GUI thread — has active message loop)
win32::send_ctrl_v();
return Ok(true);
}
// Linux — clipboard approach (works regardless of which window has focus)
if platform == "linux" {
// Restore focus to the previously-captured window (X11 only — Wayland can't)
if let Some(hwnd) = restore_hwnd {
if hwnd != 0 {
win32::set_foreground_hwnd(hwnd);
std::thread::sleep(std::time::Duration::from_millis(50));
}
}
// Set clipboard — wtype/xdotool will type from clipboard
if !win32::set_clipboard(&text) {
return Err("Failed to set clipboard on Linux".into());
}
std::thread::sleep(std::time::Duration::from_millis(30));
// On X11: send Ctrl+V to paste. On Wayland: wtype to paste from clipboard.
let is_wayland = std::env::var("WAYLAND_DISPLAY").is_ok();
if is_wayland {
// wtype types directly into the focused Wayland window
let out = std::process::Command::new("wtype")
.arg(&text)
.output();
if let Ok(o) = out {
if o.status.success() {
return Ok(true);
} else {
return Err("wtype failed to type text on Wayland".into());
}
} else {
return Err("wtype command failed on Wayland".into());
}
}
// X11: Ctrl+V via xdotool
win32::send_ctrl_v();
return Ok(true);
}
if platform == "macos" {
let escaped = text.replace('\\', "\\\\").replace('"', "\\\"");
let script = format!("tell application \"System Events\" to keystroke \"{escaped}\"");
let out = std::process::Command::new("osascript")
.args(["-e", &script])
.output();
if let Ok(o) = out {
return Ok(o.status.success());
}
}
Err("No typing backend available".into())
}

#[tauri::command]
fn get_backend_path() -> Result<String, AppError> {
let candidates = vec![
Expand Down Expand Up @@ -1418,9 +1414,11 @@ pub fn run() {
toggle_dictionary_favorite,
import_dictionary_csv,
export_dictionary_csv,
type_text,
get_foreground_hwnd,
set_foreground_hwnd,
commands::insert_text,
commands::show_overlay,
commands::hide_overlay,
widget::show_widget,
widget::hide_widget,
widget::get_widget_visible,
Expand Down
Loading