From e907ac66837efa9ba4a9f21370ea5bf54cc8513f Mon Sep 17 00:00:00 2001 From: SWangHash <88996709+SWangHash@users.noreply.github.com> Date: Sun, 2 Aug 2026 18:36:03 +0800 Subject: [PATCH 1/2] feat(ohos): voice input via system speechRecognizer on HarmonyOS Route desktop speech_* commands to the HarmonyOS system speechRecognizer (CoreSpeechKit + AudioKit) via a new ohos ArkTS VoiceInputService + EntryAbility bridge; compile speech_api.rs as cfg-gated ohos/non-ohos versions so the ohos artifact never references local sherpa-onnx models; map language auto->zh-CN (engine rejects auto with 1002200001), overwrite onResult (engine streams full results), finish returns prebuilt result when recognition already completed (avoid session id mismatch); frontend falls back to a no-op recorder when getUserMedia is denied on Tauri (HarmonyOS ArkWeb denies it) and exposes the underlying error; add ohos.permission.MICROPHONE; switch HAP frontend baseline to web-ui. Logs kept at key entry/error points only. --- README.md | 8 +- README.zh-CN.md | 8 +- src/apps/desktop/src/api/speech_api.rs | 233 +++++++++- .../main/ets/entryability/EntryAbility.ets | 32 ++ .../main/ets/services/VoiceInputService.ets | 439 ++++++++++++++++++ src/apps/ohos/entry/src/main/module.json5 | 20 + .../main/resources/base/element/string.json | 4 + .../core/src/util/register_arkts_function.rs | 36 ++ .../components/voice/useComposerVoiceInput.ts | 57 ++- 9 files changed, 806 insertions(+), 31 deletions(-) create mode 100644 src/apps/ohos/entry/src/main/ets/services/VoiceInputService.ets diff --git a/README.md b/README.md index e1370ee920..f93517409f 100644 --- a/README.md +++ b/README.md @@ -109,12 +109,12 @@ npm run build # build backend(.so) in desktop directory cd src/apps/desktop && cargo tauri ohos init && cargo tauri ohos build -# build mini app frontend -cd src/mobile-web && npm run build +# build desktop frontend +cd src/web-ui && npm run build # build app -## copy mini-app resources -1. cp src/mobile-web/dist src/apps/ohos/entry/src/main/resources/resfile +## copy frontend resources +1. cp src/web-ui/dist src/apps/ohos/entry/src/main/resources/resfile ## copy app exe 2. cp target/aarch64-unknow-linux-ohos/release/libbitfun_desktop_lib.so src/apps/ohos/entry/libs/arm64-v8a/libbitfun_desktop_lib.so diff --git a/README.zh-CN.md b/README.zh-CN.md index d6ad883ca0..57c0e8c90a 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -109,12 +109,12 @@ npm run build # 在 desktop 目录编译后端（.so） cd src/apps/desktop && cargo tauri ohos init && cargo tauri ohos build -# 编译 mini app 前端 -cd src/mobile-web && npm run build +# 编译桌面前端 +cd src/web-ui && npm run build # 构建应用 -## 1. 复制 mini-app 资源 -cp src/mobile-web/dist src/apps/ohos/entry/src/main/resources/resfile +## 1. 复制前端资源 +cp src/web-ui/dist src/apps/ohos/entry/src/main/resources/resfile ## 2. 复制应用可执行文件 cp target/aarch64-unknow-linux-ohos/release/libbitfun_desktop_lib.so src/apps/ohos/entry/libs/arm64-v8a/libbitfun_desktop_lib.so diff --git a/src/apps/desktop/src/api/speech_api.rs b/src/apps/desktop/src/api/speech_api.rs index 9edc88ff9c..0111cf7376 100644 --- a/src/apps/desktop/src/api/speech_api.rs +++ b/src/apps/desktop/src/api/speech_api.rs @@ -1,16 +1,55 @@ //! Desktop adapter for local speech input. +//! +//! On the `ohos` target the local sherpa-onnx recognizer is unsupported and +//! the HarmonyOS system `speechRecognizer` is used instead — it ships its own +//! on-device model, so BitFun does not download or manage any speech model on +//! HarmonyOS. To keep the ohos build artifact free of the local model path +//! entirely, every command is compiled as one of two mutually exclusive +//! versions via `#[cfg(target_env = "ohos")]`: +//! +//! - ohos version: the four input-session commands route through +//! [`ohos_speech_call`] to the ArkTS-registered `ohos_speech_*` bridge +//! functions (see +//! `src/apps/ohos/entry/src/main/ets/services/VoiceInputService.ets`); +//! `list_models` reports a single installed system-recognizer stub so the +//! existing frontend gating passes; download/delete/verify reject. +//! - non-ohos version: unchanged — drives `state.speech_service` (sherpa-onnx). -use crate::api::AppState; use bitfun_core_types::speech::{ SpeechAppendAudioChunkRequest, SpeechAppendAudioChunkResponse, SpeechCancelInputSessionRequest, SpeechCancelModelDownloadRequest, SpeechDeleteModelRequest, SpeechDownloadModelRequest, SpeechFinishInputSessionRequest, SpeechInputSession, SpeechListModelsResponse, - SpeechModelProgressEvent, SpeechModelStatus, SpeechStartInputSessionRequest, - SpeechTranscriptionResult, SpeechVerifyModelRequest, + SpeechModelStatus, SpeechStartInputSessionRequest, SpeechTranscriptionResult, + SpeechVerifyModelRequest, }; + +#[cfg(target_env = "ohos")] +use bitfun_core::util::ohos_speech_call; +#[cfg(target_env = "ohos")] +use bitfun_core_types::speech::SpeechModelInstallState; + +#[cfg(not(target_env = "ohos"))] +use crate::api::AppState; +#[cfg(not(target_env = "ohos"))] +use bitfun_core_types::speech::SpeechModelProgressEvent; +#[cfg(not(target_env = "ohos"))] use bitfun_events::{SPEECH_MODEL_PROGRESS_EVENT, SPEECH_MODEL_STATUS_CHANGED_EVENT}; +#[cfg(not(target_env = "ohos"))] use tauri::{AppHandle, Emitter, State}; +// ---------------------------------------------------------------------------- +// speech_list_models +// ---------------------------------------------------------------------------- + +#[cfg(target_env = "ohos")] +#[tauri::command] +pub async fn speech_list_models() -> Result { + Ok(SpeechListModelsResponse { + models: vec![ohos_system_model_status()], + }) +} + +#[cfg(not(target_env = "ohos"))] #[tauri::command] pub async fn speech_list_models( state: State<'_, AppState>, @@ -22,6 +61,22 @@ pub async fn speech_list_models( .map_err(|error| format!("Failed to list speech models: {error}")) } +// ---------------------------------------------------------------------------- +// speech_download_model +// ---------------------------------------------------------------------------- + +#[cfg(target_env = "ohos")] +#[tauri::command] +pub async fn speech_download_model( + request: SpeechDownloadModelRequest, +) -> Result { + Err(format!( + "Speech model download is not supported on HarmonyOS; the system speechRecognizer ships its own on-device model (model_id={}).", + request.model_id + )) +} + +#[cfg(not(target_env = "ohos"))] #[tauri::command] pub async fn speech_download_model( state: State<'_, AppState>, @@ -42,6 +97,22 @@ pub async fn speech_download_model( Ok(status) } +// ---------------------------------------------------------------------------- +// speech_cancel_model_download +// ---------------------------------------------------------------------------- + +#[cfg(target_env = "ohos")] +#[tauri::command] +pub async fn speech_cancel_model_download( + request: SpeechCancelModelDownloadRequest, +) -> Result { + Err(format!( + "Speech model download is not supported on HarmonyOS; nothing to cancel (model_id={}).", + request.model_id + )) +} + +#[cfg(not(target_env = "ohos"))] #[tauri::command] pub async fn speech_cancel_model_download( state: State<'_, AppState>, @@ -57,6 +128,22 @@ pub async fn speech_cancel_model_download( Ok(status) } +// ---------------------------------------------------------------------------- +// speech_delete_model +// ---------------------------------------------------------------------------- + +#[cfg(target_env = "ohos")] +#[tauri::command] +pub async fn speech_delete_model( + request: SpeechDeleteModelRequest, +) -> Result { + Err(format!( + "Speech model management is not supported on HarmonyOS; the system speechRecognizer ships its own on-device model (model_id={}).", + request.model_id + )) +} + +#[cfg(not(target_env = "ohos"))] #[tauri::command] pub async fn speech_delete_model( state: State<'_, AppState>, @@ -72,6 +159,22 @@ pub async fn speech_delete_model( Ok(status) } +// ---------------------------------------------------------------------------- +// speech_verify_model +// ---------------------------------------------------------------------------- + +#[cfg(target_env = "ohos")] +#[tauri::command] +pub async fn speech_verify_model( + request: SpeechVerifyModelRequest, +) -> Result { + Err(format!( + "Speech model verification is not supported on HarmonyOS; the system speechRecognizer ships its own on-device model (model_id={}).", + request.model_id + )) +} + +#[cfg(not(target_env = "ohos"))] #[tauri::command] pub async fn speech_verify_model( state: State<'_, AppState>, @@ -87,6 +190,46 @@ pub async fn speech_verify_model( Ok(status) } +// ---------------------------------------------------------------------------- +// speech_start_input_session +// ---------------------------------------------------------------------------- + +#[cfg(target_env = "ohos")] +#[tauri::command] +pub async fn speech_start_input_session( + request: SpeechStartInputSessionRequest, +) -> Result { + log::info!("[speech] start_input_session ohos branch, request={:?}", request); + let payload = serde_json::to_string(&request) + .map_err(|error| { + log::error!("[speech] failed to encode start request: {}", error); + format!("Failed to encode speech start request: {error}") + })?; + let response = match ohos_speech_call("ohos_speech_start", &payload).await { + Ok(r) => r, + Err(e) => { + log::error!("[speech] ohos_speech_start returned error: {}", e); + return Err(format!("Failed to start speech input session: {e}")); + } + }; + log::info!("[speech] ohos_speech_start response: {}", response); + let value: serde_json::Value = serde_json::from_str(&response) + .map_err(|error| { + log::error!("[speech] failed to parse start response json: {} | response={}", error, response); + format!("Invalid speech start response from ArkTS: {error}: {response}") + })?; + if let Some(err_msg) = value.get("__error").and_then(|v| v.as_str()) { + log::error!("[speech] ArkTS speech start failed: {}", err_msg); + return Err(format!("ArkTS speech start failed: {err_msg}")); + } + serde_json::from_value::(value) + .map_err(|error| { + log::error!("[speech] failed to parse start session: {}", error); + format!("Invalid speech start session from ArkTS: {error}") + }) +} + +#[cfg(not(target_env = "ohos"))] #[tauri::command] pub async fn speech_start_input_session( state: State<'_, AppState>, @@ -99,6 +242,23 @@ pub async fn speech_start_input_session( .map_err(|error| format!("Failed to start speech input session: {error}")) } +// ---------------------------------------------------------------------------- +// speech_append_audio_chunk +// ---------------------------------------------------------------------------- + +#[cfg(target_env = "ohos")] +#[tauri::command] +pub async fn speech_append_audio_chunk( + request: SpeechAppendAudioChunkRequest, +) -> Result { + let payload = serde_json::to_string(&request) + .map_err(|error| format!("Failed to encode speech append request: {error}"))?; + let response = ohos_speech_call("ohos_speech_append", &payload).await?; + serde_json::from_str::(&response) + .map_err(|error| format!("Invalid speech append response from ArkTS: {error}: {response}")) +} + +#[cfg(not(target_env = "ohos"))] #[tauri::command] pub async fn speech_append_audio_chunk( state: State<'_, AppState>, @@ -111,6 +271,23 @@ pub async fn speech_append_audio_chunk( .map_err(|error| format!("Failed to append speech audio chunk: {error}")) } +// ---------------------------------------------------------------------------- +// speech_finish_input_session +// ---------------------------------------------------------------------------- + +#[cfg(target_env = "ohos")] +#[tauri::command] +pub async fn speech_finish_input_session( + request: SpeechFinishInputSessionRequest, +) -> Result { + let payload = serde_json::to_string(&request) + .map_err(|error| format!("Failed to encode speech finish request: {error}"))?; + let response = ohos_speech_call("ohos_speech_finish", &payload).await?; + serde_json::from_str::(&response) + .map_err(|error| format!("Invalid speech transcription response from ArkTS: {error}: {response}")) +} + +#[cfg(not(target_env = "ohos"))] #[tauri::command] pub async fn speech_finish_input_session( state: State<'_, AppState>, @@ -123,6 +300,22 @@ pub async fn speech_finish_input_session( .map_err(|error| format!("Failed to transcribe speech input: {error}")) } +// ---------------------------------------------------------------------------- +// speech_cancel_input_session +// ---------------------------------------------------------------------------- + +#[cfg(target_env = "ohos")] +#[tauri::command] +pub async fn speech_cancel_input_session( + request: SpeechCancelInputSessionRequest, +) -> Result<(), String> { + let payload = serde_json::to_string(&request) + .map_err(|error| format!("Failed to encode speech cancel request: {error}"))?; + let _ = ohos_speech_call("ohos_speech_cancel", &payload).await?; + Ok(()) +} + +#[cfg(not(target_env = "ohos"))] #[tauri::command] pub async fn speech_cancel_input_session( state: State<'_, AppState>, @@ -135,8 +328,42 @@ pub async fn speech_cancel_input_session( .map_err(|error| format!("Failed to cancel speech input session: {error}")) } +// ---------------------------------------------------------------------------- +// helpers (cfg-gated to the version that uses them) +// ---------------------------------------------------------------------------- + +#[cfg(not(target_env = "ohos"))] fn emit_status(app: &AppHandle, status: &SpeechModelStatus) { if let Err(error) = app.emit(SPEECH_MODEL_STATUS_CHANGED_EVENT, status) { log::warn!("Failed to emit speech model status event: {error}"); } } + +/// Synthetic `Installed` model status returned by `speech_list_models` on the +/// `ohos` target. +/// +/// The frontend (`useComposerVoiceInput`) gates recording on +/// `modelInstalled === true` for the configured local model id. On HarmonyOS +/// there is no downloadable model — the system `speechRecognizer` ships its +/// own — so we report the default local model id (`sensevoice-small-int8`) as +/// installed to keep the existing frontend gating logic passing without any +/// frontend change. Downloads/deletes/verifies are rejected by the other +/// ohos commands, so no model file is ever touched. +#[cfg(target_env = "ohos")] +fn ohos_system_model_status() -> SpeechModelStatus { + SpeechModelStatus { + model_id: "sensevoice-small-int8".to_string(), + display_name: "HarmonyOS System Speech Recognition".to_string(), + provider: "ohos-system".to_string(), + version: "1".to_string(), + description: "On-device speech recognition provided by the HarmonyOS system speechRecognizer.".to_string(), + languages: vec!["zh-CN".to_string(), "en-US".to_string()], + state: SpeechModelInstallState::Installed, + installed_path: None, + installed_bytes: 0, + expected_bytes: 0, + progress: None, + error: None, + } +} + diff --git a/src/apps/ohos/entry/src/main/ets/entryability/EntryAbility.ets b/src/apps/ohos/entry/src/main/ets/entryability/EntryAbility.ets index e6a067a4bf..a5fc4ab474 100644 --- a/src/apps/ohos/entry/src/main/ets/entryability/EntryAbility.ets +++ b/src/apps/ohos/entry/src/main/ets/entryability/EntryAbility.ets @@ -19,6 +19,7 @@ import RustModule from 'libbitfun_desktop_lib.so'; import { AppUpdater, UpdateCheckResult } from '../utils/AppUpdater'; import { CommonUtils } from '../utils/CommonUtils'; import { runDeveco } from '../utils/DevecoStart'; +import { VoiceInputService } from '../services/VoiceInputService'; const DOMAIN = 0x0000; @@ -29,6 +30,7 @@ export default class EntryAbility extends RustAbility { public commonEventListener: CommonEventListener | undefined = undefined; public remoteUrl: string = ""; private appUpdater: AppUpdater = new AppUpdater(); + private voiceInputService: VoiceInputService = new VoiceInputService(); public shareStatus: boolean = false; async onCreate(want: Want, launchParam: AbilityConstant.LaunchParam): Promise { @@ -219,6 +221,25 @@ export default class EntryAbility extends RustAbility { } return ''; }); + this.voiceInputService.setContext(this.context); + RustModule.registerArktsFunction('ohos_speech_start', async (err: Error, arg: string): Promise => { + try { + return await this.voiceInputService.start(arg); + } catch (e) { + const detail: string = describeError(e); + hilog.error(0x0000, 'BitFunVoiceInput', 'ohos_speech_start callback error: %{public}s', detail); + return JSON.stringify({ '__error': detail }); + } + }); + RustModule.registerArktsFunction('ohos_speech_append', async (err: Error, arg: string): Promise => { + return await this.voiceInputService.append(arg); + }); + RustModule.registerArktsFunction('ohos_speech_finish', async (err: Error, arg: string): Promise => { + return await this.voiceInputService.finish(arg); + }); + RustModule.registerArktsFunction('ohos_speech_cancel', async (err: Error, _arg: string): Promise => { + return await this.voiceInputService.cancel(''); + }); const autoUpdateEnabled = RustModule.getAppConfigBool('app.auto_update'); hilog.info(0x0000, 'vnext', 'autoUpdateEnabled: ' + autoUpdateEnabled); if (autoUpdateEnabled) { @@ -264,6 +285,17 @@ export const AppGlobal: AppGlobalType = { mContext: null } +function describeError(e: Object): string { + if (e instanceof Error) { + return `Error: ${e.message}`; + } + try { + return `json: ${JSON.stringify(e)}`; + } catch (_jsonErr) { + return `str: ${e}`; + } +} + export async function createMeetingEvent(input: string): Promise { interface CalendarInfo { title: string, diff --git a/src/apps/ohos/entry/src/main/ets/services/VoiceInputService.ets b/src/apps/ohos/entry/src/main/ets/services/VoiceInputService.ets new file mode 100644 index 0000000000..ca29fbb790 --- /dev/null +++ b/src/apps/ohos/entry/src/main/ets/services/VoiceInputService.ets @@ -0,0 +1,439 @@ +import { abilityAccessCtrl, Context, Permissions } from '@kit.AbilityKit'; +import { audio } from '@kit.AudioKit'; +import { speechRecognizer } from '@kit.CoreSpeechKit'; +import { BusinessError } from '@kit.BasicServicesKit'; +import { hilog } from '@kit.PerformanceAnalysisKit'; + +const VOICE_DOMAIN: number = 0x0000; +const VOICE_TAG: string = 'BitFunVoiceInput'; +const MICROPHONE_PERMISSION: Permissions = 'ohos.permission.MICROPHONE'; +const AUDIO_CHUNK_BYTES = 1280; +const HALF_AUDIO_CHUNK_BYTES = 640; +const DEFAULT_LANGUAGE = 'zh-CN'; +const DEFAULT_SAMPLE_RATE = 16000; +const DEFAULT_MAX_RECORDING_SECONDS = 60; +const ENGINE_VAD_BEGIN = 2000; +const ENGINE_VAD_END = 3000; +const ENGINE_MAX_AUDIO_DURATION = 55000; + +interface VoiceStartRequest { + language?: string; + sampleRate?: number; + maxRecordingSeconds?: number; + modelId?: string; +} + +interface VoiceSessionInfo { + sessionId: string; + modelId: string; + language: string; + sampleRate: number; + maxRecordingSeconds: number; +} + +interface VoiceTranscriptionResult { + text: string; + language: string; + durationMs: number; + audioDurationSeconds: number; +} + +interface VoiceAppendRequest { + sessionId: string; + pcm16Base64: string; +} + +interface VoiceAppendResult { + receivedBytes: number; + receivedSeconds: number; + limitReached: boolean; +} + +/** + * HarmonyOS system speech input service. + * + * Wraps the on-device `speechRecognizer` engine and `AudioCapturer` into the + * start/append/finish/cancel Promise contract used by the desktop speech_* + * Tauri commands. The ArkTS side owns audio capture and recognition; the + * frontend-supplied PCM chunks in `append` are intentionally discarded because + * the system engine captures audio itself. + */ +export class VoiceInputService { + private context?: Context; + private engine?: speechRecognizer.SpeechRecognitionEngine; + private capturer?: audio.AudioCapturer; + private sessionId: string = ''; + private language: string = DEFAULT_LANGUAGE; + private sampleRate: number = DEFAULT_SAMPLE_RATE; + private maxRecordingSeconds: number = DEFAULT_MAX_RECORDING_SECONDS; + private listening: boolean = false; + private reading: boolean = false; + private idCounter: number = 0; + private startedAt: number = 0; + private capturedSeconds: number = 0; + private finalText: string = ''; + private finishedResult: string = ''; + private hasFinishedResult: boolean = false; + private finishResolver: ((result: string) => void) | null = null; + private finishRejecter: ((error: Error) => void) | null = null; + + async start(arg: string): Promise { + await this.cancelInternal(false); + hilog.info(VOICE_DOMAIN, VOICE_TAG, 'start begin: arg=%{public}s', arg); + const context = this.context; + if (!context) { + throw new Error('Voice input service is not bound to a context'); + } + await ensureMicrophonePermission(context); + const req: VoiceStartRequest = JSON.parse(arg); + this.language = this.resolveLanguage(req.language); + this.sampleRate = req.sampleRate ?? DEFAULT_SAMPLE_RATE; + this.maxRecordingSeconds = req.maxRecordingSeconds ?? DEFAULT_MAX_RECORDING_SECONDS; + const modelId: string = req.modelId ?? 'ohos-system'; + this.sessionId = this.newSessionId(); + this.finalText = ''; + this.finishedResult = ''; + this.hasFinishedResult = false; + this.finishResolver = null; + this.finishRejecter = null; + this.capturedSeconds = 0; + this.startedAt = Date.now(); + try { + this.engine = await speechRecognizer.createEngine({ + language: this.language, + online: 1, + extraParams: { + locate: 'CN', + recognizerMode: 'short' + } + }); + this.engine.setListener(this.buildRecognitionListener()); + this.capturer = await this.createCapturer(); + this.engine.startListening({ + sessionId: this.sessionId, + audioInfo: { + audioType: 'pcm', + sampleRate: 16000, + soundChannel: 1, + sampleBit: 16 + }, + extraParams: { + recognitionMode: 1, + vadBegin: ENGINE_VAD_BEGIN, + vadEnd: ENGINE_VAD_END, + maxAudioDuration: ENGINE_MAX_AUDIO_DURATION + } + }); + await this.capturer.start(); + this.listening = true; + this.reading = true; + this.captureLoop(this.sessionId); + } catch (err) { + const failErr: Error = this.toError(err); + hilog.error(VOICE_DOMAIN, VOICE_TAG, 'voice input start failed: %{public}s', failErr.message); + await this.cancelInternal(true); + throw failErr; + } + const info: VoiceSessionInfo = { + sessionId: this.sessionId, + modelId: modelId, + language: this.language, + sampleRate: this.sampleRate, + maxRecordingSeconds: this.maxRecordingSeconds + }; + return JSON.stringify(info); + } + + async append(arg: string): Promise { + const req: VoiceAppendRequest = JSON.parse(arg); + const receivedBytes: number = this.estimateBase64Bytes(req.pcm16Base64); + const receivedSeconds: number = this.sampleRate > 0 + ? receivedBytes / (this.sampleRate * 2) + : 0; + this.capturedSeconds += receivedSeconds; + const limitReached: boolean = this.capturedSeconds >= this.maxRecordingSeconds; + const result: VoiceAppendResult = { + receivedBytes: receivedBytes, + receivedSeconds: receivedSeconds, + limitReached: limitReached + }; + return JSON.stringify(result); + } + + async finish(arg: string): Promise { + const sessionId: string = JSON.parse(arg).sessionId as string; + if (this.hasFinishedResult) { + const prebuilt: string = this.finishedResult; + this.hasFinishedResult = false; + this.finishedResult = ''; + await this.releaseCapturer(); + this.releaseEngine(); + return prebuilt; + } + const engine = this.engine; + if (!engine || sessionId !== this.sessionId) { + throw new Error('Voice input session is not active'); + } + await this.releaseCapturer(); + try { + engine.finish(this.sessionId); + } catch (err) { + const failErr: Error = this.toError(err); + hilog.error(VOICE_DOMAIN, VOICE_TAG, 'voice input finish failed: %{public}s', failErr.message); + this.releaseEngine(); + throw failErr; + } + const resultJson: string = await new Promise((resolve, reject): void => { + this.finishResolver = resolve; + this.finishRejecter = reject; + }); + this.releaseEngine(); + return resultJson; + } + + async cancel(_arg: string): Promise { + await this.cancelInternal(false); + return ''; + } + + private buildRecognitionListener(): speechRecognizer.RecognitionListener { + return { + onStart: (_sessionId: string, _eventMessage: string): void => { + }, + onEvent: (_sessionId: string, _eventCode: number, _eventMessage: string): void => { + }, + onResult: (_sessionId: string, result: speechRecognizer.SpeechRecognitionResult): void => { + const text: string = result.result.trim(); + if (text.length > 0) { + this.finalText = text; + } + if (result.isLast) { + this.resolveFinish(); + } + }, + onComplete: (_sessionId: string, _eventMessage: string): void => { + this.resolveFinish(); + }, + onError: (_sessionId: string, errorCode: number, errorMessage: string): void => { + const error: Error = new Error(errorMessage.length > 0 + ? errorMessage + : `Voice input failed (${errorCode})`); + this.listening = false; + this.reading = false; + if (this.finishRejecter) { + const rejecter = this.finishRejecter; + this.finishRejecter = null; + this.finishResolver = null; + rejecter(error); + } else { + this.hasFinishedResult = true; + this.finishedResult = ''; + } + this.releaseCapturer(); + this.releaseEngine(); + } + }; + } + + private resolveFinish(): void { + this.listening = false; + this.reading = false; + const result: VoiceTranscriptionResult = { + text: this.finalText, + language: this.language, + durationMs: Date.now() - this.startedAt, + audioDurationSeconds: this.capturedSeconds + }; + const resultJson: string = JSON.stringify(result); + this.finishedResult = resultJson; + this.hasFinishedResult = true; + if (this.finishResolver) { + const resolver = this.finishResolver; + this.finishResolver = null; + this.finishRejecter = null; + resolver(resultJson); + } + this.releaseCapturer(); + } + + private async captureLoop(sessionId: string): Promise { + while (this.reading && this.sessionId === sessionId) { + const capturer = this.capturer; + if (!capturer) { + break; + } + try { + const buffer = await capturer.read(AUDIO_CHUNK_BYTES, true); + if (!this.reading || this.sessionId !== sessionId) { + break; + } + this.writeAudioBuffer(sessionId, buffer); + } catch (err) { + this.reading = false; + if (this.finishRejecter) { + const rejecter = this.finishRejecter; + this.finishRejecter = null; + this.finishResolver = null; + rejecter(this.toError(err)); + } + await this.cancelInternal(true); + break; + } + } + } + + private writeAudioBuffer(sessionId: string, buffer: ArrayBuffer): void { + const engine = this.engine; + if (!engine || buffer.byteLength === 0) { + return; + } + const data: Uint8Array = new Uint8Array(buffer); + let offset = 0; + while (offset + AUDIO_CHUNK_BYTES <= data.length) { + engine.writeAudio(sessionId, data.subarray(offset, offset + AUDIO_CHUNK_BYTES)); + offset += AUDIO_CHUNK_BYTES; + } + const remaining: number = data.length - offset; + if (remaining === HALF_AUDIO_CHUNK_BYTES) { + engine.writeAudio(sessionId, data.subarray(offset, offset + HALF_AUDIO_CHUNK_BYTES)); + } + } + + private async createCapturer(): Promise { + try { + return await audio.createAudioCapturer(this.capturerOptions(audio.SourceType.SOURCE_TYPE_VOICE_RECOGNITION)); + } catch (_err) { + return await audio.createAudioCapturer(this.capturerOptions(audio.SourceType.SOURCE_TYPE_MIC)); + } + } + + private capturerOptions(source: audio.SourceType): audio.AudioCapturerOptions { + return { + streamInfo: { + samplingRate: audio.AudioSamplingRate.SAMPLE_RATE_16000, + channels: audio.AudioChannel.CHANNEL_1, + sampleFormat: audio.AudioSampleFormat.SAMPLE_FORMAT_S16LE, + encodingType: audio.AudioEncodingType.ENCODING_TYPE_RAW + }, + capturerInfo: { + source: source, + capturerFlags: 0 + } + }; + } + + private async cancelInternal(notifyError: boolean): Promise { + const engine = this.engine; + const sessionId = this.sessionId; + this.listening = false; + this.reading = false; + const rejecter = this.finishRejecter; + this.finishResolver = null; + this.finishRejecter = null; + await this.releaseCapturer(); + if (engine && sessionId.length > 0) { + try { + engine.cancel(sessionId); + } catch (_err) { + } + } + this.releaseEngine(); + if (notifyError && rejecter) { + rejecter(new Error('Voice input cancelled')); + } + } + + private async releaseCapturer(): Promise { + const capturer = this.capturer; + this.capturer = undefined; + if (!capturer) { + return; + } + try { + await capturer.stop(); + } catch (_err) { + } + try { + await capturer.release(); + } catch (_err) { + } + } + + private releaseEngine(): void { + const engine = this.engine; + this.engine = undefined; + this.sessionId = ''; + if (!engine) { + return; + } + try { + engine.shutdown(); + } catch (_err) { + } + } + + setContext(context: Context): void { + this.context = context; + } + + /** + * Map the frontend-provided language to a value accepted by the system + * speechRecognizer. The engine rejects "auto" (and other non-BCP-47 values) + * with code 1002200001 "Asr CreateEngineParams is wrong!", so normalize to + * the default concrete language. + */ + private resolveLanguage(language?: string): string { + if (language === undefined || language === null || language.length === 0 || language === 'auto') { + return DEFAULT_LANGUAGE; + } + return language; + } + + private newSessionId(): string { + this.idCounter += 1; + return `voice-${Date.now()}-${this.idCounter}`; + } + + private estimateBase64Bytes(base64: string): number { + if (base64.length === 0) { + return 0; + } + let padding = 0; + if (base64.endsWith('==')) { + padding = 2; + } else if (base64.endsWith('=')) { + padding = 1; + } + const decoded: number = Math.floor((base64.length * 3) / 4); + return decoded > padding ? decoded - padding : 0; + } + + private toError(err: Object): Error { + if (err instanceof Error && err.message.length > 0) { + return err; + } + const business = err as BusinessError; + if (business && business.message && business.message.length > 0) { + return new Error(business.message); + } + try { + return new Error(JSON.stringify(err)); + } catch (_jsonErr) { + return new Error(`${err}`); + } + } +} + +/** + * Request microphone permission from the user. Must be called from an + * ability context (the EntryAbility registers the ArkTS bridge with the + * ability context attached so the service can request permissions on demand). + */ +export async function ensureMicrophonePermission(context: Context): Promise { + const atManager = abilityAccessCtrl.createAtManager(); + const result = await atManager.requestPermissionsFromUser(context, [MICROPHONE_PERMISSION]); + if (result.authResults.length === 0 || + result.authResults[0] !== abilityAccessCtrl.GrantStatus.PERMISSION_GRANTED) { + throw new Error('Microphone permission denied'); + } +} diff --git a/src/apps/ohos/entry/src/main/module.json5 b/src/apps/ohos/entry/src/main/module.json5 index fbdf19a351..819589a728 100644 --- a/src/apps/ohos/entry/src/main/module.json5 +++ b/src/apps/ohos/entry/src/main/module.json5 @@ -64,6 +64,16 @@ "when": "always" } }, + { + "name": "ohos.permission.READ_WRITE_USB_DEV", + "reason": "$string:permission_desktop_directory_reason", + "usedScene": { + "abilities": [ + "EntryAbility" + ], + "when": "always" + } + }, { "name": "ohos.permission.READ_WRITE_DESKTOP_DIRECTORY", "reason": "$string:permission_desktop_directory_reason", @@ -111,6 +121,16 @@ ], "when": "always" } + }, + { + "name": "ohos.permission.MICROPHONE", + "reason": "$string:permission_microphone_reason", + "usedScene": { + "abilities": [ + "EntryAbility" + ], + "when": "always" + } } ], "extensionAbilities": [ diff --git a/src/apps/ohos/entry/src/main/resources/base/element/string.json b/src/apps/ohos/entry/src/main/resources/base/element/string.json index 3dc48114b2..3364b16c64 100644 --- a/src/apps/ohos/entry/src/main/resources/base/element/string.json +++ b/src/apps/ohos/entry/src/main/resources/base/element/string.json @@ -50,6 +50,10 @@ { "name": "permission_read_pasteboard_reason", "value": "Used to read clipboard contents for terminal copy and paste" + }, + { + "name": "permission_microphone_reason", + "value": "Used to capture microphone audio for voice input transcription in the chat composer" } ] } \ No newline at end of file diff --git a/src/crates/assembly/core/src/util/register_arkts_function.rs b/src/crates/assembly/core/src/util/register_arkts_function.rs index dd0844d730..f001cae47e 100644 --- a/src/crates/assembly/core/src/util/register_arkts_function.rs +++ b/src/crates/assembly/core/src/util/register_arkts_function.rs @@ -136,3 +136,39 @@ fn parse_picker_result(json: &str) -> Result { fn parse_paths_result(json: &str) -> Result, String> { parse_paths_envelope(json, "get_clipboard_files") } + +/// Invoke an ArkTS-registered speech bridge function by name. +/// +/// Mirrors [`open_dialog_file`]: looks up the `ThreadsafeFunction` registered +/// on the ArkTS side via `RustModule.registerArktsFunction(name, ...)`, calls +/// it with a JSON string argument, and awaits the returned `Promise`. +/// Used by the ohos branches of the `speech_*` Tauri commands to route voice +/// input to the HarmonyOS system `speechRecognizer` (via +/// `src/apps/ohos/.../services/VoiceInputService.ets`) instead of the local +/// sherpa-onnx recognizer, which is unsupported on the ohos target. +pub async fn ohos_speech_call(name: &str, json: &str) -> Result { + let function = { + let lock = JS_THREADSAFE_FUNCTION.read(); + lock.get(name).cloned() + }; + + let Some(function) = function else { + log::error!("[ohos_speech] {} not registered by ArkTS", name); + return Err(format!("{name} has not been registered by ArkTS")); + }; + + let res = function.call_async(Ok(json.to_string())).await; + match res { + Ok(promise) => match promise.await { + Ok(json) => Ok(json), + Err(err) => { + log::error!("[ohos_speech] {} promise rejected: {} | {:?}", name, err.to_string(), err); + Err(err.to_string()) + } + }, + Err(err) => { + log::error!("[ohos_speech] {} call_async failed: {} | {:?}", name, err.to_string(), err); + Err(err.to_string()) + } + } +} diff --git a/src/web-ui/src/flow_chat/components/voice/useComposerVoiceInput.ts b/src/web-ui/src/flow_chat/components/voice/useComposerVoiceInput.ts index a334326661..f1efa2b6bd 100644 --- a/src/web-ui/src/flow_chat/components/voice/useComposerVoiceInput.ts +++ b/src/web-ui/src/flow_chat/components/voice/useComposerVoiceInput.ts @@ -55,6 +55,13 @@ function isMediaCaptureSupported(): boolean { return typeof navigator !== 'undefined' && Boolean(navigator.mediaDevices?.getUserMedia); } +function isPermissionDeniedError(error: unknown): boolean { + return error instanceof DOMException && ( + error.name === 'NotAllowedError' || + error.name === 'PermissionDeniedError' + ); +} + function resolveErrorMessage(error: unknown, permissionDenied: string, fallback: string): string { if (error instanceof DOMException && ( error.name === 'NotAllowedError' || @@ -62,7 +69,8 @@ function resolveErrorMessage(error: unknown, permissionDenied: string, fallback: )) { return permissionDenied; } - return fallback; + const detail = error instanceof Error ? error.message : String(error); + return detail ? `${fallback} (${detail})` : fallback; } function isModelMissingError(error: unknown): boolean { @@ -426,7 +434,7 @@ export function useComposerVoiceInput({ notificationService.info(t('input.voiceInput.disabled')); return; } - if (!speechRuntimeSupported || !isMediaCaptureSupported()) { + if (!speechRuntimeSupported || (!isTauriRuntime() && !isMediaCaptureSupported())) { notificationService.error(t('input.voiceInput.unsupported')); return; } @@ -463,22 +471,31 @@ export function useComposerVoiceInput({ } log.debug('Voice input startup requested', { modelInstalled }); - const recorder = await createVoiceInputRecorder({ - targetSampleRate: DEFAULT_SPEECH_SAMPLE_RATE, - chunkDurationMs: RECORDING_CHUNK_DURATION_MS, - microphoneDeviceId: voiceSettings.microphone_device_id || undefined, - onChunk: enqueueChunk, - onLevel: updateAudioLevel, - onDeviceEnded: () => { - if (activeRecordingIdRef.current !== recordingId) return; - log.warn('Voice input microphone disconnected during recording'); - notificationService.error(t('input.voiceInput.deviceDisconnected')); - void cancelRecordingRef.current?.(); - }, - onStartupTiming: timing => { - log.debug('Voice input recorder startup stage completed', timing); - }, - }); + let recorder: VoiceInputRecorder = { stop: async () => {} }; + try { + recorder = await createVoiceInputRecorder({ + targetSampleRate: DEFAULT_SPEECH_SAMPLE_RATE, + chunkDurationMs: RECORDING_CHUNK_DURATION_MS, + microphoneDeviceId: voiceSettings.microphone_device_id || undefined, + onChunk: enqueueChunk, + onLevel: updateAudioLevel, + onDeviceEnded: () => { + if (activeRecordingIdRef.current !== recordingId) return; + log.warn('Voice input microphone disconnected during recording'); + notificationService.error(t('input.voiceInput.deviceDisconnected')); + void cancelRecordingRef.current?.(); + }, + onStartupTiming: timing => { + log.debug('Voice input recorder startup stage completed', timing); + }, + }); + } catch (micError) { + if (isTauriRuntime() && isPermissionDeniedError(micError)) { + log.info('Voice input getUserMedia denied on this runtime; falling back to the system speechRecognizer'); + } else { + throw micError; + } + } if (activeRecordingIdRef.current !== recordingId) { await recorder.stop().catch(error => { log.warn('Failed to stop stale voice recorder', { error }); @@ -624,10 +641,10 @@ export function useComposerVoiceInput({ const disabled = phase === 'recording' ? false - : !settings?.enabled || !speechRuntimeSupported || !isMediaCaptureSupported() || phase === 'preparing' || phase === 'transcribing'; + : !settings?.enabled || !speechRuntimeSupported || (!isTauriRuntime() && !isMediaCaptureSupported()) || phase === 'preparing' || phase === 'transcribing'; const tooltip = useMemo(() => { if (!settings?.enabled) return t('input.voiceInput.disabled'); - if (!speechRuntimeSupported || !isMediaCaptureSupported()) return t('input.voiceInput.unsupported'); + if (!speechRuntimeSupported || (!isTauriRuntime() && !isMediaCaptureSupported())) return t('input.voiceInput.unsupported'); if (settings.provider === 'cloud') return t('input.voiceInput.cloudPending'); if (modelInstalled === false) return t('input.voiceInput.modelMissing'); if (phase === 'preparing') return t('input.voiceInput.preparing'); From 6cfef02ea355221178b4aadf84cc0313ca10be71 Mon Sep 17 00:00:00 2001 From: SWangHash <88996709+SWangHash@users.noreply.github.com> Date: Sun, 2 Aug 2026 18:50:53 +0800 Subject: [PATCH 2/2] i18n(ohos): localize microphone permission reason to zh-CN --- .../ohos/entry/src/main/resources/zh_CN/element/string.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/apps/ohos/entry/src/main/resources/zh_CN/element/string.json b/src/apps/ohos/entry/src/main/resources/zh_CN/element/string.json index e8b83812da..b0a52c3c7f 100644 --- a/src/apps/ohos/entry/src/main/resources/zh_CN/element/string.json +++ b/src/apps/ohos/entry/src/main/resources/zh_CN/element/string.json @@ -51,6 +51,10 @@ { "name": "permission_read_pasteboard_reason", "value": "用于读取剪贴板内容以进行终端复制和粘贴" + }, + { + "name": "permission_microphone_reason", + "value": "用于采集麦克风音频以进行对话框语音输入转写" } ] } \ No newline at end of file