diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index 84a37b5..c3c16aa 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -142,20 +142,23 @@ shell = "bash -euo pipefail -c" [tasks.pytest-cov] description = "Combined Python coverage (pytest + web-runner Pyodide runs) -> Cobertura for Codecov/DeepSource" # One Python report is combined from several data sources: the pytest runs plus the web-runner Pyodide runs. -# The pyface1 + pyeye1 pytest runs and the web-runner integration test (which drops the modules' .coverage files -# into target/pycov/ when ET_TEST_COVERAGE is set) all feed `coverage combine`, which merges them via -# config/coverage.toml's `[paths]` remap into a single Cobertura file. pytest-cov is a plugin, so it rides in -# each module's uv `dev` group -- the same venv as pytest -- and `uv run` (a mise-managed tool) installs both -# into the one project venv. UV_PYTHON mirrors test-pyface1's mise-CPython pin. +# The pyface1 + pyeye1 + pyspeech1 pytest runs and the web-runner integration test (which drops the modules' +# .coverage files into target/pycov/ when ET_TEST_COVERAGE is set) all feed `coverage combine`, which merges +# them via config/coverage.toml's `[paths]` remap into a single Cobertura file. pytest-cov is a plugin, so it +# rides in each module's uv `dev` group -- the same venv as pytest -- and `uv run` (a mise-managed tool) installs +# both into the one project venv. UV_PYTHON mirrors test-pyface1's mise-CPython pin. run = """ pyface=services/ws-modules/pyface1 pyeye=services/ws-modules/pyeye1 +pyspeech=services/ws-modules/pyspeech1 pycov=target/pycov mkdir -p "$pycov" uv run --directory "$pyface" pytest --cov=pyface1 --cov-report= cp "$pyface/.coverage" "$pycov/.coverage.pyface1_pytest" uv run --directory "$pyeye" pytest --cov=pyeye1 --cov-report= cp "$pyeye/.coverage" "$pycov/.coverage.pyeye1_pytest" +uv run --directory "$pyspeech" pytest --cov=pyspeech1 --cov-report= +cp "$pyspeech/.coverage" "$pycov/.coverage.pyspeech1_pytest" uv run --project "$pyface" coverage combine "$pycov" uv run --project "$pyface" coverage xml -o coverage-python.xml """ diff --git a/.mise/config.python.toml b/.mise/config.python.toml index e21517b..7cd4c39 100644 --- a/.mise/config.python.toml +++ b/.mise/config.python.toml @@ -61,7 +61,7 @@ depends = ["ruff-fix"] description = "Apply Python lint-fix passes" [tasks."test:python"] -depends = ["test-pyeye1", "test-pyface1"] +depends = ["test-pyeye1", "test-pyface1", "test-pyspeech1"] description = "Run Python tests" [tasks.test-pyeye1] @@ -76,6 +76,11 @@ dir = "services/ws-modules/pyface1" env = { UV_PYTHON = "{% if os() == 'windows' %}{{ vars.py3_win }}{% else %}{{ vars.py3_unix }}{% endif %}" } run = "uv run pytest" +[tasks.test-pyspeech1] +dir = "services/ws-modules/pyspeech1" +env = { UV_PYTHON = "{% if os() == 'windows' %}{{ vars.py3_win }}{% else %}{{ vars.py3_unix }}{% endif %}" } +run = "uv run pytest" + [tasks.build-et-ws-wheel] depends = ["build-et-cli"] description = "Build the generated et-ws Python package as a wheel + ws-module pkg/" @@ -143,6 +148,17 @@ uv build --wheel --out-dir pkg """ shell = "bash -euo pipefail -c" +[tasks.build-ws-pyspeech1-module] +depends = ["build-et-ws-wheel"] +description = "Build the pyspeech1 Python Silero VAD workflow module" +dir = "services/ws-modules/pyspeech1" +run = """ +rm -f pkg/*.whl +uv build --wheel --out-dir pkg +{{ vars.et_cli }} module-package-json +""" +shell = "bash -euo pipefail -c" + [tasks.build-ws-wasi-graphics-info-module] depends = ["build-et-cli"] description = "Build the WASI graphics-info Python module as a WASI Preview 2 component" @@ -175,6 +191,7 @@ run = """ uv sync --directory services/ws-modules/pydata1 uv sync --directory services/ws-modules/pyeye1 uv sync --directory services/ws-modules/pyface1 +uv sync --directory services/ws-modules/pyspeech1 """ shell = "bash -euo pipefail -c" diff --git a/.mise/config.toml b/.mise/config.toml index b4c4781..b45bf87 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -413,6 +413,9 @@ har_repo = ":http:acd17sk/MET-Metabolic-Equivalent-of-Task-AI-Android-APP" har_src = "{{ vars.har_repo }}/raw/refs/heads/main/Models/onnx/hybrid_met.onnx" mnist_dst = "services/ws-modules/wasi-graphics-info/pkg/mnist-12.onnx" mnist_src = ":http:onnx/models/raw/main/validated/vision/classification/mnist/model/mnist-12.onnx" +speech1_asset = "speech1-84f0d86.onnx" +speech1_dst = "data/model-modules/model-speech1/pkg/speech1.onnx" +speech1_src = ":http:onnx-community/silero-vad/resolve/84f0d86/onnx/model_fp16.onnx" # Pip 26.0.1 wheel via PyPI, pinned to stay compatible with RustPython. # Pinned because pip 26.1.1+ calls # `sys.addaudithook` in `_prevent_further_imports`, which RustPython 0.5.0 @@ -1431,8 +1434,26 @@ shell = "bash -euo pipefail -c" run = "rclone copyto {{ vars.mnist_src }} {{ vars.mnist_dst }} {{ vars.gh_http }}" shell = "bash -euo pipefail -c" +[tasks.fetch-speech1-rclone] +run = """ +opts="--http-url https://huggingface.co --progress --ignore-existing" +# shellcheck disable=SC2086 +rclone copyto {{ vars.speech1_src }} {{ vars.speech1_dst }} $opts +export asset='{{ vars.speech1_asset }}' +sha=$(mise exec -- yq -p toml -oy '.asset[strenv(asset)].sha256' config/upstream-cache/data.toml) +actual=$(mise exec -- coreutils sha256sum -b "{{ vars.speech1_dst }}" | mise exec -- coreutils cut -c1-64) +[ "$actual" = "$sha" ] || { echo "speech1 checksum mismatch: expected=$sha actual=$actual" >&2; exit 1; } +""" +shell = "bash -euo pipefail -c" + [tasks.download-models] -depends = ["fetch-eye1-rclone", "fetch-face1-rclone", "fetch-har-motion1-rclone", "fetch-mnist-rclone"] +depends = [ + "fetch-eye1-rclone", + "fetch-face1-rclone", + "fetch-har-motion1-rclone", + "fetch-mnist-rclone", + "fetch-speech1-rclone", +] [tasks.cargo-test] description = "Run every Rust test via nextest (web-runner excluded on gnullvm)" diff --git a/config/coverage.toml b/config/coverage.toml index 9beca98..f6c4b31 100644 --- a/config/coverage.toml +++ b/config/coverage.toml @@ -13,3 +13,4 @@ relative_files = true pydata1 = ["services/ws-modules/pydata1/pydata1", "pydata1", "**/pydata1"] pyeye1 = ["services/ws-modules/pyeye1/pyeye1", "pyeye1", "**/pyeye1"] pyface1 = ["services/ws-modules/pyface1/pyface1", "pyface1", "**/pyface1"] +pyspeech1 = ["services/ws-modules/pyspeech1/pyspeech1", "pyspeech1", "**/pyspeech1"] diff --git a/config/pyrefly.toml b/config/pyrefly.toml index 77c3880..dd54356 100644 --- a/config/pyrefly.toml +++ b/config/pyrefly.toml @@ -23,6 +23,7 @@ search-path = [ "../services/ws-modules/pydata1", "../services/ws-modules/pyeye1", "../services/ws-modules/pyface1", + "../services/ws-modules/pyspeech1", "../services/ws-pyo3-runner/python", ] diff --git a/config/upstream-cache/data.toml b/config/upstream-cache/data.toml index 1f039e2..40fff34 100644 --- a/config/upstream-cache/data.toml +++ b/config/upstream-cache/data.toml @@ -33,6 +33,12 @@ sha256 = "64184e229b263107bc2b804c6625db1341ff2bb731874b0bcc2fe6544e0bc9ff" upstream = "https://ai.google.dev/edge/mediapipe/solutions/vision/face_landmarker" url = "https://github.com/edge-toolkit/core/releases/download/hf-cache-v1/face_landmarker.task" +[asset."speech1-84f0d86.onnx"] +license = "MIT" +sha256 = "7ed49c2c170aefe81e7e91b472a761d922ef3c064b6051068c2dc0ccc98bd53b" +upstream = "https://huggingface.co/onnx-community/silero-vad" +url = "https://huggingface.co/onnx-community/silero-vad/resolve/84f0d86/onnx/model_fp16.onnx" + [asset."rustpython-wasm-08a1d6f.tar.gz"] license = "MIT" sha256 = "7b81d53f5b3a63c60fc2bc1717e87466026d796ec522b925f009532c7b0291f3" diff --git a/data/model-modules/model-speech1/pkg/package.json b/data/model-modules/model-speech1/pkg/package.json new file mode 100644 index 0000000..bd5ac82 --- /dev/null +++ b/data/model-modules/model-speech1/pkg/package.json @@ -0,0 +1,7 @@ +{ + "description": "FP16 ONNX speech detection model", + "license": "MIT", + "name": "et-model-speech1", + "version": "0.1.0", + "files": ["speech1.onnx"] +} diff --git a/pyproject.toml b/pyproject.toml index 2dce1b8..b503d9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,5 +5,6 @@ members = [ "services/ws-modules/pydata1", "services/ws-modules/pyeye1", "services/ws-modules/pyface1", + "services/ws-modules/pyspeech1", "services/ws-modules/wasi-graphics-info", ] diff --git a/services/ws-modules/pyspeech1/pkg/.gitignore b/services/ws-modules/pyspeech1/pkg/.gitignore new file mode 100644 index 0000000..704d307 --- /dev/null +++ b/services/ws-modules/pyspeech1/pkg/.gitignore @@ -0,0 +1 @@ +*.whl diff --git a/services/ws-modules/pyspeech1/pkg/et_ws_pyspeech1.js b/services/ws-modules/pyspeech1/pkg/et_ws_pyspeech1.js new file mode 100644 index 0000000..ebfb68c --- /dev/null +++ b/services/ws-modules/pyspeech1/pkg/et_ws_pyspeech1.js @@ -0,0 +1,548 @@ +// Browser adapter: microphone capture + ONNX inference for the Python workflow. +const PYODIDE_BASE_PATH = "/modules/pyodide/"; + +let pyodide; +let py; +let cfg; +let runtime = null; + +export default async function init() { + if (!globalThis.loadPyodide) { + await new Promise((resolve, reject) => { + const script = document.createElement("script"); + script.src = `${PYODIDE_BASE_PATH}pyodide.js`; + script.onload = resolve; + script.onerror = reject; + document.head.appendChild(script); + }); + } + + pyodide = await globalThis.loadPyodide({ indexURL: PYODIDE_BASE_PATH }); + await pyodide.loadPackage("micropip"); + await pyodide.pyimport("micropip").install("pydantic"); + + const installLocalWheel = async (path) => { + const response = await fetch(new URL(path, import.meta.url)); + if (!response.ok) throw new Error(`Unable to fetch ${path}: HTTP ${response.status}`); + pyodide.FS.writeFile(`/tmp/${path}`, new Uint8Array(await response.arrayBuffer())); + pyodide.runPython(`import sys\nsys.path.insert(0, "/tmp/${path}")`); + }; + + const pkg = await fetch(new URL("package.json", import.meta.url)).then((response) => response.json()); + await installLocalWheel(`${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`); + const { installWheel: installEtWs } = await import("/modules/et-ws/et_ws.js"); + await installEtWs(pyodide); + if (globalThis.__etPyCov) await globalThis.__etPyCov.start(pyodide, "pyspeech1"); + py = pyodide.pyimport("pyspeech1"); + cfg = py.config().toJs({ dict_converter: Object.fromEntries }); +} + +export const is_running = () => runtime !== null; +export const start = () => run(); + +export async function run() { + if (!py) throw new Error("pyspeech1: not initialized"); + if (runtime) return; + + setStatus("pyspeech1 speech detection ready. Press Record audio to begin."); + log(py.model_log_message()); + const state = { + client: null, + stream: null, + audioContext: null, + session: null, + stopped: false, + busy: false, + recordButton: null, + visualization: null, + }; + runtime = state; + + try { + const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + await wasmAgent.default(); + const protocol = window.location.protocol === "https:" ? "wss:" : "ws:"; + state.client = new wasmAgent.WsClient(new wasmAgent.WsClientConfig(`${protocol}//${window.location.host}/ws`)); + state.client.connect(); + for (let i = 0; state.client.get_state() !== "connected" && i < 100; i++) await sleep(100); + if (state.client.get_state() !== "connected") throw new Error("Timed out waiting for websocket connection"); + + configureOnnxRuntime(); + state.session = await globalThis.ort.InferenceSession.create(cfg.model_path, { + executionProviders: ["wasm"], + }); + validateModelInterface(state.session); + state.recordButton = createRecordingControls(state); + log("ready; waiting for the user to start recording"); + if (globalThis.__etPyCov) await globalThis.__etPyCov.stop(pyodide, "pyspeech1"); + } catch (error) { + if (!state.stopped) { + setStatus(`pyspeech1 speech detection failed\n${String(error)}`); + log(`run failed: ${String(error)}`); + cleanup(state); + throw error; + } + } +} + +export function stop() { + if (!runtime) return; + const state = runtime; + state.stopped = true; + cleanup(state); + setStatus(py?.stopped_status?.() ?? "pyspeech1 speech detection stopped."); +} + +async function captureAndInfer(state) { + state.stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); + const AudioContext = globalThis.AudioContext ?? globalThis.webkitAudioContext; + if (!AudioContext) throw new Error("Web Audio is unavailable"); + state.audioContext = new AudioContext(); + await state.audioContext.resume(); + + const sourceRate = state.audioContext.sampleRate; + const source = state.audioContext.createMediaStreamSource(state.stream); + const processor = state.audioContext.createScriptProcessor(4096, 1, 1); + const sink = state.audioContext.createGain(); + sink.gain.value = 0; + const blocks = []; + state.visualization = createWaveform(cfg.capture_seconds); + processor.onaudioprocess = (event) => { + const block = new Float32Array(event.inputBuffer.getChannelData(0)); + blocks.push(block); + addWaveformSamples(state.visualization, block); + }; + source.connect(processor); + processor.connect(sink); + sink.connect(state.audioContext.destination); + + const started = performance.now(); + try { + await sleep(cfg.capture_seconds * 1000); + } finally { + processor.disconnect(); + source.disconnect(); + sink.disconnect(); + for (const track of state.stream?.getTracks?.() ?? []) track.stop(); + state.stream = null; + await state.audioContext?.close?.(); + state.audioContext = null; + } + if (state.stopped) throw new Error("capture stopped"); + + const recordedSeconds = (performance.now() - started) / 1000; + finishWaveform(state.visualization, "ANALYZING"); + const audio = resample(concatenate(blocks), sourceRate, cfg.sample_rate); + const probabilities = await inferProbabilities(state.session, audio); + return pyodide.toPy({ probabilities, source_sample_rate: sourceRate, recorded_seconds: recordedSeconds }); +} + +async function recordAndDetect(state) { + if (state.busy || state.stopped || runtime !== state) return; + state.busy = true; + const button = state.recordButton; + button.disabled = true; + button.style.cursor = "wait"; + button.style.opacity = "0.72"; + button.dataset.state = "recording"; + button.querySelector("span:last-child").textContent = "Recording…"; + setStatus(py.starting_status()); + + try { + await py.run( + pyodide.toPy(() => captureAndInfer(state)), + pyodide.toPy((message) => state.client.send(message)), + pyodide.toPy((detected, confidence) => showDetectionResult(state.visualization, detected, confidence)), + pyodide.toPy(log), + pyodide.toPy(setStatus), + ); + finishWaveform(state.visualization, "COMPLETE"); + button.querySelector("span:last-child").textContent = "Record again"; + } catch (error) { + if (!state.stopped) { + finishWaveform(state.visualization, "ERROR"); + setStatus(`pyspeech1 speech detection failed\n${String(error)}`); + log(`recording failed: ${String(error)}`); + button.querySelector("span:last-child").textContent = "Try again"; + } + } finally { + state.busy = false; + if (!state.stopped) { + button.disabled = false; + button.style.cursor = "pointer"; + button.style.opacity = "1"; + button.dataset.state = "ready"; + } + } +} + +function createRecordingControls(state) { + document.getElementById("pyspeech1-recording-controls")?.remove(); + const canvas = document.getElementById("video-output-canvas"); + if (!(canvas instanceof HTMLCanvasElement)) throw new Error("Missing waveform canvas"); + + const controls = document.createElement("div"); + controls.id = "pyspeech1-recording-controls"; + const controlStyles = [ + "display:flex", + "align-items:center", + "justify-content:space-between", + "gap:18px", + "margin:18px auto 0", + "padding:16px 18px", + "max-width:720px", + "box-sizing:border-box", + "border:1px solid rgba(24,32,40,.13)", + "border-radius:16px", + "background:linear-gradient(135deg,rgba(255,255,255,.9),rgba(239,248,246,.88))", + "box-shadow:0 12px 34px rgba(24,32,40,.10)", + ]; + controls.style.cssText = controlStyles.join(";"); + + const copy = document.createElement("div"); + const title = document.createElement("strong"); + title.style.cssText = "display:block;font:700 15px ui-monospace,monospace;color:#182028"; + title.textContent = "Speech sample"; + const description = document.createElement("span"); + description.style.cssText = "display:block;margin-top:4px;font:13px ui-monospace,monospace;color:#64747a"; + description.textContent = `Capture a ${cfg.capture_seconds}-second clip for local analysis`; + copy.append(title, description); + + const button = document.createElement("button"); + button.type = "button"; + button.dataset.state = "ready"; + button.setAttribute("aria-label", "Record audio for speech detection"); + const buttonStyles = [ + "display:inline-flex", + "align-items:center", + "gap:10px", + "min-width:158px", + "justify-content:center", + "padding:12px 18px", + "border:0", + "border-radius:999px", + "background:linear-gradient(135deg,#102d3b,#176454)", + "color:#f4fffc", + "font:700 14px ui-monospace,monospace", + "letter-spacing:.01em", + "cursor:pointer", + "box-shadow:0 8px 20px rgba(16,83,72,.24)", + "transition:transform .15s ease,box-shadow .15s ease,opacity .15s ease", + ]; + button.style.cssText = buttonStyles.join(";"); + const recordingIndicator = document.createElement("span"); + recordingIndicator.setAttribute("aria-hidden", "true"); + const indicatorStyles = [ + "width:10px", + "height:10px", + "border-radius:50%", + "background:#ff6577", + "box-shadow:0 0 0 4px rgba(255,101,119,.16)", + ]; + recordingIndicator.style.cssText = indicatorStyles.join(";"); + const buttonLabel = document.createElement("span"); + buttonLabel.textContent = "Record audio"; + button.append(recordingIndicator, buttonLabel); + button.addEventListener("mouseenter", () => { + if (!button.disabled) button.style.transform = "translateY(-1px)"; + }); + button.addEventListener("mouseleave", () => { + button.style.transform = "none"; + }); + button.addEventListener("click", () => recordAndDetect(state)); + + controls.append(copy, button); + canvas.before(controls); + return button; +} + +async function inferProbabilities(session, samples) { + let recurrentState = new Float32Array(2 * 128); + let context = new Float32Array(cfg.context_size); + const probabilities = []; + + for (let offset = 0; offset < samples.length; offset += cfg.chunk_size) { + const input = new Float32Array(cfg.context_size + cfg.chunk_size); + input.set(context); + input.set(samples.subarray(offset, Math.min(offset + cfg.chunk_size, samples.length)), cfg.context_size); + const outputs = await session.run({ + input: new globalThis.ort.Tensor("float32", input, [1, input.length]), + state: new globalThis.ort.Tensor("float32", recurrentState, [2, 1, 128]), + sr: new globalThis.ort.Tensor("int64", BigInt64Array.of(BigInt(cfg.sample_rate)), []), + }); + probabilities.push(Number(outputs.output.data[0])); + recurrentState = new Float32Array(outputs.stateN.data); + context = input.slice(input.length - cfg.context_size); + } + return probabilities; +} + +function configureOnnxRuntime() { + const wasm = globalThis.ort?.env?.wasm; + const version = globalThis.ort?.env?.versions?.web; + if (!wasm || !version) throw new Error("onnxruntime-web environment is unavailable"); + const base = "/modules/onnxruntime-web/dist"; + wasm.numThreads = globalThis.crossOriginIsolated && globalThis.SharedArrayBuffer ? 0 : 1; + wasm.wasmPaths = { mjs: `${base}/ort-wasm-simd-threaded.mjs`, wasm: `${base}/ort-wasm-simd-threaded.wasm` }; +} + +function validateModelInterface(session) { + for (const name of ["input", "state", "sr"]) { + if (!session.inputNames.includes(name)) throw new Error(`Speech detection model is missing input ${name}`); + } + for (const name of ["output", "stateN"]) { + if (!session.outputNames.includes(name)) throw new Error(`Speech detection model is missing output ${name}`); + } +} + +function concatenate(blocks) { + const length = blocks.reduce((total, block) => total + block.length, 0); + const result = new Float32Array(length); + let offset = 0; + for (const block of blocks) { + result.set(block, offset); + offset += block.length; + } + return result; +} + +function resample(input, sourceRate, targetRate) { + if (sourceRate === targetRate) return input; + const output = new Float32Array(Math.max(1, Math.round((input.length * targetRate) / sourceRate))); + const ratio = sourceRate / targetRate; + for (let i = 0; i < output.length; i++) { + const position = i * ratio; + const left = Math.min(Math.floor(position), input.length - 1); + const right = Math.min(left + 1, input.length - 1); + const fraction = position - left; + output[i] = input[left] * (1 - fraction) + input[right] * fraction; + } + return output; +} + +function cleanup(state) { + if (runtime === state) runtime = null; + if (state.visualization?.animationFrame) cancelAnimationFrame(state.visualization.animationFrame); + for (const track of state.stream?.getTracks?.() ?? []) track.stop(); + state.audioContext?.close?.(); + state.client?.disconnect?.(); + document.getElementById("pyspeech1-recording-controls")?.remove(); + const canvas = document.getElementById("video-output-canvas"); + if (canvas) canvas.hidden = true; +} + +function createWaveform(durationSeconds) { + const canvas = document.getElementById("video-output-canvas"); + if (!(canvas instanceof HTMLCanvasElement)) throw new Error("Missing waveform canvas"); + canvas.width = 1440; + canvas.height = 880; + canvas.style.width = "100%"; + canvas.style.maxHeight = "none"; + canvas.hidden = false; + const visualization = { + canvas, + context: canvas.getContext("2d"), + durationSeconds, + startedAt: performance.now(), + peaks: [], + level: 0, + speechDetected: null, + speechConfidence: 0, + phase: "RECORDING", + animationFrame: 0, + }; + const animate = () => { + drawWaveform(visualization); + if (visualization.phase === "RECORDING") { + visualization.animationFrame = requestAnimationFrame(animate); + } + }; + animate(); + return visualization; +} + +function showDetectionResult(visualization, detected, confidence) { + if (!visualization) return; + visualization.speechDetected = Boolean(detected); + visualization.speechConfidence = Number(confidence); + drawWaveform(visualization); +} + +function addWaveformSamples(visualization, samples) { + if (!visualization) return; + const bins = 8; + const binSize = Math.max(1, Math.floor(samples.length / bins)); + let energy = 0; + for (const sample of samples) energy += sample * sample; + const rms = Math.sqrt(energy / Math.max(samples.length, 1)); + visualization.level += (rms - visualization.level) * 0.35; + + for (let bin = 0; bin < bins; bin++) { + let low = 0; + let high = 0; + const end = Math.min((bin + 1) * binSize, samples.length); + for (let index = bin * binSize; index < end; index++) { + low = Math.min(low, samples[index]); + high = Math.max(high, samples[index]); + } + visualization.peaks.push({ low, high }); + } + if (visualization.peaks.length > 720) visualization.peaks.splice(0, visualization.peaks.length - 720); +} + +function finishWaveform(visualization, phase) { + if (!visualization) return; + visualization.phase = phase; + if (visualization.animationFrame) cancelAnimationFrame(visualization.animationFrame); + visualization.animationFrame = 0; + drawWaveform(visualization); +} + +function drawWaveform(visualization) { + const { canvas, context: ctx } = visualization; + const width = canvas.width; + const height = canvas.height; + const elapsed = Math.min((performance.now() - visualization.startedAt) / 1000, visualization.durationSeconds); + const progress = visualization.phase === "RECORDING" ? elapsed / visualization.durationSeconds : 1; + + const background = ctx.createLinearGradient(0, 0, width, height); + background.addColorStop(0, "#091523"); + background.addColorStop(0.55, "#10253a"); + background.addColorStop(1, "#091923"); + ctx.fillStyle = background; + ctx.fillRect(0, 0, width, height); + + const glow = ctx.createRadialGradient(width * progress, height * 0.55, 0, width * progress, height * 0.55, 430); + glow.addColorStop(0, "rgba(37, 211, 180, 0.16)"); + glow.addColorStop(1, "rgba(37, 211, 180, 0)"); + ctx.fillStyle = glow; + ctx.fillRect(0, 0, width, height); + + ctx.strokeStyle = "rgba(166, 211, 221, 0.075)"; + ctx.lineWidth = 1; + for (let x = 48; x < width; x += 72) line(ctx, x, 92, x, height - 62); + for (let y = 112; y < height - 52; y += 54) line(ctx, 44, y, width - 44, y); + + ctx.font = "700 40px ui-monospace, SFMono-Regular, Menlo, monospace"; + ctx.fillStyle = "#edf9f7"; + ctx.fillText("LIVE AUDIO", 48, 64); + ctx.font = "600 27px ui-monospace, SFMono-Regular, Menlo, monospace"; + ctx.fillStyle = "#8da8b2"; + ctx.fillText("SPEECH DETECTION INPUT", 48, 103); + + const active = visualization.phase === "RECORDING"; + ctx.beginPath(); + ctx.arc(width - 300, 58, 10, 0, Math.PI * 2); + ctx.fillStyle = active ? "#ff6174" : "#26d3b4"; + ctx.shadowColor = ctx.fillStyle; + ctx.shadowBlur = 18; + ctx.fill(); + ctx.shadowBlur = 0; + ctx.fillStyle = active ? "#ffb0ba" : "#9af1de"; + ctx.fillText(visualization.phase, width - 275, 67); + ctx.fillStyle = "#a9bec5"; + ctx.textAlign = "right"; + ctx.fillText(`${elapsed.toFixed(1)} / ${visualization.durationSeconds.toFixed(1)}s`, width - 48, 103); + ctx.textAlign = "left"; + + const center = height * 0.52; + const plotLeft = 48; + const plotWidth = width - 96; + const peaks = visualization.peaks; + if (peaks.length) { + const gradient = ctx.createLinearGradient(plotLeft, 0, plotLeft + plotWidth, 0); + gradient.addColorStop(0, "#47a9ff"); + gradient.addColorStop(0.5, "#33e1c1"); + gradient.addColorStop(1, "#b5f25d"); + ctx.strokeStyle = gradient; + ctx.lineWidth = Math.max(2, (plotWidth / Math.max(peaks.length, 1)) * 0.66); + ctx.lineCap = "round"; + ctx.shadowColor = "rgba(51, 225, 193, 0.55)"; + ctx.shadowBlur = 13; + const scale = 285; + for (let index = 0; index < peaks.length; index++) { + const x = plotLeft + (index / Math.max(peaks.length - 1, 1)) * plotWidth * progress; + const peak = peaks[index]; + line(ctx, x, center + peak.low * scale, x, center + peak.high * scale); + } + ctx.shadowBlur = 0; + } else { + ctx.strokeStyle = "rgba(51, 225, 193, 0.7)"; + line(ctx, plotLeft, center, plotLeft + plotWidth * progress, center); + } + + if (visualization.speechDetected === true) { + const badgeWidth = 520; + const badgeHeight = 84; + const badgeX = (width - badgeWidth) / 2; + const badgeY = height - 190; + ctx.shadowColor = "rgba(42, 231, 191, 0.38)"; + ctx.shadowBlur = 30; + ctx.fillStyle = "rgba(15, 77, 67, 0.94)"; + roundedRect(ctx, badgeX, badgeY, badgeWidth, badgeHeight, 22); + ctx.shadowBlur = 0; + ctx.strokeStyle = "rgba(126, 245, 218, 0.72)"; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.roundRect(badgeX, badgeY, badgeWidth, badgeHeight, 22); + ctx.stroke(); + ctx.beginPath(); + ctx.arc(badgeX + 42, badgeY + badgeHeight / 2, 10, 0, Math.PI * 2); + ctx.fillStyle = "#62f0cf"; + ctx.shadowColor = "#62f0cf"; + ctx.shadowBlur = 18; + ctx.fill(); + ctx.shadowBlur = 0; + ctx.fillStyle = "#eafffa"; + ctx.font = "800 34px ui-monospace, SFMono-Regular, Menlo, monospace"; + ctx.fillText("SPEECH DETECTED", badgeX + 70, badgeY + 43); + ctx.fillStyle = "#9ee9d8"; + ctx.font = "600 19px ui-monospace, SFMono-Regular, Menlo, monospace"; + ctx.fillText(`${(visualization.speechConfidence * 100).toFixed(1)}% peak confidence`, badgeX + 72, badgeY + 68); + } + + const progressY = height - 36; + ctx.fillStyle = "rgba(255, 255, 255, 0.09)"; + roundedRect(ctx, 48, progressY, plotWidth, 7, 4); + const progressGradient = ctx.createLinearGradient(48, 0, width - 48, 0); + progressGradient.addColorStop(0, "#47a9ff"); + progressGradient.addColorStop(1, "#33e1c1"); + ctx.fillStyle = progressGradient; + roundedRect(ctx, 48, progressY, plotWidth * progress, 7, 4); + + ctx.fillStyle = "#8da8b2"; + ctx.font = "600 27px ui-monospace, SFMono-Regular, Menlo, monospace"; + ctx.fillText("INPUT LEVEL", 48, height - 58); + ctx.fillStyle = visualization.level > 0.025 ? "#7ce8cf" : "#78949e"; + ctx.fillText(visualization.level > 0.025 ? "VOICE ACTIVITY" : "LISTENING", 265, height - 58); +} + +function line(ctx, x1, y1, x2, y2) { + ctx.beginPath(); + ctx.moveTo(x1, y1); + ctx.lineTo(x2, y2); + ctx.stroke(); +} + +function roundedRect(ctx, x, y, width, height, radius) { + if (width <= 0) return; + ctx.beginPath(); + ctx.roundRect(x, y, width, height, radius); + ctx.fill(); +} + +function setStatus(message) { + const output = document.getElementById("module-output"); + if (output) output.value = message; +} + +function log(message) { + const logLine = `[pyspeech1] ${message}`; + console.log(logLine); + const output = document.getElementById("log"); + if (output) output.textContent = output.textContent ? `${output.textContent}\n${logLine}` : logLine; +} + +function sleep(ms) { + return new Promise((resolve) => window.setTimeout(resolve, ms)); +} diff --git a/services/ws-modules/pyspeech1/pkg/package.json b/services/ws-modules/pyspeech1/pkg/package.json new file mode 100644 index 0000000..1db93bf --- /dev/null +++ b/services/ws-modules/pyspeech1/pkg/package.json @@ -0,0 +1,14 @@ +{ + "dependencies": { + "et-model-speech1": "*", + "et-ws": "*", + "onnxruntime-web": "*", + "pyodide": "*" + }, + "description": "Python microphone speech detection", + "license": "Apache-2.0 OR MIT", + "main": "et_ws_pyspeech1.js", + "name": "et-ws-pyspeech1", + "type": "module", + "version": "0.1.0" +} diff --git a/services/ws-modules/pyspeech1/pyproject.toml b/services/ws-modules/pyspeech1/pyproject.toml new file mode 100644 index 0000000..23f78ef --- /dev/null +++ b/services/ws-modules/pyspeech1/pyproject.toml @@ -0,0 +1,31 @@ +[project] +dependencies = ["et-ws"] +description = "Python microphone speech detection" +license = "Apache-2.0 OR MIT" +name = "et-ws-pyspeech1" +requires-python = ">=3.10" +version = "0.1.0" + +[dependency-groups] +dev = ["pytest", "pytest-cov"] + +[build-system] +build-backend = "uv_build" +requires = ["uv_build==0.11.8"] + +[tool.uv.build-backend] +module-name = "pyspeech1" +module-root = "" + +[tool.uv.sources] +et-ws = { workspace = true } + +[tool.ws-module.dependencies] +et-model-speech1 = "*" +et-ws = "*" +onnxruntime-web = "*" +pyodide = "*" + +[tool.pytest.ini_options] +addopts = ["--import-mode=importlib"] +pythonpath = ["."] diff --git a/services/ws-modules/pyspeech1/pyspeech1/__init__.py b/services/ws-modules/pyspeech1/pyspeech1/__init__.py new file mode 100644 index 0000000..64a7a39 --- /dev/null +++ b/services/ws-modules/pyspeech1/pyspeech1/__init__.py @@ -0,0 +1,27 @@ +"""pyspeech1: Python support code for browser-side speech detection.""" + +from .speech_detection import ( + SPEECH_MODEL_PATH, + client_event_json, + config, + event_payload, + model_log_message, + run, + starting_status, + status_text, + stopped_status, + summarize_probabilities, +) + +__all__ = [ + "SPEECH_MODEL_PATH", + "client_event_json", + "config", + "event_payload", + "model_log_message", + "run", + "starting_status", + "status_text", + "stopped_status", + "summarize_probabilities", +] diff --git a/services/ws-modules/pyspeech1/pyspeech1/speech_detection.py b/services/ws-modules/pyspeech1/pyspeech1/speech_detection.py new file mode 100644 index 0000000..65fa85c --- /dev/null +++ b/services/ws-modules/pyspeech1/pyspeech1/speech_detection.py @@ -0,0 +1,159 @@ +"""Decision and event helpers for browser-side speech detection.""" + +from __future__ import annotations + +import math +from collections.abc import Iterable +from datetime import datetime, timezone +from typing import Any, TypedDict + +from et_ws.messages import WsClientEvent + +SPEECH_MODEL_PATH = "/modules/et-model-speech1/speech1.onnx" +SAMPLE_RATE = 16_000 +CHUNK_SIZE = 512 +CONTEXT_SIZE = 64 +CAPTURE_SECONDS = 5 +SPEECH_THRESHOLD = 0.5 +NEGATIVE_THRESHOLD = 0.35 +MIN_SPEECH_MS = 250 +MIN_SILENCE_MS = 100 + + +class SpeechSummary(TypedDict): + """A compact result derived from one microphone capture.""" + + speech_detected: bool + confidence: float + mean_probability: float + speech_ratio: float + speech_duration_ms: int + frame_count: int + processed_at: str + + +def config() -> dict[str, object]: + """Return constants used by the browser adapter.""" + return { + "model_path": SPEECH_MODEL_PATH, + "sample_rate": SAMPLE_RATE, + "chunk_size": CHUNK_SIZE, + "context_size": CONTEXT_SIZE, + "capture_seconds": CAPTURE_SECONDS, + } + + +def summarize_probabilities(values: Iterable[object]) -> SpeechSummary: + """Apply hysteresis and minimum-duration filtering.""" + probabilities = [_probability(value) for value in values] + if not probabilities: + raise ValueError("Speech detection model returned no probabilities") + + chunk_ms = CHUNK_SIZE * 1000.0 / SAMPLE_RATE + min_speech_frames = math.ceil(MIN_SPEECH_MS / chunk_ms) + min_silence_frames = math.ceil(MIN_SILENCE_MS / chunk_ms) + detected_frames = 0 + longest_segment = 0 + segment_frames = 0 + silence_frames = 0 + triggered = False + + for probability in probabilities: + if probability >= SPEECH_THRESHOLD: + if not triggered: + triggered = True + segment_frames = 0 + silence_frames = 0 + segment_frames += 1 + detected_frames += 1 + elif triggered: + segment_frames += 1 + if probability < NEGATIVE_THRESHOLD: + silence_frames += 1 + if silence_frames >= min_silence_frames: + longest_segment = max(longest_segment, segment_frames - silence_frames) + triggered = False + segment_frames = 0 + silence_frames = 0 + else: + silence_frames = 0 + + if triggered: + longest_segment = max(longest_segment, segment_frames - silence_frames) + + return { + "speech_detected": longest_segment >= min_speech_frames, + "confidence": max(probabilities), + "mean_probability": sum(probabilities) / len(probabilities), + "speech_ratio": detected_frames / len(probabilities), + "speech_duration_ms": round(longest_segment * chunk_ms), + "frame_count": len(probabilities), + "processed_at": datetime.now(timezone.utc).isoformat(), + } + + +def event_payload(summary: SpeechSummary, source_sample_rate: float, recorded_seconds: float) -> dict[str, object]: + """Build the details object sent to the websocket.""" + return { + **summary, + "label": "speech" if summary["speech_detected"] else "no_speech", + "source_sample_rate": round(source_sample_rate), + "model_sample_rate": SAMPLE_RATE, + "recorded_seconds": round(recorded_seconds, 3), + "threshold": SPEECH_THRESHOLD, + } + + +def client_event_json(details: dict[str, object]) -> str: + """Build the typed et-client-event envelope.""" + return WsClientEvent( + type="et-client-event", + capability="speech_detection", + action="inference", + details=details, + ).model_dump_json() + + +async def run(infer_capture, send_event, render_result, log, set_status) -> None: + """Capture once through JS, classify in Python, and publish the result.""" + capture = await infer_capture() + summary = summarize_probabilities(capture["probabilities"]) + details = event_payload(summary, capture["source_sample_rate"], capture["recorded_seconds"]) + render_result(summary["speech_detected"], summary["confidence"]) + set_status(status_text(summary)) + send_event(client_event_json(details)) + log(f"speech_detected={summary['speech_detected']} confidence={summary['confidence']:.3f}") + + +def status_text(summary: SpeechSummary) -> str: + """Render a concise human-readable detection result.""" + label = "SPEECH DETECTED" if summary["speech_detected"] else "NO SPEECH DETECTED" + return ( + f"pyspeech1: {label}\n" + f"peak probability: {summary['confidence']:.3f}\n" + f"mean probability: {summary['mean_probability']:.3f}\n" + f"speech ratio: {summary['speech_ratio']:.1%}\n" + f"longest speech segment: {summary['speech_duration_ms']} ms" + ) + + +def starting_status() -> str: + """Return the status shown while microphone audio is captured.""" + return f"pyspeech1 speech detection: recording for {CAPTURE_SECONDS} seconds" + + +def stopped_status() -> str: + """Return the status shown after an explicit stop.""" + return "pyspeech1 speech detection stopped." + + +def model_log_message() -> str: + """Return the model-loading log message.""" + return f"loading FP16 speech detection model from {SPEECH_MODEL_PATH}" + + +def _probability(value: Any) -> float: + probability = float(value) + if not math.isfinite(probability) or not 0.0 <= probability <= 1.0: + raise ValueError(f"invalid speech probability: {value}") + return probability diff --git a/services/ws-modules/pyspeech1/tests/test_speech_detection.py b/services/ws-modules/pyspeech1/tests/test_speech_detection.py new file mode 100644 index 0000000..6bc8185 --- /dev/null +++ b/services/ws-modules/pyspeech1/tests/test_speech_detection.py @@ -0,0 +1,53 @@ +import json +import unittest + +from pyspeech1.speech_detection import ( + CHUNK_SIZE, + SAMPLE_RATE, + client_event_json, + config, + event_payload, + summarize_probabilities, +) + + +class SpeechDetectionTests(unittest.TestCase): + def test_config_matches_speech1_inputs(self) -> None: + self.assertEqual(config()["sample_rate"], 16_000) + self.assertEqual(config()["chunk_size"], 512) + self.assertEqual(config()["context_size"], 64) + + def test_silence_is_not_speech(self) -> None: + result = summarize_probabilities([0.01] * 20) + self.assertFalse(result["speech_detected"]) + self.assertEqual(result["speech_duration_ms"], 0) + + def test_sustained_speech_is_detected(self) -> None: + result = summarize_probabilities([0.02] * 3 + [0.91] * 10 + [0.01] * 4) + self.assertTrue(result["speech_detected"]) + self.assertGreaterEqual(result["speech_duration_ms"], 250) + self.assertAlmostEqual(result["confidence"], 0.91) + + def test_short_noise_spike_is_rejected(self) -> None: + result = summarize_probabilities([0.01] * 3 + [0.99] * 3 + [0.01] * 4) + self.assertFalse(result["speech_detected"]) + + def test_invalid_or_empty_output_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "no probabilities"): + summarize_probabilities([]) + with self.assertRaisesRegex(ValueError, "invalid"): + summarize_probabilities([float("nan")]) + + def test_event_uses_speech_detection_capability(self) -> None: + summary = summarize_probabilities([0.9] * 10) + payload = event_payload(summary, 48_000, 5.0) + event = json.loads(client_event_json(payload)) + self.assertEqual(event["capability"], "speech_detection") + self.assertEqual(event["action"], "inference") + self.assertEqual(event["details"]["label"], "speech") + self.assertEqual(event["details"]["model_sample_rate"], SAMPLE_RATE) + self.assertEqual(CHUNK_SIZE, 512) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/ws-web-runner/tests/modules.rs b/services/ws-web-runner/tests/modules.rs index d415bb3..3de364c 100644 --- a/services/ws-web-runner/tests/modules.rs +++ b/services/ws-web-runner/tests/modules.rs @@ -21,6 +21,7 @@ //! - **et-ws-har1** -- accelerometer (`DeviceMotionEvent`) + ONNX model //! - **et-ws-pyface1** -- Pyodide + camera + ONNX model //! - **et-ws-pyeye1** -- Pyodide + camera + `MediaPipe` `FaceLandmarker` (tflite) -> eye boxes +//! - **et-ws-pyspeech1** -- Pyodide + microphone + ONNX model //! //! ### Non-JS module loaders / incompatible runtimes //! @@ -173,6 +174,7 @@ fn multi_agent_module(#[case] module: &str, #[case] language: Language) { #[case::video1("et-ws-video1", Language::Rust)] #[case::pyface1("et-ws-pyface1", Language::Python)] #[case::pyeye1("et-ws-pyeye1", Language::Python)] +#[case::pyspeech1("et-ws-pyspeech1", Language::Python)] fn hardware_module_load_fails(#[case] module: &str, #[case] language: Language) { if !mise_env_includes(language) { println!( diff --git a/uv.lock b/uv.lock index f710b2e..a46d298 100644 --- a/uv.lock +++ b/uv.lock @@ -9,6 +9,7 @@ members = [ "et-ws-pydata1", "et-ws-pyeye1", "et-ws-pyface1", + "et-ws-pyspeech1", "et-ws-wasi-graphics-info", ] @@ -252,6 +253,29 @@ dev = [ { name = "pytest-cov" }, ] +[[package]] +name = "et-ws-pyspeech1" +version = "0.1.0" +source = { editable = "services/ws-modules/pyspeech1" } +dependencies = [ + { name = "et-ws" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [{ name = "et-ws", editable = "generated/python-ws" }] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest" }, + { name = "pytest-cov" }, +] + [[package]] name = "et-ws-wasi-graphics-info" version = "0.1.0"