diff --git a/.mise.toml b/.mise.toml index 2b6692a..586e2c3 100644 --- a/.mise.toml +++ b/.mise.toml @@ -22,7 +22,7 @@ dprint-fmt = "dprint fmt" editorconfig-check = "ec" [tasks.fmt] -depends = ["cargo-fmt", "dprint-fmt", "taplo-fmt"] +depends = ["cargo-clippy-fix", "cargo-fmt", "dprint-fmt", "taplo-fmt"] description = "Run repository formatters" [tasks.install-nightly] @@ -55,6 +55,9 @@ run = "cargo +nightly fmt -- --check" [tasks.cargo-clippy] run = "cargo clippy --workspace" +[tasks.cargo-clippy-fix] +run = "cargo clippy --fix --allow-dirty --allow-staged --workspace" + [tasks.taplo-fmt] run = "taplo format" @@ -82,6 +85,15 @@ run = "wasm-pack build . --target web -- -Z build-std=std,panic_abort" RUSTFLAGS = "-C target-cpu=mvp -C target-feature=+mutable-globals,+sign-ext,+nontrapping-fptoint" RUSTUP_TOOLCHAIN = "nightly" +[tasks.build-ws-har1-module] +description = "Build the har1 workflow WASM module" +dir = "services/ws-modules/har1" +run = "wasm-pack build . --target web" + +[tasks.build] +depends = ["build-ws-har1-module", "build-ws-wasm-agent"] +description = "Build all WebAssembly modules" + [tasks.test-ws-wasm-agent-firefox] description = "Run headless Firefox tests for the WebSocket WASM client" dir = "services/ws-wasm-agent" diff --git a/Cargo.toml b/Cargo.toml index 51489bd..026f1b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["libs/edge-toolkit", "services/ws-server", "services/ws-wasm-agent"] +members = ["libs/edge-toolkit", "services/ws-modules/har1", "services/ws-server", "services/ws-wasm-agent"] resolver = "2" [workspace.dependencies] diff --git a/README.md b/README.md index 7f8eedd..e52d4fd 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Download the onnx from https://huggingface.co/amd/retinaface and save it in ```bash mise run build-ws-wasm-agent +mise run build-ws-har1-module mise run ws-server ``` @@ -43,7 +44,8 @@ which will normally be something like 192.168.1.x. Then on your phone, open Chrome and type in https://192.168.1.x:8433/ -Click "Load HAR model" and then "Start sensors". +Click "har demo". + For webcam inference, click "Load video CV model" and then "Start video". ## Grant diff --git a/services/ws-modules/har1/Cargo.toml b/services/ws-modules/har1/Cargo.toml new file mode 100644 index 0000000..0ee8ead --- /dev/null +++ b/services/ws-modules/har1/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "et-ws-har1" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +et-ws-wasm-agent = { path = "../../ws-wasm-agent" } +js-sys = "0.3" +serde.workspace = true +serde_json.workspace = true +tracing.workspace = true +tracing-wasm = "0.2" +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" +web-sys = { version = "0.3", features = [ + "AddEventListenerOptions", + "BinaryType", + "Document", + "DomError", + "Event", + "EventTarget", + "HtmlCanvasElement", + "MediaDevices", + "MediaStream", + "MediaStreamConstraints", + "MediaStreamTrack", + "MessageEvent", + "Navigator", + "Storage", + "WebSocket", + "Window", + "console", +] } + +[dev-dependencies] +wasm-bindgen-test = "0.3" diff --git a/services/ws-modules/har1/src/lib.rs b/services/ws-modules/har1/src/lib.rs new file mode 100644 index 0000000..6c7fe65 --- /dev/null +++ b/services/ws-modules/har1/src/lib.rs @@ -0,0 +1,545 @@ +use std::collections::VecDeque; + +use et_ws_wasm_agent::{DeviceSensors, MotionReading, WsClient, WsClientConfig}; +use js_sys::{Array, Float32Array, Function, Promise, Reflect}; +use serde_json::json; +use tracing::info; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; + +const HAR_MODEL_PATH: &str = "/static/models/human_activity_recognition.onnx"; +const HAR_SEQUENCE_LENGTH: usize = 512; +const HAR_FEATURE_COUNT: usize = 9; +const HAR_SAMPLE_INTERVAL_MS: i32 = 20; +const HAR_INFERENCE_INTERVAL_MS: f64 = 250.0; +const STANDARD_GRAVITY: f64 = 9.80665; +const GRAVITY_FILTER_ALPHA: f64 = 0.8; +const HAR_CLASS_LABELS: [&str; 6] = ["class_0", "class_1", "class_2", "class_3", "class_4", "class_5"]; + +#[wasm_bindgen(start)] +pub fn init() { + tracing_wasm::set_as_global_default(); + info!("har1 workflow module initialized"); +} + +#[wasm_bindgen] +pub async fn run() -> Result<(), JsValue> { + set_har_status("har1: entered run()")?; + log("entered run()")?; + log("using existing tracing setup")?; + + let outcome = async { + let ws_url = websocket_url()?; + set_har_status(&format!("har1: resolved websocket URL\n{ws_url}"))?; + log(&format!("resolved websocket URL: {ws_url}"))?; + let mut client = WsClient::new(WsClientConfig::new(ws_url)); + log("connecting websocket client")?; + client.connect()?; + wait_for_connected(&client).await?; + log(&format!("websocket connected with agent_id={}", client.get_client_id()))?; + + let mut sensors = DeviceSensors::new(); + log("starting har1 workflow")?; + + let result = run_inner(&client, &mut sensors).await; + let stop_result = sensors.stop(); + client.disconnect(); + + match (result, stop_result) { + (Ok(()), Ok(())) => { + log("har1 workflow finished")?; + Ok(()) + } + (Err(error), Ok(())) => Err(error), + (Ok(()), Err(error)) => Err(error), + (Err(error), Err(_)) => Err(error), + } + } + .await; + + if let Err(error) = &outcome { + let message = describe_js_error(error); + let _ = set_har_status(&format!("har1: error\n{message}")); + let _ = log(&format!("error: {message}")); + } + + outcome +} + +async fn run_inner(client: &WsClient, sensors: &mut DeviceSensors) -> Result<(), JsValue> { + set_har_status("har1: loading HAR model")?; + log(&format!("loading HAR model from {HAR_MODEL_PATH}"))?; + let session = create_har_session(HAR_MODEL_PATH).await?; + let input_name = first_string_entry(&session, "inputNames")?; + let output_name = first_string_entry(&session, "outputNames")?; + + set_har_status(&format!( + "har1: HAR model loaded\npath: {HAR_MODEL_PATH}\ninput: {input_name}\noutput: {output_name}" + ))?; + log(&format!("HAR model loaded: input={input_name} output={output_name}"))?; + + log("requesting sensor access")?; + sensors.start().await?; + log("sensors started")?; + render_sensor_output(sensors)?; + log("waiting for first motion sample")?; + wait_for_motion_sample(sensors).await?; + log("first motion sample received")?; + + let mut gravity_estimate = [0.0_f64; 3]; + let mut sample_buffer: VecDeque<[f32; HAR_FEATURE_COUNT]> = VecDeque::with_capacity(HAR_SEQUENCE_LENGTH); + let mut last_inference_at = 0.0_f64; + let mut last_class_label: Option = None; + let mut class_change_count = 0_u32; + + while class_change_count < 3 { + sleep_ms(HAR_SAMPLE_INTERVAL_MS).await?; + + if !sensors.has_motion() { + continue; + } + + let reading = sensors.motion_snapshot()?; + render_sensor_output(sensors)?; + sample_buffer.push_back(feature_vector(&reading, &mut gravity_estimate)); + if sample_buffer.len() > HAR_SEQUENCE_LENGTH { + sample_buffer.pop_front(); + } + + if sample_buffer.len() == 1 + || sample_buffer.len() == 64 + || sample_buffer.len() == 128 + || sample_buffer.len() == 256 + { + log(&format!( + "buffering HAR samples: {}/{}", + sample_buffer.len(), + HAR_SEQUENCE_LENGTH + ))?; + } + + if sample_buffer.len() < HAR_SEQUENCE_LENGTH { + continue; + } + + if sample_buffer.len() == HAR_SEQUENCE_LENGTH { + set_har_status("har1: HAR sample window full; inference loop active")?; + log("HAR sample window full; starting inference loop")?; + } + + let now = js_sys::Date::now(); + if now - last_inference_at < HAR_INFERENCE_INTERVAL_MS { + continue; + } + last_inference_at = now; + + let prediction = infer_prediction(&session, &input_name, &output_name, &sample_buffer).await?; + if last_class_label.as_deref() == Some(prediction.best_label.as_str()) { + continue; + } + + class_change_count += 1; + client.send_client_event( + "har", + "class_changed", + json!({ + "detected_class": prediction.best_label, + "previous_class": last_class_label, + "class_index": prediction.best_index, + "confidence": prediction.best_probability, + "probabilities": prediction.probabilities, + "logits": prediction.logits, + "buffered_samples": sample_buffer.len(), + "detected_at": String::from(js_sys::Date::new_0().to_iso_string()), + }), + )?; + + log(&format!( + "class change {} of 3: {} -> {}", + class_change_count, + last_class_label.as_deref().unwrap_or("none"), + prediction.best_label + ))?; + set_har_status(&format!( + "har1: inference running\nlatest class: {}\nclass changes: {}/3\nbuffered samples: {}", + prediction.best_label, + class_change_count, + sample_buffer.len() + ))?; + last_class_label = Some(prediction.best_label); + } + + set_har_status("har1: workflow complete")?; + Ok(()) +} + +fn render_sensor_output(sensors: &DeviceSensors) -> Result<(), JsValue> { + let orientation = if sensors.has_orientation() { + Some(sensors.orientation_snapshot()?) + } else { + None + }; + let motion = if sensors.has_motion() { + Some(sensors.motion_snapshot()?) + } else { + None + }; + + let mut lines = vec![ + String::from("Device sensor stream"), + format!( + "updated: {}", + String::from(js_sys::Date::new_0().to_locale_time_string("en-US")) + ), + String::new(), + String::from("orientation"), + ]; + + if let Some(orientation) = orientation { + lines.push(format!("alpha: {}", format_number(orientation.alpha(), 3))); + lines.push(format!("beta: {}", format_number(orientation.beta(), 3))); + lines.push(format!("gamma: {}", format_number(orientation.gamma(), 3))); + lines.push(format!("absolute: {}", orientation.absolute())); + } else { + lines.push(String::from("waiting for orientation event...")); + } + + lines.push(String::new()); + lines.push(String::from("motion")); + if let Some(motion) = motion { + lines.push(format!( + "acceleration: x={} y={} z={}", + format_number(motion.acceleration_x(), 3), + format_number(motion.acceleration_y(), 3), + format_number(motion.acceleration_z(), 3) + )); + lines.push(format!( + "acceleration including gravity: x={} y={} z={}", + format_number(motion.acceleration_including_gravity_x(), 3), + format_number(motion.acceleration_including_gravity_y(), 3), + format_number(motion.acceleration_including_gravity_z(), 3) + )); + lines.push(format!( + "rotation rate: alpha={} beta={} gamma={}", + format_number(motion.rotation_rate_alpha(), 3), + format_number(motion.rotation_rate_beta(), 3), + format_number(motion.rotation_rate_gamma(), 3) + )); + lines.push(format!("interval: {} ms", format_number(motion.interval_ms(), 1))); + } else { + lines.push(String::from("waiting for motion event...")); + } + + set_textarea_value("sensor-output", &lines.join("\n")) +} + +struct Prediction { + best_index: usize, + best_label: String, + best_probability: f64, + probabilities: Vec, + logits: Vec, +} + +fn log(message: &str) -> Result<(), JsValue> { + let line = format!("[har1] {message}"); + web_sys::console::log_1(&JsValue::from_str(&line)); + + if let Some(window) = web_sys::window() + && let Some(document) = window.document() + && let Some(log_el) = document.get_element_by_id("log") + { + let current = log_el.text_content().unwrap_or_default(); + let next = if current.is_empty() { + line + } else { + format!("{current}\n{line}") + }; + log_el.set_text_content(Some(&next)); + } + + Ok(()) +} + +fn set_har_status(message: &str) -> Result<(), JsValue> { + set_textarea_value("har-output", message) +} + +fn set_textarea_value(element_id: &str, message: &str) -> Result<(), JsValue> { + if let Some(window) = web_sys::window() + && let Some(document) = window.document() + && let Some(output) = document.get_element_by_id(element_id) + { + js_sys::Reflect::set( + output.as_ref(), + &JsValue::from_str("value"), + &JsValue::from_str(message), + )?; + } + + Ok(()) +} + +fn format_number(value: f64, digits: usize) -> String { + if value.is_finite() { + format!("{value:.digits$}") + } else { + String::from("n/a") + } +} + +fn describe_js_error(error: &JsValue) -> String { + error + .as_string() + .or_else(|| js_sys::JSON::stringify(error).ok().map(String::from)) + .unwrap_or_else(|| format!("{error:?}")) +} + +fn method(target: &JsValue, name: &str) -> Result { + Reflect::get(target, &JsValue::from_str(name))? + .dyn_into::() + .map_err(|_| JsValue::from_str(&format!("{name} is not callable"))) +} + +async fn wait_for_connected(client: &WsClient) -> Result<(), JsValue> { + for _ in 0..100 { + if client.get_state() == "connected" { + return Ok(()); + } + sleep_ms(100).await?; + } + + Err(JsValue::from_str("Timed out waiting for websocket connection")) +} + +async fn wait_for_motion_sample(sensors: &DeviceSensors) -> Result<(), JsValue> { + for _ in 0..100 { + if sensors.has_motion() { + return Ok(()); + } + sleep_ms(100).await?; + } + + Err(JsValue::from_str("Timed out waiting for initial motion sample")) +} + +fn websocket_url() -> Result { + let window = web_sys::window().ok_or_else(|| JsValue::from_str("No window available"))?; + let location = Reflect::get(window.as_ref(), &JsValue::from_str("location"))?; + let protocol = Reflect::get(&location, &JsValue::from_str("protocol"))? + .as_string() + .ok_or_else(|| JsValue::from_str("window.location.protocol is unavailable"))?; + let host = Reflect::get(&location, &JsValue::from_str("host"))? + .as_string() + .ok_or_else(|| JsValue::from_str("window.location.host is unavailable"))?; + let ws_protocol = if protocol == "https:" { "wss:" } else { "ws:" }; + Ok(format!("{ws_protocol}//{host}/ws")) +} + +async fn create_har_session(model_path: &str) -> Result { + let window = web_sys::window().ok_or_else(|| JsValue::from_str("No window available"))?; + let ort = Reflect::get(window.as_ref(), &JsValue::from_str("ort"))?; + if ort.is_null() || ort.is_undefined() { + return Err(JsValue::from_str("onnxruntime-web did not load")); + } + + configure_onnx_runtime_wasm(&window, &ort)?; + + let inference_session = Reflect::get(&ort, &JsValue::from_str("InferenceSession"))?; + let create = method(&inference_session, "create")?; + let options = js_sys::Object::new(); + Reflect::set( + &options, + &JsValue::from_str("executionProviders"), + &Array::of1(&JsValue::from_str("wasm")), + )?; + + let value = create.call2(&inference_session, &JsValue::from_str(model_path), &options)?; + JsFuture::from( + value + .dyn_into::() + .map_err(|_| JsValue::from_str("InferenceSession.create did not return a Promise"))?, + ) + .await +} + +fn configure_onnx_runtime_wasm(window: &web_sys::Window, ort: &JsValue) -> Result<(), JsValue> { + let env = Reflect::get(ort, &JsValue::from_str("env"))?; + let wasm = Reflect::get(&env, &JsValue::from_str("wasm"))?; + if wasm.is_null() || wasm.is_undefined() { + return Err(JsValue::from_str("onnxruntime-web environment is unavailable")); + } + + let versions = Reflect::get(&env, &JsValue::from_str("versions"))?; + let ort_version = Reflect::get(&versions, &JsValue::from_str("web"))? + .as_string() + .ok_or_else(|| JsValue::from_str("onnxruntime-web version is unavailable"))?; + let dist_base_url = format!("https://cdn.jsdelivr.net/npm/onnxruntime-web@{ort_version}/dist"); + + let supports_threads = Reflect::get(window.as_ref(), &JsValue::from_str("crossOriginIsolated"))? + .as_bool() + .unwrap_or(false) + && Reflect::has(window.as_ref(), &JsValue::from_str("SharedArrayBuffer"))?; + + Reflect::set( + &wasm, + &JsValue::from_str("numThreads"), + &JsValue::from_f64(if supports_threads { 0.0 } else { 1.0 }), + )?; + + let wasm_paths = js_sys::Object::new(); + Reflect::set( + &wasm_paths, + &JsValue::from_str("mjs"), + &JsValue::from_str(&format!("{dist_base_url}/ort-wasm-simd-threaded.mjs")), + )?; + Reflect::set( + &wasm_paths, + &JsValue::from_str("wasm"), + &JsValue::from_str(&format!("{dist_base_url}/ort-wasm-simd-threaded.wasm")), + )?; + Reflect::set(&wasm, &JsValue::from_str("wasmPaths"), &wasm_paths)?; + Ok(()) +} + +fn first_string_entry(target: &JsValue, field: &str) -> Result { + let values = Reflect::get(target, &JsValue::from_str(field))?; + let first = Reflect::get(&values, &JsValue::from_f64(0.0))?; + first + .as_string() + .ok_or_else(|| JsValue::from_str(&format!("Missing first entry for {field}"))) +} + +async fn infer_prediction( + session: &JsValue, + input_name: &str, + output_name: &str, + sample_buffer: &VecDeque<[f32; HAR_FEATURE_COUNT]>, +) -> Result { + let flat_samples = flatten_samples(sample_buffer); + let tensor = create_tensor(&flat_samples)?; + let feeds = js_sys::Object::new(); + Reflect::set(&feeds, &JsValue::from_str(input_name), &tensor)?; + + let run_value = method(session, "run")?.call1(session, &feeds)?; + let result = JsFuture::from( + run_value + .dyn_into::() + .map_err(|_| JsValue::from_str("InferenceSession.run did not return a Promise"))?, + ) + .await?; + + let output = Reflect::get(&result, &JsValue::from_str(output_name))?; + let data = Reflect::get(&output, &JsValue::from_str("data"))?; + let logits_f32 = Float32Array::new(&data).to_vec(); + let logits: Vec = logits_f32.into_iter().map(f64::from).collect(); + let probabilities = softmax(&logits); + let (best_index, best_probability) = probabilities + .iter() + .copied() + .enumerate() + .max_by(|(_, left), (_, right)| left.partial_cmp(right).unwrap_or(std::cmp::Ordering::Equal)) + .ok_or_else(|| JsValue::from_str("Model returned no prediction scores"))?; + let best_label = HAR_CLASS_LABELS + .get(best_index) + .copied() + .unwrap_or("unknown") + .to_string(); + + Ok(Prediction { + best_index, + best_label, + best_probability, + probabilities, + logits, + }) +} + +fn create_tensor(values: &[f32]) -> Result { + let window = web_sys::window().ok_or_else(|| JsValue::from_str("No window available"))?; + let ort = Reflect::get(window.as_ref(), &JsValue::from_str("ort"))?; + let tensor_ctor = Reflect::get(&ort, &JsValue::from_str("Tensor"))? + .dyn_into::() + .map_err(|_| JsValue::from_str("ort.Tensor is not callable"))?; + + let dims = Array::new(); + dims.push(&JsValue::from_f64(1.0)); + dims.push(&JsValue::from_f64(HAR_SEQUENCE_LENGTH as f64)); + dims.push(&JsValue::from_f64(HAR_FEATURE_COUNT as f64)); + + let args = Array::new(); + args.push(&JsValue::from_str("float32")); + args.push(&Float32Array::from(values).into()); + args.push(&dims.into()); + + Reflect::construct(&tensor_ctor, &args) +} + +fn flatten_samples(sample_buffer: &VecDeque<[f32; HAR_FEATURE_COUNT]>) -> Vec { + sample_buffer.iter().flat_map(|sample| sample.iter().copied()).collect() +} + +fn feature_vector(reading: &MotionReading, gravity_estimate: &mut [f64; 3]) -> [f32; HAR_FEATURE_COUNT] { + let total_acceleration = [ + reading.acceleration_including_gravity_x(), + reading.acceleration_including_gravity_y(), + reading.acceleration_including_gravity_z(), + ]; + + for (index, value) in total_acceleration.iter().enumerate() { + gravity_estimate[index] = GRAVITY_FILTER_ALPHA * gravity_estimate[index] + (1.0 - GRAVITY_FILTER_ALPHA) * value; + } + + let body_acceleration = [ + total_acceleration[0] - gravity_estimate[0], + total_acceleration[1] - gravity_estimate[1], + total_acceleration[2] - gravity_estimate[2], + ]; + + [ + to_g(body_acceleration[0]) as f32, + to_g(body_acceleration[1]) as f32, + to_g(body_acceleration[2]) as f32, + degrees_to_radians(reading.rotation_rate_beta()) as f32, + degrees_to_radians(reading.rotation_rate_gamma()) as f32, + degrees_to_radians(reading.rotation_rate_alpha()) as f32, + to_g(total_acceleration[0]) as f32, + to_g(total_acceleration[1]) as f32, + to_g(total_acceleration[2]) as f32, + ] +} + +fn degrees_to_radians(value: f64) -> f64 { + value * std::f64::consts::PI / 180.0 +} + +fn to_g(value: f64) -> f64 { + value / STANDARD_GRAVITY +} + +fn softmax(values: &[f64]) -> Vec { + if values.is_empty() { + return Vec::new(); + } + + let max_value = values.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let exps: Vec = values.iter().map(|value| (value - max_value).exp()).collect(); + let sum: f64 = exps.iter().sum(); + exps.into_iter().map(|value| value / sum).collect() +} + +async fn sleep_ms(duration_ms: i32) -> Result<(), JsValue> { + let window = web_sys::window().ok_or_else(|| JsValue::from_str("No window available"))?; + let promise = Promise::new(&mut |resolve, reject| { + let callback = Closure::once_into_js(move || { + let _ = resolve.call0(&JsValue::NULL); + }); + + if let Err(error) = + window.set_timeout_with_callback_and_timeout_and_arguments_0(callback.unchecked_ref(), duration_ms) + { + let _ = reject.call1(&JsValue::NULL, &error); + } + }); + JsFuture::from(promise).await.map(|_| ()) +} diff --git a/services/ws-server/README.md b/services/ws-server/README.md index 4928ebe..83d9abe 100644 --- a/services/ws-server/README.md +++ b/services/ws-server/README.md @@ -9,5 +9,6 @@ A Rust service using `actix-web` with WebSocket support. - OpenTelemetry tracing for all WebSocket operations - JSON message protocol - Connection lifecycle management -- Browser demo page at `/` -- Static WASM package `../ws-wasm-agent` served from `/pkg`. +- Browser interface page at `/` +- Static WASM package `../ws-wasm-agent/pkg` served from `/pkg`. +- WASM workflow modules under `../ws-modules` served from `/modules`. diff --git a/services/ws-server/src/main.rs b/services/ws-server/src/main.rs index 3538a1d..176ea50 100644 --- a/services/ws-server/src/main.rs +++ b/services/ws-server/src/main.rs @@ -275,16 +275,20 @@ fn wasm_pkg_dir() -> PathBuf { workspace_root().join("services/ws-wasm-agent/pkg") } +fn wasm_modules_dir() -> PathBuf { + workspace_root().join("services/ws-modules") +} + fn browser_static_dir() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("static") } async fn browser_index() -> Result { let path = browser_static_dir().join("index.html"); - info!("Serving browser demo page: {:?}", path); + info!("Serving browser interface page: {:?}", path); NamedFile::open(path).map_err(|e| { - error!("Failed to open browser demo page: {}", e); + error!("Failed to open browser interface page: {}", e); actix_web::error::ErrorNotFound(e) }) } @@ -365,6 +369,7 @@ async fn main() -> std::io::Result<()> { info!("Starting WebSocket server on https://localhost:8443"); info!("Serving browser assets from {:?}", browser_static_dir()); info!("Serving wasm package from {:?}", wasm_pkg_dir()); + info!("Serving wasm modules from {:?}", wasm_modules_dir()); info!("HTTPS uses an in-memory self-signed localhost certificate for development"); let agent_registry = web::Data::new(AgentRegistry::default()); @@ -377,6 +382,7 @@ async fn main() -> std::io::Result<()> { .route("/health", web::get().to(health)) .route("/ws", web::get().to(ws_handler)) .route("/files/{filename}", web::get().to(file_handler)) + .service(Files::new("/modules", wasm_modules_dir()).prefer_utf8(true)) .service(Files::new("/pkg", wasm_pkg_dir()).prefer_utf8(true)) .service(Files::new("/static", browser_static_dir()).prefer_utf8(true)) }) diff --git a/services/ws-server/static/app.js b/services/ws-server/static/app.js index 43409ac..e9ebc60 100644 --- a/services/ws-server/static/app.js +++ b/services/ws-server/static/app.js @@ -3,6 +3,7 @@ import init, { GeolocationReading, GpuInfo, GraphicsSupport, + initTracing, MicrophoneAccess, NfcScanResult, SpeechRecognitionSession, @@ -13,6 +14,7 @@ import init, { } from "/pkg/et_ws_wasm_agent.js"; const logEl = document.getElementById("log"); +const runHarButton = document.getElementById("run-har-button"); const micButton = document.getElementById("mic-button"); const videoButton = document.getElementById("video-button"); const bluetoothButton = document.getElementById("bluetooth-button"); @@ -23,14 +25,11 @@ const gpuInfoButton = document.getElementById("gpu-info-button"); const speechButton = document.getElementById("speech-button"); const nfcButton = document.getElementById("nfc-button"); const sensorsButton = document.getElementById("sensors-button"); -const harButton = document.getElementById("har-button"); const videoModelButton = document.getElementById("video-model-button"); const videoOutputButton = document.getElementById("video-output-button"); -const harExportButton = document.getElementById("har-export-button"); const agentStatusEl = document.getElementById("agent-status"); const agentIdEl = document.getElementById("agent-id"); const sensorOutputEl = document.getElementById("sensor-output"); -const harOutputEl = document.getElementById("har-output"); const videoOutputEl = document.getElementById("video-output"); const videoPreview = document.getElementById("video-preview"); const videoOutputCanvas = document.getElementById("video-output-canvas"); @@ -42,14 +41,6 @@ let speechListening = false; let sensorsActive = false; let orientationState = null; let motionState = null; -let harSession = null; -let harInputName = null; -let harOutputName = null; -let harSampleBuffer = []; -let harInferencePending = false; -let lastInferenceAt = 0; -let harSamplerId = null; -let lastHarClassLabel = null; let videoCvSession = null; let videoCvInputName = null; let videoCvOutputName = null; @@ -63,11 +54,7 @@ let videoOverlayContext = videoOutputCanvas.getContext("2d"); let videoOutputVisible = false; let videoRenderFrameId = null; let lastVideoInferenceSummary = null; -let gravityEstimate = { x: 0, y: 0, z: 0 }; let sendClientEvent = () => {}; -const HAR_SEQUENCE_LENGTH = 512; -const HAR_FEATURE_COUNT = 9; -const HAR_SAMPLE_INTERVAL_MS = 20; const VIDEO_INFERENCE_INTERVAL_MS = 750; const VIDEO_RENDER_SCORE_THRESHOLD = 0.35; const VIDEO_MODEL_PATH = "/static/models/video_cv.onnx"; @@ -80,27 +67,6 @@ const RETINAFACE_VARIANCES = [0.1, 0.2]; const RETINAFACE_MIN_SIZES = [[16, 32], [64, 128], [256, 512]]; const RETINAFACE_STEPS = [8, 16, 32]; const RETINAFACE_MEAN_BGR = [104, 117, 123]; -const STANDARD_GRAVITY = 9.80665; -const GRAVITY_FILTER_ALPHA = 0.8; -const HAR_CLASS_LABELS = [ - "class_0", - "class_1", - "class_2", - "class_3", - "class_4", - "class_5", -]; -const HAR_CHANNEL_NAMES = [ - "body_acc_x", - "body_acc_y", - "body_acc_z", - "body_gyro_x", - "body_gyro_y", - "body_gyro_z", - "total_acc_x", - "total_acc_y", - "total_acc_z", -]; const STORED_AGENT_ID_KEY = "ws_wasm_agent.agent_id"; let currentAgentId = null; @@ -108,6 +74,10 @@ const append = (line) => { logEl.textContent += `\n${line}`; }; +const describeError = (error) => ( + error instanceof Error ? error.message : String(error) +); + const updateAgentCard = (status, agentId = currentAgentId) => { currentAgentId = agentId || null; agentStatusEl.textContent = status; @@ -160,52 +130,6 @@ const formatNumber = (value, digits = 3) => ( Number.isFinite(value) ? value.toFixed(digits) : "n/a" ); -const configureOnnxRuntimeWasm = () => { - if (!window.ort?.env?.wasm) { - throw new Error("onnxruntime-web environment is unavailable."); - } - - const ortVersion = window.ort.env.versions?.web; - if (typeof ortVersion !== "string" || ortVersion.length === 0) { - throw new Error("onnxruntime-web version is unavailable."); - } - - const distBaseUrl = `https://cdn.jsdelivr.net/npm/onnxruntime-web@${ortVersion}/dist`; - const supportsWasmThreads = window.crossOriginIsolated === true - && typeof SharedArrayBuffer !== "undefined"; - - window.ort.env.wasm.numThreads = supportsWasmThreads ? 0 : 1; - window.ort.env.wasm.wasmPaths = { - mjs: `${distBaseUrl}/ort-wasm-simd-threaded.mjs`, - wasm: `${distBaseUrl}/ort-wasm-simd-threaded.wasm`, - }; - - append( - `onnxruntime-web configured: version=${ortVersion} wasm=${window.ort.env.wasm.wasmPaths.wasm} threads=${ - window.ort.env.wasm.numThreads === 1 ? "disabled" : "auto" - }`, - ); -}; - -const degreesToRadians = (value) => ( - Number.isFinite(value) ? (value * Math.PI) / 180 : 0 -); - -const softmax = (values) => { - if (!values.length) { - return []; - } - - const maxValue = Math.max(...values); - const exps = values.map((value) => Math.exp(value - maxValue)); - const sum = exps.reduce((accumulator, value) => accumulator + value, 0); - return exps.map((value) => value / sum); -}; - -const toG = (value) => ( - Number.isFinite(value) ? value / STANDARD_GRAVITY : 0 -); - const renderSensorOutput = () => { const lines = [ "Device sensor stream", @@ -249,27 +173,10 @@ const renderSensorOutput = () => { sensorOutputEl.value = lines.join("\n"); }; -const setHarOutput = (lines) => { - harOutputEl.value = Array.isArray(lines) ? lines.join("\n") : String(lines); -}; - const setVideoOutput = (lines) => { videoOutputEl.value = Array.isArray(lines) ? lines.join("\n") : String(lines); }; -const updateHarStatus = (extraLines = []) => { - const lines = [ - `model: ${harSession ? "loaded" : "not loaded"}`, - `input: ${harInputName ?? "n/a"}`, - `output: ${harOutputName ?? "n/a"}`, - "layout: [batch, time, features]", - `window: ${HAR_SEQUENCE_LENGTH}`, - `features: ${HAR_FEATURE_COUNT}`, - `buffered samples: ${harSampleBuffer.length}`, - ]; - setHarOutput(lines.concat("", extraLines)); -}; - const updateVideoStatus = (extraLines = []) => { const inputMetadata = videoCvInputName ? videoCvSession?.inputMetadata?.[videoCvInputName] @@ -291,160 +198,6 @@ const updateVideoStatus = (extraLines = []) => { setVideoOutput(lines.concat("", extraLines)); }; -const getFeatureVector = () => { - const totalAcceleration = motionState?.accelerationIncludingGravity ?? { x: 0, y: 0, z: 0 }; - const bodyAcceleration = { - x: totalAcceleration.x - gravityEstimate.x, - y: totalAcceleration.y - gravityEstimate.y, - z: totalAcceleration.z - gravityEstimate.z, - }; - - return [ - toG(bodyAcceleration.x), - toG(bodyAcceleration.y), - toG(bodyAcceleration.z), - degreesToRadians(motionState?.rotationRate?.beta), - degreesToRadians(motionState?.rotationRate?.gamma), - degreesToRadians(motionState?.rotationRate?.alpha), - toG(totalAcceleration.x), - toG(totalAcceleration.y), - toG(totalAcceleration.z), - ]; -}; - -const flattenSamplesForModel = () => { - return harSampleBuffer.slice(-HAR_SEQUENCE_LENGTH).flat(); -}; - -const exportHarWindow = () => { - if (harSampleBuffer.length < HAR_SEQUENCE_LENGTH) { - throw new Error(`Need ${HAR_SEQUENCE_LENGTH} samples before export.`); - } - - const samples = harSampleBuffer.slice(-HAR_SEQUENCE_LENGTH); - const columns = []; - - for (let timeIndex = 0; timeIndex < HAR_SEQUENCE_LENGTH; timeIndex += 1) { - for (const channelName of HAR_CHANNEL_NAMES) { - columns.push(`t${timeIndex}_${channelName}`); - } - } - - const values = []; - for (let timeIndex = 0; timeIndex < HAR_SEQUENCE_LENGTH; timeIndex += 1) { - for (let channelIndex = 0; channelIndex < HAR_CHANNEL_NAMES.length; channelIndex += 1) { - values.push(String(samples[timeIndex][channelIndex] ?? 0)); - } - } - - const csv = `${columns.join(",")},true_label\n${values.join(",")},\n`; - const blob = new Blob([csv], { type: "text/csv;charset=utf-8" }); - const url = URL.createObjectURL(blob); - const link = document.createElement("a"); - link.href = url; - link.download = "har_window.csv"; - link.click(); - URL.revokeObjectURL(url); -}; - -const inferHarPrediction = async () => { - if ( - !harSession - || !harInputName - || !harOutputName - || harInferencePending - || harSampleBuffer.length < HAR_SEQUENCE_LENGTH - ) { - updateHarStatus(); - return; - } - - const now = Date.now(); - if (now - lastInferenceAt < 250) { - return; - } - - harInferencePending = true; - lastInferenceAt = now; - - try { - const input = new window.ort.Tensor( - "float32", - Float32Array.from(flattenSamplesForModel()), - [1, HAR_SEQUENCE_LENGTH, HAR_FEATURE_COUNT], - ); - - const result = await harSession.run({ [harInputName]: input }); - const output = result[harOutputName]; - const logits = Array.from(output.data ?? []); - const probabilities = softmax(logits); - const bestProbability = Math.max(...probabilities); - const bestIndex = probabilities.indexOf(bestProbability); - const bestLabel = HAR_CLASS_LABELS[bestIndex] ?? `class_${bestIndex}`; - const allScores = probabilities.map((probability, index) => { - const label = HAR_CLASS_LABELS[index] ?? `class_${index}`; - const logit = logits[index] ?? 0; - return `${label}: p=${probability.toFixed(4)} logit=${logit.toFixed(4)}`; - }); - - if (bestLabel !== lastHarClassLabel) { - sendClientEvent("har", "class_changed", { - detected_class: bestLabel, - previous_class: lastHarClassLabel, - class_index: bestIndex, - confidence: bestProbability, - probabilities, - logits, - buffered_samples: harSampleBuffer.length, - detected_at: new Date().toISOString(), - }); - lastHarClassLabel = bestLabel; - } - - updateHarStatus([ - `prediction: ${bestLabel}`, - `confidence: ${bestProbability.toFixed(4)}`, - "all classes:", - ...allScores, - ]); - } catch (error) { - updateHarStatus([ - `inference error: ${error instanceof Error ? error.message : String(error)}`, - ]); - console.error(error); - } finally { - harInferencePending = false; - } -}; - -const pushHarSample = () => { - if (!harSession || !sensorsActive || !motionState) { - return; - } - - harSampleBuffer.push(getFeatureVector()); - if (harSampleBuffer.length > HAR_SEQUENCE_LENGTH) { - harSampleBuffer.shift(); - } - - void inferHarPrediction(); -}; - -const stopHarSampler = () => { - if (harSamplerId !== null) { - window.clearInterval(harSamplerId); - harSamplerId = null; - } - lastHarClassLabel = null; -}; - -const startHarSampler = () => { - stopHarSampler(); - harSamplerId = window.setInterval(() => { - pushHarSample(); - }, HAR_SAMPLE_INTERVAL_MS); -}; - const handleOrientation = (event) => { orientationState = { alpha: event.alpha, @@ -464,14 +217,6 @@ const handleMotion = (event) => { } : null; - if (accelerationIncludingGravity) { - gravityEstimate = { - x: GRAVITY_FILTER_ALPHA * gravityEstimate.x + (1 - GRAVITY_FILTER_ALPHA) * accelerationIncludingGravity.x, - y: GRAVITY_FILTER_ALPHA * gravityEstimate.y + (1 - GRAVITY_FILTER_ALPHA) * accelerationIncludingGravity.y, - z: GRAVITY_FILTER_ALPHA * gravityEstimate.z + (1 - GRAVITY_FILTER_ALPHA) * accelerationIncludingGravity.z, - }; - } - motionState = { acceleration: event.acceleration ? { @@ -491,7 +236,6 @@ const handleMotion = (event) => { interval: event.interval, }; renderSensorOutput(); - pushHarSample(); }; const requestSensorPermission = async (permissionTarget) => { @@ -502,6 +246,46 @@ const requestSensorPermission = async (permissionTarget) => { return permissionTarget.requestPermission(); }; +const stopSensorsFlow = () => { + window.removeEventListener("deviceorientation", handleOrientation); + window.removeEventListener("devicemotion", handleMotion); + sensorsActive = false; + sensorsButton.textContent = "Start sensors"; + append("device sensors stopped"); +}; + +const startSensorsFlow = async () => { + if ( + typeof window.DeviceOrientationEvent === "undefined" + && typeof window.DeviceMotionEvent === "undefined" + ) { + throw new Error("Device orientation and motion APIs are not supported in this browser."); + } + + const [orientationPermission, motionPermission] = await Promise.all([ + requestSensorPermission(window.DeviceOrientationEvent), + requestSensorPermission(window.DeviceMotionEvent), + ]); + + if ( + orientationPermission !== "granted" + || motionPermission !== "granted" + ) { + throw new Error( + `Sensor permission denied (orientation=${orientationPermission}, motion=${motionPermission})`, + ); + } + + orientationState = null; + motionState = null; + renderSensorOutput(); + window.addEventListener("deviceorientation", handleOrientation); + window.addEventListener("devicemotion", handleMotion); + sensorsActive = true; + sensorsButton.textContent = "Stop sensors"; + append("device sensors started; streaming locally to textbox"); +}; + const getTopK = (values, limit = 3) => { return values .map((value, index) => ({ value, index })) @@ -1469,25 +1253,11 @@ const syncVideoCvLoop = () => { }; renderSensorOutput(); -updateHarStatus([ - "local-only inference path", - "model file: /static/models/human_activity_recognition.onnx", -]); updateVideoStatus([ `model file: ${VIDEO_MODEL_PATH}`, "load the model, then start video capture to process frames in-browser.", ]); -harExportButton.addEventListener("click", () => { - try { - exportHarWindow(); - append("exported current HAR window to har_window.csv"); - } catch (error) { - append(`har export error: ${error instanceof Error ? error.message : String(error)}`); - console.error(error); - } -}); - const wsProtocol = window.location.protocol === "https:" ? "wss:" : "ws:"; const wsUrl = `${wsProtocol}//${window.location.host}/ws`; const retainedAgentId = readStoredAgentId(); @@ -1502,6 +1272,7 @@ updateAgentCard( try { await init(); + initTracing(); const config = new WsClientConfig(wsUrl); const client = new WsClient(config); @@ -1799,114 +1570,40 @@ try { sensorsButton.addEventListener("click", async () => { try { if (sensorsActive) { - window.removeEventListener("deviceorientation", handleOrientation); - window.removeEventListener("devicemotion", handleMotion); - stopHarSampler(); - sensorsActive = false; - sensorsButton.textContent = "Start sensors"; - append("device sensors stopped"); + stopSensorsFlow(); return; } - if ( - typeof window.DeviceOrientationEvent === "undefined" - && typeof window.DeviceMotionEvent === "undefined" - ) { - throw new Error("Device orientation and motion APIs are not supported in this browser."); - } - - const [orientationPermission, motionPermission] = await Promise.all([ - requestSensorPermission(window.DeviceOrientationEvent), - requestSensorPermission(window.DeviceMotionEvent), - ]); - - if ( - orientationPermission !== "granted" - || motionPermission !== "granted" - ) { - throw new Error( - `Sensor permission denied (orientation=${orientationPermission}, motion=${motionPermission})`, - ); - } - - orientationState = null; - motionState = null; - gravityEstimate = { x: 0, y: 0, z: 0 }; - harSampleBuffer = []; - renderSensorOutput(); - window.addEventListener("deviceorientation", handleOrientation); - window.addEventListener("devicemotion", handleMotion); - startHarSampler(); - sensorsActive = true; - sensorsButton.textContent = "Stop sensors"; - append("device sensors started; streaming locally to textbox"); + await startSensorsFlow(); } catch (error) { - append(`sensor error: ${error instanceof Error ? error.message : String(error)}`); + append(`sensor error: ${describeError(error)}`); console.error(error); } }); - harButton.addEventListener("click", async () => { - try { - if (!window.ort) { - throw new Error("onnxruntime-web did not load."); - } + runHarButton.addEventListener("click", async () => { + runHarButton.disabled = true; + runHarButton.textContent = "Running har demo..."; - configureOnnxRuntimeWasm(); - - harButton.disabled = true; - harButton.textContent = "Loading HAR..."; - updateHarStatus(["loading model..."]); - - harSession = await window.ort.InferenceSession.create( - "/static/models/human_activity_recognition.onnx", - { - executionProviders: ["wasm"], - }, - ); - - harInputName = harSession.inputNames[0] ?? null; - harOutputName = harSession.outputNames[0] ?? null; - - const inputMetadata = harInputName - ? harSession.inputMetadata?.[harInputName] - : null; - const runtimeDimensions = Array.isArray(inputMetadata?.dimensions) - ? inputMetadata.dimensions - : []; - - harSampleBuffer = []; - gravityEstimate = { x: 0, y: 0, z: 0 }; - harButton.textContent = "Reload HAR model"; - if (sensorsActive) { - startHarSampler(); - } - append( - `har model loaded: input=${harInputName} output=${harOutputName} runtime_dims=${ - JSON.stringify(runtimeDimensions) - } expected_dims=["batch",512,9]`, - ); - updateHarStatus([ - `runtime dimensions: ${JSON.stringify(runtimeDimensions)}`, - "expected dimensions: [batch, 512, 9]", - "feature order: body_acc xyz, body_gyro xyz, total_acc xyz", - "sampling target: 50 Hz from DeviceMotion events", - "predictions remain browser-local and are not sent over the websocket.", - ]); + try { + const cacheBust = Date.now(); + const moduleUrl = `/modules/har1/pkg/et_ws_har1.js?v=${cacheBust}`; + const wasmUrl = `/modules/har1/pkg/et_ws_har1_bg.wasm?v=${cacheBust}`; + append(`har1 module: importing ${moduleUrl}`); + const har1Module = await import(moduleUrl); + append(`har1 module: initializing ${wasmUrl}`); + await har1Module.default(wasmUrl); + append("har1 module: calling run()"); + const runPromise = har1Module.run(); + append("har1 module: run() started"); + await runPromise; + append("har1 module completed"); } catch (error) { - harSession = null; - harInputName = null; - harOutputName = null; - harSampleBuffer = []; - stopHarSampler(); - updateHarStatus([ - `model load error: ${error instanceof Error ? error.message : String(error)}`, - ]); - append(`har error: ${error instanceof Error ? error.message : String(error)}`); + append(`har1 module error: ${describeError(error)}`); console.error(error); } finally { - harButton.disabled = false; - harButton.textContent = harSession ? "Reload HAR model" : "Load HAR model"; + runHarButton.disabled = false; + runHarButton.textContent = "har demo"; } }); @@ -1963,6 +1660,7 @@ try { window.client = client; window.sendAlive = () => client.send_alive(); + window.runHarModule = () => runHarButton.click(); } catch (error) { append(`error: ${error instanceof Error ? error.message : String(error)}`); console.error(error); diff --git a/services/ws-server/static/index.html b/services/ws-server/static/index.html index 2ccf512..73335b0 100644 --- a/services/ws-server/static/index.html +++ b/services/ws-server/static/index.html @@ -121,6 +121,7 @@

WASM web agent

unassigned

+ - -

diff --git a/services/ws-wasm-agent/src/lib.rs b/services/ws-wasm-agent/src/lib.rs index 525f9ef..18dd24c 100644 --- a/services/ws-wasm-agent/src/lib.rs +++ b/services/ws-wasm-agent/src/lib.rs @@ -15,16 +15,19 @@ const MAX_OFFLINE_QUEUE_LEN: usize = 1000; /// Default cadence for client-side app-level `Alive` messages sent to the websocket server. /// This should remain comfortably lower than the server's idle connection timeout. const DEFAULT_ALIVE_INTERVAL_MS: u32 = 5_000; +const SENSOR_PERMISSION_GRANTED: &str = "granted"; // Initialize logging for WASM -#[wasm_bindgen(start)] -pub fn init() { - // Initialize tracing +pub fn init_logging() { tracing_wasm::set_as_global_default(); - info!("WebSocket client initialized"); } +#[wasm_bindgen(js_name = initTracing)] +pub fn init_tracing() { + init_logging(); +} + // Connection state #[derive(Debug, Clone, PartialEq)] pub enum ConnectionState { @@ -97,6 +100,47 @@ pub struct NfcScanResult { record_summary: String, } +#[derive(Clone, Default)] +struct OrientationReadingState { + alpha: Option, + beta: Option, + gamma: Option, + absolute: Option, +} + +#[derive(Clone, Default)] +struct MotionReadingState { + acceleration_x: Option, + acceleration_y: Option, + acceleration_z: Option, + acceleration_including_gravity_x: Option, + acceleration_including_gravity_y: Option, + acceleration_including_gravity_z: Option, + rotation_rate_alpha: Option, + rotation_rate_beta: Option, + rotation_rate_gamma: Option, + interval_ms: Option, +} + +#[wasm_bindgen] +pub struct OrientationReading { + inner: OrientationReadingState, +} + +#[wasm_bindgen] +pub struct MotionReading { + inner: MotionReadingState, +} + +#[wasm_bindgen] +pub struct DeviceSensors { + active: bool, + orientation_state: Rc>>, + motion_state: Rc>>, + orientation_listener: Option>, + motion_listener: Option>, +} + #[wasm_bindgen] impl MicrophoneAccess { #[wasm_bindgen(js_name = request)] @@ -357,6 +401,235 @@ impl GeolocationReading { } } +#[wasm_bindgen] +impl OrientationReading { + pub fn alpha(&self) -> f64 { + self.inner.alpha.unwrap_or(0.0) + } + + pub fn beta(&self) -> f64 { + self.inner.beta.unwrap_or(0.0) + } + + pub fn gamma(&self) -> f64 { + self.inner.gamma.unwrap_or(0.0) + } + + pub fn absolute(&self) -> bool { + self.inner.absolute.unwrap_or(false) + } +} + +#[wasm_bindgen] +impl MotionReading { + #[wasm_bindgen(js_name = accelerationX)] + pub fn acceleration_x(&self) -> f64 { + self.inner.acceleration_x.unwrap_or(0.0) + } + + #[wasm_bindgen(js_name = accelerationY)] + pub fn acceleration_y(&self) -> f64 { + self.inner.acceleration_y.unwrap_or(0.0) + } + + #[wasm_bindgen(js_name = accelerationZ)] + pub fn acceleration_z(&self) -> f64 { + self.inner.acceleration_z.unwrap_or(0.0) + } + + #[wasm_bindgen(js_name = accelerationIncludingGravityX)] + pub fn acceleration_including_gravity_x(&self) -> f64 { + self.inner.acceleration_including_gravity_x.unwrap_or(0.0) + } + + #[wasm_bindgen(js_name = accelerationIncludingGravityY)] + pub fn acceleration_including_gravity_y(&self) -> f64 { + self.inner.acceleration_including_gravity_y.unwrap_or(0.0) + } + + #[wasm_bindgen(js_name = accelerationIncludingGravityZ)] + pub fn acceleration_including_gravity_z(&self) -> f64 { + self.inner.acceleration_including_gravity_z.unwrap_or(0.0) + } + + #[wasm_bindgen(js_name = rotationRateAlpha)] + pub fn rotation_rate_alpha(&self) -> f64 { + self.inner.rotation_rate_alpha.unwrap_or(0.0) + } + + #[wasm_bindgen(js_name = rotationRateBeta)] + pub fn rotation_rate_beta(&self) -> f64 { + self.inner.rotation_rate_beta.unwrap_or(0.0) + } + + #[wasm_bindgen(js_name = rotationRateGamma)] + pub fn rotation_rate_gamma(&self) -> f64 { + self.inner.rotation_rate_gamma.unwrap_or(0.0) + } + + #[wasm_bindgen(js_name = intervalMs)] + pub fn interval_ms(&self) -> f64 { + self.inner.interval_ms.unwrap_or(0.0) + } +} + +impl Default for DeviceSensors { + fn default() -> Self { + Self::new() + } +} + +#[wasm_bindgen] +impl DeviceSensors { + #[wasm_bindgen(constructor)] + pub fn new() -> DeviceSensors { + DeviceSensors { + active: false, + orientation_state: Rc::new(RefCell::new(None)), + motion_state: Rc::new(RefCell::new(None)), + orientation_listener: None, + motion_listener: None, + } + } + + pub async fn start(&mut self) -> Result<(), JsValue> { + if self.active { + return Ok(()); + } + + let window = web_sys::window().ok_or_else(|| JsValue::from_str("No window available"))?; + + if js_sys::Reflect::get(&window, &JsValue::from_str("DeviceOrientationEvent"))?.is_undefined() + && js_sys::Reflect::get(&window, &JsValue::from_str("DeviceMotionEvent"))?.is_undefined() + { + return Err(JsValue::from_str( + "Device orientation and motion APIs are not supported in this browser.", + )); + } + + let orientation_permission = request_sensor_permission(js_sys::Reflect::get( + &window, + &JsValue::from_str("DeviceOrientationEvent"), + )?) + .await?; + let motion_permission = + request_sensor_permission(js_sys::Reflect::get(&window, &JsValue::from_str("DeviceMotionEvent"))?).await?; + + if orientation_permission != SENSOR_PERMISSION_GRANTED || motion_permission != SENSOR_PERMISSION_GRANTED { + return Err(JsValue::from_str(&format!( + "Sensor permission denied (orientation={orientation_permission}, motion={motion_permission})" + ))); + } + + *self.orientation_state.borrow_mut() = None; + *self.motion_state.borrow_mut() = None; + + let orientation_state = self.orientation_state.clone(); + let orientation_listener = Closure::wrap(Box::new(move |event: Event| { + let value: JsValue = event.into(); + *orientation_state.borrow_mut() = Some(OrientationReadingState { + alpha: js_number_field(&value, "alpha"), + beta: js_number_field(&value, "beta"), + gamma: js_number_field(&value, "gamma"), + absolute: js_bool_field(&value, "absolute"), + }); + }) as Box); + + let motion_state = self.motion_state.clone(); + let motion_listener = Closure::wrap(Box::new(move |event: Event| { + let value: JsValue = event.into(); + let acceleration = js_nested_object(&value, "acceleration"); + let acceleration_including_gravity = js_nested_object(&value, "accelerationIncludingGravity"); + let rotation_rate = js_nested_object(&value, "rotationRate"); + + *motion_state.borrow_mut() = Some(MotionReadingState { + acceleration_x: acceleration.as_ref().and_then(|v| js_number_field(v, "x")), + acceleration_y: acceleration.as_ref().and_then(|v| js_number_field(v, "y")), + acceleration_z: acceleration.as_ref().and_then(|v| js_number_field(v, "z")), + acceleration_including_gravity_x: acceleration_including_gravity + .as_ref() + .and_then(|v| js_number_field(v, "x")), + acceleration_including_gravity_y: acceleration_including_gravity + .as_ref() + .and_then(|v| js_number_field(v, "y")), + acceleration_including_gravity_z: acceleration_including_gravity + .as_ref() + .and_then(|v| js_number_field(v, "z")), + rotation_rate_alpha: rotation_rate.as_ref().and_then(|v| js_number_field(v, "alpha")), + rotation_rate_beta: rotation_rate.as_ref().and_then(|v| js_number_field(v, "beta")), + rotation_rate_gamma: rotation_rate.as_ref().and_then(|v| js_number_field(v, "gamma")), + interval_ms: js_number_field(&value, "interval"), + }); + }) as Box); + + let target: &web_sys::EventTarget = window.as_ref(); + target.add_event_listener_with_callback("deviceorientation", orientation_listener.as_ref().unchecked_ref())?; + target.add_event_listener_with_callback("devicemotion", motion_listener.as_ref().unchecked_ref())?; + + self.orientation_listener = Some(orientation_listener); + self.motion_listener = Some(motion_listener); + self.active = true; + info!("Device sensors started"); + Ok(()) + } + + pub fn stop(&mut self) -> Result<(), JsValue> { + if !self.active { + return Ok(()); + } + + let window = web_sys::window().ok_or_else(|| JsValue::from_str("No window available"))?; + let target: &web_sys::EventTarget = window.as_ref(); + + if let Some(listener) = self.orientation_listener.as_ref() { + target.remove_event_listener_with_callback("deviceorientation", listener.as_ref().unchecked_ref())?; + } + + if let Some(listener) = self.motion_listener.as_ref() { + target.remove_event_listener_with_callback("devicemotion", listener.as_ref().unchecked_ref())?; + } + + self.orientation_listener = None; + self.motion_listener = None; + self.active = false; + info!("Device sensors stopped"); + Ok(()) + } + + #[wasm_bindgen(js_name = isActive)] + pub fn is_active(&self) -> bool { + self.active + } + + #[wasm_bindgen(js_name = hasOrientation)] + pub fn has_orientation(&self) -> bool { + self.orientation_state.borrow().is_some() + } + + #[wasm_bindgen(js_name = hasMotion)] + pub fn has_motion(&self) -> bool { + self.motion_state.borrow().is_some() + } + + #[wasm_bindgen(js_name = orientationSnapshot)] + pub fn orientation_snapshot(&self) -> Result { + self.orientation_state + .borrow() + .clone() + .map(|inner| OrientationReading { inner }) + .ok_or_else(|| JsValue::from_str("No orientation reading available yet")) + } + + #[wasm_bindgen(js_name = motionSnapshot)] + pub fn motion_snapshot(&self) -> Result { + self.motion_state + .borrow() + .clone() + .map(|inner| MotionReading { inner }) + .ok_or_else(|| JsValue::from_str("No motion reading available yet")) + } +} + #[wasm_bindgen] impl GraphicsSupport { #[wasm_bindgen(js_name = detect)] @@ -971,6 +1244,47 @@ fn string_or_unknown(value: String) -> String { } } +async fn request_sensor_permission(target: JsValue) -> Result { + if target.is_null() || target.is_undefined() { + return Ok(SENSOR_PERMISSION_GRANTED.to_string()); + } + + let request_permission = js_sys::Reflect::get(&target, &JsValue::from_str("requestPermission"))?; + if request_permission.is_null() || request_permission.is_undefined() { + return Ok(SENSOR_PERMISSION_GRANTED.to_string()); + } + + let request_permission = request_permission + .dyn_into::() + .map_err(|_| JsValue::from_str("requestPermission is not callable"))?; + let promise = request_permission + .call0(&target)? + .dyn_into::() + .map_err(|_| JsValue::from_str("requestPermission did not return a Promise"))?; + let result = JsFuture::from(promise).await?; + Ok(result + .as_string() + .unwrap_or_else(|| SENSOR_PERMISSION_GRANTED.to_string())) +} + +fn js_number_field(value: &JsValue, field: &str) -> Option { + js_sys::Reflect::get(value, &JsValue::from_str(field)) + .ok() + .and_then(|field_value| field_value.as_f64()) +} + +fn js_bool_field(value: &JsValue, field: &str) -> Option { + js_sys::Reflect::get(value, &JsValue::from_str(field)) + .ok() + .and_then(|field_value| field_value.as_bool()) +} + +fn js_nested_object(value: &JsValue, field: &str) -> Option { + js_sys::Reflect::get(value, &JsValue::from_str(field)) + .ok() + .filter(|nested| !nested.is_null() && !nested.is_undefined()) +} + fn extract_speech_event_transcript(event: &JsValue) -> Option<(String, f64, bool)> { let results = js_sys::Reflect::get(event, &JsValue::from_str("results")).ok()?; let length = js_sys::Reflect::get(&results, &JsValue::from_str("length")) @@ -1626,6 +1940,24 @@ impl WsClient { } } +impl WsClient { + pub fn send_client_event( + &self, + capability: impl Into, + action: impl Into, + details: serde_json::Value, + ) -> Result<(), JsValue> { + let message = WsMessage::ClientEvent { + capability: capability.into(), + action: action.into(), + details, + }; + let payload = serde_json::to_string(&message) + .map_err(|error| JsValue::from_str(&format!("Failed to serialize client event: {error}")))?; + self.send(&payload) + } +} + // Implement Clone for WsClient (required for closures) impl Clone for WsClient { fn clone(&self) -> WsClient {