diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index ddcc4bf1..84a37b51 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -140,19 +140,22 @@ goawk -f "$keep" "$covdir/wasi.lcov" >> lcov.info shell = "bash -euo pipefail -c" [tasks.pytest-cov] -description = "Combined Python coverage (pyface1 pytest + web-runner Pyodide runs) -> Cobertura for Codecov/DeepSource" -# One Python report is combined from three data sources: pyface1's pytest run plus the web-runner Pyodide runs. -# The web-runner integration test drops pydata1's and pyface1's .coverage files into target/pycov/ when -# ET_TEST_COVERAGE is set; `coverage combine` then merges all three via config/coverage.toml's `[paths]` remap into -# a single Cobertura file. pytest-cov is a plugin, so it rides in pyface1's uv `dev` group -- the same venv as -# pytest -- and `uv run` (a mise-managed tool) installs both into the one project venv. UV_PYTHON mirrors -# test-pyface1's mise-CPython pin. +description = "Combined Python coverage (pytest + web-runner Pyodide runs) -> Cobertura for Codecov/DeepSource" +# One Python report is combined from several data sources: the pytest runs plus the web-runner Pyodide runs. +# The pyface1 + pyeye1 pytest runs and the web-runner integration test (which drops the modules' .coverage files +# into target/pycov/ when ET_TEST_COVERAGE is set) all feed `coverage combine`, which merges them via +# config/coverage.toml's `[paths]` remap into a single Cobertura file. pytest-cov is a plugin, so it rides in +# each module's uv `dev` group -- the same venv as pytest -- and `uv run` (a mise-managed tool) installs both +# into the one project venv. UV_PYTHON mirrors test-pyface1's mise-CPython pin. run = """ pyface=services/ws-modules/pyface1 +pyeye=services/ws-modules/pyeye1 pycov=target/pycov mkdir -p "$pycov" uv run --directory "$pyface" pytest --cov=pyface1 --cov-report= cp "$pyface/.coverage" "$pycov/.coverage.pyface1_pytest" +uv run --directory "$pyeye" pytest --cov=pyeye1 --cov-report= +cp "$pyeye/.coverage" "$pycov/.coverage.pyeye1_pytest" uv run --project "$pyface" coverage combine "$pycov" uv run --project "$pyface" coverage xml -o coverage-python.xml """ diff --git a/.mise/config.js.toml b/.mise/config.js.toml index ac65e321..acb815c6 100644 --- a/.mise/config.js.toml +++ b/.mise/config.js.toml @@ -8,6 +8,10 @@ # there's no value in installing them when `js` isn't loaded. [tools] +# MediaPipe tasks-vision runtime for pyeye1. +# pyeye1 loads @mediapipe/tasks-vision (FaceLandmarker) at runtime via /modules/@mediapipe/tasks-vision, the +# same npm-package-served-as-a-module pattern as onnxruntime-web (resolved in libs/edge-toolkit/src/config.rs). +"npm:@mediapipe/tasks-vision" = "latest" "npm:onnxruntime-web" = "latest" # oxlint-tsgolint is oxlint's type-aware engine. # It's a typescript-go type checker oxlint shells out to under `--type-aware`. npm-only: it ships no GitHub diff --git a/.mise/config.maint.toml b/.mise/config.maint.toml index 8d40fc74..6d7d1845 100644 --- a/.mise/config.maint.toml +++ b/.mise/config.maint.toml @@ -4,6 +4,9 @@ # # - publish-face1-to-hf-cache: refreshes the Hugging Face asset mirror in # edge-toolkit/core's hf-cache-v release. +# - publish-eye1-to-hf-cache: mirrors Google's face_landmarker.task +# (MediaPipe FaceLandmarker, used as-is) to the same hf-cache-v release +# as the pyeye1 model. # - publish-rp-wasm-to-release: builds upstream RustPython main with # wasm-pack + freeze-stdlib, tars the pkg/ output, and uploads to # edge-toolkit/core's rp-v release (same release that holds the @@ -13,6 +16,7 @@ # Not in ALL_LANGS (maint isn't a language); invoked manually: # # MISE_ENV=maint mise run publish-face1-to-hf-cache +# MISE_ENV=maint mise run publish-eye1-to-hf-cache # MISE_ENV=maint mise run publish-rp-wasm-to-release [tools] @@ -131,6 +135,39 @@ echo "updated config/upstream-cache/data.toml + .mise/config.toml; conftest will """ shell = "bash -euo pipefail -c" +[tasks.publish-eye1-to-hf-cache] +depends = ["bootstrap-hf-cache-release"] +description = "Refresh eye1 (MediaPipe FaceLandmarker) mirror in hf-cache-v release (manual / on bump)" +# Refresh the eye1 MediaPipe FaceLandmarker model mirror in the hf-cache release. +# Downloads Google's Apache-2.0 face_landmarker.task from the mediapipe-models bucket (used as-is, no +# conversion), computes the SHA256, uploads the .task + .task.sha256 sidecar to the release, then augtool-sets +# data.toml's eye1 sha256 so the in-repo manifest stays in sync (checksums.rego catches drift on next +# `mise run check`). eye1_asset has a stable name, so config.toml needs no edit. +run = """ +work="$HF_CACHE_WORK/eye1" +src="https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task" +asset="{{ vars.eye1_asset }}" + +coreutils rm -rf "$work" +coreutils mkdir -p "$work" +cd "$work" + +rclone copyurl "$src" "$asset" +coreutils sha256sum "$asset" > "${asset}.sha256" +actual=$(coreutils cut -c1-64 "${asset}.sha256") + +gh release upload "{{ vars.a_hf_cache_tag }}" --clobber --repo "$GH_REPO" "$asset" "${asset}.sha256" + +cksum_path="{{ config_root }}/config/upstream-cache/data.toml" +augtool --transform="Toml incl $cksum_path" --autosave < +# release by publish-eye1-to-hf-cache in config.maint.toml. data.toml records the SHA256; checksums.rego +# cross-checks both sides. +eye1_asset = "face_landmarker.task" +eye1_dst = "data/model-modules/model-eye1/pkg/face_landmarker.task" +eye1_src = ":http:edge-toolkit/core/releases/download/{{ vars.a_hf_cache_tag }}/{{ vars.eye1_asset }}" har_dst = "data/model-modules/model-har-motion1/pkg/har-motion1.onnx" har_repo = ":http:acd17sk/MET-Metabolic-Equivalent-of-Task-AI-Android-APP" har_src = "{{ vars.har_repo }}/raw/refs/heads/main/Models/onnx/hybrid_met.onnx" @@ -1002,9 +1009,16 @@ description = "Flag dependencies whose upstream repo is archived or unmaintained # cargo-unmaintained is deliberately NOT a mise [tool]. # It has no prebuilt and a from-source `cargo install` is too heavy to pay on every `mise install`. The # `dependencies` workflow installs it via taiki-e/install-action; this task just runs the binary so the -# audit command lives beside the others. A GITHUB_TOKEN in the env lets it check each upstream repo's -# archival status. -run = "cargo unmaintained" +# audit command lives beside the others. cargo-unmaintained needs a GitHub token to check each upstream +# repo's archival status: the CI workflow already sets GITHUB_TOKEN, so the `:=` default is a no-op there; +# locally (where the env var is unset) it falls back to `gh auth token`, so a dev never has to export one by +# hand. `set -u` is satisfied because `${VAR:=...}` assigns before the reference. +run = """ +: "${GITHUB_TOKEN:=$(gh auth token)}" +export GITHUB_TOKEN +cargo unmaintained +""" +shell = "bash -euo pipefail -c" [tasks."gen:dockerignore"] description = "Regenerate .dockerignore from .gitignore + Docker-only excludes" @@ -1058,7 +1072,7 @@ description = "Pre-download CI-required dependencies + the MNIST model" [tasks.prefetch] # "dprint-check" is here to trigger download of the dprint plugin WASM files. -depends = ["dprint-check", "fetch-face1-rclone", "fetch-har-motion1-rclone", "prefetch-ci"] +depends = ["dprint-check", "fetch-eye1-rclone", "fetch-face1-rclone", "fetch-har-motion1-rclone", "prefetch-ci"] description = "Pre-download all dependencies and models (superset of prefetch-ci)" [tasks."prefetch:rust"] @@ -1389,6 +1403,19 @@ actual=$(mise exec -- coreutils sha256sum -b "{{ vars.face1_dst }}" | mise exec """ shell = "bash -euo pipefail -c" +[tasks.fetch-eye1-rclone] +# Same shape as fetch-face1-rclone (rclone copy + Windows-safe SHA256 verify against data.toml). +# The FaceMesh .task is mirrored to the hf-cache-v release by `MISE_ENV=maint mise run +# publish-eye1-to-hf-cache`; refresh it there on an upstream bump. +run = """ +rclone copyto {{ vars.eye1_src }} {{ vars.eye1_dst }} {{ vars.gh_http }} +export asset='{{ vars.eye1_asset }}' +sha=$(mise exec -- yq -p toml -oy '.asset[strenv(asset)].sha256' config/upstream-cache/data.toml) +actual=$(mise exec -- coreutils sha256sum -b "{{ vars.eye1_dst }}" | mise exec -- coreutils cut -c1-64) +[ "$actual" = "$sha" ] || { echo "eye1 checksum mismatch: expected=$sha actual=$actual" >&2; exit 1; } +""" +shell = "bash -euo pipefail -c" + [tasks.fetch-har-motion1-rclone] run = "rclone copyto {{ vars.har_src }} {{ vars.har_dst }} {{ vars.gh_http }}" shell = "bash -euo pipefail -c" @@ -1401,7 +1428,7 @@ run = "rclone copyto {{ vars.mnist_src }} {{ vars.mnist_dst }} {{ vars.gh_http } shell = "bash -euo pipefail -c" [tasks.download-models] -depends = ["fetch-face1-rclone", "fetch-har-motion1-rclone", "fetch-mnist-rclone"] +depends = ["fetch-eye1-rclone", "fetch-face1-rclone", "fetch-har-motion1-rclone", "fetch-mnist-rclone"] [tasks.cargo-test] description = "Run every Rust test via nextest (web-runner excluded on gnullvm)" diff --git a/CLAUDE.md b/CLAUDE.md index 8b590c06..aa404f30 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -301,7 +301,7 @@ Languages: - **Rust -> WASM** (wasm-pack): audio1, bluetooth, comm1, data1, face-detection, geolocation, graphics-info, har1, nfc, sensor1, speech-recognition, video1 - **Dart -> JS**: dart-comm1 -- **Python (Pyodide)**: pydata1, pyface1 +- **Python (Pyodide)**: pydata1, pyeye1, pyface1 - **C# (.NET WASM)**: dotnet-data1 - **Java (TeaVM -> JS)**: java-data1 - **Zig -> WASM**: zig-data1 diff --git a/Cargo.toml b/Cargo.toml index dca93b9a..ad5d5697 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -324,52 +324,52 @@ ignore = [ # Upstream contain-rs/bit-set is unmaintained; the newer 0.9.1 is flagged too. # Pulled by the Deno stack (deno_core, and naga/wgpu-core via deno_webgpu). "bit-set", - # 1153 days since last update at bitvecto-rs/bitvec. + # Unmaintained upstream at bitvecto-rs/bitvec. # Pulled by sourcemap (via swc tooling under deno_ast). "bitvec", - # 536 days since last update at dsherret/capacity_builder. + # Unmaintained upstream at dsherret/capacity_builder. # Pulled by deno_ast. "capacity_builder", - # 1702 days since last update at jethrogb/rust-cexpr. + # Unmaintained upstream at jethrogb/rust-cexpr. # Pulled by bindgen (build-time only via aws-lc-sys / ort-sys). "cexpr", - # 577 days since last update at dajoha/color-print. + # Unmaintained upstream at dajoha/color-print. # Pulled by deno_node via color-print. "color-print-proc-macro", # Archived upstream at matklad/countme. "countme", - # 676 days since last update at denoland/deno_native_certs. + # Unmaintained upstream at denoland/deno_native_certs. # Pulled by deno_node + deno_tls. "deno_native_certs", # No `repository` declared. Pulled by deno_node for `os.userInfo()`. "deno_whoami", - # 1853 days since last update at xdg-rs/dirs. + # Unmaintained upstream at xdg-rs/dirs. "dirs-sys-next", - # 3182 days since last update at faradayio/exec-rs. + # Unmaintained upstream at faradayio/exec-rs. # Pulled by Deno's `node_shim` (process re-exec on Node compatibility shim). "exec", # cargo-unmaintained can't find a crate of this name in the denoland/deno monorepo its `repository` field names. # It is a workspace member there, so the "not in " heuristic misfires. Pulled by the Deno registry stack. "fast-registry-json", - # 933 days since last update at HdrHistogram/HdrHistogram_rust. + # Unmaintained upstream at HdrHistogram/HdrHistogram_rust. # Pulled by deno_node (perf hooks). "hdrhistogram", - # 460 days since last update at xacrimon/dashmap. + # Unmaintained upstream at xacrimon/dashmap. # Pulled by deno_maybe_sync. (Marked stable / feature-complete by upstream.) "dashmap", # jni-sys requirement ^0.3.0 (used 0.3.1, latest 0.4.1); pulled by ndk-sys, an Android-only dep. # Reached via wgpu-hal (deno_webgpu -> wgpu); never compiled on our target platforms. "jni-sys", - # 495 days since last update at rust-lang-nursery/lazy-static.rs. + # Unmaintained upstream at rust-lang-nursery/lazy-static.rs. # Pulled by actix-web-thiserror-derive (build proc-macro). "lazy_static", - # ndk-sys last updated ~365 days ago; Android-only, reached via wgpu-hal (deno_webgpu -> wgpu). + # ndk-sys unmaintained upstream; Android-only, reached via wgpu-hal (deno_webgpu -> wgpu). # Never compiled on our target platforms. "ndk-sys", - # 391 days since last update at rust-num/num-complex. + # Unmaintained upstream at rust-num/num-complex. # Pulled by nalgebra (via wgpu / naga). Stable foundational math crate. "num-complex", - # 2215 days since last update at SSheldon/rust-objc. + # Unmaintained upstream at SSheldon/rust-objc. # Pulled by `metal` (macOS GPU bindings via wgpu). "objc", # Not in declared repo (RustCrypto/utils). @@ -378,7 +378,7 @@ ignore = [ "opaque-debug", # cargo-unmaintained doesn't follow `hg` repositories. "oorandom", - # 1052 days since last update at hjiayz/p12. + # Unmaintained upstream at hjiayz/p12. # Pulled by deno_node_crypto for PKCS#12 cert bundles. "p12", # Archived upstream at dtolnay/paste. @@ -389,7 +389,7 @@ ignore = [ # Pulled by getset via neli via local-ip-address. Compile-time only. "proc-macro-error-attr2", "proc-macro-error2", - # 603 days since last update at timvisee/qr2term-rs. + # Unmaintained upstream at timvisee/qr2term-rs. "qr2term", # False positive. # rand_chacha lives in rust-random/rand as a workspace subdir (rand_chacha/). @@ -398,10 +398,14 @@ ignore = [ # Archived upstream at rustls/pemfile. Pulled by deno_native_certs. # rustls itself has moved to inline pemfile parsing. "rustls-pemfile", - # 2154 days since last update at arcnmx/serde-value. + # Unmaintained upstream at denoland/rustls-tokio-stream. + # Its derive-io (=0.4.1) and socket2 (^0.5) requirements lag latest; pulled by the Deno net/TLS stack + # (deno_net -> deno_runtime). + "rustls-tokio-stream", + # Unmaintained upstream at arcnmx/serde-value. # Pulled by deno_ffi. "serde-value", - # 732 days since last update at TedDriggs/serde_default; its `darling` requirement lags the latest release. + # Unmaintained upstream at TedDriggs/serde_default; its `darling` requirement lags the latest release. # A direct dependency of edge-toolkit for the serde-attribute `Default` derive. "serde_default", # Archived upstream at bodil/smartstring. @@ -409,9 +413,9 @@ ignore = [ "smartstring", # https://github.com/rustsec/advisory-db/issues/2132 "serde_yaml", - # 700 days since last update at dalek-cryptography/subtle. + # Unmaintained upstream at dalek-cryptography/subtle. "subtle", - # 1400 days since last update at hucsmn/suffix_array; its `bincode` requirement lags the latest release. + # Unmaintained upstream at hucsmn/suffix_array; its `bincode` requirement lags the latest release. # Pulled by qbsdiff (binary diffing) via deno_runtime. "suffix_array", # Archived upstream at rust-analyzer/text-size. @@ -426,7 +430,7 @@ ignore = [ # Declared repository alacritty/vte is a single-crate repo with no `utf8parse` member. # The crate is effectively orphaned upstream. "utf8parse", - # 517 days since last update at daxpedda/web-time. + # Unmaintained upstream at daxpedda/web-time. "web-time", # Not in declared repo (gfx-rs/wgpu): a per-platform helper crate from the wgpu monorepo. # cargo-unmaintained can't find the individual member at the repo root -- false positive; pulled by wgpu-core. diff --git a/config/coverage.toml b/config/coverage.toml index 50a8dbb5..9beca985 100644 --- a/config/coverage.toml +++ b/config/coverage.toml @@ -1,5 +1,5 @@ # coverage.py config for the combined Python report (referenced via COVERAGE_RCFILE by the pytest-cov task). -# Three data sources feed one report: pyface1's pytest run, and the web-runner Pyodide runs of pydata1 + pyface1 +# Data sources feed one report: the pytest runs, and the web-runner Pyodide runs of pydata1 + pyface1 + pyeye1 # (whose .coverage files the web-runner integration test drops into target/pycov/). `[tool.coverage.paths]` remaps # each source's recorded path to the repo-relative one so `coverage combine` merges them into a single file entry: # pytest records relative paths (pyface1/...); the Pyodide runs record absolute in-interpreter paths @@ -11,4 +11,5 @@ relative_files = true [tool.coverage.paths] pydata1 = ["services/ws-modules/pydata1/pydata1", "pydata1", "**/pydata1"] +pyeye1 = ["services/ws-modules/pyeye1/pyeye1", "pyeye1", "**/pyeye1"] pyface1 = ["services/ws-modules/pyface1/pyface1", "pyface1", "**/pyface1"] diff --git a/config/pyrefly.toml b/config/pyrefly.toml index 63370557..77c38808 100644 --- a/config/pyrefly.toml +++ b/config/pyrefly.toml @@ -19,7 +19,12 @@ project-excludes = [ ] # Package roots so intra-package imports (`pyface1.face_detection`) resolve as real modules, not `__unknown__.*`. -search-path = ["../services/ws-modules/pydata1", "../services/ws-modules/pyface1", "../services/ws-pyo3-runner/python"] +search-path = [ + "../services/ws-modules/pydata1", + "../services/ws-modules/pyeye1", + "../services/ws-modules/pyface1", + "../services/ws-pyo3-runner/python", +] # Third-party deps and generated packages that are absent from (or opaque to) the lint environment. # `replace-imports-with-any` (rather than a blanket ignore) keeps import resolution deterministic across every OS diff --git a/config/upstream-cache/data.toml b/config/upstream-cache/data.toml index b608b37e..0076ecd5 100644 --- a/config/upstream-cache/data.toml +++ b/config/upstream-cache/data.toml @@ -24,6 +24,15 @@ sha256 = "3b29de703b4641cb995754bc6294bd58b0fa5d859c838e9484c86d00251f8a08" upstream = "https://huggingface.co/amd/retinaface" url = "https://github.com/edge-toolkit/core/releases/download/hf-cache-v1/RetinaFace_int-7b69507.onnx" +# pyeye1's MediaPipe FaceLandmarker model bundle (Apache-2.0), Google's face_landmarker.task used as-is. +# Pinned to the stable float16/1 revision in the mediapipe-models bucket; publish-eye1-to-hf-cache mirrors the +# same blob to the hf-cache release, so its upload matches this SHA256. +[asset."face_landmarker.task"] +license = "Apache-2.0" +sha256 = "64184e229b263107bc2b804c6625db1341ff2bb731874b0bcc2fe6544e0bc9ff" +upstream = "https://ai.google.dev/edge/mediapipe/solutions/vision/face_landmarker" +url = "https://github.com/edge-toolkit/core/releases/download/hf-cache-v1/face_landmarker.task" + [asset."rustpython-wasm-08a1d6f.tar.gz"] license = "MIT" sha256 = "7b81d53f5b3a63c60fc2bc1717e87466026d796ec522b925f009532c7b0291f3" diff --git a/data/model-modules/.gitignore b/data/model-modules/.gitignore index e1a699ac..e7e8c9e8 100644 --- a/data/model-modules/.gitignore +++ b/data/model-modules/.gitignore @@ -1 +1,2 @@ *.onnx +*.task diff --git a/data/model-modules/model-eye1/pkg/package.json b/data/model-modules/model-eye1/pkg/package.json new file mode 100644 index 00000000..87a83c97 --- /dev/null +++ b/data/model-modules/model-eye1/pkg/package.json @@ -0,0 +1,8 @@ +{ + "description": "MediaPipe FaceLandmarker model bundle", + "name": "et-model-eye1", + "version": "0.1.0", + "files": [ + "face_landmarker.task" + ] +} diff --git a/libs/edge-toolkit/src/config.rs b/libs/edge-toolkit/src/config.rs index 9eb75ece..7032c240 100644 --- a/libs/edge-toolkit/src/config.rs +++ b/libs/edge-toolkit/src/config.rs @@ -153,6 +153,24 @@ pub fn default_modules_folders() -> Vec { ); } } + // MediaPipe tasks-vision (pyeye1's FaceLandmarker runtime). A scoped npm package: serve its own dir as a + // single module at /modules/@mediapipe/tasks-vision (the modules service reads its package.json name). + match mise_npm_package_path("@mediapipe/tasks-vision") { + Some(path) => { + log::info!("Resolved npm:@mediapipe/tasks-vision modules path: {}", path.display()); + paths.push(path); + } + None => { + log::warn!( + "{}", + concat!( + "npm:@mediapipe/tasks-vision install path not found via `mise where` -- ", + "requests to /modules/@mediapipe/tasks-vision/* will 404. ", + "Run `mise install npm:@mediapipe/tasks-vision` and verify the package layout.", + ) + ); + } + } paths } @@ -261,6 +279,16 @@ pub fn mise_npm_modules_path(package: &str) -> Option { find_npm_modules_path_in(&install, package) } +/// The package's own directory (`/`) rather than its parent `node_modules`. +/// +/// Use this to serve a single scoped npm package (e.g. `@mediapipe/tasks-vision`) as one module at +/// `/modules/`: pointing the modules service at the whole `node_modules` tree wouldn't find a +/// nested `@scope/pkg` (the scope dir has no `package.json`), so we push the package dir directly. +#[must_use] +pub fn mise_npm_package_path(package: &str) -> Option { + mise_npm_modules_path(package).map(|node_modules| node_modules.join(package)) +} + /// Pure-filesystem version of [`mise_npm_modules_path`]: given an /// `` root and a `` name, return the `node_modules` /// directory that contains ``. Supports the mise npm backends: diff --git a/libs/edge-toolkit/tests/npm_mod.rs b/libs/edge-toolkit/tests/npm_mod.rs index 442eb33f..406ab00e 100644 --- a/libs/edge-toolkit/tests/npm_mod.rs +++ b/libs/edge-toolkit/tests/npm_mod.rs @@ -56,6 +56,29 @@ fn resolves_aube_backend_layout() { assert_eq!(found.as_deref(), Some(modules.as_path())); } +#[test] +fn resolves_scoped_package_layout() { + // A scoped npm package (e.g. @mediapipe/tasks-vision) lives at + // /lib/node_modules/@mediapipe/tasks-vision. The resolver joins the full "@scope/pkg" name, so it + // returns the node_modules dir; mise_npm_package_path then appends the name to get the package dir served + // as one module. + let install = TempDir::new().unwrap(); + let modules = install.path().join("lib/node_modules"); + let package = modules.join("@mediapipe/tasks-vision"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("package.json"), "{}").unwrap(); + + let found = find_npm_modules_path_in(install.path(), "@mediapipe/tasks-vision"); + assert_eq!(found.as_deref(), Some(modules.as_path())); + assert!( + found + .unwrap() + .join("@mediapipe/tasks-vision") + .join("package.json") + .is_file() + ); +} + #[test] fn returns_none_when_neither_layout_has_the_package() { let install = TempDir::new().unwrap(); diff --git a/pyproject.toml b/pyproject.toml index 9c61a44b..2dce1b86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,7 @@ members = [ "generated/python-rest", "generated/python-ws", "services/ws-modules/pydata1", + "services/ws-modules/pyeye1", "services/ws-modules/pyface1", "services/ws-modules/wasi-graphics-info", ] diff --git a/services/ws-modules/face-detection/src/lib.rs b/services/ws-modules/face-detection/src/lib.rs index 5168d654..29a6823c 100644 --- a/services/ws-modules/face-detection/src/lib.rs +++ b/services/ws-modules/face-detection/src/lib.rs @@ -45,6 +45,16 @@ const RETINAFACE_NMS_THRESHOLD: f64 = 0.4; const RETINAFACE_VARIANCES: [f64; 2] = [0.1, 0.2]; const RETINAFACE_MIN_SIZES: [&[f64]; 3] = [&[16.0, 32.0], &[64.0, 128.0], &[256.0, 512.0]]; const RETINAFACE_STEPS: [f64; 3] = [8.0, 16.0, 32.0]; +// RetinaFace emits five landmark points per face, in this fixed order (subject's perspective, so the +// "left" eye/mouth-corner appears on the right of a non-mirrored frame). The eye points are indices 0 and 1. +const LANDMARK_LABELS: [&str; 5] = ["left_eye", "right_eye", "nose", "left_mouth", "right_mouth"]; + +#[derive(Clone)] +struct Landmark { + label: &'static str, + pixel_x: f64, + pixel_y: f64, +} #[derive(Clone)] struct Detection { @@ -52,6 +62,7 @@ struct Detection { class_index: i32, score: f64, box_coords: [f64; 4], + landmarks: Vec, } #[derive(Clone)] @@ -375,6 +386,11 @@ async fn infer_once( "class_index": entry.class_index, "score": entry.score, "box": entry.box_coords, + "landmarks": entry + .landmarks + .iter() + .map(|point| json!({ "label": point.label, "x": point.pixel_x, "y": point.pixel_y })) + .collect::>(), })) .collect::>(), "changed": changed, @@ -464,11 +480,23 @@ fn decode_retinaface_outputs( clamp((decoded[3] * FACE_INPUT_HEIGHT_F64) / resize_ratio, 0.0, source_height), ]; + let points = decode_retinaface_landmarks(&landm_values[index * 10..index * 10 + 10], priors[index]); + let landmarks = LANDMARK_LABELS + .into_iter() + .zip(points) + .map(|(label, point)| Landmark { + label, + pixel_x: clamp((point[0] * FACE_INPUT_WIDTH_F64) / resize_ratio, 0.0, source_width), + pixel_y: clamp((point[1] * FACE_INPUT_HEIGHT_F64) / resize_ratio, 0.0, source_height), + }) + .collect(); + detections.push(Detection { label: String::from("face"), class_index: 0, score, box_coords, + landmarks, }); } @@ -525,6 +553,20 @@ fn decode_retinaface_box(loc: [f64; 4], prior: [f64; 4]) -> [f64; 4] { ] } +// Decode the five RetinaFace landmark points from their offsets and prior box. Each point is a plain centre +// offset (no width/height exponent), so decode is simpler than the box: the normalised coordinate is the +// prior centre plus the offset scaled by the first variance and the prior size. +fn decode_retinaface_landmarks(landm: &[f32], prior: [f64; 4]) -> [[f64; 2]; 5] { + let mut points = [[0.0_f64; 2]; 5]; + for (point, slot) in points.iter_mut().enumerate() { + *slot = [ + prior[0] + f64::from(landm[point * 2]) * RETINAFACE_VARIANCES[0] * prior[2], + prior[1] + f64::from(landm[point * 2 + 1]) * RETINAFACE_VARIANCES[0] * prior[3], + ]; + } + points +} + fn compute_iou(left: &Detection, right: &Detection) -> f64 { let x1 = left.box_coords[0].max(right.box_coords[0]); let y1 = left.box_coords[1].max(right.box_coords[1]); @@ -870,11 +912,30 @@ fn face_render(detections: &[Detection]) -> Result<(), JsValue> { context.fill_rect(left, (top - 24.0).max(0.0), text_width, 22.0); context.set_fill_style_str("#fffdfa"); context.fill_text(&label, left + 5.0, (top - 8.0).max(16.0))?; + + for point in &detection.landmarks { + context.begin_path(); + context.arc(point.pixel_x, point.pixel_y, 3.0, 0.0, std::f64::consts::TAU)?; + context.set_fill_style_str(landmark_color(point.label)); + context.fill(); + } } Ok(()) } +// Colour for each RetinaFace landmark, keyed by the label the decoder emits. +fn landmark_color(label: &str) -> &'static str { + match label { + "left_eye" => "#ff4d4d", + "right_eye" => "#4dff88", + "nose" => "#ffd24d", + "left_mouth" => "#4db8ff", + "right_mouth" => "#c86bff", + _ => "#fffdfa", + } +} + fn image_data_to_tensor(image_data: &ImageData) -> Vec { const CHANNEL_MEAN: [f32; 3] = [104.0, 117.0, 123.0]; diff --git a/services/ws-modules/face-detection/src/test_face_detection.rs b/services/ws-modules/face-detection/src/test_face_detection.rs index 887ee0fd..a573ebdd 100644 --- a/services/ws-modules/face-detection/src/test_face_detection.rs +++ b/services/ws-modules/face-detection/src/test_face_detection.rs @@ -22,6 +22,7 @@ fn detection(score: f64, box_coords: [f64; 4]) -> Detection { class_index: 0, score, box_coords, + landmarks: Vec::new(), } } @@ -109,3 +110,23 @@ fn retinaface_zero_offsets_decode_to_prior_box() { assert!((decoded[2] - 0.625).abs() < 1e-6); assert!((decoded[3] - 0.75).abs() < 1e-6); } + +#[test] +fn retinaface_zero_offsets_decode_landmarks_to_prior_centre() { + let points = decode_retinaface_landmarks(&[0.0_f32; 10], [0.5, 0.5, 0.25, 0.5]); + + for point in points { + assert!((point[0] - 0.5).abs() < 1e-6); + assert!((point[1] - 0.5).abs() < 1e-6); + } +} + +#[test] +fn retinaface_landmarks_apply_offset_variance_and_prior_size() { + let landm = [1.0_f32, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + let points = decode_retinaface_landmarks(&landm, [0.5, 0.5, 0.25, 0.5]); + + assert!((points[0][0] - (0.5 + 1.0 * 0.1 * 0.25)).abs() < 1e-6); + assert!((points[0][1] - (0.5 + 2.0 * 0.1 * 0.5)).abs() < 1e-6); + assert!((points[1][0] - 0.5).abs() < 1e-6); +} diff --git a/services/ws-modules/pyeye1/pkg/.gitignore b/services/ws-modules/pyeye1/pkg/.gitignore new file mode 100644 index 00000000..d415af97 --- /dev/null +++ b/services/ws-modules/pyeye1/pkg/.gitignore @@ -0,0 +1,2 @@ +*.whl +package.json diff --git a/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js b/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js new file mode 100644 index 00000000..d441af1a --- /dev/null +++ b/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js @@ -0,0 +1,210 @@ +// et_ws_pyeye1.js - Browser adapter for the MediaPipe FaceLandmarker eye-detection Pyodide workflow. +// Interface: default(), run(), start(), stop(), is_running() +// +// MediaPipe FaceLandmarker (a maintained, offline .task bundle that internally runs a face detector then a +// mesh model) does the inference in JS; the Python layer turns its normalized landmarks into eye boxes. + +const PYODIDE_BASE_PATH = "/modules/pyodide/"; + +// Stage-2 eye-box outline colours, keyed by the label the Python decoder emits. +const EYE_COLORS = { + left_eye: "#ff4d4d", + right_eye: "#4dff88", +}; + +let pyodide; +let py; +let cfg; +let runtime = null; + +export default async function init() { + if (!globalThis.loadPyodide) { + await new Promise((resolve, reject) => { + const script = document.createElement("script"); + script.src = `${PYODIDE_BASE_PATH}pyodide.js`; + script.onload = resolve; + script.onerror = reject; + document.head.appendChild(script); + }); + } + + pyodide = await globalThis.loadPyodide({ indexURL: PYODIDE_BASE_PATH }); + + // pydantic backs the typed WsClientEvent message in the et_ws wheel. + await pyodide.loadPackage("micropip"); + const micropip = pyodide.pyimport("micropip"); + await micropip.install("pydantic"); + + // Install pyeye1's own wheel from pkg/ next to this shim. + const installLocalWheel = async (path) => { + const bytes = new Uint8Array(await fetch(new URL(path, import.meta.url)).then((r) => r.arrayBuffer())); + pyodide.FS.writeFile(`/tmp/${path}`, bytes); + pyodide.runPython(`import sys\nsys.path.insert(0, "/tmp/${path}")`); + }; + + const pkg = await fetch(new URL("package.json", import.meta.url)).then((r) => r.json()); + await installLocalWheel(`${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`); + // et-ws is its own ws-module mounted at /modules/et-ws/; delegate its wheel install to its shim. + const { installWheel: installEtWs } = await import("/modules/et-ws/et_ws.js"); + await installEtWs(pyodide); + + if (globalThis.__etPyCov) await globalThis.__etPyCov.start(pyodide, "pyeye1"); + py = pyodide.pyimport("pyeye1"); + cfg = py.config().toJs({ dict_converter: Object.fromEntries }); +} + +export const is_running = () => runtime !== null; +export const start = () => run(); + +export async function run() { + if (!py) throw new Error("pyeye1: not initialized"); + if (runtime) return; + + setStatus(py.starting_status()); + log(py.model_log_message()); + + let client = null; + let stream = null; + let state = null; + + try { + const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + await wasmAgent.default(); + const { WsClient, WsClientConfig } = wasmAgent; + const protocol = window.location.protocol === "https:" ? "wss:" : "ws:"; + client = new WsClient(new WsClientConfig(`${protocol}//${window.location.host}/ws`)); + client.connect(); + for (let i = 0; client.get_state() !== "connected" && i < 100; i++) await sleep(100); + if (client.get_state() !== "connected") throw new Error("Timed out waiting for websocket connection"); + log(`websocket connected with agent_id=${client.get_agent_id()}`); + + stream = await navigator.mediaDevices.getUserMedia({ audio: false, video: true }); + const video = element("video-preview", HTMLVideoElement); + video.srcObject = stream; + video.hidden = false; + for (let i = 0; video.videoWidth === 0 && i < 50; i++) await sleep(100); + if (video.videoWidth === 0 || video.videoHeight === 0) throw new Error("Video stream metadata did not load"); + await video.play(); + + // Load the MediaPipe tasks-vision runtime (served offline from its module mount) and build the landmarker. + const vision = await import(cfg.bundle_path); + const { FaceLandmarker, FilesetResolver } = vision; + const fileset = await FilesetResolver.forVisionTasks(cfg.wasm_path); + const landmarker = await FaceLandmarker.createFromOptions(fileset, { + baseOptions: { modelAssetPath: cfg.model_path }, + runningMode: "VIDEO", + numFaces: 1, + }); + + state = { client, stream, landmarker }; + runtime = state; + + await py.run( + pyodide.toPy(() => inferEyes(state)), + pyodide.toPy((message) => client.send(message)), + pyodide.toPy(render), + pyodide.toPy(sleep), + pyodide.toPy(log), + pyodide.toPy(setStatus), + pyodide.toPy(() => runtime !== state), + ); + } catch (err) { + log(`pyeye1 run failed: ${String(err)}`); + throw err; + } finally { + if (globalThis.__etPyCov) await globalThis.__etPyCov.stop(pyodide, "pyeye1"); + cleanup(state ?? { client, stream }); + } +} + +export function stop() { + if (!runtime) return; + cleanup(runtime); + log("pyeye1 eye detection demo stopped"); +} + +async function inferEyes(state) { + const video = element("video-preview", HTMLVideoElement); + if (video.videoWidth <= 0 || video.videoHeight <= 0) throw new Error("Video stream is not ready yet."); + + const result = state.landmarker.detectForVideo(video, performance.now()); + const faces = (result.faceLandmarks ?? []).map((face) => { + const flat = []; + for (const point of face) flat.push(point.x, point.y); + return flat; + }); + + return JSON.stringify({ faces, width: video.videoWidth, height: video.videoHeight }); +} + +function render(resultsJson) { + const video = element("video-preview", HTMLVideoElement); + if (video.videoWidth === 0 || video.videoHeight === 0) return; + + const canvas = element("video-output-canvas", HTMLCanvasElement); + const ctx = canvas.getContext("2d"); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + canvas.hidden = false; + ctx.drawImage(video, 0, 0, canvas.width, canvas.height); + ctx.font = "16px ui-monospace, monospace"; + + for (const result of JSON.parse(resultsJson)) { + const [faceLeft, faceTop, faceRight, faceBottom] = result.face_box; + ctx.lineWidth = 1; + ctx.strokeStyle = "#7a8794"; + ctx.strokeRect(faceLeft, faceTop, Math.max(faceRight - faceLeft, 1), Math.max(faceBottom - faceTop, 1)); + + ctx.lineWidth = 3; + for (const eye of result.eyes) { + const [left, top, right, bottom] = eye.box; + const color = EYE_COLORS[eye.label] ?? "#fffdfa"; + ctx.strokeStyle = color; + ctx.strokeRect(left, top, Math.max(right - left, 1), Math.max(bottom - top, 1)); + ctx.fillStyle = color; + ctx.fillText(eye.label === "left_eye" ? "L" : "R", left, Math.max(top - 4, 12)); + } + } +} + +function cleanup(state) { + if (runtime === state) runtime = null; + state?.landmarker?.close?.(); + for (const track of state?.stream?.getTracks?.() ?? []) track.stop(); + state?.client?.disconnect?.(); + + const video = document.getElementById("video-preview"); + if (video) { + video.pause(); + video.srcObject = null; + video.hidden = true; + } + + const canvas = document.getElementById("video-output-canvas"); + if (canvas) { + canvas.hidden = true; + canvas.getContext("2d")?.clearRect(0, 0, canvas.width, canvas.height); + } +} + +function setStatus(message) { + const output = document.getElementById("module-output"); + if (output) output.value = message; +} + +function log(message) { + const line = `[pyeye1] ${message}`; + console.log(line); + const logEl = document.getElementById("log"); + if (logEl) logEl.textContent = logEl.textContent ? `${logEl.textContent}\n${line}` : line; +} + +function sleep(ms) { + return new Promise((resolve) => window.setTimeout(resolve, ms)); +} + +function element(id, type) { + const found = document.getElementById(id); + if (!(found instanceof type)) throw new Error(`Missing #${id} element`); + return found; +} diff --git a/services/ws-modules/pyeye1/pyeye1/__init__.py b/services/ws-modules/pyeye1/pyeye1/__init__.py new file mode 100644 index 00000000..1ba455fb --- /dev/null +++ b/services/ws-modules/pyeye1/pyeye1/__init__.py @@ -0,0 +1,29 @@ +"""pyeye1: MediaPipe FaceLandmarker eye-detection support code (face -> eye bounding boxes).""" + +from .eye_detection import ( + EYE_MODEL_PATH, + build_results, + config, + decode_eye_boxes, + event_payload, + face_bounds, + model_log_message, + run, + starting_status, + status_text, + stopped_status, +) + +__all__ = [ + "EYE_MODEL_PATH", + "build_results", + "config", + "decode_eye_boxes", + "event_payload", + "face_bounds", + "model_log_message", + "run", + "starting_status", + "status_text", + "stopped_status", +] diff --git a/services/ws-modules/pyeye1/pyeye1/eye_detection.py b/services/ws-modules/pyeye1/pyeye1/eye_detection.py new file mode 100644 index 00000000..d4d6c764 --- /dev/null +++ b/services/ws-modules/pyeye1/pyeye1/eye_detection.py @@ -0,0 +1,240 @@ +"""Eye bounding-box post-processing for the pyeye1 MediaPipe FaceLandmarker workflow. + +The browser shim runs Google's MediaPipe FaceLandmarker (a maintained, offline `.task` bundle that internally +runs a face detector then a mesh model) and hands the resulting normalized face landmarks to this module, which +turns each eye's contour-landmark cluster into a tight bounding box in source-image pixels. +""" + +from __future__ import annotations + +import json +import math +import time +from collections.abc import Iterable, Sequence +from datetime import datetime +from typing import Any, TypedDict + +from et_ws.messages import WsClientEvent + +# Served by the MediaPipe tasks-vision runtime module and the model module (see config()). +EYE_MODEL_PATH = "/modules/et-model-eye1/face_landmarker.task" +VISION_BUNDLE_PATH = "/modules/@mediapipe/tasks-vision/vision_bundle.mjs" +VISION_WASM_PATH = "/modules/@mediapipe/tasks-vision/wasm" + +# FaceLandmarker returns 478 landmarks (the 468-point mesh plus 10 iris points), each x/y normalized to [0, 1] +# against the input frame. The eye-box needs only the mesh contour indices (all < 468). +MESH_LANDMARK_COUNT = 478 +CONTOUR_LANDMARK_COUNT = 468 +# Eye-contour landmark indices in the MediaPipe mesh (subject's perspective, so "left" appears on the right of +# a non-mirrored frame). A tight per-eye box is the min/max over each cluster. +RIGHT_EYE_INDICES = (33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246) +LEFT_EYE_INDICES = (263, 249, 390, 373, 374, 380, 381, 382, 362, 398, 384, 385, 386, 387, 388, 466) + +INFERENCE_INTERVAL_MS = 750 +RENDER_INTERVAL_MS = 60 +MAX_INFERENCES = 20 +MAX_RUNTIME_MS = 30_000 + + +Box = list[float] + + +class Eye(TypedDict): + """One detected eye: its label and bounding box in source-image pixels.""" + + label: str + box: Box + + +class FaceEyes(TypedDict): + """One detected face: its overall landmark bounds and the two eye boxes within it.""" + + face_box: Box + eyes: list[Eye] + + +async def run(infer, send_event, render, sleep_ms, log, set_status, should_stop) -> None: + """Run the browser eye detection workflow using JS platform callbacks. + + `infer()` returns a JSON string `{"faces": [[x0, y0, x1, y1, ...], ...], "width": W, "height": H}` where each + face is the flat list of normalized landmark coordinates from MediaPipe FaceLandmarker. + """ + inference_count = 0 + started_at = time.monotonic() + results: list[FaceEyes] = [] + + set_status(starting_status()) + + while not should_stop(): + elapsed_ms = (time.monotonic() - started_at) * 1000.0 + if inference_count >= MAX_INFERENCES or elapsed_ms >= MAX_RUNTIME_MS: + break + + try: + capture = json.loads(await infer()) + results = build_results(capture["faces"], capture["width"], capture["height"]) + inference_count += 1 + + set_status(status_text(results)) + render(results_json(results)) + send_event(client_event_json(event_payload(results, capture["width"], capture["height"]))) + except Exception as exc: + message = f"pyeye1 eye detection: inference error\n{exc}" + set_status(message) + log(f"inference error: {exc}") + + remaining_ms = INFERENCE_INTERVAL_MS + while remaining_ms > 0 and not should_stop(): + render(results_json(results)) + delay = min(RENDER_INTERVAL_MS, remaining_ms) + await sleep_ms(delay) + remaining_ms -= delay + + if inference_count >= MAX_INFERENCES: + log(f"workflow finished automatically after {MAX_INFERENCES} inferences") + elif (time.monotonic() - started_at) * 1000.0 >= MAX_RUNTIME_MS: + log("workflow finished automatically after 30 seconds") + set_status(stopped_status()) + + +def config() -> dict[str, object]: + """Return browser-facing constants: the model asset plus the MediaPipe runtime bundle + wasm roots.""" + return { + "model_path": EYE_MODEL_PATH, + "bundle_path": VISION_BUNDLE_PATH, + "wasm_path": VISION_WASM_PATH, + } + + +def starting_status() -> str: + """Return the status line shown while the workflow starts up.""" + return "pyeye1 eye detection: starting" + + +def stopped_status() -> str: + """Return the status line shown once the workflow has stopped.""" + return "pyeye1 eye detection demo stopped." + + +def model_log_message() -> str: + """Return the log line emitted when loading the model.""" + return f"loading MediaPipe FaceLandmarker from {EYE_MODEL_PATH}" + + +def decode_eye_boxes(landmarks: Iterable[Any], image_width: float, image_height: float) -> dict[str, Box]: + """Decode the left and right eye bounding boxes from one face's normalized FaceLandmarker landmarks.""" + values = [float(value) for value in landmarks] + if len(values) < CONTOUR_LANDMARK_COUNT * 2: + raise ValueError("FaceLandmarker output did not contain the expected face-mesh landmarks") + + width = require_positive_finite(image_width, "image_width") + height = require_positive_finite(image_height, "image_height") + return { + "left_eye": eye_box(values, LEFT_EYE_INDICES, width, height), + "right_eye": eye_box(values, RIGHT_EYE_INDICES, width, height), + } + + +def eye_box(values: Sequence[float], indices: Sequence[int], width: float, height: float) -> Box: + """Map one eye's contour landmarks from normalized [0, 1] coords to a source-pixel bounding box.""" + xs = [clamp(values[index * 2] * width, 0.0, width) for index in indices] + ys = [clamp(values[index * 2 + 1] * height, 0.0, height) for index in indices] + return [min(xs), min(ys), max(xs), max(ys)] + + +def face_bounds(landmarks: Sequence[float], width: float, height: float) -> Box: + """Return the bounding box of all of a face's landmarks, in source pixels (context for the eye overlay).""" + xs = [clamp(landmarks[index] * width, 0.0, width) for index in range(0, len(landmarks), 2)] + ys = [clamp(landmarks[index] * height, 0.0, height) for index in range(1, len(landmarks), 2)] + return [min(xs), min(ys), max(xs), max(ys)] + + +def build_results(faces: Sequence[Any], width: float, height: float) -> list[FaceEyes]: + """Combine each detected face's landmarks into its overall bounds plus its two eye boxes.""" + width = require_positive_finite(width, "width") + height = require_positive_finite(height, "height") + results: list[FaceEyes] = [] + for face in faces: + values = [float(value) for value in face] + boxes = decode_eye_boxes(values, width, height) + results.append( + { + "face_box": face_bounds(values, width, height), + "eyes": [ + {"label": "left_eye", "box": boxes["left_eye"]}, + {"label": "right_eye", "box": boxes["right_eye"]}, + ], + } + ) + return results + + +def results_json(results: Sequence[FaceEyes]) -> str: + """Serialise the per-face eye results to JSON for the renderer.""" + return json.dumps(list(results)) + + +def client_event_json(details: dict[str, object]) -> str: + """Build the et-client-event JSON envelope for an eye-detection inference.""" + return WsClientEvent( + type="et-client-event", + capability="eye_detection", + action="inference", + details=details, + ).model_dump_json() + + +def status_text(results: Sequence[FaceEyes]) -> str: + """Render the browser status text used by the eye detection demo.""" + eye_count = sum(len(result["eyes"]) for result in results) + lines = [ + "pyeye1 eye detection demo", + f"model file: {EYE_MODEL_PATH}", + f"faces: {len(results)}", + f"eyes: {eye_count}", + f"processed at: {datetime.now().strftime('%X')}", + ] + + if results and results[0]["eyes"]: + box = results[0]["eyes"][0]["box"] + lines.extend(["", f"first eye: {box[0]:.1f}, {box[1]:.1f}, {box[2]:.1f}, {box[3]:.1f}"]) + + return "\n".join(lines) + + +def event_payload(results: Sequence[FaceEyes], width: float, height: float) -> dict[str, object]: + """Build the WebSocket client event payload for one eye-detection inference.""" + width = require_non_negative_finite(width, "width") + height = require_non_negative_finite(height, "height") + eye_count = sum(len(result["eyes"]) for result in results) + return { + "mode": "eye_detection", + "detected_class": "eyes" if eye_count else "no_detection", + "faces": len(results), + "eyes": eye_count, + "results": list(results), + "processed_at": datetime.now().strftime("%X"), + "model_path": EYE_MODEL_PATH, + "source_resolution": {"width": float(width), "height": float(height)}, + } + + +def clamp(value: float, minimum: float, maximum: float) -> float: + """Clamp `value` to the inclusive range [minimum, maximum].""" + return max(minimum, min(value, maximum)) + + +def require_positive_finite(value: float, name: str) -> float: + """Return `value` as a float, raising if it isn't positive and finite.""" + value = float(value) + if not math.isfinite(value) or value <= 0.0: + raise ValueError(f"{name} must be a positive finite number") + return value + + +def require_non_negative_finite(value: float, name: str) -> float: + """Return `value` as a float, raising if it's negative or non-finite.""" + value = float(value) + if not math.isfinite(value) or value < 0.0: + raise ValueError(f"{name} must be a non-negative finite number") + return value diff --git a/services/ws-modules/pyeye1/pyproject.toml b/services/ws-modules/pyeye1/pyproject.toml new file mode 100644 index 00000000..fc1f3a70 --- /dev/null +++ b/services/ws-modules/pyeye1/pyproject.toml @@ -0,0 +1,35 @@ +[project] +dependencies = ["et-ws"] +description = "Python eye detection (MediaPipe FaceLandmarker -> eye bounding boxes)" +license = "Apache-2.0 OR MIT" +name = "et-ws-pyeye1" +requires-python = ">=3.10" +version = "0.1.0" + +[dependency-groups] +dev = ["pytest", "pytest-cov"] + +[build-system] +build-backend = "uv_build" +requires = ["uv_build==0.11.8"] + +[tool.uv.build-backend] +module-name = "pyeye1" +module-root = "" + +# `et-ws` is the generated Pydantic client used for the typed WsClientEvent message. +# It is wired as a uv workspace source so `uv run pytest` resolves it locally; the wheel itself is its own +# ws-module that Pyodide micropip-installs at runtime from its mount (`/modules/et-ws/`). +[tool.uv.sources] +et-ws = { workspace = true } + +[tool.ws-module.dependencies] +"@mediapipe/tasks-vision" = "*" +et-model-eye1 = "*" +et-ws = "*" +pyodide = "*" + +[tool.pytest.ini_options] +pythonpath = ["."] +# Optional: helps with modern import resolution +addopts = ["--import-mode=importlib"] diff --git a/services/ws-modules/pyeye1/tests/test_eye_detection.py b/services/ws-modules/pyeye1/tests/test_eye_detection.py new file mode 100644 index 00000000..141040fb --- /dev/null +++ b/services/ws-modules/pyeye1/tests/test_eye_detection.py @@ -0,0 +1,81 @@ +import unittest + +from pyeye1.eye_detection import ( + CONTOUR_LANDMARK_COUNT, + LEFT_EYE_INDICES, + MESH_LANDMARK_COUNT, + RIGHT_EYE_INDICES, + build_results, + decode_eye_boxes, + face_bounds, +) + + +def landmarks_with(points: dict[int, tuple[float, float]]) -> list[float]: + """Build a flat 478*2 normalized landmark list, placing (x, y) at the given landmark indices.""" + values = [0.0] * (MESH_LANDMARK_COUNT * 2) + for index, (x, y) in points.items(): + values[index * 2] = x + values[index * 2 + 1] = y + return values + + +class EyeDetectionTests(unittest.TestCase): + def test_eye_index_sets_are_within_the_mesh(self) -> None: + self.assertEqual(len(RIGHT_EYE_INDICES), 16) + self.assertEqual(len(LEFT_EYE_INDICES), 16) + for index in (*RIGHT_EYE_INDICES, *LEFT_EYE_INDICES): + self.assertGreaterEqual(index, 0) + self.assertLess(index, CONTOUR_LANDMARK_COUNT) + + def test_decode_eye_boxes_maps_normalized_landmarks_to_source_pixels(self) -> None: + first = RIGHT_EYE_INDICES[0] + last = RIGHT_EYE_INDICES[-1] + # Place two right-eye contour points at normalized (0.25, 0.5) and (0.75, 0.5) in a 640x480 frame. + values = landmarks_with({first: (0.25, 0.5), last: (0.75, 0.5)}) + # Give the left-eye cluster a non-zero location so its box isn't pinned at the origin. + for index in LEFT_EYE_INDICES: + values[index * 2] = 0.6 + values[index * 2 + 1] = 0.6 + + boxes = decode_eye_boxes(values, 640.0, 480.0) + right = boxes["right_eye"] + # Other right-eye indices sit at 0 -> min stays at origin; the two placed points set the max. + self.assertAlmostEqual(right[0], 0.0) + self.assertAlmostEqual(right[2], 0.75 * 640.0) + self.assertAlmostEqual(right[3], 0.5 * 480.0) + self.assertAlmostEqual(boxes["left_eye"][0], 0.6 * 640.0) + + def test_decode_eye_boxes_clamps_out_of_range_coords(self) -> None: + values = landmarks_with(dict.fromkeys(RIGHT_EYE_INDICES, (1.5, -0.2))) + boxes = decode_eye_boxes(values, 100.0, 200.0) + # 1.5 clamps to width 100, -0.2 clamps to 0. + self.assertAlmostEqual(boxes["right_eye"][0], 100.0) + self.assertAlmostEqual(boxes["right_eye"][2], 100.0) + self.assertAlmostEqual(boxes["right_eye"][1], 0.0) + + def test_decode_eye_boxes_rejects_short_output(self) -> None: + with self.assertRaisesRegex(ValueError, "face-mesh landmarks"): + decode_eye_boxes([0.0] * 10, 640.0, 480.0) + + def test_decode_eye_boxes_rejects_bad_dimensions(self) -> None: + with self.assertRaisesRegex(ValueError, "image_width"): + decode_eye_boxes(landmarks_with({}), 0.0, 480.0) + + def test_face_bounds_covers_all_landmarks(self) -> None: + values = landmarks_with({10: (0.1, 0.2), 20: (0.9, 0.8)}) + bounds = face_bounds(values, 1000.0, 500.0) + self.assertAlmostEqual(bounds[0], 0.0) + self.assertAlmostEqual(bounds[2], 0.9 * 1000.0) + self.assertAlmostEqual(bounds[3], 0.8 * 500.0) + + def test_build_results_pairs_each_face_with_two_eyes(self) -> None: + faces = [landmarks_with(dict.fromkeys((*RIGHT_EYE_INDICES, *LEFT_EYE_INDICES), (0.5, 0.5)))] + results = build_results(faces, 640.0, 480.0) + self.assertEqual(len(results), 1) + self.assertEqual([eye["label"] for eye in results[0]["eyes"]], ["left_eye", "right_eye"]) + self.assertEqual(len(results[0]["face_box"]), 4) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/ws-modules/pyface1/pkg/et_ws_pyface1.js b/services/ws-modules/pyface1/pkg/et_ws_pyface1.js index 688da912..55ca49d2 100644 --- a/services/ws-modules/pyface1/pkg/et_ws_pyface1.js +++ b/services/ws-modules/pyface1/pkg/et_ws_pyface1.js @@ -3,6 +3,15 @@ const PYODIDE_BASE_PATH = "/modules/pyodide/"; +// One colour per RetinaFace landmark, keyed by the label the Python decoder emits. +const LANDMARK_COLORS = { + left_eye: "#ff4d4d", + right_eye: "#4dff88", + nose: "#ffd24d", + left_mouth: "#4db8ff", + right_mouth: "#c86bff", +}; + let pyodide; let py; let cfg; @@ -176,6 +185,13 @@ function render(detectionsJson) { ctx.fillRect(left, Math.max(top - 24, 0), ctx.measureText(label).width + 10, 22); ctx.fillStyle = "#fffdfa"; ctx.fillText(label, left + 5, Math.max(top - 8, 16)); + + for (const point of detection.landmarks ?? []) { + ctx.beginPath(); + ctx.arc(point.x, point.y, 3, 0, Math.PI * 2); + ctx.fillStyle = LANDMARK_COLORS[point.label] ?? "#fffdfa"; + ctx.fill(); + } } } diff --git a/services/ws-modules/pyface1/pyface1/face_detection.py b/services/ws-modules/pyface1/pyface1/face_detection.py index 6463100d..da667d3f 100644 --- a/services/ws-modules/pyface1/pyface1/face_detection.py +++ b/services/ws-modules/pyface1/pyface1/face_detection.py @@ -30,14 +30,27 @@ Prior = tuple[float, float, float, float] DecodedBox = tuple[float, float, float, float] +# RetinaFace emits five landmark points per face, in this fixed order (subject's perspective, so the +# "left" eye/mouth-corner appears on the right of a non-mirrored frame). The eye points are indices 0 and 1. +LANDMARK_LABELS = ("left_eye", "right_eye", "nose", "left_mouth", "right_mouth") + + +class Landmark(TypedDict): + """One facial landmark point: its label and its coordinates in source-image pixels.""" + + label: str + x: float + y: float + class Detection(TypedDict): - """One detected face: label, class index, score, and bounding box.""" + """One detected face: label, class index, score, bounding box, and the five landmark points.""" label: str class_index: int score: float box: Box + landmarks: list[Landmark] class DetectionSummary(TypedDict): @@ -243,12 +256,23 @@ def decode_outputs( clamp((decoded[3] * FACE_INPUT_HEIGHT) / resize_ratio, 0.0, source_height), ] + points = decode_landmarks(landm[index * 10 : index * 10 + 10], priors[index]) + landmarks: list[Landmark] = [ + { + "label": LANDMARK_LABELS[point_index], + "x": clamp((point[0] * FACE_INPUT_WIDTH) / resize_ratio, 0.0, source_width), + "y": clamp((point[1] * FACE_INPUT_HEIGHT) / resize_ratio, 0.0, source_height), + } + for point_index, point in enumerate(points) + ] + detections.append( { "label": "face", "class_index": 0, "score": score, "box": box, + "landmarks": landmarks, } ) @@ -365,6 +389,26 @@ def decode_box(loc: Sequence[float], prior: Sequence[float]) -> DecodedBox: ) +def decode_landmarks(landm: Sequence[float], prior: Sequence[float]) -> list[tuple[float, float]]: + """Decode the five RetinaFace landmark points from their offsets and prior box. + + Each point is a plain centre offset (no width/height exponent), so decode is simpler than the box: the + normalised coordinate is the prior centre plus the offset scaled by the first variance and the prior size. + """ + if len(landm) != 10: + raise ValueError("landm must contain exactly 10 values") + if len(prior) != 4: + raise ValueError("prior must contain exactly 4 values") + + return [ + ( + prior[0] + landm[point * 2] * RETINAFACE_VARIANCES[0] * prior[2], + prior[1] + landm[point * 2 + 1] * RETINAFACE_VARIANCES[0] * prior[3], + ) + for point in range(5) + ] + + def apply_nms(detections: list[Detection], threshold: float) -> list[Detection]: """Apply non-maximum suppression, keeping the highest-scoring boxes.""" threshold = require_non_negative_finite(threshold, "threshold") diff --git a/services/ws-modules/pyface1/tests/test_face_detection.py b/services/ws-modules/pyface1/tests/test_face_detection.py index 7007ce8d..058d58cd 100644 --- a/services/ws-modules/pyface1/tests/test_face_detection.py +++ b/services/ws-modules/pyface1/tests/test_face_detection.py @@ -4,8 +4,10 @@ from pyface1.face_detection import ( FACE_INPUT_HEIGHT, FACE_INPUT_WIDTH, + LANDMARK_LABELS, build_priors, decode_box, + decode_landmarks, decode_outputs, output_values, preprocess_geometry, @@ -43,6 +45,45 @@ def test_zero_offsets_decode_to_prior_box(self) -> None: self.assertAlmostEqual(decoded[2], 0.625, places=6) self.assertAlmostEqual(decoded[3], 0.75, places=6) + def test_zero_offsets_decode_landmarks_to_prior_centre(self) -> None: + points = decode_landmarks([0.0] * 10, [0.5, 0.5, 0.25, 0.5]) + + self.assertEqual(len(points), len(LANDMARK_LABELS)) + for point in points: + self.assertAlmostEqual(point[0], 0.5, places=6) + self.assertAlmostEqual(point[1], 0.5, places=6) + + def test_decode_landmarks_applies_offset_variance_and_prior_size(self) -> None: + landm = [1.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + points = decode_landmarks(landm, [0.5, 0.5, 0.25, 0.5]) + + self.assertAlmostEqual(points[0][0], 0.5 + 1.0 * 0.1 * 0.25, places=6) + self.assertAlmostEqual(points[0][1], 0.5 + 2.0 * 0.1 * 0.5, places=6) + self.assertAlmostEqual(points[1][0], 0.5, places=6) + + def test_decode_landmarks_rejects_bad_lengths(self) -> None: + with self.assertRaisesRegex(ValueError, "landm"): + decode_landmarks([0.0] * 8, [0.5, 0.5, 0.25, 0.5]) + with self.assertRaisesRegex(ValueError, "prior"): + decode_landmarks([0.0] * 10, [0.5, 0.5]) + + def test_decode_outputs_attaches_five_labelled_landmarks(self) -> None: + priors = build_priors(float(FACE_INPUT_HEIGHT), float(FACE_INPUT_WIDTH)) + loc = [0.0] * (len(priors) * 4) + landm = [0.0] * (len(priors) * 10) + # Force a single high-confidence detection at prior 0 (softmax picks index 1 as the face class). + conf = [0.0] * (len(priors) * 2) + conf[1] = 20.0 + + summary = decode_outputs(loc, conf, landm, 1.0, float(FACE_INPUT_WIDTH), float(FACE_INPUT_HEIGHT)) + + self.assertEqual(len(summary["detections"]), 1) + landmarks = summary["detections"][0]["landmarks"] + self.assertEqual([point["label"] for point in landmarks], list(LANDMARK_LABELS)) + for point in landmarks: + self.assertGreaterEqual(point["x"], 0.0) + self.assertGreaterEqual(point["y"], 0.0) + def test_output_values_rejects_trailing_shape_data(self) -> None: with self.assertRaisesRegex(ValueError, "unexpected shapes"): output_values([0.0, 1.0, 2.0], "loc", 4) diff --git a/services/ws-web-runner/tests/modules.rs b/services/ws-web-runner/tests/modules.rs index 919c7455..0caf1046 100644 --- a/services/ws-web-runner/tests/modules.rs +++ b/services/ws-web-runner/tests/modules.rs @@ -20,6 +20,7 @@ //! - **et-ws-face-detection** -- camera (`getUserMedia`) + ONNX model //! - **et-ws-har1** -- accelerometer (`DeviceMotionEvent`) + ONNX model //! - **et-ws-pyface1** -- Pyodide + camera + ONNX model +//! - **et-ws-pyeye1** -- Pyodide + camera + `MediaPipe` `FaceLandmarker` (tflite) -> eye boxes //! //! ### Non-JS module loaders / incompatible runtimes //! @@ -149,6 +150,7 @@ fn multi_agent_module(#[case] module: &str, #[case] language: Language) { #[case::speech_recognition("et-ws-speech-recognition", Language::Rust)] #[case::video1("et-ws-video1", Language::Rust)] #[case::pyface1("et-ws-pyface1", Language::Python)] +#[case::pyeye1("et-ws-pyeye1", Language::Python)] fn hardware_module_load_fails(#[case] module: &str, #[case] language: Language) { if !mise_env_includes(language) { println!( diff --git a/uv.lock b/uv.lock index fc14e3f0..f710b2ef 100644 --- a/uv.lock +++ b/uv.lock @@ -7,6 +7,7 @@ members = [ "et-rest-client", "et-ws", "et-ws-pydata1", + "et-ws-pyeye1", "et-ws-pyface1", "et-ws-wasi-graphics-info", ] @@ -205,6 +206,29 @@ requires-dist = [ { name = "pyodide-http" }, ] +[[package]] +name = "et-ws-pyeye1" +version = "0.1.0" +source = { editable = "services/ws-modules/pyeye1" } +dependencies = [ + { name = "et-ws" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [{ name = "et-ws", editable = "generated/python-ws" }] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest" }, + { name = "pytest-cov" }, +] + [[package]] name = "et-ws-pyface1" version = "0.1.0"