diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 253bc92a..902ed172 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -68,7 +68,7 @@ reviews: Root CMake build. Must use C++20, -Wall -Wextra -Werror. Core library is INTERFACE (header-only). Only platform .cpp files need compilation. - - path: "scripts/**" + - path: "moondeck/**" instructions: | MoonDeck dev console and build/test/run scripts. Scripts must resolve - ROOT path correctly relative to their location in scripts/subdir/. + ROOT path correctly relative to their location in moondeck/subdir/. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 18c2fb5e..23cf38d4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ on: paths: - 'src/**' - 'esp32/**' - - 'scripts/build/**' + - 'moondeck/build/**' - 'CMakeLists.txt' - 'library.json' - '.github/workflows/release.yml' @@ -79,7 +79,7 @@ jobs: # warning on the direct ${{ }} expansion inside a shell string. env: TAG: ${{ inputs.tag || github.ref_name }} - run: uv run scripts/build/verify_version.py --tag "$TAG" + run: uv run moondeck/ci/verify_version.py --tag "$TAG" # The shipping firmware list, read from the generated web-installer/firmwares.json # (projected from build_esp32.py's FIRMWARES dict, drift-guarded by @@ -160,7 +160,7 @@ jobs: TAG: ${{ steps.tag.outputs.tag }} run: | set -euo pipefail - V=$(python scripts/build/compute_version.py --tag "$TAG") + V=$(python moondeck/build/compute_version.py --tag "$TAG") echo "version=$V" >> "$GITHUB_OUTPUT" - name: Build firmware @@ -188,7 +188,7 @@ jobs: # We run our own builder (not the action's default `idf.py build`) # so the sdkconfig fragments and EXCLUDE_COMPONENTS go through the # same code path as local builds. --release burns the channel tag in. - command: python ../scripts/build/build_esp32.py --firmware ${{ matrix.firmware }} --release "${{ steps.tag.outputs.tag }}" --version "${{ steps.ver.outputs.version }}" + command: python ../moondeck/build/build_esp32.py --firmware ${{ matrix.firmware }} --release "${{ steps.tag.outputs.tag }}" --version "${{ steps.ver.outputs.version }}" - name: Stage release artifacts run: | @@ -234,7 +234,7 @@ jobs: # runners don't ship uv by default — install it before package_desktop. - uses: astral-sh/setup-uv@v3 - name: Build + package macOS arm64 - run: uv run scripts/build/package_desktop.py + run: uv run moondeck/ci/package_desktop.py - uses: actions/upload-artifact@v4 with: name: desktop-macos @@ -250,7 +250,7 @@ jobs: # Same uv prerequisite as build-macos — see the comment there. - uses: astral-sh/setup-uv@v3 - name: Build + package Windows x64 - run: uv run scripts/build/package_desktop.py + run: uv run moondeck/ci/package_desktop.py - uses: actions/upload-artifact@v4 with: name: desktop-windows @@ -326,7 +326,7 @@ jobs: TAG: ${{ steps.tag.outputs.tag }} run: | set -euo pipefail - V=$(uv run python scripts/build/compute_version.py --tag "$TAG") + V=$(uv run python moondeck/build/compute_version.py --tag "$TAG") echo "version=$V" >> "$GITHUB_OUTPUT" - name: Generate ESP Web Tools manifests (release-asset URLs) @@ -344,7 +344,7 @@ jobs: # The shipping firmware list — the same web-installer/firmwares.json the # build matrix reads, so manifests and builds can't drift. for F in $(jq -r '.firmwares[] | select(.ships) | .name' web-installer/firmwares.json); do - uv run python scripts/build/generate_manifest.py \ + uv run python moondeck/build/generate_manifest.py \ --firmware "$F" \ --version "$V" \ --release-url "$BASE" \ @@ -567,7 +567,7 @@ jobs: # installer staged above under pages/install/ survives (a plain # --site-dir pages would wipe it — mkdocs cleans its output dir). # history/ and backlog/ are excluded in mkdocs.yml (internal docs). - uv run scripts/docs/build_docs.py --site-dir "$RUNNER_TEMP/docs-site" + uv run moondeck/docs/build_docs.py --site-dir "$RUNNER_TEMP/docs-site" cp -r "$RUNNER_TEMP/docs-site/." pages/ ls -la pages/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ee68d4eb..c7f50542 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ name: Test on: pull_request: paths: &test-paths - - 'scripts/**' + - 'moondeck/**' - 'web-installer/**' - 'src/core/ImprovFrame.h' - 'src/core/ImprovOpReassembler.h' diff --git a/.gitignore b/.gitignore index 1042d4c2..bd58df0f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,11 +39,11 @@ *.out *.app -# Build directories (anchored to root, not scripts/build/) +# Build directories (anchored to root, not moondeck/build/) /build/ /build-*/ -# Packaging output from scripts/build/package_desktop.py — the .tar.gz / .zip +# Packaging output from moondeck/build/package_desktop.py — the .tar.gz / .zip # bundles attached to GitHub releases. CI is the authoritative producer # (release.yml on tag/main push); local runs are scratch for dev verification. # Never commit these — they bloat the repo and can drift from source. @@ -74,10 +74,10 @@ Testing/ .cache/ # MoonDeck state -scripts/moondeck.json +moondeck/moondeck.json # Flash-event breadcrumb dropped by flash_esp32.py and consumed by MoonDeck # on the next device refresh to link last_port → device. -scripts/.last_flash.json +moondeck/.last_flash.json # Generated files src/ui/ui_embedded.h @@ -117,11 +117,11 @@ __pycache__/ # MkDocs — built docs site (rendered fresh in CI, never committed) /site/ -# Generated test inventory pages — built into the site by scripts/docs/mkdocs_hooks.py +# Generated test inventory pages — built into the site by moondeck/docs/mkdocs_hooks.py # (the CLI generate_test_docs.py can still write them locally, but they are not committed). /docs/tests/*.md # Generated per-module technical pages — Doxygen+moxygen from each `.h`, built into -# the site by scripts/docs/mkdocs_hooks.py (via gen_api.py). Regenerated every build. +# the site by moondeck/docs/mkdocs_hooks.py (via gen_api.py). Regenerated every build. /docs/moonmodules/core/moxygen/ /docs/moonmodules/light/moxygen/ diff --git a/CLAUDE.md b/CLAUDE.md index e8c40fec..4cd8d096 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,7 +12,7 @@ See `docs/architecture.md` for system design. This file contains only rules and - **Industry standards, our own code.** Reach for the established, recognisable solution: the textbook *algorithm* (a DC-blocker high-pass, a Hann window, RMS, an integer-square-root) AND the textbook *name* for every variable, function, and UI control. That's *Common patterns first* applied to both domains, core and light: take the textbook approach over a clever or borrowed one. Study the prior art hard, whatever sharpens our thinking: repos, datasheets, vendor sites. Respect it, learn from it, credit it by name in the `history/` digests and per-module "Prior art" sections. Then write every line fresh against our own architecture: **carry the ideas forward, but write our own code rather than copying theirs or tracing their structure.** The method that guarantees it: spec from primary sources (ESP32 / peripheral / sensor datasheets, Espressif docs, reference standards), pin the behaviour as tests (unit + scenario), and let the worker-bee agents implement against the process ([CLAUDE.md](CLAUDE.md)) and architecture ([architecture.md](docs/architecture.md)). The result is independent by construction, not a renamed copy. - **Minimalism means elegance, not fewest features.** Minimalism here is *consistency, reusability, no duplication, minimal memory, and the fastest hot path* — not "the smallest feature set" or "the fewest lines." A recognizable, proven construct (recursion, a textbook algorithm, a clean data structure) that ticks those boxes is *more* minimal than a hand-rolled special case, even when it's more code, because it's reusable and consistent rather than a one-off. So **reach for the established, beautiful solution**, and when you *know* a complex system will need a capability, build the cleanest *complete* version of it rather than a crippled subset that forces hacks elsewhere later (a JSON reader that can't read arrays is not "minimal" — it pushes ugliness outward). Guardrail: the construct must be *recognizable* (see *Common patterns first*), not bespoke cleverness — a contributor with general experience should recognise *what* it is within 30 seconds, even if the full *why* takes longer. Beauty and consistency win over raw line count. - **Complexity lives in core; domain modules stay simple.** When something is genuinely hard — a recursive parser, a bounded arena, a scheduling or mapping algorithm — it belongs in the **core**, written once as the elegant standard construct (see *Minimalism means elegance*), so every **domain module** (a light effect/driver/layout/modifier, DevicesModule, …) gets to stay short and obvious by leaning on it. A domain module that is accreting parsing, buffer juggling, or clever bookkeeping is a smell: that complexity wants to move down into a core primitive the module then calls in a line or two. The test: a domain module should read like *what it does*, not *how the hard parts work* — the "how" is core's job. (Example: the device-list persistence — the recursive JSON reader is core; DevicesModule's restore is "parse, fill my array," a dozen plain lines.) This is the same coin as *Core grows slower than the domain*: core earns its size by absorbing complexity that would otherwise be duplicated, badly, across many simple modules. -- **Core grows slower than the domain.** Adding a domain module is expected growth: a new unit of domain capability adds lines because it adds a feature, and that's fine. The **core** (domain-neutral infrastructure: `src/core/`, the platform layer, the docs that describe them) is held to a higher bar: while the system is still being built the core does grow, but each core change should buy proportionally more than a domain addition: new infrastructure that many modules use, not a one-off. A core change is suspect until that leverage is shown. Watch the ratio; core is meant to be the lean base under a wider domain, not the other way around. +- **Core grows slower than the domain.** Adding a domain module is expected growth: a new unit of domain capability adds lines because it adds a feature, and that's fine. The **core** (domain-neutral infrastructure: `src/core/`, the platform layer, the docs that describe them) is held to a higher bar: each core change buys proportionally more than a domain addition — new infrastructure that many modules use, not a one-off. A core change is suspect until that leverage is shown. Core is the lean base under a wider domain, not the other way around. The bar is a *kind* test, not a line-count race: the domain is effectively unbounded (each light effect/driver/device model is a self-contained feature), whereas **core earns growth only by adding a recognizable, industry-standard, reusable primitive** (a streaming write, a positional read, a bounded arena, a recursive JSON reader) — never a bespoke one-off. Such a primitive is welcome *because it is the textbook construct many modules lean on*, not because it is small; a core change that only one caller needs is the smell this rule catches. (This is *Common patterns first* and *Industry standards, our own code* applied to the core/domain growth ratio.) - **Default to subtraction.** The reflex on most changes (a bug fix, a review finding, a refactor) should be *can this remove or replace code, or land net-neutral?*, not *what do I add?* If a change only ever grows the line count and the doc count, that's the smell this rule exists to catch. Prefer removing code over adding it; a deletion that preserves behaviour is the best kind of change. - **Continuous refactor, no hacks.** Improvement is not a scheduled phase; it happens *the moment* a hack, a divergence, or a duplicated pattern is spotted, in whatever change is already open. The bar is absolute: **never** leave a hack, a workaround, or a bespoke one-off in place because "it works for now" — the fix is the *recognisable, standard* one. So when you reach for a clever shortcut, an environment sniff, a duplicated block, a stub that papers over a broken dependency, stop and ask *what's the textbook construct here?* and do that instead. This is the union of three principles applied as a working reflex rather than a checklist: *[Common patterns first](#principles)* (use the construct a new contributor recognises in 30s), *[Industry standards, our own code](#principles)* (the textbook algorithm AND the textbook name, written fresh against our architecture), and *[Minimalism means elegance](#principles)* (consistency, reuse, no duplication, the fast hot path). What this bullet adds over those: the **timing** (on sight, continuously, not deferred to a "cleanup later" that never comes) and the **no-hacks floor** (a workaround is never the destination; if the standard fix is genuinely out of scope right now, the hack doesn't ship — it's backlogged with the standard fix named, per *[Mandatory subtraction](#process-rules)*). The product-owner-initiated counterpart, for larger restructures, is the *[Refactor for simplicity](#process-rules)* process rule; this principle is the small-scale, agent-initiated version of the same instinct. - **No duplication, in code or docs.** Same logic in two places belongs in one shared function; same fact in two docs belongs in one place the other links to. A comment or doc paragraph that restates what the code already says is duplication too; delete it. (Reuse a recognisable shape rather than inventing one; see *Common patterns first* above.) @@ -28,7 +28,7 @@ See `docs/architecture.md` for system design. This file contains only rules and The design rationale for each rule below lives in [docs/architecture.md](docs/architecture.md). The one-liners here are what the agent holds in working memory. -**Tests must pass.** Run `ctest` (unit tests) and `uv run scripts/scenario/run_scenario.py` (scenarios) before considering work complete. The Python wrapper invokes the C++ runner and persists per-target observations back to each scenario JSON (drift visibility); direct `./build/test/mm_scenarios` is fine for ad-hoc pass/fail checks but skips the JSON write-back. New core logic needs a corresponding module test. Full pipeline needs a scenario test. See [docs/testing.md](docs/testing.md) for the strategy and test inventory. +**Tests must pass.** Run `ctest` (unit tests) and `uv run moondeck/scenario/run_scenario.py` (scenarios) before considering work complete. The Python wrapper invokes the C++ runner and persists per-target observations back to each scenario JSON (drift visibility); direct `./build/test/mm_scenarios` is fine for ad-hoc pass/fail checks but skips the JSON write-back. New core logic needs a corresponding module test. Full pipeline needs a scenario test. See [docs/testing.md](docs/testing.md) for the strategy and test inventory. **Warnings are errors.** Build with `-Wall -Wextra -Werror`. No warning is "harmless"; if it's noise, fix it or silence it explicitly with a `-Wno-…` justified in code. @@ -58,7 +58,7 @@ Then check the recommendation against [§ Principles](#principles) (minimalism, **Plan before implementing.** Use `/plan` mode before every feature. Review plans for: unnecessary files, inheritance where structs suffice, modifications outside the relevant directory. Reject and regenerate bad plans. **Save every approved plan** to `docs/history/plans/` named `Plan-YYYYMMDD - .md` (ISO-8601 date order so the directory sorts chronologically, e.g. `Plan-20260620 - Improv-as-REST.md` for 2026-06-20), as the first implementation step. The plan is the design record that complements `decisions.md` (the lesson record): the plan says what we set out to build and why; decisions.md captures what we learned doing it. **These saved plans are a reference archive for the product owner — agents WRITE a plan when creating one, but do NOT read the existing plan files for context unless the product owner explicitly points to one** (they're under the "Never automatically" rule below alongside the rest of `docs/history/`). Plans are **kept, not pruned** — they are the permanent design-intent record. When a plan's design ships (or doesn't), mark its outcome in the filename with a trailing parenthetical — `… (shipped).md` once it lands, `… (attempted, abandoned).md` if it was tried and dropped — so the directory shows at a glance what's done; an unmarked plan is still in flight. -**Use `uv` for every Python invocation.** Never type `python` or `python3` directly; always go through `uv run` (e.g. `uv run scripts/build/build_desktop.py`, `uv run python -c "…"`). This applies to shell commands, CMake `add_custom_command` / `execute_process`, documentation examples, and anything that shells out. In CMake, resolve `find_program(UV_EXECUTABLE NAMES uv REQUIRED HINTS "$ENV{USERPROFILE}/.local/bin" "$ENV{HOME}/.local/bin")` once and use `${UV_EXECUTABLE} run python …` thereafter. Reason: uv manages the project venv and is the project standard ([scripts/MoonDeck.md](scripts/MoonDeck.md)); bare `python3` isn't on PATH on Windows (and macOS Python Launcher pops a Store prompt). If you catch yourself about to type `python`, stop and prefix with `uv run`. +**Use `uv` for every Python invocation.** Never type `python` or `python3` directly; always go through `uv run` (e.g. `uv run moondeck/build/build_desktop.py`, `uv run python -c "…"`). This applies to shell commands, CMake `add_custom_command` / `execute_process`, documentation examples, and anything that shells out. In CMake, resolve `find_program(UV_EXECUTABLE NAMES uv REQUIRED HINTS "$ENV{USERPROFILE}/.local/bin" "$ENV{HOME}/.local/bin")` once and use `${UV_EXECUTABLE} run python …` thereafter. Reason: uv manages the project venv and is the project standard ([moondeck/MoonDeck.md](moondeck/MoonDeck.md)); bare `python3` isn't on PATH on Windows (and macOS Python Launcher pops a Store prompt). If you catch yourself about to type `python`, stop and prefix with `uv run`. The **one exception** is `esp32/main/CMakeLists.txt`: ESP-IDF builds use IDF's bundled Python venv, not the project venv; adding uv to ESP-IDF docker would be a bigger CI lift than the portability win pays for. That file uses `find_package(Python3 REQUIRED COMPONENTS Interpreter)` and invokes `${Python3_EXECUTABLE}`, so CMake locates whichever Python IDF set up (`.venv\Scripts\python.exe` on Windows IDF, `.venv/bin/python3` on macOS/Linux IDF). The shared `src/ui/embed_ui.cmake` script takes a `PYTHON_CMD` parameter that callers pass: desktop passes `${UV_EXECUTABLE};run;python`, ESP32 passes `${Python3_EXECUTABLE}`. @@ -68,7 +68,7 @@ The **one exception** is `esp32/main/CMakeLists.txt`: ESP-IDF builds use IDF's b **Anti-stalling.** If a build error or test failure takes more than 2 attempts to fix, STOP. Do not rewrite surrounding files. Ask the product owner for guidance or rollback with Git and re-approach. -**Git: only on explicit request.** Do not `git add`, `git commit`, or `git push` on your own initiative. Only execute these when the product owner explicitly asks (e.g. "commit now", "push it"). The product owner controls when changes are staged, committed, and pushed. +**Git: only on explicit request.** Do not `git add`, `git commit`, or `git push` on your own initiative. Only execute these when the product owner explicitly asks (e.g. "commit now", "push it"). The product owner controls when changes are staged, committed, and pushed. Two workflow specifics: (1) **one combined commit per cycle** — all uncommitted branch work lands as a *single* commit, never split into standalone/partial commits (not even a small hygiene or doc change; leave those uncommitted to fold into the next combined commit), with the message's change sections grouping the areas. (2) **A verified hotfix goes straight to `main`** — a small, already-tested CI/build/manifest fix is committed directly on `main`, no feature branch or PR (the "branch first on the default branch" rule is for *feature* work that warrants review); feature work still branches. **Gate lists are initiated by the product owner, not by agents. Never start a gate list on your own; always ask first.** The full set of gates (especially the Opus reviewer at PR-merge and the ESP32 build at commit) easily takes 10 minutes per event and burns real tokens; agents must not initiate them unprompted. When the product owner says "run pre-commit" / "commit now" / "pre-merge" / "ready to release" the relevant list below runs. Do NOT start any list automatically because you finished a feature, because tests pass, because a milestone feels reached, because completion seems imminent, or because an earlier instruction implied a sequence ending in "commit". If you finished feature work and are unsure whether the product owner wants to proceed, **ask**: "Feature work is done, should I run pre-commit, or do you want to look first?" Treat each list as a gate the product owner opens; the agent's job is finishing the work and reporting status so the product owner can decide when to open it. @@ -107,21 +107,21 @@ The narrow safety net: "this snapshot is internally consistent." **Conditional (run if trigger matches):** -2. Desktop build, `cmake --build build` (zero warnings), if any file that compiles into the desktop binary changed: `src/`, `test/`, `CMakeLists.txt` (root or `test/`), `library.json`. A YAML / docs / `scripts/` / `.claude/` change cannot break the build. +2. Desktop build, `cmake --build build` (zero warnings), if any file that compiles into the desktop binary changed: `src/`, `test/`, `CMakeLists.txt` (root or `test/`), `library.json`. A YAML / docs / `moondeck/` / `.claude/` change cannot break the build. 3. Unit tests, `ctest --output-on-failure` (all pass), same trigger as Desktop build. No build, no tests. -4. Scenario tests, `uv run scripts/scenario/run_scenario.py` (all pass; wraps `mm_scenarios` and persists per-target `observed.<target>` blocks back to each scenario JSON for drift tracking), same trigger as Desktop build, plus any `test/scenarios/*.json` change. +4. Scenario tests, `uv run moondeck/scenario/run_scenario.py` (all pass; wraps `mm_scenarios` and persists per-target `observed.<target>` blocks back to each scenario JSON for drift tracking), same trigger as Desktop build, plus any `test/scenarios/*.json` change. 5. Platform boundary, `check_platform_boundary.py`, if any file under `src/` (excluding `src/platform/`) changed. 6. ESP32 build, `build_esp32.py`, if any file under `src/` (excluding `src/platform/desktop/`), `esp32/`, `CMakeLists.txt`, or `library.json` changed. 7. KPI collection, `collect_kpi.py --commit`, if any file under `src/` changed. **The one-liner MUST include `tick:Xus(FPS:Y)` for every supported target** (PC + ESP32 today; Teensy/RPi when added). If a target's tick/FPS is missing (e.g. ESP32 wasn't monitored recently and `esp32/monitor.log` is stale), re-run a short live capture before committing, or note explicitly in the commit body why the value is absent. -8. Device-model catalog, `check_devices.py`, fast (<1s), if `web-installer/deviceModels.json` or `scripts/check/check_devices.py` changed. Validates the installer catalog: required fields, `firmwares` a non-empty list of non-empty strings (`firmwares[0]` is the default), every `image` resolves on disk, each entry's `System.deviceModel` control equals its entry `name`, module `type`s are factory-registered (or boot-wired singletons), `pins` controls live only on `*LedDriver` modules, and `supported` capabilities stay within the known vocabulary. -9. Firmware list, `check_firmwares.py`, fast (<1s), if `scripts/build/build_esp32.py`, `web-installer/firmwares.json`, or `scripts/check/check_firmwares.py` changed. Regenerates the firmware projection from the `FIRMWARES` dict and fails on drift from the committed `web-installer/firmwares.json` (so a `FIRMWARES` edit without regenerating is caught). Trigger includes `build_esp32.py` because that dict is the upstream source. -10. Host-side unit tests (Python + JS), fast (<2s), the suites the C++ ctest can't reach (MoonDeck / build-script logic, web-installer logic). Run `uv run --with pytest --with pyserial pytest test/python -q` if any file under `scripts/` or `test/python/` changed, and `node --test "test/js/**/*.test.mjs"` if any file under `web-installer/` or `test/js/` changed. (Same suites the PR-triggered `.github/workflows/test.yml` runs.) Today these pin the Improv frame wire format — `test/python` + `test/js` assert a shared golden vector so the device C++, Python, and JS frame builders can't drift. New Python/JS unit suites land under `test/python` / `test/js` and run here. +8. Device-model catalog, `check_devices.py`, fast (<1s), if `web-installer/deviceModels.json` or `moondeck/check/check_devices.py` changed. Validates the installer catalog: required fields, `firmwares` a non-empty list of non-empty strings (`firmwares[0]` is the default), every `image` resolves on disk, each entry's `System.deviceModel` control equals its entry `name`, module `type`s are factory-registered (or boot-wired singletons), `pins` controls live only on `*LedDriver` modules, and `supported` capabilities stay within the known vocabulary. +9. Firmware list, `check_firmwares.py`, fast (<1s), if `moondeck/build/build_esp32.py`, `web-installer/firmwares.json`, or `moondeck/check/check_firmwares.py` changed. Regenerates the firmware projection from the `FIRMWARES` dict and fails on drift from the committed `web-installer/firmwares.json` (so a `FIRMWARES` edit without regenerating is caught). Trigger includes `build_esp32.py` because that dict is the upstream source. +10. Host-side unit tests (Python + JS), fast (<2s), the suites the C++ ctest can't reach (MoonDeck / build-script logic, web-installer logic). Run `uv run --with pytest --with pyserial pytest test/python -q` if any file under `moondeck/` or `test/python/` changed, and `node --test "test/js/**/*.test.mjs"` if any file under `web-installer/` or `test/js/` changed. (Same suites the PR-triggered `.github/workflows/test.yml` runs.) Today these pin the Improv frame wire format — `test/python` + `test/js` assert a shared golden vector so the device C++, Python, and JS frame builders can't drift. New Python/JS unit suites land under `test/python` / `test/js` and run here. -A commit that touches *only* `.github/`, `docs/`, `README.md`, `CLAUDE.md`, or `.claude/` therefore runs only the spec check (plus the board-catalog and firmware checks when their specific files changed); the build/test/ESP32/KPI gates are no-ops because their triggers don't fire. A `scripts/` change adds the Python host-side unit-test gate, and a `web-installer/` change adds the JS one, but both still skip the C++ build/ESP32/KPI gates. This is the intended pre-commit cost for CI-only or doc-only changes. +A commit that touches *only* `.github/`, `docs/`, `README.md`, `CLAUDE.md`, or `.claude/` therefore runs only the spec check (plus the board-catalog and firmware checks when their specific files changed); the build/test/ESP32/KPI gates are no-ops because their triggers don't fire. A `moondeck/` change adds the Python host-side unit-test gate, and a `web-installer/` change adds the JS one, but both still skip the C++ build/ESP32/KPI gates. This is the intended pre-commit cost for CI-only or doc-only changes. **Recommended (manual, not blocking):** -- **Improv smoke test**, `uv run scripts/build/improv_smoke_test.py --port <port>` (or MoonDeck → ESP32 → **Improv Smoke Test**), recommended when a connected ESP32 is available and any of these changed: `src/core/ImprovFrame.h`, `src/platform/esp32/platform_esp32_improv.cpp`, `web-installer/index.html`, `src/ui/install-picker.js`, `scripts/build/improv_*.py`. Three-step end-to-end check (probe + WiFi provision + LAN reachability). Not a blocking gate because it needs hardware that isn't always plugged in; pair with `preview_installer`'s flash-ready mode for the browser-side equivalent. +- **Improv smoke test**, `uv run moondeck/build/improv_smoke_test.py --port <port>` (or MoonDeck → ESP32 → **Improv Smoke Test**), recommended when a connected ESP32 is available and any of these changed: `src/core/ImprovFrame.h`, `src/platform/esp32/platform_esp32_improv.cpp`, `web-installer/index.html`, `src/ui/install-picker.js`, `moondeck/build/improv_*.py`. Three-step end-to-end check (probe + WiFi provision + LAN reachability). Not a blocking gate because it needs hardware that isn't always plugged in; pair with `preview_installer`'s flash-ready mode for the browser-side equivalent. **After all gates pass:** stop and wait for the product owner to explicitly say "commit now" (or equivalent). Do not commit on your own initiative. @@ -135,7 +135,7 @@ A commit that touches *only* `.github/`, `docs/`, `README.md`, `CLAUDE.md`, or ` - **Core** (`src/core/`, `src/platform/`): e.g. `- HttpServerModule: added 409 guard to prevent overlapping OTA jobs` - **Light domain** (`src/light/`): e.g. `- PreviewDriver: replaced strided sampling with max-pooling to fix empty frames` - **UI** (`src/ui/`): e.g. `- app.js: auto-fit camera distance on first preview frame` - - **Scripts / MoonDeck** (`scripts/`): e.g. `- MoonDeck: added per-scenario dropdown to scenario card` + - **Scripts / MoonDeck** (`moondeck/`): e.g. `- MoonDeck: added per-scenario dropdown to scenario card` - **Tests** (`test/`): e.g. `- test_preview_driver: updated default assertions for new fps/detail/decompress values` - **Docs / CI** (`docs/`, `README.md`, `CLAUDE.md`, `.github/`, `CMakeLists.txt`): e.g. `- README: consolidated ESP32 install info to web installer` - **Reviews**: if any review findings were processed in this commit, one bullet per finding, prefixed by reviewer: 🐇 for CodeRabbit findings, 👾 for internal Reviewer agent findings. Each bullet states what was flagged, what was done (fixed / accepted / deferred), and, if not fixed, why. Omit if no review findings were processed. @@ -143,7 +143,7 @@ A commit that touches *only* `.github/`, `docs/`, `README.md`, `CLAUDE.md`, or ` **Not at commit-time** (these run at PR-merge): Reviewer agent; live perf analysis + `docs/performance.md` update; documentation sync sweep; permission review. -**On-demand reviewer.** The product owner can ask for a reviewer pass mid-branch on a single risky commit (say "run reviewer" before commit), and the agent runs it on the staged diff with the same scope as the merge-time gate. Default is fast (no reviewer); the on-demand path is a safety valve when something specific feels off. +**Reviewer at commit-time — large commits, or on demand.** The Reviewer's home is PR-merge (it sees drift best across N commits, per gate 5 below), but a *single* large commit already hides the same drift a whole branch does, so it earns a review before it lands. Trigger: **run the Reviewer on the staged diff when the commit is large** — a rough bar is **≥ ~10 changed files spanning multiple areas** (e.g. core + light + scripts + UI), or a diff big/multi-threaded enough that "these changes each look fine alone but do they cohere?" is a real question. Same scope as the merge-time gate (gate 5). Below that bar the pass is **on demand** — the product owner says "run reviewer" before commit when a specific change feels risky. Small, single-area commits skip it (the mechanical commit gates are the net there). Either way the Reviewer only *reports*; findings are fixed, or accepted with a one-line reason in the commit body's **Reviews** section (👾), before "commit now". As with every gate list, the product owner initiates — the agent finishing a large diff should *ask* "this is large, run the Reviewer first?", not launch it unprompted. #### Event 2: PR merge into `main` @@ -241,7 +241,7 @@ The project uses Claude Code agents in defined roles. The user is the **Product |--|-------|-------|-------|------| | 🤖 | **Architect** | Opus | System design | Reviews against architecture, designs components, validates boundaries | | 👽 | **Developer** | Sonnet | Implementation | Writes code in worktrees, follows all rules, one step at a time | -| 👾 | **Reviewer** | Fable when available, otherwise Opus | Pre-merge check | Runs at PR merge over the whole branch diff (Event 2, gate 5); available on-demand pre-commit on the staged diff when the product owner asks. Complements CodeRabbit (which handles line-level bugs in the PR). | +| 👾 | **Reviewer** | **Fable** (Opus only if Fable is unavailable) | Pre-merge check | Runs at PR merge over the whole branch diff (Event 2, gate 5), and pre-commit on the staged diff when the commit is large or the product owner asks (see *Reviewer at commit-time*). The model is fixed, not a per-run choice. Complements CodeRabbit (which handles line-level bugs in the PR). | | 🛸 | **Tester** | Sonnet | Verification | Writes tests, verifies architectural rules in code | | 💀 | **Runner** | Haiku | Quick checks | Runs MoonDeck scripts, platform boundary checks, build verification | @@ -249,6 +249,6 @@ Agents work in parallel on independent steps. Agents never commit; only the prod ## Build -How to build, flash, run, monitor, and check the project for every target: [docs/building.md](docs/building.md). Per-script reference: [scripts/MoonDeck.md](scripts/MoonDeck.md). +How to build, flash, run, monitor, and check the project for every target: [docs/building.md](docs/building.md). Per-script reference: [moondeck/MoonDeck.md](moondeck/MoonDeck.md). -Agents use the CLI (`uv run scripts/<group>/<name>.py`); humans typically use MoonDeck (`uv run scripts/moondeck.py`, port 8420). Same scripts under both. Every gate in [Lifecycle Events](#lifecycle-events) ultimately invokes one of these scripts. +Agents use the CLI (`uv run moondeck/<group>/<name>.py`); humans typically use MoonDeck (`uv run moondeck/moondeck.py`, port 8420). Same scripts under both. Every gate in [Lifecycle Events](#lifecycle-events) ultimately invokes one of these scripts. diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c598345..3c7a4b2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,7 +38,7 @@ else() add_compile_options(-Wall -Wextra -Werror) endif() -# `uv` is the project's Python launcher (see CLAUDE.md / scripts/MoonDeck.md). +# `uv` is the project's Python launcher (see CLAUDE.md / moondeck/MoonDeck.md). # The build invokes Python helpers for build_info.h generation and UI embedding; # resolving them through `uv run python …` keeps Windows (where `python3` isn't # on PATH) on the same path as macOS / Linux without a `find_package(Python3)` @@ -54,6 +54,8 @@ add_library(mm_core STATIC src/core/Control.cpp src/core/HttpServerModule.cpp src/core/FilesystemModule.cpp + src/core/FileManagerModule.cpp + src/core/MqttModule.cpp src/core/Scheduler.cpp src/core/moonlive/MoonLive.cpp src/core/moonlive/MoonLiveCompiler.cpp @@ -77,8 +79,8 @@ target_link_libraries(mm_platform PUBLIC $<$<PLATFORM_ID:Windows>:ws2_32>) # Generate build_info.h from library.json (carries version, build date, board name). add_custom_command( OUTPUT ${CMAKE_SOURCE_DIR}/src/core/build_info.h - COMMAND ${UV_EXECUTABLE} run python ${CMAKE_SOURCE_DIR}/scripts/build/generate_build_info.py - DEPENDS ${CMAKE_SOURCE_DIR}/library.json ${CMAKE_SOURCE_DIR}/scripts/build/generate_build_info.py + COMMAND ${UV_EXECUTABLE} run python ${CMAKE_SOURCE_DIR}/moondeck/build/generate_build_info.py + DEPENDS ${CMAKE_SOURCE_DIR}/library.json ${CMAKE_SOURCE_DIR}/moondeck/build/generate_build_info.py COMMENT "Generating build_info.h" ) add_custom_target(build_info_gen DEPENDS ${CMAKE_SOURCE_DIR}/src/core/build_info.h) diff --git a/README.md b/README.md index 15fca3fe..0f3e1900 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Drive large LED installations and DMX lighting from ESP32, Teensy, Raspberry Pi, 📦 **Release + downloads:** [latest release](https://github.com/MoonModules/projectMM/releases/latest) -🛠️ **Building / hacking on it?** [MoonDeck](scripts/MoonDeck.md), our browser-based dev console (build · flash · test · live device discovery), comes in the repo. +🛠️ **Building / hacking on it?** [MoonDeck](moondeck/MoonDeck.md), our browser-based dev console (build · flash · test · live device discovery), comes in the repo. Open Chrome or Edge, plug in your device, and you'll see lights in under a minute. @@ -32,6 +32,10 @@ If you like projectMM, give it a ⭐️, fork it, or open an issue or pull reque 🎵 **Audio-reactive**: an I²S microphone drives a 16-band FFT spectrum + sound level, consumed by audio-reactive effects — all built fresh from the mic datasheet and textbook DSP. +🏠 **Home-automation control**: a device joins Homebridge (and any MQTT hub) over a dependency-free MQTT 3.1.1 client — on/off, brightness, and a HomeKit colour wheel that picks the nearest palette. See [the MQTT module docs](docs/moonmodules/core/ui/ui.md#mqtt). + +📁 **On-device File Manager**: browse and edit the device filesystem from the browser — a lazy folder tree with an inline editor, drag-drop upload, and create/delete, plus [firmware upload OTA](docs/moonmodules/core/ui/ui.md#firmware-update) (flash a `.bin` over the LAN, no USB). See [the File Manager docs](docs/moonmodules/core/ui/ui.md#file-manager). + 🛡️ **Robust to any input**: add, delete, replace, or reconfigure any module in any order, at any grid size, and the device keeps running — degraded or idle, never crashed. Every crash that's ever found becomes a regression test, so it stays fixed. 🖥️ **One source tree, many targets**: the same code runs on ESP32, Teensy, Raspberry Pi, and macOS / Windows / Linux. @@ -108,10 +112,10 @@ You need [uv](https://docs.astral.sh/uv/) (Python launcher), CMake 3.20+, and a Once prerequisites are in place, launch MoonDeck, the browser-based dev console: ```sh -uv run scripts/moondeck.py +uv run moondeck/moondeck.py ``` -Open `http://localhost:8420`: PC tab to build / run / test, ESP32 tab to flash, Live tab to discover devices. Full per-command reference: [scripts/MoonDeck.md](scripts/MoonDeck.md). +Open `http://localhost:8420`: PC tab to build / run / test, ESP32 tab to flash, Live tab to discover devices. Full per-command reference: [moondeck/MoonDeck.md](moondeck/MoonDeck.md). ![Moondeck Pc](docs/assets/ui/moondeck_pc.png) diff --git a/docs/architecture.md b/docs/architecture.md index 7bd41030..b6f4649e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -109,7 +109,7 @@ ModuleFactory is core infrastructure ([`src/core/ModuleFactory.h`](../src/core/M **Self-reporting.** Every MoonModule reports its own footprint and cost: `classSize()` (the `sizeof` of the class instance, captured at registration), `dynamicBytes()` (heap allocated during `onBuildState`), and `loopTimeUs()` (average time its `loop` took, accumulated per tick). These surface in `/api/system`, console output, and scenario tests: the same numbers for an effect, a driver, or a system service, because they're a base-class feature, not a light-domain one. -Each MoonModule has two documentation surfaces under [`docs/moonmodules/`](moonmodules/): an end-user **summary page** — one 4-column table row in its group's page (effects/modifiers/layouts/drivers, or core/light UI/supporting) — and a **generated technical page** built from the header's `///` comments. See [coding-standards § Documentation model](coding-standards.md#documentation-model) for the full model. +Each MoonModule has two documentation surfaces under `docs/moonmodules/`: an end-user **summary page** — one 4-column table row in its group's page (effects/modifiers/layouts/drivers, or core/light UI/supporting) — and a **generated technical page** built from the header's `///` comments. See [coding-standards § Documentation model](coding-standards.md#documentation-model) for the full model. ## Controls @@ -218,7 +218,7 @@ Only abstract what you actually need. Currently: Abstractions are added when a concrete implementation needs them, not pre-designed. -**Platform boundary (hard rule).** All `#ifdef`, `#if defined`, platform-specific `#include`s, and hardware API calls live exclusively in `src/platform/`. Everything outside `src/platform/` compiles on every target without modification. Compile-time platform branching uses `if constexpr` on `platform_config.h` flags, never a preprocessor `#ifdef`. The boundary is enforced by [`scripts/check/check_platform_boundary.py`](../scripts/check/check_platform_boundary.py), a commit gate (see [CLAUDE.md § Lifecycle Events](../CLAUDE.md#lifecycle-events)). +**Platform boundary (hard rule).** All `#ifdef`, `#if defined`, platform-specific `#include`s, and hardware API calls live exclusively in `src/platform/`. Everything outside `src/platform/` compiles on every target without modification. Compile-time platform branching uses `if constexpr` on `platform_config.h` flags, never a preprocessor `#ifdef`. The boundary is enforced by [`moondeck/check/check_platform_boundary.py`](../moondeck/check/check_platform_boundary.py), a commit gate (see [CLAUDE.md § Lifecycle Events](../CLAUDE.md#lifecycle-events)). ## Firmware vs deviceModel vs board diff --git a/docs/assets/boards/shelly.jpg b/docs/assets/boards/shelly.jpg new file mode 100644 index 00000000..9cc52105 Binary files /dev/null and b/docs/assets/boards/shelly.jpg differ diff --git a/docs/assets/extra.css b/docs/assets/extra.css index b632901f..e4dbf88f 100644 --- a/docs/assets/extra.css +++ b/docs/assets/extra.css @@ -1,5 +1,5 @@ /* Catalog tables (effects / modifiers / layouts / drivers), rendered from prose - ### blocks by scripts/docs/mkdocs_hooks.py. Give the four columns sensible + ### blocks by moondeck/docs/mkdocs_hooks.py. Give the four columns sensible proportions and let the preview image fill its column instead of the tiny hand-authored width="300". diff --git a/docs/backlog/README.md b/docs/backlog/README.md index 97b8d199..65be5733 100644 --- a/docs/backlog/README.md +++ b/docs/backlog/README.md @@ -1,6 +1,6 @@ # Backlog — index -The forward-looking half of the docs (the backward-looking half is [`../history/`](../history/)). This folder is **not** present-tense and agents don't read it automatically — only when planning new work. See [CLAUDE.md § Documentation](../../CLAUDE.md) for how `backlog/` and `history/` relate. +The forward-looking half of the docs (the backward-looking half is [`../history/`](../history/README.md)). This folder is **not** present-tense and agents don't read it automatically — only when planning new work. See [CLAUDE.md § Documentation](../../CLAUDE.md) for how `backlog/` and `history/` relate. This README is the **landing page**: the rest of the system links here, not into individual items, so the present-tense docs stay present-tense. @@ -42,7 +42,7 @@ A map of everything in the three files, by theme. ## In-flight draft specs -A spec for a not-yet-built module can live here as a plain draft `.md` (alongside the design studies below) until the module ships — at which point its final spec is written in [`../moonmodules/`](../moonmodules/) and the draft is deleted. None are in flight right now. +A spec for a not-yet-built module can live here as a plain draft `.md` (alongside the design studies below) until the module ships — at which point its final spec is written under `../moonmodules/` (e.g. [drivers](../moonmodules/light/drivers/drivers.md)) and the draft is deleted. None are in flight right now. ## Design studies diff --git a/docs/backlog/backlog-core.md b/docs/backlog/backlog-core.md index b8b5d696..475da5ef 100644 --- a/docs/backlog/backlog-core.md +++ b/docs/backlog/backlog-core.md @@ -34,6 +34,26 @@ Full design + the reasoned transport split: [Plan-20260629 — UDP device discov ## ESP32 performance and memory +### Flash budget — the 4 MB classic ESP32 is the ceiling (investigation) + +The binary has grown ~1.4 → ~1.48 MB as effects, audio sync, IR, and Ethernet landed. Per-board headroom against the app (OTA) partition slot: + +| Board | app slot | binary | used | +|---|---|---|---| +| **classic esp32 (4 MB)** | 1.75 MB | ~1.48 MB | **~84 %** ⚠️ | +| esp32s3-n8r8 (8 MB) | 3.00 MB | ~1.48 MB | ~49 % | +| 16 MB boards | 4.00 MB | ~1.48 MB | ~37 % | + +Only the **4 MB classic** is tight (the partition comment itself notes "~200 KB headroom"); the 8/16 MB boards have years of room. So this is a *classic-ESP32-only* constraint, not a global one — the fix should shrink the small-flash build without touching the flagship boards. + +Levers, roughly by payoff-per-effort: + +1. **`-Os` for the size-bound builds.** The ESP32 build currently runs the IDF default optimization (`CONFIG_COMPILER_OPTIMIZATION_DEBUG`, ~`-Og`), **not** `CONFIG_COMPILER_OPTIMIZATION_SIZE` (`-Os`). Setting size-opt on the classic (and any small-flash) firmware typically buys 5–15 % flash for free — measure the tick/FPS delta, since `-Os` can cost a little hot-path speed; if it does, gate it to the flash-bound firmwares only, not the S3/P4 where headroom is fine. +2. **`MM_MINIMAL` feature profile for the 4 MB target.** The classic board doesn't need every effect/driver compiled in. A build profile that compiles out heavy optional modules (the same `firmware_cmake_args()` seam the eth/wifi gating already uses) keeps the flagship boards full-featured while the small board ships a curated subset — the standard "small board, smaller build" pattern. +3. **Repartition the 4 MB classic.** If A/B OTA isn't required on the classic, a single-app-slot layout nearly doubles the app ceiling (1.75 → ~3.5 MB). Trade-off: OTA loses its rollback slot. Decide per-board, not globally. + +Start with (1) — it's a one-line sdkconfig change with a measurable payoff and no code churn; (2)/(3) only if (1) plus normal growth still crowds the classic. The UI embed is already gzipped (`app.js` ~140 KB → ~40 KB), so UI growth is cheap in flash; the pressure is C++ `.text`. + ### E1.31 multicast receive (IGMP join) NetworkReceiveEffect accepts E1.31 via unicast only — the same scope MoonLight ships. Multicast senders address the per-universe group `239.255.{universe_hi}.{universe_lo}`, which a receiver must join via IGMP; the platform `UdpSocket` has no `IP_ADD_MEMBERSHIP` support yet (lwIP `setsockopt` on ESP32, plain `setsockopt` on desktop, plus a join-per-accepted-universe bookkeeping question). Add when a multicast-only sender actually shows up on a bench; until then the spec documents "point sACN senders at the device's IP". @@ -49,7 +69,7 @@ This determines the practical LED limit for WiFi-only boards. Until the `sdkconf ### Network round-trip test — drop/reorder measurement (deferred) -`scripts/scenario/run_network_roundtrip.py` measures PC→device→PC **latency and jitter** per protocol (ArtNet/E1.31/DDP) by timing how long a sent colour takes to appear in the device's preview stream. It deliberately does **not** measure per-frame **drops or reorder**, because the path can't track individual frames cleanly: `NetworkSendDriver` re-clocks at its own fps (decoupled from receive) and `NetworkReceiveEffect` holds-last-frame, so frames don't pass through 1:1 — a sequence number embedded in frame N may be re-sent 0, 1, or several times downstream. The min/median/max spread the test already reports *is* the jitter signal (it surfaced multi-second outliers on the classic ESP32). To measure true drop/reorder, the firmware would need a sequence-faithful echo path (e.g. a `NetworkReceiveEffect` echo mode that re-emits each received frame 1:1 back to the sender, bypassing the fps re-clock), then the PC could match sent↔received sequence numbers. The test's docstring lists this under "extend later" alongside per-frame sequence matching and the device→device chain. +`moondeck/scenario/run_network_roundtrip.py` measures PC→device→PC **latency and jitter** per protocol (ArtNet/E1.31/DDP) by timing how long a sent colour takes to appear in the device's preview stream. It deliberately does **not** measure per-frame **drops or reorder**, because the path can't track individual frames cleanly: `NetworkSendDriver` re-clocks at its own fps (decoupled from receive) and `NetworkReceiveEffect` holds-last-frame, so frames don't pass through 1:1 — a sequence number embedded in frame N may be re-sent 0, 1, or several times downstream. The min/median/max spread the test already reports *is* the jitter signal (it surfaced multi-second outliers on the classic ESP32). To measure true drop/reorder, the firmware would need a sequence-faithful echo path (e.g. a `NetworkReceiveEffect` echo mode that re-emits each received frame 1:1 back to the sender, bypassing the fps re-clock), then the PC could match sent↔received sequence numbers. The test's docstring lists this under "extend later" alongside per-frame sequence matching and the device→device chain. ### Async ArtNet send — decouple the wire from the render tick (PSRAM-only) @@ -74,7 +94,7 @@ On Olimex ESP32-Gateway flashed with `esp32-eth`, Ethernet sometimes takes **a m What we know: - `build/esp32-esp32-eth/sdkconfig` and `build/esp32-esp32-eth-wifi/sdkconfig` are **byte-identical** (3,617 lines each, `cmp -s` confirms). So lwIP buffer pools, DHCP timeouts, and Ethernet driver settings are the same. - Same hardware (Olimex ESP32-Gateway Rev G), same RMII pin/clock config (`EMAC_CLK_OUT` on GPIO17), same `ethInit()` code in `src/platform/esp32/platform_esp32.cpp`. -- The only difference at link time: `esp32-eth` passes `EXCLUDE_COMPONENTS=esp_wifi;wpa_supplicant;esp_coex` to ESP-IDF (see `scripts/build/build_esp32.py:31`). +- The only difference at link time: `esp32-eth` passes `EXCLUDE_COMPONENTS=esp_wifi;wpa_supplicant;esp_coex` to ESP-IDF (see `moondeck/build/build_esp32.py:31`). - `esp_coex` (WiFi/Bluetooth coexistence) was an early hypothesis: even though Ethernet doesn't share the radio, `esp_coex`'s init might warm a shared clock path that helps Ethernet auto-negotiation, and the eth-only build excludes it. **Disproven — see below.** **Firmware is ruled out (the evidence is contradictory across reboots with the *same* build, which by itself proves build-independence).** Over one session, on the same Olimex + cable: @@ -96,10 +116,10 @@ Bottom line: intermittent, build-independent, reset-correlated → a hardware/PH ### MoonDeck doc-asset endpoint hardening (backlog) -`scripts/moondeck.py::_serve_doc_asset` accepts any ROOT-relative path and serves the file. Path traversal *is* blocked (`asset_path.relative_to(ROOT.resolve())`), but inside the repo any file is served — including local-only artefacts like `scripts/build/wifi_credentials.json` if present. MoonDeck binds to all interfaces by design (the existing comment in `main()` explicitly enables LAN reach), so anyone on the LAN can hit the endpoint. +`moondeck/moondeck.py::_serve_doc_asset` accepts any ROOT-relative path and serves the file. Path traversal *is* blocked (`asset_path.relative_to(ROOT.resolve())`), but inside the repo any file is served — including local-only artefacts like `moondeck/build/wifi_credentials.json` if present. MoonDeck binds to all interfaces by design (the existing comment in `main()` explicitly enables LAN reach), so anyone on the LAN can hit the endpoint. Two improvements when this matters: -- **Subdirectory whitelist** — only serve under `docs/` (and image asset paths the markdown renderer needs). Reject `scripts/build/wifi_credentials.json` etc. with 403. +- **Subdirectory whitelist** — only serve under `docs/` (and image asset paths the markdown renderer needs). Reject `moondeck/build/wifi_credentials.json` etc. with 403. - **Extension whitelist** — only image / CSS / JS mime types via a small allowlist. - **Optional bind-to-localhost flag** — `--bind 127.0.0.1` for users who don't want LAN reachability. Default stays "" (all interfaces) since the LAN-reach is the documented design. @@ -254,7 +274,7 @@ What to build (~4 h): - **Platform layer** — implement `ota_begin`/`ota_write`/`ota_end` in [platform_esp32.cpp](../src/platform/esp32/platform_esp32_ota.cpp) wrapping `esp_ota_*` directly (already declared in [platform.h](../src/platform/platform.h), currently bypassed by `esp_https_ota_perform`). - **HTTP route** — `POST /api/firmware` in [HttpServerModule.cpp](../src/core/HttpServerModule.cpp); streams `conn.read(chunkBuf, 4096)` → `ota_write`; shares `g_otaStatus`/`g_otaPct` with the URL path. - **`device_ip` control on SystemModule** — lets a script discover the device without mDNS. -- **`scripts/build/flash_over_network.py`** — finds the latest `.bin` in `build/esp32-<board>/`, POSTs it, polls for `rebooting`. +- **`moondeck/build/flash_over_network.py`** — finds the latest `.bin` in `build/esp32-<board>/`, POSTs it, polls for `rebooting`. - **UI file-upload affordance** — `<input type="file" accept=".bin">` in a `<details>` expander on the Firmware card. ### HTTP file serving blocks the render tick (backlog) @@ -269,7 +289,7 @@ What to build (~4 h): - **UI page load time** — scenario step measuring HTTP response time for `/`, `/api/state`, `/api/system` via the live runner. Verifies acceptable load time on ESP32. - **Module teardown memory** — scenario that tears down all modules and verifies heap returns to pre-setup baseline. Confirms no lifecycle leaks. - **JavaScript test harness** — `vitest` + `jsdom` for the browser UI: pure helpers in `install-picker.js` (`isCompatible`, `parseFirmwaresFromAssets`, `relativeTime`) **and `app.js`'s conditional-control DOM logic** (`syncVisibleControls` — reconciles which control rows are rendered when a `hidden` flag flips). The C++/backend half of conditional controls IS unit-tested (`conditional_controls.h` + per-module tests pin the binding + `hidden` flag), but the **UI re-render half is not** — `syncVisibleControls` was the source of a real re-render-loop freeze (Network static-IP toggle) caught only on hardware. A `jsdom` test that builds a card, flips a control's `hidden`, runs the reconcile, and asserts the right rows appear/disappear (and that it converges — the unchanged→no-op fast path) would have caught it. **Attempted and reverted (2026-06-17):** stood up vitest + 13 passing tests for the install-picker pure helpers, but the high-value half (`syncVisibleControls`) needs either an `app.js` module seam or extracting its reconcile logic into a separate served `.js` (6 embed/route wiring edits for a firmware-served file). Judged not worth adding a whole Node/npm toolchain to a C++/Python repo to test ~3 small pure functions; the toolchain earns its place only once the `syncVisibleControls` DOM test (and a real body of JS logic) lands with it. **Do it as its own focused branch**, deciding the app.js seam first (it's already `type="module"`, so extracting `reconcileControlRows` into a served file — wired through `embed_ui.cmake` + the two HttpServerModule routes like the other UI .js — is the clean shape). Pure-helper `_test` exports + the reconcile extraction are the two pieces; both were prototyped in that reverted attempt. -- **Browser-level Improv automation** (deferred) — `scripts/build/improv_smoke_test.py` (added 2026-06-03) exercises the device-side Improv listener over plain serial; what's missing is the browser-side equivalent — Playwright driving Chrome's Web Serial, clicking through ESP Web Tools' install modal, filling the WiFi creds form, asserting `PROVISIONED`. Catches "ESP Web Tools changed its Improv handling in a way that broke our manifest format" failures the serial-only smoke test can't see. Hard to set up reliably (headless Chrome with Web Serial is finicky, needs a wired ESP32 in CI). Pick this up if a regression in the browser flow ever escapes the manual dev-environment test (preview_installer flash-ready mode at <http://localhost:8000/>). +- **Browser-level Improv automation** (deferred) — `moondeck/build/improv_smoke_test.py` (added 2026-06-03) exercises the device-side Improv listener over plain serial; what's missing is the browser-side equivalent — Playwright driving Chrome's Web Serial, clicking through ESP Web Tools' install modal, filling the WiFi creds form, asserting `PROVISIONED`. Catches "ESP Web Tools changed its Improv handling in a way that broke our manifest format" failures the serial-only smoke test can't see. Hard to set up reliably (headless Chrome with Web Serial is finicky, needs a wired ESP32 in CI). Pick this up if a regression in the browser flow ever escapes the manual dev-environment test (preview_installer flash-ready mode at <http://localhost:8000/>). ### Live full-suite run leaks state between scenarios (test infra) @@ -293,7 +313,7 @@ The build IDF is `v6.1-dev-399-gd1b91b79b5`, a dev-branch snapshot (2025-11-05) ### Three-level device model: MCU → Board → Device (config provenance) -The model itself is now a shipped design — see architecture.md § Config provenance (the three levels + the `txPowerSetting` example + "default only at the level that fixes it"). The catalog that carries it is `install/deviceModels.json` (schema). **MoonDeck device-profile save/restore is shipped** — capture a device's pin/peripheral config (`/api/save-profile`) and re-apply it after a reflash or to a clone (`/api/apply-profile`), stored per-device in `moondeck.json`. The remaining forward-looking pieces — a `devices.json`/MCU-layer split and annotated-pin images — stay gated by the sequencing rule (no catalog field ahead of a consumer). +The model itself is now a shipped design — see architecture.md § Config provenance (the three levels + the `txPowerSetting` example + "default only at the level that fixes it"). The catalog that carries it is `web-installer/deviceModels.json` (schema). The remaining forward-looking pieces — a `devices.json`/MCU-layer split and annotated-pin images — stay gated by the sequencing rule (no catalog field ahead of a consumer). ### Persistence overlay: partial-save / schema-change audit (backlog) @@ -346,6 +366,16 @@ Forward-looking companion to the shipped UI spec, [moonmodules/core/ui/ui.md](.. - Core affinity badge (C0/C1) — only meaningful when core pinning lands - Module `category()` field — taxonomy beyond `role()` for the picker (decision: derive from `role()` for now) +### File Manager follow-ups + +The shipped File Manager (see [ui.md](../moonmodules/core/ui/ui.md)) is a lazy expand/collapse tree over `/api/dir` + a size-capped text editor over `/api/file`, with drag-drop upload (tier 1) + per-file download + a filesystem-usage bar. Deferred capabilities, each self-contained: + +- **Upload — recursive folder tier.** Single-file upload of **any size** streams to the file (`fsWriteStream`, binary-safe, `kUploadMax` + free-space guarded) and downloads stream back (`fsReadAt`), so text/config/binary all work. Remaining: **multi-file / recursive folder drops** — client-side `mkdir` + walk via `dataTransfer.items` `webkitGetAsEntry()`. +- **Download — folder as `.zip`.** Per-file download streams (`<a download>` on `/api/file`). A folder download needs the browser to walk `/api/dir` recursively, fetch each file, and build a `.zip` client-side — which means bundling a zip library into `app.js`. That's a permanent app.js size bump, and app.js is embedded in firmware, so it weighs on the flash budget (see the flash-budget item above). Gate on real demand; symmetric with the folder-upload tier. +- **Last-modified dates.** Needs a time source (NTP) + LittleFS mtime storage; the tree is name + size until both land. Backlogged with the NTP work. +- **`.ml` syntax highlighting in the editor.** MoonLive source wants *highlighting* (a colour layer over the textarea), not the JSON-style reformat — a bigger editor change (a highlight overlay or a small tokenizer), added when MoonLive `.ml` files land on disk. The editor already has an extension seam (`fmPrettify`) for the reformat case; highlighting is the separate, larger tier. +- **Keyboard + screen-reader access for the tree.** The `fm-row` tree entries are mouse-only today (click to select/expand/open). Making them keyboard-operable (focusable, Enter/Space activation) with ARIA tree semantics (`role="treeitem"`, `aria-expanded` from `isOpen`, `aria-level`) is a real accessibility win but adds JS that lands in the firmware-embedded `app.js` (flash cost). Do it when a11y is a stated goal, together across the whole generic UI, not just this panel. + ### Open design questions These don't block the shipped baseline but should be answered before 1.0: diff --git a/docs/backlog/docs-system-overhaul.md b/docs/backlog/docs-system-overhaul.md index 9de1c713..32a824a8 100644 --- a/docs/backlog/docs-system-overhaul.md +++ b/docs/backlog/docs-system-overhaul.md @@ -8,7 +8,7 @@ Follows the *Refactor for simplicity* process rule: alternatives enumerated, gai - **~259K words / 19.5K lines** of `.md`. Biggest buckets: `docs/history/` (97K words, incl. 47 plan files), `docs/backlog/` (57K), `docs/moonmodules/` (38K), `docs/tests/` (25K, **auto-generated**), top-level `docs/` (29K). - **GitHub Pages today publishes only the web installer** (`docs/install/`), *not the docs*. So the docs are read as raw `.md` on github.com — no nav, no search, no landing page. This is the root of "impossible for end users to read": **there is no doc *site* at all yet.** -- **One generation loop already works and proves the model:** `scripts/docs/generate_test_docs.py` reads test metadata (`// @module` tags in `test/unit/*.cpp`, JSON fields in `test/scenarios/*.json`) and emits `docs/tests/{unit,scenario}-tests.md`. The same parser (`_test_metadata.py`) feeds MoonDeck's `/api/tests` UI. Source of truth = the test file. This is the pattern to extend, not replace. +- **One generation loop already works and proves the model:** `moondeck/docs/generate_test_docs.py` reads test metadata (`// @module` tags in `test/unit/*.cpp`, JSON fields in `test/scenarios/*.json`) and emits `docs/tests/{unit,scenario}-tests.md`. The same parser (`_test_metadata.py`) feeds MoonDeck's `/api/tests` UI. Source of truth = the test file. This is the pattern to extend, not replace. - **Duplication hotspots** — each fact lives in N places, changing one forces the others: | Fact | Lives in | Sync today | @@ -54,7 +54,7 @@ Rejected: **classic Doxygen HTML** (separate site, dated UI, not integrated), ** - Move the *most* end-user-hostile prose (deep architecture) below a "Developer" fold so a user's top-down path never hits it unless they drill. - **Outcome:** the "top-to-down, easy navigate" structure, still additive. -### Phase 2 — Fold generated tests into the site + surface them to users — *DONE (scripts/docs/mkdocs_hooks.py)* +### Phase 2 — Fold generated tests into the site + surface them to users — *DONE (moondeck/docs/mkdocs_hooks.py)* *As shipped: the two inventory pages are generated at build time (never committed). Each effect/modifier keeps a compact card (heading, GIF, params, source link) with a one-line `[Tests]` link into its inventory section — an early attempt to expand that link into the full inline case list bloated the cards and was reverted; tests are a link, not a dump.* - Run the existing test-doc generation *at site-build time* via `mkdocs-gen-files` (stop committing `docs/tests/*.md` — the 25K generated words leave the repo, generated fresh each build). Kills the "forgot to regenerate" drift class entirely. @@ -100,5 +100,5 @@ Prove each on **one module**, then sweep. Order by leverage: ## Source -- Investigation basis: `scripts/docs/generate_test_docs.py`, `scripts/check/check_specs.py`, `.github/workflows/release.yml` (Pages deploy), the `docs/` tree. +- Investigation basis: `moondeck/docs/generate_test_docs.py`, `moondeck/check/check_specs.py`, `.github/workflows/release.yml` (Pages deploy), the `docs/` tree. - Prior art / tooling: [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/), [pymdownx snippets](https://facelessuser.github.io/pymdown-extensions/extensions/snippets/), [Doxide](https://github.com/lawmurray/doxide). diff --git a/docs/backlog/folder-structure-proposal.md b/docs/backlog/folder-structure-proposal.md index fb7cb191..142dff84 100644 --- a/docs/backlog/folder-structure-proposal.md +++ b/docs/backlog/folder-structure-proposal.md @@ -88,7 +88,7 @@ test/scenarios/light/{effects,layouts,drivers,…}/ (mirror) | `docs/moonmodules/` | per-library pages (`effects_<library>.md`) with compact rows + the `check_specs.py` rewrite; delete the ~21 per-module effect `.md`s. | Medium | **Migration Stage 2** (already planned) | | `src/` | **unchanged** (already `domain/type`, flat). | none | — | -**Not reshaped** (correctly orthogonal): `test/js`, `test/python` (host-side; test scripts/installer, not modules), `src/platform/{desktop,esp32}` (platform split, already consistent), `src/{core,light}/moonlive` (feature sub-tree). +**Not reshaped** (correctly orthogonal): `test/js`, `test/python` (host-side; test moondeck/installer, not modules), `src/platform/{desktop,esp32}` (platform split, already consistent), `src/{core,light}/moonlive` (feature sub-tree). ## Sequencing diff --git a/docs/backlog/leddriver-analysis-top-down.md b/docs/backlog/leddriver-analysis-top-down.md index 31876841..f2527398 100644 --- a/docs/backlog/leddriver-analysis-top-down.md +++ b/docs/backlog/leddriver-analysis-top-down.md @@ -392,7 +392,7 @@ Each platform reaches hello-world when: 1. A 100-pixel solid-color test frame renders on a real strip. 2. The on-board loopback test (5.2) passes on real hardware. 3. The fx2lafw cross-check (5.3) passes — same bytes, same reset gap. -4. The fps/jitter KPI line (per [collect_kpi.py](../../scripts/collect/collect_kpi.py)) is captured and stored. +4. The fps/jitter KPI line (per [collect_kpi.py](../../moondeck/check/collect_kpi.py)) is captured and stored. 5. (ESP32 only) the test still passes with WiFi associated and a packet flood running. ## 7. Product-owner decisions diff --git a/docs/backlog/rename-scripts-to-moondeck.md b/docs/backlog/rename-scripts-to-moondeck.md deleted file mode 100644 index 03c676ce..00000000 --- a/docs/backlog/rename-scripts-to-moondeck.md +++ /dev/null @@ -1,49 +0,0 @@ -# Rename top-level `scripts/` → `moondeck/` - -Forward-looking (backlog is exempt from present-tense). Decided by the PO on 2026-07-02, to be done as its **own separate commit** in the next cycle — deliberately NOT folded into the docs-site / `web-installer/` commit (that one is scoped to the `docs/` separation; this is a larger orthogonal sweep and stays isolated for a clean, revertible diff). - -## Why - -Consistency with the `web-installer/` move: top-level folders should name what they hold. MoonDeck is the human-facing dev console (`uv run moondeck.py`); the folder is its home. - -> Note (agent pushback recorded, PO overrode): `scripts/` is the more *recognizable* convention and it holds more than MoonDeck (`build/`, `check/`, `docs/`, `run/` groups the CLI gates invoke directly — see CLAUDE.md § Build). The rename narrows an accurate name. PO's call stands; captured here so the trade-off is on record, not to relitigate. - -## Scope (measured) - -~77 files / ~376 occurrences of the string `scripts/`. Load-bearing categories: - -- **CLAUDE.md** — every gate command (`uv run scripts/check/…`, `scripts/build/…`, `scripts/docs/…`), the Build section, the `scripts/**` commit-gate triggers. -- **Both CI workflows** — `.github/workflows/{release,test}.yml`: `run:` steps, `paths:` filters (`scripts/**`, `scripts/build/**`). -- **CMake** — `find_program(UV…)` + `execute_process` / `add_custom_command` that shell into `scripts/…`, and `src/ui/embed_ui.cmake`. -- **`moondeck_config.json`** — its own `"script": "<group>/<name>.py"` entries (relative to the scripts dir; check whether the loader prepends `scripts/` or the config is already relative). -- **Cross-references** in `scripts/MoonDeck.md`, `docs/building.md`, module specs, and every script's own docstring/help text. -- **The deferred/allow lists** in `.claude/settings.local*.json` (gitignored live file re-accumulates; the tracked `.cleaned.json` needs updating). - -## The gotcha that WILL bite (learned from the `web-installer/` sweep) - -A naive `s{scripts/}{moondeck/}` regex **misses split-component path construction**. The `web-installer` sweep left three JS test files broken because they built the path as `join(ROOT, "docs", "install", …)` — separate string args, not the literal `docs/install/`. The equivalent here: - -- **Python:** `Path(...) / "scripts" / ...`, `os.path.join(ROOT, "scripts", ...)`. -- **JS:** `join(ROOT, "scripts", ...)`. -- **CMake:** `${CMAKE_SOURCE_DIR}/scripts` and `"${CMAKE_SOURCE_DIR}" "scripts"` forms. - -Grep for **all** of these split forms (`"scripts"`, `'scripts'`, `/ "scripts"`, `, "scripts"`) BEFORE running the literal-path replace, or things break silently and only a test run catches them. - -Also protect against false positives: any identifier or word containing `scripts` that is NOT the folder (e.g. `livescripts`, `ESPLiveScript`, `description`, `subscripts`) — anchor the replace to `scripts/` with trailing slash and to the split-string forms, never a bare substring. - -## Verification (before the commit lands) - -- All gates green: `check_specs`, `check_devices`, `check_firmwares`, `ctest`, scenarios, `uv run --with pytest … test/python`, `node --test test/js/**`. -- Desktop + at least one ESP32 build (CMake reaches the renamed scripts via `find_program`/`execute_process`). -- MoonDeck starts and loads its config (`moondeck_config.json` script paths resolve). -- Docs site builds (`uv run <newpath>/docs/build_docs.py`). -- The gitignored `.claude/settings.local.json` allow-list entries pointing at `scripts/…` still match (they re-accumulate; the tracked `.cleaned.json` is the baseline to update). - -## Decision - -- **Separate commit, next cycle.** Not in the docs-site commit. -- Delete this backlog note once the rename ships (per *Mandatory subtraction* — the git commit is the record). - -## Source - -- Basis: the reference sweep done for the `web-installer/` move (this session); `git grep scripts/` for the live count. diff --git a/docs/backlog/rename-to-moonlight.md b/docs/backlog/rename-to-moonlight.md index 2ac14878..3d9ac392 100644 --- a/docs/backlog/rename-to-moonlight.md +++ b/docs/backlog/rename-to-moonlight.md @@ -11,7 +11,7 @@ The hard part is the **name collision**: while the move is in progress, `MoonMod ## Blast radius (measured) -- **`projectMM` → `MoonLight`:** ~579 references across ~135 files. Concentrated in `docs/` (history/plans, moonmodules specs, install), then `src/` (core, light/effects, ui), `scripts/build`, and `test/`. +- **`projectMM` → `MoonLight`:** ~579 references across ~135 files. Concentrated in `docs/` (history/plans, moonmodules specs, install), then `src/` (core, light/effects, ui), `moondeck/build`, and `test/`. - **Predecessor links to repoint** (`github.com/MoonModules/MoonLight` → `github.com/ewowi/MoonLight`): ~23 URLs, all in `docs/` prior-art / history sections. - **`MoonLight` as prior-art prose** (NOT the new product name): ~25 mentions in `docs/history` + light specs — these must NOT be swept into the rename; they describe the predecessor. - **`MoonLive`** (the on-device scripting engine, 29 files): a *different* name, NOT part of this rename. Any sweep must exclude it. @@ -20,7 +20,7 @@ The hard part is the **name collision**: while the move is in progress, `MoonMod These are the ones that break running devices, OTA, or the installer if mistimed — they change **at the switch**, not before: -- **Binary name** `projectMM.bin` (`library.json`, `scripts/build/flash_esp32.py`, `generate_manifest.py`, `package_desktop.py`) — renaming changes OTA asset names and the web-installer manifest; old + new must line up with the release that ships under the new repo. +- **Binary name** `projectMM.bin` (`library.json`, `moondeck/build/flash_esp32.py`, `generate_manifest.py`, `package_desktop.py`) — renaming changes OTA asset names and the web-installer manifest; old + new must line up with the release that ships under the new repo. - **OTA download URLs** `github.com/MoonModules/projectMM/releases/...` (`docs/install/install.js`, `FirmwareUpdateModule`) — deployed devices fetch updates from here; the URL only resolves to the new repo after the repo rename. - **mDNS / device identity** the `MM-XXXX` hostname prefix (`SystemModule.h`) — devices on the LAN announce as `MM-….local`. Changing the prefix (e.g. to `ML-`) renames every device's network identity; deliberately deferred (own decision: keep `MM-` or move to `ML-`). - **Repo URL** in docs/READMEs/`package_desktop.py` source links. @@ -62,11 +62,11 @@ Decoupling and groundwork that's safe while both repos still hold their current - **npm/pip/package identifiers:** none exist (no `package.json`, no `pyproject.toml`/`setup.py`) — nothing to rename. 3. **Centralise the name** — ✅ **investigated, decided: skip *for the rename*; the sweep (step 4) covers it. The constant belongs to the library direction, not here (see note).** Centralising *to shrink the switch* doesn't pay: the centralisable runtime-identity is tiny (3 C++ network wire-strings — ArtNet/E1.31 source-name + CID — plus the boot printf), those are fixed-length protocol fields whose `memcpy(…, 9)`-style copies need rewriting *regardless*, and a constant added for a one-time event sits unused afterward — the single-purpose abstraction *Default to subtraction* / *Core grows slower than the domain* say not to add. The mechanical sweep (step 4) flips every literal — wire strings, the golden-vector tests that assert them, CMake target, UI HTML, repo URL — in **one auditable commit** where the assertion and the wire bytes move in lockstep (better than a constant, which would split them). So no constant; the sweep is the centralisation. *(Step-2 decisions stand: no value changes until the switch.)* - > **Could we reuse `library.json`'s `name` now where a literal sits (subtraction, not a new constant)?** Surveyed `scripts/` for it — verdict: **no genuine low-hanging fruit.** ~95% of `projectMM` literals there are the **binary name** (`build/…/projectMM`, `.bin`, `.exe`, `.log`, `pkill projectMM`, crash `.ips`) which must track the **CMake target**, not `library.json` (wiring them to the product name would break the path to the file on disk); plus one **wire literal** (`_net_probe.py` ArtNet source-name, must byte-match the device) and ~15 prose/docstrings. The only product-name candidates — `generate_manifest.py`'s manifest `name`/`home_assistant_domain` — must *stay* `projectMM` today (Step 2), don't currently read `library.json`, and flip alongside `library.json` in the sweep anyway, so wiring them is new plumbing for zero present benefit. The principle (reuse an existing source of truth over a hardcoded literal) is right; it just has no payoff here because the literals are either binary-coupled or static-until-the-switch. (The real home for product-identity reuse is still the library API — see the box above.) - + > **Could we reuse `library.json`'s `name` now where a literal sits (subtraction, not a new constant)?** Surveyed `moondeck/` for it — verdict: **no genuine low-hanging fruit.** ~95% of `projectMM` literals there are the **binary name** (`build/…/projectMM`, `.bin`, `.exe`, `.log`, `pkill projectMM`, crash `.ips`) which must track the **CMake target**, not `library.json` (wiring them to the product name would break the path to the file on disk); plus one **wire literal** (`_net_probe.py` ArtNet source-name, must byte-match the device) and ~15 prose/docstrings. The only product-name candidates — `generate_manifest.py`'s manifest `name`/`home_assistant_domain` — must *stay* `projectMM` today (Step 2), don't currently read `library.json`, and flip alongside `library.json` in the sweep anyway, so wiring them is new plumbing for zero present benefit. The principle (reuse an existing source of truth over a hardcoded literal) is right; it just has no payoff here because the literals are either binary-coupled or static-until-the-switch. (The real home for product-identity reuse is still the library API — see the box above.) + > > **The constant has a real future home: projectMM/MoonLight as a library.** When the project is offered as an embeddable library, a consumer will want one runtime identity to read (an "About"/banner string, the protocol source-name they can query) — *that* is the ongoing, widely-referenced use a `kProjectName` constant genuinely earns (the test the rename failed). But build it **then**, against a real library API surface (it may want to be a small `ProjectInfo` — name + version + url — not a bare string), per *Concrete first, abstract later* — not speculatively now. Tracked as a seed in [backlog-core](backlog-core.md); when the library work starts, introduce the identity constant as part of its public API and let the wire-strings + UI derive from it. -4. **Author the mechanical sweep script** — ✅ **Done:** [`scripts/rename/rename_to_moonlight.py`](../../scripts/rename/rename_to_moonlight.py), dry-run by default (`--apply` writes; reserved for switch-day Phase 3.3, *after* the repo rename). What the dry-run against today's tree established: replaces two tokens (`ProjectMM` the enum, then `projectMM`) — a plain token swap is correct for *every* form (repo URL, host path, `projectMM.bin`, product name, `deviceName` slug) since `projectMM` is never a substring of another token; file list comes from `git ls-files` so build output (`build/`, `esp32/build/`) is excluded without a brittle blocklist; `docs/history` (era record) + the rename doc itself are content-excluded. Verified: **542 hits across 113 files**, and `MoonLive` / predecessor `MoonLight` / `namespace mm` are provably never touched (0 files where their count changes). The enum rename is safe — device classification keys on the `"modules"` marker, not the label string. The script de-risks switch-day; it is NOT run with `--apply` until then. -5. **Prep MoonDeck / `moondeck.json` / bench registry** — ✅ **investigated; nothing to change now, two things flagged for switch-day.** (a) **The functional chain stays `projectMM` until the switch (and flips together in the sweep):** `moondeck_config.json`'s `process_name: "projectMM"` ↔ the CMake binary `projectMM` ↔ the `build/<host>/projectMM` run/log path ↔ `pkill projectMM`. These are tracked files the sweep rewrites in one pass, so they stay consistent — changing `process_name` early would break MoonDeck's process detection against today's binary, so don't. (b) **The sweep cannot reach the gitignored bench registry** `scripts/moondeck.json` (it's private, per [[bench-setup]]; the sweep uses `git ls-files`). Its `"board": "projectMM testbench …"` values reference catalog `name`s that *do* flip — so after the switch they'd mismatch only on your bench. **Switch-day local-tooling note: hand-update `scripts/moondeck.json` board names** (and re-provision bench devices if you want the new mDNS identity) — the sweep covers tracked files only. The MoonDeck prose (`MoonDeck.md`, code comments) flips in the normal sweep. +4. **Author the mechanical sweep script** — ✅ **Done:** [`moondeck/rename/rename_to_moonlight.py`](../../moondeck/rename/rename_to_moonlight.py), dry-run by default (`--apply` writes; reserved for switch-day Phase 3.3, *after* the repo rename). What the dry-run against today's tree established: replaces two tokens (`ProjectMM` the enum, then `projectMM`) — a plain token swap is correct for *every* form (repo URL, host path, `projectMM.bin`, product name, `deviceName` slug) since `projectMM` is never a substring of another token; file list comes from `git ls-files` so build output (`build/`, `esp32/build/`) is excluded without a brittle blocklist; `docs/history` (era record) + the rename doc itself are content-excluded. Verified: **542 hits across 113 files**, and `MoonLive` / predecessor `MoonLight` / `namespace mm` are provably never touched (0 files where their count changes). The enum rename is safe — device classification keys on the `"modules"` marker, not the label string. The script de-risks switch-day; it is NOT run with `--apply` until then. +5. **Prep MoonDeck / `moondeck.json` / bench registry** — ✅ **investigated; nothing to change now, two things flagged for switch-day.** (a) **The functional chain stays `projectMM` until the switch (and flips together in the sweep):** `moondeck_config.json`'s `process_name: "projectMM"` ↔ the CMake binary `projectMM` ↔ the `build/<host>/projectMM` run/log path ↔ `pkill projectMM`. These are tracked files the sweep rewrites in one pass, so they stay consistent — changing `process_name` early would break MoonDeck's process detection against today's binary, so don't. (b) **The sweep cannot reach the gitignored bench registry** `moondeck/moondeck.json` (it's private, per [[bench-setup]]; the sweep uses `git ls-files`). Its `"board": "projectMM testbench …"` values reference catalog `name`s that *do* flip — so after the switch they'd mismatch only on your bench. **Switch-day local-tooling note: hand-update `moondeck/moondeck.json` board names** (and re-provision bench devices if you want the new mDNS identity) — the sweep covers tracked files only. The MoonDeck prose (`MoonDeck.md`, code comments) flips in the normal sweep. ## Phase 2 — the COMING TIME (staged, still pre-switch) diff --git a/docs/building.md b/docs/building.md index c7b2c373..9427f494 100644 --- a/docs/building.md +++ b/docs/building.md @@ -4,12 +4,12 @@ How to get the system running on a desktop, an ESP32, a Teensy, or a Raspberry P ## MoonDeck — the dev console -Everything that builds, flashes, runs, tests, monitors, or checks the project — for every target — lives as a script under `scripts/`. The full per-script reference is [scripts/MoonDeck.md](../scripts/MoonDeck.md). +Everything that builds, flashes, runs, tests, monitors, or checks the project — for every target — lives as a script under `moondeck/`. The full per-script reference is [moondeck/MoonDeck.md](../moondeck/MoonDeck.md). The scripts have two front ends with the same code and arguments: -- **CLI** — `uv run scripts/<group>/<name>.py`. What agents use; what CI uses. Composes with shell, captures exit codes, parses output. -- **MoonDeck** — `uv run scripts/moondeck.py`, then open `http://localhost:8420`. A browser dev console wrapping the same scripts: status dots, run/stop toggles for long-running processes, grouped tabs, output panes. The human control deck. +- **CLI** — `uv run moondeck/<group>/<name>.py`. What agents use; what CI uses. Composes with shell, captures exit codes, parses output. +- **MoonDeck** — `uv run moondeck/moondeck.py`, then open `http://localhost:8420`. A browser dev console wrapping the same scripts: status dots, run/stop toggles for long-running processes, grouped tabs, output panes. The human control deck. Use whichever fits. Neither path is "more official" than the other; the scripts are the source of truth and the front ends are interfaces. New work adds a script first; both interfaces follow. @@ -19,7 +19,7 @@ MoonDeck has three tabs: - **ESP32** — chip type and USB port selection. Build, flash, monitor. - **Live** — device discovery and monitoring against running devices on the network. -Script definitions and configuration live in `scripts/moondeck_config.json` (committed). Script documentation lives in `scripts/MoonDeck.md`, one section per script. Runtime state (selected devices, ports) persists in `scripts/moondeck.json` (gitignored). +Script definitions and configuration live in `moondeck/moondeck_config.json` (committed). Script documentation lives in `moondeck/MoonDeck.md`, one section per script. Runtime state (selected devices, ports) persists in `moondeck/moondeck.json` (gitignored). ## Tooling overview @@ -52,9 +52,9 @@ The project is structured as a small set of CMake libraries: a core library (pla Desktop and RPi both build with the root `CMakeLists.txt`. RPi can cross-compile against the same tree or build natively on the device — same source. ```sh -uv run scripts/build/build_desktop.py # build -uv run scripts/run/run_desktop.py # run as detached background process -uv run scripts/test/test_desktop.py # unit tests +uv run moondeck/build/build_desktop.py # build +uv run moondeck/run/run_desktop.py # run as detached background process +uv run moondeck/test/test_desktop.py # unit tests ``` Or use MoonDeck's PC tab for the same operations with a status dot per card. The desktop run detaches and outlives the launching script — the same model as flashing an ESP32, where the device runs independently afterwards. @@ -103,13 +103,13 @@ git config --global core.longpaths true git clone --depth 1 --branch release/v6.1 https://github.com/espressif/esp-idf.git "$env:USERPROFILE\esp\esp-idf" ``` -Then run the one-time Python environment setup — either open MoonDeck (`uv run scripts/moondeck.py`), go to the ESP32 tab, and click **Setup ESP-IDF**, or run it directly: +Then run the one-time Python environment setup — either open MoonDeck (`uv run moondeck/moondeck.py`), go to the ESP32 tab, and click **Setup ESP-IDF**, or run it directly: ```sh -uv run scripts/build/setup_esp_idf.py # one-time -uv run scripts/build/build_esp32.py --firmware esp32 # WiFi-only -uv run scripts/build/flash_esp32.py --firmware esp32 --port /dev/tty.usbserial-XXXX -uv run scripts/run/monitor_esp32.py --port /dev/tty.usbserial-XXXX +uv run moondeck/build/setup_esp_idf.py # one-time +uv run moondeck/build/build_esp32.py --firmware esp32 # WiFi-only +uv run moondeck/build/flash_esp32.py --firmware esp32 --port /dev/tty.usbserial-XXXX +uv run moondeck/run/monitor_esp32.py --port /dev/tty.usbserial-XXXX ``` `setup_esp_idf.py` runs the upstream installer for the host: `install.sh` on macOS/Linux, `install.bat` on Windows. Both create the same `~/.espressif/python_env/...` venv and download the same toolchains (~1.5 GB more) — only the wrapper differs. The Windows installer needs roughly 5 minutes on a fast link. It also offers to move a drifted checkout onto the pinned commit (see [ESP-IDF version](#esp-idf-version)); pass `--no-checkout` to keep it warn-only. @@ -170,7 +170,7 @@ Each row below states where we are and the trigger to move. | **PSA Crypto** — legacy mbedTLS crypto APIs deprecated in favour of the PSA API | No direct exposure: we never call mbedTLS ourselves; OTA uses `esp_https_ota` + `esp_crt_bundle_attach` ([platform_esp32_ota.cpp](../src/platform/esp32/platform_esp32_ota.cpp)), which wrap crypto internally. | Nothing to migrate while we stay on high-level components. **Watch** only: if a future feature needs hashing/signing directly (e.g. signed-OTA verification, a device identity), write it against the **PSA API** from the start, not legacy mbedTLS. Trigger: first direct crypto use. | | **`network_provisioning`** — Espressif's Unified Provisioning subsystem, renamed from `wifi_provisioning` in v6.0. Transports: **BLE (GATT)** + **Wi-Fi SoftAP**. Clients: official iOS/Android apps for both, plus `esp_prov` (a Python CLI on Linux/macOS/Windows). Transport-agnostic but ships no web/serial client. | We provision over [Improv](../src/core/ImprovProvisioningModule.h) — serial (USB) + BLE, driven from the **browser** (ESP Web Tools) or a serial CLI. That covers the *web-installer / no-app* onboarding well. What we **don't** have is the IDF-native **phone-app + SoftAP** flow (open the ESP app, pick the device's AP, hand it credentials) that most shipping ESP32 products offer. So this is a real coverage gap, not a duplicate: the two standards meet only on BLE and own different front-ends. | **Adopt to close the gap — this is a planned capability, not a maybe.** `network_provisioning` is in v6.0 (clears the floor) and is *the* IDF-native standard, so embracing it is exactly the stance above. Add it as a **sibling provisioning module** beside ImprovProvisioning (both live as Peripheral/System modules; the device can offer whichever transports its chip supports), reusing the same WiFi-credential plumbing. Not a replacement for Improv — they cover different front-ends (browser vs phone-app), and a product can want either. The phone-app + SoftAP path is the part that makes ESP32 deployment feel product-grade. Trigger: scheduled as one of the v6.0-adoption iterations (see below). | | **CMake Build System v2** — the named successor to the current build system; technical preview in v6.0/6.1, has its own migration guide | Standard `idf.py` build (v1). Our component is a thin `idf_component_register()` wrapper, so the migration surface is small. | **Watch until it's GA** (not while it's preview — adopting a preview build system would be the opposite of "common patterns first"). Trigger: v2 ships as the default. Then dry-run a build under v2, fix any `idf_component_register()` / Kconfig-dependency fallout, switch. Low risk given how little custom CMake we have. | -| **Built-in MCP server** (`idf.py mcp-server`) — lets an AI assistant drive build/flash/monitor/debug directly | Not used. Agents and humans both go through the `scripts/<group>/*.py` layer (the uniform interface in [scripts/MoonDeck.md](../scripts/MoonDeck.md)), which wraps pin-drift checks, per-firmware build dirs, and KPI collection. | **Evaluate, don't default to it.** The risk is a *second control path* that bypasses our script policy, and it's ESP32-only (no desktop), so it can't be the uniform path. If adopted, wrap it *behind* a script (`scripts/run/idf_mcp.py`) so the policy layer still applies, rather than pointing the agent at raw `idf.py`. Trigger: a concrete debug workflow the scripts can't cover. | +| **Built-in MCP server** (`idf.py mcp-server`) — lets an AI assistant drive build/flash/monitor/debug directly | Not used. Agents and humans both go through the `moondeck/<group>/*.py` layer (the uniform interface in [moondeck/MoonDeck.md](../moondeck/MoonDeck.md)), which wraps pin-drift checks, per-firmware build dirs, and KPI collection. | **Evaluate, don't default to it.** The risk is a *second control path* that bypasses our script policy, and it's ESP32-only (no desktop), so it can't be the uniform path. If adopted, wrap it *behind* a script (`moondeck/run/idf_mcp.py`) so the policy layer still applies, rather than pointing the agent at raw `idf.py`. Trigger: a concrete debug workflow the scripts can't cover. | The general rule: **anything already in v6.0 we adopt proactively** (it clears the floor, so there's no reason to wait — EIM and `network_provisioning` are both here), while **preview / not-yet-in-v6.0 features wait** until they're stable *and* in our floor. Each adoption is its own commit with its own hardware re-test, and none may regress the Teensy / desktop paths. @@ -186,11 +186,11 @@ Tracked in [backlog](backlog/README.md). `build_esp32.py --firmware` selects one of the shipping variants. The key combines chip name + feature flags + (for SKU-sensitive chips) module. ("Firmware" here is the compiled binary; the physical product (deviceModel) is a separate concept — see [architecture.md § Firmware vs deviceModel vs board](architecture.md#firmware-vs-devicemodel-vs-board).) `build_esp32.py --help` lists the full set. -The canonical list is the **`FIRMWARES` dict** in [`scripts/build/build_esp32.py`](../scripts/build/build_esp32.py) — the single source of truth, carrying each variant's `chip`, sdkconfig `fragments`, `eth_only`, `ships`, and `description`. Its machine-readable projection is [`web-installer/firmwares.json`](../web-installer/firmwares.json) (generated by `generate_firmwares.py`, drift-guarded by `check_firmwares.py`), which the CI release matrix, the ESP Web Tools manifest loops, and MoonDeck all read — so the list lives in exactly one place. `esp32p4-eth-wifi` has `ships: false` (its C6-slave Kconfig isn't reproducible in CI yet), so it builds from the CLI but stays out of the release matrix. +The canonical list is the **`FIRMWARES` dict** in [`moondeck/build/build_esp32.py`](../moondeck/build/build_esp32.py) — the single source of truth, carrying each variant's `chip`, sdkconfig `fragments`, `eth_only`, `ships`, and `description`. Its machine-readable projection is [`web-installer/firmwares.json`](../web-installer/firmwares.json) (generated by `generate_firmwares.py`, drift-guarded by `check_firmwares.py`), which the CI release matrix, the ESP Web Tools manifest loops, and MoonDeck all read — so the list lives in exactly one place. `esp32p4-eth-wifi` has `ships: false` (its C6-slave Kconfig isn't reproducible in CI yet), so it builds from the CLI but stays out of the release matrix. ESP-IDF v6.x has no `CONFIG_ESP_WIFI_ENABLED` switch (the symbol is forced on for WiFi-capable SoCs), so dropping WiFi at compile time happens via `EXCLUDE_COMPONENTS` plus `MM_NO_WIFI` (set when `MM_ETH_ONLY=1`, applied in `esp32/main/CMakeLists.txt`). The `esp32-eth` variant takes this path; the default `esp32` keeps both stacks compiled in and uses the runtime cascade in `NetworkModule` (Ethernet first, WiFi fallback when no PHY responds). -Each firmware has its own build dir at `build/esp32-<firmware>/`, so all variants can coexist on disk. `build_esp32.py` points `idf.py -B` at the per-firmware dir; switching firmwares is just a different `--firmware` argument, no clean rebuild penalty. Same-firmware rebuilds stay incremental, as before. Disk usage scales with the number of firmwares built (≈100 MB each), and a future rename would orphan the old dir — clean with `scripts/build/clean_esp32.py --firmware <name>` or `--all`. +Each firmware has its own build dir at `build/esp32-<firmware>/`, so all variants can coexist on disk. `build_esp32.py` points `idf.py -B` at the per-firmware dir; switching firmwares is just a different `--firmware` argument, no clean rebuild penalty. Same-firmware rebuilds stay incremental, as before. Disk usage scales with the number of firmwares built (≈100 MB each), and a future rename would orphan the old dir — clean with `moondeck/build/clean_esp32.py --firmware <name>` or `--all`. If a firmware *key* changes its feature set (e.g. the classic `esp32` collapse turned a WiFi-only key into WiFi+Ethernet), its existing build dir would otherwise keep the old `MM_NO_ETH` / `MM_ETH_ONLY` in `CMakeCache.txt` — CMake `-D` flags are written to the cache, and omitting one on a later configure does *not* clear it, so the stale value would silently build the old feature set (Ethernet stubbed out, no link, no LED, and a flash erase wouldn't help because it's a compile-time define). `build_esp32.py` guards this: before reusing a dir it compares the cached feature flags to what the firmware wants and wipes the dir for a clean reconfigure on a mismatch (printing the reason). Same-feature rebuilds are untouched, so the incremental fast path is preserved. @@ -198,7 +198,7 @@ Each ESP32-S3 SKU has its own firmware key because the sdkconfig fragment encode The Ethernet PHY type and pin map are runtime config, not baked into the build: each firmware carries the driver(s) its chip can host (RMII EMAC for classic/P4, W5500 SPI for S3), and `deviceModels.json` supplies the per-board PHY/pins (pushed into NetworkModule's eth controls at provision). The classic chip default is the common LAN8720 RMII wiring (reset GPIO 5, MDIO addr 0, clock GPIO 17 — e.g. the Olimex ESP32-Gateway), so a board with the same PHY but a different pinout (e.g. WT32-ETH01 with reset on GPIO 16) just needs a different `deviceModels.json` entry — no rebuild. -`--profile` is accepted one release for migration: `--profile default` → `--firmware esp32`, `--profile eth-only` → `--firmware esp32-eth`. The legacy `build_esp32_ethonly.py` wrapper still works (it now forwards `--firmware esp32-eth`). +`--profile` is accepted one release for migration: `--profile default` → `--firmware esp32`, `--profile eth-only` → `--firmware esp32-eth`. ### Why not Arduino diff --git a/docs/coding-standards.md b/docs/coding-standards.md index 871144d6..cb50dde6 100644 --- a/docs/coding-standards.md +++ b/docs/coding-standards.md @@ -95,7 +95,7 @@ All targets build warnings-as-errors: `-Wall -Wextra -Werror` on Clang/GCC (macO ## Static checks -- **Platform boundary** (`scripts/check/check_platform_boundary.py`) — scans all files outside `src/platform/` for `#ifdef` / `#if defined` with platform macros and `#include` of platform-specific headers (`esp_*`, `freertos/*`, `driver/*`, `SDL.h`, `wiringPi.h`, …). Fails if any are found. The platform boundary rule itself: [architecture.md § Platform abstraction](architecture.md#platform-abstraction). +- **Platform boundary** (`moondeck/check/check_platform_boundary.py`) — scans all files outside `src/platform/` for `#ifdef` / `#if defined` with platform macros and `#include` of platform-specific headers (`esp_*`, `freertos/*`, `driver/*`, `SDL.h`, `wiringPi.h`, …). Fails if any are found. The platform boundary rule itself: [architecture.md § Platform abstraction](architecture.md#platform-abstraction). - **Hot path lint** — flags allocation calls (`new`, `malloc`, `make_unique`, `make_shared`, `push_back`, `std::string` constructors) inside functions identified as hot path (render loop and callees). A code-review convention, enforced by hand. The hot path rule itself: [architecture.md § Hot path discipline](architecture.md#hot-path-discipline). - **Code formatting** — `clang-format` with a project `.clang-format` file. Applied in CI; code that doesn't match fails the check. Run locally via editor integration or `clang-format -i`. @@ -116,13 +116,13 @@ Where each kind of fact lives. The guiding rule: **document a thing once, in the ### The two surfaces -1. **Summary page (hand-written, end-user).** One `.md` per module *group*, a 4-column table — **name + description · gif/image · controls · links** (per module: tests · its technical page · attribution · anchors to any extra prose). One row per module, authored as `### ` prose blocks that a build-time hook ([`scripts/docs/mkdocs_hooks.py`](../scripts/docs/mkdocs_hooks.py)) renders as the table. Catalog controls live here because a catalog module's user surface is its runtime `controls_.add(...)` calls, which no static tool sees. Each group is nested in its own subfolder, consistent with the catalog: +1. **Summary page (hand-written, end-user).** One `.md` per module *group*, a 4-column table — **name + description · gif/image · controls · links** (per module: tests · its technical page · attribution · anchors to any extra prose). One row per module, authored as `###` prose blocks that a build-time hook ([`moondeck/docs/mkdocs_hooks.py`](../moondeck/docs/mkdocs_hooks.py)) renders as the table. Catalog controls live here because a catalog module's user surface is its runtime `controls_.add(...)` calls, which no static tool sees. Each group is nested in its own subfolder, consistent with the catalog: - `light/{effects,modifiers,layouts,drivers}/<name>.md` — the four light-catalog pages (may later split, e.g. `effects_wled.md` / `effects_moonmodules.md`). - `core/ui/`, `core/supporting/`, `light/supporting/` — the core-UI, core-supporting, and light-supporting summary pages. Cross-file design rationale that no single `.h` owns (module interactions, buffer-lifecycle coupling) is a prose section beneath a summary page's table — the only home for it, so a module needs no page of its own. -2. **Technical page (generated).** `docs/moonmodules/{core,light}/moxygen/<Module>.md`, produced from the `.h` by [`scripts/docs/gen_api.py`](../scripts/docs/gen_api.py): **Doxygen** (the de-facto-standard C++ parser) emits XML, **moxygen** renders Markdown through a custom Handlebars template ([`scripts/docs/moxygen-templates/`](../scripts/docs/moxygen-templates/)). Layout is controlled only through Doxygen config + that template — moxygen's own levers, the output is used verbatim, never post-processed. Each page carries the module's **class, variables, and members** from their `///` comments and links to its `.h`; a summary page's per-module link points here. +2. **Technical page (generated).** `docs/moonmodules/{core,light}/moxygen/<Module>.md`, produced from the `.h` by [`moondeck/docs/gen_api.py`](../moondeck/docs/gen_api.py): **Doxygen** (the de-facto-standard C++ parser) emits XML, **moxygen** renders Markdown through a custom Handlebars template ([`moondeck/docs/moxygen-templates/`](../moondeck/docs/moxygen-templates/)). Layout is controlled only through Doxygen config + that template — moxygen's own levers, the output is used verbatim, never post-processed. Each page carries the module's **class, variables, and members** from their `///` comments and links to its `.h`; a summary page's per-module link points here. The pages are **gitignored, regenerated on build** (flipping to committed-and-drift-gated is a one-line `.gitignore` change plus a gate like `check_firmwares.py`, if PR-review of the generated output ever earns it). Doxygen (a brew/apt binary) and moxygen (via npx) are the one justified non-uv dependency, like ESP-IDF's Python (see CLAUDE.md); absent locally the pages skip and the rest of the site builds, present in CI they render. @@ -134,4 +134,4 @@ A module's story therefore lives in exactly two places: its `.h` (technical, gen ### Test inventories: their own generator, not moxygen -`docs/tests/*.md` is generated by [`scripts/docs/generate_test_docs.py`](../scripts/docs/generate_test_docs.py), not moxygen. Unit tests are `TEST_CASE("…")` macros tagged with `// @module` and per-case `//` descriptions — a convention Doxygen documents nothing of (it parses C++ *entities*, not macro string literals) — and scenario tests are JSON, which moxygen cannot read. moxygen is for the `.h` module pages; the test generator owns the test pages. +`docs/tests/*.md` is generated by [`moondeck/docs/generate_test_docs.py`](../moondeck/docs/generate_test_docs.py), not moxygen. Unit tests are `TEST_CASE("…")` macros tagged with `// @module` and per-case `//` descriptions — a convention Doxygen documents nothing of (it parses C++ *entities*, not macro string literals) — and scenario tests are JSON, which moxygen cannot read. moxygen is for the `.h` module pages; the test generator owns the test pages. diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index 4b908f6b..d6d8499d 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -257,7 +257,7 @@ keep going. - **Run it on your computer** instead of (or alongside) an ESP32 — macOS, Windows, Linux: [project overview → Getting started](../README.md#getting-started). - **Manage several devices, build, and flash from one console** with MoonDeck, our - developer tool: [MoonDeck guide](../scripts/MoonDeck.md). + developer tool: [MoonDeck guide](../moondeck/MoonDeck.md). - **Build from source** or target Teensy / Raspberry Pi: [building.md](building.md). Stuck, or something didn't work? Open an diff --git a/docs/history/README.md b/docs/history/README.md index 7789ef64..8700da2b 100644 --- a/docs/history/README.md +++ b/docs/history/README.md @@ -1,6 +1,6 @@ # History — index -The backward-looking half of the docs (the forward-looking half is [`../backlog/`](../backlog/)). This folder is **not** present-tense and agents don't read it automatically — only when planning new work. See [CLAUDE.md § Documentation](../../CLAUDE.md) for how `history/` and `backlog/` relate. +The backward-looking half of the docs (the forward-looking half is [`../backlog/`](../backlog/README.md)). This folder is **not** present-tense and agents don't read it automatically — only when planning new work. See [CLAUDE.md § Documentation](../../CLAUDE.md) for how `history/` and `backlog/` relate. *Living index — kept current as the friend-repo digests are updated each month; the git log carries exact dates.* diff --git a/docs/history/decisions.md b/docs/history/decisions.md index 3a46b6e7..359d84af 100644 --- a/docs/history/decisions.md +++ b/docs/history/decisions.md @@ -449,7 +449,7 @@ The plan-18 branch landed plans 17 + 18 + six unplanned follow-ups (19, 19.1, 20 - **Per-`ControlType` behaviour belongs with the type, not with the caller.** Three call sites (`HttpServerModule::writeControls`, `FilesystemModule::writeValue`, `scenario_runner.cpp::writeJsonValAsValue` and twin apply paths) were each carrying their own `switch (c.type)` over the same enum. Each was 50-60 lines, hand-maintained in parallel, and silently drifted (the scenario runner had stopped recognising new types added for the FS path). The fix was to extract the switch into a small set of free functions in `src/core/Control.cpp` (`writeControlValue`, `writeControlMetadata`, `applyControlValue`, `isPersistable`, `hasDefault`, `controlTypeName`) and route the three call sites through them. Two cross-cutting requirements made this work: (1) `JsonSink` gained a third "fixed-buffer" mode (alongside socket and heap-grow) so the FS path could share the serializer without growing a per-value allocation, and (2) the apply path got an `ApplyPolicy` parameter (`Strict` for HTTP, `Clamp` for FS load) so the tolerant-load semantics survived the consolidation. Codified in `docs/coding-standards.md § Per-type behaviour lives with the type` — applies whenever the same `switch` appears in 2+ places. Counter-example also in that section: if only one caller needs the behaviour, keep it at the call site (a one-shot switch is cheaper than a function with one user). -- **Local Improv testing closes the dev-loop gap that made every Improv change a high-friction commit.** Before this branch, the only way to verify the Improv flow end-to-end was to tag a release, wait for CI, deploy to GitHub Pages, then flash from the live web installer — each iteration of the docs/install page or release-picker.js took minutes and burned a release tag. `scripts/run/preview_installer.py` now has a "flash-ready" mode that stages every local `build/esp32-*/projectMM.bin` under `releases/latest/` in the preview server's tree, generates matching Pages-relative manifests via the same `generate_manifest.py` production uses, and serves the install page at `localhost:8000`. The picker resolves the GitHub `latest` tag to the staged local bins via `toLocalUrl`, and Web Serial works on `localhost` without the secure-origin requirement that gates the public site — so an end-to-end flash + Improv WiFi provision is verifiable without touching any release. Paired with `scripts/build/improv_smoke_test.py` (probe + WiFi provision + LAN reachability, all three steps as a single CLI), which gives a deterministic pass/fail for the device-side Improv state machine and is wired into MoonDeck. Lesson: when a feature can only be tested in production, the iteration cost compounds into "don't touch it" — building the local test loop, even if it takes a day, pays for itself within a few commits. +- **Local Improv testing closes the dev-loop gap that made every Improv change a high-friction commit.** Before this branch, the only way to verify the Improv flow end-to-end was to tag a release, wait for CI, deploy to GitHub Pages, then flash from the live web installer — each iteration of the docs/install page or release-picker.js took minutes and burned a release tag. `moondeck/run/preview_installer.py` now has a "flash-ready" mode that stages every local `build/esp32-*/projectMM.bin` under `releases/latest/` in the preview server's tree, generates matching Pages-relative manifests via the same `generate_manifest.py` production uses, and serves the install page at `localhost:8000`. The picker resolves the GitHub `latest` tag to the staged local bins via `toLocalUrl`, and Web Serial works on `localhost` without the secure-origin requirement that gates the public site — so an end-to-end flash + Improv WiFi provision is verifiable without touching any release. Paired with `moondeck/build/improv_smoke_test.py` (probe + WiFi provision + LAN reachability, all three steps as a single CLI), which gives a deterministic pass/fail for the device-side Improv state machine and is wired into MoonDeck. Lesson: when a feature can only be tested in production, the iteration cost compounds into "don't touch it" — building the local test loop, even if it takes a day, pays for itself within a few commits. ## Lessons from this branch (Board injection follow-ups) @@ -713,7 +713,7 @@ The installer was reworked so a board catalog (`boards.json`) sets a device up f **A periodic re-broadcast to let late joiners "catch up" is a hack wearing a keepalive costume.** The 3D preview sends a coordinate table (positions) once, then per-frame colour. The original implementation re-sent the *whole table every ~1 second* "so a client that connected after the last rebuild catches it." It looked fine — a fresh page recovered within a second — so it shipped and sat there. But it's a workaround, not the mechanism: it rebuilt the full table from the layout **every tick-second forever**, on the hot path, whether or not anyone had connected and whether or not anything changed — and it papered over a missing request/response with polling. The correct construct is event-driven: send the table **when it actually changes** (`onBuildState` — grid/layout/LUT rebuild) and **when a client asks** (a new WS connection bumps `BinaryBroadcaster::clientGeneration()`, which `PreviewDriver::loop()` watches and re-sends on change). That's strictly *less* code than the timer and zero idle cost. How it sneaked past review: the workaround *worked* in casual testing and its cost was invisible until a later change made each rebuild heavier and the per-module tick was profiled. Lessons: (1) "re-send periodically so it eventually syncs" is the polling-instead-of-events smell — ask "what's the event that should trigger this?" and trigger on *that*; (2) a recurring rebuild on the hot path must justify itself every tick, so "every second, just in case" fails *Data over objects / fastest hot path* on sight; (3) this is *Continuous refactor, no hacks* — the fix isn't a scheduled cleanup, it's "the moment you see a keepalive timer standing in for a request, replace it." The guard is a test that advances the clock several seconds with no client change and asserts the table is **not** re-sent (the old timer would have). -**When a working seam regresses after your "fix," suspect the fix — and measure with a tool faithful to what the user sees.** The resumable preview send (`sendBufferedFrame`/`drainPreviewSend` — stream the producer buffer a memory-adaptive chunk per `loop20ms`, drop-new backpressure, downsample + display cap) shipped working on all three boards. A later attempt to *also* route the coordinate table + downsampled colour frame through the resumable path (removing the synchronous `sendAllOrClose` spin-and-close) looked principled but **regressed every board into an intermittent stall**, through several variants. Three lessons compounded: (1) **Stop at the first failed fix on a working path.** Each "improvement" to a seam that already worked added a new failure; the discipline (CLAUDE.md *Anti-stalling*) is to revert to the known-good state at attempt two, not re-engineer. The committed synchronous coord/downsampled path *closes a wedged client past a spin budget and lets it reconnect* — not elegant, but proven; the elegant unification wasn't worth a regression nobody could pin. (2) **A measurement tool must be faithful to the real client or it invents and hides bugs.** A plain one-shot WebSocket probe *gave up on close* where a browser *reconnects*, so it reported stalls users never saw AND missed blips they did — it sent the debugging in circles for a whole session. The fix was a browser-faithful probe (`scripts/diag/preview_health.py`: reads binary frames, sends the 25 s keepalive ping, auto-reconnects with backoff — exactly `app.js`'s `connectWs`); only then did probe and eyes agree. (3) **A stale process masquerades as a code bug.** "No preview on the PC build" with a corrupt-looking on-the-wire coordinate count survived every fresh rebuild — because a **38-hour-old desktop binary** still held port 8080; the freshly-built one couldn't bind and the browser/probe hit the stale one. The tell was the *uptime*, spotted by the product owner, not the diff. Lesson: before bisecting a "rebuild didn't fix it" bug, confirm the artifact under test is the one you built (check the process uptime / `build` timestamp / what's actually bound to the port). The faithful probe is now the standing way to measure preview health on any target (PC + the three boards), wired into MoonDeck's Live tab. +**When a working seam regresses after your "fix," suspect the fix — and measure with a tool faithful to what the user sees.** The resumable preview send (`sendBufferedFrame`/`drainPreviewSend` — stream the producer buffer a memory-adaptive chunk per `loop20ms`, drop-new backpressure, downsample + display cap) shipped working on all three boards. A later attempt to *also* route the coordinate table + downsampled colour frame through the resumable path (removing the synchronous `sendAllOrClose` spin-and-close) looked principled but **regressed every board into an intermittent stall**, through several variants. Three lessons compounded: (1) **Stop at the first failed fix on a working path.** Each "improvement" to a seam that already worked added a new failure; the discipline (CLAUDE.md *Anti-stalling*) is to revert to the known-good state at attempt two, not re-engineer. The committed synchronous coord/downsampled path *closes a wedged client past a spin budget and lets it reconnect* — not elegant, but proven; the elegant unification wasn't worth a regression nobody could pin. (2) **A measurement tool must be faithful to the real client or it invents and hides bugs.** A plain one-shot WebSocket probe *gave up on close* where a browser *reconnects*, so it reported stalls users never saw AND missed blips they did — it sent the debugging in circles for a whole session. The fix was a browser-faithful probe (`moondeck/diag/preview_health.py`: reads binary frames, sends the 25 s keepalive ping, auto-reconnects with backoff — exactly `app.js`'s `connectWs`); only then did probe and eyes agree. (3) **A stale process masquerades as a code bug.** "No preview on the PC build" with a corrupt-looking on-the-wire coordinate count survived every fresh rebuild — because a **38-hour-old desktop binary** still held port 8080; the freshly-built one couldn't bind and the browser/probe hit the stale one. The tell was the *uptime*, spotted by the product owner, not the diff. Lesson: before bisecting a "rebuild didn't fix it" bug, confirm the artifact under test is the one you built (check the process uptime / `build` timestamp / what's actually bound to the port). The faithful probe is now the standing way to measure preview health on any target (PC + the three boards), wired into MoonDeck's Live tab. **Don't hold a vendor library's async handle across your own event loop — it races the library's internal timers.** A UI refresh intermittently crashed the device (`assert failed: xQueueSemaphoreTake queue.c:1709 (( pxQueue ))` — a null FreeRTOS queue — inside the espressif mDNS component's `mdns_query_async_get_results`, plus an `Interrupt wdt timeout`). The mDNS *browse* (discovering peers for the "Your devices" list — distinct from mDNS *advertise*, which serves `<deviceName>.local` and was never the problem) used the async API: `mdns_query_async_new` returns a handle that `DevicesModule` held across ticks, polling it each `loop1s` with a 0 ms timeout. The trap: the mDNS component's **own task** owns that handle's queue and **frees it when the query's window (3 s) expires** — so a poll landing in the gap after expiry asserts on a freed queue. It was intermittent and grid-size-sensitive (a bigger grid lengthens the tick, widening the gap) and looked like "refresh crashes it" only because a refresh's activity coincided with the poll. **First fix attempt was wrong:** I assumed a *service-table mutation* (live rename re-registering `_http._tcp`) tore the handle down and added a cancel-before-mutate guard — it didn't fix it, because the freeing party is the component's expiry timer, not our code. **Real fix:** stop using the async-handle API entirely — replace the start/poll/stop trio with one synchronous `mdnsBrowse()` (`mdns_query_ptr`) that queries, delivers results, and frees everything in a single call, holding **no handle across ticks**, so the race window can't exist. The catch that synchronous introduced: `mdns_query_ptr` blocks the *full* timeout (it waits the whole window for late responders, no early return) and `loop1s` is charged to the tick — an 80 ms query tanked the tick. So **throttle**: browse one service type every ~8th tick with a ~60 ms timeout — one brief hiccup every ~8 s, invisible for discovery, FPS untouched in between. Lessons: (1) a library's async/iterator handle is only valid between *its* lifecycle events — if you can't see/where those fire (here, an internal expiry timer on another task), don't hold the handle across your loop; prefer a self-contained synchronous call that owns the whole lifecycle. (2) An *intermittent, load-dependent* crash whose backtrace sits in a vendor component is a **lifecycle race**, not a component bug — but find the *actual* concurrent actor before "fixing" (my first guess at the actor was wrong and the fix did nothing). (3) Trading async for synchronous trades a race for a blocking cost — budget it (throttle + bound the timeout) so the cure isn't a tick-killer. (4) Desktop stubs these mDNS calls to no-ops, so it's a hardware-only fix the unit suite can't reach; the reproduction (concurrent WS churn at a large grid → crash before, stable after, uptime climbing) is the proof, in the commit, not a desktop test. @@ -837,3 +837,23 @@ The move: **lift `setControl` onto the `Scheduler`** (which owns the module tree ## An RMT/IR done-callback runs in ISR context: signal only, decode on the task The first NEC-decoder version crash-looped the board (USB dropped, port re-enumerated) because the RMT `on_recv_done` callback both **decoded** the frame (calling non-`IRAM_ATTR` helpers) and **re-armed** `rmt_receive()` — from interrupt context. On ESP32, an ISR that jumps to flash-resident code while the cache is disabled, or re-enters a driver call, faults. The existing working RX capture (`rmtWs2812RxCapture`) already showed the safe shape and I didn't follow it at first: **the ISR only records the symbol count and signals a queue; the decode + re-arm happen on the task** (in `irRead`, called from the render loop). Rule: an RMT/GPIO done-callback is ISR context — it may touch only its stack, the passed event data, and an ISR-safe FreeRTOS primitive (`xQueueSendFromISR`); everything else (parsing, logging, re-arming the peripheral) moves to the waking task. When in doubt, copy the codebase's already-proven callback, don't re-derive it. + +## An MQTT (or any integration) topic identity must be a stable hardware id, never the editable name + +The MQTT module first derived its topic prefix from the user-editable `deviceName` (`projectMM/<deviceName>`) — attractive because it reads well and a rename "follows" on the device. On the bench a rename (ShellyOne → ShellyTwo) instantly repointed every topic and orphaned the Homebridge config: the hub kept publishing to the old topics, the device now listened on new ones, and the light went "not responding." Researching WLED (which projectMM already emulates), Tasmota, ESPHome, and Home-Assistant MQTT discovery found the finding unanimous: **the machine-facing identity is anchored to a stable hardware id (MAC / chip-id), and the human display name is a separate field.** WLED's default device topic is `wled/<last6-of-MAC>`, rename-stable; the friendly `serverDescription` is decoupled. HA discovery *requires* a stable `unique_id` precisely so a friendly-name change doesn't break automations, and explicitly forbids device name / hostname as identity (they change). + +The fix: topics derive from `projectMM/<last6-of-MAC>` (stable for the chip's life), and the friendly `deviceName` rides a separate retained `<prefix>/name` topic. A rename now updates only the display label; control never breaks. General rule: **when a config value is a control-plane identity that external systems bind to (a topic, a discovery id, an API key path), derive it from something immutable and keep the human-readable name a separate, published-but-non-identifying field.** The convenience of "the name is the identity" is a trap — it couples a display concern to a wire contract. + +## Don't promote one specific board's pins to "the chip default" + +`NetworkModule::ethType` defaulted to a per-chip `ethConfigDefault`, and the classic-ESP32 / P4 / S31 entries in that table were, on inspection, **one specific board's wiring each** (the classic default was "e.g. Olimex," the P4 default was the Waveshare NANO's pins, the S31 default was the Function-CoreBoard's) dressed up as a chip property. Two failures fell out: a WiFi-only board (the LOLIN-D32 Shelly) wasted an Ethernet-PHY init on hardware it lacks, and — worse — the QuinLED Dig-Octa, which sets custom MDC/MDIO but has *no software eth reset* (GPIO5 is an LED output on that board, per its published pinout), would have inherited the classic default reset=GPIO5 and driven an **LED pin as an Ethernet reset**. The reviewer's instinct ("are these industry standards, or a device we happened to use?") was exactly right. + +The fix: `ethType` defaults to **None** — Ethernet is opt-in, and every board that has it declares its own `ethType` + board-wiring pins explicitly in `deviceModels.json`; a validator (`check_devices.py`) enforces "ethType set → board-wiring pins present." The pin *defaults* still seed a board that opts in, but nothing relies on them silently. General rule: **a "default" that is really one board's values is bespoke masquerading as standard (a § Principles violation); make the capability opt-in and require each consumer to state its own values, so a missing declaration fails loudly instead of inheriting a stranger's wiring.** + +## A new firmware default only reaches a factory-fresh device; persistence wins on an upgrade + +After changing the `ethType` firmware default to None and reflashing an already-provisioned board, the device *still* reported the old value — because persistence (`FilesystemModule` overlays every saved control at boot) restores the value captured during the earlier provisioning, and a plain `flash write` keeps the filesystem. The same shape bit the MQTT prefix: an old-firmware device carried a stale stored prefix a reflash didn't clear. This is *correct* persistence behavior (saved config wins over a constructor default), but it means **a default change is invisible to any device that has already run** — only an erase-flash or an explicit re-set surfaces it. General rule: when you change a default to fix a bug, remember the fix lands only on factory-fresh devices; an already-provisioned fleet needs an explicit migration (re-inject, an erase, or a one-shot "clear if stale" on upgrade) — and when debugging "my fix didn't take," check the persisted `.config/*.json` before the firmware default. + +## Fixing "blocking in the tick" at the connect misses the same violation at the write + +The MQTT client's first version blocked the render loop twice — a synchronous `connect()` (with DNS) *and* an unbounded blocking `write()` retry — both inside `loop1s`, which runs in `Scheduler::tick`. The first review caught the connect and it was reworked to non-blocking `connectStart`/`connectPoll`; the *re-review* caught that `conn_.write()` was still there, one layer up: a zero-window broker (accepts TCP, stops reading) fills the send buffer and the next `write` never returns — the render loop freezes permanently, and the keepalive timeout can't even fire because control never comes back. The fix mirrored the connect one: sends go through a non-blocking `writeSome` with an all-or-reset rule (control packets are ≤256 B, so atomic-send is realistic). General rule: **a "no blocking in the hot path" audit must sweep every syscall the path can reach, not just the obvious one** — connect, DNS, read, *and* write. Fixing the loudest blocker while an equally-blocking sibling survives is a partial fix that reads as complete; the re-review exists to catch exactly that. diff --git a/docs/history/plans/Plan-20260519 - ESP32 Deployment.md b/docs/history/plans/Plan-20260519 - ESP32 Deployment.md index deab2d05..eb2214a8 100644 --- a/docs/history/plans/Plan-20260519 - ESP32 Deployment.md +++ b/docs/history/plans/Plan-20260519 - ESP32 Deployment.md @@ -122,7 +122,7 @@ Actual `wifi_credentials.h` is gitignored. 1. `cmake --build build` — desktop still builds, zero warnings 2. `cd build && ctest --output-on-failure` — all tests pass 3. `./build/test/mm_scenarios` — scenario passes -4. `python scripts/check/check_platform_boundary.py` — passes +4. `python moondeck/check/check_platform_boundary.py` — passes 5. `cd esp32 && idf.py set-target esp32 && idf.py build` — ESP32 builds 6. Flash + monitor: WiFi connects, serial shows "mmv3 running", ArtNet packets arrive at receiver 7. Lights visible on hub75 panel from ESP32 diff --git a/docs/history/plans/Plan-20260520 - Adaptive Memory Allocation & Memory Scenario Testing.md b/docs/history/plans/Plan-20260520 - Adaptive Memory Allocation & Memory Scenario Testing.md index 9eec5a52..a3fb8b96 100644 --- a/docs/history/plans/Plan-20260520 - Adaptive Memory Allocation & Memory Scenario Testing.md +++ b/docs/history/plans/Plan-20260520 - Adaptive Memory Allocation & Memory Scenario Testing.md @@ -118,7 +118,7 @@ Scenarios that verify memory behavior. Both in-process and live. - Each step: measure heap, check bounds - On ESP32: observe degradation cascade kicking in at some grid size -**Live runner** (`scripts/scenario/run_live_scenario.py`): +**Live runner** (`moondeck/scenario/run_live_scenario.py`): - Parse heap/maxBlock bounds from scenario JSON - Report memory deltas per step @@ -177,7 +177,7 @@ test/scenarios/memory-boot.json test/scenarios/memory-1to1.json test/scenarios/memory-shuffled.json test/scenarios/memory-scaling.json -scripts/scenario/run_live_scenario.py # memory bounds support +moondeck/scenario/run_live_scenario.py # memory bounds support docs/architecture-light.md # memory tiers, invariants docs/moonmodules/core/MoonModule.md docs/moonmodules/light/Layer.md diff --git a/docs/history/plans/Plan-20260520 - Live Scenario Testing (Item 8).md b/docs/history/plans/Plan-20260520 - Live Scenario Testing (Item 8).md index 01b91497..e9acedac 100644 --- a/docs/history/plans/Plan-20260520 - Live Scenario Testing (Item 8).md +++ b/docs/history/plans/Plan-20260520 - Live Scenario Testing (Item 8).md @@ -80,7 +80,7 @@ Actually, for live scenarios we just need the values. The Python runner calls `G ### 5. Python live scenario runner -`scripts/scenario/run_live_scenario.py`: +`moondeck/scenario/run_live_scenario.py`: - Connects to a device via HTTP (host:port) - Reads scenario JSON (same format as in-process) - Executes steps: @@ -113,10 +113,10 @@ src/light/DriverGroup.h # MODIFY: addChild/removeChild, role() src/light/LayoutGroup.h # MODIFY: addChild/removeChild, role() src/light/Layer.h # MODIFY: addChild/removeChild, role() src/main.cpp # MODIFY: register module types with factory -scripts/scenario/run_live_scenario.py # NEW: Python HTTP scenario runner -scripts/moondeck_config.json # MODIFY: add Live tab entries -scripts/moondeck_ui/index.html # MODIFY: Live tab content -scripts/moondeck_ui/app.js # MODIFY: device discovery UI +moondeck/scenario/run_live_scenario.py # NEW: Python HTTP scenario runner +moondeck/moondeck_config.json # MODIFY: add Live tab entries +moondeck/moondeck_ui/index.html # MODIFY: Live tab content +moondeck/moondeck_ui/app.js # MODIFY: device discovery UI test/scenarios/control-change.json # NEW: scenario with set_control steps docs/moonmodules/core/HttpServerModule.md # MODIFY: new endpoints docs/testing.md # MODIFY: live scenario section diff --git a/docs/history/plans/Plan-20260521 - Control-list-driven JSON persistence (item 11).md b/docs/history/plans/Plan-20260521 - Control-list-driven JSON persistence (item 11).md index 31aefb6d..3c55e4c9 100644 --- a/docs/history/plans/Plan-20260521 - Control-list-driven JSON persistence (item 11).md +++ b/docs/history/plans/Plan-20260521 - Control-list-driven JSON persistence (item 11).md @@ -234,11 +234,11 @@ saveSubtree(m): | 1 | Desktop build | `cmake --build /Users/ewoud/Developer/GitHub/ewowi/projectMM/build` (zero warnings) | | 2 | Unit tests | `cd build && ctest --output-on-failure` | | 3 | Scenario tests | `./build/test/mm_scenarios` (SIGABRT exit pre-existing on HEAD — accept) | -| 4 | Platform boundary | `python3 scripts/check/check_platform_boundary.py` — verify no platform leakage in FilesystemModule.h | -| 5 | Spec check | `python3 scripts/check/check_specs.py` — confirms FilesystemModule.md describes the implemented API | -| 6 | ESP32 build | `python3 scripts/build/build_esp32.py` — clean. Verify partition + LittleFS still work. | +| 4 | Platform boundary | `python3 moondeck/check/check_platform_boundary.py` — verify no platform leakage in FilesystemModule.h | +| 5 | Spec check | `python3 moondeck/check/check_specs.py` — confirms FilesystemModule.md describes the implemented API | +| 6 | ESP32 build | `python3 moondeck/build/build_esp32.py` — clean. Verify partition + LittleFS still work. | | 7 | Reviewer agent | Opus reviewer over staged diff. Flag: no heap alloc in `loop1s()` save path (only stack buffers); platform boundary clean; no duplication of JSON helpers; JsonUtil.h stays at ~50 lines (not growing into a JSON library). | -| 8 | KPI collection | `python3 scripts/check/collect_kpi.py --commit` | +| 8 | KPI collection | `python3 moondeck/check/collect_kpi.py --commit` | | 9 | Live scenarios | Run on ESP32 hardware: existing 7 scenarios pass. Manual: set deviceName via REST → reboot → verify deviceName persisted. | | 10 | Documentation | spec + architecture + testing updated; item 11 removed from `docs/plan.md`. | diff --git a/docs/history/plans/Plan-20260522 - Nest child module cards inside their parent card's box.md b/docs/history/plans/Plan-20260522 - Nest child module cards inside their parent card's box.md index cf58d5b1..f6207e0e 100644 --- a/docs/history/plans/Plan-20260522 - Nest child module cards inside their parent card's box.md +++ b/docs/history/plans/Plan-20260522 - Nest child module cards inside their parent card's box.md @@ -26,7 +26,7 @@ Two files changed: `src/ui/app.js` and `src/ui/style.css`. No backend change. `s ## Verification -- **Build** — `python3 scripts/build/build_desktop.py` from a clean `build/` (the old build cache held stale `projectMM-v3` paths from the directory rename and was removed). Zero warnings; `ui_embed` regenerated `ui_embedded.h` from the edited assets. +- **Build** — `python3 moondeck/build/build_desktop.py` from a clean `build/` (the old build cache held stale `projectMM-v3` paths from the directory rename and was removed). Zero warnings; `ui_embed` regenerated `ui_embedded.h` from the edited assets. - **Rendered DOM** — drove headless Chrome via CDP, selected the `Layer` root (`localStorage['mm_selectedRoot']`), asserted on the live DOM: root card is `Layer`; its child order is exactly `["card-title", "card-children", "card-footer"]`; the `.card-children` wrapper holds 2 child cards (`Noise`, `Mirror`) as direct descendants; `+ add child` sits below the children block. All assertions passed. - **Tests** — `ctest` 1/1 passed, `./build/test/mm_scenarios` 8/8 passed. UI-only change, no C++ touched, so test results unaffected as expected. diff --git a/docs/history/plans/Plan-20260522 - UI rewrite to ui-spec.md baseline (item 12).md b/docs/history/plans/Plan-20260522 - UI rewrite to ui-spec.md baseline (item 12).md index 7b1d97fb..bd2283bf 100644 --- a/docs/history/plans/Plan-20260522 - UI rewrite to ui-spec.md baseline (item 12).md +++ b/docs/history/plans/Plan-20260522 - UI rewrite to ui-spec.md baseline (item 12).md @@ -230,9 +230,9 @@ Per CLAUDE.md pre-commit checklist (10 steps). Specific to this plan: 1. `cmake --build build` — zero warnings (UI changes don't affect build but engine changes do) 2. `ctest --output-on-failure` — existing tests pass + 2 new (`test_movechild`, `test_module_factory`) 3. `./build/test/mm_scenarios` — exit 0 -4. `python3 scripts/check/check_platform_boundary.py` — PASS (new `platform::reboot` correctly placed) -5. `python3 scripts/check/check_specs.py` — `10+ modules ok` (HttpServer/SystemModule/MoonModule specs updated) -6. `python3 scripts/build/build_esp32.py` — clean; `ui_embedded.h` regenerated +4. `python3 moondeck/check/check_platform_boundary.py` — PASS (new `platform::reboot` correctly placed) +5. `python3 moondeck/check/check_specs.py` — `10+ modules ok` (HttpServer/SystemModule/MoonModule specs updated) +6. `python3 moondeck/build/build_esp32.py` — clean; `ui_embedded.h` regenerated 7. Reviewer agent (Opus) over the staged diff 8. KPI one-liner with PC + ESP32 tick/FPS per CLAUDE.md step 8 9. Hardware smoke test at `http://192.168.1.210/`: diff --git a/docs/history/plans/Plan-20260523 - Top-level shape change to `Layouts`, `Layers`, `Drivers`.md b/docs/history/plans/Plan-20260523 - Top-level shape change to `Layouts`, `Layers`, `Drivers`.md index afe43f24..dd243bcb 100644 --- a/docs/history/plans/Plan-20260523 - Top-level shape change to `Layouts`, `Layers`, `Drivers`.md +++ b/docs/history/plans/Plan-20260523 - Top-level shape change to `Layouts`, `Layers`, `Drivers`.md @@ -178,7 +178,7 @@ The UI is module-driven — it renders whatever the tree says. `acceptsChildren` Wait — that adds noise. Let me reconsider: - **(b) is the lightest:** `acceptsChildren` for `"Layers"` is hardcoded to `[Layer]` (by typeName, not role). The UI already special-cases this kind of containment via `acceptsChildren`. The role chip on each `Layer` card stays Generic (⚙️). Slightly cluttered emoji-wise but no role-enum change. - - **(a) is cleaner long-term:** add `ModuleRole::Layer` to the enum. The UI ROLE_EMOJI map gets a new entry (need to pick an emoji — 🪟 / 🎞️ / 🧱 are candidates, will ask the product owner). [check_specs.py](scripts/check/check_specs.py) might depend on the role list; verify. + - **(a) is cleaner long-term:** add `ModuleRole::Layer` to the enum. The UI ROLE_EMOJI map gets a new entry (need to pick an emoji — 🪟 / 🎞️ / 🧱 are candidates, will ask the product owner). [check_specs.py](moondeck/check/check_specs.py) might depend on the role list; verify. **Plan picks (a)** because we're already changing the shape and adding a role is cheaper than a special-case in the UI. **One emoji to pick during implementation.** @@ -212,13 +212,13 @@ Wait — that adds noise. Let me reconsider: ## Implementation order -1. **Add `ModuleRole::Layer`** to [src/core/MoonModule.h](src/core/MoonModule.h). Update `roleName()`. Verify [scripts/check/check_specs.py](scripts/check/check_specs.py) doesn't have a hardcoded role list. Build to check for warnings. +1. **Add `ModuleRole::Layer`** to [src/core/MoonModule.h](src/core/MoonModule.h). Update `roleName()`. Verify [moondeck/check/check_specs.py](moondeck/check/check_specs.py) doesn't have a hardcoded role list. Build to check for warnings. 2. **Rename `LayoutGroup` → `Layouts`** (class + file via `git mv` + factory key). Update all `#include`s, all `static_cast<LayoutGroup*>`, all references in tests + scenarios + spec. Build + run all tests; expect green (no behaviour change). 3. **Rename `DriverGroup` → `Drivers`** (same treatment). 4. **Add `class Layers`** in [src/light/Layers.h](src/light/Layers.h). Add `setLayouts()` to `Layer`. main.cpp creates `Layers` containing one `Layer`. Run all tests; live-verify with a desktop run that the pipeline still produces frames. -5. **Add `start/end` controls to `Layer`** — uint16 (or int16 if available) with sensible bounds. Default = whole layout. `rebuildLUT()` honours them when not at default. Update [test_layer*.cpp](test) and add a test asserting "Layer with default start/end matches old Layer behaviour byte-for-byte." +5. **Add `start/end` controls to `Layer`** — uint16 (or int16 if available) with sensible bounds. Default = whole layout. `rebuildLUT()` honours them when not at default. Update `test_layer*.cpp` and add a test asserting "Layer with default start/end matches old Layer behaviour byte-for-byte." 6. **UI emoji pick** for `ModuleRole::Layer` — ask the product owner. Add to `ROLE_EMOJI` map in [src/ui/app.js](src/ui/app.js). -7. **Update specs** ([Layer.md](docs/moonmodules/light/Layer.md), new [Layouts.md](docs/moonmodules/light/Layouts.md), new [Layers.md](docs/moonmodules/light/Layers.md), new [Drivers.md](docs/moonmodules/light/drivers/Drivers.md), [architecture-light.md](docs/architecture-light.md), [plan.md](docs/plan.md), [README.md](README.md) if needed). Run [check_specs.py](scripts/check/check_specs.py). +7. **Update specs** ([Layer.md](docs/moonmodules/light/Layer.md), new [Layouts.md](docs/moonmodules/light/Layouts.md), new [Layers.md](docs/moonmodules/light/Layers.md), new [Drivers.md](docs/moonmodules/light/drivers/Drivers.md), [architecture-light.md](docs/architecture-light.md), [plan.md](docs/plan.md), [README.md](README.md) if needed). Run [check_specs.py](moondeck/check/check_specs.py). 8. **Migration**: FilesystemModule deletes `.config/LayoutGroup.json` and `.config/DriverGroup.json` if present, logs a warning. 9. **All pre-commit gates 1–6** (build, ctest, scenarios, platform boundary, specs, ESP32). Reviewer agent (gate 7) after. @@ -227,9 +227,9 @@ Wait — that adds noise. Let me reconsider: - [ ] `cmake --build build` — zero warnings, builds clean. - [ ] `ctest` — all unit tests pass, including the new `test_layers_container.cpp` cases. - [ ] `./build/test/mm_scenarios` — all scenarios pass (after their `LayoutGroup`/`DriverGroup` type-name updates). -- [ ] [check_platform_boundary.py](scripts/check/check_platform_boundary.py) — PASS. -- [ ] [check_specs.py](scripts/check/check_specs.py) — all specs ok. -- [ ] [build_esp32.py](scripts/build/build_esp32.py) — clean ESP32 build. +- [ ] [check_platform_boundary.py](moondeck/check/check_platform_boundary.py) — PASS. +- [ ] [check_specs.py](moondeck/check/check_specs.py) — all specs ok. +- [ ] [build_esp32.py](moondeck/build/build_esp32.py) — clean ESP32 build. - [ ] Live desktop run: `/api/types` shows `Layouts`, `Layers`, `Drivers` (no longer `LayoutGroup`, `DriverGroup`). `/api/state` shows the new tree shape. Effects render correctly through the new wiring. Tick time within run-to-run jitter of the previous commit. - [ ] UI side-nav reads `Layouts`, `Layers`, `Drivers`. Cards under `Layers` contain one `Layer` with effects+modifiers inside. Drag-reorder still works within each container. - [ ] One snapshot ESP32 run verifies no regression — same scenario, same FPS within jitter. diff --git a/docs/history/plans/Plan-20260524 - Release 1.0 distribution - web installer + GitHub Releases.md b/docs/history/plans/Plan-20260524 - Release 1.0 distribution - web installer + GitHub Releases.md index b865b407..f6e4fc53 100644 --- a/docs/history/plans/Plan-20260524 - Release 1.0 distribution - web installer + GitHub Releases.md +++ b/docs/history/plans/Plan-20260524 - Release 1.0 distribution - web installer + GitHub Releases.md @@ -20,7 +20,7 @@ Closed scope (decided before this plan, not revisited here): - **Desktop binaries = macOS arm64 + Windows x64.** No Linux, no macOS x64. - **No OTA in 1.0.** Users re-flash via Web Tools when a new release lands. - **Tag matches `library.json` version.** CI fails fast on drift — maintainer bumps version, commits, tags as one act. -- **MoonDeck developer flow unchanged.** `git clone` + `uv run scripts/moondeck.py` stays the dev bootstrap; the slow part is the prerequisite chain (uv, ESP-IDF), not the clone. +- **MoonDeck developer flow unchanged.** `git clone` + `uv run moondeck/moondeck.py` stays the dev bootstrap; the slow part is the prerequisite chain (uv, ESP-IDF), not the clone. ## Architecture @@ -29,7 +29,7 @@ git tag v1.0.0 └─> .github/workflows/release.yml ├─ verify-version (tag == library.json["version"]?) ├─ build-esp32 (matrix: esp32, esp32-eth, esp32-eth-wifi, esp32s3-n16r8) - │ └─ scripts/build/build_esp32.py --board <key> + │ └─ moondeck/build/build_esp32.py --board <key> ├─ build-macos (macos-14, cmake Release, tar.gz) ├─ build-windows (windows-latest, cmake/MSVC Release, zip) └─ release @@ -68,14 +68,14 @@ The base file used to carry 7 Olimex-specific Eth lines (`CONFIG_ETH_USE_ESP32_E **Files to edit:** -- [scripts/build/build_esp32.py](../../scripts/build/build_esp32.py) — replace the `--profile` argument logic with a `BOARDS` dict + `--board` argument. `--profile` becomes a deprecated alias (`eth-only` → `esp32-eth`, `default` → `esp32`) for one release. Replace `profile_cmake_args()` with `board_cmake_args(board)`. -- [scripts/build/build_esp32_ethonly.py](../../scripts/build/build_esp32_ethonly.py) — forward `--board esp32-eth` instead of `--profile eth-only`. Kept for any external scripting that already calls the filename. +- [moondeck/build/build_esp32.py](../../moondeck/build/build_esp32.py) — replace the `--profile` argument logic with a `BOARDS` dict + `--board` argument. `--profile` becomes a deprecated alias (`eth-only` → `esp32-eth`, `default` → `esp32`) for one release. Replace `profile_cmake_args()` with `board_cmake_args(board)`. +- [moondeck/build/build_esp32_ethonly.py](../../moondeck/build/build_esp32_ethonly.py) — forward `--board esp32-eth` instead of `--profile eth-only`. Kept for any external scripting that already calls the filename. - [esp32/sdkconfig.defaults](../../esp32/sdkconfig.defaults) — remove the 7 Eth-block lines. File becomes board-neutral WiFi-default. - [esp32/sdkconfig.defaults.eth](../../esp32/sdkconfig.defaults.eth) — renamed from `.olimex_gw`. Self-sufficient (carries every Eth setting the working Olimex build needs); comment names Olimex as the default pin map and points at the 2.0 PHY-runtime-config plan. - [esp32/sdkconfig.defaults.esp32s3-n16r8](../../esp32/sdkconfig.defaults.esp32s3-n16r8) — renamed from `.esp32s3_n16r8` (hyphen instead of underscore — matches the board key). No content change. - Profile-change marker: rename `esp32/build/.mm_profile` → `.mm_board`. Migrate on first run (if the legacy file exists, read it once, treat as the equivalent board, then write the new marker). -- [scripts/moondeck.py](../../scripts/moondeck.py) — add `extra_args` forwarding (3 lines) so a config entry can pass static flags to its script. -- [scripts/moondeck_config.json](../../scripts/moondeck_config.json) + [scripts/MoonDeck.md](../../scripts/MoonDeck.md) — replace the "Build" / "Build (Ethernet-only)" pair with four board buttons, each baking a `--board` arg via `extra_args`. +- [moondeck/moondeck.py](../../moondeck/moondeck.py) — add `extra_args` forwarding (3 lines) so a config entry can pass static flags to its script. +- [moondeck/moondeck_config.json](../../moondeck/moondeck_config.json) + [moondeck/MoonDeck.md](../../moondeck/MoonDeck.md) — replace the "Build" / "Build (Ethernet-only)" pair with four board buttons, each baking a `--board` arg via `extra_args`. - docs/moonmodules/core/NetworkModule.md — update the Ethernet-only section to reference `--board esp32-eth`. ### Step 2 — Version-drift guard (0.5 h) @@ -84,7 +84,7 @@ CI must verify `git tag == library.json["version"]` and fail before building. No **Files to create:** -- [scripts/build/verify_version.py](../../scripts/build/verify_version.py) — short script that reads `GITHUB_REF_NAME` (without leading `v`) and `library.json` `"version"`, fails the workflow if they differ. +- [moondeck/ci/verify_version.py](../../moondeck/ci/verify_version.py) — short script that reads `GITHUB_REF_NAME` (without leading `v`) and `library.json` `"version"`, fails the workflow if they differ. **Action at release time:** @@ -96,7 +96,7 @@ Static-link what we can; accept dynamic libc++ on macOS (Apple doesn't ship a st **Files to create / edit:** -- [scripts/build/package_desktop.py](../../scripts/build/package_desktop.py) — new. Reads version from `library.json`, detects host platform, runs the right CMake invocation, packages. +- [moondeck/ci/package_desktop.py](../../moondeck/ci/package_desktop.py) — new. Reads version from `library.json`, detects host platform, runs the right CMake invocation, packages. - macOS arm64: `cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=arm64 && cmake --build build --config Release`. Tarball as `dist/projectMM-macos-arm64-vX.Y.Z.tar.gz` with the binary + a short `README.txt`. - Windows x64: `cmake -B build -G "Visual Studio 17 2022" -A x64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded && cmake --build build --config Release`. Zip as `dist/projectMM-windows-x64-vX.Y.Z.zip`. - [CMakeLists.txt](../../CMakeLists.txt) — gate the warning flags by compiler: @@ -122,7 +122,7 @@ Don't hardcode the offset table — read from `esp32/build/flasher_args.json` pr **Files to create:** -- [scripts/build/generate_manifest.py](../../scripts/build/generate_manifest.py) — takes `--board <key> --version <ver> --release-url <url> --flasher-args <path> --out <path>`, writes the manifest JSON with parts ordered by offset. +- [moondeck/build/generate_manifest.py](../../moondeck/build/generate_manifest.py) — takes `--board <key> --version <ver> --release-url <url> --flasher-args <path> --out <path>`, writes the manifest JSON with parts ordered by offset. Schema: @@ -168,7 +168,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: python scripts/build/verify_version.py + - run: python moondeck/ci/verify_version.py build-esp32: needs: verify-version @@ -190,7 +190,7 @@ jobs: esp_idf_version: v5.4 target: ${{ startsWith(matrix.board, 'esp32s3') && 'esp32s3' || 'esp32' }} path: 'esp32' - command: python ../scripts/build/build_esp32.py --board ${{ matrix.board }} + command: python ../moondeck/build/build_esp32.py --board ${{ matrix.board }} - name: Stage artifacts run: | mkdir -p dist @@ -209,7 +209,7 @@ jobs: runs-on: macos-14 steps: - uses: actions/checkout@v4 - - run: python scripts/build/package_desktop.py + - run: python moondeck/ci/package_desktop.py - uses: actions/upload-artifact@v4 with: { name: desktop-macos, path: dist/ } @@ -218,7 +218,7 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v4 - - run: python scripts/build/package_desktop.py + - run: python moondeck/ci/package_desktop.py - uses: actions/upload-artifact@v4 with: { name: desktop-windows, path: dist/ } @@ -237,7 +237,7 @@ jobs: V=$(jq -r .version library.json) BASE=https://github.com/${{ github.repository }}/releases/download/v$V for B in esp32 esp32-eth esp32-eth-wifi esp32s3-n16r8; do - python scripts/build/generate_manifest.py \ + python moondeck/build/generate_manifest.py \ --board $B --version $V --release-url $BASE \ --flasher-args dist/flasher-$B.json --out dist/manifest-$B.json done @@ -372,23 +372,23 @@ These run as the per-release additions to CLAUDE.md's Event 3 (Release tag) gate **New:** - [.github/workflows/release.yml](../../.github/workflows/release.yml) -- [scripts/build/package_desktop.py](../../scripts/build/package_desktop.py) -- [scripts/build/generate_manifest.py](../../scripts/build/generate_manifest.py) -- [scripts/build/verify_version.py](../../scripts/build/verify_version.py) +- [moondeck/ci/package_desktop.py](../../moondeck/ci/package_desktop.py) +- [moondeck/build/generate_manifest.py](../../moondeck/build/generate_manifest.py) +- [moondeck/ci/verify_version.py](../../moondeck/ci/verify_version.py) - docs/install/index.html - docs/install/README.md **Edited:** -- [scripts/build/build_esp32.py](../../scripts/build/build_esp32.py) — add `--board`, deprecate `--profile`. -- [scripts/build/build_esp32_ethonly.py](../../scripts/build/build_esp32_ethonly.py) — forwards to `--board esp32-eth`. +- [moondeck/build/build_esp32.py](../../moondeck/build/build_esp32.py) — add `--board`, deprecate `--profile`. +- [moondeck/build/build_esp32_ethonly.py](../../moondeck/build/build_esp32_ethonly.py) — forwards to `--board esp32-eth`. - [esp32/sdkconfig.defaults](../../esp32/sdkconfig.defaults) — drop the 7 Eth lines. - [esp32/sdkconfig.defaults.eth](../../esp32/sdkconfig.defaults.eth) — renamed from `.olimex_gw`. Self-sufficient (carries the full Olimex pin set). - [esp32/sdkconfig.defaults.esp32s3-n16r8](../../esp32/sdkconfig.defaults.esp32s3-n16r8) — renamed from `.esp32s3_n16r8`. No content change. - [CMakeLists.txt](../../CMakeLists.txt) — MSVC-gated warning flags + static MSVC runtime. - [library.json](../../library.json) — bump `0.1.0` → `1.0.0` at release time. -- [scripts/moondeck.py](../../scripts/moondeck.py) — `extra_args` forwarding. -- [scripts/moondeck_config.json](../../scripts/moondeck_config.json) + [scripts/MoonDeck.md](../../scripts/MoonDeck.md) — four board buttons. +- [moondeck/moondeck.py](../../moondeck/moondeck.py) — `extra_args` forwarding. +- [moondeck/moondeck_config.json](../../moondeck/moondeck_config.json) + [moondeck/MoonDeck.md](../../moondeck/MoonDeck.md) — four board buttons. - docs/moonmodules/core/NetworkModule.md — `--board esp32-eth` reference. - [README.md](../../README.md) — Quick Start with installer URL. - docs/building.md — boards table. @@ -400,8 +400,8 @@ These run as the per-release additions to CLAUDE.md's Event 3 (Release tag) gate ## Verification -- **Local board builds:** `python scripts/build/build_esp32.py --board esp32`, `--board esp32-eth`, `--board esp32-eth-wifi`, and `--board esp32s3-n16r8` all complete with zero warnings. Each writes `esp32/build/projectMM.bin` + `flasher_args.json` for the right chip. -- **Local desktop:** `python scripts/build/package_desktop.py` on macOS produces a tarball under `dist/` that runs on a fresh Mac. +- **Local board builds:** `python moondeck/build/build_esp32.py --board esp32`, `--board esp32-eth`, `--board esp32-eth-wifi`, and `--board esp32s3-n16r8` all complete with zero warnings. Each writes `esp32/build/projectMM.bin` + `flasher_args.json` for the right chip. +- **Local desktop:** `python moondeck/ci/package_desktop.py` on macOS produces a tarball under `dist/` that runs on a fresh Mac. - **Local installer (surface 1):** the C+ recipe in `docs/install/README.md` — pull branch CI artifacts with `gh run download`, serve locally, flash each of the four boards over USB through the install page. - **RC dry run (surface 2):** push `v1.0.0-rcN` tag, all workflow jobs green, pre-release page populated with 22 files, Pages staging + deploy correctly skipped, manual flash per board succeeds against the rcN release URLs. - **Real release (surface 3):** tag `v1.0.0`, Pages flips, the live `https://ewowi.github.io/projectMM/install/` flashes each board cleanly. diff --git a/docs/history/plans/Plan-20260525 - Release-channel picker + first-boot WiFi provisioning.md b/docs/history/plans/Plan-20260525 - Release-channel picker + first-boot WiFi provisioning.md index a205b21e..e1e7c996 100644 --- a/docs/history/plans/Plan-20260525 - Release-channel picker + first-boot WiFi provisioning.md +++ b/docs/history/plans/Plan-20260525 - Release-channel picker + first-boot WiFi provisioning.md @@ -22,7 +22,7 @@ projectMM has three installer surfaces, two of which don't exist in v3 yet: Plan-18 builds all three with a coherent identity: - **A shared release-channel picker JS module** powers both installers (Tracks 1+2): visitor (browser, or device-UI tab) picks **Stable** or **Pre-release (beta)** → release → board → click Install. Same code, two surfaces, one mental model. -- **Improv WiFi over USB-serial** (Track 3) handles first-boot provisioning. Browser drives the protocol immediately after a flash via ESP Web Tools; a Python CLI (`scripts/build/improv_provision.py`) drives it for headless / rack / CI use. Same protocol, two transports. +- **Improv WiFi over USB-serial** (Track 3) handles first-boot provisioning. Browser drives the protocol immediately after a flash via ESP Web Tools; a Python CLI (`moondeck/build/improv_provision.py`) drives it for headless / rack / CI use. Same protocol, two transports. **Step 0 of v1 of this plan empirically falsified plan-17 risk #4**: GitHub release-asset URLs (both `github.com/.../releases/download/` and the `release-assets.githubusercontent.com` redirect target) return **no `Access-Control-Allow-Origin` headers**. Cross-origin browser fetches are blocked. WLED works around this with a third-party CORS proxy (`proxy.corsfix.com`); ESPHome self-hosts every binary; projectMM-v1 sidestepped the problem entirely by not having a web installer. v3 chooses **self-host on Pages** (Option 1 from the CORS replan discussion): the release workflow stages the last 5 stable + 5 prerelease releases' binaries into Pages content. End-of-line origin = same as the install page, no CORS. @@ -51,7 +51,7 @@ Track 3 (Improv) closed scope: - **Serial source**: UART0 only on every board. ESP32-S3-DevKitC-1's UART USB port works (UART0 routed to the on-board USB-UART bridge); the S3's native USB-Serial-JTAG port doesn't. AP-fallback remains the only path for users with USB-CDC-only connections. - **Lifecycle**: always-on listener, no task suspension. Provision requests are rejected (with Improv's wrong-state error frame) when `platform::wifiStaConnected() == true`; scan + info requests stay available so a browser can identify a running device. - **What it surfaces**: one read-only `provision_status` Control matching `FirmwareUpdateModule`'s shape. No buttons, no re-provision affordance — the protocol is the entry point. -- **Rack / CI mode**: `scripts/build/improv_provision.py` — pyserial CLI speaking the Improv protocol. Single-port mode today (`--port + --ssid + --password`); a future `--from-list <devicelist.json>` mode is a separate plan once v3 has a devicelist schema. +- **Rack / CI mode**: `moondeck/build/improv_provision.py` — pyserial CLI speaking the Improv protocol. Single-port mode today (`--port + --ssid + --password`); a future `--from-list <devicelist.json>` mode is a separate plan once v3 has a devicelist schema. - **AP-fallback flow stays unchanged.** Improv adds a third credential-entry path alongside the AP fallback UI and the persistence-loaded values; all three converge on `NetworkModule::ssid_` / `password_`. Prior art (the design isn't bespoke): @@ -223,13 +223,13 @@ releasePicker.init({ Keep the SRI-pinned ESP Web Tools `<script>` tag and the browser-warning card. -**Step 2.2 — Per-release manifest URLs become relative (0.5 h).** [scripts/build/generate_manifest.py](scripts/build/generate_manifest.py) already accepts `--release-url`; passing `--release-url .` produces relative paths like `./firmware-esp32-v…bin`. This is the form needed for Pages-hosted manifests (same-origin). The current release-asset manifest stays absolute (used by the OTA picker which still passes binary URLs to the device). +**Step 2.2 — Per-release manifest URLs become relative (0.5 h).** [moondeck/build/generate_manifest.py](moondeck/build/generate_manifest.py) already accepts `--release-url`; passing `--release-url .` produces relative paths like `./firmware-esp32-v…bin`. This is the form needed for Pages-hosted manifests (same-origin). The current release-asset manifest stays absolute (used by the OTA picker which still passes binary URLs to the device). Add a second invocation of the manifest generator in the workflow's "Generate ESP Web Tools manifests" step, writing a second set into `pages-manifests/`: ```bash for B in esp32 esp32-eth esp32-eth-wifi esp32s3-n16r8; do - python scripts/build/generate_manifest.py \ + python moondeck/build/generate_manifest.py \ --board "$B" --version "$V" \ --release-url . \ --flasher-args "dist/flasher-$B.json" \ @@ -372,7 +372,7 @@ void setWifiCredentials(const char* ssid, const char* password) { 2. `mm::ModuleFactory::registerType<mm::ImprovProvisioningModule>("ImprovProvisioningModule", "core/ImprovProvisioningModule.md");` in `registerModuleTypes()`. 3. Create + setters + `scheduler.addModule(...)` **after** `networkModule` (Improv depends on NetworkModule existing so its `setNetworkModule` setter has a valid pointer; the cold-boot order doesn't matter — both modules' `setup()` runs in the same scheduler phase). -**Step 3.6 — Python rack CLI: `scripts/build/improv_provision.py` (0.5 h).** ~150-line pyserial script. Argparse: +**Step 3.6 — Python rack CLI: `moondeck/build/improv_provision.py` (0.5 h).** ~150-line pyserial script. Argparse: ```bash improv_provision.py --port /dev/tty.usbserial-X --ssid <SSID> --password <PW> [--timeout 30] @@ -382,9 +382,9 @@ improv_provision.py --port /dev/tty.usbserial-X --ssid <SSID> --password <PW> [- - Loop reading frames until "provisioning success" (with URL) or "provisioning fail" (with error code) or timeout. - Print: `provisioned <device> on <SSID>; UI at <URL>`. Exit 0 on success, non-zero on any error. -A future `--from-list scripts/devicelist.json` mode for true rack provisioning is **out of scope for plan-18** — v3 doesn't have a devicelist schema yet; single-port mode covers single-device and "shell-loop a hub" today. +A future `--from-list moondeck/devicelist.json` mode for true rack provisioning is **out of scope for plan-18** — v3 doesn't have a devicelist schema yet; single-port mode covers single-device and "shell-loop a hub" today. -Add to [scripts/MoonDeck.md](../../scripts/MoonDeck.md) under a new "Provisioning" section. No MoonDeck button — the script is CLI-only by design. +Add to [moondeck/MoonDeck.md](../../moondeck/MoonDeck.md) under a new "Provisioning" section. No MoonDeck button — the script is CLI-only by design. **Step 3.7 — `docs/moonmodules/core/ImprovProvisioningModule.md` + hardware verification (0.4 h).** @@ -398,7 +398,7 @@ Hardware verification (product-owner gate): 1. Flash to an ESP32 and an ESP32-S3 DevKitC-1 (via the silkscreen UART USB port on the S3). 2. From Chrome desktop, open <https://www.improv-wifi.com/>, click Connect, pick the device's serial port. Expect device name + chip + version to appear in the browser; SSID scan returns nearby networks; enter creds → device shows `received credentials` → `connecting` → `connected: <ssid>`; URL `http://<ip>/` clickable; opens the device UI. -3. Same flow via the Python CLI: `python3 scripts/build/improv_provision.py --port <port> --ssid <ssid> --password <pw>`. Expect exit 0 + the IP printed. +3. Same flow via the Python CLI: `python3 moondeck/build/improv_provision.py --port <port> --ssid <ssid> --password <pw>`. Expect exit 0 + the IP printed. 4. Confirm ESP_LOGI output still appears on the serial monitor throughout (the step 3.2 risk). 5. Wipe credentials (`POST /api/control` on `ssid` = empty + reboot) → confirm device boots AP-fallback as before. Re-run Improv via browser → confirm it provisions cleanly. The AP-fallback path stays intact. @@ -417,7 +417,7 @@ Track 3 total: **3.0 h**. Sequential within the track. Hardware verification (st - [src/ui/release-picker.js](src/ui/release-picker.js) — shared release-picker module. - [src/core/FirmwareUpdateModule.h](src/core/FirmwareUpdateModule.h) — on-device OTA MoonModule (header-only). - [src/core/ImprovProvisioningModule.h](src/core/ImprovProvisioningModule.h) — Improv listener MoonModule (header-only). **Track 3.** -- [scripts/build/improv_provision.py](scripts/build/improv_provision.py) — pyserial CLI for headless / rack provisioning. **Track 3.** +- [moondeck/build/improv_provision.py](moondeck/build/improv_provision.py) — pyserial CLI for headless / rack provisioning. **Track 3.** - [docs/moonmodules/core/ImprovProvisioningModule.md](docs/moonmodules/core/ImprovProvisioningModule.md) — spec page. **Track 3.** - [docs/history/plan-18.md](docs/history/plan-18.md) — this plan's archive. @@ -437,9 +437,9 @@ Track 3 total: **3.0 h**. Sequential within the track. Hardware verification (st - `main.cpp` — register FirmwareUpdateModule **+ ImprovProvisioningModule (Track 3)**. - [docs/install/index.html](docs/install/index.html) — Track 2: use shared module. - [.github/workflows/release.yml](.github/workflows/release.yml) — Track 2: cumulative content staging, remove RC-Pages gate. -- [scripts/build/generate_manifest.py](scripts/build/generate_manifest.py) — comment about the `--release-url .` use case. +- [moondeck/build/generate_manifest.py](moondeck/build/generate_manifest.py) — comment about the `--release-url .` use case. - [docs/install/README.md](docs/install/README.md) — Track 2: simplify recipes. -- [scripts/MoonDeck.md](scripts/MoonDeck.md) — document `improv_provision.py` (Track 3). +- [moondeck/MoonDeck.md](moondeck/MoonDeck.md) — document `improv_provision.py` (Track 3). - [docs/plan.md](docs/plan.md) — remove installer stub. ## Reuse map @@ -448,11 +448,11 @@ Track 3 total: **3.0 h**. Sequential within the track. Hardware verification (st |---|---|---| | [projectMM-v1 OTA module](../../projectMM-v1/src/modules/system/FirmwareUpdateModule.h) | MoonModule with two display controls + file-scope statics polled by `loop1s()` | Working pattern from v1. v3 ports the architecture using v3 idioms (`controls_.addReadOnly`, anon namespace statics). | | projectMM-v1 `populateGhList()` | `fetch(api.github.com/.../releases) + sessionStorage cache + prerelease filter + per-asset install button` | Identical data shape; the picker UX is recognisable from v1. | -| [projectMM-v1 `pal::http_fetch_to_ota`](../../projectMM-v1/src/platform/esp32) | `esp_https_ota_config_t` + `esp_crt_bundle_attach` + perform loop | Working ESP-IDF idiom. Cherry-pick the working numbers (stack size, core pinning) into v3. | +| projectMM-v1 `pal::http_fetch_to_ota` (`projectMM-v1/src/platform/esp32`) | `esp_https_ota_config_t` + `esp_crt_bundle_attach` + perform loop | Working ESP-IDF idiom. Cherry-pick the working numbers (stack size, core pinning) into v3. | | [src/ui/embed_ui.cmake](src/ui/embed_ui.cmake) | The whole UI-embedding pipeline | Extend to one more file. No new architecture. | | [src/core/SystemModule.h:85](src/core/SystemModule.h#L85) | `controls_.addReadOnly("name", buf, sizeof(buf))` for live-updating diagnostic strings | Same shape for `update_status` and `update_pct`. | | [src/core/HttpServerModule.cpp:428+](src/core/HttpServerModule.cpp#L428) | `mm::json::parseString(body, "key", buf, sizeof(buf))` pattern | Reuse for parsing the `{"url":"..."}` body. | -| [scripts/build/generate_manifest.py](scripts/build/generate_manifest.py) | The whole script | Stays unchanged. Track 2 just calls it twice — once with absolute URLs (release assets) and once with `--release-url .` (Pages copy). | +| [moondeck/build/generate_manifest.py](moondeck/build/generate_manifest.py) | The whole script | Stays unchanged. Track 2 just calls it twice — once with absolute URLs (release assets) and once with `--release-url .` (Pages copy). | | projectMM-v1 `deploy/wifi.py` + `flashfs.py --wifi` | The "one set of credentials, applied to a rack of devices" use case | The use case is preserved. The mechanism changes: v1 baked credentials into a LittleFS partition image and re-flashed it over USB (device halted). Track 3 talks Improv to running devices via UART — same end state, no flash required, generalises to any-firmware-with-Improv. | | `improv/improv` (ESP Component Registry; source: `improv-wifi/sdk-cpp` on GitHub) | The Improv protocol parser + callbacks | Standard upstream library. We don't reimplement the protocol; we install the listener task that feeds it bytes. | | [src/platform/esp32/platform_esp32.cpp:870-891](src/platform/esp32/platform_esp32.cpp#L870-L891) (`http_fetch_to_ota` task) | The xTaskCreate + status-buffer pattern | Improv's listener task is identical shape: heap struct + `xTaskCreate` + status-buffer ownership. Reuses the pattern, not the code. | @@ -474,7 +474,7 @@ Added under plan-18 (Option A — "cheap + extract parser"): - **`test/test_improv_frame.cpp`** — 13 cases / 216 assertions over the Improv framing layer. Parser feed-byte-at-a-time, bad-checksum detection, oversize-length rejection, resync on garbage, the "stray 'I' restarts the magic search" edge case, builder/parser round-trip across all four frame types, back-to-back frames. The parser was extracted into [src/core/ImprovFrame.h](../../src/core/ImprovFrame.h) precisely to make this test cheap (no MCU, no `improv/improv` host port). The ESP32 task at [platform_esp32.cpp::improvTask](../../src/platform/esp32/platform_esp32.cpp) now consumes the same parser, so the unit test covers the bytes-in path that runs on-device. - **`test/test_network_module.cpp`** — 4 cases on `NetworkModule::setWifiCredentials`: SSID + password copy, dirty-flag toggling, null-SSID no-op, null-password tolerance, oversize-SSID truncation. The desktop `wifiStaInit` stub returns false safely so the test runs without bringing up a radio. This is the bridge Improv uses to hand credentials to the network state machine. -- **`scripts/build/improv_provision.py --self-test`** — host-side framing/payload round-trip. No serial port needed; re-runnable in CI. Catches a regression in the Python frame builder before any device is involved. +- **`moondeck/build/improv_provision.py --self-test`** — host-side framing/payload round-trip. No serial port needed; re-runnable in CI. Catches a regression in the Python frame builder before any device is involved. Documented gaps (not covered by automated tests, called out so reviewers know what hardware verification is buying): diff --git a/docs/history/plans/Plan-20260620 - Improv-as-REST - push device-model config over serial.md b/docs/history/plans/Plan-20260620 - Improv-as-REST - push device-model config over serial.md index 4ad1ec26..ccd813ee 100644 --- a/docs/history/plans/Plan-20260620 - Improv-as-REST - push device-model config over serial.md +++ b/docs/history/plans/Plan-20260620 - Improv-as-REST - push device-model config over serial.md @@ -49,7 +49,7 @@ Three confirmed seams make this a wiring job, not new infrastructure: ## Verification -- **Host:** `cmake --build build` (0 warnings), `ctest` (incl. the new apply-core test), `uv run scripts/scenario/run_scenario.py`, `check_specs.py`, `check_platform_boundary.py`, `check_devices.py`. +- **Host:** `cmake --build build` (0 warnings), `ctest` (incl. the new apply-core test), `uv run moondeck/scenario/run_scenario.py`, `check_specs.py`, `check_platform_boundary.py`, `check_devices.py`. - **ESP32 build** (`build_esp32.py --firmware esp32s3-n16r8` + `esp32` classic) — compiles the new vendor RPC under `-Werror`. - **Serial APPLY_OP probe** — send a hand-built `APPLY_OP` frame to a connected S3, confirm the op applies (e.g. set Grid width 8) + the device acks. Pins the wire contract without the browser. - **Real install (the actual fix):** from the local preview, flash the S3 with erase + apply-defaults; confirm it comes up as 8×8 + AudioSpectrum + RandomMap with the serial monitor showing the ops applied, **no handoff link involved**. Repeat on the P4. Confirm no duplicate AudioModule. Serial push is now the *only* install-time path, so preview and deployed behave identically — the HTTPS-vs-HTTP difference that caused the original bug no longer exists. diff --git a/docs/history/plans/Plan-20260621 - Improv frame-contract unit tests (pytest + node-test).md b/docs/history/plans/Plan-20260621 - Improv frame-contract unit tests (pytest + node-test).md index 338f47c9..7d829671 100644 --- a/docs/history/plans/Plan-20260621 - Improv frame-contract unit tests (pytest + node-test).md +++ b/docs/history/plans/Plan-20260621 - Improv frame-contract unit tests (pytest + node-test).md @@ -5,7 +5,7 @@ MoonDeck (Python) and the web installer (JS) have **no unit tests** today (no pytest dep, no package.json). The single highest-value target is the **Improv wire frame**, implemented **three times** that must agree byte-for-byte: - device C++ — `src/core/ImprovFrame.h` (parser) + `src/platform/esp32/platform_esp32_improv.cpp` (handlers) -- Python — `scripts/build/improv_provision.py::build_frame` / `checksum` +- Python — `moondeck/build/improv_provision.py::build_frame` / `checksum` - installer JS — `docs/install/install-orchestrator.js::buildImprovFrame` + the APPLY_OP chunker All three use `IMPROV` magic + version 1 + type + length + payload + **sum-mod-256** checksum (verified: `ImprovFrame.h:115`, JS `& 0xff`, Py `& 0xFF`). Drift here silently breaks provisioning. Pin it with a **golden vector** asserted on both the Python and JS sides (and hand-checked against C++). @@ -26,7 +26,7 @@ All three use `IMPROV` magic + version 1 + type + length + payload + **sum-mod-2 4. **New** `test/python/test_improv_frame.py` — pytest over `improv_provision.build_frame`/`checksum` (imported via `sys.path`), same golden vector. (`import serial` is already lazy/`try`-guarded, so the import is clean without pyserial.) 5. **Edit** `pyproject.toml` — add `pytest` dev dependency. 6. **New** `.github/workflows/test.yml` — PR-triggered: `uv run pytest test/python` + `node --test test/js`. -7. **Edit** `CLAUDE.md` Event 1 (commit gates) — add the pytest + node:test step (trigger: `scripts/**`, `docs/install/**`, `test/python/**`, `test/js/**` changed). +7. **Edit** `CLAUDE.md` Event 1 (commit gates) — add the pytest + node:test step (trigger: `moondeck/**`, `docs/install/**`, `test/python/**`, `test/js/**` changed). ## Golden vector diff --git a/docs/history/plans/Plan-20260622 - Non-blocking preview send (shipped).md b/docs/history/plans/Plan-20260622 - Non-blocking preview send (shipped).md index f77fbcb9..d898db6b 100644 --- a/docs/history/plans/Plan-20260622 - Non-blocking preview send (shipped).md +++ b/docs/history/plans/Plan-20260622 - Non-blocking preview send (shipped).md @@ -59,7 +59,7 @@ The preview is a *live view* — one viewer at a time is the real use case, and ## Verification -- **Host:** `cmake --build build` (-Werror), `ctest`, `uv run scripts/scenario/run_scenario.py`, `check_specs.py`, `check_platform_boundary.py`. +- **Host:** `cmake --build build` (-Werror), `ctest`, `uv run moondeck/scenario/run_scenario.py`, `check_specs.py`, `check_platform_boundary.py`. - **ESP32 build** (`esp32p4-eth` + `esp32s3-n16r8` + classic `esp32`). - **Live — find where it breaks (test-first):** websockets probe — sweep Grid 48²→64²→128²→195²→256²→512², recording at each size: WS open?, frame point-count (full vs downsampled), `/api/system` tick, preview fps. Locate the real break point **per board** and set the cap from that. Assert the WS never closes and the tick never stalls. - **Classic board (the key test):** sweep toward 16K+, find where internal RAM / drain-latency forces downsampling, confirm the device degrades-never-crashes. Sets the classic-tier cap. diff --git a/docs/history/plans/Plan-20260624 - Dev-channel update badge.md b/docs/history/plans/Plan-20260624 - Dev-channel update badge.md index aa507d17..c40fd77e 100644 --- a/docs/history/plans/Plan-20260624 - Dev-channel update badge.md +++ b/docs/history/plans/Plan-20260624 - Dev-channel update badge.md @@ -30,9 +30,9 @@ Reuse the existing `-D` override pattern (`MM_FIRMWARE_NAME`/`MM_RELEASE` alread Extend `checkFirmwareUpdate`: if the device version is a prerelease (`parse(...).prerelease.length > 0`) and no stable update is shown, fetch the `latest` release (cache key `projectMM.update.dev.v1`), read its version from `manifest-<firmware>.json` (`.version`), `isNewer(latestDev, deviceVersion)` → badge → click opens Firmware with `latest` pre-selected. Stable update takes precedence. Best-effort, cached, compatible-`.bin` check applies. ## Files -- `scripts/build/compute_version.py` (NEW) + `test/python/test_compute_version.py` (NEW) -- `scripts/build/generate_build_info.py` — `MM_VERSION` overridable `#ifndef` -- `scripts/build/build_esp32.py` — `--version` → `-DMM_VERSION` +- `moondeck/build/compute_version.py` (NEW) + `test/python/test_compute_version.py` (NEW) +- `moondeck/build/generate_build_info.py` — `MM_VERSION` overridable `#ifndef` +- `moondeck/build/build_esp32.py` — `--version` → `-DMM_VERSION` - `.github/workflows/release.yml` — compute V, fetch-depth 0, thread through - `src/ui/app.js` — dev-channel branch - `src/ui/semver.js` tests — `2.1.0-dev.7 > 2.1.0-dev.6`, `2.1.0-dev.1 > 2.1.0-dev` diff --git a/docs/history/plans/Plan-20260624 - Semver version + update-available badge.md b/docs/history/plans/Plan-20260624 - Semver version + update-available badge.md index 54c345f1..c1780f8b 100644 --- a/docs/history/plans/Plan-20260624 - Semver version + update-available badge.md +++ b/docs/history/plans/Plan-20260624 - Semver version + update-available badge.md @@ -19,7 +19,7 @@ On top of that clean version, add a status-bar **"firmware update available" bad ### 1. Semver-clean version (build pipeline + firmware) - `library.json`: version `2.0.0` → `2.1.0-dev`. `build_info.h` is gitignored + generated from this, so `MM_VERSION` follows. -- `scripts/build/verify_version.py`: a stable `vX.Y.Z` tag matches `library.json` **with any prerelease suffix stripped** (so `v2.1.0` ↔ `2.1.0-dev` passes — the release of what was in dev; a wrong *core* like `v2.2.0` ↔ `2.1.0-dev` still fails). Keep the `latest`-skips behaviour. Doc the ritual. +- `moondeck/ci/verify_version.py`: a stable `vX.Y.Z` tag matches `library.json` **with any prerelease suffix stripped** (so `v2.1.0` ↔ `2.1.0-dev` passes — the release of what was in dev; a wrong *core* like `v2.2.0` ↔ `2.1.0-dev` still fails). Keep the `latest`-skips behaviour. Doc the ritual. - `src/core/FirmwareUpdateModule.h` (`setup()`): `version` control = **just `kVersion`** (pure semver). Drop the `(kRelease)` concatenation. Update inline comment + spec doc. - `docs/moonmodules/core/FirmwareUpdateModule.md`: `version` description → "pure semver; a `-dev`/prerelease suffix marks a moving/pre-release build." @@ -38,12 +38,13 @@ On top of that clean version, add a status-bar **"firmware update available" bad - Graceful: any failure → badge hidden, `console.warn` only. ## Files -- `library.json` · `scripts/build/verify_version.py` · `src/core/FirmwareUpdateModule.h` · `docs/moonmodules/core/FirmwareUpdateModule.md` +- `library.json` · `moondeck/ci/verify_version.py` · `src/core/FirmwareUpdateModule.h` · `docs/moonmodules/core/FirmwareUpdateModule.md` - `src/ui/semver.js` (NEW) · `src/ui/index.html` · `src/ui/style.css` · `src/ui/app.js` - `test/js/semver.test.mjs` (NEW) ## Verification -- Host: `node --test "test/js/**/*.test.mjs"`; `node --check` the JS + extract-check index.html; `cmake --build build` + `ctest`; `uv run scripts/scenario/run_scenario.py`; `uv run scripts/check/check_specs.py`; a `test/python` verify_version case (`v2.1.0` ↔ `2.1.0-dev` OK; `v2.2.0` ↔ `2.1.0-dev` fails). + +- Host: `node --test "test/js/**/*.test.mjs"`; `node --check` the JS + extract-check index.html; `cmake --build build` + `ctest`; `uv run moondeck/scenario/run_scenario.py`; `uv run moondeck/check/check_specs.py`; a `test/python` verify_version case (`v2.1.0` ↔ `2.1.0-dev` OK; `v2.2.0` ↔ `2.1.0-dev` fails). - Bench/preview: Firmware card shows clean semver; badge appears on an older device, opens Firmware pre-selected; no badge on newest stable; no error offline. ## Existing releases — already semver-compatible (no migration) diff --git a/docs/history/plans/Plan-20260626 - Composable modifiers (chain the whole stack).md b/docs/history/plans/Plan-20260626 - Composable modifiers (chain the whole stack).md index c5b3ab4f..31202616 100644 --- a/docs/history/plans/Plan-20260626 - Composable modifiers (chain the whole stack).md +++ b/docs/history/plans/Plan-20260626 - Composable modifiers (chain the whole stack).md @@ -119,7 +119,7 @@ This resolves the Plan-agent's concern: dynamic modifiers do **not** force the f - **Per-frame seam scratch buffer**: the live remap needs a logical-sized scratch (allocated when `hasLive_`, off the hot path). Confirm it degrades gracefully on OOM (fall back to static LUT, no live motion, status warning). ## Verification -- `ctest` + `uv run scripts/scenario/run_scenario.py` green at each commit boundary. +- `ctest` + `uv run moondeck/scenario/run_scenario.py` green at each commit boundary. - New `unit_Layer_modifier_chain` proves Region∘Multiply composes (A∘B ≠ B∘A, disabled-middle skipped). - Live on the bench: build a Region + Multiply(mirror) stack on the S3, confirm the carved-then-mirrored result in the preview; add a Rotate on top, confirm smooth (not stepped) dynamic rotation with the static chain underneath. - Perf: capture single-layer (no modifier) tick on S3/classic/P4 — must match today (max-speed guarantee); capture a static 2-chain and a dynamic (Rotate) chain to quantify the live-seam cost. diff --git a/docs/history/plans/Plan-20260626 - MoonLive Stage 0 (native codegen spike) (shipped).md b/docs/history/plans/Plan-20260626 - MoonLive Stage 0 (native codegen spike) (shipped).md index cbdad8e3..ad27a87d 100644 --- a/docs/history/plans/Plan-20260626 - MoonLive Stage 0 (native codegen spike) (shipped).md +++ b/docs/history/plans/Plan-20260626 - MoonLive Stage 0 (native codegen spike) (shipped).md @@ -96,7 +96,7 @@ A bare recursive-descent slice in `src/core/moonlive/` (neutral): tokenize → p ## Validation -- `ctest` + `uv run scripts/scenario/run_scenario.py` green at each step. +- `ctest` + `uv run moondeck/scenario/run_scenario.py` green at each step. - **The hardware acceptance is the point:** S3 solid blue (1a), S3 hue sweep (1b), S3 blue-from-source (2) — eyeballed in the preview, the product owner confirms. - Desktop build zero-warnings (`-Wall -Wextra -Werror`); platform-boundary check passes (all ISA bytes behind `src/platform/`). - `check_specs.py` green — `docs/moonmodules/core/MoonLive.md` written (it is the module's home; carries the neutral engine API, the allocExec contract, the §3.9 boundary, and the ESPLiveScript/ARTI-FX/MoonLight prior-art block staged in the analysis doc). diff --git a/docs/history/plans/Plan-20260627 - MoonLive Stage 1 controls (shipped).md b/docs/history/plans/Plan-20260627 - MoonLive Stage 1 controls (shipped).md index cadaa6a4..777ff434 100644 --- a/docs/history/plans/Plan-20260627 - MoonLive Stage 1 controls (shipped).md +++ b/docs/history/plans/Plan-20260627 - MoonLive Stage 1 controls (shipped).md @@ -174,7 +174,7 @@ under each ISA's scratch count (2-3 controls fine; flag if a statement needs man - **`@effect dim=3D`** role/dimension annotation — Stage 4/5. ## Verification -- `ctest` (unit) + `uv run scripts/scenario/run_scenario.py` (scenarios) green at each **step** +- `ctest` (unit) + `uv run moondeck/scenario/run_scenario.py` (scenarios) green at each **step** boundary (each step is independently testable); host codegen proven (step 4) before device backends (step 7). The full commit-gate set runs once at the end, before the single combined commit. - Disassemble the host + Xtensa + RISC-V output for a control script against the real toolchains; diff --git a/docs/history/plans/Plan-20260627 - MoonLive Stage 3 (IR seam + assembler, second statement).md b/docs/history/plans/Plan-20260627 - MoonLive Stage 3 (IR seam + assembler, second statement).md index e5e65b33..2d1822ed 100644 --- a/docs/history/plans/Plan-20260627 - MoonLive Stage 3 (IR seam + assembler, second statement).md +++ b/docs/history/plans/Plan-20260627 - MoonLive Stage 3 (IR seam + assembler, second statement).md @@ -79,7 +79,7 @@ The set is **closed under the rest of the ladder**: 2D/3D addressing is more ind ## Validation -- `ctest` + `uv run scripts/scenario/run_scenario.py` green at each step; desktop-first so steps 1–3 are pure in-process. +- `ctest` + `uv run moondeck/scenario/run_scenario.py` green at each step; desktop-first so steps 1–3 are pure in-process. - **Golden-bytes regression** (assembler `fill` == hand blob) is the anchor proving the assembler reproduces hand-quality code. - Hardware: S3 (Xtensa) + P4 (RISC-V) run `setRGB`/`random16` live via the scenario, including the out-of-range-index safety step. - Build zero-warnings; platform-boundary check (assembler bytes behind `src/platform/`); `check_specs` green. diff --git a/docs/history/plans/Plan-20260628 - ESP32-S31 board + IDF pin + System sdkDate (shipped).md b/docs/history/plans/Plan-20260628 - ESP32-S31 board + IDF pin + System sdkDate (shipped).md index 28e68137..593f35d4 100644 --- a/docs/history/plans/Plan-20260628 - ESP32-S31 board + IDF pin + System sdkDate (shipped).md +++ b/docs/history/plans/Plan-20260628 - ESP32-S31 board + IDF pin + System sdkDate (shipped).md @@ -52,7 +52,7 @@ unchanged — the risk is in the *board* layer (Ethernet PHY, pin map, PSRAM mod The pin is named in several files (from the inventory); update **all** so they agree: -- **`scripts/build/setup_esp_idf.py`** — `PINNED_IDF_COMMIT` → `0d9287800812c95662921c2c5e812023939e3d58`, +- **`moondeck/build/setup_esp_idf.py`** — `PINNED_IDF_COMMIT` → `0d9287800812c95662921c2c5e812023939e3d58`, `PINNED_IDF_VERSION` → `v6.1-dev-5215-g0d928780081`. **Plus the convergence change:** after the drift warning, prompt "Check out the pinned commit now? [Y/n]" and run `git checkout <PINNED_IDF_COMMIT>` + `git submodule update --init --recursive` when accepted. Add a `--no-checkout` flag to opt out (the @@ -65,7 +65,7 @@ The pin is named in several files (from the inventory); update **all** so they a - **`docs/building.md`** — the clone instructions (`--branch v6.1-dev` → `--branch release/v6.1`), the "Tested IDF version" line, the pinned-commit line, and the "moving to a different release" procedure. Add a one-liner that S31 needs `install.sh esp32s31`. -- **`scripts/MoonDeck.md`** — the `setup_esp_idf` section: document the new checkout-offer behaviour + +- **`moondeck/MoonDeck.md`** — the `setup_esp_idf` section: document the new checkout-offer behaviour + the `install.sh esp32s31` note for S31 builders. **Re-verify gate (do this before Part B):** with the new IDF active, build the existing targets @@ -80,7 +80,7 @@ Follow the established new-firmware pattern (inventory mapped every touch point) the real PSRAM-mode / flash / EMAC symbol names), starting from the P4 (RISC-V) fragment as the closest analog. Touch points: -- **`scripts/build/build_esp32.py`** — add the `FIRMWARES["esp32s31"]` entry: `"chip": "esp32s31"`, +- **`moondeck/build/build_esp32.py`** — add the `FIRMWARES["esp32s31"]` entry: `"chip": "esp32s31"`, `"fragments": ["sdkconfig.defaults", "sdkconfig.defaults.esp32s31"]`, `"eth_only": False`, `"ships": True`, a one-line description. Add `"esp32s31": "ESP32-S31"` to `TARGET_TO_FAMILY` (line 153). Confirm `set-target` (line 505) passes the chip through; the preview target may need @@ -103,7 +103,7 @@ analog. Touch points: `deviceModel` control = the entry name, an LED-driver module with `pins`, and a NetworkModule with the S31 Ethernet config (mirroring the P4 entry's `ethType`/`ethPhyAddr`/`eth*Gpio`). Passes `check_devices.py`. -- **Regenerate the projection** — `uv run scripts/build/generate_firmwares.py --out +- **Regenerate the projection** — `uv run moondeck/build/generate_firmwares.py --out docs/install/firmwares.json` (so `esp32s31` lands in the installer-read JSON; `check_firmwares.py` guards drift). `generate_manifest.py` derives chip family from `TARGET_TO_FAMILY` automatically — no change. @@ -128,9 +128,9 @@ IDF outside `src/platform/`): ## Files (summary) -- IDF pin + convergence: `scripts/build/setup_esp_idf.py`, `.github/workflows/release.yml`, - `docs/building.md`, `scripts/MoonDeck.md`. -- S31 firmware: `scripts/build/build_esp32.py`, `esp32/sdkconfig.defaults.esp32s31` (new), +- IDF pin + convergence: `moondeck/build/setup_esp_idf.py`, `.github/workflows/release.yml`, + `docs/building.md`, `moondeck/MoonDeck.md`. +- S31 firmware: `moondeck/build/build_esp32.py`, `esp32/sdkconfig.defaults.esp32s31` (new), `src/platform/esp32/platform_config.h` (S31 eth pins), `docs/install/deviceModels.json`, `docs/install/firmwares.json` (regenerated), `assets/boards/<s31 photo>` (new). - System sdkDate: `src/platform/platform.h`, `src/platform/{esp32,desktop}/platform_{esp32,desktop}.cpp`, @@ -151,10 +151,10 @@ IDF outside `src/platform/`): ## Verification -- **Part A:** `uv run scripts/build/setup_esp_idf.py` shows no drift warning (installed == new pin); +- **Part A:** `uv run moondeck/build/setup_esp_idf.py` shows no drift warning (installed == new pin); the checkout-offer path moves a deliberately-stale checkout onto the pin. Re-verify gate: `esp32`, `esp32s3-n16r8`, `esp32p4-eth` build zero-warning green on the new IDF. -- **Part B:** `uv run scripts/build/build_esp32.py --firmware esp32s31` builds clean → +- **Part B:** `uv run moondeck/build/build_esp32.py --firmware esp32s31` builds clean → `flash_esp32.py` to `/dev/tty.usbserial-20213420` → device boots, joins WiFi/Eth, serves the UI. Author a MoonLive control script (`uint8_t speed = 50; // @control 0..99 setRGB(speed,0,0,255);`), confirm the slider appears + moves the pixel live with no recompile (RISC-V codegen, P4-proven). diff --git a/docs/history/plans/Plan-20260630 - Doc consolidation - per-type compact pages (effects-modifiers-layouts).md b/docs/history/plans/Plan-20260630 - Doc consolidation - per-type compact pages (effects-modifiers-layouts).md index 379c5bf4..e0a10e48 100644 --- a/docs/history/plans/Plan-20260630 - Doc consolidation - per-type compact pages (effects-modifiers-layouts).md +++ b/docs/history/plans/Plan-20260630 - Doc consolidation - per-type compact pages (effects-modifiers-layouts).md @@ -34,7 +34,7 @@ Today: `check_specs.py` rglobs each module `.h` → requires a matching per-modu - **New:** `docs/moonmodules/light/effects.md`, `modifiers.md`, `layouts.md` (compact-row pages with library sections). - **Delete:** the ~17 `docs/moonmodules/light/effects/*.md`, ~5 `modifiers/*.md`, ~4 `layouts/*.md` (folded into the pages). -- **Edit:** `src/main.cpp` (registered `.md` path per effect/modifier/layout → the page), `scripts/check/check_specs.py` (page-scoped control-name check + the type→page map), `scripts/docs/generate_test_docs.py` if it links per-module `.md` anchors, and any doc that links a deleted per-module page (re-point to `effects.md#name`). +- **Edit:** `src/main.cpp` (registered `.md` path per effect/modifier/layout → the page), `moondeck/check/check_specs.py` (page-scoped control-name check + the type→page map), `moondeck/docs/generate_test_docs.py` if it links per-module `.md` anchors, and any doc that links a deleted per-module page (re-point to `effects.md#name`). - **Unchanged:** `src/light/effects/*.h` etc. (code), `docs/moonmodules/light/drivers/*.md` (per-driver), `docs/moonmodules/core/*`. ## Origin sections (from each module's `tags()` today) diff --git a/docs/history/plans/Plan-20260702 - Docs Phase 1+2 (nav fold + tests in build).md b/docs/history/plans/Plan-20260702 - Docs Phase 1+2 (nav fold + tests in build).md index fe78b674..57ac27f9 100644 --- a/docs/history/plans/Plan-20260702 - Docs Phase 1+2 (nav fold + tests in build).md +++ b/docs/history/plans/Plan-20260702 - Docs Phase 1+2 (nav fold + tests in build).md @@ -10,10 +10,10 @@ Phase 0's `nav:` already splits User guide vs Developer reference and excludes ` ## Phase 2 — fold test-doc generation into the build + surface tests per module -Two deliverables, both leaning on existing infrastructure (`scripts/docs/_test_metadata.py` already parses tests and feeds both the doc generator and MoonDeck; `render_unit_tests()`/`render_scenarios()` in `generate_test_docs.py` are importable pure functions; `cases_for_module()`/`paths_for_module()` already return per-module test data). +Two deliverables, both leaning on existing infrastructure (`moondeck/docs/_test_metadata.py` already parses tests and feeds both the doc generator and MoonDeck; `render_unit_tests()`/`render_scenarios()` in `generate_test_docs.py` are importable pure functions; `cases_for_module()`/`paths_for_module()` already return per-module test data). ### 2a — generate the test pages at build time (stop committing them) -- Add **`mkdocs-gen-files`** (standard MkDocs plugin) with a small `scripts/docs/gen_pages.py` hook that, at `mkdocs build` time, calls the existing `render_unit_tests(collect_unit_files())` / `render_scenarios(collect_scenario_files())` and writes `tests/unit-tests.md` + `tests/scenario-tests.md` into MkDocs' virtual file tree. +- Add **`mkdocs-gen-files`** (standard MkDocs plugin) with a small `moondeck/docs/gen_pages.py` hook that, at `mkdocs build` time, calls the existing `render_unit_tests(collect_unit_files())` / `render_scenarios(collect_scenario_files())` and writes `tests/unit-tests.md` + `tests/scenario-tests.md` into MkDocs' virtual file tree. - **Delete the committed `docs/tests/unit-tests.md` + `docs/tests/scenario-tests.md`** (~25K words leave the repo — pure subtraction). They regenerate fresh on every build. - **Retire the commit-time drift gate** for these files: `generate_test_docs.py --check` (CLAUDE.md Event 1 context) and any CI drift check become unnecessary — the pages can't drift when they're built from source every time. Keep `generate_test_docs.py` itself (CLI still useful for a quick local look, and MoonDeck may reference it), but it no longer *writes into the repo* as the source of truth for the site. Decision to settle in review: keep the CLI writing to `docs/tests/` for non-site consumers, or make it print-only. Leaning: keep it writing (MoonDeck/CLI convenience) but drop the CI `--check` gate, and gitignore `docs/tests/*.md` so a stray local run doesn't dirty the tree. - Add `mkdocs-gen-files` to `build_docs.py`'s inline PEP-723 deps and the CI docs build. @@ -29,12 +29,12 @@ Two deliverables, both leaning on existing infrastructure (`scripts/docs/_test_m - Continues on `next-iteration` (PO's workflow: exercise the branch before merging). No dependency on Phase 0 being *merged* — only on its files, which are on the branch. ## Files -- **New:** `scripts/docs/gen_pages.py` (the mkdocs-gen-files hook). -- **Edit:** `mkdocs.yml` (add `gen-files` [+ `macros` if 2b uses it] plugins; nav tweak), `scripts/docs/build_docs.py` (inline dep), `.gitignore` (`docs/tests/*.md`), `docs/moonmodules/light/effects/effects.md` + `modifiers/modifiers.md` (drop the hand `[Tests]` links, replaced by the injected section), CLAUDE.md (drop the test-doc `--check` gate note), `docs/testing.md` (describe build-time generation). +- **New:** `moondeck/docs/gen_pages.py` (the mkdocs-gen-files hook). +- **Edit:** `mkdocs.yml` (add `gen-files` [+ `macros` if 2b uses it] plugins; nav tweak), `moondeck/docs/build_docs.py` (inline dep), `.gitignore` (`docs/tests/*.md`), `docs/moonmodules/light/effects/effects.md` + `modifiers/modifiers.md` (drop the hand `[Tests]` links, replaced by the injected section), CLAUDE.md (drop the test-doc `--check` gate note), `docs/testing.md` (describe build-time generation). - **Delete:** `docs/tests/unit-tests.md`, `docs/tests/scenario-tests.md` (regenerated at build). ## Verification -- `uv run scripts/docs/build_docs.py` builds; `tests/unit-tests.html` + `tests/scenario-tests.html` exist in `site/` (generated, not from committed source). +- `uv run moondeck/docs/build_docs.py` builds; `tests/unit-tests.html` + `tests/scenario-tests.html` exist in `site/` (generated, not from committed source). - Each effect/modifier page shows its Tests section; a no-test module shows "no tests yet" (none should, after the 24 additions). - 0 broken anchors; the 254 intentional out-of-`docs/` source-link warnings unchanged (Phase-4 marker). - ctest/scenarios/spec-check still green (Phase 2 touches no C++; the deleted `.md` were generated artifacts). diff --git a/docs/history/plans/Plan-20260702 - Docs Phase 3 (drift validation, not snippet de-dup).md b/docs/history/plans/Plan-20260702 - Docs Phase 3 (drift validation, not snippet de-dup).md index b0d9df0a..ee059ad4 100644 --- a/docs/history/plans/Plan-20260702 - Docs Phase 3 (drift validation, not snippet de-dup).md +++ b/docs/history/plans/Plan-20260702 - Docs Phase 3 (drift validation, not snippet de-dup).md @@ -24,7 +24,7 @@ This is the pragmatic Phase 3: it solves the actual "change one thing, the other ## Deliverables -Extend `scripts/check/check_specs.py` (the existing spec-drift gate — already parses `.h` control names and `.md` prose) with two new drift checks: +Extend `moondeck/check/check_specs.py` (the existing spec-drift gate — already parses `.h` control names and `.md` prose) with two new drift checks: ### 3a — control-range drift - Parse each control's numeric bounds from the `.h` (`addUint8("floor", floor, 0, 255)` → `floor: 0–255`; also `addInt`, `addUint16`, etc.). @@ -53,5 +53,5 @@ Extend `scripts/check/check_specs.py` (the existing spec-drift gate — already - No change to rendered docs (checks only); catalog tables/build unaffected. ## Files -- **Edit:** `scripts/check/check_specs.py` (two drift checks), `docs/testing.md` (document them), any `.md` where the checks surface real drift (fix the prose). +- **Edit:** `moondeck/check/check_specs.py` (two drift checks), `docs/testing.md` (document them), any `.md` where the checks surface real drift (fix the prose). - **Maybe new:** `test/python/test_check_specs_drift.py` (unit-test the new checks). diff --git a/docs/history/plans/Plan-20260702 - Docs Phase 4b (Doxide pilot, attempted, abandoned).md b/docs/history/plans/Plan-20260702 - Docs Phase 4b (Doxide pilot, attempted, abandoned).md index de256cf1..2d6b394c 100644 --- a/docs/history/plans/Plan-20260702 - Docs Phase 4b (Doxide pilot, attempted, abandoned).md +++ b/docs/history/plans/Plan-20260702 - Docs Phase 4b (Doxide pilot, attempted, abandoned).md @@ -87,7 +87,7 @@ It's the only option that is simultaneously robust (zero parse errors on our rea **Still true regardless of tool:** autodoc is for **services & infrastructure** only. For catalog module types the card remains the doc (controls are runtime `add()` calls no static tool sees). moxygen *can* show a catalog class's raw C++ members — useful to a developer — but that's a *secondary* developer view, not the user-facing card. ### Phase-4b implementation shape (next commit) -- CI: `apt install doxygen` + `npx moxygen` in the deploy-pages job (and a local `scripts/docs/` wrapper so `build_docs` can run it where doxygen is present; skip gracefully where it isn't, like cxxdox's local-mac gap — but moxygen degrades better since doxygen is brew-installable on mac). +- CI: `apt install doxygen` + `npx moxygen` in the deploy-pages job (and a local `moondeck/docs/` wrapper so `build_docs` can run it where doxygen is present; skip gracefully where it isn't, like cxxdox's local-mac gap — but moxygen degrades better since doxygen is brew-installable on mac). - A `Doxyfile` scoped to a curated `services & infrastructure` file list (core/ + light-base), `GENERATE_XML`, `EXTRACT_ALL`, `JAVADOC_AUTOBRIEF`. - moxygen → `.md` into a `Developer / Source` MkDocs nav section. - Enrich the highest-value headers' comments with `///`/`///<` incrementally (as files are touched), but structure works from plain `//` on day one. diff --git a/docs/history/plans/Plan-20260702 - Docs Phase 4b (source-generated technical docs, superseded by Docs v2).md b/docs/history/plans/Plan-20260702 - Docs Phase 4b (source-generated technical docs, superseded by Docs v2).md index 760c5eed..0210cf92 100644 --- a/docs/history/plans/Plan-20260702 - Docs Phase 4b (source-generated technical docs, superseded by Docs v2).md +++ b/docs/history/plans/Plan-20260702 - Docs Phase 4b (source-generated technical docs, superseded by Docs v2).md @@ -51,8 +51,8 @@ The ordering is strict — push everything into the source that can go there: A module whose entire story fits in `///` comments has **no `.md`** — just its generated API page. ## Files -- **Edit:** `scripts/docs/mkdocs_hooks.py` (Doxygen+moxygen generation + link retargeting), `mkdocs.yml` (nav for the generated API section; moxygen template path), `.github/workflows/release.yml` (`apt install doxygen`), the curated infra `.h` files (progressively `///`-enrich, starting Control.h + a few), the shrunk infra overview `.md` (Control.md → cross-file-only + matrix + lineage), `docs/coding-standards.md` (§ Documentation model: record the inversion + the moxygen mechanism, fill the "autodoc TBD" placeholder), CLAUDE.md pointer. -- **New:** a committed `Doxyfile` (scoped, compact settings) + the moxygen custom template under `scripts/docs/`. +- **Edit:** `moondeck/docs/mkdocs_hooks.py` (Doxygen+moxygen generation + link retargeting), `mkdocs.yml` (nav for the generated API section; moxygen template path), `.github/workflows/release.yml` (`apt install doxygen`), the curated infra `.h` files (progressively `///`-enrich, starting Control.h + a few), the shrunk infra overview `.md` (Control.md → cross-file-only + matrix + lineage), `docs/coding-standards.md` (§ Documentation model: record the inversion + the moxygen mechanism, fill the "autodoc TBD" placeholder), CLAUDE.md pointer. +- **New:** a committed `Doxyfile` (scoped, compact settings) + the moxygen custom template under `moondeck/docs/`. ## Verification - Build generates an API page per infra module; a card `.h` link lands on it in-site; `Control` page reproduces Control.md's type reference from `///<`. diff --git a/docs/history/plans/Plan-20260702 - Docs site Phase 0 (MkDocs Material).md b/docs/history/plans/Plan-20260702 - Docs site Phase 0 (MkDocs Material).md index 221e9e07..df9bb21e 100644 --- a/docs/history/plans/Plan-20260702 - Docs site Phase 0 (MkDocs Material).md +++ b/docs/history/plans/Plan-20260702 - Docs site Phase 0 (MkDocs Material).md @@ -10,7 +10,7 @@ End users currently read 259K words of docs as raw `.md` on github.com — no la 1. **`mkdocs.yml`** at repo root — Material for MkDocs, instant search, `pymdownx.snippets` (for later phases), and a `nav:` tree imposing a top-down end-user → developer order over the *existing* files. No file is moved or rewritten; `nav:` just orders what's there. `history/` and `backlog/` stay out of the published nav (internal). 2. **`docs/index.md`** — the landing page end users lack: what projectMM is → a prominent "Flash an ESP32" call-to-action routing to `/install/` → first-light → effects → developer drill-down. Preserves every affordance of the old `docs/landing/index.html` (Flash button, GitHub/Releases links) but as the docs home, with Docs/Getting-started links now resolving to *rendered* pages instead of github.com blobs. -3. **`scripts/docs/build_docs.py`** — a PEP-723 (`# /// script`) uv wrapper around `mkdocs build`, matching the existing `scripts/docs/` convention and the uv-everywhere rule. (As shipped: CI runs a plain `build`, which fails on missing pages / bad nav; broken links and stale anchors stay `warn`-level per the `validation:` block in `mkdocs.yml`. `--strict` is a *local* anchor-audit option, documented in `scripts/MoonDeck.md`, not the CI gate — the intentional out-of-`docs/` source links would make `--strict` fail the build.) +3. **`moondeck/docs/build_docs.py`** — a PEP-723 (`# /// script`) uv wrapper around `mkdocs build`, matching the existing `moondeck/docs/` convention and the uv-everywhere rule. (As shipped: CI runs a plain `build`, which fails on missing pages / bad nav; broken links and stale anchors stay `warn`-level per the `validation:` block in `mkdocs.yml`. `--strict` is a *local* anchor-audit option, documented in `moondeck/MoonDeck.md`, not the CI gate — the intentional out-of-`docs/` source links would make `--strict` fail the build.) 4. **CI wiring** — the existing `deploy-pages` job (in `.github/workflows/release.yml`, gated to `main`) builds the MkDocs site into `pages/` root **instead of** copying the single `docs/landing/index.html`. `pages/install/` staging is untouched. Add `docs/**` + `mkdocs.yml` to the workflow's `paths:` so a docs-only change to `main` redeploys. ## Decisions locked (PO) @@ -28,13 +28,13 @@ End users currently read 259K words of docs as raw `.md` on github.com — no la ## Verification -- `uv run scripts/docs/build_docs.py --strict` builds locally with no warnings; every nav entry resolves; search index generates. +- `uv run moondeck/docs/build_docs.py --strict` builds locally with no warnings; every nav entry resolves; search index generates. - Old `docs/landing/index.html` retired (its Flash button + links live on in `docs/index.md`); `check_specs.py` + the doc-generation `--check` still green (Phase 0 touches no specs/tests). - CI `deploy-pages` publishes the rendered docs at `/`, installer still reachable at `/install/`. ## Files -- **New:** `mkdocs.yml`, `docs/index.md`, `scripts/docs/build_docs.py`. +- **New:** `mkdocs.yml`, `docs/index.md`, `moondeck/docs/build_docs.py`. - **Edit:** `.github/workflows/release.yml` (build MkDocs into `pages/` root; add docs paths to `paths:`). - **Delete:** `docs/landing/index.html` (folded into `docs/index.md`) — and drop its `docs/landing/**` from the workflow `paths:`. diff --git a/docs/history/plans/Plan-20260702 - Docs v2 (two-surface module docs).md b/docs/history/plans/Plan-20260702 - Docs v2 (two-surface module docs).md index 273a2f45..67286d3a 100644 --- a/docs/history/plans/Plan-20260702 - Docs v2 (two-surface module docs).md +++ b/docs/history/plans/Plan-20260702 - Docs v2 (two-surface module docs).md @@ -65,7 +65,7 @@ Re-staged (PO): ship a **working system first** (all generated pages + summary p ## Files -- **Edit:** `scripts/docs/gen_api.py` (domain-nested output + module discovery), `scripts/docs/mkdocs_hooks.py` (link retarget), `scripts/check/check_specs.py` (exemption path), `mkdocs.yml` (nav), `.gitignore` (moxygen dirs), `docs/architecture.md` (line ~112), the ~29 `src/**/*.h` (enrich `///`), the 4 catalog summary pages (link retarget). +- **Edit:** `moondeck/docs/gen_api.py` (domain-nested output + module discovery), `moondeck/docs/mkdocs_hooks.py` (link retarget), `moondeck/check/check_specs.py` (exemption path), `mkdocs.yml` (nav), `.gitignore` (moxygen dirs), `docs/architecture.md` (line ~112), the ~29 `src/**/*.h` (enrich `///`), the 4 catalog summary pages (link retarget). - **New:** 3 summary pages (`core/ui/`, `core/supporting/`, `light/supporting/`). - **Delete:** ~29 per-module `.md` (12 core + `ui.md` + 10 light-supporting + 7 catalog-detail — as each is absorbed), `docs/poc/`. diff --git a/docs/history/plans/Plan-20260704 - File Manager drag-drop + download + usage bar.md b/docs/history/plans/Plan-20260704 - File Manager drag-drop + download + usage bar.md new file mode 100644 index 00000000..975507ee --- /dev/null +++ b/docs/history/plans/Plan-20260704 - File Manager drag-drop + download + usage bar.md @@ -0,0 +1,59 @@ +# Plan — File Manager: desktop drag-drop (tier 1) + filesystem usage bar in the panel + +## Context +Two follow-ups on the shipped File Manager (PR #36): +1. **Drag-and-drop from the desktop filesystem** — tier 1 only: text/config/`.ml` files ≤ `kFileApiCap` (8 KB). Drop onto a tree folder → upload via the existing `/api/file` write endpoint. +2. **Filesystem usage bar in the panel** — the LittleFS used/total progress bar currently on the FilesystemModule card moves *visually* below the tree in the File Manager. FilesystemModule keeps owning + computing it (it owns the fs mount); the File Manager just renders it, read from `/api/state`. The bar is hidden on the FilesystemModule card so it lives in one place. + +## Item 1 — Drag-drop tier 1 + +### Backend: byte-exact write (fixes the NUL-truncation for real) +- `HttpServerModule.cpp` POST route (line ~200): replace `std::strlen(body)` with the true body length. `headerEnd` + `totalRead` are in scope: `size_t bodyLen = headerEnd ? (size_t)(totalRead - (int)(body - buf)) : 0;` — pass that to `handleWriteFile`. This makes writes byte-exact (a body with an embedded NUL no longer truncates), which the shipped editor Save also benefits from. +- With byte-exact writes, the editor's binary **read-only guard** stays (a `<textarea>` still can't safely round-trip binary), but the *write path* is now correct for any bytes a client sends. Keep the guard; it's about the textarea, not the endpoint. +- No new endpoint — `/api/file?path=<dir>/<name>` POST already creates/overwrites. + +### UI: drop handler on tree folder rows +- In `renderFileManager`, add `dragover`/`dragleave`/`drop` listeners on each **folder** `rowEl` (and the root tree container, so a drop on empty space targets root). +- `dragover`: `e.preventDefault()` + add a `.fm-row--drop` highlight class so the target is obvious. +- `drop`: `e.preventDefault()`, for each `File` in `e.dataTransfer.files`: + - **Size guard (tier 1):** if `file.size > 8192`, skip it with a visible note (status text / alert) — tier 1 is text/config only; binary/large is backlogged. Log what was skipped (no silent truncation, per the principles). + - Read via `await file.text()` (tier 1 = text), POST to `/api/file?path=<joinFsPath(folderPath, file.name)>`. + - On success, `st.expanded.add(folderPath)` to reveal it; after all files, `renderFileManager` to re-list. +- Reuse `joinFsPath`, the existing POST shape from `＋ file`/editor Save. Extract a small `fmUploadFile(destDir, file)` helper so the drop loop stays readable. +- CSS: `.fm-row--drop { outline: 2px dashed var(--accent); }` (or a background tint). + +### UI: per-file download (device → desktop) +- True drag-*out* is not portable (browser `DownloadURL` is Chrome-only + needs contents up-front). The standard equivalent is a **download link**: `/api/file?path=…` GET already serves the file contents + length, so a per-row `⤓` is `<a href="/api/file?path=<childPath>" download="<name>">` — forces a save-to-desktop with the right filename, every browser, any file type, **zero backend change**. +- Add a `⤓` button/link to each **file** row (next to the existing per-row affordances), styled like `.fm-del`. Folders get no `⤓` (folder-as-zip is backlogged — needs a bundled client-side zip lib + recursion, a real app.js/flash cost). + +### Not doing (tier 1 scope guards) +- No binary/large (>8KB) upload — `file.text()` + size cap; a too-big or binary file is skipped with a note. +- No recursive folder drops — `dataTransfer.items` webkitGetAsEntry() recursion is tier 3. +- No chunked write — the 8KB cap keeps it a single POST. +- No folder download (zip) — needs a bundled zip lib + recursion; backlogged next to folder-upload, both flagged for flash cost. + +## Item 2 — Filesystem usage bar below the tree + +### FilesystemModule: hide the control on its own card +- `FilesystemModule.cpp` onBuildControls: after `addProgress("filesystem", ...)`, `controls_.setHidden(controls_.count()-1, true)` — same generic-hidden pattern the File Manager's op controls use. FilesystemModule still computes/refreshes `fsUsedVal_` in loop1s; the control stays in `/api/state` (hidden is a UI-render hint only), so the File Manager can read it. +- `lastSaved` stays visible on the FilesystemModule card (it's persistence-engine state, not a fs-browser concern). + +### File Manager panel: render the bar below the tree +- In `renderFileManager`, after the tree, read the FilesystemModule's `filesystem` progress control from `state` (find the module by type `"FilesystemModule"`, its control named `"filesystem"` → `value` + `total`). +- Render a `<progress value=used max=total>` + a label (mirror `fmtProgressLabel` — "X KB / Y KB"). Reuse the existing progress styling; wrap in a `.fm-usage` row. +- If FilesystemModule or the control isn't present (e.g. desktop with fs total 0), render nothing (graceful). + +### Docs +- `ui.md`: File Manager entry — note the usage bar below the tree + drag-drop (tier 1). FilesystemModule entry — drop the `filesystem` usage bullet (it's shown in the File Manager now), keep `lastSaved`. +- `backlog-core § File Manager follow-ups`: mark drag-drop tier 1 shipped; keep tiers 2/3 (binary/large, folders). The Content-Length write note is now done — remove it. + +## Tests +- `unit_HttpServerModule_apply` or a new small test: a POST to `/api/file` with a body containing a NUL byte writes the **full** length (byte-exact), not truncated at the NUL — pins the `strlen`→`contentLen` fix. (If no socket harness, at minimum a `handleWriteFile` length-path unit check.) +- Drag-drop itself is browser DOM — not unit-testable in ctest; covered by desktop manual smoke + the byte-exact write test. +- `unit_FileManagerModule` unchanged (module surface didn't change). + +## Verification +- Desktop: drag a small `.json`/`.txt`/`.ml` from Finder onto a folder in the tree → appears, editable. Drag a >8KB file → skipped with a note. Usage bar shows below the tree; gone from the FilesystemModule card. +- ESP32-S3: same, on real LittleFS; confirm the usage bar reads sane used/total. +- Gates: desktop build, ctest, scenarios, spec check, ESP32 build, KPI. +- Save plan to `docs/history/plans/Plan-20260704 - File Manager drag-drop + usage bar.md`. diff --git a/docs/history/plans/Plan-20260704 - FileManagerModule (shipped).md b/docs/history/plans/Plan-20260704 - FileManagerModule (shipped).md new file mode 100644 index 00000000..142df392 --- /dev/null +++ b/docs/history/plans/Plan-20260704 - FileManagerModule (shipped).md @@ -0,0 +1,100 @@ +# Plan — FileManagerModule: browse / create / delete / edit the device filesystem + +## Context + +The product owner wants to browse and manage the device's LittleFS filesystem from the web UI, +Windows-Explorer-like: navigate folders, see file name + size, create/delete/edit files and +folders. Today the only filesystem-facing module is **FilesystemModule** — but that is the +*persistence engine* (writes `/.config/<Type>.json`, loads at boot, reconciles the module tree). +A file *manager* is a different job; merging them would repeat the v1 `StatefulModule` +"one class, five jobs" anti-pattern (decisions.md). + +**Product-owner decisions (2026-07-04):** +- **Two modules to start.** New `FileManagerModule` for browse/create/delete/edit; FilesystemModule + stays the untouched persistence engine. *Phasing note:* a later merge stays open **if evidence + warrants** (e.g. the manager wants to edit the `/.config/*.json` the engine owns) — but not up + front, so we avoid conflating infrastructure with a feature. +- **Breadcrumb drill-in navigation**, not the always-expanded Explorer tree. The full hierarchy is + reachable (click a folder to enter, breadcrumb to go up); one directory shown at a time. The + always-expanded tree from the reference screenshot is a **follow-up** once the plumbing is proven. +- **Text/config file editing, size-capped.** Edit text files up to a cap (a few KB — the + `/.config` JSONs, small scripts) via a textarea + `fsWriteAtomic`. Binary/oversized files show + (name/size) but aren't edited. +- **Dates/NTP backlogged.** Show name + size now (both available). Real "last modified" needs a + time source (NTP/SNTP) AND LittleFS mtime storage — a separate backlog item; the column shows + "—" / is omitted until then. + +## Files + +1. **`src/platform/platform.h` + platform impls** — the listing seam needs **size** (today + `FsListCb = void(*)(const char* name, bool isDir, void* user)` has no size). Extend it: + `using FsListCb = void(*)(const char* name, bool isDir, uint32_t sizeBytes, void* user);` and + fill `sizeBytes` in both impls (ESP32 LittleFS `stat`, desktop `std::filesystem::file_size`; + dirs report 0). This is the only platform-layer change. (Keep it a *listing* callback — no new + per-file `fsStat` seam unless a caller needs a single stat, which this feature doesn't.) + +2. **New `src/core/FileManagerModule.h` (+ `.cpp` if bodies grow)** — a domain-neutral core + module. `role() → Peripheral` (added via UI / boot-wired near System — decide at wiring; NOT the + persistence engine). Holds the **current directory path** as state; exposes: + - A `path` control (breadcrumb / current dir; read-only display + navigation via the actions). + - A **List control** backed by a `FileListSource : ListSource` — `writeListRow` emits + `{name, isDir, size}` per entry in the current dir (via `platform::fsList`), `writeListRowDetail` + adds nothing extra for now. This is the DevicesModule ListSource pattern exactly. + - Actions (buttons / API): **enter** a subdir, **up** a level, **mkdir**, **rm** (file or empty + dir), **read** a file (into an editor buffer), **write** a file (atomic). Bounded + robust: a + bad path, a too-large file, a non-empty dir delete all fail cleanly with a status, never crash + (the Robustness rule). + - Status line reports the current dir + the last action result. + - **Complexity stays in core / the module stays simple:** the recursive/parse-heavy work already + lives in `platform::fs*` + the ListSource; the module is "list this dir, do this one op." + +3. **HTTP API** — the browse/read/write/mkdir/rm operations need endpoints the UI calls. Reuse the + existing `/api/control` + List-control refresh where it fits (navigation = a control write that + changes `path` and rebuilds the list); file **read**/**write** need a small dedicated path + (`GET /api/file?path=…` → contents, `POST /api/file` → atomic write) since a file body isn't a + control value. Keep these thin and transport-only; the actual fs work is `platform::fs*`. Guard: + path-sanitise (no `..` escape outside the mount root), size-cap the write. + +4. **UI (`src/ui/app.js` + CSS)** — a File Manager view for the module: a breadcrumb bar (current + path, click a crumb to jump up), a row list (folder/file icon, name, size; click a folder to + enter, click a file to open the editor), a create-folder + create-file affordance, a + delete affordance per row, and a modal/inline **text editor** (textarea, Save = atomic write, + size-capped, read-only for binary/oversized). Modern + intuitive; this is the one genuinely + *custom* (non-generic) UI in the cut — justified because a file manager can't be a generic + control grid. Keep it a recognisable master-detail list, not a bespoke tree. + +5. **Docs** — `ui.md` File Manager entry (summary + the controls/actions); moxygen page from the + `.h` `///` comments. A backlog note for the follow-ups (Explorer tree, NTP+mtime dates). + +6. **Tests** — `unit_FileManagerModule`: list a dir (name/size/isDir), mkdir, create + read-back a + file, delete a file, delete-non-empty-dir rejected, path-traversal (`../`) rejected, oversized + write rejected, list a missing dir doesn't crash. Driven on desktop (`fsSetRoot` to a temp dir, + the existing test seam) so the real `platform::fs*` path is exercised. + +## Not doing (scope guards) + +- **No always-expanded Explorer tree** — breadcrumb drill-in now; the tree is a follow-up. +- **No dates / NTP / mtime** — name + size only; backlogged. +- **No binary/large-file editing** — text, size-capped; binary shown not edited. +- **No merge into FilesystemModule** — two modules; merge only later if evidenced. +- **No move/rename/copy** in the first cut (create/delete/edit only) — add once browse+edit lands. + +## Verification + +- Desktop build (`-Werror`) + ESP32 build clean; `ctest` (the new FileManager tests, driven via + `fsSetRoot` temp dir); scenarios green; `check_specs` (control↔doc), `check_devices` if a board + gains the module. +- Bench: on a real ESP32, browse `/`, enter `.config`, open a `<Type>.json`, see name+size, edit a + small text file and confirm the write survives (read back / reboot). Create + delete a folder. +- Robustness: `../` path rejected, non-empty-dir delete rejected, oversized write rejected — each a + clean status, no crash. + +## Follow-ups (backlog) + +- **Explorer-style expandable tree** (nested folders, expand/collapse, details pane) — the + presentation upgrade over breadcrumb drill-in. +- **Real "last modified" dates** — an NTP/SNTP time seam + LittleFS mtime storage (LittleFS doesn't + store mtime by default; needs the attribute API). Both a separate item. +- **Move / rename / copy** operations. +- **Possible FileManager ↔ FilesystemModule convergence** — revisit once the manager is used + against the `/.config` files, if a single "files" surface proves cleaner than two modules. diff --git a/docs/history/plans/Plan-20260704 - Streamed file upload+download (any size).md b/docs/history/plans/Plan-20260704 - Streamed file upload+download (any size).md new file mode 100644 index 00000000..75dc9e0b --- /dev/null +++ b/docs/history/plans/Plan-20260704 - Streamed file upload+download (any size).md @@ -0,0 +1,76 @@ +# Plan — streamed file upload (any size, fixed small buffer) + +## The bottleneck (measured) +Uploads are limited by `HttpServerModule`'s `uint8_t buf[2048]` — it holds the WHOLE request +(headers + body), so a body can't exceed ~1.8 KB (the UI's 8 KB guard is optimistic; the real +cap is smaller). `kFileApiCap = 8192` bounds the *buffered* read/write helpers but never actually +binds for uploads because the 2 KB buffer bites first. `fsWriteAtomic` already writes via a temp +file (fopen/fwrite/fsync/rename) — so the DEVICE never needs the whole file in RAM; only the HTTP +layer's whole-request buffering forces it. Fix: stream the body to the file, never buffering it. + +## Design — a pull-based streaming atomic write (fits the existing seam) + +Rather than leak a stateful `FILE*` across the platform seam (open/write/close/abort — 4 calls, +error-prone), add ONE seam function that keeps the atomic temp-file dance in core and pulls data +from a caller source until exhausted. Recognizable "sink pulls from source" streaming shape. + +**platform.h (new):** +``` +// Streamed atomic write: open a temp file, pull chunks from `src` until it returns 0, then +// atomically rename into place. `src(buf, cap, user)` fills up to cap bytes, returns the count +// (0 = end). Returns false (and discards the temp file) on any write/short-read/rename failure. +using FsWriteSrc = size_t(*)(char* buf, size_t cap, void* user); +bool fsWriteStream(const char* path, FsWriteSrc src, void* user); +``` +**platform_esp32_fs.cpp / platform_desktop.cpp:** same fopen(tmp,"wb") → loop { n = src(chunk, +sizeof chunk, user); if !n break; fwrite } → fflush/fsync/fclose → rename. A fixed local +`char chunk[1024]` — bounded RAM regardless of file size. `fsWriteAtomic` stays (small callers use +it); `fsWriteStream` is the large/streamed path. (Could re-express fsWriteAtomic on top of +fsWriteStream later; not now — concrete-first, don't churn the working small-write path.) + +## HTTP layer — stream the /api/file POST body to the file + +The trick: route EARLY (on headers) for this one endpoint, before the "buffer whole body" step. +In the request loop, once `\r\n\r\n` is seen and the method+path parse to `POST /api/file`: +- Parse Content-Length. Reject > a sane ceiling (`kUploadMax`, e.g. 256 KB — a guard against a + runaway/hostile upload filling LittleFS; the write still fails cleanly if the FS is full, but a + ceiling keeps a single request bounded). NOT 8 KB — that cap goes away for uploads. +- Any body bytes already in `buf` after the header terminator are the FIRST chunk; the source + callback yields those, then reads the rest straight off the socket (`conn.read`) in ≤1 KB chunks, + tracking remaining = Content-Length, with the same bounded-wait patience as today. +- `fsWriteStream(path, srcFromSocket, &ctx)` → 200 `{"ok":true}` / 400 / 500. +- Everything else (control/modules/wled JSON POSTs — all small) keeps the existing buffered path + untouched. Only `/api/file` POST gets the streaming branch. + +`handleWriteFile` (the buffered version) can stay for the editor Save (small text) OR both routes +funnel through the streamed path. Simplest: the streamed path handles ALL `/api/file` POSTs (editor +Save included — it's just a small stream), and the old `strlen`/buffered `handleWriteFile` is +removed. Net: one write path, no size cap besides `kUploadMax`. (Subtraction — deletes the buffered +special-case + the kFileApiCap-on-write.) + +## UI +- Raise/remove `FM_UPLOAD_CAP` — set it to the new `kUploadMax` (256 KB) or drop the client guard + and let the device 413/400 a too-big file with a message. Keep reading as text for now (tier 1 + was text); a follow-up reads binary via ArrayBuffer. Actually: `file.text()` handles up to any + size fine client-side — the cap was the device's. So just raise FM_UPLOAD_CAP to kUploadMax. +- The editor's binary read-only guard stays (textarea can't round-trip binary). + +## kFileApiCap (the READ/serve side) +- Downloads + editor-load still use `static char fileBuf[kFileApiCap+1]` (8 KB) — a file larger + than 8 KB currently serves truncated. Since uploads can now exceed 8 KB, a >8 KB file could exist + and would DOWNLOAD truncated. So the read/serve path needs the same streaming treatment OR a + bigger cap. Cleanest symmetric fix: stream the file → socket on GET too (read chunk → conn.write), + no `fileBuf`. Do this in the same change so up/down are symmetric and neither truncates. + +## Tests +- `fsWriteStream` round-trip (a multi-chunk source writes the full content, incl a NUL); a source + that errors mid-stream leaves no partial file (atomic). Desktop unit. +- Byte-exact NUL test already covers content integrity; extend for >1 chunk. + +## Gates + verify +- Build/ctest/scenarios/spec/esp32/kpi. Bench: upload the 19.6 KB deviceModels.json to the S3, + re-download it, diff — must be byte-identical. Confirm a large file both up- and downloads whole. + +## Open question for PO +- `kUploadMax` ceiling: 256 KB? (LittleFS state partition is ~384 KB–2 MB depending on board; a + single config file is realistically < 64 KB. 256 KB is generous but bounded.) diff --git a/docs/history/plans/Plan-20260705 - Homebridge MQTT control.md b/docs/history/plans/Plan-20260705 - Homebridge MQTT control.md new file mode 100644 index 00000000..cafdd6cd --- /dev/null +++ b/docs/history/plans/Plan-20260705 - Homebridge MQTT control.md @@ -0,0 +1,203 @@ +# Plan — Homebridge (MQTT) control + shared on/off, HomeKit color-wheel → palette + +## Context + +A user wants **Homebridge** to control projectMM devices over **MQTT** (the `homebridge-mqttthing` +plugin). Today the device is controllable from the WLED native app and IR, but has no MQTT service. +We build our **own** MQTT 3.1.1 client — no libraries — fully test-guarded with golden-vector frame +tests, the same way `ImprovFrame.h` is pinned. Home Assistant is a **separate later increment** +(via the existing WLED `/json` shim, ~12 lines — HA's WLED integration reuses the same API the WLED +app does); it is *mentioned* here, not designed. + +Underpinning this is a **shared `on` control on the Drivers module** — independent of brightness — +that IR, MQTT, the WLED app, and (later) HA all drive through the one apply-core +`Scheduler::setControl(module, control, json)`. Adding it lets us **delete** the WLED shim's on/off +fudge (a genuine subtraction). The user's on/off request thus lands once and is reused everywhere. + +For palette control, HomeKit has no "palette" concept but has a native **color wheel** — so the +HomeKit hue drives a **nearest-palette-by-hue** selection (each built-in palette carries a +representative hue; incoming hue picks the closest). The color wheel becomes a natural palette +selector, no non-native control on the HomeKit tile. + +**Decisions locked (product owner):** hand-rolled MQTT (no libs); controls = on/off + brightness + +palette; palette via HomeKit color wheel using nearest-palette-by-hue; broker accepts a **hostname** +(add DNS `getaddrinfo` to the platform layer — first DNS use, a reusable primitive); `MqttPacket.h` +lives in **`src/core/`** (a control transport, sibling of `ImprovFrame.h`/`WledPacket.h`; the pixel +formats ArtNet/DDP/E131 live in `light/`). Order: on/off + Homebridge first; HA later. + +## Increments (core deliverable = 1–3; palette-by-hue = 4; HA deferred) + +1. **`on` control on Drivers** + WLED-shim subtraction + tests — foundation, independently useful + (improves the existing WLED app immediately). +2. **IR on/off learnable toggle** — small, rides on #1. +3. **Platform `TcpConnection::connect(host,port)` (with DNS) + `MqttPacket.h` + `MqttModule` + (on/off + brightness)** + Homebridge config — the core deliverable. +4. **Palette over MQTT via HomeKit color wheel** — `nearestPaletteForHue` + HSV topic mapping. +5. *(Deferred, not designed here)* HA via the WLED `/json` shim. + +## Files + +### New +| File | Why | +|---|---| +| `src/core/MqttPacket.h` | MQTT 3.1.1 wire format: constants + inline `build*`/`parse*` + a byte-at-a-time inbound parser, the `ImprovFrame.h` shape. In `core/` — a control transport (sibling of `ImprovFrame.h`/`WledPacket.h`), not a pixel format. | +| `src/core/MqttModule.h` + `.cpp` | The service module (`.h`+`.cpp` per the core convention; it has real socket-lifecycle logic). | +| `test/unit/core/unit_MqttPacket.cpp` | Golden-vector + round-trip, mirrors `unit_ImprovFrame.cpp`. | +| `test/unit/core/unit_MqttModule.cpp` | Subscribe→apply routing via an injected inbound byte feed + FakeDrivers. | +| `docs/moonmodules/core/MqttModule.md` | The `registerType` doc; carries the Homebridge how-to snippet. | + +### Edited +| File | Change | +|---|---| +| `src/light/drivers/Drivers.h` | Add `bool on = true` control; gate the correction LUT on it (§2). | +| `src/light/Palette.h` | Add representative-hue-per-palette + pure `nearestPaletteForHue(hue)→index` (§4). | +| `src/core/IrModule.h` | Add a `Toggle` action kind + an `on/off` action row (§3). | +| `src/core/HttpServerModule.cpp` | **Subtraction**: `applyWledState` writes the real `on`; `writeWledStateBody` reads it; add `driversOn()` beside `driversBrightness()` (§2). | +| `src/platform/platform.h` + desktop + esp32 impls | `TcpConnection::connect(host,port,timeoutMs)` with DNS (§3a). | +| `src/main.cpp` | `registerType<MqttModule>` + construct/inject/`markWiredByCode()`/`networkModule->addChild(mqtt)`. | +| `test/CMakeLists.txt` | Add the two new unit-test sources. | +| `test/unit/core/unit_IrModule.cpp` | Toggle test case (§5). | +| `test/unit/core/unit_HttpServerModule_apply.cpp` | Update WLED on/off expectations to the real-`on` behaviour (§5). | + +## 2. The `on` control (shared foundation) + +Add `bool on = true;` to `Drivers.h` (beside `brightness` at :239); register +`controls_.addBool("on", on);` first in `onBuildControls()` (:274) so it renders at top. Gate the +LUT via one helper both call sites use (no duplicated ternary): +`uint8_t effectiveBrightness() const { return on ? brightness : 0; }`, then +`correction_.rebuild(effectiveBrightness(), lightPreset)` in `onUpdate` (:291) and `setup` (:303), +and add `"on"` to the `onUpdate` branch that matches `brightness`/`lightPreset` (:289) so it rebuilds ++ propagates via the same `onCorrectionChanged()` loop. `controlChangeTriggersBuildState` stays false +→ on/off is as fluent as brightness. `on=false` scales the whole LUT to black +(`briLut[v]=(v*0)/255`, Correction.h:39) while **preserving** the `brightness` value — `on=true` +restores instantly. Textbook "compute the effective value where it's consumed," no shadow variable, +no hot-path touch. + +**WLED subtraction** (HttpServerModule.cpp): add `driversOn()` mirroring `driversBrightness()` +(:991; `ControlType::Bool`, name `"on"`, default `true` if absent). `writeWledStateBody` (:1019): +`driversOn() ? "true" : "false"` instead of `bri>0`. `applyWledState` (:1063-1067): **delete** the +`if(!on)bri=0; else if(bri<0)bri=…128` fudge; replace with a direct +`applySetControl("Drivers","on", on?"{\"value\":true}":"{\"value\":false}")` plus the untouched +`bri` clamp. Net deletion of the restore-128 heuristic + `bri` coupling + `bri<0` sentinel. + +## 3. IR on/off toggle + +Delta can't toggle a bool. Add `enum class ActionKind : uint8_t { Delta, Toggle };` and a +`ActionKind kind = ActionKind::Delta;` field on `Action` (IrModule.h:107) — existing rows keep their +aggregate form, `kind` defaults. Add one row +`{"on/off","code on/off","Drivers","on",0,ActionKind::Toggle}` and `"on/off"` to `kLearnOptions` +(:122; `kActionCount+1` covers it). In `runAction` (:160) branch before the delta math: for +`Toggle`, read `controlIntValue(c)!=0` (a Bool is a 1-byte object, reads fine through the existing +`uint8_t*` path at :195) and `setControl` the inverse. Everything else (learn select, `codeStr_` +persistence) works unchanged — the toggle is just another data row. + +### 3a. Platform `TcpConnection::connect` (with DNS) +MQTT needs a **persistent, non-blocking, outbound** TCP client to a **hostname** — none exists +(all socket paths are inbound-`accept` or IP-only). Add to `platform::TcpConnection`: +`bool connect(const char* host, uint16_t port, uint32_t timeoutMs);` — resolve `host` with +`getaddrinfo` (first DNS use; brokers are named), then the proven non-blocking-connect-with-`select` +block lifted from `httpRequest` (platform_desktop.cpp:597-620 + esp32 equivalent), leaving the +socket **non-blocking** after connect (MQTT uses the existing non-blocking `read()`/`writeSome()`). +Desktop + ESP32 (lwip) impls; caller gates on `networkReady()`. *Follow-up subtraction (not now):* +`httpRequest`'s inline connect could later call this. + +### 3b. `MqttPacket.h` (golden-vector tested, `ImprovFrame.h` shape) +`namespace mm`, dependency-free, inline. Packets: type nibbles (CONNECT/CONNACK/PUBLISH/SUBSCRIBE/ +SUBACK/PINGREQ/PINGRESP/DISCONNECT), proto `"MQTT"` level `0x04`; **remaining-length varint** +encode/decode (the fiddly bit — textbook 7-bit continuation, tested at 0/127/128/16383/16384); +`buildConnect(clientId,user,pass,keepalive,…)`, `parseConnack`, `buildPublish(topic,payload,…)` +(QoS0), `buildSubscribe(packetId,topic,…)`, `parseSuback`, `buildPingreq`, `buildDisconnect`; and an +`MqttInboundParser` byte-at-a-time state machine (like `ImprovFrameParser`) exposing a completed +PUBLISH's topic/payload — the seam that makes the receive path host-testable with no socket. + +### 3c. `MqttModule` (Improv/DevicesModule template) +`MoonModule` subclass, honours `enabled` (a user genuinely disables MQTT), `userEditable()=false`. +Injects `SystemModule` (default prefix = deviceName); reaches Drivers via +`Scheduler::instance()->setControl` (like IR — no HttpServer dependency). **Controls:** `broker` +(Text, hostname/IP), `port` (Uint16, 1883), `username` (Text), `password` (`addPassword` — reuses +the WiFi-password secret serialization, Control.h:320, no new obfuscation), `prefix` (Text, default +`projectMM/<deviceName>`), `mqtt_status` (ReadOnly). **Lifecycle** all on `loop1s()` (off the hot +path): connect lazily gated on `networkReady() && enabled` with reconnect backoff; CONNECT→CONNACK→ +SUBSCRIBE; PINGREQ every keepalive/2 (reconnect if no PINGRESP); drain `read()` into the parser and +route `set` PUBLISHes to Drivers via `setControl`; publish `get` topics on change + on connect (so +mqttthing never shows "No Response"). Fixed-size member buffers; `conn_` is a `TcpConnection` member. + +**Topics** (mqttthing "lightbulb"): +``` +<prefix>/on/set ← "true"/"false" → Drivers.on +<prefix>/on/get → publish current on +<prefix>/brightness/set ← 0..100 → *255/100 → Drivers.brightness +<prefix>/brightness/get → publish brightness*100/255 +<prefix>/hsv/set ← "h,s,v" → hue → nearestPaletteForHue → Drivers.palette (§4) +<prefix>/hsv/get → publish "<rep-hue>,100,<bri%>" for the chosen palette +``` + +## 4. Palette via HomeKit color wheel (nearest-palette-by-hue) + +In `src/light/Palette.h`, alongside `palettes::kBuiltins`: a parallel `constexpr uint16_t` +representative hue per built-in (Lava≈15 red-orange, Ocean≈210 blue, Forest≈120 green, Party≈300 +magenta, Rainbow = a low-sat/degenerate case → default index 0), and a pure, unit-tested +`uint8_t nearestPaletteForHue(uint16_t hue)` — minimal **circular** hue distance (wrap at 360). +**Boundary:** the MQTT module is core and must not `#include "light/Palette.h"`. Resolve by exposing +the conversion through the already-core-reachable `Palettes::` static API (add +`static uint8_t Palettes::nearestForHue(uint16_t)` delegating to the light-side pure function) — the +same way IR/WLED reach light state without a light include. MQTT converts hue→index, then +`setControl("Drivers","palette",{index})`. `hsv/get` publishes the chosen palette's representative +hue so the HomeKit tile snaps to a sensible color. Saturation ignored for now (very-low-sat may map +to index 0). *This is the palette increment; on/off + brightness ship first.* + +## 5. Tests + +| Test | Unit/HW | Pins | +|---|---|---| +| `unit_MqttPacket.cpp` | Unit | Golden vectors CONNECT/SUBSCRIBE/PUBLISH/PINGREQ (byte-exact); PUBLISH round-trip; remaining-length varint at 0/127/128/16383/16384; CONNACK/SUBACK/PINGRESP parse; **fragmented-PUBLISH** feed across `read()` boundaries reassembles. | +| `unit_MqttModule.cpp` | Unit | Rig = Scheduler + FakeDrivers (on/brightness/palette) + `feedForTest(bytes,len)` (mirrors `injectCodeForTest`). PUBLISH `on/set`"false"→on=false; `brightness/set`"50"→127; `hsv/set` a blue hue→Ocean index. No socket/broker. | +| Drivers on/off | Unit | `on=false` → `correction_.briLut[255]==0` **and** `brightness` unchanged; `on=true` → LUT restored to `brightness`. | +| IR toggle | Unit | Learn a code to `on/off`; fire → FakeDrivers.on flips; fire again → flips back. | +| `nearestPaletteForHue` | Unit | red→Lava, blue→Ocean, green→Forest, 359 wraps ≈ 0. | +| WLED apply update | Unit | `applyWledState("{\"on\":false}")` → Drivers.on=false, brightness untouched (was: bri→0). | +| Broker end-to-end | **HW** | mosquitto + homebridge-mqttthing driving a real device. Not ctest. | + +## 6. Homebridge config (user pastes) + +`docs/moonmodules/core/MqttModule.md` carries the minimal `homebridge-mqttthing` "lightbulb" block +(on/off + brightness + the HSV color wheel for palette), prefix `projectMM/<deviceName>`: +```json +{ + "accessory": "mqttthing", "type": "lightbulb", "name": "projectMM", + "url": "mqtt://<broker>:1883", "username": "<user>", "password": "<pass>", + "topics": { + "getOn": "projectMM/MM-3A7F/on/get", "setOn": "projectMM/MM-3A7F/on/set", + "getBrightness": "projectMM/MM-3A7F/brightness/get", "setBrightness": "projectMM/MM-3A7F/brightness/set", + "getHSV": "projectMM/MM-3A7F/hsv/get", "setHSV": "projectMM/MM-3A7F/hsv/set" + }, + "onValue": "true", "offValue": "false" +} +``` +HA gets a one-line mention (works via the WLED `/json` shim, a separate increment). + +## 7. Verification + +1. Desktop build `cmake --build build` (-Wall -Wextra -Werror; the `getaddrinfo`/`connect` path clean + on desktop + ESP32 toolchains). +2. `ctest` (the six unit tests above) + `uv run moondeck/scenario/run_scenario.py` (no regression). +3. Broker/HW: flash a WiFi board → set broker/port/user/pass/enabled in the UI → run `mosquitto` → + `mosquitto_sub -t 'projectMM/#'` to watch state → install homebridge-mqttthing (§6) → toggle + on/off + brightness + color from the Home app; confirm the strip responds, the color wheel snaps + palettes, and the tile never shows "No Response". +4. Platform boundary: all MQTT socket I/O via `platform::TcpConnection` + `networkReady()` only; the + one new socket capability (`connect`+DNS) lives in `src/platform/`; `MqttPacket.h` is pure byte + math, no platform include. + +## 8. Subtraction / principle check + +**Removes:** the WLED on/off fudge (restore-128 heuristic + `bri` coupling + `bri<0` sentinel, +HttpServerModule.cpp:1063-1067). **Adds:** one `on` control (shared by IR/MQTT/WLED/HA — the first +slice of the backlog LightsControl global state); one IR row + `kind` tag; `TcpConnection::connect` ++ DNS (a reusable primitive — any future named-host client, incl. HA push); `MqttPacket.h` + +`MqttModule` (expected domain growth); `nearestPaletteForHue` (a pure light-domain helper); docs + +tests. Mirrors `ImprovFrame.h`/Improv/DevicesModule (*Common patterns first*); MQTT 3.1.1 written +fresh (*Industry standards, our own code*); no hot-path touch (*Data over objects*); every control +live (*No reboot to apply*). Nothing fights a hard rule. + +Save the approved plan to `docs/history/plans/Plan-20260705 - Homebridge MQTT control.md`. diff --git a/docs/history/plans/Plan-20260705 - Rename scripts to moondeck.md b/docs/history/plans/Plan-20260705 - Rename scripts to moondeck.md new file mode 100644 index 00000000..7df34fc6 --- /dev/null +++ b/docs/history/plans/Plan-20260705 - Rename scripts to moondeck.md @@ -0,0 +1,39 @@ +# Plan — Rename `scripts/` → `moondeck/` (+ `moondeck/ci/` split) + +PO-approved 2026-07-05 (refines the 2026-07-02 decision in `docs/backlog/rename-scripts-to-moondeck.md`). Its own isolated commit, next cycle — not folded into feature work. + +## Decision (what ships) + +1. **`scripts/` → `moondeck/`** wholesale. The folder is MoonDeck's home; it holds the console (`moondeck.py`, `moondeck_ui/`, `moondeck_config.json`, `MoonDeck.md`) plus the build/check/scenario/run tooling MoonDeck invokes (28 scripts are MoonDeck cards; the rest are helpers of carded scripts or the build graph MoonDeck triggers). +2. **`moondeck/build/{package_desktop,verify_version}.py` → `moondeck/ci/`.** These two are the *only* truly-CI-only scripts (git-tag verification + release packaging, invoked by `.github/workflows/`, never a MoonDeck action, never build-wired). A `ci/` subfolder labels them accurately so `moondeck/` reads as MoonDeck's world. +3. **Keep function-based subfolders** (`build/`, `check/`, `docs/`, `scenario/`, `run/`, `diag/`, `report/`, `test/`). NOT reorganized by UI tab: the tab→folder map is many-to-many (the `pc` tab draws from 7 folders; `build/` serves `pc`+`esp32`+CI), so tab-based folders would couple code layout to UI layout and force placement choices for multi-tab scripts. The tab a card shows on is declared in `moondeck_config.json` (`"tab"`) — UI grouping is data, not directories. Function grouping is also the recognizable convention (*Common patterns first*). + +### Considered and rejected +- **One folder per UI tab** — couples code to UI, bespoke, fights the many-to-many tab↔folder reality. +- **Split all CI/build-graph scripts out** — `compute_version.py` (CMake-invoked on every build, incl. MoonDeck's Build) and `generate_manifest.py` (imported by `generate_firmwares` ← `build_esp32`) are build-wired, so they must stay in `moondeck/`; only 2 files cleanly separate. +- **Collapse singleton folders** (`diag/`, `report/`, `test/`) — deferred; not part of this rename (keep the diff purely a move). + +## What NOT to rename (the entanglement that justified keeping the name generic — noted, PO kept the rename) +`moondeck/` held more than MoonDeck, but the data shows nearly all of it is MoonDeck's dependency tree; the handful that isn't (CI-only → now `ci/`; the build-graph helpers → stay) is an acceptable minority under a folder named for its dominant purpose. + +## Blast radius (measured, this tree) + +- **Literal `scripts/`**: 29 `.py`, 47 `.md`, 3 `.yml`, 2 CMake — replace `scripts/` → `moondeck/`. +- **Split-path `"scripts"` token** (the gotcha that broke the web-installer sweep): **10 Python test files** use `ROOT / "scripts" / …` — replace the `"scripts"` string token → `"moondeck"`. No JS split-paths. CMake uses the literal `/moondeck/` (covered). +- **`moondeck_config.json`** script paths are relative to the scripts dir (`"build/…"`, `"check/…"`) — unaffected by the folder rename EXCEPT the two moving to `ci/`: `build/package_desktop.py`→`ci/package_desktop.py`, `build/verify_version.py`→`ci/verify_version.py` (only if they get cards — they don't today, so only `.github/workflows` refs change). +- **`.github/workflows/{release,test}.yml`**: `run:` steps + `paths:` filters (`scripts/**` → `moondeck/**`), and the two CI-only paths → `moondeck/ci/…`. +- **False-positive guard**: 161 `scripts`-as-substring hits (`description`, `scripting`, `LiveScript`, `livescripts`) must NOT be touched — anchor every replace to `scripts/` (trailing slash) or the exact `"scripts"` token, never a bare substring. + +## Execution + +Scripted sweep (one reusable, revertible script), then full gate verification: +1. `git mv scripts moondeck` +2. `git mv moondeck/ci/package_desktop.py moondeck/ci/` + `verify_version.py` (mkdir `ci/` first) +3. Anchored replaces: `scripts/`→`moondeck/` in `.py`/`.md`/`.yml`/CMake; `"scripts"`→`"moondeck"` in the 10 test files; `build/package_desktop.py`→`ci/package_desktop.py` and `build/verify_version.py`→`ci/verify_version.py` in `.github/workflows` + any doc that names them. +4. Update `.claude/settings.local.cleaned.json` allow entries pointing at `scripts/…` (the gitignored live file re-accumulates). + +## Verification (all green before commit) +`check_specs`, `check_devices`, `check_firmwares`, `ctest`, scenarios, `pytest test/python`, `node --test test/js/**`, desktop + one ESP32 build (CMake reaches the renamed dir via `find_program`/`execute_process`), MoonDeck starts + loads `moondeck_config.json` (all card paths resolve), docs site builds (`uv run moondeck/docs/build_docs.py`). + +## After +Delete `docs/backlog/rename-scripts-to-moondeck.md` (shipped → git is the record, *Mandatory subtraction*). Mark this plan `(shipped)` once it lands. diff --git a/docs/history/release-notes-v1.0.0.md b/docs/history/release-notes-v1.0.0.md index acd009e6..aa86895c 100644 --- a/docs/history/release-notes-v1.0.0.md +++ b/docs/history/release-notes-v1.0.0.md @@ -64,7 +64,7 @@ Run it, open `http://localhost:8080/`. Teensy, Raspberry Pi, and Linux build fro ## Building from source -Develop and build with **MoonDeck**, the browser dev console (`uv run scripts/moondeck.py`): +Develop and build with **MoonDeck**, the browser dev console (`uv run moondeck/moondeck.py`): ![MoonDeck](https://raw.githubusercontent.com/MoonModules/projectMM/v1.0.0/docs/assets/screenshots/moondeck_pc.png) diff --git a/docs/moonmodules/core/archive/FirmwareUpdateModule.md b/docs/moonmodules/core/archive/FirmwareUpdateModule.md index 543094bf..d3198967 100644 --- a/docs/moonmodules/core/archive/FirmwareUpdateModule.md +++ b/docs/moonmodules/core/archive/FirmwareUpdateModule.md @@ -8,7 +8,7 @@ A thin status surface for OTA flashing. The flash itself is driven by `POST /api | Name | Type | Description | |---|---|---| -| `version` | read-only string | Pure semver (`MM_VERSION`). A stable release is a clean `X.Y.Z` (e.g. `2.0.0`); a moving `latest` build is a monotonic prerelease `<core>-dev.<N>` (e.g. `2.1.0-dev.7`, where `N` is the commit count since the last `vX.Y.Z` tag — see `scripts/build/compute_version.py`), so successive `latest` builds are orderable (semver.org §9/§11); a local/dev build carries library.json's bare `<core>-dev`. The prerelease suffix marks a not-yet-released build; a clean `X.Y.Z` is a stable release. The release channel is derivable from the version (prerelease suffix → not stable), so it is not mixed into this string — the version stays a clean, machine-comparable semver, which the UI's "update available" check compares against the newest GitHub release (stable, and the moving `latest` for devices already on a `-dev` build). | +| `version` | read-only string | Pure semver (`MM_VERSION`). A stable release is a clean `X.Y.Z` (e.g. `2.0.0`); a moving `latest` build is a monotonic prerelease `<core>-dev.<N>` (e.g. `2.1.0-dev.7`, where `N` is the commit count since the last `vX.Y.Z` tag — see `moondeck/build/compute_version.py`), so successive `latest` builds are orderable (semver.org §9/§11); a local/dev build carries library.json's bare `<core>-dev`. The prerelease suffix marks a not-yet-released build; a clean `X.Y.Z` is a stable release. The release channel is derivable from the version (prerelease suffix → not stable), so it is not mixed into this string — the version stays a clean, machine-comparable semver, which the UI's "update available" check compares against the newest GitHub release (stable, and the moving `latest` for devices already on a `-dev` build). | | `build` | read-only string | Build date/time (`MM_BUILD_DATE`). | | `firmware` | read-only string | Build-time firmware variant key from `src/core/build_info.h` (`MM_FIRMWARE_NAME`): `esp32`, `esp32-eth`, `esp32-16mb`, `esp32s3-n16r8`, … for the shipped firmware variants (the full list is the `FIRMWARES` dict in `build_esp32.py`); `desktop-macos-arm64` / `desktop-windows-x64` for packaged desktop binaries; `desktop-dev` for unpackaged local desktop builds. A device carrying the legacy `esp32-eth-wifi` key OTA-maps to `esp32`. Identifies which release asset matches the device — the same key appears in the firmware filenames published by `release.yml`. The compiled binary; the physical hardware it runs on is SystemModule's `deviceModel` control. `install-picker.js`'s `isCompatible()` reads this string. | | `firmwarePartition` | progress (used/total) | Running app image size / total firmware (app) partition size — how full the partition is. Named distinctly from the `firmware` string control so a `controls.find(c => c.name === "firmware")` caller resolves the string, not this progress value. | diff --git a/docs/moonmodules/core/archive/ImprovProvisioningModule.md b/docs/moonmodules/core/archive/ImprovProvisioningModule.md index 93d73196..747eb759 100644 --- a/docs/moonmodules/core/archive/ImprovProvisioningModule.md +++ b/docs/moonmodules/core/archive/ImprovProvisioningModule.md @@ -2,11 +2,11 @@ **What Improv is.** [Improv-Wifi](https://www.improv-wifi.com/) (from Nabu Casa, the Home Assistant / ESPHome company) is an open standard for handing a device its WiFi credentials over a *local* link — USB serial here (it also has a BLE variant) — at the moment it has no network yet. That's the bootstrap chicken-and-egg it solves: a freshly-flashed ESP32 isn't on your WiFi, so you can't reach it over the network to tell it the WiFi password; Improv carries that first handoff over the cable the browser is already connected to from flashing. The name is short for *improvise* — the device has no pre-configured network, so it improvises its first connection from whatever local link is already there. projectMM extends this past credentials with the `APPLY_OP` vendor RPC — "Improv = REST over serial", see below — reusing the same already-there-before-the-network link to push the whole device config (including the deviceModel identity, which is just one of the config controls). -Browser-driven WiFi provisioning over USB-serial, using the [Improv-WiFi](https://www.improv-wifi.com/) protocol. Bridges credentials from a Chrome / Edge / Opera tab — or from `scripts/build/improv_provision.py` for rack/CI use — into `NetworkModule::setWifiCredentials`, which writes the same buffers the AP-fallback UI flow uses. The protocol parser + UART task live in the platform layer; this module is the status surface that polls a ready-flag and bridges credentials to NetworkModule on the scheduler thread. +Browser-driven WiFi provisioning over USB-serial, using the [Improv-WiFi](https://www.improv-wifi.com/) protocol. Bridges credentials from a Chrome / Edge / Opera tab — or from `moondeck/build/improv_provision.py` for rack/CI use — into `NetworkModule::setWifiCredentials`, which writes the same buffers the AP-fallback UI flow uses. The protocol parser + UART task live in the platform layer; this module is the status surface that polls a ready-flag and bridges credentials to NetworkModule on the scheduler thread. A code-wired child of NetworkModule. The wiring calls `markWiredByCode()` so the persistence-apply step preserves the child across reboots even on devices whose `Network.json` predates the addition (see [Persistence — code-wired children](../../../architecture.md#persistence)). -The browser flow runs immediately after a Web Serial flash (ESP Web Tools recognises Improv-capable firmware and offers a "Connect to Wi-Fi?" dialog automatically). The CLI flow uses [`scripts/build/improv_provision.py`](../../../../scripts/build/improv_provision.py) over the same USB cable for headless or rack provisioning. +The browser flow runs immediately after a Web Serial flash (ESP Web Tools recognises Improv-capable firmware and offers a "Connect to Wi-Fi?" dialog automatically). The CLI flow uses [`moondeck/build/improv_provision.py`](../../../../moondeck/build/improv_provision.py) over the same USB cable for headless or rack provisioning. ## Controls @@ -56,10 +56,10 @@ ESP32 tab → pick the device's port → hit **Improv WiFi**. The script reads t ```bash # Reuse the host's currently-joined WiFi (same path as the MoonDeck button): -uv run scripts/build/improv_provision.py --port /dev/tty.usbserial-XXXX +uv run moondeck/build/improv_provision.py --port /dev/tty.usbserial-XXXX # Or override to push a different network's credentials: -uv run scripts/build/improv_provision.py \ +uv run moondeck/build/improv_provision.py \ --port /dev/tty.usbserial-XXXX \ --ssid "MyWiFi" \ --password "hunter2" @@ -70,7 +70,7 @@ For multiple devices on a USB hub: ```bash for port in /dev/tty.usbserial-*; do - uv run scripts/build/improv_provision.py --port "$port" + uv run moondeck/build/improv_provision.py --port "$port" done ``` diff --git a/docs/moonmodules/core/ui/ui.md b/docs/moonmodules/core/ui/ui.md index 27468333..050a0d80 100644 --- a/docs/moonmodules/core/ui/ui.md +++ b/docs/moonmodules/core/ui/ui.md @@ -47,6 +47,59 @@ Detail: [technical](../moxygen/DevicesModule.md) [Tests](../../../tests/unit-tests.md#devicesmodule) +### MQTT + +Bridges the light to an MQTT broker so a home-automation hub (Homebridge) can control it. It drives the same `Drivers` controls IR and the WLED app do — every command routes through `Scheduler::setControl`, so MQTT is a transport, not new control logic. The MQTT 3.1.1 client is our own (dependency-free, golden-vector-tested in `MqttPacket.h`), speaking to the broker over the platform TCP socket. Disabled until a broker is set; works over WiFi or Ethernet. + +- `broker` — the broker hostname (e.g. `homeassistant.lan`) or IP. A hostname is resolved via DNS. +- `port` — broker port (default 1883). +- `username` / `password` — broker credentials (optional; the password is stored obfuscated like the WiFi password). +- The topic prefix is `projectMM/<mac>` — a **stable** identifier (the last 6 hex of the device's MAC), fixed for the device's life. Renaming the device does **not** change its topics, so a hub's config never breaks on a rename (the WLED/Tasmota/Home-Assistant convention). Not a stored control. +- read-only — `mqtt_status` (`disabled` / `idle` / `connecting` / `connected` / `disconnected` / an error). + +**Topics** (for a device whose MAC ends `563cfe`): the device SUBSCRIBEs to the `set` topics and PUBLISHes the `get` topics on change (and on connect, so a controller never reads "No Response"). It also publishes its friendly `deviceName` on the retained `name` topic, so a hub can show the human name while the topics stay MAC-stable: + +| direction | topic | payload | +|---|---|---| +| set → device | `projectMM/563cfe/on/set` | `true` / `false` | +| device → get | `projectMM/563cfe/on/get` | `true` / `false` | +| set → device | `projectMM/563cfe/brightness/set` | `0`–`100` | +| device → get | `projectMM/563cfe/brightness/get` | `0`–`100` | +| set → device | `projectMM/563cfe/hsv/set` | `h,s,v` (hue `0`–`359`, sat/val `0`–`100`) | +| device → get | `projectMM/563cfe/hsv/get` | `h,s,v` | +| device → get | `projectMM/563cfe/name` | the friendly `deviceName` (retained) | + +The HomeKit colour wheel has no "palette" concept, so `hsv/set`'s hue+saturation pick the **nearest palette** (each built-in palette has a representative colour; the closest one is selected) and the value drives brightness — the colour wheel becomes a natural palette selector. + +**Homebridge** — install [`homebridge-mqttthing`](https://github.com/arachnetech/homebridge-mqttthing) and add a `lightbulb` accessory. Use the device's own MAC suffix (read it from the `mqtt_status`/topics, or `mosquitto_sub -t 'projectMM/#'`) in place of `563cfe`: + +```json +{ + "accessory": "mqttthing", + "type": "lightbulb", + "name": "projectMM", + "url": "mqtt://<broker>:1883", + "username": "<user>", + "password": "<pass>", + "topics": { + "getOn": "projectMM/563cfe/on/get", + "setOn": "projectMM/563cfe/on/set", + "getBrightness": "projectMM/563cfe/brightness/get", + "setBrightness": "projectMM/563cfe/brightness/set", + "getHSV": "projectMM/563cfe/hsv/get", + "setHSV": "projectMM/563cfe/hsv/set" + }, + "onValue": "true", + "offValue": "false" +} +``` + +Home Assistant does not need MQTT: it adopts the device through its built-in WLED integration over the existing WLED `/json` API. + +Detail: [technical](../moxygen/MqttModule.md) + +[Tests](../../../tests/unit-tests.md#mqttmodule) + ### Firmware update Over-the-air firmware flashing — the one operation that swaps the binary and needs a power cycle (every *config* change applies live; a firmware OTA does not). @@ -62,12 +115,28 @@ Detail: [technical](../moxygen/FirmwareUpdateModule.md) Persists control values as JSON and restores them on boot, overlaying loaded values through each control's pointer during `onBuildControls()`. The home of the no-reboot live-reconfiguration behaviour. -- read-only — `lastSaved`, `filesystem` (usage). +- read-only — `lastSaved`. Detail: [technical](../moxygen/FilesystemModule.md) [Tests](../../../tests/unit-tests.md#filesystemmodule) +### File Manager + +A boot-wired system tool (distinct from Filesystem, which is the persistence *engine*): browse and manage the device filesystem. It renders a dedicated panel — a lazy expand/collapse folder **tree** (the standard VS Code / Explorer shape: each folder loads its children on first expand) plus an inline text editor. Browsing is UI-side over the `/api/dir` listing endpoint; file contents come over `/api/file`; so the module itself stays minimal, exposing only the operations. Dot-prefixed entries (the `.config` persistence dir) are hidden unless `show hidden` is on. + +- `show hidden` — reveal dot-prefixed files/folders (e.g. `.config`); forwarded to `/api/dir` as its `hidden` filter. +- Click a folder's row to select it and toggle its expansion (▸/▾); click a selected file to open the editor. +- The toolbar acts on the selected node: **＋ folder** creates a folder inside it, **＋ file** creates an empty file (click it to edit), **🗑 delete** removes the selected file or empty folder (press-twice to confirm), **⟳** refreshes. +- **Drag files from the desktop** onto a folder (or the tree) to upload them — the body streams straight to the file (any size, binary-safe; capped only by a sanity limit and the free space, which it reports if short); a per-file **⤓** streams it back to the desktop. +- The editor loads a file's text, pretty-prints JSON on open, and saves atomically; a binary file (contains a NUL) loads read-only (use ⤓ to fetch it intact). Upload and download both stream, so neither truncates. +- A filesystem-usage bar (used / total, read from the platform) sits below the tree, with a `last saved` line under it (how long ago config was persisted — read from the FilesystemModule engine). +- Create / delete are HTTP calls (`POST` / `DELETE /api/dir?path=`), not controls — the path rides the request, so nothing is stored on the device per op. + +Last-modified dates (needs an NTP time source + LittleFS mtime), binary/large + folder upload, folder-as-zip download, and `.ml` syntax highlighting are backlogged ([backlog-core § File Manager follow-ups](../../../backlog/backlog-core.md#file-manager-follow-ups)). + +Detail: [technical](../moxygen/FileManagerModule.md) + ### Audio A System peripheral (added by the user, not auto-wired): an I²S microphone (or line-in ADC) feeding the FFT that audio-reactive effects consume via `AudioModule::latestFrame()`. It also syncs audio over UDP, WLED-compatible: broadcast the local analysis for the WLED ecosystem, or receive a peer's audio to drive effects with no local mic. Idle until real GPIOs are entered. @@ -101,9 +170,9 @@ A System peripheral (added per board, not auto-wired): an IR remote receiver tha <img src="../../../assets/core/IrModule.jpeg" width="300" alt="An IR receiver and the common 21-key RGB remote it decodes"> - `pin` — the IR receiver GPIO (unset until entered; on the SE16 it shares GPIO 5 with the Ethernet MISO via the board switch, on the LightCrafter it is its own GPIO 4 alongside Ethernet). -- `learn` — pick an action (off / brightness up / brightness down / palette next / palette prev); the next received code binds to it, then learning disarms. -- `code brightness up` / `code brightness down` / `code palette next` / `code palette prev` — read-only, the learned code for each action (persisted). +- `learn` — pick an action to bind (`on/off` / brightness up / brightness down / palette next / palette prev); the next received code binds to it, then learning disarms. The first option, `off`, is the disarmed state (bind nothing), not a light action. +- `code on/off` / `code brightness up` / `code brightness down` / `code palette next` / `code palette prev` — read-only, the learned code for each action (persisted). -The actions nudge `Drivers.brightness` (±16, clamped 0–255) and step `Drivers.palette` (next/previous). The status line reports setup state ("set pin to receive" / "ready"), the learn prompt, a binding ("learned … = 0x…"), a fired action ("Drivers.brightness → N"), and an unbound code ("received 0x… (unassigned)"). +The actions toggle `Drivers.on` (master power), nudge `Drivers.brightness` (±16, clamped 0–255) and step `Drivers.palette` (next/previous). The status line reports setup state ("set pin to receive" / "ready"), the learn prompt, a binding ("learned … = 0x…"), a fired action ("Drivers.brightness → N", "Drivers.on → off"), and an unbound code ("received 0x… (unassigned)"). Detail: [technical](../moxygen/IrModule.md) diff --git a/docs/moonmodules/light/drivers/NetworkSendDriver.md b/docs/moonmodules/light/drivers/NetworkSendDriver.md index e9dae14c..fac2240c 100644 --- a/docs/moonmodules/light/drivers/NetworkSendDriver.md +++ b/docs/moonmodules/light/drivers/NetworkSendDriver.md @@ -29,4 +29,4 @@ The whole frame goes out inline in `loop()` — ~35 ms over Ethernet / ~90 ms ov The driver is added as a child of the `Drivers` container at runtime — not boot-wired, but added per board through the catalog (`POST /api/modules`, the board's [`deviceModels.json`](../../../../web-installer/deviceModels.json) `modules` entry), so a device only carries the outputs its board actually has. Once added it receives `setSourceBuffer` / `setCorrection` from `Drivers::passBufferToDrivers` (which wires every child generically, boot-added or runtime-added) and applies the shared `const Correction*` before every send — the same correction the RMT LED driver applies, so network and wired outputs show identical colours. The added child persists across reboot via `FilesystemModule`. -The live send tier — `uv run scripts/scenario/run_network_live.py` ([MoonDeck.md § run_network_live](../../../../scripts/MoonDeck.md#run_network_live)) — relays between real boards with the protocol control cycled round-robin, exercising the wire path no host test reaches. +The live send tier — `uv run moondeck/scenario/run_network_live.py` ([MoonDeck.md § run_network_live](../../../../moondeck/MoonDeck.md#run_network_live)) — relays between real boards with the protocol control cycled round-robin, exercising the wire path no host test reaches. diff --git a/docs/moonmodules/light/drivers/RmtLedDriver.md b/docs/moonmodules/light/drivers/RmtLedDriver.md index 8320f5e3..f882ec4e 100644 --- a/docs/moonmodules/light/drivers/RmtLedDriver.md +++ b/docs/moonmodules/light/drivers/RmtLedDriver.md @@ -19,6 +19,8 @@ Bits are sent **MSB-first** within each byte; channel order (GRB, GRBW, …) is The source buffer is split into **consecutive slices**, one per pin, in list order: pin 1 takes lights `[0, n₁)`, pin 2 takes `[n₁, n₁+n₂)`, and so on. Slice sizes come from `ledsPerPin`; pins without an explicit count split the unassigned remainder evenly (the last pin takes the rounding remainder). Counts are clamped so the sum never exceeds the buffer; lights beyond the last slice are not emitted. With `ledsPerPin` empty the whole buffer splits evenly over all pins — the zero-config case. +Each pin is also capped at a **WS2812 per-pin ceiling** (2048 lights). A single 1-wire WS2812 line clocks a fixed ~30 µs/light, so 2048 lights is already ~61 ms/frame (~16 FPS) — the floor before output turns to a slideshow. A pin over the ceiling is **clamped to it** (the driver still drives the first 2048, output stays lit — it does not idle) and a Warning status is raised ("some LEDs not driven…"), cleared once the count drops back under. This guards the common misconfig — a whole grid funneled onto one pin — which otherwise pins the tick at ~490 ms (FPS 1). The intended way to drive fewer lights is the start/count window, not this safety cap. The cap is a per-protocol value: a clocked 2-wire type (APA102/SK9822 over SPI) has no such fixed-rate limit and would pass a far higher one. (Clamping accumulates offsets from the clamped counts, so on a multi-pin over-ceiling config the later strips show a shifted window, not just a truncated one — accepted as a degraded, warned state for a config no one wires on purpose; the one-pin headline case is exact.) + ## Concurrent show (blocks the render tick for the longest strand) `loop()` encodes the whole frame once, then starts every pin's transmission (`platform::rmtWs2812Transmit`) before waiting on each (`platform::rmtWs2812Wait`) — the RMT channels clock out concurrently, so the render tick is charged roughly the **longest** strand, not the sum (~3 ms per 100 pixels on the longest slice), plus one shared reset gap. The transmission runs synchronously on the render task, so a large strand or a WiFi-interrupt-sensitive install can show timing artifacts. diff --git a/docs/moonmodules/light/effects/effects.md b/docs/moonmodules/light/effects/effects.md index d41f7458..792ec6ee 100644 --- a/docs/moonmodules/light/effects/effects.md +++ b/docs/moonmodules/light/effects/effects.md @@ -4,6 +4,8 @@ Every effect, one block each: its preview, what it does, and what each control m **Jump to:** [MoonLight](#moonlight-effects) · [MoonModules](#moonmodules-effects) · [WLED](#wled-effects) · [FastLED](#fastled-effects) · [projectMM-native](#projectmm-native-effects) +**Migrating an effect — behaviour is the spec.** A ported effect must reproduce the original's **exact** visual behaviour: end users have relied on these for years, so a port that looks different is a regression, not an improvement. Don't get creative with defaults, oscillator math, colour mapping, or geometry, and don't silently drop a parameter that *is* the mechanism (the PaintBrush straight-vs-curved-lines bug was a dropped partial-line `length`; Game of Life was wrong the first time by not porting the real algorithm). Study the source for the algorithm, defaults, and visual result; pin it with unit + scenario tests; then write our **own** implementation against `EffectBase`/our primitives — carry the behaviour forward, don't trace or copy the structure (see [*Industry standards, our own code*](../../../../CLAUDE.md#principles)). Credit the origin as prior art in the block below. + > Some WLED-origin effects show a preview gif from [WLED-Utils](https://github.com/scottrbailey/WLED-Utils) by scottrbailey (the canonical WLED effect gif set, cross-linked with credit); these show WLED's rendering. Effects with a local `../../../assets/…` gif show our own output. ## MoonLight effects @@ -213,6 +215,7 @@ A 3D Rubik's Cube projected onto the volume: it scrambles, then plays its soluti - `turnsPerSecond` — how fast the cube turns. - `cubeSize` — the cube order (2×2 up to 8×8). - `randomTurning` — turn endlessly at random instead of scramble-then-solve. +- `usePalette` — colour the six faces from the system-wide palette instead of the classic Rubik's colours. Origin: MoonLight · by WildCats08 / [@Brandon502](https://github.com/Brandon502) · via [MoonLight](https://github.com/MoonModules/MoonLight/blob/main/src/MoonLight/Nodes/Effects/E_MoonLight.h) · source [RubiksCubeEffect.h](../../../../src/light/effects/RubiksCubeEffect.h) diff --git a/docs/moonmodules/light/layouts/layouts.md b/docs/moonmodules/light/layouts/layouts.md index aaccffcf..a38ff5e4 100644 --- a/docs/moonmodules/light/layouts/layouts.md +++ b/docs/moonmodules/light/layouts/layouts.md @@ -1,6 +1,6 @@ # Layouts -Every layout, one block each: what it does and what each control means — together. A layout maps light indices to physical `(x, y, z)` positions — it defines the *shape* an [effect](../effects/effects.md) draws onto and a [driver](../drivers/) sends out. The [Layouts](../moxygen/Layouts.md) container holds one or more layout children and composes them into one coordinate space; a [Layer](../moxygen/Layer.md) renders over that combined space. (For how this page maps to the source/asset folders, see the [folder-structure decision](../../../backlog/folder-structure-proposal.md).) +Every layout, one block each: what it does and what each control means — together. A layout maps light indices to physical `(x, y, z)` positions — it defines the *shape* an [effect](../effects/effects.md) draws onto and a [driver](../drivers/drivers.md) sends out. The [Layouts](../moxygen/Layouts.md) container holds one or more layout children and composes them into one coordinate space; a [Layer](../moxygen/Layer.md) renders over that combined space. (For how this page maps to the source/asset folders, see the [folder-structure decision](../../../backlog/folder-structure-proposal.md).) ## MoonLight layouts diff --git a/docs/performance.md b/docs/performance.md index 3d53fba4..69a51093 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -319,7 +319,7 @@ Board: the default `esp32` (WiFi + Ethernet — the largest classic variant, mea Run from project root after a clean build: ```bash -uv run scripts/build/build_esp32.py --firmware esp32 +uv run moondeck/build/build_esp32.py --firmware esp32 idf.py -B build/esp32-esp32 \ -DSDKCONFIG=build/esp32-esp32/sdkconfig \ size-components | head -40 diff --git a/docs/testing.md b/docs/testing.md index 955e27ab..ebda8bd6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -5,7 +5,7 @@ What we test and how. The detailed inventory of every test lives in two auto-gen - **[Unit tests](tests/unit-tests.md)** — one row per `TEST_CASE`, grouped by module. Generated from `test/unit/{core,light}/unit_*.cpp`. - **[Scenario tests](tests/scenario-tests.md)** — one section per scenario JSON, grouped by module. Generated from `test/scenarios/{core,light}/scenario_*.json`. Each scenario runs in two tiers (in-process + live) unless flagged `live_only` (construct-mode scenarios are also skipped on live, since `main.cpp` owns the live shape). -Both are produced by `scripts/docs/generate_test_docs.py`; the source of truth is the test files themselves (see [Adding Tests](#adding-tests) below). +Both are produced by `moondeck/docs/generate_test_docs.py`; the source of truth is the test files themselves (see [Adding Tests](#adding-tests) below). ## Testing strategy @@ -54,11 +54,11 @@ A test lives under the subfolder of its **primary** `@module`'s source domain (e ### Host-side tests (Python + JS) -The C++ `ctest` / scenario suites can't reach the **Python** (MoonDeck, build scripts) or **JS** (web installer) code, so those get their own host-side unit tier — `test/python/` (pytest, run `uv run --with pytest --with pyserial pytest test/python`) and `test/js/` (Node's built-in runner, run `node --test "test/js/**/*.test.mjs"`; no `package.json`/`npm install`). Both run in `.github/workflows/test.yml` on every PR and are commit gates (CLAUDE.md Event 1, gate 10) when `scripts/` / `web-installer/` / the test dirs change. Python test files carry their deps in a PEP-723 `# /// script` block (the repo's convention — there's no central `pyproject.toml`); pyserial is a dep only because `improv_provision.py`'s import guard exits without it, not because the frame logic needs it. +The C++ `ctest` / scenario suites can't reach the **Python** (MoonDeck, build scripts) or **JS** (web installer) code, so those get their own host-side unit tier — `test/python/` (pytest, run `uv run --with pytest --with pyserial pytest test/python`) and `test/js/` (Node's built-in runner, run `node --test "test/js/**/*.test.mjs"`; no `package.json`/`npm install`). Both run in `.github/workflows/test.yml` on every PR and are commit gates (CLAUDE.md Event 1, gate 10) when `moondeck/` / `web-installer/` / the test dirs change. Python test files carry their deps in a PEP-723 `# /// script` block (the repo's convention — there's no central `pyproject.toml`); pyserial is a dep only because `improv_provision.py`'s import guard exits without it, not because the frame logic needs it. #### What's covered today: the Improv frame wire format -The Improv serial frame is implemented **three times** — device C++ (`src/core/ImprovFrame.h`), Python (`scripts/build/improv_provision.py`), and installer JS (`web-installer/improv-frame.js`) — so a drift in any one silently breaks provisioning. Both suites assert the **same golden vector** so the Python and JS builders provably agree (hand-verified against the C++ sum-mod-256 checksum): +The Improv serial frame is implemented **three times** — device C++ (`src/core/ImprovFrame.h`), Python (`moondeck/build/improv_provision.py`), and installer JS (`web-installer/improv-frame.js`) — so a drift in any one silently breaks provisioning. Both suites assert the **same golden vector** so the Python and JS builders provably agree (hand-verified against the C++ sum-mod-256 checksum): ``` buildImprovFrame(type=0x03, payload=[0x01]) == 49 4d 50 52 4f 56 01 03 01 01 e3 @@ -87,9 +87,9 @@ buildImprovFrame(type=0x03, payload=[0x01]) == 49 4d 50 52 4f 56 01 03 01 01 e The JS suite proves the installer *chunks* an op correctly; the **device side that reassembles those chunks** is pinned by the C++ `unit_ImprovOpReassembler` suite (`src/core/ImprovOpReassembler.h`, the pure state machine behind the device's `APPLY_OP` handler — extracted from `platform_esp32_improv.cpp` so it's desktop-testable). It covers the full receive contract: in-order multi-chunk reassembly + NUL-termination, **duplicate-chunk rejection** and **out-of-order/skipped-seq rejection** (the guard against an installer retry corrupting the buffer), **overflow** rejection at the buffer-minus-NUL boundary, mid-stream `seq 0` abandoning a partial op, and clean recovery after every error. Encode (JS) + reassemble (C++) together prove APPLY_OP end to end without hardware. -**`test/python/test_installer_manifests.py`** (pytest) — pins the web installer's per-release file contract. For every `ships: true` firmware in `web-installer/firmwares.json` it runs `scripts/build/generate_manifest.py` (with a synthetic `flasher_args.json`, so no firmware build is needed) and asserts the manifest is valid (a `chipFamily` + non-empty `parts[]`) AND that **every part filename matches one of the globs the release workflow stages onto Pages** (`firmware-*.bin` / `shared-ota-data.bin` / `partition-table-*.bin`). A manifest that names a file outside those globs points at something the deploy never stages → the installer 404s at fetch-firmware (the failure that shipped a broken v2.0.0 installer). The test guards the manifest-generation ↔ staged-files contract; the *deploy mechanics* that stage them (per-tag, in `release.yml`) are workflow shell logic a unit test can't reach, so the two are complementary. +**`test/python/test_installer_manifests.py`** (pytest) — pins the web installer's per-release file contract. For every `ships: true` firmware in `web-installer/firmwares.json` it runs `moondeck/build/generate_manifest.py` (with a synthetic `flasher_args.json`, so no firmware build is needed) and asserts the manifest is valid (a `chipFamily` + non-empty `parts[]`) AND that **every part filename matches one of the globs the release workflow stages onto Pages** (`firmware-*.bin` / `shared-ota-data.bin` / `partition-table-*.bin`). A manifest that names a file outside those globs points at something the deploy never stages → the installer 404s at fetch-firmware (the failure that shipped a broken v2.0.0 installer). The test guards the manifest-generation ↔ staged-files contract; the *deploy mechanics* that stage them (per-tag, in `release.yml`) are workflow shell logic a unit test can't reach, so the two are complementary. -**`test/python/test_check_specs_drift.py`** (pytest) — pins the two spec-drift guards in `scripts/check/check_specs.py` (the spec-check commit gate). Some facts live in both the `.h` and the module doc in different forms: a control's **numeric range** (`addUint8("floor", floor, 0, 255)` vs the prose "noise floor (0–255)") and the **author/source URL** (`// Author: … — <url>` vs the `Origin:` markdown link). Neither can be single-sourced — they're the same fact for two audiences — so instead the gate *validates* them: if the doc restates a control's range and it conflicts with the `.h`, or an `.h` author URL is missing from the doc, the spec check flags it. The checks are block-scoped on the consolidated catalog pages (a control name shared across modules, `fps`/`fadeRate`, matches only its own module's block), and tolerant of the human range spellings (`1–8` / `1-8` / `1 to 8`) — this suite pins both the catch and the no-false-alarm behaviour. +**`test/python/test_check_specs_drift.py`** (pytest) — pins the two spec-drift guards in `moondeck/check/check_specs.py` (the spec-check commit gate). Some facts live in both the `.h` and the module doc in different forms: a control's **numeric range** (`addUint8("floor", floor, 0, 255)` vs the prose "noise floor (0–255)") and the **author/source URL** (`// Author: … — <url>` vs the `Origin:` markdown link). Neither can be single-sourced — they're the same fact for two audiences — so instead the gate *validates* them: if the doc restates a control's range and it conflicts with the `.h`, or an `.h` author URL is missing from the doc, the spec check flags it. The checks are block-scoped on the consolidated catalog pages (a control name shared across modules, `fps`/`fadeRate`, matches only its own module's block), and tolerant of the human range spellings (`1–8` / `1-8` / `1 to 8`) — this suite pins both the catch and the no-false-alarm behaviour. MoonDeck's pure logic (catalog reverse-lookup, state migration) and the installer's op-walk / storage are the next host-side candidates as they accrete regression risk. @@ -238,11 +238,11 @@ A contract changes only when there's a reason: a code change improved performanc ```bash # After an optimisation: tighten the ceiling for pc-* -uv run scripts/scenario/run_scenario.py --update-contract \ +uv run moondeck/scenario/run_scenario.py --update-contract \ --reason "Layer LUT inline copy" # After accepting an ESP32 cost: loosen the ceiling on that target -uv run scripts/scenario/run_live_scenario.py --host 192.168.1.210 \ +uv run moondeck/scenario/run_live_scenario.py --host 192.168.1.210 \ --update-contract --reason "added DMX driver overhead" ``` @@ -315,10 +315,10 @@ Unknown JSON keys are ignored by both runners (C++ and Python), so adding a new ### Generated docs and the shared parser ```text -scripts/docs/ +moondeck/docs/ ├── _test_metadata.py # one parser used by both consumers below └── generate_test_docs.py # writes docs/tests/unit-tests.md + scenario-tests.md -scripts/moondeck.py # serves the same data as HTML in MoonDeck views +moondeck/moondeck.py # serves the same data as HTML in MoonDeck views ``` Both the markdown generator and the MoonDeck endpoints (`/api/test-modules`, `/api/unit-tests/<Module>`, `/api/scenarios/<name>`) import from `_test_metadata.py`. Adding a new metadata field (e.g. `@since`) means one edit there; both consumers pick it up. @@ -326,8 +326,8 @@ Both the markdown generator and the MoonDeck endpoints (`/api/test-modules`, `/a Run the generator after touching any test file: ```bash -uv run scripts/docs/generate_test_docs.py # writes both docs -uv run scripts/docs/generate_test_docs.py --check # exits non-zero on drift (CI-friendly) +uv run moondeck/docs/generate_test_docs.py # writes both docs +uv run moondeck/docs/generate_test_docs.py --check # exits non-zero on drift (CI-friendly) ``` `--check` exits non-zero when the docs are stale — a contributor who adds a `TEST_CASE` without re-running the generator gets flagged. Run it in CI or before a commit to keep the generated docs in sync. @@ -340,11 +340,11 @@ Run them with: ```bash # Replace build/macos with build/linux or build/windows per host. The -# MoonDeck path below and `uv run scripts/test/test_desktop.py` resolve the +# MoonDeck path below and `uv run moondeck/test/test_desktop.py` resolve the # host build dir automatically; the raw ctest / mm_tests calls don't. ctest --test-dir build/macos --output-on-failure # all ./build/macos/test/mm_tests -tc="<case-name>" # one test case -uv run scripts/test/test_desktop.py --module Layer # filtered by module +uv run moondeck/test/test_desktop.py --module Layer # filtered by module ``` Or via MoonDeck (PC tab → Unit Test card). Pick a module from the shared module dropdown above the card to filter the run. Tests button shows the per-module inventory. @@ -358,9 +358,9 @@ Inventory: **[docs/tests/scenario-tests.md](tests/scenario-tests.md)** (auto-gen Run them with: ```bash -uv run scripts/scenario/run_scenario.py # all -uv run scripts/scenario/run_scenario.py --name scenario_Layer_base_pipeline # one by stem -uv run scripts/scenario/run_scenario.py --module Layer # all for one module +uv run moondeck/scenario/run_scenario.py # all +uv run moondeck/scenario/run_scenario.py --name scenario_Layer_base_pipeline # one by stem +uv run moondeck/scenario/run_scenario.py --module Layer # all for one module ``` Or via MoonDeck (PC tab → Scenarios card). The module dropdown is shared with the Unit Test card above it: pick a module once and both card's run-set narrows. Steps button shows the per-scenario step list. @@ -383,10 +383,10 @@ A step can also set `"measure": true` on a non-measure op (e.g. mark the last `a Live scenarios run the same JSON against a running device via the REST API. ```bash -uv run scripts/scenario/run_live_scenario.py --host localhost:8080 # desktop -uv run scripts/scenario/run_live_scenario.py --host 192.168.1.210 # ESP32 -uv run scripts/scenario/run_live_scenario.py --update-baseline # save -uv run scripts/scenario/run_live_scenario.py --compare-baseline # check +uv run moondeck/scenario/run_live_scenario.py --host localhost:8080 # desktop +uv run moondeck/scenario/run_live_scenario.py --host 192.168.1.210 # ESP32 +uv run moondeck/scenario/run_live_scenario.py --update-baseline # save +uv run moondeck/scenario/run_live_scenario.py --compare-baseline # check ``` MoonDeck's Live tab wraps the same workflow: the Network bar at the top selects the LAN, Discover/Refresh populates the device list, the Live Scenarios card runs the selected scenario against every checked device. @@ -399,7 +399,7 @@ Scenarios that add modules (e.g. `scenario_Layer_base_pipeline`, `scenario_Layer Memory tracking works on ESP32: `freeHeap` and `freeInternalHeap` report real values. Desktop returns 0 (unlimited). The control-change scenario verifies no memory leaks by checking that heap returns to baseline after a mirror toggle. -One live-tier test lives outside the scenario JSON schema because it spans **multiple devices**: `uv run scripts/scenario/run_network_live.py` runs a lights-over-UDP matrix (ArtNet, E1.31 and DDP) over every online board in moondeck.json — each board is once the sender, all others listen, and reception is asserted by reading each device's `/ws` preview stream (see [MoonDeck.md § run_network_live](../scripts/MoonDeck.md#run_network_live)). A device matrix needs loops and per-round state the declarative scenario JSON can't express, so it follows the `improv_smoke_test.py` script shape instead. +One live-tier test lives outside the scenario JSON schema because it spans **multiple devices**: `uv run moondeck/scenario/run_network_live.py` runs a lights-over-UDP matrix (ArtNet, E1.31 and DDP) over every online board in moondeck.json — each board is once the sender, all others listen, and reception is asserted by reading each device's `/ws` preview stream (see [MoonDeck.md § run_network_live](../moondeck/MoonDeck.md#run_network_live)). A device matrix needs loops and per-round state the declarative scenario JSON can't express, so it follows the `improv_smoke_test.py` script shape instead. ## Hardware Verification @@ -414,10 +414,10 @@ All live scenarios pass on both desktop and ESP32 with `min_pct: 80` relative bo ## Adding Tests -**Unit test:** add a `TEST_CASE` to the appropriate `test/unit/{core,light}/unit_<ExactModuleName>[_<topic>].cpp` file. Each file carries `// @module <ExactCamelCaseName>` at the top, plus a single `//` description line above each `TEST_CASE`. Add a new file when no existing test covers your module — pick the subfolder matching the module's `src/` domain. After adding cases, run `uv run scripts/docs/generate_test_docs.py` so the generated inventory matches. +**Unit test:** add a `TEST_CASE` to the appropriate `test/unit/{core,light}/unit_<ExactModuleName>[_<topic>].cpp` file. Each file carries `// @module <ExactCamelCaseName>` at the top, plus a single `//` description line above each `TEST_CASE`. Add a new file when no existing test covers your module — pick the subfolder matching the module's `src/` domain. After adding cases, run `uv run moondeck/docs/generate_test_docs.py` so the generated inventory matches. **Scenario test:** create a JSON file under `test/scenarios/{core,light}/` named `scenario_<ExactModuleName>_<topic>.json`. The top-level needs `name` (matching the filename stem), `module`, optional `also`, `description`, `mode` (`construct` or `mutate`), and optional `"live_only": true` if the scenario can only run against a real device. Each `steps[]` entry has an `op` (`add_module`, `set_control`, `measure`), a `name`, a `description` field that the doc generator picks up, optional `"measure": true` to run a measurement after the op, and optional `bounds` (`fps` and/or `heap`). The scenario runner auto-discovers all `.json` files under `test/scenarios/` recursively. **Regression test:** when fixing a bug, add a test that reproduces it. The test's description (the `//` line for unit tests, the `description` JSON field for scenarios) should mention the root cause so the connection stays traceable in the generated inventory. -**Doc check:** `scripts/docs/generate_test_docs.py --check` exits non-zero if regeneration would change `docs/tests/*.md` — a CI-friendly way to catch metadata that drifted from the source. +**Doc check:** `moondeck/docs/generate_test_docs.py --check` exits non-zero if regeneration would change `docs/tests/*.md` — a CI-friendly way to catch metadata that drifted from the source. diff --git a/esp32/main/CMakeLists.txt b/esp32/main/CMakeLists.txt index 53d8cea1..82c5a71c 100644 --- a/esp32/main/CMakeLists.txt +++ b/esp32/main/CMakeLists.txt @@ -5,6 +5,8 @@ idf_component_register( "../../src/core/Control.cpp" "../../src/core/HttpServerModule.cpp" "../../src/core/FilesystemModule.cpp" + "../../src/core/FileManagerModule.cpp" + "../../src/core/MqttModule.cpp" "../../src/core/Scheduler.cpp" "../../src/core/moonlive/MoonLive.cpp" "../../src/core/moonlive/MoonLiveCompiler.cpp" @@ -37,7 +39,7 @@ idf_component_register( target_compile_options(${COMPONENT_LIB} PRIVATE -Wall -Wextra -Werror) -# Firmware-variant defines, set by scripts/build/build_esp32.py via firmware_cmake_args(). +# Firmware-variant defines, set by moondeck/build/build_esp32.py via firmware_cmake_args(). # See docs/architecture.md § Firmware vs board — "firmware" is the compiled # binary variant; the physical "board" is a separate concept the device # cannot identify on its own. @@ -125,8 +127,8 @@ add_custom_target(ui_embed DEPENDS ${UI_DIR}/ui_embedded.h) set(BUILD_INFO_DIR ${COMPONENT_DIR}/../../src/core) add_custom_command( OUTPUT ${BUILD_INFO_DIR}/build_info.h - COMMAND ${Python3_EXECUTABLE} ${COMPONENT_DIR}/../../scripts/build/generate_build_info.py - DEPENDS ${COMPONENT_DIR}/../../library.json ${COMPONENT_DIR}/../../scripts/build/generate_build_info.py + COMMAND ${Python3_EXECUTABLE} ${COMPONENT_DIR}/../../moondeck/build/generate_build_info.py + DEPENDS ${COMPONENT_DIR}/../../library.json ${COMPONENT_DIR}/../../moondeck/build/generate_build_info.py COMMENT "Generating build_info.h" ) add_custom_target(build_info_gen DEPENDS ${BUILD_INFO_DIR}/build_info.h) diff --git a/esp32/sdkconfig.defaults.esp32p4-eth b/esp32/sdkconfig.defaults.esp32p4-eth index 857cf52a..53684b34 100644 --- a/esp32/sdkconfig.defaults.esp32p4-eth +++ b/esp32/sdkconfig.defaults.esp32p4-eth @@ -3,7 +3,7 @@ # Schematic / pin map confirmed against the Waveshare wiki + the ESPHome # device page (devices.esphome.io/devices/waveshare-esp32-p4-nano). # -# Layered onto sdkconfig.defaults by scripts/build/build_esp32.py. +# Layered onto sdkconfig.defaults by moondeck/build/build_esp32.py. # # The RMII / PHY *pins* are NOT set here — they live in C (the per-target # ethPins struct in src/platform/esp32/platform_config.h, read by ethInit), diff --git a/esp32/sdkconfig.defaults.esp32p4-eth-wifi b/esp32/sdkconfig.defaults.esp32p4-eth-wifi index 4b9078f3..b9564812 100644 --- a/esp32/sdkconfig.defaults.esp32p4-eth-wifi +++ b/esp32/sdkconfig.defaults.esp32p4-eth-wifi @@ -2,7 +2,7 @@ # Board: https://www.waveshare.com/wiki/ESP32-P4-Nano-StartPage # # Layered onto sdkconfig.defaults + sdkconfig.defaults.esp32p4-eth by -# scripts/build/build_esp32.py. The eth fragment brings the EMAC / RMII / flash / +# moondeck/build/build_esp32.py. The eth fragment brings the EMAC / RMII / flash / # PSRAM / partition layout; this fragment adds WiFi over the on-board ESP32-C6 # co-processor. # diff --git a/esp32/sdkconfig.defaults.esp32s31 b/esp32/sdkconfig.defaults.esp32s31 index 58824356..f79573a9 100644 --- a/esp32/sdkconfig.defaults.esp32s31 +++ b/esp32/sdkconfig.defaults.esp32s31 @@ -2,7 +2,7 @@ # Board: the Espressif ESP32-S31 Function-CoreBoard (RISC-V dual-core, Wi-Fi 6, # BT 5.4, on-chip 1 Gbps EMAC, 16 MB flash, PSRAM). # -# Layered onto sdkconfig.defaults by scripts/build/build_esp32.py. +# Layered onto sdkconfig.defaults by moondeck/build/build_esp32.py. # # The RMII / PHY *pins* are NOT set here — they live in C (the per-target ethPins # struct in src/platform/esp32/platform_config.h, read by ethInit), so the pin map diff --git a/esp32/sdkconfig.defaults.eth b/esp32/sdkconfig.defaults.eth index f2033b44..ff82b504 100644 --- a/esp32/sdkconfig.defaults.eth +++ b/esp32/sdkconfig.defaults.eth @@ -1,5 +1,5 @@ # Ethernet (internal EMAC, RMII) — enables the RMII *driver* for the classic -# ESP32 / P4. Layered onto sdkconfig.defaults by scripts/build/build_esp32.py. +# ESP32 / P4. Layered onto sdkconfig.defaults by moondeck/build/build_esp32.py. # # Only the driver enable + DMA buffers live here (compile-time). The PHY addr, # reset GPIO, MDC/MDIO and RMII clock are RUNTIME config now — set per board in diff --git a/esp32/sdkconfig.defaults.eth-spi b/esp32/sdkconfig.defaults.eth-spi index e1ac48fc..adeaa98d 100644 --- a/esp32/sdkconfig.defaults.eth-spi +++ b/esp32/sdkconfig.defaults.eth-spi @@ -3,7 +3,7 @@ # which SPI pins is runtime config (boards.json → NetworkModule → setEthConfig), # not baked here. Boards: SE16, LightCrafter (WIZ850IO). # -# Layered onto sdkconfig.defaults by scripts/build/build_esp32.py. The filename's +# Layered onto sdkconfig.defaults by moondeck/build/build_esp32.py. The filename's # `.eth-spi` segment is what flips the build to an Ethernet build (clears MM_NO_ETH # — build_esp32.py matches ".eth" in any fragment name), so platform::hasEthernet # becomes true and ethInit()'s W5500 path compiles. diff --git a/mkdocs.yml b/mkdocs.yml index 8842ca20..16305f4f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -6,7 +6,7 @@ # # Built in CI (release.yml deploy-pages job) into pages/ root; the web # installer keeps its /install/ path untouched. Build locally with -# uv run scripts/docs/build_docs.py --strict +# uv run moondeck/docs/build_docs.py --strict site_name: projectMM site_description: High-performance LED & DMX lighting control for ESP32 and beyond. # Canonical deployed URL: the custom domain, with the /projectMM/ repo subpath @@ -80,12 +80,12 @@ plugins: extra_css: - assets/extra.css -# Build-time hooks (scripts/docs/mkdocs_hooks.py): generate the test-inventory pages +# Build-time hooks (moondeck/docs/mkdocs_hooks.py): generate the test-inventory pages # from the test files (never committed → can't drift), render each catalog page's # prose ### blocks as a 4-column table, and rewrite out-of-docs source links to # GitHub URLs. See the module docstring. hooks: - - scripts/docs/mkdocs_hooks.py + - moondeck/docs/mkdocs_hooks.py # history/ and backlog/ are internal (agent-facing, transient) — kept OFF the top # nav (they're not in the `nav:` tree) but still BUILT into the site, so the many @@ -118,8 +118,10 @@ nav: - Getting started: - Install & first light: gettingstarted.md # The web installer is a separate app deployed at /install/ (staged - # verbatim by the release workflow, not built by MkDocs). Link out to it. - - Web installer: /projectMM/install/ + # verbatim by the release workflow, not built by MkDocs). A full URL (not a + # site-relative /projectMM/install/) so MkDocs treats it as the external + # resource it is, rather than warning about an unresolvable absolute path. + - Web installer: https://moonmodules.org/projectMM/install/ - Effects & building shows: - Effects: moonmodules/light/effects/effects.md - Layouts: moonmodules/light/layouts/layouts.md diff --git a/scripts/MoonDeck.md b/moondeck/MoonDeck.md similarity index 72% rename from scripts/MoonDeck.md rename to moondeck/MoonDeck.md index fe908366..d7ac2870 100644 --- a/scripts/MoonDeck.md +++ b/moondeck/MoonDeck.md @@ -1,8 +1,8 @@ # MoonDeck Script Reference -MoonDeck is projectMM's browser-based developer console: one page that builds, flashes, runs, tests, monitors, and checks the project across every target, and discovers and drives devices on the network. Every action it offers is a thin wrapper around a script under `scripts/`, so the CLI (`uv run scripts/<group>/<name>.py`) and MoonDeck run exactly the same code — agents typically use the CLI, humans use MoonDeck. For what MoonDeck *is* and where it sits in the workflow see [docs/building.md § MoonDeck](../docs/building.md#moondeck--the-dev-console); this page is the per-script reference. +MoonDeck is projectMM's browser-based developer console: one page that builds, flashes, runs, tests, monitors, and checks the project across every target, and discovers and drives devices on the network. Every action it offers is a thin wrapper around a script under `moondeck/`, so the CLI (`uv run moondeck/<group>/<name>.py`) and MoonDeck run exactly the same code — agents typically use the CLI, humans use MoonDeck. For what MoonDeck *is* and where it sits in the workflow see [docs/building.md § MoonDeck](../docs/building.md#moondeck--the-dev-console); this page is the per-script reference. -Launch it with `uv run scripts/moondeck.py` and open <http://localhost:8420>. The console has three tabs — **PC** (desktop build / run / test), **ESP32** (chip + port, build / flash / monitor), and **Live** (discovery and live runs against networked devices) — above a network bar and per-device board pickers. Script definitions live in `scripts/moondeck_config.json` (committed); runtime state (selected network, devices, ports) persists in `scripts/moondeck.json` (gitignored). +Launch it with `uv run moondeck/moondeck.py` and open <http://localhost:8420>. The console has three tabs — **PC** (desktop build / run / test), **ESP32** (chip + port, build / flash / monitor), and **Live** (discovery and live runs against networked devices) — above a network bar and per-device deviceModel pickers. Script definitions live in `moondeck/moondeck_config.json` (committed); runtime state (selected network, devices, ports) persists in `moondeck/moondeck.json` (gitignored). Below: the UI behaviours common to every card, described once, then one section per script grouped by the tab it appears on. Each section gives the equivalent CLI invocation, so the page doubles as the command reference for running anything without the browser. @@ -14,10 +14,8 @@ Below: the UI behaviours common to every card, described once, then one section - **Destructive-action confirm** — scripts flagged `destructive: true` (e.g. Erase Flash) pop a native confirm dialog before running. - **Tab persistence** — selected tab survives page refresh. - **Process detection** — on page load, checks if projectMM or idf.py is already running and shows Stop button. -- **Network bar** (top of the sidebar): switch between known networks. Each network holds its own device list, last-used serial port, and WiFi credentials (consumed by Improv). On startup, MoonDeck auto-selects the network whose subnet matches the host's current LAN — moving the laptop between networks usually requires no clicks. Manual override (the dropdown) pins the selection until the pinned network's subnet stops matching the host. Add / Rename buttons next to the dropdown manage the catalog. State persisted in `scripts/moondeck.json` under `networks` + `active_network`. -- **Board picker** on each device row: dropdown of physical boards from [web-installer/deviceModels.json](../web-installer/deviceModels.json) — the same catalog the web installer uses. When the device's firmware uniquely identifies one board (e.g. `esp32-eth` → Olimex Gateway), MoonDeck auto-deduces and mirrors the value to the device's `deviceModel` control on [SystemModule](../docs/moonmodules/core/SystemModule.md) via `POST /api/control` on next discover. For firmwares with no unique board (`esp32` runs on multiple), the user picks; MoonDeck pushes that value too. A device-reported board not in the catalog still shows up as `<key> (unknown)` so the value survives. MoonDeck's picker is a **text dropdown for an already-running device** — distinct from the web installer's flash-time *picture* board picker; both read the same catalog, but MoonDeck doesn't need the per-board `image`/`url` fields (those are installer-picker UX). Selecting a board pushes its full catalog config — each entry is a list of `{type, id, parent_id?, controls?}` module units (the [nested catalog schema](../web-installer/README.md), add-then-configure), so MoonDeck adds the board's modules (`POST /api/modules`) then sets their controls (`POST /api/control`); see `_push_board_to_device` in [moondeck.py](moondeck.py). -- **Pin profiles** on each device row (Save / Apply): a profile captures the device's current pin/peripheral config — the driver, board, network and audio modules and their control values, read from `GET /api/state` (effects/layouts and the always-on Preview are excluded; a profile is the *physical wiring*, not animation state). **Save** (`POST /api/save-profile`) stores it as a named entry under that device in `moondeck.json`; **Apply** (`POST /api/apply-profile`) re-pushes it through the same add-then-configure fan-out the board picker uses (`_apply_modules_to_device`, shared with `_push_board_to_device`). The captured unit shape is identical to a `deviceModels.json` entry's `modules`, so save→apply round-trips. Use it to restore GPIOs after a reflash wipes config, or to clone a working setup onto a second identical rig. - +- **Network bar** (top of the sidebar): switch between known networks. Each network holds its own device list, last-used serial port, and WiFi credentials (consumed by Improv). On startup, MoonDeck auto-selects the network whose subnet matches the host's current LAN — moving the laptop between networks usually requires no clicks. Manual override (the dropdown) pins the selection until the pinned network's subnet stops matching the host. Add / Rename buttons next to the dropdown manage the catalog. State persisted in `moondeck/moondeck.json` under `networks` + `active_network`. +- **Device-model picker** on each device row: dropdown of device models from [web-installer/deviceModels.json](../web-installer/deviceModels.json) — the same catalog the web installer uses. When the device's firmware uniquely identifies one deviceModel (e.g. `esp32-eth` → Olimex Gateway), MoonDeck auto-deduces and mirrors the value to the device's `deviceModel` control on [SystemModule](../docs/moonmodules/core/SystemModule.md) via `POST /api/control` on next discover. For firmwares with no unique deviceModel (`esp32` runs on multiple), the user picks; MoonDeck pushes that value too. A device-reported deviceModel not in the catalog still shows up as `<key> (unknown)` so the value survives. MoonDeck's picker is a **text dropdown for an already-running device** — distinct from the web installer's flash-time *picture* deviceModel picker; both read the same catalog, but MoonDeck doesn't need the per-deviceModel `image`/`url` fields (those are installer-picker UX). Selecting a deviceModel pushes its full catalog config — each entry is a list of `{type, id, parent_id?, controls?}` module units (the [nested catalog schema](../web-installer/README.md), add-then-configure), so MoonDeck adds the deviceModel's modules (`POST /api/modules`) then sets their controls (`POST /api/control`); see `_push_device` in [moondeck.py](moondeck.py). ## PC Tab @@ -28,7 +26,7 @@ Below: the UI behaviours common to every card, described once, then one section Build the desktop target using CMake. ```bash -uv run scripts/build/build_desktop.py +uv run moondeck/build/build_desktop.py ``` Runs `cmake -B build/<host> -DCMAKE_BUILD_TYPE=Release` then `cmake --build build/<host>`, where `<host>` is `macos`, `linux`, or `windows` depending on the OS this script runs on. The per-host directory keeps an experimental Linux build from clobbering a macOS one on the same machine, and mirrors the ESP32 side's `build/esp32-<board>/` shape. @@ -38,7 +36,7 @@ Runs `cmake -B build/<host> -DCMAKE_BUILD_TYPE=Release` then `cmake --build buil Run the desktop test suite. ```bash -uv run scripts/test/test_desktop.py +uv run moondeck/test/test_desktop.py ``` Runs `./build/<host>/test/mm_tests -s` (doctest with all test cases shown) — same per-host build dir as the desktop build above. @@ -48,7 +46,7 @@ Runs `./build/<host>/test/mm_tests -s` (doctest with all test cases shown) — s Launch the desktop executable as a detached background process and exit. The app keeps running across other MoonDeck scripts and outlives MoonDeck itself — the same model as flashing an ESP32, where the device runs independently of this console. ```bash -uv run scripts/run/run_desktop.py +uv run moondeck/run/run_desktop.py ``` Re-running is idempotent: any existing `projectMM` instance is stopped first, then a fresh one is launched. Output goes to `build/<host>/projectMM.log`. Build first. @@ -64,7 +62,7 @@ While the app is running, MoonDeck shows the button as **Stop** (a 5-second poll Locally preview the web installer page at <https://moonmodules.org/projectMM/install/> without tagging a release. Stages `web-installer/index.html` + `src/ui/install-picker.js` into `build/install-preview/` and serves them via Python's `http.server` on port 8421. ```bash -uv run scripts/run/preview_installer.py +uv run moondeck/run/preview_installer.py # open http://localhost:8421/ in Chrome / Edge / Opera ``` @@ -75,23 +73,63 @@ Long-running — MoonDeck shows **Stop** while the server is up. Two modes, pick Add `?nocache=1` to the URL to bypass the picker's 5-minute sessionStorage cache while editing. +### check_specs + +Verify every implemented MoonModule has a matching, up-to-date spec. + +```bash +uv run moondeck/check/check_specs.py +``` + +Scans `src/` for MoonModule `.h` files and checks each has a `docs/moonmodules/*.md` page whose control names / source facts still agree with the header. The always-run commit gate (fast, <1s) — catches `.h` ↔ doc drift even on doc-only commits. + ### check_platform_boundary Verify that platform-specific code stays inside `src/platform/`. ```bash -uv run scripts/check/check_platform_boundary.py +uv run moondeck/check/check_platform_boundary.py ``` Scans all source files outside `src/platform/` for forbidden includes and platform `#ifdef`s. +### check_devices + +Validate the installer device-model catalog (`web-installer/deviceModels.json`). + +```bash +uv run moondeck/check/check_devices.py +``` + +Checks each entry's required fields, that `firmwares` is a non-empty list, every `image` resolves on disk, the `System.deviceModel` control equals the entry name, module `type`s are factory-registered, `pins` controls live only on `*LedDriver` modules, and `flashBaud` (if set) is a standard esptool rate. The catalog's counterpart to `check_specs` for module docs. + +### check_firmwares + +Verify the firmware projection (`web-installer/firmwares.json`) matches the `FIRMWARES` source. + +```bash +uv run moondeck/check/check_firmwares.py +``` + +Regenerates the firmware list from `build_esp32.py`'s `FIRMWARES` dict and fails on drift from the committed `firmwares.json` — so a `FIRMWARES` edit without regenerating is caught. + +### collect_kpi + +Collect the per-target KPI line (tick/FPS, memory, sizes) for the commit message. + +```bash +uv run moondeck/check/collect_kpi.py +``` + +Captures a live tick from a connected ESP32 (and the PC scenario ticks) plus source/test line counts, emitting the `tick:Xus(FPS:Y)` one-liner the commit gate records. + ### scenario_pipeline Run scenario tests. Replays JSON scenario files in-process. ```bash -uv run scripts/scenario/run_scenario.py # run all -uv run scripts/scenario/run_scenario.py --name scenario_Layer_base_pipeline # run one +uv run moondeck/scenario/run_scenario.py # run all +uv run moondeck/scenario/run_scenario.py --name scenario_Layer_base_pipeline # run one ``` Scenarios are JSON files in `test/scenarios/`. Use the dropdown to run a single scenario or leave it on **all** to run the full suite. @@ -103,8 +141,8 @@ For a full description of each scenario, see the [scenario inventory](/api/docs/ Generate a human-readable history report from `git log` + `gh release list`. Writes a single markdown file at `build/history.md` (gitignored — the report is an artifact, not source; storing it in the repo would duplicate what git already carries). ```bash -uv run scripts/report/history_report.py # default: build/history.md -uv run scripts/report/history_report.py --out /tmp/h.md +uv run moondeck/report/history_report.py # default: build/history.md +uv run moondeck/report/history_report.py --out /tmp/h.md ``` Output shape: @@ -120,11 +158,11 @@ The MoonDeck button writes the file, prints a `MOONDECK_VIEW: /api/history-repor Capture UI screenshots of every module that has controls and save them to `docs/assets/`. ```bash -uv run scripts/docs/install_playwright.py # one-time (or use Install Playwright button in MoonDeck) -uv run scripts/docs/screenshot_modules.py # requires projectMM running on localhost:8080 -uv run scripts/docs/screenshot_modules.py --host 192.168.1.210:8080 -uv run scripts/docs/screenshot_modules.py --gif # also record 3-second GIF previews -uv run scripts/docs/screenshot_modules.py --force # re-capture and overwrite existing screenshots +uv run moondeck/docs/install_playwright.py # one-time (or use Install Playwright button in MoonDeck) +uv run moondeck/docs/screenshot_modules.py # requires projectMM running on localhost:8080 +uv run moondeck/docs/screenshot_modules.py --host 192.168.1.210:8080 +uv run moondeck/docs/screenshot_modules.py --gif # also record 3-second GIF previews +uv run moondeck/docs/screenshot_modules.py --force # re-capture and overwrite existing screenshots ``` The **GIF** and **Force** checkboxes in MoonDeck toggle these flags. @@ -148,27 +186,27 @@ After capture, run `update_module_docs` to insert the references into the module Insert screenshot and GIF references into `docs/moonmodules/**/*.md` files. ```bash -uv run scripts/docs/update_module_docs.py # update all -uv run scripts/docs/update_module_docs.py --dry-run # preview without writing +uv run moondeck/docs/update_module_docs.py # update all +uv run moondeck/docs/update_module_docs.py --dry-run # preview without writing ``` For each `.md` file, if `docs/assets/<type-folder>/<TypeName>.png` exists and the file doesn't already contain a screenshot reference, inserts the image after the first heading. If a matching `<TypeName>.gif` also exists, inserts the GIF reference on the next line. Safe to re-run — skips files that already have all references. -Also inserts MoonDeck tab screenshots and the installer screenshot into `scripts/MoonDeck.md` and `README.md` at fixed anchor points (defined in the `EXTRA_SHOTS` list in the script). +Also inserts MoonDeck tab screenshots and the installer screenshot into `moondeck/MoonDeck.md` and `README.md` at fixed anchor points (defined in the `EXTRA_SHOTS` list in the script). -Reports unreferenced screenshots — any PNG or GIF in `docs/assets/` not mentioned anywhere in `docs/` or `scripts/`. +Reports unreferenced screenshots — any PNG or GIF in `docs/assets/` not mentioned anywhere in `docs/` or `moondeck/`. ### build_docs **Preview Docs Site** — serve the documentation site (Material for MkDocs) from the `docs/` tree with live-reload, so you can view and iterate on it. Long-running: MoonDeck shows **Stop** while the server is up (like Installer Preview); a stray `mkdocs serve` is killed before a new one starts. The button passes `--serve`. ```bash -uv run scripts/docs/build_docs.py --serve # what the button runs → http://localhost:8422/projectMM/ (auto-reload) -uv run scripts/docs/build_docs.py # one-shot build to site/ (CI parity; no server) -uv run scripts/docs/build_docs.py --strict # promote every warning to an error (local anchor audit) +uv run moondeck/docs/build_docs.py --serve # what the button runs → http://localhost:8422/projectMM/ (auto-reload) +uv run moondeck/docs/build_docs.py # one-shot build to site/ (CI parity; no server) +uv run moondeck/docs/build_docs.py --strict # promote every warning to an error (local anchor audit) ``` -The preview binds **:8422** — the [Installer Preview](#preview_installer) owns :8421 and MoonDeck :8420, so all three servers run at once. Config is `mkdocs.yml`; deps (`mkdocs-material`) are declared inline in the script, so `uv run` provisions them on first use. The two test-inventory pages and each effect/modifier's inline test list are generated from the test files at build time (`scripts/docs/mkdocs_hooks.py`), so they're never committed and can't drift; `history/` and `backlog/` are built but kept off the nav. Warnings for links to repo files outside `docs/` (rewritten to GitHub URLs) and pre-existing stale anchors are expected — the build still succeeds. +The preview binds **:8422** — the [Installer Preview](#preview_installer) owns :8421 and MoonDeck :8420, so all three servers run at once. Config is `mkdocs.yml`; deps (`mkdocs-material`) are declared inline in the script, so `uv run` provisions them on first use. The two test-inventory pages and each effect/modifier's inline test list are generated from the test files at build time (`moondeck/docs/mkdocs_hooks.py`), so they're never committed and can't drift; `history/` and `backlog/` are built but kept off the nav. Warnings for links to repo files outside `docs/` (rewritten to GitHub URLs) and pre-existing stale anchors are expected — the build still succeeds. ## Live Tab @@ -180,11 +218,11 @@ The preview binds **:8422** — the [Installer Preview](#preview_installer) owns Run scenario tests against a live running device via HTTP. ```bash -uv run scripts/scenario/run_live_scenario.py # all scenarios vs localhost:8080 -uv run scripts/scenario/run_live_scenario.py --host 192.168.1.210 # vs ESP32 -uv run scripts/scenario/run_live_scenario.py --name scenario_MoonModule_control_change # one scenario -uv run scripts/scenario/run_live_scenario.py --update-baseline # save baseline -uv run scripts/scenario/run_live_scenario.py --compare-baseline # detect regressions +uv run moondeck/scenario/run_live_scenario.py # all scenarios vs localhost:8080 +uv run moondeck/scenario/run_live_scenario.py --host 192.168.1.210 # vs ESP32 +uv run moondeck/scenario/run_live_scenario.py --name scenario_MoonModule_control_change # one scenario +uv run moondeck/scenario/run_live_scenario.py --update-baseline # save baseline +uv run moondeck/scenario/run_live_scenario.py --compare-baseline # detect regressions ``` Executes scenario steps (add_module, set_control, delete_module) via REST API. Collects per-step FPS and heap measurements. Compares against stored baselines to detect performance regressions. Use the dropdown to run a single scenario or leave it on **all** to run the full suite. @@ -196,9 +234,9 @@ For a full description of each scenario, see the [scenario inventory](/api/docs/ End-to-end lights-over-UDP matrix test across every online board in moondeck.json's active network — the live proof for [NetworkReceiveEffect](../docs/moonmodules/light/effects/NetworkReceiveEffect.md) and [NetworkSendDriver](../docs/moonmodules/light/drivers/NetworkSendDriver.md). Each round one device is the sender and every other device listens: the PC seeds the sender **three times — once per protocol (ArtNet, E1.31, DDP), each with its own colour** — asserting the sender's `/ws` preview stream shows each one, then points the sender's own NetworkSendDriver at each listener with the protocol control cycled round-robin and asserts the listener's preview shows the sender's corrected colour (brightness + channel order replicated host-side). With one device online only the PC→device sweep runs. ```bash -uv run scripts/scenario/run_network_live.py # full matrix over all online devices -uv run scripts/scenario/run_network_live.py --device MM-70BC # only rounds with this sender -uv run scripts/scenario/run_network_live.py --tolerance 1 # loosen the per-channel byte match +uv run moondeck/scenario/run_network_live.py # full matrix over all online devices +uv run moondeck/scenario/run_network_live.py --device MM-70BC # only rounds with this sender +uv run moondeck/scenario/run_network_live.py --tolerance 1 # loosen the per-channel byte match ``` Everything it mutates (grid size → 16×16 for the run, NetworkSend `ip`/`protocol`/`enabled`, the temporarily added NetworkReceive effect) is restored afterwards, also on failure. Exit codes: `0` = all legs passed, `1` = a leg failed, `2` = environment problem (no online devices / no moondeck.json). Desktop listeners may need the OS firewall to allow UDP 6454/5568/4048. @@ -208,8 +246,8 @@ Everything it mutates (grid size → 16×16 for the run, NetworkSend `ip`/`proto Minimal **PC→device→PC latency probe** across **all three protocols**: per device, the PC sends one solid-colour frame over ArtNet, then E1.31, then DDP, each time timing how long until that colour appears in the device's `/ws` preview stream (PC → NetworkReceiveEffect → PreviewDriver → PC). The receiver autodetects each protocol on its own port, so there's no device reconfig between them. Reports min / median / max over N repeats per protocol and a per-device median-per-protocol comparison line — the spread is the signal for the latency / hiccup symptom, the protocol comparison shows which transport is fastest on a given board, and running across boards makes the per-chip difference visible (a classic ESP32 measures slower than an S3). Runs against **every device checked in the Live tab** (the same `selected` set the matrix test uses); unreachable checked devices are warned and skipped. The measured time includes the PreviewDriver's own fps quantisation (≈42 ms at the 24 fps default), so it's "state visible within" latency, not wire latency; raise the device's Preview fps to tighten it. Deliberately minimal — per-frame sequence matching, the device→device chain, and jitter/drop histograms are left as later extensions. ```bash -uv run scripts/scenario/run_network_roundtrip.py # every checked device, 10 probes each -uv run scripts/scenario/run_network_roundtrip.py --host 192.168.1.156 --repeats 20 # one explicit device +uv run moondeck/scenario/run_network_roundtrip.py # every checked device, 10 probes each +uv run moondeck/scenario/run_network_roundtrip.py --host 192.168.1.156 --repeats 20 # one explicit device ``` Captures and restores each device's grid and removes the temporary NetworkReceive on exit (also on failure). Exit codes match the matrix test: `0` = at least one device measured, `1` = none returned a frame, `2` = environment problem (no checked/reachable devices). @@ -219,10 +257,10 @@ Captures and restores each device's grid and removes the temporary NetworkReceiv Browser-faithful **3D-preview stream health probe** — measures the device's `/ws` preview the way a real browser tab experiences it, so the numbers match what a person watching the [PreviewDriver](../docs/moonmodules/light/drivers/PreviewDriver.md) preview sees. A plain one-shot WebSocket reader gives up the moment the device closes the socket, so it reports stalls a browser never shows (the browser reconnects) and misses the brief blips a browser does show; this probe replicates the real client in [app.js](../src/ui/app.js)'s `connectWs` — reads the binary frames, sends a `"ping"` text frame every 25 s, and **auto-reconnects on close with 500 ms→5 s backoff** — so a momentary device-side close registers as a short blip, not a frozen preview. Pure WebSocket client: **no device-side changes**, it observes the unmodified stream the device already broadcasts. Reports, per device: colour frames + sustained fps, reconnects (each a visible blip), `maxgap` (the longest stretch with no colour frame — the real "did it freeze?" number), and a `SMOOTH` / `CHOPPY` / `DEAD` verdict. Diagnostic, not a gate — it always exits `0`; read the verdict. Runs against **every device checked in the Live tab** (or an explicit `--host`); with no host it sweeps every online device on the active network. ```bash -uv run scripts/diag/preview_health.py # every online device, 30s each -uv run scripts/diag/preview_health.py --host 192.168.1.156 # one explicit device -uv run scripts/diag/preview_health.py localhost:8080 --grid 128 # PC build, force a 128×128 grid first -uv run scripts/diag/preview_health.py 192.168.1.132 --seconds 60 # longer window to catch rare stalls +uv run moondeck/diag/preview_health.py # every online device, 30s each +uv run moondeck/diag/preview_health.py --host 192.168.1.156 # one explicit device +uv run moondeck/diag/preview_health.py localhost:8080 --grid 128 # PC build, force a 128×128 grid first +uv run moondeck/diag/preview_health.py 192.168.1.132 --seconds 60 # longer window to catch rare stalls ``` When the verdict is `CHOPPY`/`DEAD`, the *cause* (which close path fired on the device) needs device-side serial logging — that scaffolding is added on-demand during diagnosis, separate from this always-on probe. Stamps nothing on the device; safe to run against a live preview a browser is also watching (subject to the 4-client `/ws` limit). @@ -250,7 +288,7 @@ The Firmware dropdown drives **Build** and **Flash**. Each board has its own bui Set up ESP-IDF Python environment. ```bash -uv run scripts/build/setup_esp_idf.py +uv run moondeck/build/setup_esp_idf.py ``` Finds the ESP-IDF installation and runs `install.sh` to create the Python venv. Run once after installing ESP-IDF or after a Python version change. When the installed checkout has drifted from the pinned commit (`PINNED_IDF_COMMIT`), it offers to check the pin out (a new dev converges on the validated IDF); `--no-checkout` keeps it warn-only for a dev migrating to a newer release. Building for the ESP32-S31 (a RISC-V preview target) needs its toolchain fetched once with `(cd ~/esp/esp-idf && ./install.sh esp32s31)` — the default install only pulls the classic-`esp32` toolchains. @@ -260,7 +298,7 @@ Finds the ESP-IDF installation and runs `install.sh` to create the Python venv. Clean the ESP32 build directory. ```bash -uv run scripts/build/clean_esp32.py +uv run moondeck/build/clean_esp32.py ``` Removes one ESP32 per-firmware build dir (`--firmware <name>`) or every `build/esp32-*/` plus a leftover `esp32/build/` if present (`--all`). Run a per-firmware clean after ESP-IDF updates, Python version changes, or anything else that should force a from-scratch build of that variant. Other firmwares' build dirs aren't touched. @@ -279,10 +317,10 @@ Build one of the shipping ESP32 firmware variants. The MoonDeck **Build** button CLI equivalent: ```bash -uv run scripts/build/build_esp32.py --firmware esp32 -uv run scripts/build/build_esp32.py --firmware esp32-eth -uv run scripts/build/build_esp32.py --firmware esp32-16mb -uv run scripts/build/build_esp32.py --firmware esp32s3-n16r8 +uv run moondeck/build/build_esp32.py --firmware esp32 +uv run moondeck/build/build_esp32.py --firmware esp32-eth +uv run moondeck/build/build_esp32.py --firmware esp32-16mb +uv run moondeck/build/build_esp32.py --firmware esp32s3-n16r8 ``` Auto-detects ESP-IDF installation, sets target if needed, builds, and shows flash/RAM usage summary. Each firmware writes into `build/esp32-<firmware>/`, so switching firmwares (or building several in one session) keeps every variant on disk — no clean rebuild on switch. @@ -291,7 +329,7 @@ The Ethernet PHY type and pin map are runtime config, not baked in: each firmwar Each ESP32-S3 SKU has its own firmware key because the sdkconfig fragment encodes flash size, partition layout, and PSRAM mode — flashing an `n16r8` binary onto a different module (e.g. N8R2) misaligns the partition table or fails PSRAM init. New SKUs become new keys (e.g. `esp32s3-n8r8`); we don't ship a generic `esp32s3` shortcut. -`--profile` is deprecated and accepted one release for migration: `--profile default` → `--firmware esp32`, `--profile eth-only` → `--firmware esp32-eth`. The legacy `build_esp32_ethonly.py` wrapper still works (it now forwards `--firmware esp32-eth`). +`--profile` is deprecated and accepted one release for migration: `--profile default` → `--firmware esp32`, `--profile eth-only` → `--firmware esp32-eth`. ### flash_esp32 @@ -299,19 +337,21 @@ Flash firmware to an ESP32 device. Reads `build/esp32-<firmware>/projectMM.bin` The MoonDeck button forwards the Firmware dropdown as `--firmware`. Flash exits cleanly with a "no build for <firmware> — run Build first" message when that dir doesn't exist. The log line up front confirms which build is being flashed and how old it is, e.g.: +**Flash baud** defaults to **921600** here (the CLI/MoonDeck path assumes a modern bench bridge — ~2x faster than the installer's safe 460800). A board with a flaky bridge pins a lower `flashBaud` in `deviceModels.json` to slow down (the LOLIN's CH340 → 460800); `--baud` overrides either. To resolve that per the *exact* board rather than the shared firmware, MoonDeck maps the selected Port → the device last flashed on it (its `last_port`) and forwards that device's deviceModel as `--device-model` — so one board's opt-down never leaks to a firmware-sibling with a fine bridge. + ```text ==> flashing esp32 build (1267 KB, built 3m ago) to /dev/tty.usbserial-0001 ``` ```bash -uv run scripts/build/flash_esp32.py --firmware esp32 --port /dev/tty.usbserial-0001 +uv run moondeck/build/flash_esp32.py --firmware esp32 --port /dev/tty.usbserial-0001 ``` `--firmware` is required — there's no longer a single canonical `esp32/build/` to fall back to. For a rack flash, loop over ports AND specify the firmware explicitly: ```bash for port in /dev/tty.usbserial-*; do - uv run scripts/build/flash_esp32.py --firmware esp32 --port "$port" + uv run moondeck/build/flash_esp32.py --firmware esp32 --port "$port" done ``` @@ -320,7 +360,7 @@ done Wipe the entire flash on an ESP32 device, including the LittleFS partition where persisted state lives (WiFi credentials, module list, control values). Flagged `destructive: true` so MoonDeck prompts a confirmation dialog before running. ```bash -uv run scripts/build/erase_flash_esp32.py --port /dev/tty.usbserial-0001 +uv run moondeck/build/erase_flash_esp32.py --port /dev/tty.usbserial-0001 ``` Typical use: forcing a fresh-first-boot after firmware experiments leave the LittleFS partition in a state the new firmware can't migrate from, or before testing the post-flash Improv provisioning flow as if the device just came out of the factory. After erase, re-run **Build** then **Flash** — the device boots with empty persistence and goes straight to AP-fallback / Improv-awaiting-credentials. @@ -330,7 +370,7 @@ Typical use: forcing a fresh-first-boot after firmware experiments leave the Lit Monitor serial output. Long-running — shows Stop button. ```bash -uv run scripts/run/monitor_esp32.py --port /dev/tty.usbserial-0001 +uv run moondeck/run/monitor_esp32.py --port /dev/tty.usbserial-0001 ``` Reads serial at 115200 baud. Output streams to MoonDeck's log and is saved to `esp32/monitor.log` for later inspection (useful when crashes flood the output). @@ -339,40 +379,40 @@ Reads serial at 115200 baud. Output streams to MoonDeck's log and is saved to `e Push WiFi credentials to a running projectMM device over USB-serial. Uses the [Improv-WiFi](https://www.improv-wifi.com/serial/) protocol — the same wire format the browser flow at improv-wifi.com uses. Device must be running a firmware that includes the Improv listener. -**One-click flow**: pick the device's port in MoonDeck, hit **Improv WiFi**. The script reads SSID + password from the **active network's WiFi block in `scripts/moondeck.json`** (the one shown in the network bar at the top of the sidebar). If that block is empty, it falls back to detecting the host machine's currently-joined WiFi (macOS Keychain / Linux NetworkManager / Windows `netsh`). The device replies with its new URL when STA comes up — typically 5-10 s end to end. +**One-click flow**: pick the device's port in MoonDeck, hit **Improv WiFi**. The script reads SSID + password from the **active network's WiFi block in `moondeck/moondeck.json`** (the one shown in the network bar at the top of the sidebar). If that block is empty, it falls back to detecting the host machine's currently-joined WiFi (macOS Keychain / Linux NetworkManager / Windows `netsh`). The device replies with its new URL when STA comes up — typically 5-10 s end to end. -**Board dropdown (pre-association injection)**: pick your physical board next to the Firmware dropdown and the flow forwards `--board` — the script then resolves the board's `deviceModels.json` settings and pushes the TX-power cap over the `SET_TX_POWER` vendor RPC **before** the credentials, plus `SET_DEVICE_MODEL` after success. This matters for brown-out-prone weak-powered boards (cap 8 dBm): at full TX power they fail their very first WiFi association, so the cap can't wait for the post-online HTTP injection. Leave the dropdown on "(any board)" for boards without special settings. +**Device-model dropdown (pre-association injection)**: pick your device model next to the Firmware dropdown and the flow forwards `--device-model` — the script then resolves the deviceModel's `deviceModels.json` settings and pushes the TX-power cap over the `SET_TX_POWER` vendor RPC **before** the credentials, plus `SET_DEVICE_MODEL` after success. This matters for brown-out-prone weak-powered device models (cap 8 dBm): at full TX power they fail their very first WiFi association, so the cap can't wait for the post-online HTTP injection. Leave the dropdown on "(any model)" for device models without special settings. ```bash # Equivalent CLI for a weak-powered board (cap resolved from deviceModels.json): -uv run scripts/build/improv_provision.py --port /dev/cu.usbmodem-XXX --board "ESP32-S3 N16R8 Dev" +uv run moondeck/build/improv_provision.py --port /dev/cu.usbmodem-XXX --device-model "ESP32-S3 N16R8 Dev" # Or set the cap explicitly without a catalog entry: -uv run scripts/build/improv_provision.py --port /dev/cu.usbmodem-XXX --tx-power 8 +uv run moondeck/build/improv_provision.py --port /dev/cu.usbmodem-XXX --tx-power 8 ``` ```bash # Use host's currently-joined WiFi (one click in MoonDeck → equivalent CLI): -uv run scripts/build/improv_provision.py --port /dev/tty.usbserial-XXXX +uv run moondeck/build/improv_provision.py --port /dev/tty.usbserial-XXXX # Override SSID + password (rack / CI / different network): -uv run scripts/build/improv_provision.py \ +uv run moondeck/build/improv_provision.py \ --port /dev/tty.usbserial-XXXX \ --ssid "MyWiFi" \ --password "hunter2" # Self-test the framing — no serial port needed (CI / pre-commit): -uv run scripts/build/improv_provision.py --self-test +uv run moondeck/build/improv_provision.py --self-test ``` Exits 0 with `==> provisioned: http://<ip>/` on success. On a USB hub, shell-loop over the ports: ```bash for port in /dev/tty.usbserial-*; do - uv run scripts/build/improv_provision.py --port "$port" + uv run moondeck/build/improv_provision.py --port "$port" done ``` -The host-WiFi reader lives at [scripts/build/host_wifi.py](build/host_wifi.py) and runs standalone for diagnosis (`uv run scripts/build/host_wifi.py` prints the resolved SSID + password). It first checks `scripts/moondeck.json`'s active network's `wifi` block; if empty, falls back to OS auto-detect. The first macOS auto-detect run pops a Keychain access dialog — the OS doing its job; we don't try to bypass it. The retired `scripts/build/wifi_credentials.json` source is gone — credentials now live per-network in moondeck.json, so moving the laptop between networks is just a dropdown switch. +The host-WiFi reader lives at [moondeck/build/host_wifi.py](build/host_wifi.py) and runs standalone for diagnosis (`uv run moondeck/build/host_wifi.py` prints the resolved SSID + password). It first checks `moondeck/moondeck.json`'s active network's `wifi` block; if empty, falls back to OS auto-detect. The first macOS auto-detect run pops a Keychain access dialog — the OS doing its job; we don't try to bypass it. The retired `moondeck/build/wifi_credentials.json` source is gone — credentials now live per-network in moondeck.json, so moving the laptop between networks is just a dropdown switch. Replaces v1's `deploy/wifi.py` + `deploy/flashfs.py --wifi` partition-baking flow — the device stays running, no flash mode required. Full module + protocol details: [docs/moonmodules/core/ImprovProvisioningModule.md](../docs/moonmodules/core/ImprovProvisioningModule.md). @@ -429,7 +469,7 @@ Exit codes: `0` = all checks passed, `1` = device-side failure (probe or provisi - [src/platform/esp32/platform_esp32_improv.cpp](../src/platform/esp32/platform_esp32_improv.cpp) — the UART listener task - [web-installer/index.html](../web-installer/index.html) — the web installer page - [src/ui/install-picker.js](../src/ui/install-picker.js) — the picker driving the install flow -- [scripts/build/improv_*.py](build/) — the host-side framing helpers +- [moondeck/build/improv_*.py](build/) — the host-side framing helpers Pair with `preview_installer`'s flash-ready mode (above) for a complete dev-environment proof that the install flow works before deploying to GitHub Pages. @@ -439,7 +479,7 @@ Pair with `preview_installer`'s flash-ready mode (above) for a complete dev-envi Print the most recent projectMM crash report and run log. ```bash -uv run scripts/run/show_crash_log.py +uv run moondeck/run/show_crash_log.py ``` On macOS, finds the newest `projectMM-*.ips` in `~/Library/Logs/DiagnosticReports/`, parses the JSON crash report, and prints the exception type, signal, faulting thread, and top 20 stack frames. If no crash report exists it falls back to the last 40 lines of `build/<host>/projectMM.log` so the run log is always reachable from one place. diff --git a/scripts/build/_idf_win_shim.py b/moondeck/build/_idf_win_shim.py similarity index 100% rename from scripts/build/_idf_win_shim.py rename to moondeck/build/_idf_win_shim.py diff --git a/scripts/build/build_desktop.py b/moondeck/build/build_desktop.py similarity index 100% rename from scripts/build/build_desktop.py rename to moondeck/build/build_desktop.py diff --git a/scripts/build/build_esp32.py b/moondeck/build/build_esp32.py similarity index 100% rename from scripts/build/build_esp32.py rename to moondeck/build/build_esp32.py diff --git a/scripts/build/clean_esp32.py b/moondeck/build/clean_esp32.py similarity index 100% rename from scripts/build/clean_esp32.py rename to moondeck/build/clean_esp32.py diff --git a/scripts/build/compute_version.py b/moondeck/build/compute_version.py similarity index 100% rename from scripts/build/compute_version.py rename to moondeck/build/compute_version.py diff --git a/scripts/build/erase_flash_esp32.py b/moondeck/build/erase_flash_esp32.py similarity index 100% rename from scripts/build/erase_flash_esp32.py rename to moondeck/build/erase_flash_esp32.py diff --git a/moondeck/build/flash_esp32.py b/moondeck/build/flash_esp32.py new file mode 100644 index 00000000..821fe147 --- /dev/null +++ b/moondeck/build/flash_esp32.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Flash a built ESP32 firmware to a device. + +Reads ``build/esp32-<firmware>/projectMM.bin``. The per-firmware build dir +(written by ``build_esp32.py``) makes "which firmware am I flashing" an +on-disk fact rather than an in-memory marker — switching firmwares is a +``--firmware`` change, not a clean-rebuild. + +Prints the artifact size + age before flashing so a stale build (one +from yesterday vs an edit five minutes ago) is visible in the log. +""" + +import argparse +import re +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent.parent +ESP32_DIR = ROOT / "esp32" + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from build_esp32 import find_idf, idf_env, idf_cmd, FIRMWARES, build_dir_for + + +CATALOG = ROOT / "web-installer" / "deviceModels.json" +# MoonDeck / CLI flashing defaults FAST: this path is the DIY bench, where the operator +# knows their board and the USB bridge is almost always a modern one that sustains 921600 +# (~2x faster). A board with a flaky bridge opts DOWN via its catalog `flashBaud` (e.g. a +# CH340 clone that stalls at 921600). This is the opposite of the web installer, which +# defaults to the safe 460800 because it serves unknown walk-up hardware (see +# install-orchestrator.js) — same catalog, different audience, different default. +DEFAULT_FLASH_BAUD = 921600 + + +def _catalog_flash_baud(firmware: str, device_model: str | None = None) -> int: + """The flash baud to use, from the deviceModel catalog. + + A deviceModel pins its own baud with a `flashBaud` field — down for a flaky bridge + (a CH340 clone at 460800) or up/explicit for a verified one (the S31 at 921600). + + Resolution: + - When the EXACT `device_model` is known (MoonDeck maps it from the port's device): + that entry's `flashBaud` if it sets one, else DEFAULT_FLASH_BAUD. Keying on the + exact board stops a per-model opt-down leaking to firmware-siblings — the LOLIN's + 460800 must not slow the Dig-Uno, which shares the `esp32` firmware but has a fine + bridge and no flashBaud of its own (so it gets the fast default, not the LOLIN's). + - When no device_model is given (a plain --firmware flash): the LOWEST `flashBaud` + among models sharing this firmware, so an opt-down still protects a board flashed + without a known model; else DEFAULT_FLASH_BAUD. + An unreadable catalog always yields DEFAULT_FLASH_BAUD. + """ + import json + try: + catalog = json.loads(CATALOG.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return DEFAULT_FLASH_BAUD + if device_model: + for e in catalog: + if isinstance(e, dict) and e.get("name") == device_model: + b = e.get("flashBaud") + return b if isinstance(b, int) else DEFAULT_FLASH_BAUD + # device_model given but not in the catalog — fall through to firmware/default. + bauds = { + e["flashBaud"] for e in catalog + if isinstance(e, dict) and firmware in (e.get("firmwares") or []) + and isinstance(e.get("flashBaud"), int) + } + return min(bauds) if bauds else DEFAULT_FLASH_BAUD + + +def _fmt_age(seconds: float) -> str: + """Compact human-readable age (5s, 12m, 3h, 2d).""" + s = int(seconds) + if s < 60: return f"{s}s" + if s < 3600: return f"{s // 60}m" + if s < 86400: return f"{s // 3600}h" + return f"{s // 86400}d" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--port", required=True, help="Serial port") + parser.add_argument("--firmware", required=True, choices=sorted(FIRMWARES), + help="Firmware variant to flash. The build for this " + "firmware must exist at build/esp32-<firmware>/ — " + "i.e. you must have run Build with the same " + "--firmware first.") + parser.add_argument("--baud", type=int, default=None, + help="esptool flash baud rate. When omitted, defaults to 921600 " + "(~2x faster — the DIY-bench assumption is a modern bridge), " + "unless the deviceModel's catalog 'flashBaud' pins it lower " + "for a flaky bridge (a CH340 clone that stalls at 921600 with " + "'chip stopped responding'). Pass --baud to override either. " + "(The web installer defaults to 460800 instead — it serves " + "unknown walk-up hardware.)") + parser.add_argument("--device-model", dest="device_model", default=None, + help="The deviceModel of the board being flashed (MoonDeck passes " + "it, mapped from the port's device). Lets the baud resolve by " + "the EXACT board so a per-model flashBaud opt-down doesn't leak " + "to firmware-siblings. Optional — omitted flashes resolve by " + "firmware then the fast default.") + args = parser.parse_args() + + # Default fast (921600) for the DIY bench; a deviceModel with a flaky bridge pins its + # own `flashBaud` in the catalog to slow down. An explicit --baud always wins; else the + # catalog value (exact deviceModel first, then lowest across firmware-siblings) applies; + # else the fast default. Data-driven per-board, like the rest of deviceModels.json. + baud = args.baud if args.baud is not None \ + else _catalog_flash_baud(args.firmware, args.device_model) + + if not ESP32_DIR.exists(): + print(f"ESP32 project directory not found: {ESP32_DIR}") + sys.exit(1) + + build_dir = build_dir_for(args.firmware) + image = build_dir / "projectMM.bin" + + if not image.exists(): + print(f"ERROR: no build for {args.firmware!r} at " + f"{build_dir.relative_to(ROOT)}/.") + print(f" Run Build with --firmware {args.firmware} first, then " + f"Flash again.") + sys.exit(2) + + size_kb = image.stat().st_size // 1024 + age = _fmt_age(time.time() - image.stat().st_mtime) + print(f"==> flashing {args.firmware} build ({size_kb} KB, built {age} ago) " + f"to {args.port}") + + idf_path = find_idf() + if not idf_path: + print("ESP-IDF not found. Install it or set IDF_PATH.") + sys.exit(1) + + env = idf_env(idf_path) + cmd = idf_cmd(idf_path) + # -B + -DSDKCONFIG mirror build_esp32.py so idf.py flash reads the + # per-firmware sdkconfig (the chip target lives in there). Without + # -DSDKCONFIG, idf.py reads esp32/sdkconfig at the project root, + # which may belong to a different firmware. + b_arg = [ + "-B", str(build_dir), + "-DSDKCONFIG=" + str(build_dir / "sdkconfig"), + ] + + # -b sets the esptool flash baud (idf.py's own default is also 460800). + # --baud 921600 matches the web installer for ~2x speed, but isn't the + # default because some USB bridges can't sustain it (see --baud help). + # Tee the output: esptool prints the board's efuse MAC during its connect + # phase ("MAC: xx:xx:..."), which we parse to key the flash breadcrumb by the + # board's stable identity — so MoonDeck records last_port on the exact device, + # even when two boards share a firmware. Streaming keeps the live progress on + # the console; we just also scan it. + print(f"==> flash baud: {baud}") + mac = "" + proc = subprocess.Popen(cmd + b_arg + ["flash", "-p", args.port, "-b", str(baud)], + cwd=ESP32_DIR, env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + for line in proc.stdout: + sys.stdout.write(line) # keep the live flash output visible + if not mac: + m = re.search(r"MAC:\s*([0-9A-Fa-f:]{17})", line) + if m: + mac = m.group(1).upper() + proc.wait() + if proc.returncode == 0: + _record_flash_event(args.port, args.firmware, mac) + sys.exit(proc.returncode) + + +def _record_flash_event(port: str, firmware: str, mac: str) -> None: + """Drop a `moondeck/.last_flash.json` breadcrumb so MoonDeck can link the + just-flashed serial port to the exact device. `mac` (the board's efuse MAC, + parsed from esptool's flash output) is the stable identity MoonDeck matches + on — a firmware-only match is ambiguous when two boards share a firmware. + MoonDeck's discover/refresh consumes it and clears it. Stored beside + moondeck.json so the whole "MoonDeck state" lives in one place.""" + import json, time + marker = ROOT / "moondeck" / ".last_flash.json" + marker.write_text(json.dumps({ + "port": port, + "firmware": firmware, + "mac": mac, + "ts": time.time(), + })) + + +if __name__ == "__main__": + main() diff --git a/scripts/build/generate_build_info.py b/moondeck/build/generate_build_info.py similarity index 89% rename from scripts/build/generate_build_info.py rename to moondeck/build/generate_build_info.py index a6279502..e700a365 100644 --- a/scripts/build/generate_build_info.py +++ b/moondeck/build/generate_build_info.py @@ -9,11 +9,11 @@ mechanism as MM_RELEASE / MM_FIRMWARE_NAME) so a published build carries a precise version: the core semver for a stable release, or a monotonic `<core>-dev.<N>` for a - moving `latest` build (see scripts/build/compute_version.py). + moving `latest` build (see moondeck/build/compute_version.py). MM_BUILD_DATE — __DATE__ " " __TIME__, evaluated by the compiler. MM_FIRMWARE_NAME — set by the build system as a -D flag (see - scripts/build/build_esp32.py firmware_cmake_args() and - scripts/build/package_desktop.py). The header carries an + moondeck/build/build_esp32.py firmware_cmake_args() and + moondeck/build/package_desktop.py). The header carries an #ifndef "unknown" fallback for builds that didn't set it. "Firmware" is the compiled-binary variant; the physical board is a separate concept the device cannot self-identify. @@ -40,7 +40,7 @@ content = f'''#pragma once -// Auto-generated from library.json by scripts/build/generate_build_info.py +// Auto-generated from library.json by moondeck/build/generate_build_info.py // -- do not edit by hand. Regenerated when library.json changes. // MM_VERSION defaults to the in-tree library.json semver, but the release @@ -62,8 +62,8 @@ // vs board. The physical hardware ("board") is a separate concept the // device cannot identify on its own. // -// ESP32: scripts/build/build_esp32.py firmware_cmake_args() -> -DMM_FIRMWARE_NAME="<key>" -// Desktop: scripts/build/package_desktop.py for release builds; local +// ESP32: moondeck/build/build_esp32.py firmware_cmake_args() -> -DMM_FIRMWARE_NAME="<key>" +// Desktop: moondeck/build/package_desktop.py for release builds; local // CMake builds fall through to "unknown" today (no harm: // local builds aren't published). #ifndef MM_FIRMWARE_NAME diff --git a/scripts/build/generate_firmwares.py b/moondeck/build/generate_firmwares.py similarity index 100% rename from scripts/build/generate_firmwares.py rename to moondeck/build/generate_firmwares.py diff --git a/scripts/build/generate_manifest.py b/moondeck/build/generate_manifest.py similarity index 100% rename from scripts/build/generate_manifest.py rename to moondeck/build/generate_manifest.py diff --git a/scripts/build/host_wifi.py b/moondeck/build/host_wifi.py similarity index 98% rename from scripts/build/host_wifi.py rename to moondeck/build/host_wifi.py index ebb0c4b9..5437e2c1 100644 --- a/scripts/build/host_wifi.py +++ b/moondeck/build/host_wifi.py @@ -8,11 +8,11 @@ Cross-platform: macOS / Linux / Windows. Standalone — only stdlib (subprocess, shutil, json, pathlib); no extra dependencies. Used by -`scripts/build/improv_provision.py` so a one-click MoonDeck button can +`moondeck/build/improv_provision.py` so a one-click MoonDeck button can push the host's credentials to a fresh ESP32 over USB-serial; also re-runnable on its own with `python3 host_wifi.py` for diagnosis. -Primary credentials source: `scripts/moondeck.json` under the active +Primary credentials source: `moondeck/moondeck.json` under the active network's `wifi` block (`{networks: [{name, wifi: {ssid, password}, …}], active_network: "..."}`). When MoonDeck is the entry point the user has already picked a network there, so credentials follow that choice — moving @@ -325,7 +325,7 @@ def _windows_credentials() -> Tuple[Optional[str], Optional[str]]: def _moondeck_active_network_credentials() -> Tuple[Optional[str], Optional[str]]: """Read SSID + password from the active network's wifi block in - scripts/moondeck.json. Returns (None, None) when moondeck.json is absent, + moondeck/moondeck.json. Returns (None, None) when moondeck.json is absent, has no networks, or the active network has no wifi set. Never raises — this is the primary credentials source so it must degrade silently to the OS auto-detect fallback. @@ -358,7 +358,7 @@ def get_host_wifi() -> Tuple[Optional[str], Optional[str]]: """Return `(ssid, password)` for WiFi provisioning. Resolution order: - 1. `scripts/moondeck.json` active network's `wifi` block — the primary + 1. `moondeck/moondeck.json` active network's `wifi` block — the primary source when MoonDeck is being used. The user picks the network in the MoonDeck dropdown; credentials follow. 2. OS auto-detect (macOS / Linux / Windows) — the fallback when diff --git a/scripts/build/improv_probe.py b/moondeck/build/improv_probe.py similarity index 98% rename from scripts/build/improv_probe.py rename to moondeck/build/improv_probe.py index 414f5484..e9ecbbaa 100644 --- a/scripts/build/improv_probe.py +++ b/moondeck/build/improv_probe.py @@ -20,7 +20,7 @@ No credentials are exchanged. The device's WiFi state isn't touched. Usage: - uv run scripts/build/improv_probe.py --port /dev/tty.usbserial-X + uv run moondeck/build/improv_probe.py --port /dev/tty.usbserial-X Exit codes: 0 = both RPCs answered, 1 = partial / no response, 2 = couldn't open the port. @@ -87,7 +87,7 @@ def main() -> int: # Step 1: GET_DEVICE_INFO. The device responds with an RPC_RESPONSE frame # carrying 4 strings: firmware name, version, chip family, device name. frame = build_frame(TYPE_RPC, build_simple_rpc(RPC_GET_DEVICE_INFO)) - print(f" → GET_DEVICE_INFO") + print(" → GET_DEVICE_INFO") ser.write(frame) ser.flush() @@ -142,7 +142,7 @@ def main() -> int: # device URL when the state is `provisioned`. if not sent_state: frame2 = build_frame(TYPE_RPC, build_simple_rpc(RPC_GET_CURRENT_STATE)) - print(f" → GET_CURRENT_STATE") + print(" → GET_CURRENT_STATE") ser.write(frame2) ser.flush() sent_state = True diff --git a/scripts/build/improv_provision.py b/moondeck/build/improv_provision.py similarity index 94% rename from scripts/build/improv_provision.py rename to moondeck/build/improv_provision.py index 2b816689..8f63223f 100644 --- a/scripts/build/improv_provision.py +++ b/moondeck/build/improv_provision.py @@ -168,7 +168,7 @@ def self_test() -> int: The script has no other test harness — the C++ parser at src/core/ImprovFrame.h + test/test_improv_frame.cpp covers the device side; this covers the host-CLI side. Re-runnable: `uv run - scripts/build/improv_provision.py --self-test`. + moondeck/build/improv_provision.py --self-test`. """ # Frame builder produces magic + version + type + len + payload + checksum. frame = build_frame(TYPE_RPC, b"\x42") @@ -219,11 +219,11 @@ def main() -> int: "on Windows; empty for open networks)") ap.add_argument("--timeout", type=float, default=45.0, help="Max seconds to wait for a final response (default: 45)") - ap.add_argument("--board", default=None, metavar="NAME", - help="Board name from web-installer/deviceModels.json (e.g. " - "'ESP32-S3 N16R8 Dev'). Resolves the board's TX-power cap " + ap.add_argument("--device-model", dest="device_model", default=None, metavar="NAME", + help="deviceModel name from web-installer/deviceModels.json (e.g. " + "'ESP32-S3 N16R8 Dev'). Resolves the deviceModel's TX-power cap " "(controls.Network.txPowerSetting) automatically and " - "pushes the board name via SET_DEVICE_MODEL after " + "pushes the name via SET_DEVICE_MODEL after " "provisioning — the same injection the web installer " "does. An explicit --tx-power overrides the lookup.") ap.add_argument("--tx-power", type=int, default=None, metavar="DBM", @@ -250,7 +250,7 @@ def main() -> int: if host_ssid is None: ap.error("--ssid not given and host WiFi not detected. " "Pass --ssid explicitly, or join a network on this " - "machine first (run `python3 scripts/build/host_wifi.py` " + "machine first (run `python3 moondeck/build/host_wifi.py` " "to verify detection).") args.ssid = host_ssid print(f"==> using host SSID: {args.ssid}") @@ -274,7 +274,7 @@ def main() -> int: print(f"ERROR: password too long ({len(args.password)} > 63 bytes)", file=sys.stderr) return 2 - if args.board: + if args.device_model: # deviceModels.json is the single source of truth for per-board injection — # same file the web installer and MoonDeck read. import json @@ -285,16 +285,16 @@ def main() -> int: except (OSError, json.JSONDecodeError) as e: print(f"ERROR: cannot read {boards_file}: {e}", file=sys.stderr) return 2 - entry = next((b for b in catalog if b.get("name") == args.board), None) + entry = next((b for b in catalog if b.get("name") == args.device_model), None) if entry is None: names = ", ".join(b.get("name", "?") for b in catalog) - print(f"ERROR: board {args.board!r} not in deviceModels.json ({names})", + print(f"ERROR: deviceModel {args.device_model!r} not in deviceModels.json ({names})", file=sys.stderr) return 2 cap = entry.get("controls", {}).get("Network", {}).get("txPowerSetting") if args.tx_power is None and isinstance(cap, int): args.tx_power = cap - print(f"==> board {args.board!r}: TX-power cap {cap} dBm from deviceModels.json") + print(f"==> deviceModel {args.device_model!r}: TX-power cap {cap} dBm from deviceModels.json") try: ser = serial.Serial(args.port, baudrate=115200, timeout=0.1) @@ -360,16 +360,16 @@ def main() -> int: idx += slen url = urls[0] if urls else "(no URL reported)" print(f"==> provisioned: {url}") - if args.board: + if args.device_model: # SET_DEVICE_MODEL vendor RPC (0xFE): [cmd][1+len][len][name] — # the same post-provision push the web installer does, so # the device persists its physical-board identity. - name = args.board.encode("utf-8") + name = args.device_model.encode("utf-8") ser.write(build_frame(TYPE_RPC, bytes([0xFE, 1 + len(name), len(name)]) + name)) ser.flush() time.sleep(0.5) # let the device's serial task consume it - print(f"==> pushed SET_DEVICE_MODEL {args.board!r}") + print(f"==> pushed SET_DEVICE_MODEL {args.device_model!r}") ser.close() return 0 diff --git a/scripts/build/improv_smoke_test.py b/moondeck/build/improv_smoke_test.py similarity index 97% rename from scripts/build/improv_smoke_test.py rename to moondeck/build/improv_smoke_test.py index a552190a..8c9f38c0 100755 --- a/scripts/build/improv_smoke_test.py +++ b/moondeck/build/improv_smoke_test.py @@ -30,10 +30,10 @@ - src/platform/esp32/platform_esp32_improv.cpp - web-installer/index.html - src/ui/install-picker.js - - scripts/build/improv_*.py + - moondeck/build/improv_*.py Usage: - uv run scripts/build/improv_smoke_test.py --port /dev/tty.usbserial-X + uv run moondeck/build/improv_smoke_test.py --port /dev/tty.usbserial-X Exit codes: 0 = all three checks passed, 1 = device-side failure (probe or provision), 2 = environment failure (couldn't open port, or @@ -192,14 +192,14 @@ def finish(exit_code: int) -> int: # ----- Step 3: reachable (optional) ----------------------------------- if args.no_network: - print(f"==> [3/3] network skipped (--no-network)") + print("==> [3/3] network skipped (--no-network)") return finish(0) if not url: # Provisioning succeeded but we couldn't parse the URL from the # output — older firmware variants may not report the URL line. # Don't fail (device IS provisioned), but skip the reachability # check with a clear message. - print(f"==> [3/3] network skipped (no device URL in provision output)") + print("==> [3/3] network skipped (no device URL in provision output)") summary["failure"] = "provision succeeded but device URL not parseable from output" return finish(0) diff --git a/scripts/build/setup_esp_idf.py b/moondeck/build/setup_esp_idf.py similarity index 98% rename from scripts/build/setup_esp_idf.py rename to moondeck/build/setup_esp_idf.py index 6ff55370..23466e92 100644 --- a/scripts/build/setup_esp_idf.py +++ b/moondeck/build/setup_esp_idf.py @@ -94,7 +94,7 @@ def main(): if args.no_checkout: print(f" To pin: (cd {idf_path} && git checkout {PINNED_IDF_COMMIT})\n") else: - answer = input(f" Check out the pinned commit now? [Y/n] ").strip().lower() + answer = input(" Check out the pinned commit now? [Y/n] ").strip().lower() if answer in ("", "y", "yes"): if _checkout_pinned(idf_path): print(f" Checked out {PINNED_IDF_COMMIT[:12]} ({PINNED_IDF_VERSION}).\n") diff --git a/scripts/check/check_devices.py b/moondeck/check/check_devices.py similarity index 66% rename from scripts/check/check_devices.py rename to moondeck/check/check_devices.py index bcf398bf..8f50d470 100644 --- a/scripts/check/check_devices.py +++ b/moondeck/check/check_devices.py @@ -31,11 +31,24 @@ MAIN_CPP = ROOT / "src" / "main.cpp" DOCS = ROOT / "docs" +# The device stores the injected deviceModel in SystemModule's deviceModel_[32] buffer (31 usable +# chars + NUL); a longer catalog name is truncated on-device and stops matching. Keep in step with +# that buffer in src/core/SystemModule.h. +DEVICE_MODEL_MAX = 31 + # Capability vocabulary. supported = what a module drives today; keep this list # in lockstep with the modules that actually exist. planned = peripherals with no # module yet (the backlog seed) — open-ended by design, so it is NOT whitelisted, # only type-checked. Adding a new supported capability means a module backs it. -SUPPORTED_VOCAB = {"LEDs", "WiFi", "Ethernet", "Audio", "IR"} +SUPPORTED_VOCAB = {"LEDs", "WiFi", "Ethernet", "Audio", "IR", "MQTT", "Hue"} + +# Flash bauds a board may pin via `flashBaud` — the standard esptool rates. The default +# differs by audience: the CLI / MoonDeck path defaults FAST (921600 — DIY bench, modern +# bridge), the web installer defaults SAFE (460800 — unknown walk-up hardware). A board +# sets `flashBaud` to override its resolved default in either direction — down for a flaky +# bridge (the LOLIN's CH340), up where a slow default needs raising. Keep in step with +# flash_esp32.py (_catalog_flash_baud) and install-orchestrator.js. +FLASH_BAUDS = {115200, 230400, 460800, 921600} # Boot-wired singletons: present on every device, added by code, so the catalog # references them by id without the factory creating them. Their catalog `type` @@ -84,6 +97,13 @@ def main(): if name in names_seen: errors.append(f"{where}: duplicate board name") names_seen.add(name) + # The name is injected into the device's SystemModule.deviceModel control, whose buffer is + # deviceModel_[32] (31 chars + NUL). A longer name is silently truncated on-device, so it no + # longer matches the catalog — MoonDeck then can't map it back and shows a duplicate + # "(unknown)" entry. Cap the source data so it always round-trips whole. + if isinstance(name, str) and len(name) > DEVICE_MODEL_MAX: + errors.append(f"{where}: name is {len(name)} chars; max {DEVICE_MODEL_MAX} " + f"(SystemModule.deviceModel_[32] truncates longer names on-device)") # --- firmwares (firmwares[0] is the default the picker pre-selects) --- fws = e.get("firmwares") @@ -104,6 +124,14 @@ def main(): errors.append(f"{where}: firmwares entries must not be whitespace-only " f"or have leading/trailing whitespace") + # --- flashBaud (optional) — a board opts into a faster flash baud only when + # its USB bridge is verified to sustain it (flash_esp32.py reads this). --- + baud = e.get("flashBaud") + # `bool` is an int subclass and `1.0 in {int}` is True, so guard the type + # explicitly — a float/bool flashBaud would stringify wrong for esptool. + if baud is not None and (type(baud) is not int or baud not in FLASH_BAUDS): + errors.append(f"{where}: flashBaud must be one of {sorted(FLASH_BAUDS)}, got {baud!r}") + # --- image resolves on disk + is a local path --- img = e.get("image") if img is not None: @@ -170,6 +198,38 @@ def main(): if isinstance(controls, dict) and "pins" in controls and not str(mtype).endswith("LedDriver"): errors.append(f"{where}: module '{mtype}' has a 'pins' control but is not a *LedDriver") + # Ethernet is explicit, not defaulted: a board that turns Ethernet ON (NetworkModule with + # a non-None ethType) must declare its board-wiring GPIOs, so the firmware never falls + # back to a per-chip default that is really one specific board's pins. (The Dig-Octa is + # why: GPIO5 is the classic-ESP32 default reset but that board uses it as an LED output; + # inheriting the default would drive an LED pin as an Ethernet reset.) Only the pins that + # are genuine BOARD WIRING are required — MDC/MDIO may stay at the IDF default (omit or + # -1) on RMII, since that's a real standard, not a board-specific value. + if mtype == "NetworkModule" and isinstance(controls, dict): + et = controls.get("ethType") + # ethType must be an int (a JSON string like "2" would silently skip the rule below and + # also isn't what the device deserializes into the Select) — reject a stringified value. + # `bool` is an int subclass, so exact-type-check (as flashBaud does) or a JSON `true` + # would pass as ethType 1 (LAN8720). + if et is not None and type(et) is not int: + errors.append(f"{where}: NetworkModule ethType must be an integer, got {et!r}") + if type(et) is int and et != 0: + # RMII LAN8720(1)/IP101(2): rst + clock. RGMII YT8531(4): mdc/mdio/rst. + # W5500 SPI(3): the four SPI bus pins. -1 is an allowed explicit value ("unused / + # IDF default"); what the rule forbids is OMITTING a board-wiring pin. + required = { + 1: ["ethRstGpio", "ethClockGpio"], + 2: ["ethRstGpio", "ethClockGpio"], + 4: ["ethMdcGpio", "ethMdioGpio", "ethRstGpio"], + 3: ["ethSpiMiso", "ethSpiMosi", "ethSpiSck", "ethSpiCs"], + }.get(et, []) + missing = [p for p in required if p not in controls] + if missing: + errors.append(f"{where}: NetworkModule sets ethType={et} but omits board-wiring " + f"pin(s) {missing} — Ethernet must be pinned explicitly, not " + f"inherited from a per-chip default (use -1 for a genuinely " + f"unused pin)") + # Every entry must carry the deviceModel identity (a System unit with the # `deviceModel` control). An empty `modules` list is also a failure — it has no # identity at all — so this is gated only on board_control_seen, not on `mods`. diff --git a/scripts/check/check_firmwares.py b/moondeck/check/check_firmwares.py similarity index 91% rename from scripts/check/check_firmwares.py rename to moondeck/check/check_firmwares.py index d64b3670..c608322b 100644 --- a/scripts/check/check_firmwares.py +++ b/moondeck/check/check_firmwares.py @@ -18,7 +18,7 @@ COMMITTED = ROOT / "web-installer" / "firmwares.json" # Reuse the generator's projection so the checker and generator can't disagree. -sys.path.insert(0, str(ROOT / "scripts" / "build")) +sys.path.insert(0, str(ROOT / "moondeck" / "build")) from generate_firmwares import build_doc # noqa: E402 @@ -37,7 +37,7 @@ def main(): if actual != expected: print(f"Firmware check: {n} variants, DRIFT") print(" web-installer/firmwares.json is stale — regenerate with:") - print(" uv run scripts/build/generate_firmwares.py --out web-installer/firmwares.json") + print(" uv run moondeck/build/generate_firmwares.py --out web-installer/firmwares.json") sys.exit(1) print(f"Firmware check: {n} variants, 0 issue(s)") diff --git a/scripts/check/check_platform_boundary.py b/moondeck/check/check_platform_boundary.py similarity index 100% rename from scripts/check/check_platform_boundary.py rename to moondeck/check/check_platform_boundary.py diff --git a/scripts/check/check_specs.py b/moondeck/check/check_specs.py similarity index 99% rename from scripts/check/check_specs.py rename to moondeck/check/check_specs.py index 708c6fc5..d2cc3603 100644 --- a/scripts/check/check_specs.py +++ b/moondeck/check/check_specs.py @@ -17,7 +17,7 @@ # src/{core,light} — gen_api discovers them). A summary/overview may link its # `moxygen/<stem>.md` in place of a `## Source` section; this set validates that link # points at a real generated page. Import by path so this check needs no PYTHONPATH tweak. -sys.path.insert(0, str(ROOT / "scripts" / "docs")) +sys.path.insert(0, str(ROOT / "moondeck" / "docs")) from gen_api import _discover_headers # noqa: E402 _API_STEMS = {Path(h).stem for h in _discover_headers()} diff --git a/scripts/check/collect_kpi.py b/moondeck/check/collect_kpi.py similarity index 97% rename from scripts/check/collect_kpi.py rename to moondeck/check/collect_kpi.py index 45872fa9..f187419b 100644 --- a/scripts/check/collect_kpi.py +++ b/moondeck/check/collect_kpi.py @@ -5,8 +5,8 @@ """Collect KPI data for commit descriptions. Usage: - uv run scripts/check/collect_kpi.py # full report - uv run scripts/check/collect_kpi.py --commit # commit message format + uv run moondeck/check/collect_kpi.py # full report + uv run moondeck/check/collect_kpi.py --commit # commit message format """ import argparse @@ -47,7 +47,7 @@ # no FPU, so the simplex float math dominates). MIN_ESP32_FPS_LED_PRODUCT = 10 * 16384 # 163840 -sys.path.insert(0, str(ROOT / "scripts" / "build")) +sys.path.insert(0, str(ROOT / "moondeck" / "build")) def run(cmd, cwd=None, timeout=30): # encoding="utf-8" + errors="replace" — subprocess defaults to the locale @@ -145,12 +145,12 @@ def _flush(): if buffer_lights: kpi["lights"] = max(buffer_lights) - check = ROOT / "scripts" / "check" / "check_platform_boundary.py" + check = ROOT / "moondeck" / "check" / "check_platform_boundary.py" if check.exists(): _, rc = run([sys.executable, str(check)]) kpi["boundary"] = "PASS" if rc == 0 else "FAIL" - specs = ROOT / "scripts" / "check" / "check_specs.py" + specs = ROOT / "moondeck" / "check" / "check_specs.py" if specs.exists(): out, rc = run([sys.executable, str(specs)]) kpi["specs_check"] = "PASS" if rc == 0 else "FAIL" @@ -225,7 +225,7 @@ def collect_esp32(): pass # Read tick/FPS from monitor.log. Do a live capture against the canonical - # port (scripts/moondeck.json) when the log is stale OR when it exists but + # port (moondeck/moondeck.json) when the log is stale OR when it exists but # has no parseable tick line — instead of silently skipping ESP32 KPI. log = ESP32_DIR / "monitor.log" stale = (not log.exists()) or (time.time() - log.stat().st_mtime) > 300 @@ -266,7 +266,7 @@ def _extract_esp32_tick(log, kpi): def _live_capture(log, seconds=15): """Capture ESP32 serial output to monitor.log for ~seconds. Returns True on success.""" import json - cfg = ROOT / "scripts" / "moondeck.json" + cfg = ROOT / "moondeck" / "moondeck.json" if not cfg.exists(): return False try: diff --git a/scripts/build/package_desktop.py b/moondeck/ci/package_desktop.py similarity index 100% rename from scripts/build/package_desktop.py rename to moondeck/ci/package_desktop.py diff --git a/scripts/build/verify_version.py b/moondeck/ci/verify_version.py similarity index 100% rename from scripts/build/verify_version.py rename to moondeck/ci/verify_version.py diff --git a/scripts/diag/preview_health.py b/moondeck/diag/preview_health.py similarity index 96% rename from scripts/diag/preview_health.py rename to moondeck/diag/preview_health.py index c8c2a198..d204f6d5 100644 --- a/scripts/diag/preview_health.py +++ b/moondeck/diag/preview_health.py @@ -29,8 +29,8 @@ device on moondeck.json's active network. Usage: - uv run scripts/diag/preview_health.py <ip[:port]> [--seconds N] [--grid SIZE] - uv run scripts/diag/preview_health.py # all online devices on the active network + uv run moondeck/diag/preview_health.py <ip[:port]> [--seconds N] [--grid SIZE] + uv run moondeck/diag/preview_health.py # all online devices on the active network --seconds N measurement window per device (default 30) --grid SIZE POST Grid width/height=SIZE, depth=1 before measuring (default: leave grid as-is) """ @@ -45,11 +45,11 @@ import time import urllib.request -# Repo root: scripts/diag/preview_health.py → ../../ +# Repo root: moondeck/diag/preview_health.py → ../../ _ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -# --- minimal RFC 6455 client (one connection), mirroring scripts/scenario/_preview_ws.py ----------- +# --- minimal RFC 6455 client (one connection), mirroring moondeck/scenario/_preview_ws.py ----------- class _Ws: """One /ws connection to host "ip[:port]". Reads server frames; can send a text frame (the @@ -235,7 +235,8 @@ def walk(m): def _online_devices(): """ip[:port] for every online device on moondeck.json's active network.""" - cfg = json.load(open(os.path.join(_ROOT, "scripts", "moondeck.json"))) + with open(os.path.join(_ROOT, "moondeck", "moondeck.json")) as f: + cfg = json.load(f) nets = cfg.get("networks", []) active = cfg.get("active_network") net = next((n for n in nets if n.get("name") == active), nets[0] if nets else None) diff --git a/scripts/docs/_test_metadata.py b/moondeck/docs/_test_metadata.py similarity index 97% rename from scripts/docs/_test_metadata.py rename to moondeck/docs/_test_metadata.py index af579d0d..6be95252 100644 --- a/scripts/docs/_test_metadata.py +++ b/moondeck/docs/_test_metadata.py @@ -1,7 +1,7 @@ """Shared parsers for unit-test and scenario metadata. -Both `scripts/docs/generate_test_docs.py` (writes markdown to disk) and -`scripts/moondeck.py` (serves HTML views in the MoonDeck UI) read the same +Both `moondeck/docs/generate_test_docs.py` (writes markdown to disk) and +`moondeck/moondeck.py` (serves HTML views in the MoonDeck UI) read the same `@module` / `@also` / per-TEST_CASE descriptions from unit tests and the same `module` / `also` / per-step `description` JSON fields from scenarios. This module owns that parsing so adding a new field — `@since`, a new step key — diff --git a/scripts/docs/build_docs.py b/moondeck/docs/build_docs.py similarity index 88% rename from scripts/docs/build_docs.py rename to moondeck/docs/build_docs.py index f053d7bf..a1dd71c1 100644 --- a/scripts/docs/build_docs.py +++ b/moondeck/docs/build_docs.py @@ -11,11 +11,11 @@ Phase 0 of the docs overhaul (docs/backlog/docs-system-overhaul.md): render the existing docs/ tree as a navigable site, config in mkdocs.yml. Dependencies are declared inline (PEP 723) so `uv run` provisions them — same pattern as the other -scripts/docs/ tools and the uv-everywhere project rule; no requirements file. +moondeck/docs/ tools and the uv-everywhere project rule; no requirements file. Link validation is governed by the `validation:` block in mkdocs.yml, not by --strict. The docs deliberately link OUT to repo files MkDocs can't see -(../src/*.h, ../CLAUDE.md, ../scripts/*) — the "drill into source" links that +(../src/*.h, ../CLAUDE.md, ../moondeck/*) — the "drill into source" links that resolve in the deployed tree; MkDocs warns on them because they're outside docs_dir. Those (and the pre-existing stale cross-doc anchors) are `warn`, so the normal build is the CI gate: it fails on a missing page or broken nav, not @@ -28,10 +28,10 @@ clash. Override with --port. Usage: - uv run scripts/docs/build_docs.py # build to site/ (CI gate) - uv run scripts/docs/build_docs.py --serve # live-preview at :8422 - uv run scripts/docs/build_docs.py --serve --port 9000 # serve on a custom port - uv run scripts/docs/build_docs.py --strict # local: fail on ANY warning (anchor audit) + uv run moondeck/docs/build_docs.py # build to site/ (CI gate) + uv run moondeck/docs/build_docs.py --serve # live-preview at :8422 + uv run moondeck/docs/build_docs.py --serve --port 9000 # serve on a custom port + uv run moondeck/docs/build_docs.py --strict # local: fail on ANY warning (anchor audit) """ import argparse @@ -108,9 +108,9 @@ def main() -> int: site = args.site_dir or "site/" print() print(f"==> docs built to {site}") - print(f" preview locally: uv run scripts/docs/build_docs.py --serve" + print(f" preview locally: uv run moondeck/docs/build_docs.py --serve" f" → http://localhost:{SERVE_PORT}/projectMM/") - print(f" deployed (after merge to main): https://moonmodules.org/projectMM/") + print(" deployed (after merge to main): https://moonmodules.org/projectMM/") elif args.serve: # A serve that just exited: remind where it was. print() diff --git a/scripts/docs/gen_api.py b/moondeck/docs/gen_api.py similarity index 99% rename from scripts/docs/gen_api.py rename to moondeck/docs/gen_api.py index b616f785..04b1699c 100644 --- a/scripts/docs/gen_api.py +++ b/moondeck/docs/gen_api.py @@ -4,7 +4,7 @@ *views* of the `///` comments in it — nothing is hand-restated. Pipeline: Doxygen (the de-facto-standard parser — robust on the C++20 that a Tree-sitter tool choked on) emits XML; moxygen renders it to Markdown with our custom template -(scripts/docs/moxygen-templates/). Called by mkdocs_hooks.py's on_files, the output +(moondeck/docs/moxygen-templates/). Called by mkdocs_hooks.py's on_files, the output injected into the virtual tree under moonmodules/{core,light}/moxygen/<Module>.md — the domain-nested layout the § Documentation model standard defines. diff --git a/scripts/docs/generate_test_docs.py b/moondeck/docs/generate_test_docs.py similarity index 95% rename from scripts/docs/generate_test_docs.py rename to moondeck/docs/generate_test_docs.py index ea591492..d4ca702c 100644 --- a/scripts/docs/generate_test_docs.py +++ b/moondeck/docs/generate_test_docs.py @@ -12,10 +12,10 @@ same bytes (verified by --check). Usage: - uv run scripts/docs/generate_test_docs.py # writes both files - uv run scripts/docs/generate_test_docs.py --unit # unit-tests.md only - uv run scripts/docs/generate_test_docs.py --scenario # scenario-tests.md only - uv run scripts/docs/generate_test_docs.py --check # exit 1 if regen would change files + uv run moondeck/docs/generate_test_docs.py # writes both files + uv run moondeck/docs/generate_test_docs.py --unit # unit-tests.md only + uv run moondeck/docs/generate_test_docs.py --scenario # scenario-tests.md only + uv run moondeck/docs/generate_test_docs.py --check # exit 1 if regen would change files """ import argparse @@ -45,7 +45,7 @@ def render_unit_tests(files: list[dict]) -> str: lines.append("# Unit Tests") lines.append("") lines.append( - "Auto-generated from `test/unit/{core,light}/unit_*.cpp` by `scripts/docs/generate_test_docs.py`. " + "Auto-generated from `test/unit/{core,light}/unit_*.cpp` by `moondeck/docs/generate_test_docs.py`. " "**Do not edit by hand** — update the source file's `@module` / `@also` " "and per-TEST_CASE `//` descriptions instead, then regenerate." ) @@ -269,7 +269,7 @@ def render_scenarios(files: list[dict]) -> str: lines.append("") lines.append( "Auto-generated from `test/scenarios/{core,light}/scenario_*.json` by " - "`scripts/docs/generate_test_docs.py`. **Do not edit by hand** — " + "`moondeck/docs/generate_test_docs.py`. **Do not edit by hand** — " "update the JSON file's top-level fields and per-step `description` " "/ `bounds` / `contract` / `observed` instead, then regenerate." ) @@ -278,8 +278,8 @@ def render_scenarios(files: list[dict]) -> str: "Scenario tests are the integration tier in the [test strategy](../testing.md): " "each one is a JSON script that drives the full pipeline (PC or live ESP32) " "and captures tick / heap per step against per-target contracts. " - "Run them with `scripts/scenario/run_scenario.py` (PC) or " - "`scripts/scenario/run_live_scenario.py` (live device). " + "Run them with `moondeck/scenario/run_scenario.py` (PC) or " + "`moondeck/scenario/run_live_scenario.py` (live device). " "See [testing.md § Performance contracts](../testing.md#performance-contracts-contracttarget) " "for the contract semantics." ) @@ -389,7 +389,7 @@ def main() -> int: ok = False if args.check and not ok: - print("Run scripts/docs/generate_test_docs.py to regenerate.") + print("Run `uv run moondeck/docs/generate_test_docs.py` to regenerate.") return 1 return 0 diff --git a/scripts/docs/install_playwright.py b/moondeck/docs/install_playwright.py similarity index 94% rename from scripts/docs/install_playwright.py rename to moondeck/docs/install_playwright.py index c70dc509..f463d6e0 100644 --- a/scripts/docs/install_playwright.py +++ b/moondeck/docs/install_playwright.py @@ -9,7 +9,7 @@ skips the download if the browser is already installed. Usage: - uv run scripts/docs/install_playwright.py + uv run moondeck/docs/install_playwright.py """ import subprocess diff --git a/scripts/docs/mkdocs_hooks.py b/moondeck/docs/mkdocs_hooks.py similarity index 98% rename from scripts/docs/mkdocs_hooks.py rename to moondeck/docs/mkdocs_hooks.py index 1f55579d..74240670 100644 --- a/scripts/docs/mkdocs_hooks.py +++ b/moondeck/docs/mkdocs_hooks.py @@ -8,7 +8,7 @@ plain one-line links so the effect cards stay compact (a link, not a case dump). 2. on_page_markdown — repoint links that escape docs/ into repo files - (`../src/*.h`, `../CLAUDE.md`, `../scripts/*`) to absolute GitHub blob URLs, + (`../src/*.h`, `../CLAUDE.md`, `../moondeck/*`) to absolute GitHub blob URLs, since the site can't host `src/` (pre-Doxide) and a relative link would 404. 3. on_post_build (serve-only) — stage the web installer into the built site under @@ -19,7 +19,7 @@ import re from pathlib import Path -# scripts/docs/ is on sys.path when mkdocs runs from repo root; import the shared parser. +# moondeck/docs/ is on sys.path when mkdocs runs from repo root; import the shared parser. import sys _HERE = Path(__file__).resolve().parent if str(_HERE) not in sys.path: @@ -36,7 +36,7 @@ # ---- source-link rewrite: docs link OUT to repo files the site can't host ---- -# The blob base for links that escape docs/ into the repo (src/, scripts/, test/, +# The blob base for links that escape docs/ into the repo (src/, moondeck/, test/, # CLAUDE.md, README.md, web-installer/). The site can't serve these — src/ isn't # published — so a relative link 404s on the deployed site. Rewrite to an absolute # GitHub blob URL so "source [Foo.h]" resolves everywhere (locally-served preview + @@ -52,7 +52,7 @@ _API_MODULES: dict[str, str] = {} # Repo top-level dirs/files a doc may link into but the site doesn't host. -_OUT_OF_DOCS = ("src/", "scripts/", "test/", "esp32/", "web-installer/", +_OUT_OF_DOCS = ("src/", "moondeck/", "test/", "esp32/", "web-installer/", ".github/", "CLAUDE.md", "README.md", "library.json", "CMakeLists.txt") # A markdown link into a repo file the site doesn't host. Two authored shapes, both @@ -103,8 +103,8 @@ def _sub(m: re.Match) -> str: # Fallback for links authored relative to the REPO ROOT (common in the # transient history/plans + backlog notes, written before their file sat # this deep under docs/): strip leading `../` and see if the remainder is - # itself an out-of-docs repo path. Catches `../../scripts/x.py` from a - # docs/history/plans/ page, which resolves to a nonexistent docs/scripts/x.py, + # itself an out-of-docs repo path. Catches `../../moondeck/x.py` from a + # docs/history/plans/ page, which resolves to a nonexistent docs/moondeck/x.py, # AND a bare `src/x.js` / `docs/plan.md` with no `../` at all (same intent, # written root-relative) — both 404 in-site, so send them to the GitHub blob. stripped = re.sub(r'^(?:\.\./)+', '', rel) diff --git a/scripts/docs/moxygen-templates/cpp/class.md b/moondeck/docs/moxygen-templates/cpp/class.md similarity index 100% rename from scripts/docs/moxygen-templates/cpp/class.md rename to moondeck/docs/moxygen-templates/cpp/class.md diff --git a/scripts/docs/moxygen-templates/cpp/index.md b/moondeck/docs/moxygen-templates/cpp/index.md similarity index 100% rename from scripts/docs/moxygen-templates/cpp/index.md rename to moondeck/docs/moxygen-templates/cpp/index.md diff --git a/scripts/docs/moxygen-templates/cpp/namespace.md b/moondeck/docs/moxygen-templates/cpp/namespace.md similarity index 100% rename from scripts/docs/moxygen-templates/cpp/namespace.md rename to moondeck/docs/moxygen-templates/cpp/namespace.md diff --git a/scripts/docs/moxygen-templates/cpp/page.md b/moondeck/docs/moxygen-templates/cpp/page.md similarity index 100% rename from scripts/docs/moxygen-templates/cpp/page.md rename to moondeck/docs/moxygen-templates/cpp/page.md diff --git a/scripts/docs/screenshot_modules.py b/moondeck/docs/screenshot_modules.py similarity index 97% rename from scripts/docs/screenshot_modules.py rename to moondeck/docs/screenshot_modules.py index b15d5e68..f04e9810 100644 --- a/scripts/docs/screenshot_modules.py +++ b/moondeck/docs/screenshot_modules.py @@ -22,9 +22,9 @@ Usage: # one effect, raw (no modifier), good preview size, with its GIF, overwriting: - uv run scripts/docs/screenshot_modules.py --filter Ripples --no-modifier --grid 32 --gif --force + uv run moondeck/docs/screenshot_modules.py --filter Ripples --no-modifier --grid 32 --gif --force # everything, same look: - uv run scripts/docs/screenshot_modules.py --no-modifier --grid 32 --gif --force + uv run moondeck/docs/screenshot_modules.py --no-modifier --grid 32 --gif --force Preview quality: --no-modifier removes the default MultiplyModifier so effects render RAW, not @@ -46,7 +46,7 @@ and repo transfers — if it's "missing" it was simply never installed here. Running server: the script captures against whatever is on --host (default -:8080). Start ONE fresh server: uv run scripts/build/build_desktop.py && ./build/<host>/projectMM +:8080). Start ONE fresh server: uv run moondeck/build/build_desktop.py && ./build/<host>/projectMM (or via MoonDeck's PC tab — same per-host build dir, so they share the binary). GOTCHA: a leftover binary on :8080 captures the WRONG images silently — e.g. a `build/macos/projectMM` from a MoonDeck run still bound to the port serves the OLD code, so a renamed/changed effect screenshots as its @@ -159,28 +159,28 @@ class _ExtrasOnlyDone(Exception): "moondeck_pc.png", f"{MOONDECK_URL}/?tab=pc", ".tab-content.active", - ["README.md", "scripts/MoonDeck.md"], + ["README.md", "moondeck/MoonDeck.md"], "## PC Tab", ), ( "moondeck_esp32.png", f"{MOONDECK_URL}/?tab=esp32", ".tab-content.active", - ["scripts/MoonDeck.md"], + ["moondeck/MoonDeck.md"], "## ESP32 Tab", ), ( "moondeck_live.png", f"{MOONDECK_URL}/?tab=live", ".tab-content.active", - ["scripts/MoonDeck.md"], + ["moondeck/MoonDeck.md"], "## Live Tab", ), ( "installer.png", INSTALLER_URL, "body", - ["README.md", "scripts/MoonDeck.md"], + ["README.md", "moondeck/MoonDeck.md"], "### preview_installer", ), ] @@ -373,10 +373,10 @@ def check_server_freshness(host: str) -> None: print(f" ⚠️ STALE SERVER: {host} is missing {len(missing)} type(s) the " f"source registers: {', '.join(missing[:6])}" + (" …" if len(missing) > 6 else "")) - print(f" The running binary is older than the current source. Most likely a") - print(f" second projectMM (a stale build/<host>/projectMM) is still on :8080.") - print(f" Fix: pkill -f projectMM then rebuild + run ONE server:") - print(f" uv run scripts/build/build_desktop.py && ./build/<host>/projectMM") + print(" The running binary is older than the current source. Most likely a") + print(" second projectMM (a stale build/<host>/projectMM) is still on :8080.") + print(" Fix: pkill -f projectMM then rebuild + run ONE server:") + print(" uv run moondeck/build/build_desktop.py && ./build/<host>/projectMM") # --------------------------------------------------------------------------- @@ -641,7 +641,7 @@ def main() -> int: _get(f"http://{args.host}/api/state", timeout=3) except Exception as e: print(f"Cannot reach projectMM at {args.host}: {e}") - print("Start the server first: uv run scripts/moondeck.py (then build+run from PC tab)") + print("Start the server first: uv run moondeck/moondeck.py (then build+run from PC tab)") return 1 # Module-related state — only meaningful when projectMM is reachable. @@ -669,7 +669,7 @@ def main() -> int: if uncaptured: print(f" ⚠️ {len(uncaptured)} registered effect/modifier(s) are NOT in this " f"script's MODULES list, so they get no screenshot: {', '.join(uncaptured)}") - print(f" Add them to MODULES (near the top of this file) to capture them.") + print(" Add them to MODULES (near the top of this file) to capture them.") # Optional pipeline tweaks for nicer effect previews (raw, higher-res). if args.no_modifier or args.grid: @@ -852,7 +852,7 @@ def _sweep_orphans(modules: list) -> None: ok = screenshot_module(page, args.host, actual_name, nav_root, out_path) if ok: - print(f"png ", end="", flush=True) + print("png ", end="", flush=True) captured.append(type_name) else: print("png-failed ", end="", flush=True) @@ -862,7 +862,7 @@ def _sweep_orphans(modules: list) -> None: ok = capture_preview_gif(page, args.host, actual_name, nav_root, gif_path) if ok: - print(f"gif ", end="", flush=True) + print("gif ", end="", flush=True) gif_captured.append(type_name) else: print("gif-failed ", end="", flush=True) @@ -925,7 +925,7 @@ def _sweep(modules: list) -> None: print(f" {name}: {reason}") if captured: print("\nNext steps:") - print(" Add module screenshots: uv run scripts/docs/update_module_docs.py") + print(" Add module screenshots: uv run moondeck/docs/update_module_docs.py") if "ui_overview" in captured: print(" Add UI overview to docs/architecture.md # Web UI section:") print(" ![UI overview](assets/ui/ui_overview.png)") diff --git a/scripts/docs/update_module_docs.py b/moondeck/docs/update_module_docs.py similarity index 95% rename from scripts/docs/update_module_docs.py rename to moondeck/docs/update_module_docs.py index 34db0d43..9c966f47 100644 --- a/scripts/docs/update_module_docs.py +++ b/moondeck/docs/update_module_docs.py @@ -18,7 +18,7 @@ are not referenced anywhere in docs/. Usage: - uv run scripts/docs/update_module_docs.py [--dry-run] + uv run moondeck/docs/update_module_docs.py [--dry-run] """ import argparse @@ -55,11 +55,11 @@ def asset_dir_for(type_name: str): # anchor_text: heading/line after which to insert (exact prefix match) EXTRA_SHOTS = [ ("moondeck_pc.png", "README.md", "![MoonDeck]"), - ("moondeck_pc.png", "scripts/MoonDeck.md", "## PC Tab"), - ("moondeck_esp32.png", "scripts/MoonDeck.md", "## ESP32 Tab"), - ("moondeck_live.png", "scripts/MoonDeck.md", "## Live Tab"), + ("moondeck_pc.png", "moondeck/MoonDeck.md", "## PC Tab"), + ("moondeck_esp32.png", "moondeck/MoonDeck.md", "## ESP32 Tab"), + ("moondeck_live.png", "moondeck/MoonDeck.md", "## Live Tab"), ("installer.png", "README.md", "**ESP32 — flash from your browser.**"), - ("installer.png", "scripts/MoonDeck.md", "### preview_installer"), + ("installer.png", "moondeck/MoonDeck.md", "### preview_installer"), ] @@ -248,11 +248,11 @@ def main() -> int: print(f" {p}") # --- Unreferenced screenshots check --- - # Collect all *.md text across docs/ and scripts/ (MoonDeck.md lives there). + # Collect all *.md text across docs/ and moondeck/ (MoonDeck.md lives there). all_docs_text = "" for md in sorted((ROOT / "docs").rglob("*.md")): all_docs_text += md.read_text(encoding="utf-8") - for md in sorted((ROOT / "scripts").rglob("*.md")): + for md in sorted((ROOT / "moondeck").rglob("*.md")): all_docs_text += md.read_text(encoding="utf-8") all_docs_text += (ROOT / "README.md").read_text(encoding="utf-8") diff --git a/scripts/moondeck.py b/moondeck/moondeck.py similarity index 74% rename from scripts/moondeck.py rename to moondeck/moondeck.py index 991fe43a..9fbb1105 100644 --- a/scripts/moondeck.py +++ b/moondeck/moondeck.py @@ -20,7 +20,7 @@ STATE_FILE = SCRIPTS_DIR / "moondeck.json" # Shared test-metadata parsers live next to the doc generator. Both this server -# and scripts/docs/generate_test_docs.py import from there so the two views of +# and moondeck/docs/generate_test_docs.py import from there so the two views of # the same source files (HTML in MoonDeck, markdown in docs/tests/) can't drift. sys.path.insert(0, str(SCRIPTS_DIR / "docs")) import _test_metadata as test_meta # noqa: E402 @@ -41,25 +41,25 @@ def _app_version(): APP_VERSION = _app_version() # --------------------------------------------------------------------------- -# Boards catalog (single source of truth, shared with the web installer) +# Device-model catalog (single source of truth, shared with the web installer) # --------------------------------------------------------------------------- -BOARDS_FILE = ROOT / "web-installer" / "deviceModels.json" +DEVICE_MODELS_FILE = ROOT / "web-installer" / "deviceModels.json" -def _load_boards(): +def _load_device_models(): """Load web-installer/deviceModels.json. Returns [] on missing/malformed file — - `_deduce_board` then always returns "" (no firmware uniquely identifies + `_deduce_device_model` then always returns "" (no firmware uniquely identifies a board), MoonDeck JS shows only the empty default. The web installer Step 2 picker will share this file. """ try: - return json.loads(BOARDS_FILE.read_text(encoding="utf-8")) + return json.loads(DEVICE_MODELS_FILE.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return [] -BOARDS = _load_boards() +DEVICE_MODELS = _load_device_models() FIRMWARES_FILE = ROOT / "web-installer" / "firmwares.json" @@ -79,7 +79,7 @@ def _load_firmwares(): # --------------------------------------------------------------------------- -# Script definitions (loaded from scripts.json) +# Script definitions (loaded from moondeck_config.json) # --------------------------------------------------------------------------- SCRIPTS_FILE = SCRIPTS_DIR / "moondeck_config.json" @@ -126,6 +126,23 @@ def _walk_modules(modules): yield from _walk_modules(m.get("children", [])) +def _device_sort_key(d): + """Sort devices by name (case-insensitive), IP as tiebreaker — the same order the on-device + DevicesModule uses (src/core/DevicesModule.h sortByName / ciLess), so MoonDeck's list and the + device's own list read the same. Used everywhere a device list is stored, so the persisted + moondeck.json is already in display order.""" + return (d.get("deviceName", "").lower(), d.get("ip", "")) + + +def _device_key(d): + """The device's identity: its MAC (from SystemModule.mac), lower-cased. The MAC is per-chip and + the only unchangeable identifier — an IP is a DHCP lease, not an identity, so it is NOT used as a + key. A device without a MAC (a WLED peer, or a projectMM device on firmware predating the `mac` + control) has NO stable identity: it's shown while online in a live scan but not persisted. Returns + "" for such a device — callers persist/dedup only truthy keys.""" + return (d.get("mac") or "").strip().lower() + + def _probe_device(ip, port=8080, timeout=0.4): """Probe a single IP for /api/state. Returns device info or None. @@ -134,12 +151,12 @@ def _probe_device(ip, port=8080, timeout=0.4): silently drop packets (firewalled hosts), and a subnet scan should not stall seconds on those. - Returns: { ip, deviceName, firmware, board } + Returns: { ip, deviceName, firmware, deviceModel } - `firmware` is the variant flashed (value of the `firmware` control on SystemModule, set from kFirmwareName in build_info.h). Used to deduce - `board` when the device hasn't been told its board yet. See + `deviceModel` when the device hasn't been told its model yet. See docs/architecture.md § Firmware vs board. - - `board` is the physical hardware key. Preferred source: the device's + - `deviceModel` is the physical-hardware identity (a catalog entry). Preferred source: the device's own `deviceModel` control on SystemModule (the value MoonDeck pushed earlier and the device persisted). Fall back to firmware-based deduction (catalog lookup) when the device hasn't been told yet — then MoonDeck @@ -156,48 +173,62 @@ def _probe_device(ip, port=8080, timeout=0.4): modules = data.get("modules", []) device_name = "" firmware = "" - device_board = "" + device_model = "" + mac = "" for m in _walk_modules(modules): - # deviceName, firmware AND deviceModel all live on SystemModule now - # (the deviceModel identity was folded in from the former BoardModule). + # deviceName, deviceModel (the board) and mac live on SystemModule; the flashed + # firmware-variant lives on FirmwareUpdateModule (`firmware`, alongside version/build). if m.get("type") == "SystemModule": for c in m.get("controls", []): if c.get("name") == "deviceName": device_name = c.get("value", "") or "" - elif c.get("name") == "firmware": - firmware = c.get("value", "") or "" elif c.get("name") == "deviceModel": - device_board = c.get("value", "") or "" + device_model = c.get("value", "") or "" + elif c.get("name") == "mac": + mac = c.get("value", "") or "" + elif m.get("type") == "FirmwareUpdateModule": + for c in m.get("controls", []): + if c.get("name") == "firmware": + firmware = c.get("value", "") or "" return { - "ip": f"{ip}:{port}", + # `mac` is the stable device IDENTITY (per-chip, survives DHCP IP changes); the merge + # keys on it. `ip` is just the current location — omit the port when it's HTTP's + # default (80, the ESP32 devices), keep a non-default port (8080, the desktop build). + "mac": mac, + "ip": ip if port == 80 else f"{ip}:{port}", "deviceName": device_name, "firmware": firmware, - "board": device_board or _deduce_board(firmware), + # `deviceModel` is the merged value (device-reported, else the firmware-deduced + # fallback). `_reportedModel` is what the DEVICE actually said (empty if it hasn't been + # told) — the merge compares against this to know whether to push the model to the + # device; using the already-deduced value would make them equal and skip the push. + "deviceModel": device_model or _deduce_device_model(firmware), + "_reportedModel": device_model, } except Exception: return None -def _deduce_board(firmware: str) -> str: - """Firmware → board name when exactly one catalog entry claims this - firmware. Returns "" when zero (unknown firmware) or multiple boards +def _deduce_device_model(firmware: str) -> str: + """Firmware → deviceModel name when exactly one catalog entry claims this + firmware. Returns "" when zero (unknown firmware) or multiple device models claim it (ambiguous — user picks). Catalog lives at web-installer/deviceModels.json; see docs/architecture.md § Firmware vs board. """ if not firmware: return "" - matches = [b["name"] for b in BOARDS if firmware in b.get("firmwares", [])] + matches = [b["name"] for b in DEVICE_MODELS if firmware in b.get("firmwares", [])] return matches[0] if len(matches) == 1 else "" -def _push_board_to_device(ip: str, board: str) -> bool: +def _push_device(ip: str, model: str) -> bool: """POST /api/control on the device for every per-board control in deviceModels.json. - For boards that have a catalog entry in web-installer/deviceModels.json: fans + For device models that have a catalog entry in web-installer/deviceModels.json: fans out the full `controls.<Module>.<control>` block (matching the web installer's and the device-side `?deviceModel=` Inject path — same generic - iteration, so adding a new field to a board entry Just Works without - code changes here). For boards without a catalog entry (custom names, + iteration, so adding a new field to a deviceModel entry Just Works without + code changes here). For device models without a catalog entry (custom names, unknown firmware): still pushes `System.deviceModel` so the bare name lands — keeps the legacy single-field behaviour as the fallback. @@ -207,18 +238,18 @@ def _push_board_to_device(ip: str, board: str) -> bool: field shape. `ip` is the "host:port" string from the device record (already includes - the port discovery picked). `board` is the catalog key MoonDeck wants + the port discovery picked). `deviceModel` is the catalog key MoonDeck wants the device to remember (empty string means "clear" — no push). """ - if not board: + if not model: return True # nothing to push; not a failure import urllib.request import urllib.error - # Look up the catalog entry. BOARDS is loaded at module init; we don't + # Look up the catalog entry. DEVICE_MODELS is loaded at module init; we don't # re-read deviceModels.json per push so a tight discover-refresh cycle # doesn't hammer the disk. If the user edits deviceModels.json, restart # MoonDeck (same as every other catalog change). - entry = next((b for b in BOARDS if b.get("name") == board), None) + entry = next((b for b in DEVICE_MODELS if b.get("name") == model), None) if entry is not None: modules = entry.get("modules") or [] else: @@ -226,7 +257,7 @@ def _push_board_to_device(ip: str, board: str) -> bool: # control (the identity lives on SystemModule now — no parent_id, it's the # boot-wired top-level module, so _apply just sets the control, no add). modules = [{"type": "System", "id": "System", - "controls": {"deviceModel": board}}] + "controls": {"deviceModel": model}}] return _apply_modules_to_device(ip, modules) @@ -235,8 +266,7 @@ def _apply_modules_to_device(ip: str, modules: list) -> bool: """Add-then-configure a list of module-with-controls units on a device. Each unit is `{type, id, parent_id?, controls?}` — the SAME shape deviceModels.json - catalog entries use and a saved device-profile stores, so both the board push - (_push_board_to_device) and a profile restore share this one fan-out. Per + catalog entries use, so the deviceModel push (_push_device) drives it. Per module: add it first when it has a parent_id (a fresh flash has no user-added modules like AudioModule, so a control write would 404), then set its controls. A module without parent_id is boot-wired/top-level (Board under System, @@ -280,84 +310,8 @@ def _post(path: str, body_obj: dict) -> bool: return True -# Module types whose controls a device-profile captures: the drivers + the -# system/network/audio config a user sets by hand. Effects/layouts/layers are -# animation state, not the device's physical pin wiring, so they're left out — -# a profile is "the GPIO/peripheral setup", re-applied after a reflash wipes it. -# SystemModule carries the device identity (deviceName + deviceModel, the latter -# folded in from the former BoardModule); its read-only telemetry controls are -# dropped by the _NON_REPLAYABLE_CONTROL_TYPES filter, leaving just the Text config. -_PROFILE_MODULE_TYPES = { - "SystemModule", "NetworkModule", "AudioModule", - "RmtLedDriver", "LcdLedDriver", "ParlioLedDriver", "NetworkSendDriver", -} - -# Control types that must NOT go into a captured profile: password values are -# XOR-obfuscated in /api/state (re-pushing double-encodes them), and display / -# display-int / progress are device-derived read-outs. These mirror the device's -# own isPersistable() exclusions (src/core/Control.cpp); strings are the JSON -# `type` values from controlTypeName(). -_NON_REPLAYABLE_CONTROL_TYPES = {"password", "display", "display-int", "progress"} - - -def _capture_device_profile(ip: str) -> "list | None": - """Read /api/state and flatten the module tree into profile units. - - Returns a list of `{type, id, parent_id?, controls}` units (the same shape - _apply_modules_to_device + deviceModels.json use), or None if the device is - unreachable. Only the config-bearing module types in _PROFILE_MODULE_TYPES are - captured (the physical pin/peripheral setup), and each module's controls list - `[{name, value}]` is collapsed to a `{name: value}` dict. parent_id comes from - the tree position so restore re-creates user-added modules under the right - container. The catalog `type` is the short id the device reports as the module - type; we keep it verbatim (e.g. "RmtLedDriver"), matching deviceModels.json. - """ - import urllib.request - import urllib.error - host = ip.split(":")[0] - port = ip.split(":")[1] if ":" in ip else "8080" - try: - with urllib.request.urlopen(f"http://{host}:{port}/api/state", timeout=1.0) as resp: - state = json.loads(resp.read()) - except (urllib.error.URLError, OSError, json.JSONDecodeError): - return None - - units: list = [] - - def _collect(modules, parent_id) -> None: - for m in modules or []: - mtype = m.get("type") - mid = m.get("name") # /api/state reports the instance id under "name" - if mtype in _PROFILE_MODULE_TYPES and mid: - controls = {} - for c in m.get("controls", []): - cn = c.get("name") - # Skip non-replayable control types: password values are - # XOR-obfuscated in /api/state (replaying them would double- - # encode), and display / display-int / progress are device- - # derived read-outs the device overwrites every tick. These are - # exactly the types the device's own isPersistable() (Control.cpp) - # refuses to save — mirror that here so a profile only carries - # writable config. Type strings come from controlTypeName(). - if c.get("type") in _NON_REPLAYABLE_CONTROL_TYPES: - continue - if cn is not None and "value" in c: - controls[cn] = c["value"] - unit = {"type": mtype, "id": mid} - if parent_id: - unit["parent_id"] = parent_id - if controls: - unit["controls"] = controls - units.append(unit) - # recurse with THIS module's id as the parent for its children - _collect(m.get("children", []), m.get("name")) - - _collect(state.get("modules", []), None) - return units - - -def _push_boards_in_parallel(pushes): - """Fire _push_board_to_device for each (ip, board) tuple in parallel. +def _push_devices_in_parallel(pushes): + """Fire _push_device for each (ip, deviceModel) tuple in parallel. Discovery + refresh probe a /24, so the push count is bounded by the device count (single digits in practice). A small thread pool keeps @@ -370,7 +324,7 @@ def _push_boards_in_parallel(pushes): import concurrent.futures with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: # list() forces all futures to start; the with-block waits for them. - list(pool.map(lambda p: _push_board_to_device(*p), pushes)) + list(pool.map(lambda p: _push_device(*p), pushes)) def discover_devices(subnet=""): @@ -398,13 +352,20 @@ def discover_devices(subnet=""): # subnet scan finds the LAN-IP entry, the explicit localhost probe finds the # other. Keep the LAN IP (usable from any device) and drop the redundant # localhost entry so the discovered list shows real network addresses. + # Compare on the HOST (ip without port), since a port-80 device is now stored bare. localIp = _lan_ip() - hasLanEntry = localIp and any(d["ip"].startswith(localIp + ":") for d in devices) + hasLanEntry = localIp and any(d["ip"].split(":", 1)[0] == localIp for d in devices) if hasLanEntry: - devices = [d for d in devices if not d["ip"].startswith("localhost:")] + devices = [d for d in devices if d["ip"].split(":", 1)[0] != "localhost"] + + # Attribute a recent flash's port to the device it flashed (by MAC) — same as refresh, so + # a flash → discover also records last_port. Whichever scan finds the device first links it. + _link_last_flash(devices) - # Sort by IP - devices.sort(key=lambda d: d["ip"]) + # Sort by device name, case-insensitive — matching the on-device DevicesModule list + # (src/core/DevicesModule.h sortByName / ciLess) so both lists read the same. IP is the + # tiebreaker so un-named / duplicate-named devices still have a stable order. + devices.sort(key=_device_sort_key) return devices, subnet @@ -413,10 +374,10 @@ def discover_devices(subnet=""): def _consume_last_flash() -> dict | None: - """Read the breadcrumb scripts/.last_flash.json that flash_esp32.py drops - after a successful flash. Returns {port, firmware} when the marker is - recent (< TTL); deletes the file so the link only happens once. Returns - None when there's no recent marker.""" + """Read the breadcrumb moondeck/.last_flash.json that flash_esp32.py drops + after a successful flash. Returns {port, mac, firmware} when the marker is + recent (< TTL); the caller deletes the file once it links the event so it + only applies once. Returns None when there's no recent marker.""" if not _LAST_FLASH_FILE.exists(): return None try: @@ -428,16 +389,17 @@ def _consume_last_flash() -> dict | None: with suppress(OSError): _LAST_FLASH_FILE.unlink() return None - return {"port": data.get("port", ""), "firmware": data.get("firmware", "")} + return {"port": data.get("port", ""), "mac": data.get("mac", ""), + "firmware": data.get("firmware", "")} def refresh_devices(known_devices): """Probe known devices to check online/offline status. - Preserves user-set fields (`board`, `last_port`) across refreshes: the - probe result carries fresh `firmware`/`deviceName` and a deduced `board` + Preserves user-set fields (`deviceModel`, `last_port`) across refreshes: the + probe result carries fresh `firmware`/`deviceName` and a deduced `deviceModel` (set only when firmware unambiguously identifies hardware), but a user-set - `board` for a firmware that can run on multiple boards (e.g. `esp32` on + `deviceModel` for a firmware that can run on multiple device models (e.g. `esp32` on LOLIN D32 vs generic DevKit) must survive a refresh. Same for `last_port`, which is set by the flash-event breadcrumb (see _consume_last_flash). """ @@ -447,7 +409,9 @@ def probe(device): host, port = ip.rsplit(":", 1) fresh = _probe_device(host, int(port)) else: - fresh = _probe_device(ip) + # A bare IP is a port-80 device (that's why it's stored without a port) — probe 80, NOT + # _probe_device's 8080 default, or the ESP32 refresh would fail and mark it offline. + fresh = _probe_device(ip, 80) if not fresh: return None # Merge: probe wins for live-readable fields; user-set / flash-tracked @@ -458,8 +422,8 @@ def probe(device): fresh["online"] = True if "selected" in device: fresh["selected"] = device["selected"] - if not fresh.get("board") and device.get("board"): - fresh["board"] = device["board"] + if not fresh.get("deviceModel") and device.get("deviceModel"): + fresh["deviceModel"] = device["deviceModel"] if device.get("last_port"): fresh["last_port"] = device["last_port"] return fresh @@ -470,24 +434,73 @@ def probe(device): with ThreadPoolExecutor(max_workers=16) as pool: refreshed = [r for r in pool.map(probe, known_devices) if r] - # Link a recent flash event to the device whose firmware matches. The - # flash flow writes scripts/.last_flash.json after a successful flash; - # here we attribute it to a refreshed device with the same firmware - # variant (newest "online with matching firmware" wins — usually the - # only candidate). After linking we consume the marker so the same - # event doesn't keep applying on every refresh. - last_flash = _consume_last_flash() - if last_flash and last_flash["firmware"]: - matches = [d for d in refreshed if d.get("firmware") == last_flash["firmware"]] - if len(matches) == 1: - matches[0]["last_port"] = last_flash["port"] - with suppress(OSError): - _LAST_FLASH_FILE.unlink() - # If 0 matches (device hasn't booted yet) or 2+ matches (ambiguous), - # leave the marker for the next refresh to retry / re-evaluate. + _link_last_flash(refreshed) return refreshed +def _mac_matches(flash_mac: str, device_mac: str) -> bool: + """True if a probed device's self-reported MAC identifies the just-flashed chip. + + esptool reads the raw 6-byte efuse MAC (e.g. 30:ED:A0:F3:D4:68), which is what the + breadcrumb stores. Most chips' `esp_efuse_mac_get_default()` returns that same value, + so an exact compare is the common case. But on some chips (the ESP32-S31) that API + returns the EUI-64 encoding of the efuse MAC — FF:FE inserted after the 3-byte OUI — + truncated back to 6 bytes: 30:ED:A0:F3:D4:68 → 30:ED:A0:FF:FE:F3. The device then + reports *that* everywhere (its `mac` control, its MM-xxxx name), so a raw-efuse + breadcrumb never matches it. Accept both the raw form and its EUI-64-truncated + derivation, computed forward from the efuse MAC (the reverse is lossy — the dropped + bytes are gone). A pure widening of the exact compare: it never breaks a real match. + """ + flash_mac = (flash_mac or "").strip().lower() + device_mac = (device_mac or "").strip().lower() + if not flash_mac or not device_mac: + return False + if flash_mac == device_mac: + return True + b = flash_mac.split(":") + if len(b) != 6: + return False + # EUI-64(efuse) = OUI(3) + ff:fe + rest(3); the device reports its first 6 bytes. + eui64_trunc = ":".join([b[0], b[1], b[2], "ff", "fe", b[3]]) + return device_mac == eui64_trunc + + +def _link_last_flash(probed: list) -> None: + """Set `last_port` on the device a recent flash targeted. flash_esp32.py writes + moondeck/.last_flash.json {port, mac, firmware} after a successful flash; the port is + attributed to the probed device with that MAC (the board's stable identity — unambiguous + even when two boards share a firmware). A legacy breadcrumb without a MAC falls back to a + firmware match, kept only for the single-candidate case. The marker is consumed (deleted) + once linked so it applies once; if the device isn't in this scan yet it's left for the next. + Called by both discover and refresh — whichever finds the device first links it.""" + last_flash = _consume_last_flash() + if not last_flash: + return + flash_mac = (last_flash.get("mac") or "").strip().lower() + if flash_mac: + matches = [d for d in probed if _mac_matches(flash_mac, d.get("mac"))] + elif last_flash.get("firmware"): # legacy breadcrumb without a MAC + matches = [d for d in probed if d.get("firmware") == last_flash["firmware"]] + else: + matches = [] + if len(matches) == 1: + port = last_flash["port"] + # A physical serial port maps to exactly ONE device at a time — boards get swapped on + # the same USB port, and ports drift between sessions. So before attributing it, strip + # this port from any OTHER device that still carries it (a stale link from a previous + # flash), else two records share a last_port and port→device lookups (baud resolution, + # the flash chip) can pick the wrong, stale board. + for d in probed: + if d is not matches[0] and d.get("last_port") == port: + d.pop("last_port", None) + matches[0]["last_port"] = port + with suppress(OSError): + _LAST_FLASH_FILE.unlink() # linked → consume so it applies once + # 0 matches (device hasn't rebooted into the new firmware yet) or 2+ (ambiguous legacy + # firmware match) → _consume_last_flash read but did NOT delete the file, so the marker + # stays on disk for the next scan to retry, until its TTL lapses. + + # --------------------------------------------------------------------------- # State management # --------------------------------------------------------------------------- @@ -600,7 +613,7 @@ def save_state(state): invites stale values when the device is reflashed/renamed via another host. They are re-read from `/api/state` on each refresh and live only in the in-memory device lists until the next save. User-set fields - (`board`, `last_port`, `selected`, `online`) persist. Iterates per network. + (`deviceModel`, `last_port`, `selected`, `online`) persist. Iterates per network. Write is atomic + serialized: a temp file in the same dir → fsync → rename. The rename is atomic on POSIX (same filesystem); fsync makes the bytes @@ -640,9 +653,9 @@ def save_state(state): def _strip_network_volatiles(network: dict) -> dict: """Return a copy of a network with volatile per-device fields stripped. - `board` is conditionally volatile: when it equals the value `_deduce_board` + `deviceModel` is conditionally volatile: when it equals the value `_deduce_device_model` produces from the device's current firmware, the next probe will re-derive - it for free — no need to persist. When the user picked a board manually + it for free — no need to persist. When the user picked a deviceModel manually (firmware doesn't deduce to anything, e.g. `esp32` could be LOLIN D32 or generic), the picker's choice is the only source so we must keep it. """ @@ -655,8 +668,8 @@ def _strip_network_volatiles(network: dict) -> dict: # firmware-deduced value (recomputable). Keep when the user picked # it (firmware deduces "" so the picker's value is the only source). firmware = d.get("firmware") or "" - if "board" in c and (not c["board"] or c["board"] == _deduce_board(firmware)): - del c["board"] + if "deviceModel" in c and (not c["deviceModel"] or c["deviceModel"] == _deduce_device_model(firmware)): + del c["deviceModel"] cleaned.append(c) out["devices"] = cleaned return out @@ -674,6 +687,29 @@ def _active_network(state: dict) -> dict | None: return None +def _strip_probe_transient(device: dict) -> dict: + """A copy of `device` without `_reportedModel` — the raw model string a probe attaches + for the push-compare, which must not be persisted to moondeck.json. One idiom for the + drop so discover and refresh can't diverge on which key(s) count as probe-transient.""" + return {k: v for k, v in device.items() if k != "_reportedModel"} + + +def _device_model_for_port(port: str) -> str: + """The deviceModel of the device last flashed via `port`, or "" if none. + + Maps a serial port back to a board by the `last_port` breadcrumb (set at flash time, + keyed per-device by MAC) in the active network. Used so a port-based flash can resolve + its baud by the exact deviceModel rather than the shared firmware. Returns "" when the + port isn't linked to any device, or the linked device has no deviceModel set.""" + if not port: + return "" + net = _active_network(load_state()) + for d in (net.get("devices") or []) if net else []: + if d.get("last_port") == port: + return d.get("deviceModel") or "" + return "" + + def _subnet_from_host_subnet(host_subnet: str) -> str: """Normalise `_get_local_subnet()` output (e.g. "192.168.1") to the network record's `subnet` field shape ("192.168.1.0/24").""" @@ -736,6 +772,13 @@ def kill_script(script_id: str): os.killpg(os.getpgid(proc.pid), signal.SIGTERM) except (OSError, ProcessLookupError): pass + # Close the pty master fd if the SSE stream never attached to close it (a killed-before- + # streamed proc — e.g. a fast Run→Run double click). Harmless if already closed. + if proc is not None: + stream = getattr(proc, "_mm_read_stream", None) + if stream is not None and getattr(proc, "_mm_master_fd", None) is not None: + with suppress(OSError): + stream.close() # Clean up any orphaned processes (e.g. projectMM after os.execv) script_def = next((s for s in SCRIPTS if s["id"] == script_id), None) @@ -762,7 +805,7 @@ def is_process_running(name: str) -> bool: def list_serial_ports() -> list[str]: """List available serial ports. - POSIX hosts: glob the conventional /dev/tty* device files (no deps). + POSIX hosts: glob the conventional /dev serial device files (no deps). Windows: read HKLM\\HARDWARE\\DEVICEMAP\\SERIALCOMM via winreg (stdlib). SERIALCOMM is the authoritative table the OS itself maintains for present COM ports — what pyserial reads under the hood — so a registry @@ -772,7 +815,11 @@ def list_serial_ports() -> list[str]: """ ports: list[str] = [] import glob - ports.extend(glob.glob("/dev/tty.usb*")) + # macOS exposes each USB serial as BOTH /dev/tty.X (call-in, blocks on DCD) and /dev/cu.X + # (call-out, non-blocking). Flashing (esptool / idf.py) uses the cu.* node, and flash_esp32.py + # records last_port as cu.*, so list cu.* here too — otherwise the port dropdown (tty.*) never + # matches a stored last_port and the selection silently falls back to "--". + ports.extend(glob.glob("/dev/cu.usb*")) ports.extend(glob.glob("/dev/ttyUSB*")) ports.extend(glob.glob("/dev/ttyACM*")) if sys.platform == "win32": @@ -909,12 +956,12 @@ def do_GET(self): elif self.path == "/api/test-modules": self._send_json({"modules": test_meta.list_test_modules()}) - elif self.path == "/api/boards": + elif self.path == "/api/device-models": # Serves web-installer/deviceModels.json (loaded at startup). The web - # installer (Step 2) will fetch the same file directly from - # Pages; MoonDeck reads it locally and exposes it here so the - # JS UI shares one source of truth with the Python deduce path. - self._send_json({"boards": BOARDS}) + # installer (Step 2) fetches the same file directly from Pages; + # MoonDeck reads it locally and exposes it here so the JS UI shares + # one source of truth with the Python deduce path. + self._send_json({"deviceModels": DEVICE_MODELS}) elif self.path.startswith("/api/unit-tests/"): self._serve_unit_tests_for_module() @@ -955,6 +1002,9 @@ def do_GET(self): elif self.path.startswith("/api/doc-asset/"): self._serve_doc_asset() + elif self.path.startswith("/firmware/") and self.path.endswith(".bin"): + self._serve_firmware_bin() + else: self._serve_static() @@ -981,9 +1031,57 @@ def _merge(s): result = mutate_state(_merge) self._send_json(result) - elif self.path == "/api/push-board": - # Push a single (ip, board) to a device. Called by the JS when the - # user picks a board from the per-device dropdown — saveState + elif self.path == "/api/ota": + # Wireless flash of a LOCAL build: MoonDeck serves build/esp32-<fw>/projectMM.bin over + # its own HTTP (the GET /firmware/<fw>.bin route) and hands the device that URL via the + # device's POST /api/firmware/url — the device pulls + flashes it (esp_https_ota accepts + # the plain-http LAN URL). Same "flash my local build" as USB, over WiFi. Body: {ip, firmware}. + import urllib.request + import urllib.error + import re + body = self._read_body() + params = json.loads(body) if body else {} + ip = params.get("ip", "") + firmware = params.get("firmware", "") + if not ip or not firmware: + self._send_json({"error": "ip and firmware required"}, 400) + return + # Validate both inputs before they reach a filesystem path or an outbound POST. `firmware` + # is a build-dir name (same "/" + ".." guard _serve_firmware_bin uses); `ip` must be a + # bare host/IP (a device on the LAN), not an arbitrary URL/authority — this handler POSTs + # to `http://<ip>/…`, so an unchecked value would let a caller aim MoonDeck at any host. + if "/" in firmware or ".." in firmware: + self._send_json({"error": "bad firmware name"}, 400) + return + if not re.fullmatch(r"[A-Za-z0-9.\-]{1,253}", ip): + self._send_json({"error": "bad ip/host"}, 400) + return + bin_path = ROOT / "build" / f"esp32-{firmware}" / "projectMM.bin" + if not bin_path.exists(): + self._send_json({"error": f"no build for {firmware!r} — run Build first"}, 404) + return + host_ip = _lan_ip() + if not host_ip: + self._send_json({"error": "MoonDeck can't determine its LAN IP (offline?)"}, 502) + return + bin_url = f"http://{host_ip}:{PORT}/firmware/{firmware}.bin" + # POST the URL to the device; it fetches + flashes. The device 202/200s immediately and + # reboots on success — the ota status is polled from the device's Firmware module. + try: + req = urllib.request.Request( + f"http://{ip}/api/firmware/url", + data=json.dumps({"url": bin_url}).encode(), + method="POST", headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=5) as resp: + self._send_json({"ok": True, "url": bin_url, "device_status": resp.status}) + except urllib.error.HTTPError as e: + self._send_json({"error": f"device rejected OTA (HTTP {e.code})"}, 502) + except (urllib.error.URLError, OSError) as e: + self._send_json({"error": f"device unreachable: {e}"}, 502) + + elif self.path == "/api/push-device": + # Push a single (ip, deviceModel) to a device. Called by the JS when the + # user picks a deviceModel from the per-device dropdown — saveState # alone persists the value in moondeck.json but the device also # needs to hear about it (the device persists its `deviceModel` control, # now on SystemModule, to /.config/SystemModule.json). The bulk push from discover / @@ -992,72 +1090,11 @@ def _merge(s): body = self._read_body() params = json.loads(body) if body else {} ip = params.get("ip", "") - board = params.get("board", "") + model = params.get("deviceModel", "") if not ip: self._send_json({"error": "ip required"}, 400) return - ok = _push_board_to_device(ip, board) - self._send_json({"ok": ok}) - - elif self.path == "/api/save-profile": - # Capture a device's current pin/peripheral config (drivers, board, - # network, audio) and store it as a named profile under the device - # record in moondeck.json. Lets a user re-apply the GPIO setup after a - # reflash wipes config, or clone it to a second identical rig. - body = self._read_body() - params = json.loads(body) if body else {} - ip = params.get("ip", "") - name = (params.get("name") or "").strip() - if not ip or not name: - self._send_json({"error": "ip and name required"}, 400) - return - modules = _capture_device_profile(ip) - if modules is None: - self._send_json({"error": "device unreachable"}, 502) - return - # Store under the device whose ip matches, in the ACTIVE network only — - # scoping to one network avoids a collision when two networks have - # overlapping private subnets (both a 192.168.1.x device). - def _store(state) -> None: - net = _active_network(state) - if not net: - return - for d in net.get("devices") or []: - if d.get("ip") == ip: - profiles = [p for p in d.get("profiles", []) - if p.get("name") != name] # replace same-named - profiles.append({"name": name, "modules": modules}) - d["profiles"] = profiles - mutate_state(_store) - self._send_json({"ok": True, "moduleCount": len(modules)}) - - elif self.path == "/api/apply-profile": - # Re-apply a stored profile to a device (same or a second identical - # board) via the shared add-then-configure fan-out. - body = self._read_body() - params = json.loads(body) if body else {} - ip = params.get("ip", "") - name = (params.get("name") or "").strip() - if not ip or not name: - self._send_json({"error": "ip and name required"}, 400) - return - # Scope the profile lookup to the device the UI applied it from, in the - # ACTIVE network only (its profile dropdown lists that device's own - # profiles). Matching by name across every device — or across networks - # with overlapping subnets — would restore the wrong pin map when two - # devices share a profile name like "default". - modules = None - net = _active_network(load_state()) - for d in (net.get("devices") or []) if net else []: - if d.get("ip") != ip: - continue - for p in d.get("profiles", []): - if p.get("name") == name: - modules = p.get("modules") - if modules is None: - self._send_json({"error": "profile not found"}, 404) - return - ok = _apply_modules_to_device(ip, modules) + ok = _push_device(ip, model) self._send_json({"ok": ok}) elif self.path == "/api/discover": @@ -1070,7 +1107,7 @@ def _store(state) -> None: # under the lock via mutate_state. devices, scanned_subnet = discover_devices(subnet) target_subnet = _subnet_from_host_subnet(scanned_subnet) - pushes = [] # (ip, board) tuples populated by _merge_discover + pushes = [] # (ip, deviceModel) tuples populated by _merge_discover def _merge_discover(state): # Attribute found devices to the network whose subnet matches @@ -1089,46 +1126,53 @@ def _merge_discover(state): existing.append(net) if net is None: return # nothing to merge — state unchanged - # Merge found devices into the network: keep existing user - # fields (board, last_port, selected), update online + probe - # fields. Drop devices no longer reachable — they stay - # offline elsewhere if previously known, here we trust the - # fresh scan as authoritative for what's on this subnet. - by_ip = {d["ip"]: d for d in net.get("devices", [])} + # Merge found devices into the network, keyed by MAC (the only persistent identity — + # see _device_key). A found device with a MAC updates/creates its record, carrying the + # user fields (deviceModel, last_port, selected) forward. A found device WITHOUT a MAC is + # shown this scan but not persisted (nothing stable to track it by). + by_key = {_device_key(d): d for d in net.get("devices", []) if _device_key(d)} merged = [] for fresh in devices: - ip = fresh.get("ip", "") - keep = by_ip.get(ip, {}) - out = {**fresh, "online": True, + key = _device_key(fresh) + if not key: + # No MAC → not a MoonDeck-manageable device (a WLED peer, or projectMM + # firmware predating the `mac` control). Dropped from the list entirely — not + # shown, not persisted — since MoonDeck can't flash it anyway. This is + # deliberate (the product-owner "MAC or it's not a device" rule), not an omission. + continue + keep = by_key.get(key, {}) + out = {**_strip_probe_transient(fresh), "online": True, "selected": keep.get("selected", False)} - if keep.get("board") and not out.get("board"): - out["board"] = keep["board"] + if keep.get("deviceModel") and not out.get("deviceModel"): + out["deviceModel"] = keep["deviceModel"] if keep.get("last_port"): out["last_port"] = keep["last_port"] merged.append(out) - # The device's `board` (from probe) is what's persisted - # device-side. If the merged value differs — typically - # because MoonDeck just deduced one from firmware on a - # device that hadn't been told yet — schedule a push so - # the next probe reads the value back from the device. - device_board = (fresh.get("board") or "") - merged_board = (out.get("board") or "") - if merged_board and merged_board != device_board: - pushes.append((ip, merged_board)) - # Devices that existed previously but weren't found in the - # scan stay in the network as offline (the user may want to - # keep them around for when the device comes back). Discover - # is additive — refresh is the verb that prunes. - found_ips = {d.get("ip") for d in devices} - for ip, dev in by_ip.items(): - if ip not in found_ips: + # Compare the device's OWN reported model (`_reportedModel`, empty if it hasn't + # been told) against the merged value. When they differ — typically MoonDeck just + # deduced a model from firmware, or the user picked one, for a device that hasn't + # heard it — schedule a push so the device persists it and reports it back next scan. + reported_model = (fresh.get("_reportedModel") or "") + merged_model = (out.get("deviceModel") or "") + if merged_model and merged_model != reported_model: + pushes.append((fresh.get("ip", ""), merged_model)) + # Previously-stored (MAC'd) devices not found in this scan stay as offline, so a + # known board isn't lost just because it's powered off. Discover is additive; refresh + # prunes. (MAC-less entries never entered by_key, so they simply fall away.) + found_keys = {_device_key(d) for d in devices if _device_key(d)} + for key, dev in by_key.items(): + if key not in found_keys: merged.append({**dev, "online": False}) + # Sort the FULL list (online + carried-over offline) by name so the persisted + # moondeck.json is stored in the same order it displays — matching the on-device + # DevicesModule (sortByName). IP is the tiebreaker for un-named devices. + merged.sort(key=_device_sort_key) net["devices"] = merged result = mutate_state(_merge_discover) # Fire pushes outside the lock — the state write has already # landed; pushes are best-effort device-side mirroring. - _push_boards_in_parallel(pushes) + _push_devices_in_parallel(pushes) self._send_json(result) elif self.path == "/api/refresh": @@ -1149,7 +1193,7 @@ def _merge_discover(state): self._send_json(state) return refreshed = refresh_devices(snapshot) - pushes = [] # (ip, board) tuples populated by _merge_refresh + pushes = [] # (ip, deviceModel) tuples populated by _merge_refresh def _merge_refresh(state): # Re-resolve `net` under the second lock — the network may have @@ -1163,25 +1207,32 @@ def _merge_refresh(state): # refresh_devices returns only devices that responded — devices # marked offline (didn't respond) are dropped from the list it # returns. Carry them forward as offline so the UI doesn't lose - # known-but-unreachable entries. - refreshed_ips = {d.get("ip") for d in refreshed} - merged = list(refreshed) + # known-but-unreachable entries. Persist only MAC'd devices (the identity rule): + # a responding device without a MAC is shown but not stored; a prior MAC'd device + # not seen this refresh stays as offline. + refreshed_keys = {_device_key(d) for d in refreshed if _device_key(d)} + merged = [_strip_probe_transient(d) + for d in refreshed if _device_key(d)] for prior in (target.get("devices") or []): - if prior.get("ip") not in refreshed_ips: + pk = _device_key(prior) + if pk and pk not in refreshed_keys: merged.append({**prior, "online": False}) + # Sort by name so the persisted list matches the display + the on-device + # DevicesModule order (see the discover merge above). + merged.sort(key=_device_sort_key) target["devices"] = merged - # Schedule a board push for every online device with a + # Schedule a deviceModel push for every online device with a # non-empty board. Redundant writes are cheap on the device # (Text-control write hits a 2s debounce — repeated identical # writes coalesce into one disk write). Catches the case # where the device lost its persisted value but MoonDeck # still has the user-set / deduced one. for dev in merged: - if dev.get("online") and dev.get("board") and dev.get("ip"): - pushes.append((dev["ip"], dev["board"])) + if dev.get("online") and dev.get("deviceModel") and dev.get("ip"): + pushes.append((dev["ip"], dev["deviceModel"])) result = mutate_state(_merge_refresh) - _push_boards_in_parallel(pushes) + _push_devices_in_parallel(pushes) self._send_json(result) else: @@ -1194,14 +1245,16 @@ def _handle_run(self, script_id: str, params: dict): self._send_json({"error": "unknown script"}, 404) return - # Refuse to launch with a required selector unset: the underlying - # script declares the arg `required=True`, so running it bare just - # leaks argparse's "the following arguments are required" usage error - # into the log. Surface a clear message naming what to pick instead. + # Refuse to launch with a genuinely-required selector unset: the underlying + # script declares the arg `required=True`, so running it bare just leaks + # argparse's "the following arguments are required" usage error into the log. + # Surface a clear message naming what to pick instead. Only port + firmware + # are mandatory; `scenario` and `module` are OPTIONAL filters whose empty / + # "all" value the scenario scripts accept as "run everything" (run_scenario.py + # / run_live_scenario.py both `--name`/`--module default=None`), so they are + # NOT in this list — requiring them here wrongly blocks the "all" case. REQUIRED = [("needs_port", "port", "a serial port"), - ("needs_firmware", "firmware", "a firmware variant"), - ("needs_scenario", "scenario", "a scenario"), - ("needs_module", "module", "a module")] + ("needs_firmware", "firmware", "a firmware variant")] missing = [label for flag, key, label in REQUIRED if script_def.get(flag) and not params.get(key)] if missing: @@ -1226,20 +1279,32 @@ def _handle_run(self, script_id: str, params: dict): cmd.extend(["--firmware", params["firmware"]]) if script_def.get("needs_port") and params.get("port"): cmd.extend(["--port", params["port"]]) + # For a port-based flash, hand the script the deviceModel of the device last + # flashed via that port (matched by last_port in the active network), so its + # baud resolves by the EXACT board — a per-model flashBaud opt-down (the LOLIN's + # 460800) then can't leak to a firmware-sibling with a fine bridge (the Dig-Uno). + # No-op when the port maps to no known device or that device has no model. + if script_def.get("pass_port_device_model"): + model = _device_model_for_port(params["port"]) + if model: + cmd.extend(["--device-model", model]) if script_def.get("needs_scenario") and params.get("scenario"): cmd.extend(["--name", params["scenario"]]) if script_def.get("needs_module") and params.get("module"): cmd.extend(["--module", params["module"]]) - # pass_board: forward the board picked in the UI's provisioning - # dropdown (state.provisionBoard) so improv_provision.py injects that + # pass_device_model: forward the deviceModel picked in the UI's provisioning + # dropdown (state.provisionDeviceModel) so improv_provision.py injects that # board's TX-power cap BEFORE provisioning (the weak-power brown-out fix). - # No firmware-deduce fallback: the only pass_board script + # No firmware-deduce fallback: the only pass_device_model script # (improv_provision) doesn't declare needs_firmware, so params never # carries a firmware to deduce from — the dropdown is the sole source. - if script_def.get("pass_board"): - board = params.get("board") - if board: - cmd.extend(["--board", board]) + if script_def.get("pass_device_model"): + # The UI sends this as `device_model` (app.js sets params.device_model); accept the + # camelCase form too so either key reaches the flag. Without the snake_case key the + # value was silently dropped — the TX-power cap then never injected before provisioning. + device_model = params.get("device_model") or params.get("deviceModel") + if device_model: + cmd.extend(["--device-model", device_model]) # improv_provision.py's CLI flag if params.get("host"): cmd.extend(["--host", params["host"]]) for flag in script_def.get("flags", []): @@ -1247,16 +1312,50 @@ def _handle_run(self, script_id: str, params: dict): cmd.append(flag["arg"]) try: - popen_kwargs = dict( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd=str(ROOT), - ) - if not _IS_WIN: + # Force live output: a child (build_esp32 → idf.py → ninja, esptool, …) block-buffers + # its stdout when it detects a PIPE instead of a terminal, so nothing reaches the SSE + # stream until it exits — a long ESP32 build dumps its whole log at once. On Unix we + # run it under a pseudo-terminal (pty) so every process in the tree sees a TTY and + # line-buffers naturally (the standard `unbuffer`/`script` trick); the SSE reader reads + # the pty master. PYTHONUNBUFFERED belts-and-suspenders the Python layer, and is the + # only lever on Windows (no pty there — ninja still chunks, but flash/monitor improve). + env = {**os.environ, "PYTHONUNBUFFERED": "1"} + popen_kwargs = dict(cwd=str(ROOT), env=env) + read_stream = None # what the SSE reader reads (pty master, or proc.stdout) + master_fd = None # pty master to close on teardown (Unix only) + slave_fd = None # pty slave — closed in parent right after spawn (or on failure) + if _IS_WIN: + popen_kwargs["stdout"] = subprocess.PIPE + popen_kwargs["stderr"] = subprocess.STDOUT + popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP + else: + import pty + master_fd, slave_fd = pty.openpty() + popen_kwargs["stdout"] = slave_fd + popen_kwargs["stderr"] = slave_fd popen_kwargs["start_new_session"] = True + try: + proc = subprocess.Popen(cmd, **popen_kwargs) + except Exception: + # Popen failed (bad cmd, uv not on PATH): close both pty fds so a failed launch + # doesn't leak two fds per attempt in this long-running server, then re-raise. + for fd in (master_fd, slave_fd): + if fd is not None: + with suppress(OSError): + os.close(fd) + raise + if _IS_WIN: + read_stream = proc.stdout else: - popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP - proc = subprocess.Popen(cmd, **popen_kwargs) + os.close(slave_fd) # parent doesn't write; child owns the slave end + # Buffered binary reader over the pty master. Its read raises OSError (EIO) when + # the child exits and closes the slave — the SSE loop treats that as clean EOF. + read_stream = os.fdopen(master_fd, "rb", buffering=0) + # Stash the read stream + master fd on the proc so _handle_stream can find them (kept in + # _running keyed by proc). kill_script closes the master fd too, so a killed-before- + # streamed proc doesn't orphan it. + proc._mm_read_stream = read_stream + proc._mm_master_fd = master_fd with _lock: _running[script_id] = proc self._send_json({"status": "started", "pid": proc.pid}) @@ -1278,9 +1377,19 @@ def _handle_stream(self, script_id: str): self.send_header("Connection", "keep-alive") self.end_headers() + # Read from the pty master (Unix) or the pipe (Windows) — see _handle_run. On a pty, the + # master read raises OSError (EIO) once the child exits and closes the slave; treat that as + # EOF, same as the pipe's b"" sentinel. + stream = getattr(proc, "_mm_read_stream", None) or proc.stdout try: - for line in iter(proc.stdout.readline, b""): - text = line.decode("utf-8", errors="replace").rstrip("\n") + while True: + try: + line = stream.readline() + except OSError: + break # pty EIO on child exit → EOF + if not line: + break # pipe EOF + text = line.decode("utf-8", errors="replace").rstrip("\r\n") self.wfile.write(f"data: {json.dumps(text)}\n\n".encode()) self.wfile.flush() @@ -1293,8 +1402,17 @@ def _handle_stream(self, script_id: str): except (BrokenPipeError, ConnectionResetError): pass finally: + # Close the pty master fd (Unix) so the kernel reclaims it; the pipe closes with proc. + master_fd = getattr(proc, "_mm_master_fd", None) + if master_fd is not None: + with suppress(OSError): + stream.close() + # Pop only if THIS proc is still the registered one — a re-run under the same id + # (fast Run→Run) may have replaced it while this stream was finishing; popping then + # would drop the NEW proc and break its Stop button. with _lock: - _running.pop(script_id, None) + if _running.get(script_id) is proc: + _running.pop(script_id, None) def _serve_doc(self): """Serve any docs/**/*.md file as styled HTML with deep-link anchor support. @@ -1563,6 +1681,26 @@ def _serve_doc_asset(self): self.end_headers() self.wfile.write(data) + def _serve_firmware_bin(self): + """Serve a local firmware image for a device OTA: GET /firmware/<fw>.bin → + build/esp32-<fw>/projectMM.bin. The device (handed this URL by /api/ota) fetches it over + the LAN and flashes it. Only the exact <fw>.bin shape is served, mapped to the known build + dir — no arbitrary path, so this can't read outside build/esp32-*/.""" + fw = self.path[len("/firmware/"):-len(".bin")] + if not fw or "/" in fw or ".." in fw: + self.send_error(400, "bad firmware name") + return + bin_path = ROOT / "build" / f"esp32-{fw}" / "projectMM.bin" + if not bin_path.exists(): + self.send_error(404, f"no build for {fw}") + return + data = bin_path.read_bytes() + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + def _serve_markdown_as_html(self, md_path, anchor): """Render a markdown file to HTML for the View pane. Handles headings (with id slugs for deep-linking), fenced code blocks, diff --git a/scripts/moondeck_config.json b/moondeck/moondeck_config.json similarity index 95% rename from scripts/moondeck_config.json rename to moondeck/moondeck_config.json index 5443f15a..3eb6e3a2 100644 --- a/scripts/moondeck_config.json +++ b/moondeck/moondeck_config.json @@ -55,6 +55,14 @@ "long_running": true, "process_name": "preview_installer.py" }, + { + "id": "check_specs", + "tab": "pc", + "group": "check", + "label": "Spec Check", + "help": "check_specs", + "script": "check/check_specs.py" + }, { "id": "check_platform_boundary", "tab": "pc", @@ -205,7 +213,8 @@ "help": "flash_esp32", "script": "build/flash_esp32.py", "needs_port": true, - "needs_firmware": true + "needs_firmware": true, + "pass_port_device_model": true }, { "id": "erase_flash_esp32", @@ -236,7 +245,7 @@ "help": "improv_provision", "script": "build/improv_provision.py", "needs_port": true, - "pass_board": true + "pass_device_model": true }, { "id": "improv_probe", diff --git a/scripts/moondeck_ui/app.js b/moondeck/moondeck_ui/app.js similarity index 77% rename from scripts/moondeck_ui/app.js rename to moondeck/moondeck_ui/app.js index 200696df..2e99464a 100644 --- a/scripts/moondeck_ui/app.js +++ b/moondeck/moondeck_ui/app.js @@ -12,18 +12,18 @@ let scripts = []; let firmwares = []; let scenarios = []; // [{name, module, also}] let testModules = []; // ["CamelCaseName", ...] -// Boards catalog loaded from /api/boards (served by moondeck.py from -// web-installer/deviceModels.json). Replaces the previously-hardcoded boardOptions -// list — same file the web installer (Step 2) will fetch directly from -// GitHub Pages. Empty until init() loads it; renderDevices waits on init. -let boards = []; // [{ key, label, firmwares: [...] }] (firmwares[0] is the default) +// Device-model catalog loaded from /api/device-models (served by moondeck.py from +// web-installer/deviceModels.json) — the same file the web installer fetches. Empty until +// init() loads it; renderDevices waits on init. +let deviceModels = []; // [{ name, firmwares: [...], ... }] — `name` is the identifier + label + // (single-name catalog, matched by b.name); firmwares[0] is the default. // State shape (post-networks refactor): // { networks: [{name, subnet, wifi: {ssid, password}, port, devices: [...]}], // active_network: "Home", // active_network_user_pinned: bool, // set true when user picks the dropdown // firmware, scenario, module, tab, flag_* } // `firmware` is the variant flashed onto the ESP32 (esp32 / esp32-eth / -// esp32-16mb / esp32s3-n16r8) — separate from the per-device `board` +// esp32-16mb / esp32s3-n16r8) — separate from the per-device `deviceModel` // (physical hardware) inside each network's devices list. See // docs/architecture.md § Firmware vs board. // Devices and the active serial port now live INSIDE the active network. @@ -36,6 +36,16 @@ function getActiveNetwork() { return (state.networks || []).find(n => n.name === state.active_network) || null; } +// A fetch abort signal that fires after `ms`. AbortSignal.timeout is unsupported on older browsers +// (and throws if called), which would abort the fetch before it starts — fall back to an +// AbortController + setTimeout so the request still runs. +function timeoutSignal(ms) { + if (typeof AbortSignal !== "undefined" && AbortSignal.timeout) return AbortSignal.timeout(ms); + const c = new AbortController(); + setTimeout(() => c.abort(), ms); + return c.signal; +} + // --------------------------------------------------------------------------- // Init // --------------------------------------------------------------------------- @@ -59,6 +69,11 @@ async function init() { delete state.board; } if (!firmwares.includes(state.firmware)) state.firmware = ""; + // Migrate the provisioning-picker key from its old name (board → device model). + if (state.provisionBoard !== undefined && state.provisionDeviceModel === undefined) { + state.provisionDeviceModel = state.provisionBoard; + delete state.provisionBoard; + } const scenResp = await fetch("/api/scenarios"); const scenData = await scenResp.json(); @@ -69,15 +84,15 @@ async function init() { testModules = modData.modules || []; try { - const boardsResp = await fetch("/api/boards"); - const boardsData = await boardsResp.json(); - boards = boardsData.boards || []; + const modelsResp = await fetch("/api/device-models"); + const modelsData = await modelsResp.json(); + deviceModels = modelsData.deviceModels || []; } catch { - boards = []; // empty catalog → picker shows only "(unknown board)" + deviceModels = []; // empty catalog → picker shows only "(unknown model)" } renderFirmwareSelect(); - renderBoardSelect(); + renderDeviceModelSelect(); renderScripts(); renderNetworkBar(); try { renderDevices(); } catch (e) { console.error("renderDevices:", e); } @@ -189,6 +204,11 @@ document.getElementById("view-forward").addEventListener("click", () => viewNavA document.getElementById("view-refresh").addEventListener("click", () => { if (viewFrame.src) viewFrame.src = viewFrame.src; }); +// Open the currently-viewed URL (the device UI) in a real browser tab — escapes the sandboxed +// iframe entirely, so anything the frame restricts (modals, popups, downloads) works natively. +document.getElementById("view-open").addEventListener("click", () => { + if (viewFrame.src) window.open(viewFrame.src, "_blank", "noopener"); +}); function showInView(url) { viewFrame.src = url; @@ -507,7 +527,7 @@ async function runScriptOnce(script, btn, extraParams) { if (script.needs_port) params.port = (getActiveNetwork()?.port) || ""; if (script.needs_scenario) params.scenario = state.scenario; if (script.needs_module) params.module = state.module; - if (script.pass_board) params.board = state.provisionBoard || ""; + if (script.pass_device_model) params.device_model = state.provisionDeviceModel || ""; for (const flag of (script.flags || [])) { const stateKey = `flag_${script.id}_${flag.id}`; params[`flag_${flag.id}`] = stateKey in state ? state[stateKey] : flag.default; @@ -585,31 +605,31 @@ async function runScriptOnce(script, btn, extraParams) { // ESP32 controls // --------------------------------------------------------------------------- -// Board picker for provisioning scripts (pass_board): deviceModels.json entries -// whose `firmwares` include the selected firmware. "(any board)" = no +// Device-model picker for provisioning scripts (pass_device_model): deviceModels.json entries +// whose `firmwares` include the selected firmware. "(any model)" = no // injection. Distinct state key from the LEGACY `state.board` (which meant -// firmware — see the migration in init) and from per-device boards. -function renderBoardSelect() { - const select = document.getElementById("board-select"); +// firmware — see the migration in init) and from per-device deviceModels. +function renderDeviceModelSelect() { + const select = document.getElementById("devicemodel-select"); if (!select) return; select.innerHTML = ""; - const candidates = boards.filter(b => (b.firmwares || []).includes(state.firmware)); - const options = [["", "(any board)"], ...candidates.map(b => [b.name, b.name])]; - if (state.provisionBoard && !options.some(([v]) => v === state.provisionBoard)) { - state.provisionBoard = ""; // firmware changed; stale pick no longer applies + const candidates = deviceModels.filter(b => (b.firmwares || []).includes(state.firmware)); + const options = [["", "(any model)"], ...candidates.map(b => [b.name, b.name])]; + if (state.provisionDeviceModel && !options.some(([v]) => v === state.provisionDeviceModel)) { + state.provisionDeviceModel = ""; // firmware changed; stale pick no longer applies } for (const [val, lbl] of options) { const opt = document.createElement("option"); opt.value = val; opt.textContent = lbl; - if (val === (state.provisionBoard || "")) opt.selected = true; + if (val === (state.provisionDeviceModel || "")) opt.selected = true; select.appendChild(opt); } // onchange (assigned, not addEventListener) so a re-render replaces the // handler instead of stacking another — addEventListener here would fire // saveState() once per past render. select.onchange = async () => { - state.provisionBoard = select.value; + state.provisionDeviceModel = select.value; await saveState(); }; } @@ -631,10 +651,24 @@ function renderFirmwareSelect() { select.addEventListener("change", async () => { state.firmware = select.value; await saveState(); - renderBoardSelect(); // board candidates follow the firmware + renderDeviceModelSelect(); // device-model candidates follow the firmware }); } +// Show which device MoonDeck last flashed on `port` — matched by the `last_port` +// breadcrumb on the active network's devices (set at flash time, keyed per-device by +// MAC). Names the board you're about to reflash so the port isn't just an opaque +// /dev/cu.* string. Blank when the port maps to no known device. +function updatePortDeviceHint(port) { + const hint = document.getElementById("port-device-hint"); + if (!hint) return; + const devices = getActiveNetwork()?.devices || []; + const dev = port ? devices.find(d => d.last_port === port) : null; + if (!dev) { hint.textContent = ""; return; } + const model = dev.deviceModel ? ` · ${dev.deviceModel}` : ""; + hint.textContent = `last flashed: ${dev.deviceName || dev.ip}${model}`; +} + async function refreshPorts() { const resp = await fetch("/api/ports"); const data = await resp.json(); @@ -651,6 +685,7 @@ async function refreshPorts() { if (port === current) opt.selected = true; select.appendChild(opt); } + updatePortDeviceHint(current); // `.onchange = ...` (not addEventListener) so the handler is REPLACED on // each refresh rather than stacking — refreshPorts re-runs on every // network switch + every manual Refresh click, so addEventListener @@ -658,6 +693,7 @@ async function refreshPorts() { select.onchange = async () => { const active = getActiveNetwork(); if (active) active.port = select.value; + updatePortDeviceHint(select.value); await saveState(); }; } @@ -698,6 +734,9 @@ document.getElementById("refresh-devices-btn")?.addEventListener("click", async state = await resp.json(); renderNetworkBar(); renderDevices(); + // /api/refresh can re-attribute last_port (it consumes the flash breadcrumb + can strip a + // stale link), so refresh the "last flashed: X" hint under the port dropdown too. + updatePortDeviceHint(getActiveNetwork()?.port || ""); const after = getActiveNetwork(); const onCount = after ? after.devices.filter(d => d.online).length : 0; appendLog(`${onCount}/${after ? after.devices.length : 0} online\n`); @@ -819,7 +858,11 @@ function renderDevices() { } el.innerHTML = ""; for (const device of devices) { - const label = document.createElement("label"); + // A plain <div>, NOT a <label>: a <label> wrapper toggles its checkbox on a click + // ANYWHERE inside it (native behaviour), so clicking the IP/name to open the device + // UI also flipped the selection. The checkbox has its own change handler, so it + // toggles only when clicked directly. + const label = document.createElement("div"); label.className = "device-item"; const dot = document.createElement("span"); @@ -835,76 +878,127 @@ function renderDevices() { }); // Two-row layout: row 1 carries identity at-a-glance (dot, checkbox, - // IP). Row 2 carries secondary info (deviceName + fw:<firmware>) plus - // the board picker + remove button. Splitting keeps the IP — the - // single most-clicked-on field — next to the checkbox and stops the - // board picker from being pushed off the right edge by long - // device-name / firmware strings. - const ipText = document.createElement("span"); - ipText.textContent = device.ip; - const tooltipLines = [device.ip]; + // deviceName). Row 2 carries secondary info (IP + fw:<firmware>) plus + // the device picker + remove button. The deviceName is the prominent + // line — it's how a human recognises the device ("the S31") — with the + // IP demoted to the info row (still clickable to open the device UI). + // A device with no name yet falls back to showing its IP as the primary. + // A <button> (not a clickable <span>) so it's keyboard-focusable and Enter/Space activate it + // — the .link-button class strips the button chrome to read as inline text. + const nameText = document.createElement("button"); + nameText.type = "button"; + nameText.className = "device-name link-button"; + nameText.textContent = device.deviceName || device.ip; + const tooltipLines = ["Open this device's UI in the view pane", device.ip]; if (device.last_port) tooltipLines.push(`last flashed via ${device.last_port}`); - ipText.title = tooltipLines.join("\n"); - ipText.style.cursor = "pointer"; - ipText.addEventListener("click", (e) => { - if (e.target === ipText) showInView("http://" + device.ip); - }); + nameText.title = tooltipLines.join("\n"); + nameText.addEventListener("click", () => showInView("http://" + device.ip)); + // Info row: clickable IP · fw. The IP is its own button so activating it opens the + // device UI in the view pane (same as the name), while the fw text stays inert. const infoText = document.createElement("span"); infoText.className = "device-info"; - // last_port lives in the tooltip — it's flash-history, not identity. - const infoParts = []; - if (device.deviceName) infoParts.push(device.deviceName); - if (device.firmware) infoParts.push(`fw:${device.firmware}`); - infoText.textContent = infoParts.join(" · "); - - // Board picker — options derived from deviceModels.json (loaded via - // /api/boards). Auto-deduced for firmwares that map to a single - // board (probe sets device.board); user-set when the firmware can - // run on multiple boards (e.g. `esp32` on LOLIN D32 vs generic + const ipLink = document.createElement("button"); + ipLink.type = "button"; + ipLink.className = "device-ip-link link-button"; + ipLink.textContent = device.ip; + ipLink.title = "Open this device's UI in the view pane"; + ipLink.addEventListener("click", () => showInView("http://" + device.ip)); + infoText.appendChild(ipLink); + if (device.firmware) { + infoText.appendChild(document.createTextNode(` · fw:${device.firmware}`)); + } + + // last_port — the serial port this exact device (by MAC) was last flashed via. Click it to + // set it as the active network's flash port, so "flash the SE16" is one click → run Flash. + // The port lives per-device (keyed by MAC), the flash uses the network-level port; this + // chip bridges them. Absent until the device has been flashed once through MoonDeck. + if (device.last_port) { + const portChip = document.createElement("button"); + portChip.className = "device-port-chip"; + portChip.textContent = "⚡ " + device.last_port.replace("/dev/cu.", "").replace("/dev/tty.", ""); + portChip.title = `Select this device model for flashing: set the port` + + (device.firmware ? `, firmware (${device.firmware})` : "") + + (device.deviceModel ? ` and deviceModel` : ""); + const active = getActiveNetwork(); + if (active && active.port === device.last_port) portChip.classList.add("active"); + // Selecting a device model for flashing means matching what it actually runs: the port, AND the + // firmware + deviceModel MoonDeck learned from discovery — so Build/Flash target the right + // binary for this device model (flashing the wrong firmware would brick it). One click, all in sync. + portChip.addEventListener("click", async (e) => { + e.preventDefault(); + const a = getActiveNetwork(); + if (a) a.port = device.last_port; + // Set the flash target to THIS device's firmware/deviceModel — and clear it when this + // device doesn't have one, so the next Build/Flash can't reuse the PREVIOUS device's + // values (flashing the wrong firmware would brick it). The reset is unconditional. + state.firmware = (device.firmware && firmwares.includes(device.firmware)) ? device.firmware : ""; + state.provisionDeviceModel = device.deviceModel || ""; + await saveState(); + refreshPorts(); + renderFirmwareSelect(); + renderDeviceModelSelect(); + renderDevices(); + }); + infoText.appendChild(document.createTextNode(" ")); + infoText.appendChild(portChip); + } + + // Device-model picker — options derived from deviceModels.json (loaded via + // /api/device-models). Auto-deduced for firmwares that map to a single + // deviceModel (probe sets device.deviceModel); user-set when the firmware can + // run on multiple device models (e.g. `esp32` on LOLIN D32 vs generic // DevKit). A device-reported value that isn't in the catalog gets // prepended as <key> (unknown) so the selection survives — without // that, <select> would silently snap to "" and the next saveState // would clobber the device's stored value with empty. - const boardPicker = document.createElement("select"); - boardPicker.className = "device-board"; - boardPicker.title = "Physical board (pick when firmware can't tell us)"; - const boardOptions = [ - ["", "(unknown board)"], + const deviceModelPicker = document.createElement("select"); + deviceModelPicker.className = "devicemodel-picker"; + deviceModelPicker.title = "Device model (pick when firmware can't tell us)"; + // Only offer deviceModels that can run this device's firmware — a model whose `firmwares` + // don't include it is not a valid choice for this unit (same filter the provisioning picker + // uses). If the firmware is unknown (empty), we can't filter, so show all. + const fwModels = device.firmware + ? deviceModels.filter(b => (b.firmwares || []).includes(device.firmware)) + : deviceModels; + const deviceModelOptions = [ + ["", "(unknown model)"], // Each entry is [value, label] for the <select>; with the // single-name catalog (no separate key/label), they're identical. - ...boards.map(b => [b.name, b.name]), + ...fwModels.map(b => [b.name, b.name]), ]; - const deviceBoard = device.board || ""; - if (deviceBoard && !boardOptions.some(([k]) => k === deviceBoard)) { - boardOptions.push([deviceBoard, `${deviceBoard} (unknown)`]); + const deviceModel = device.deviceModel || ""; + if (deviceModel && !deviceModelOptions.some(([k]) => k === deviceModel)) { + // Keep the current value selectable even if it's filtered out (firmware mismatch) or + // not in the catalog — marked (unknown) so the selection survives. + deviceModelOptions.push([deviceModel, `${deviceModel} (unknown)`]); } - for (const [val, lbl] of boardOptions) { + for (const [val, lbl] of deviceModelOptions) { const opt = document.createElement("option"); opt.value = val; opt.textContent = lbl; - if (deviceBoard === val) opt.selected = true; - boardPicker.appendChild(opt); + if (deviceModel === val) opt.selected = true; + deviceModelPicker.appendChild(opt); } - // Push a board's full deviceModels.json defaults to the device (POST - // /api/push-board → _push_board_to_device fans out controls.<Module>.<control>). + // Push a deviceModel's full deviceModels.json defaults to the device (POST + // /api/push-device → _push_device fans out controls.<Module>.<control>). // `onDone(ok)` lets the explicit button below show success/failure; the picker // change path passes nothing (fire-and-forget, recovered on next refresh). - const pushBoard = (board, onDone) => { + const pushDevice = (deviceModel, onDone) => { // Success is the device-side result in the JSON body ({"ok": bool} from - // _push_board_to_device) — HTTP 200 alone can wrap a failed push (a device + // _push_device) — HTTP 200 alone can wrap a failed push (a device // timeout / non-2xx mid-fan-out), so r.ok would falsely report success. // 10s AbortSignal timeout so a stalled request can't wedge the button forever. - fetch("/api/push-board", { + fetch("/api/push-device", { method: "POST", headers: {"Content-Type": "application/json"}, - body: JSON.stringify({ip: device.ip, board}), - signal: AbortSignal.timeout(10000), + body: JSON.stringify({ip: device.ip, deviceModel}), + signal: timeoutSignal(10000), }).then(r => r.json()).then(j => onDone && onDone(!!j.ok)) .catch(() => onDone && onDone(false)); }; - boardPicker.addEventListener("change", () => { - device.board = boardPicker.value; + deviceModelPicker.addEventListener("change", () => { + device.deviceModel = deviceModelPicker.value; saveState(); // Mirror the change to the device immediately. Without this, the // device's SystemModule wouldn't hear about the picker until the @@ -912,29 +1006,59 @@ function renderDevices() { // UI to update right after they pick. Fire-and-forget; failure // (timeout / device offline) is recovered on the next refresh // when discover/refresh's bulk push catches up. - pushBoard(boardPicker.value); + pushDevice(deviceModelPicker.value); }); - // Explicit "inject defaults" — re-push the SELECTED board's full config on demand, + // Explicit "inject" — re-push the SELECTED device model's full config on demand, // without having to change the picker. Distinct intent from the implicit on-change // push: re-apply after a reflash wiped config, or re-assert defaults a user edited // away. Brief inline feedback so a no-op (timeout / offline) isn't silent. const injectBtn = document.createElement("button"); injectBtn.className = "device-inject"; - injectBtn.textContent = "inject defaults"; + injectBtn.textContent = "inject"; injectBtn.title = "Push the selected device-model's deviceModels.json defaults to this device now"; injectBtn.addEventListener("click", (e) => { e.preventDefault(); - const board = boardPicker.value; - if (!board) { injectBtn.textContent = "pick a board first"; setTimeout(() => injectBtn.textContent = "inject defaults", 1500); return; } + const deviceModel = deviceModelPicker.value; + if (!deviceModel) { injectBtn.textContent = "pick a device model"; setTimeout(() => injectBtn.textContent = "inject", 1500); return; } injectBtn.disabled = true; injectBtn.textContent = "injecting…"; - pushBoard(board, (ok) => { + pushDevice(deviceModel, (ok) => { injectBtn.textContent = ok ? "injected ✓" : "failed ✗"; - setTimeout(() => { injectBtn.textContent = "inject defaults"; injectBtn.disabled = false; }, 1800); + setTimeout(() => { injectBtn.textContent = "inject"; injectBtn.disabled = false; }, 1800); }); }); + // "OTA" — wireless flash of the local build for this device's firmware. MoonDeck serves + // build/esp32-<fw>/projectMM.bin and hands the device its URL (POST /api/ota); the device + // pulls + flashes over WiFi. No USB. Needs a local build for the device's firmware. + const otaBtn = document.createElement("button"); + otaBtn.className = "device-inject"; // same compact style as inject + const otaFw = device.firmware && device.firmware !== "unknown" ? device.firmware : ""; + otaBtn.textContent = "OTA"; + otaBtn.disabled = !otaFw; + otaBtn.title = otaFw + ? `Flash the local ${otaFw} build to this device over WiFi (no USB)` + : "OTA needs the device's firmware known (discover it first)"; + otaBtn.addEventListener("click", async (e) => { + e.preventDefault(); + if (!otaFw) return; + otaBtn.disabled = true; + otaBtn.textContent = "OTA…"; + try { + const res = await fetch("/api/ota", { + method: "POST", headers: {"Content-Type": "application/json"}, + body: JSON.stringify({ip: device.ip, firmware: otaFw}), + signal: timeoutSignal(15000), + }); + const j = await res.json(); + otaBtn.textContent = j.ok ? "flashing ✓" : "failed ✗"; + } catch (err) { + otaBtn.textContent = "failed ✗"; + } + setTimeout(() => { otaBtn.textContent = "OTA"; otaBtn.disabled = !otaFw; }, 2500); + }); + const removeBtn = document.createElement("button"); removeBtn.className = "device-remove"; removeBtn.textContent = "x"; @@ -951,17 +1075,17 @@ function renderDevices() { // Three-row layout, each row a flex-basis:100% wrapper inside // .device-item (which is flex-wrap:wrap): - // row 1 — dot · checkbox · IP · X (identity + remove) - // row 2 — deviceName · fw:firmware (secondary info) - // row 3 — [board picker] (right-aligned) + // row 1 — dot · checkbox · deviceName · X (identity + remove) + // row 2 — IP · fw:firmware (secondary info) + // row 3 — [device picker] (right-aligned) // Splitting like this keeps each row narrow enough to fit the // sidebar without truncation, and puts the most-clicked items - // (checkbox, IP, board picker) at predictable y-positions. + // (checkbox, name, device picker) at predictable y-positions. const row1 = document.createElement("div"); row1.className = "device-row device-row-identity"; row1.appendChild(dot); row1.appendChild(cb); - row1.appendChild(ipText); + row1.appendChild(nameText); row1.appendChild(removeBtn); const row2 = document.createElement("div"); @@ -969,87 +1093,14 @@ function renderDevices() { row2.appendChild(infoText); const row3 = document.createElement("div"); - row3.className = "device-row device-row-board"; - row3.appendChild(boardPicker); + row3.className = "device-row devicemodel-row"; + row3.appendChild(deviceModelPicker); row3.appendChild(injectBtn); - - // row 4 — pin-profile save/apply. A profile is the device's captured - // GPIO/peripheral config (drivers, board, network, audio); saving stores - // it under this device in moondeck.json, applying re-pushes it (handy - // after a reflash wipes config, or to clone to a second identical rig). - const row4 = document.createElement("div"); - row4.className = "device-row device-row-profile"; - - const profileSel = document.createElement("select"); - profileSel.className = "device-profile-select"; - profileSel.title = "Saved pin profiles for this device"; - const phOpt = document.createElement("option"); - phOpt.value = ""; phOpt.textContent = "(profiles)"; - profileSel.appendChild(phOpt); - for (const p of (device.profiles || [])) { - const o = document.createElement("option"); - o.value = p.name; o.textContent = p.name; - profileSel.appendChild(o); - } - - const saveBtn = document.createElement("button"); - saveBtn.className = "device-profile-btn"; - saveBtn.textContent = "Save"; - saveBtn.title = "Capture this device's current pins as a named profile"; - saveBtn.addEventListener("click", async (e) => { - e.preventDefault(); - const name = prompt(`Save pin profile for ${device.deviceName || device.ip} as:`); - if (!name) return; - saveBtn.disabled = true; saveBtn.textContent = "…"; - try { - const res = await fetch("/api/save-profile", { - method: "POST", headers: {"Content-Type": "application/json"}, - body: JSON.stringify({ip: device.ip, name}), - }); - const j = await res.json(); - if (j.ok) { - // The server captured + persisted the real modules. Re-fetch the - // authoritative state so device.profiles holds the actual captured - // config — NOT a {modules: []} placeholder, which the next - // saveState() would POST back and clobber the server's real copy. - try { - const sr = await fetch("/api/state"); - state = await sr.json(); - } catch (_) { /* keep current state; dropdown refreshes on next load */ } - renderDevices(); - } else { alert("Save failed: " + (j.error || "unknown")); } - } catch (err) { alert("Save failed: " + err); } - finally { saveBtn.disabled = false; saveBtn.textContent = "Save"; } - }); - - const applyBtn = document.createElement("button"); - applyBtn.className = "device-profile-btn"; - applyBtn.textContent = "Apply"; - applyBtn.title = "Re-apply the selected profile to this device"; - applyBtn.addEventListener("click", async (e) => { - e.preventDefault(); - const name = profileSel.value; - if (!name) { alert("Pick a profile to apply"); return; } - applyBtn.disabled = true; applyBtn.textContent = "…"; - try { - const res = await fetch("/api/apply-profile", { - method: "POST", headers: {"Content-Type": "application/json"}, - body: JSON.stringify({ip: device.ip, name}), - }); - const j = await res.json(); - if (!j.ok) alert("Apply failed: " + (j.error || "device unreachable")); - } catch (err) { alert("Apply failed: " + err); } - finally { applyBtn.disabled = false; applyBtn.textContent = "Apply"; } - }); - - row4.appendChild(profileSel); - row4.appendChild(saveBtn); - row4.appendChild(applyBtn); + row3.appendChild(otaBtn); label.appendChild(row1); label.appendChild(row2); label.appendChild(row3); - label.appendChild(row4); el.appendChild(label); } } diff --git a/scripts/moondeck_ui/index.html b/moondeck/moondeck_ui/index.html similarity index 79% rename from scripts/moondeck_ui/index.html rename to moondeck/moondeck_ui/index.html index 8d627d17..8e4939f0 100644 --- a/scripts/moondeck_ui/index.html +++ b/moondeck/moondeck_ui/index.html @@ -54,7 +54,7 @@ <h1>MoonDeck</h1> <div id="scripts-esp32-setup" class="script-grid"></div> <div class="esp32-controls"> <label>Firmware: <select id="firmware-select"></select></label> - <label>Device: <select id="board-select" title="Device model — provisioning injects its deviceModels.json settings (e.g. the weak-power TX-power cap) before WiFi association"></select></label> + <label>Device model: <select id="devicemodel-select" title="Device model — provisioning injects its deviceModels.json settings (e.g. the weak-power TX-power cap) before WiFi association"></select></label> </div> <div id="scripts-esp32-build" class="script-grid"></div> <div class="esp32-controls"> @@ -63,6 +63,10 @@ <h1>MoonDeck</h1> <button id="refresh-ports" title="Refresh">↻</button> </div> <select id="port-select"><option value="">--</option></select> + <!-- Names the device MoonDeck last flashed on the selected port (from + moondeck.json's last_port breadcrumb) so you know which board you're + about to reflash. Empty when the port has no known device. --> + <div id="port-device-hint" class="port-device-hint"></div> </div> <div id="scripts-esp32" class="script-grid"></div> </section> @@ -88,6 +92,7 @@ <h1>MoonDeck</h1> <button id="view-back" title="Back">←</button> <button id="view-forward" title="Forward">→</button> <button id="view-refresh" title="Refresh">↻</button> + <button id="view-open" title="Open in a new tab">↗</button> </div> <button id="clear-log">Clear</button> </div> @@ -96,7 +101,11 @@ <h1>MoonDeck</h1> <pre id="log"></pre> </div> <div id="pane-view" class="pane-content"> - <iframe id="view-frame" sandbox="allow-same-origin allow-scripts allow-forms"></iframe> + <!-- allow-modals: the embedded device UI uses window.prompt/alert/confirm (e.g. the + File Manager's new-folder/new-file name prompt). Without this flag a sandboxed + frame silently blocks them ("Use of window.prompt is not allowed…"). It permits + modal dialogs only — it does NOT relax the origin/script isolation. --> + <iframe id="view-frame" title="Device UI preview" sandbox="allow-same-origin allow-scripts allow-forms allow-modals"></iframe> </div> </main> </div> diff --git a/scripts/moondeck_ui/style.css b/moondeck/moondeck_ui/style.css similarity index 82% rename from scripts/moondeck_ui/style.css rename to moondeck/moondeck_ui/style.css index 7ac76e57..b081c199 100644 --- a/scripts/moondeck_ui/style.css +++ b/moondeck/moondeck_ui/style.css @@ -58,8 +58,8 @@ nav { display: flex; gap: 4px; } /* Left sidebar */ .sidebar { - width: 240px; - min-width: 240px; + width: 312px; + min-width: 312px; background: #16213e; border-right: 1px solid #0f3460; overflow-y: auto; @@ -98,6 +98,17 @@ nav { display: flex; gap: 4px; } } .esp32-controls select { width: 100%; } +/* Which device MoonDeck last flashed on the selected port — a small muted line under + the port dropdown. Empty (collapses) when the port maps to no known device. */ +.port-device-hint { + font-size: 11px; + color: #7a8a9a; + margin-top: 3px; + min-height: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} select { background: #1a1a2e; @@ -278,7 +289,7 @@ select { } /* Each device card uses three rows: identity (dot/cb/ip/x), info (name + fw), - board (picker). Sized full-width and laid out with flex so each row can + device model (picker). Sized full-width and laid out with flex so each row can align its own children independently. */ .device-row { display: flex; @@ -297,15 +308,53 @@ select { } .device-status.online { background: #4caf50; } .device-status.offline { background: #666; } +/* Device name — the prominent identity line (how a human recognises the device, + e.g. "MM-S31"). Bigger + bolder than the secondary IP/fw info row below it. */ +/* A <button> that reads as inline text: strip the native chrome so device-name / device-ip-link + keep their text styling, while staying keyboard-focusable (Enter/Space activate). */ +.link-button { + appearance: none; + background: none; + border: none; + padding: 0; + margin: 0; + font: inherit; + color: inherit; + text-align: left; + cursor: pointer; +} +.link-button:focus-visible { outline: 2px solid #8ab4f8; outline-offset: 2px; } +.device-name { + font-size: 14px; + font-weight: 600; + color: #e8eef5; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + min-width: 0; +} .device-info { color: #7a8a9a; font-size: 11px; } +/* The IP in the info row — clickable to open the device UI in the view pane. Underline + on hover marks it as the link, distinct from the inert fw text beside it. */ +.device-ip-link { cursor: pointer; } +.device-ip-link:hover { color: #8ab4f8; text-decoration: underline; } + +/* last_port chip — click to set this device's known flash port as the network port. */ +.device-port-chip { + font-size: 10px; background: #1c2535; color: #8ab4d8; + border: 1px solid #2a3547; border-radius: 4px; + padding: 1px 5px; cursor: pointer; font-family: inherit; +} +.device-port-chip:hover { background: #243044; color: #a9cdf0; } +.device-port-chip.active { background: #2d4a2d; color: #9fe09f; border-color: #3a5a3a; } -/* Picker + inject button share the board row; the picker flexes, the button +/* Picker + inject button share the deviceModel row; the picker flexes, the button stays its natural width. */ -.device-row-board { display: flex; gap: 6px; align-items: center; } -.device-board { +.devicemodel-row { display: flex; gap: 6px; align-items: center; } +.devicemodel-picker { font-size: 11px; background: #1c2535; color: #c0c0c0; border: 1px solid #2a3548; border-radius: 3px; padding: 1px 4px; /* Flexes to fill the row; max-width/ellipsis clamp long labels (e.g. @@ -317,6 +366,8 @@ select { text-overflow: ellipsis; white-space: nowrap; } +/* The per-device inject button — dark, bordered, compact, sitting beside the deviceModel + picker so the row reads as one control set rather than a styled dropdown + native button. */ .device-inject { flex: 0 0 auto; font-size: 11px; background: #1c2535; color: #8ab4f8; @@ -325,6 +376,7 @@ select { } .device-inject:hover:not(:disabled) { border-color: #8ab4f8; } .device-inject:disabled { opacity: 0.6; cursor: default; } + .device-remove { background: none; border: none; color: #666; cursor: pointer; font-size: 11px; padding: 0 4px; diff --git a/scripts/rename/rename_to_moonlight.py b/moondeck/rename/rename_to_moonlight.py similarity index 95% rename from scripts/rename/rename_to_moonlight.py rename to moondeck/rename/rename_to_moonlight.py index 7540e6ce..2ae690eb 100644 --- a/scripts/rename/rename_to_moonlight.py +++ b/moondeck/rename/rename_to_moonlight.py @@ -19,8 +19,8 @@ becomes MoonModules/MoonLight, which only resolves once this repo holds that name. So --apply belongs in Phase 3.3, after 3.2 — not before. - uv run scripts/rename/rename_to_moonlight.py # dry-run: list every hit, change nothing - uv run scripts/rename/rename_to_moonlight.py --apply # WRITE the changes (switch-day only) + uv run moondeck/rename/rename_to_moonlight.py # dry-run: list every hit, change nothing + uv run moondeck/rename/rename_to_moonlight.py --apply # WRITE the changes (switch-day only) Why a script and not a bare `sed`: the value here is not clever per-form logic (a plain token replace is correct for every form — repo URL, host path, @@ -67,7 +67,7 @@ # corrupt its meaning ("the predecessor at MoonModules/MoonLight vacates…"). "docs/backlog/rename-to-moonlight.md", # The rename script itself (it names the tokens it replaces). - "scripts/rename/rename_to_moonlight.py", + "moondeck/rename/rename_to_moonlight.py", ] # File extensions the sweep considers (text formats that carry the name). Binary @@ -100,7 +100,7 @@ def iter_files(): # Tracked files only — `git ls-files` respects .gitignore, so build output # (build/, esp32/build/) is never swept and we don't maintain a blocklist. # Caveat: gitignored files are therefore NOT covered — notably the private - # bench registry scripts/moondeck.json, whose "board" names must be + # bench registry moondeck/moondeck.json, whose "board" names must be # hand-updated at switch-time (see the plan's Phase 1 step 5). out = subprocess.run( ["git", "ls-files"], cwd=ROOT, capture_output=True, text=True, check=True diff --git a/scripts/report/history_report.py b/moondeck/report/history_report.py similarity index 99% rename from scripts/report/history_report.py rename to moondeck/report/history_report.py index 34c2a463..acd675f8 100644 --- a/scripts/report/history_report.py +++ b/moondeck/report/history_report.py @@ -309,7 +309,7 @@ def main() -> int: parts = [ "# projectMM history report", "", - "Auto-generated by `scripts/report/history_report.py`. Re-run from " + "Auto-generated by `moondeck/report/history_report.py`. Re-run from " "MoonDeck's PC tab → **History Report** button. Source data: " "`git log` + `gh release list`.", "", diff --git a/scripts/run/monitor_esp32.py b/moondeck/run/monitor_esp32.py similarity index 100% rename from scripts/run/monitor_esp32.py rename to moondeck/run/monitor_esp32.py diff --git a/scripts/run/preview_installer.py b/moondeck/run/preview_installer.py similarity index 97% rename from scripts/run/preview_installer.py rename to moondeck/run/preview_installer.py index 29485b10..ec59a317 100644 --- a/scripts/run/preview_installer.py +++ b/moondeck/run/preview_installer.py @@ -58,7 +58,7 @@ # entry point working. STAGE_INSTALL = STAGE_DIR / "install" BUILD_ROOT = ROOT / "build" -GENERATE_MANIFEST = ROOT / "scripts" / "build" / "generate_manifest.py" +GENERATE_MANIFEST = ROOT / "moondeck" / "build" / "generate_manifest.py" # Stage under the tag the picker WILL surface from the live GitHub Releases # API: `latest` is always the newest CI build. Local bins replace the real # `latest` bins for the duration of the preview — that's the point, we want @@ -299,12 +299,12 @@ def main() -> int: print(f"==> staged `{LOCAL_TAG}` release with firmwares: {', '.join(firmwares)}") print(f" pick the `{LOCAL_TAG}` tag in the picker to flash a USB-connected ESP32") else: - print(f"==> no firmwares could be staged (all skipped — see warnings above)") - print(f" falling back to render-only mode") + print("==> no firmwares could be staged (all skipped — see warnings above)") + print(" falling back to render-only mode") else: print(f"==> no local ESP32 builds found under {BUILD_ROOT.relative_to(ROOT)}") - print(f" render-only mode — `Install` will fail") - print(f" run `uv run scripts/build/build_esp32.py --firmware <variant>` first to enable end-to-end flash") + print(" render-only mode — `Install` will fail") + print(" run `uv run moondeck/build/build_esp32.py --firmware <variant>` first to enable end-to-end flash") print(f"==> serving at http://localhost:{PORT}/") print(f" installer: http://localhost:{PORT}/install/") diff --git a/scripts/run/run_desktop.py b/moondeck/run/run_desktop.py similarity index 100% rename from scripts/run/run_desktop.py rename to moondeck/run/run_desktop.py diff --git a/scripts/run/show_crash_log.py b/moondeck/run/show_crash_log.py similarity index 100% rename from scripts/run/show_crash_log.py rename to moondeck/run/show_crash_log.py diff --git a/scripts/scenario/_net_probe.py b/moondeck/scenario/_net_probe.py similarity index 98% rename from scripts/scenario/_net_probe.py rename to moondeck/scenario/_net_probe.py index 553a8441..b0b663fa 100644 --- a/scripts/scenario/_net_probe.py +++ b/moondeck/scenario/_net_probe.py @@ -21,7 +21,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent.parent -MOONDECK_STATE = ROOT / "scripts" / "moondeck.json" +MOONDECK_STATE = ROOT / "moondeck" / "moondeck.json" ARTNET_PORT = 6454 E131_PORT = 5568 diff --git a/scripts/scenario/_observed.py b/moondeck/scenario/_observed.py similarity index 97% rename from scripts/scenario/_observed.py rename to moondeck/scenario/_observed.py index 152531e8..0e95b496 100644 --- a/scripts/scenario/_observed.py +++ b/moondeck/scenario/_observed.py @@ -1,6 +1,6 @@ """Shared widen-only range update for observed.<target> blocks. -Both runners (scripts/scenario/run_scenario.py and run_live_scenario.py) +Both runners (moondeck/scenario/run_scenario.py and run_live_scenario.py) persist a per-target rolling [min, max] range for each scalar in the observed block. The range expands when a new measurement falls outside its current bounds; otherwise the JSON isn't rewritten — drastically diff --git a/scripts/scenario/_preview_ws.py b/moondeck/scenario/_preview_ws.py similarity index 100% rename from scripts/scenario/_preview_ws.py rename to moondeck/scenario/_preview_ws.py diff --git a/scripts/scenario/run_live_scenario.py b/moondeck/scenario/run_live_scenario.py similarity index 99% rename from scripts/scenario/run_live_scenario.py rename to moondeck/scenario/run_live_scenario.py index 11b75496..591f9e05 100644 --- a/scripts/scenario/run_live_scenario.py +++ b/moondeck/scenario/run_live_scenario.py @@ -26,9 +26,9 @@ def _mod_path(name: str) -> str: BASELINE_FILE = ROOT / "test" / "scenario-baseline.json" # Reuse the shared test-metadata parser so scenario discovery stays in one place. -sys.path.insert(0, str(ROOT / "scripts" / "docs")) +sys.path.insert(0, str(ROOT / "moondeck" / "docs")) import _test_metadata as test_meta # noqa: E402 -sys.path.insert(0, str(ROOT / "scripts" / "scenario")) +sys.path.insert(0, str(ROOT / "moondeck" / "scenario")) import _observed # noqa: E402 @@ -241,8 +241,8 @@ def run_scenario(client: Client, scenario_path: Path, settle_s: float = 1.5, pending_contract_originals: dict = {} if mode == "construct": - print(f"\n SKIP (mode=construct — runs in-process only; the live device's " - f"main.cpp owns the top-level shape)") + print("\n SKIP (mode=construct — runs in-process only; the live device's " + "main.cpp owns the top-level shape)") results["skipped"] = True return results if mode != "mutate": @@ -626,7 +626,7 @@ def run_scenario(client: Client, scenario_path: Path, settle_s: float = 1.5, # current bounds. Routine runs that stay in range produce no JSON # diff. When --update-contract is set, the historical range no # longer reflects the new promise, so reset to the current point. - # See scripts/scenario/_observed.py. + # See moondeck/scenario/_observed.py. sample = { "tick_us": int(tick_us), "free_heap": int(heap), @@ -765,11 +765,11 @@ def run_scenario(client: Client, scenario_path: Path, settle_s: float = 1.5, print(f" WROTE {scenario_path.name} ({' + '.join(what)})") # Summary - print(f"\n---") + print("\n---") if results["passed"]: - print(f"PASSED") + print("PASSED") else: - print(f"FAILED") + print("FAILED") return results diff --git a/scripts/scenario/run_network_live.py b/moondeck/scenario/run_network_live.py similarity index 99% rename from scripts/scenario/run_network_live.py rename to moondeck/scenario/run_network_live.py index fcbba1c2..3b5bd29c 100644 --- a/scripts/scenario/run_network_live.py +++ b/moondeck/scenario/run_network_live.py @@ -2,7 +2,7 @@ """Live lights-over-UDP matrix test across every online board in moondeck.json. Proves the multi-protocol network path (ArtNet, E1.31/sACN, DDP) end-to-end on -real firmware. Devices come from scripts/moondeck.json (the MoonDeck device +real firmware. Devices come from moondeck/moondeck.json (the MoonDeck device list, active network, online only). Each round one device is the SENDER and every other device LISTENS: @@ -23,7 +23,7 @@ all legs passed, 1 = a leg failed, 2 = environment problem (no devices, moondeck.json missing). -Run: uv run scripts/scenario/run_network_live.py [--device NAME] [--host IP] +Run: uv run moondeck/scenario/run_network_live.py [--device NAME] [--host IP] """ import argparse diff --git a/scripts/scenario/run_network_roundtrip.py b/moondeck/scenario/run_network_roundtrip.py similarity index 99% rename from scripts/scenario/run_network_roundtrip.py rename to moondeck/scenario/run_network_roundtrip.py index 9503983f..98431fac 100644 --- a/scripts/scenario/run_network_roundtrip.py +++ b/moondeck/scenario/run_network_roundtrip.py @@ -25,7 +25,7 @@ back (a real failure), 2 = environment problem (no device, moondeck.json missing). -Run: uv run scripts/scenario/run_network_roundtrip.py [--host IP] [--repeats N] +Run: uv run moondeck/scenario/run_network_roundtrip.py [--host IP] [--repeats N] """ import argparse diff --git a/scripts/scenario/run_scenario.py b/moondeck/scenario/run_scenario.py similarity index 98% rename from scripts/scenario/run_scenario.py rename to moondeck/scenario/run_scenario.py index 12e20f91..105ad51a 100644 --- a/scripts/scenario/run_scenario.py +++ b/moondeck/scenario/run_scenario.py @@ -26,9 +26,9 @@ ROOT = Path(__file__).resolve().parent.parent.parent # Reuse the shared test-metadata parser so scenario discovery stays in one place. -sys.path.insert(0, str(ROOT / "scripts" / "docs")) +sys.path.insert(0, str(ROOT / "moondeck" / "docs")) import _test_metadata as test_meta # noqa: E402 -sys.path.insert(0, str(ROOT / "scripts" / "scenario")) +sys.path.insert(0, str(ROOT / "moondeck" / "scenario")) import _observed # noqa: E402 _HOST = {"darwin": "macos", "win32": "windows"}.get(sys.platform, "linux") @@ -122,7 +122,7 @@ def _run_one(path: Path, update_contract: bool, update_reason: str | None) -> in # — drops JSON churn on routine runs to near-zero while preserving full # drift visibility. When --update-contract was passed, reset the range # to the current single point (the historical range was for the - # previous contract). See scripts/scenario/_observed.py. + # previous contract). See moondeck/scenario/_observed.py. existing_obs = step.get("observed", {}).get(target) if update_contract: new_obs = _observed.reset(observations[name], today) diff --git a/scripts/test/test_desktop.py b/moondeck/test/test_desktop.py similarity index 98% rename from scripts/test/test_desktop.py rename to moondeck/test/test_desktop.py index d206bc4a..5647710b 100644 --- a/scripts/test/test_desktop.py +++ b/moondeck/test/test_desktop.py @@ -16,7 +16,7 @@ ROOT = Path(__file__).resolve().parent.parent.parent # Reuse the shared test-metadata parser instead of re-implementing @module / # @case discovery here. Same parser feeds the doc generator and MoonDeck. -sys.path.insert(0, str(ROOT / "scripts" / "docs")) +sys.path.insert(0, str(ROOT / "moondeck" / "docs")) import _test_metadata as test_meta # noqa: E402 diff --git a/scripts/build/build_esp32_ethonly.py b/scripts/build/build_esp32_ethonly.py deleted file mode 100644 index c36acf5d..00000000 --- a/scripts/build/build_esp32_ethonly.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# dependencies = ["pyserial"] -# /// -"""Build the ESP32 target for Ethernet only (WiFi compiled out). - -Thin wrapper over build_esp32.py — MoonDeck's runner historically only -forwarded --env/--port, so a fixed --board is injected here. Kept for any -external scripting that already calls this filename. Standalone use: prefer -`build_esp32.py --board esp32-eth` directly. -""" - -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) -import build_esp32 # noqa: E402 - -if __name__ == "__main__": - if "--board" not in sys.argv and "--profile" not in sys.argv: - sys.argv += ["--board", "esp32-eth"] - build_esp32.main() diff --git a/scripts/build/flash_esp32.py b/scripts/build/flash_esp32.py deleted file mode 100644 index 0a28b95d..00000000 --- a/scripts/build/flash_esp32.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python3 -"""Flash a built ESP32 firmware to a device. - -Reads ``build/esp32-<firmware>/projectMM.bin``. The per-firmware build dir -(written by ``build_esp32.py``) makes "which firmware am I flashing" an -on-disk fact rather than an in-memory marker — switching firmwares is a -``--firmware`` change, not a clean-rebuild. - -Prints the artifact size + age before flashing so a stale build (one -from yesterday vs an edit five minutes ago) is visible in the log. -""" - -import argparse -import subprocess -import sys -import time -from pathlib import Path - -ROOT = Path(__file__).resolve().parent.parent.parent -ESP32_DIR = ROOT / "esp32" - -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from build_esp32 import find_idf, idf_env, idf_cmd, FIRMWARES, build_dir_for - - -def _fmt_age(seconds: float) -> str: - """Compact human-readable age (5s, 12m, 3h, 2d).""" - s = int(seconds) - if s < 60: return f"{s}s" - if s < 3600: return f"{s // 60}m" - if s < 86400: return f"{s // 3600}h" - return f"{s // 86400}d" - - -def main(): - parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - parser.add_argument("--port", required=True, help="Serial port") - parser.add_argument("--firmware", required=True, choices=sorted(FIRMWARES), - help="Firmware variant to flash. The build for this " - "firmware must exist at build/esp32-<firmware>/ — " - "i.e. you must have run Build with the same " - "--firmware first.") - parser.add_argument("--baud", type=int, default=460800, - help="esptool flash baud rate (default: 460800 — reliable on " - "every board). The web installer uses 921600 (~2x faster); " - "pass --baud 921600 to match it, but some USB bridges " - "(CP210x/CH340) drop to 'chip stopped responding' mid-flash " - "at that rate, so it's opt-in, not the default.") - args = parser.parse_args() - - if not ESP32_DIR.exists(): - print(f"ESP32 project directory not found: {ESP32_DIR}") - sys.exit(1) - - build_dir = build_dir_for(args.firmware) - image = build_dir / "projectMM.bin" - - if not image.exists(): - print(f"ERROR: no build for {args.firmware!r} at " - f"{build_dir.relative_to(ROOT)}/.") - print(f" Run Build with --firmware {args.firmware} first, then " - f"Flash again.") - sys.exit(2) - - size_kb = image.stat().st_size // 1024 - age = _fmt_age(time.time() - image.stat().st_mtime) - print(f"==> flashing {args.firmware} build ({size_kb} KB, built {age} ago) " - f"to {args.port}") - - idf_path = find_idf() - if not idf_path: - print("ESP-IDF not found. Install it or set IDF_PATH.") - sys.exit(1) - - env = idf_env(idf_path) - cmd = idf_cmd(idf_path) - # -B + -DSDKCONFIG mirror build_esp32.py so idf.py flash reads the - # per-firmware sdkconfig (the chip target lives in there). Without - # -DSDKCONFIG, idf.py reads esp32/sdkconfig at the project root, - # which may belong to a different firmware. - b_arg = [ - "-B", str(build_dir), - "-DSDKCONFIG=" + str(build_dir / "sdkconfig"), - ] - - # -b sets the esptool flash baud (idf.py's own default is also 460800). - # --baud 921600 matches the web installer for ~2x speed, but isn't the - # default because some USB bridges can't sustain it (see --baud help). - r = subprocess.run(cmd + b_arg + ["flash", "-p", args.port, "-b", str(args.baud)], - cwd=ESP32_DIR, env=env) - if r.returncode == 0: - _record_flash_event(args.port, args.firmware) - sys.exit(r.returncode) - - -def _record_flash_event(port: str, firmware: str) -> None: - """Drop a `scripts/.last_flash.json` breadcrumb so MoonDeck can link the - just-flashed serial port to whichever device appears online next. - MoonDeck's _probe_device consumes it on the next refresh and clears it. - Stored as JSON in the same directory as moondeck.json so the entire - "MoonDeck state" lives in one place.""" - import json, time - marker = ROOT / "scripts" / ".last_flash.json" - marker.write_text(json.dumps({ - "port": port, - "firmware": firmware, - "ts": time.time(), - })) - - -if __name__ == "__main__": - main() diff --git a/src/core/AudioModule.h b/src/core/AudioModule.h index 387453ca..b4d2ff37 100644 --- a/src/core/AudioModule.h +++ b/src/core/AudioModule.h @@ -62,7 +62,7 @@ /// projectMM's own implementation, designed from the INMP441 datasheet /// (https://invensense.tdk.com/wp-content/uploads/2015/02/INMP441.pdf) and standard /// DSP rather than traced from any one project. See -/// docs/moonmodules/core/AudioModule.md for the full DSP rationale, the source-seam +/// docs/moonmodules/core/moxygen/AudioModule.md for the full DSP rationale, the source-seam /// backlog (line-in / PDM / analog / I2C codecs), and the forward-looking adaptive /// noise-gate analysis. diff --git a/src/core/DevicesModule.h b/src/core/DevicesModule.h index c979097e..cf13777e 100644 --- a/src/core/DevicesModule.h +++ b/src/core/DevicesModule.h @@ -69,7 +69,7 @@ namespace mm { /// **Prior art:** the industry-standard mDNS-SD / DNS-SD (Bonjour, Avahi) /// announce-and-browse pattern, plus MoonLight's UDP presence broadcast carried /// forward as the 44-byte WLED-compatible packet on UDP 65506. See -/// docs/moonmodules/core/DevicesModule.md for the WLED-interop screenshots + the +/// docs/moonmodules/core/moxygen/DevicesModule.md for the WLED-interop screenshots + the /// wire shape. class DevicesModule : public MoonModule, public ListSource { public: diff --git a/src/core/FileManagerModule.cpp b/src/core/FileManagerModule.cpp new file mode 100644 index 00000000..0f8335e3 --- /dev/null +++ b/src/core/FileManagerModule.cpp @@ -0,0 +1,56 @@ +#include "core/FileManagerModule.h" + +#include "core/FilesystemModule.h" // instance()->lastSavedStr() for the "last saved" readout +#include "platform/platform.h" // fs* primitives + +namespace mm { + +void FileManagerModule::onBuildControls() { + // Only `show hidden` is a control: the whole File Manager surface is the tree panel (app.js + // renderFileManager), which lists over /api/dir and reads the gauges from /api/state; the + // mkdir/delete OPS are their own HTTP endpoints (POST/DELETE /api/dir?path=) — not persisted + // controls — so a create/delete carries its path in the request, not in device storage. The + // `show hidden` flag keeps it bound for the API while the generic control list skips it. + controls_.addBool("show hidden", showHidden_); // reveal dot-prefixed entries (e.g. .config) + controls_.setHidden(controls_.count() - 1, true); + // Filesystem-usage gauge (used / total bytes), read from the platform. Shown below the tree in + // the panel — the File Manager is where filesystem space is relevant, so it owns the control. + // Bound only when the platform reports a real partition (desktop / a no-data-partition chip + // reports 0). loop1s refreshes the used value; the total is fixed. + totalBytes_ = static_cast<uint32_t>(platform::filesystemTotal()); + usedBytes_ = static_cast<uint32_t>(platform::filesystemUsed()); + if (totalBytes_ > 0) { + controls_.addProgress("filesystem", usedBytes_, totalBytes_); + controls_.setHidden(controls_.count() - 1, true); // renders as the usage bar in the panel, not generically + } + // "last saved" readout — how long ago config was persisted. The value is OWNED by the + // FilesystemModule engine (non-UI); the File Manager just displays it here (this is where + // filesystem state is topical). Bind the control straight to the engine's live buffer — no + // per-instance copy — the same no-copy pattern SystemModule uses for its static strings. The + // engine is the boot-wired singleton (alive for the device's life), and its loop1s keeps the + // string current. Bound only when the engine exists (it's constructed before this module). + if (FilesystemModule* fs = FilesystemModule::instance()) { + controls_.addReadOnly("lastSaved", fs->lastSavedStr()); + controls_.setHidden(controls_.count() - 1, true); // shown in the panel header, not generically + } + MoonModule::onBuildControls(); +} + +void FileManagerModule::loop1s() { + if (totalBytes_ > 0) usedBytes_ = static_cast<uint32_t>(platform::filesystemUsed()); +} + +void FileManagerModule::setup() { + MoonModule::setup(); + // `show hidden` is a transient view preference, not device config — force it off on every boot + // regardless of any persisted value (setup() runs after persistence overlays it). A file manager + // opens with hidden entries hidden; the user re-toggles per session. + showHidden_ = false; +} + +// mkdir/delete are HTTP endpoints (POST/DELETE /api/dir?path=) in HttpServerModule: a create/delete +// carries its path in the request and touches the filesystem directly, so this module holds no op +// state and writes nothing to persisted config. The path guard (reject `..`, root at mount) lives +// once in HttpServerModule::parseFilePath, shared with /api/file + /api/dir GET. + +} // namespace mm diff --git a/src/core/FileManagerModule.h b/src/core/FileManagerModule.h new file mode 100644 index 00000000..c3875934 --- /dev/null +++ b/src/core/FileManagerModule.h @@ -0,0 +1,57 @@ +#pragma once +// Core service module — `.h` interface, bodies in FileManagerModule.cpp (the core `.h`+`.cpp` +// convention: it bridges the platform fs layer + has real logic, so implementation edits recompile +// only the .cpp, not every TU that includes this header). + +#include "core/MoonModule.h" + +#include <cstddef> +#include <cstdint> + +namespace mm { + +/// Browse and manage the device filesystem from the UI — the counterpart to FilesystemModule, +/// which is the *persistence engine* (writes `/.config/*.json`, loads at boot). This module is the +/// user-facing **file manager**: an expand/collapse folder tree, each node's name + size, and +/// create / delete / edit of files and folders. +/// +/// **Browsing lives in the UI.** The tree is a client-side lazy tree (the standard VS Code / +/// Explorer / web-file-tree shape): each folder loads *its own* children on first expand from the +/// `/api/dir?path=` endpoint (a single-level listing — the same `platform::fsList` the seam offers), +/// and expand-state is UI state. The module owns none of that; it only exposes the *operations*, +/// keeping the domain module simple and the recursion/paging out of core state and off `/api/state`. +/// +/// **Hidden entries.** A leading `.` (e.g. the `.config` persistence dir) is hidden unless the +/// `show hidden` toggle is on — the standard dotfile convention. The toggle is a transient +/// per-session view preference (a bool the tree forwards to `/api/dir` as the `hidden` filter): +/// `setup()` forces it off on every boot, so a file manager always opens with hidden entries hidden. +/// +/// **Operations are HTTP endpoints, not controls.** `POST /api/dir?path=` creates a folder, +/// `DELETE /api/dir?path=` removes a file or empty folder, and a file's contents are read/written +/// over `/api/file` — the path rides the request query, so an op carries its target in the request +/// rather than a stored control (nothing to persist to flash per op). All share the one path guard +/// (`HttpServerModule::parseFilePath`: reject `..`, root at the mount), and each fails cleanly on a +/// bad path / non-empty-dir delete — never crashes (the Robustness rule). This module itself owns +/// only the `show hidden` toggle + the read-only usage gauges. +/// +/// **Not shown yet:** last-modified dates need a time source (NTP) + LittleFS mtime storage, both +/// backlogged — the tree is name + size for now. +/// +/// **Prior art:** the lazy-loaded folder tree is the standard file-explorer shape (a node loads its +/// children when expanded); the `/api/dir` + `/api/file` split mirrors the listing-vs-contents split +/// every file API draws (WebDAV PROPFIND vs GET). +class FileManagerModule : public MoonModule { +public: + ModuleRole role() const override { return ModuleRole::Peripheral; } + + void onBuildControls() override; + void setup() override; + void loop1s() override; + +private: + bool showHidden_ = false; // reveal dot-prefixed entries (forwarded to /api/dir by the UI) + uint32_t usedBytes_ = 0; // "filesystem" progress: bytes used, refreshed in loop1s + uint32_t totalBytes_ = 0; // "filesystem" progress: partition total, read once at build +}; + +} // namespace mm diff --git a/src/core/FilesystemModule.cpp b/src/core/FilesystemModule.cpp index 20624a3e..4ebca202 100644 --- a/src/core/FilesystemModule.cpp +++ b/src/core/FilesystemModule.cpp @@ -40,24 +40,14 @@ void FilesystemModule::setup() { platform::filesystemUsed(), platform::filesystemTotal()); } -void FilesystemModule::onBuildControls() { - controls_.addReadOnly("lastSaved", lastSaveStr_, sizeof(lastSaveStr_)); - // Filesystem-partition usage bar (bytes used / total). Lives here — on the module that owns the - // filesystem — not on SystemModule. Read the total once; loop1s refreshes the used value. Bound - // only when the platform reports a real partition (desktop / a chip without a data partition - // reports 0, so the bar is omitted rather than showing 0/0). - totalFsVal_ = static_cast<uint32_t>(platform::filesystemTotal()); - fsUsedVal_ = static_cast<uint32_t>(platform::filesystemUsed()); - if (totalFsVal_ > 0) { - controls_.addProgress("filesystem", fsUsedVal_, totalFsVal_); - } - MoonModule::onBuildControls(); -} +// FilesystemModule is a non-UI persistence engine: it holds no controls (hence no +// onBuildControls override), so it renders no card in the module tree — a card here would +// confuse an end user next to the File Manager. Its one piece of status, "last saved", is +// displayed by FileManagerModule, which reads it via FilesystemModule::instance()->lastSavedStr(). +// The filesystem-usage gauge likewise lives on FileManagerModule (that's where filesystem state +// is topical). void FilesystemModule::loop1s() { - // Refresh the usage bar first — cheap, and it should track saves even before the mount/scheduler - // guards below (the total is fixed; only the used value moves as files are written). - if (totalFsVal_ > 0) fsUsedVal_ = static_cast<uint32_t>(platform::filesystemUsed()); if (!mounted_ || !scheduler_) return; updateLastSavedStr(); if (!dirtyPending_) return; diff --git a/src/core/FilesystemModule.h b/src/core/FilesystemModule.h index edcafd3a..8c30b9a1 100644 --- a/src/core/FilesystemModule.h +++ b/src/core/FilesystemModule.h @@ -95,15 +95,26 @@ class FilesystemModule : public MoonModule { /// the UI before the 2s debounce expires. bool respectsEnabled() const override { return false; } + /// Non-UI: a pure persistence engine with no controls — not shown as a card (its "last saved" + /// status is displayed by FileManagerModule). See MoonModule::appearsInUi. + bool appearsInUi() const override { return false; } + void setScheduler(Scheduler* s); void setup() override; - - /// Binds the read-only "lastSaved" display (how long ago the config was last written, - /// or "never" before any save) and the "filesystem" partition-usage progress bar (bound - /// only when the platform reports a real data partition; 0 → omitted). - void onBuildControls() override; void loop1s() override; + /// The engine's live "last saved" buffer — "never" before the first save, else "5m ago". This + /// is the persistence engine's status; FileManagerModule binds its read-only "lastSaved" control + /// straight to this buffer (no copy — same no-per-instance-copy pattern SystemModule uses for + /// its statics), and this module's loop1s keeps it current. Returns the mutable buffer because + /// addReadOnly takes a char* it points the control at. FilesystemModule itself has NO controls — + /// it's a non-UI engine, not a card in the module tree. + char* lastSavedStr() { return lastSaveStr_; } + + /// The live singleton (the main-wired instance registered in setScheduler), or null before + /// boot wiring. FileManagerModule reads lastSavedStr() through this. + static FilesystemModule* instance() { return instance_; } + /// Synchronous save of every dirty subtree, bypassing the debounce. Same work /// loop1s does once the debounce expires. Exposed for tests so they can assert /// the file appears without wall-clock waits; production callers shouldn't need this. @@ -125,9 +136,7 @@ class FilesystemModule : public MoonModule { bool everSaved_ = false; ///< false until the first successful save uint32_t lastDirtyMs_ = 0; uint32_t lastSaveMs_ = 0; - char lastSaveStr_[24] = "never"; ///< "lastSaved" read-only control value - uint32_t fsUsedVal_ = 0; ///< "filesystem" progress: bytes used, refreshed in loop1s - uint32_t totalFsVal_ = 0; ///< "filesystem" progress: partition total, read once in onBuildControls + char lastSaveStr_[24] = "never"; ///< "last saved" status string; FileManagerModule reads it via lastSavedStr() /// Shared load/save buffer — load runs once at boot (phase 2), save runs in loop1s after /// the 2s debounce. Mutually exclusive, so one buffer is enough. Kept off the task stack /// since 2KB plus recursive applyNode/writeNode frames is uncomfortably close to the ESP32 diff --git a/src/core/FirmwareUpdateModule.h b/src/core/FirmwareUpdateModule.h index ff29ad5c..9636d4d1 100644 --- a/src/core/FirmwareUpdateModule.h +++ b/src/core/FirmwareUpdateModule.h @@ -29,6 +29,16 @@ inline char g_otaStatus[64] = "idle"; inline uint32_t g_otaBytesRead = 0; inline uint32_t g_otaBytesTotal = 0; +// True while an OTA is running (as opposed to idle / a terminal "success"/"failed:…"). +// The URL and upload flash paths both gate their 409 "already in progress" guard on this, +// so the set of in-flight states lives in one place instead of a duplicated strcmp chain. +inline bool otaInFlight() { + return std::strcmp(g_otaStatus, "starting") == 0 || + std::strcmp(g_otaStatus, "downloading") == 0 || + std::strcmp(g_otaStatus, "flashing") == 0 || + std::strcmp(g_otaStatus, "rebooting") == 0; +} + /// A thin status surface for OTA flashing — surfaces flash progress as live /// read-only controls plus the per-module status banner /// @@ -65,7 +75,7 @@ inline uint32_t g_otaBytesTotal = 0; /// /// **Prior art:** `esp_https_ota` is the standard ESP-IDF OTA-from-HTTP component used by /// every ESP32 OTA flow since IDF v4.x; the install-picker UI is the new layer on top. See -/// docs/moonmodules/core/FirmwareUpdateModule.md for the `POST /api/firmware/url` wire +/// docs/moonmodules/core/moxygen/FirmwareUpdateModule.md for the `POST /api/firmware/url` wire /// contract, the compatibility rules, and the flash lifecycle + error taxonomy. class FirmwareUpdateModule : public MoonModule { public: diff --git a/src/core/HttpServerModule.cpp b/src/core/HttpServerModule.cpp index 8ff15d7a..f44e1b24 100644 --- a/src/core/HttpServerModule.cpp +++ b/src/core/HttpServerModule.cpp @@ -21,6 +21,8 @@ #include <climits> #include <cstdarg> #include <cstdio> +#include <cstdlib> // strtol — bounded Content-Length parse +#include <cerrno> // errno / ERANGE — Content-Length overflow check #include <cstring> #include <cstdint> @@ -101,14 +103,52 @@ void HttpServerModule::handleConnection(platform::TcpConnection& conn) { // the permissive JSON helpers (a silent partial control write). If the full declared // body still hasn't arrived within the budget, reject with 400 rather than process it. auto* headerEnd = std::strstr(req, "\r\n\r\n"); + int contentLen = 0; // declared body length (0 if no Content-Length); used by the streaming route if (headerEnd) { auto* clh = std::strstr(req, "Content-Length:"); if (clh) { - int contentLen = std::atoi(clh + 15); + // Bounded parse (not atoi): a malformed/negative/overflowing Content-Length must not + // flow downstream, where it's cast to size_t — a negative int would become a huge + // length that UploadSource/handleFirmwareUpload would treat as "gigabytes still to + // come". We reject anything that isn't a clean unsigned integer: strtol with an end + // pointer catches non-numeric, trailing junk ("123abc"), and ERANGE overflow; then we + // reject negative and clamp to a firmware-sized ceiling (8 MB > any image we flash), + // returning 400 rather than acting on it. The value ends at CR/LF/space or the string end. + constexpr long kContentLenMax = 8L * 1024 * 1024; + const char* valStart = clh + 15; + while (*valStart == ' ' || *valStart == '\t') valStart++; // skip OWS after the colon + char* valEnd = nullptr; + errno = 0; + const long parsed = std::strtol(valStart, &valEnd, 10); + const bool consumedDigits = valEnd != valStart; + const bool endsCleanly = *valEnd == '\r' || *valEnd == '\n' || *valEnd == ' ' || + *valEnd == '\t' || *valEnd == '\0'; + if (!consumedDigits || !endsCleanly || errno == ERANGE || + parsed < 0 || parsed > kContentLenMax) { + sendResponse(conn, 400, "application/json", + "{\"error\":\"invalid content-length\"}"); + return; + } + contentLen = static_cast<int>(parsed); int headerSize = static_cast<int>(headerEnd + 4 - req); int bodyNeeded = headerSize + contentLen; - if (bodyNeeded > static_cast<int>(sizeof(buf) - 1)) - bodyNeeded = static_cast<int>(sizeof(buf) - 1); // cap to buffer + // Only the STREAMING routes (/api/file, /api/firmware/upload) may carry a body larger than + // buf — they take the buffered prefix and pull the remainder straight off the socket. For + // every OTHER route the body is parsed whole from buf, so a body over the buffer must be + // REJECTED (413), not truncated: a capped read would parse a JSON prefix as if complete + // (its own bodyNeeded check wouldn't fire, since the cap makes the short read "enough"). + // The request line sits at the start of req; a substring match on the path is sufficient. + const bool isStreamingRoute = + std::strncmp(req, "POST /api/file", 14) == 0 || + std::strncmp(req, "POST /api/firmware/upload", 25) == 0; + if (bodyNeeded > static_cast<int>(sizeof(buf) - 1)) { + if (!isStreamingRoute) { + sendResponse(conn, 413, "application/json", + "{\"error\":\"request body too large\"}"); + return; + } + bodyNeeded = static_cast<int>(sizeof(buf) - 1); // streaming: buffer the prefix only + } for (int empties = 0; totalRead < bodyNeeded;) { int n = conn.read(buf + totalRead, sizeof(buf) - 1 - totalRead); if (n > 0) { totalRead += n; empties = 0; } @@ -133,7 +173,7 @@ void HttpServerModule::handleConnection(platform::TcpConnection& conn) { // and is not part of the path. Browsers send `/?foo=bar` for query-on- // root; without this split the GET / route falls through to 404. The web // installer's Inject button hits us as `/?deviceModel=<name>` to hand off the - // deviceModels.json entry — see docs/moonmodules/core/SystemModule.md. + // deviceModels.json entry — see docs/moonmodules/core/moxygen/SystemModule.md. char* queryStart = std::strchr(path, '?'); if (queryStart) *queryStart = 0; @@ -161,6 +201,11 @@ void HttpServerModule::handleConnection(platform::TcpConnection& conn) { else if (std::strcmp(path, "/api/state") == 0) serveState(conn); else if (std::strcmp(path, "/api/system") == 0) serveSystem(conn); else if (std::strcmp(path, "/api/types") == 0) serveTypes(conn); + // File Manager: GET /api/dir?path=<rel>[&hidden=1] → one directory's children as JSON + // [{name,isDir,size}] (the lazy tree loads a node's children on expand). + else if (std::strcmp(path, "/api/dir") == 0) serveDirListing(conn, queryStart ? queryStart + 1 : ""); + // File Manager: GET /api/file?path=<rel> → the file's contents (text, size-capped). + else if (std::strcmp(path, "/api/file") == 0) serveFileContents(conn, queryStart ? queryStart + 1 : ""); // WLED-compatibility shim: the native WLED apps (and Home Assistant's WLED // integration) discover a device via mDNS `_wled._tcp` then VALIDATE it by // GETting /json/info and checking it's WLED-shaped. Serving a minimal @@ -189,6 +234,19 @@ void HttpServerModule::handleConnection(platform::TcpConnection& conn) { std::strcmp(path + pathLen - 8, "/replace") == 0; if (std::strcmp(path, "/api/control") == 0 && body) { handleSetControl(conn, body); + } else if (std::strcmp(path, "/api/file") == 0 && body) { + // File Manager: POST /api/file?path=<rel>, the body → streamed atomic write. `body` + // points at the bytes already buffered (initialLen); the full length is Content-Length, + // and handleWriteFile pulls any remainder straight off the socket — so an upload of any + // size streams to the file without a whole-request buffer or a strlen (binary-safe). + const size_t initialLen = static_cast<size_t>(totalRead) - static_cast<size_t>(body - req); + handleWriteFile(conn, queryStart ? queryStart + 1 : "", body, initialLen, + static_cast<size_t>(contentLen)); + } else if (std::strcmp(path, "/api/dir") == 0) { + // File Manager: POST /api/dir?path=<rel> → mkdir. The path is the whole operation (a + // create is a filesystem action, not a stored control), so it rides the request query + // — same path-as-query shape as /api/file, no persisted control holds it. + handleMakeDir(conn, queryStart ? queryStart + 1 : ""); } else if (std::strcmp(path, "/api/modules") == 0 && body) { handleAddModule(conn, body); } else if (isMoveRoute && body) { @@ -222,6 +280,12 @@ void HttpServerModule::handleConnection(platform::TcpConnection& conn) { handleReboot(conn); } else if (std::strcmp(path, "/api/firmware/url") == 0 && body) { handleFirmwareUrl(conn, body); + } else if (std::strcmp(path, "/api/firmware/upload") == 0 && body) { + // OTA from an uploaded .bin body (no URL, no host to serve it) — the browser POSTs the + // firmware image straight to the device, which streams it into the OTA partition. Same + // streamed-body handling as /api/file (initial buffered bytes + socket remainder). + const size_t initialLen = static_cast<size_t>(totalRead) - static_cast<size_t>(body - req); + handleFirmwareUpload(conn, body, initialLen, static_cast<size_t>(contentLen)); } else { sendResponse(conn, 404, "text/plain", "Not found"); } @@ -229,6 +293,9 @@ void HttpServerModule::handleConnection(platform::TcpConnection& conn) { // DELETE /api/modules/ModuleName if (std::strncmp(path, "/api/modules/", 13) == 0) { handleDeleteModule(conn, path + 13); + } else if (std::strcmp(path, "/api/dir") == 0) { + // File Manager: DELETE /api/dir?path=<rel> → remove a file or empty dir. + handleRemoveEntry(conn, queryStart ? queryStart + 1 : ""); } else { sendResponse(conn, 404, "text/plain", "Not found"); } @@ -298,6 +365,303 @@ void HttpServerModule::sendResponse(platform::TcpConnection& conn, int status, c conn.write(reinterpret_cast<const uint8_t*>(body), bodyLen); } +// --- File Manager file read/write (the /api/file endpoints) --- +// +// A file body isn't a control value, so these are their own small endpoints (not /api/control). +// The path comes as a query param `path=<rel>`; parseFilePath vets it (reject "..", root at the +// mount) — the single traversal guard shared by every filesystem HTTP entry (read, write, dir +// listing, mkdir, delete). +// +// Read + write both stream: the write pulls the request body chunk-by-chunk straight to the file +// (fsWriteStream), the read pulls the file into a size-fit buffer — so a file of any size up- and +// downloads intact without a fixed cap. kUploadMax is a per-request sanity ceiling; a legit upload +// is additionally rejected up front if it wouldn't fit the free filesystem space. +static constexpr size_t kUploadMax = 256 * 1024; // 256 KB — sanity bound on one upload + +// Copy the `path=` query value into `out` (decoding %XX and '+' minimally), rooted at the mount. +// Returns false on a missing/empty path or a ".." traversal attempt. +// +// Deliberately NOT a `.config`/dotfile denylist (PO decision): the File Manager is a device-admin +// tool on a trusted LAN, and reading the persisted `.config/*.json` is a feature (inspect/back up +// the device's own config), not a leak — there are no third-party secrets on the device, and the +// WiFi password is XOR-obfuscated in what it writes. The weak-protection is `show hidden` defaulting +// off (FileManagerModule), so `.config` isn't shown unless the operator asks. Reviewers periodically +// flag this as a secrets-exposure — it's an accepted design, not an oversight; leave it. +bool HttpServerModule::parseFilePath(const char* query, char* out, size_t cap) { + const char* p = query ? std::strstr(query, "path=") : nullptr; + if (!p) return false; + p += 5; // past "path=" + size_t i = 0; + // The path may be its own query (stop at '&') and percent-encoded ('/' → %2F, ' ' → %20). + while (*p && *p != '&' && i + 1 < cap) { + char c = *p; + if (c == '%' && p[1] && p[2]) { // %XX → byte + auto hex = [](char h) -> int { + if (h >= '0' && h <= '9') return h - '0'; + if (h >= 'a' && h <= 'f') return h - 'a' + 10; + if (h >= 'A' && h <= 'F') return h - 'A' + 10; + return -1; + }; + const int hi = hex(p[1]), lo = hex(p[2]); + if (hi >= 0 && lo >= 0) { c = static_cast<char>((hi << 4) | lo); p += 2; } + } else if (c == '+') { + c = ' '; + } + out[i++] = c; + p++; + } + // Reject an overlong path outright rather than routing on a truncated prefix: if the loop stopped + // because the buffer filled (still more path bytes to come, i.e. not at '\0' or the '&' delimiter), + // the decoded value is incomplete and must not be treated as a valid path. + if (*p && *p != '&') return false; + out[i] = 0; + if (i == 0 || std::strstr(out, "..")) return false; // empty or traversal → reject + if (out[0] != '/') { // relative → root at the mount + char rooted[160]; + const int n = std::snprintf(rooted, sizeof(rooted), "/%s", out); + if (n <= 0 || static_cast<size_t>(n) >= cap) return false; + std::strncpy(out, rooted, cap - 1); out[cap - 1] = 0; + } + return true; +} + +// --- File Manager directory listing (the /api/dir endpoint) --- +// +// One directory's children as a JSON array, the source the lazy tree loads a node's children from. +// Single-level only (platform::fsList) — the recursion is the UI's job, one fetch per expanded node, +// the standard file-tree shape. The `hidden` query flag (hidden=1) includes dot-prefixed entries. +// The listing streams straight to the socket (as serveState does) — no whole-listing buffer. The +// fsList C callback carries the streaming sink + the hidden filter + a first-row flag via `user`. +namespace { +struct DirListState { + JsonSink* sink; + bool showHidden; + bool first = true; +}; +void dirListTrampoline(const char* name, bool isDir, uint32_t size, void* user) { + auto* st = static_cast<DirListState*>(user); + if (!st->showHidden && name[0] == '.') return; // dotfile convention + if (!st->first) st->sink->append(","); + st->first = false; + st->sink->append("{\"name\":"); + st->sink->writeJsonString(name); + st->sink->appendf(",\"isDir\":%s,\"size\":%lu}", + isDir ? "true" : "false", static_cast<unsigned long>(size)); +} +} // namespace + +void HttpServerModule::serveDirListing(platform::TcpConnection& conn, const char* query) { + char path[160]; + if (!parseFilePath(query, path, sizeof(path))) { + sendResponse(conn, 400, "application/json", "{\"error\":\"bad path\"}"); + return; + } + const char* header = + "HTTP/1.1 200 OK\r\n" + "Content-Type: application/json\r\n" + "Connection: close\r\n" + "Access-Control-Allow-Origin: *\r\n" + "\r\n"; + conn.write(reinterpret_cast<const uint8_t*>(header), std::strlen(header)); + + JsonSink sink(conn); + DirListState st{&sink, query && std::strstr(query, "hidden=1") != nullptr, true}; + sink.append("["); + platform::fsList(path, &dirListTrampoline, &st); + sink.append("]"); + sink.flush(); +} + +// POST /api/dir?path=<rel> → mkdir. The path rides the query and is vetted by parseFilePath (the +// same `..`-reject + root-at-mount guard /api/file and /api/dir GET use). A create is a filesystem +// action, not a stored control — no persisted `path` control holds it, so no flash write. +void HttpServerModule::handleMakeDir(platform::TcpConnection& conn, const char* query) { + char path[160]; + if (!parseFilePath(query, path, sizeof(path))) { + sendResponse(conn, 400, "application/json", "{\"error\":\"bad path\"}"); + return; + } + if (platform::fsMkdir(path)) sendResponse(conn, 200, "application/json", "{\"ok\":true}"); + else sendResponse(conn, 500, "application/json", "{\"error\":\"mkdir failed\"}"); +} + +// DELETE /api/dir?path=<rel> → remove a file or EMPTY dir (fsRemove fails cleanly on a non-empty +// dir). Same path guard as handleMakeDir. +void HttpServerModule::handleRemoveEntry(platform::TcpConnection& conn, const char* query) { + char path[160]; + if (!parseFilePath(query, path, sizeof(path))) { + sendResponse(conn, 400, "application/json", "{\"error\":\"bad path\"}"); + return; + } + if (platform::fsRemove(path)) sendResponse(conn, 200, "application/json", "{\"ok\":true}"); + else sendResponse(conn, 500, "application/json", "{\"error\":\"delete failed (folder not empty?)\"}"); +} + +void HttpServerModule::serveFileContents(platform::TcpConnection& conn, const char* query) { + char path[160]; + if (!parseFilePath(query, path, sizeof(path))) { + sendResponse(conn, 400, "application/json", "{\"error\":\"bad path\"}"); + return; + } + // Stream the file straight to the socket in fixed 1 KB chunks (fsReadAt) with an explicit + // Content-Length header — no whole-file buffer, and NUL-safe (sendResponse strlen()s its body, so + // it can't carry binary). Symmetric with the streamed upload: a file of any size downloads whole. + const long size = platform::fsSize(path); + if (size < 0) { sendResponse(conn, 404, "application/json", "{\"error\":\"not found\"}"); return; } + char header[160]; + const int hn = std::snprintf(header, sizeof(header), + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %ld\r\n" + "Connection: close\r\nAccess-Control-Allow-Origin: *\r\n\r\n", size); + conn.write(reinterpret_cast<const uint8_t*>(header), static_cast<size_t>(hn)); + char chunk[1024]; + for (long offset = 0; offset < size;) { + const size_t want = static_cast<size_t>(size - offset) < sizeof(chunk) + ? static_cast<size_t>(size - offset) : sizeof(chunk); + const int got = platform::fsReadAt(path, offset, chunk, want); + if (got <= 0) break; // read error / early EOF — the client sees a short (truncated) body + conn.write(reinterpret_cast<const uint8_t*>(chunk), static_cast<size_t>(got)); + offset += got; + } +} + +// Source state for the streamed upload: yields the body bytes already sitting in the request buffer, +// then reads the remainder straight off the socket — feeding fsWriteStream in fixed chunks so the +// device never holds the whole upload in RAM. +namespace { +// This drain runs SYNCHRONOUSLY on the loop20ms() tick, which is inside Scheduler::tick — so it +// blocks rendering for the duration of the transfer (LEDs freeze until the upload completes or a +// bound trips). Accepted trade-off: an upload is user-initiated and transient (and a firmware upload +// reboots the device anyway), so a brief freeze is fine where a persistent one wouldn't be. The two +// bounds cap how long that freeze can last, because neither alone is enough: +// - kUploadIdleMs: max wait for the NEXT byte, reset on every successful read. Scales to +// any size the endpoint accepts — a big but steady upload (256 KB over slow LittleFS + +// weak WiFi) never trips it, because progress keeps resetting the clock. But idle-only +// lets a slowloris trickle one byte just under the idle limit forever, freezing rendering +// (and the HTTP server) for as long as it keeps dribbling. +// - kUploadHardMs: an absolute whole-request ceiling that closes that hole. Sized well +// above a legit worst case (256 KB / ~50 KB/s ≈ 5 s, plus wide margin) so a real slow +// upload finishes, but far below the days a byte-per-idle-window trickler would need. +// A single budget can't do both jobs; the pair does (idle scales, hard caps the total). The +// zero-freeze fix (drain a chunk per tick, like drainPreviewSend) is backlogged; the bounded +// synchronous drain is the accepted interim. +constexpr uint32_t kUploadIdleMs = 5000; // max gap between successful reads before abort +constexpr uint32_t kUploadHardMs = 60000; // absolute whole-request ceiling (anti-slowloris) +struct UploadSource { + platform::TcpConnection* conn; + const char* initial; // body bytes already read into the request buffer + size_t initialLeft; // how many of those remain to hand out + size_t remaining; // total body bytes still to deliver (Content-Length − delivered) + uint32_t hardDeadline; // absolute millis by which the whole body must arrive +}; +size_t uploadPull(char* out, size_t cap, void* user, bool* abort) { + auto* s = static_cast<UploadSource*>(user); + if (s->remaining == 0) return 0; // all body delivered → clean EOF + // Whole-request ceiling, checked on EVERY pull (not only while the socket is dry): a paced + // trickler that always keeps one byte ready makes each read return > 0 immediately, so a cap + // tested only in the wait loop would never fire. Enforcing it here makes it truly absolute. + if (static_cast<int32_t>(platform::millis() - s->hardDeadline) >= 0) { *abort = true; return 0; } + // Drain the already-buffered prefix first. + if (s->initialLeft) { + const size_t n = s->initialLeft < cap ? s->initialLeft : cap; + std::memcpy(out, s->initial, n); + s->initial += n; s->initialLeft -= n; s->remaining -= n; + return n; + } + // Then pull the rest off the socket, bounded by BOTH the per-pull idle deadline (recomputed + // here, only advances while we wait — bounds a stall) and the request-lifetime hardDeadline + // (set once at construction — bounds the total). If the body is still incomplete when the + // socket closes early or either deadline lapses, signal *abort — fsWriteStream then discards + // the temp file rather than committing a truncated upload (a 0 here is NOT a clean end). Both + // compares are subtraction-based, wraparound-safe across the ~49.7-day millis() rollover. + const size_t want = s->remaining < cap ? s->remaining : cap; + const uint32_t deadline = platform::millis() + kUploadIdleMs; + for (;;) { + const int r = s->conn->read(reinterpret_cast<uint8_t*>(out), want); + if (r > 0) { s->remaining -= static_cast<size_t>(r); return static_cast<size_t>(r); } + if (r == 0) { *abort = true; return 0; } // peer closed with body remaining + // Idle timeout (the hard whole-request cap is enforced at the top of uploadPull, so it + // covers the pacing case this wait loop can't). Both compares are wraparound-safe. + if (static_cast<int32_t>(platform::millis() - deadline) >= 0) { *abort = true; return 0; } + platform::delayMs(1); + } +} +} // namespace + +void HttpServerModule::handleWriteFile(platform::TcpConnection& conn, const char* query, + const char* initialBody, size_t initialLen, size_t contentLen) { + char path[160]; + if (!parseFilePath(query, path, sizeof(path))) { + sendResponse(conn, 400, "application/json", "{\"error\":\"bad path\"}"); + return; + } + if (contentLen > kUploadMax) { + sendResponse(conn, 413, "application/json", "{\"error\":\"file too large\"}"); + return; + } + // Reject up front if it wouldn't fit the free filesystem space (friendlier than filling the FS + // and failing mid-write — fsWriteStream also fails cleanly + discards the temp if it does fill). + // total − used = free. An overwrite would reclaim the old file's space, but treat free + // conservatively (don't credit the overwrite) so the check never over-promises. + const size_t total = platform::filesystemTotal(); + const size_t used = platform::filesystemUsed(); + const size_t freeBytes = total > used ? total - used : 0; + if (total > 0 && contentLen > freeBytes) { + char msg[96]; + std::snprintf(msg, sizeof(msg), "{\"error\":\"not enough space (%lu free)\"}", + static_cast<unsigned long>(freeBytes)); + sendResponse(conn, 507, "application/json", msg); // 507 Insufficient Storage + return; + } + // Never hand the source more than Content-Length of the already-buffered bytes: a buffer can hold + // bytes past the body (a pipelined next request), which must not be written into the file. + const size_t initial = initialLen < contentLen ? initialLen : contentLen; + UploadSource src{&conn, initialBody, initial, contentLen, + platform::millis() + kUploadHardMs}; + if (platform::fsWriteStream(path, &uploadPull, &src)) { + sendResponse(conn, 200, "application/json", "{\"ok\":true}"); + } else { + sendResponse(conn, 500, "application/json", "{\"error\":\"write failed\"}"); + } +} + +// OTA from an uploaded .bin body: stream the request body straight into the OTA partition +// (platform::otaWriteStream), reusing the exact uploadPull the file-upload path uses — the only +// difference is the sink (OTA partition vs a file). On success the device reboots into the new +// image; the 200 goes out first (otaWriteStream's ~600 ms pre-reboot delay covers the round-trip). +void HttpServerModule::handleFirmwareUpload(platform::TcpConnection& conn, const char* initialBody, + size_t initialLen, size_t contentLen) { + if constexpr (!platform::hasOta) { + sendResponse(conn, 501, "application/json", "{\"error\":\"OTA not supported on this platform\"}"); + return; + } + // Same 409 concurrency guard as handleFirmwareUrl: one OTA at a time (both write g_ota* state). + if (otaInFlight()) { + sendResponse(conn, 409, "application/json", "{\"error\":\"ota already in progress\"}"); + return; + } + const size_t initial = initialLen < contentLen ? initialLen : contentLen; + UploadSource src{&conn, initialBody, initial, contentLen, platform::millis() + kUploadHardMs}; + g_otaBytesTotal = static_cast<uint32_t>(contentLen); // the UI's "Y KB" (Content-Length up front) + g_otaBytesRead = 0; // clear any stale count from a prior OTA + // Stream the body into the OTA partition. otaWriteStream commits the image + flips the boot + // pointer but does NOT reboot — it returns so we can send a 200 first, then reboot the same + // way /api/reboot does (response, close, brief drain, platform::reboot). That gives the browser + // a clean "flashed" response instead of an aborted socket it can't tell from a real failure. + const bool ok = platform::otaWriteStream(&uploadPull, &src, contentLen, + g_otaStatus, sizeof(g_otaStatus), &g_otaBytesRead); + if (!ok) { + char msg[96]; + std::snprintf(msg, sizeof(msg), "{\"error\":\"ota failed: %.60s\"}", g_otaStatus); + sendResponse(conn, 500, "application/json", msg); + return; + } + FilesystemModule::flushPending(); + sendResponse(conn, 200, "application/json", "{\"ok\":true}"); + conn.close(); + platform::delayMs(200); + platform::reboot(); // noreturn — boots the flashed image +} + void HttpServerModule::serveFile(platform::TcpConnection& conn, const char* filename, const char* contentType) { // Try disk first (desktop development — live editing without rebuild) char filepath[256]; @@ -388,7 +752,10 @@ void HttpServerModule::buildStateJson(JsonSink& sink) { bool first = true; for (uint8_t m = 0; m < scheduler_->moduleCount(); m++) { auto* mod = scheduler_->module(m); - if (!mod || mod == this) continue; // skip self + // Skip modules that opt out of the UI via appearsInUi() — the one mechanism for + // "not a card in /api/state": HttpServerModule (the server itself) and FilesystemModule + // (a pure persistence engine, no controls) both return false. + if (!mod || !mod->appearsInUi()) continue; if (!first) sink.append(","); first = false; writeModuleJson(sink, mod); @@ -544,25 +911,11 @@ void HttpServerModule::handleSetControl(platform::TcpConnection& conn, const cha } } +// The Scheduler owns the module tree, so the tree-walk-by-name lives there (firstByName); +// this only adds the scheduler_ null-guard the request handlers rely on (scheduler_ is unset +// until setScheduler() runs), then delegates — one recursive lookup, not two. MoonModule* HttpServerModule::findModuleByName(const char* name) { - if (!name || name[0] == 0 || !scheduler_) return nullptr; - - for (uint8_t m = 0; m < scheduler_->moduleCount(); m++) { - auto* mod = scheduler_->module(m); - if (!mod) continue; - auto* found = findInTree(mod, name); - if (found) return found; - } - return nullptr; -} - -MoonModule* HttpServerModule::findInTree(MoonModule* mod, const char* name) { - if (mod->name() && std::strcmp(mod->name(), name) == 0) return mod; - for (uint8_t i = 0; i < mod->childCount(); i++) { - auto* found = findInTree(mod->child(i), name); - if (found) return found; - } - return nullptr; + return scheduler_ ? scheduler_->firstByName(name) : nullptr; } void HttpServerModule::serveSystem(platform::TcpConnection& conn) { @@ -607,7 +960,7 @@ void HttpServerModule::serveSystem(platform::TcpConnection& conn) { // WLED fork) does — while `product:"MoonModules"` says what this actually is. We speak // WLED's info shape to interoperate, not to impersonate. Built fresh against WLED's // public JSON, not copied. (Reference real WLED carries far more; this is the trimmed, -// known-sufficient field set — see docs/moonmodules/core/HttpServerModule.md.) +// known-sufficient field set — see docs/moonmodules/core/moxygen/HttpServerModule.md.) void HttpServerModule::serveWledInfo(platform::TcpConnection& conn) { const char* header = "HTTP/1.1 200 OK\r\n" @@ -661,18 +1014,6 @@ void HttpServerModule::writeWledInfoBody(JsonSink& sink, const char* name, const // output brightness the WLED app's slider maps to. Read generically through the control // list (by name + Uint8 type, via the stored pointer) so this core module needs no // light-domain include — the same domain-neutral reach applySetControl uses to write. -uint8_t HttpServerModule::driversBrightness() { - MoonModule* d = findModuleByName("Drivers"); - if (!d) return 0; - const ControlList& cl = d->controls(); - for (uint8_t i = 0; i < cl.count(); i++) { - const ControlDescriptor& c = cl[i]; - if (c.ptr && c.type == ControlType::Uint8 && std::strcmp(c.name, "brightness") == 0) - return *static_cast<const uint8_t*>(c.ptr); - } - return 0; -} - // The WLED state object, written into an open sink. `on` + `bri` mirror Drivers // brightness (off = brightness 0). `seg[0].col[0]` is the colour the WLED app tints the // device card with: we send the LIVE first-LED RGB from Drivers, so the card mirrors what @@ -680,7 +1021,7 @@ uint8_t HttpServerModule::driversBrightness() { // black/off or there's no output (so a dark device still reads as a distinct projectMM, // not an indistinct black card). void HttpServerModule::writeWledStateBody(JsonSink& sink) { - const uint8_t bri = driversBrightness(); + const uint8_t bri = driversBrightness(scheduler_); uint8_t rgb[3] = {0, 0, 0}; bool haveLed = false; if (MoonModule* d = findModuleByName("Drivers")) haveLed = d->firstOutputRgb(rgb); @@ -689,7 +1030,7 @@ void HttpServerModule::writeWledStateBody(JsonSink& sink) { } sink.appendf("{\"on\":%s,\"bri\":%u," "\"seg\":[{\"id\":0,\"col\":[[%u,%u,%u]]}]}", - bri > 0 ? "true" : "false", bri, rgb[0], rgb[1], rgb[2]); + driversOn(scheduler_) ? "true" : "false", bri, rgb[0], rgb[1], rgb[2]); } void HttpServerModule::serveWledState(platform::TcpConnection& conn) { @@ -726,19 +1067,18 @@ void HttpServerModule::serveWledStateInfo(platform::TcpConnection& conn) { sink.flush(); } -// Apply a WLED state-set body ({on?, bri?}) to the Drivers brightness control through the -// shared apply-core (the same path /api/control and Improv APPLY_OP use). `on:false` → 0; -// `on:true` with no `bri` → restore a visible default; `bri:N` → set N. Shared by the HTTP -// POST /json/state handler and the inbound-WebSocket path (the app uses both channels). +// Apply a WLED state-set body ({on?, bri?}) to the Drivers controls through the shared apply-core +// (the same path /api/control and Improv APPLY_OP use). `on` and `bri` are independent: `on` sets +// the real master-power control (so toggling off preserves the brightness level), `bri` sets the +// level. Shared by the HTTP POST /json/state handler and the inbound-WebSocket path. void HttpServerModule::applyWledState(const char* body) { - int bri = -1; - if (mm::json::hasKey(body, "bri")) bri = mm::json::parseInt(body, "bri"); if (mm::json::hasKey(body, "on")) { - const bool on = mm::json::parseBool(body, "on"); - if (!on) bri = 0; - else if (bri < 0) bri = driversBrightness() > 0 ? -1 : 128; // turn on → visible default + applySetControl("Drivers", "on", + mm::json::parseBool(body, "on") ? "{\"value\":true}" : "{\"value\":false}"); } - if (bri >= 0) { + if (mm::json::hasKey(body, "bri")) { + int bri = mm::json::parseInt(body, "bri"); + if (bri < 0) bri = 0; if (bri > 255) bri = 255; char valueJson[32]; std::snprintf(valueJson, sizeof(valueJson), "{\"value\":%d}", bri); @@ -886,7 +1226,7 @@ HttpServerModule::OpResult HttpServerModule::applyClearChildren(const char* pare // field — both feed the one applyAddModule() core, but the two transports parse different // JSON keys, so an HTTP payload is NOT a drop-in APPLY_OP (rename parent_id → parent). The // serial op stays terse because every byte counts against the 128-byte frame budget; the -// discrepancy is documented in docs/moonmodules/core/ImprovProvisioningModule.md. +// discrepancy is documented in docs/moonmodules/core/moxygen/ImprovProvisioningModule.md. HttpServerModule::OpResult HttpServerModule::applyOp(const char* opJson) { if (!opJson) return OpResult::BadRequest; char op[16] = {}; @@ -1168,10 +1508,7 @@ void HttpServerModule::handleFirmwareUrl(platform::TcpConnection& conn, const ch // shows garbled progress. Check g_otaStatus for an in-flight state and // reject early with 409. Successful OTAs reboot, so the only path that // re-enables a new attempt after an in-flight one is an explicit error. - if (std::strcmp(g_otaStatus, "starting") == 0 || - std::strcmp(g_otaStatus, "downloading") == 0 || - std::strcmp(g_otaStatus, "flashing") == 0 || - std::strcmp(g_otaStatus, "rebooting") == 0) { + if (otaInFlight()) { sendResponse(conn, 409, "application/json", "{\"error\":\"ota already in progress\"}"); return; diff --git a/src/core/HttpServerModule.h b/src/core/HttpServerModule.h index a11c7b97..9adb1d2c 100644 --- a/src/core/HttpServerModule.h +++ b/src/core/HttpServerModule.h @@ -26,8 +26,11 @@ class Scheduler; /// role-suffix-stripped `displayName`, `acceptsChildRoles`, and per-type `defaults` captured /// from a fresh probe instance). Mutations: `POST /api/control` `{module,control,value}`, /// `POST /api/modules` create, `POST /api/modules/{name}/move` reorder, `.../replace` swap, -/// `POST /api/reboot`, `DELETE /api/modules/{name}`. All JSON responses stream through a -/// `JsonSink` — no fixed-buffer ceiling, so a tree of any size serialises correctly. +/// `POST /api/reboot`, `DELETE /api/modules/{name}`. File Manager: `GET /api/dir?path=` lists a +/// directory, `POST /api/dir?path=` creates a folder, `DELETE /api/dir?path=` removes a file or +/// empty folder, `GET|POST /api/file?path=` reads / writes a file body (the path rides the query, +/// so a filesystem op carries its target in the request, not a stored control). All JSON responses +/// stream through a `JsonSink` — no fixed-buffer ceiling, so a tree of any size serialises correctly. /// /// **WebSocket:** `GET /ws` with `Upgrade: websocket` does the RFC 6455 handshake (SHA-1 + /// base64), up to 4 concurrent clients. Server pushes full state JSON as text frames from @@ -107,10 +110,14 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster { uint32_t clientGeneration() const override { return wsClientGeneration_; } /// Keep running even when "disabled" via the UI — otherwise the user has no way - /// to re-enable themselves through the same UI. The `enabled` checkbox on this - /// card has no effect; that's intentional. + /// to re-enable themselves through the same UI. bool respectsEnabled() const override { return false; } + /// Non-UI: this IS the server that renders /api/state — it doesn't list itself as a card. + /// The "not a UI module" opt-out (shared with FilesystemModule), read by the state serializer's + /// module loop to skip this module. + bool appearsInUi() const override { return false; } + void onBuildControls() override; void setup() override; void teardown() override; @@ -147,6 +154,17 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster { /// to the three above. The wire shape the Improv APPLY_OP frame carries. OpResult applyOp(const char* opJson); + /// Decode a `path=<rel>` query value into `out` (%XX + '+' decoding), rooted at the mount. + /// Returns false on a missing/empty path, a `..` traversal, or an overlong (buffer-filling) + /// value. The single filesystem-path guard shared by every fs HTTP entry (read/write/dir/ + /// mkdir/delete). Public + static so it's unit-testable without a socket fixture. + static bool parseFilePath(const char* query, char* out, size_t cap); + + /// Apply a WLED `{on?, bri?}` state body onto the Drivers `on` / `brightness` controls through + /// the shared apply-core (`on` and `bri` independent — off preserves the level). The transport- + /// free entry the HTTP `POST /json/state`, the inbound-`/ws` path, and the unit tests all drive. + void applyWledState(const char* body); + private: platform::TcpServer server_; Scheduler* scheduler_ = nullptr; @@ -207,6 +225,22 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster { void sendPreflightResponse(platform::TcpConnection& conn); void serveFile(platform::TcpConnection& conn, const char* filename, const char* contentType); + // File Manager: read/write an arbitrary filesystem path (the /api/file endpoints). `query` is + // the request's query string, read for `path=<rel>`; the path is vetted (no traversal, rooted + // at the mount) and size-capped. A file body isn't a control value, so these are their own + // endpoints rather than /api/control. + void serveFileContents(platform::TcpConnection& conn, const char* query); + // Streamed atomic upload: `initialBody`/`initialLen` are the body bytes already in the request + // buffer; `contentLen` is the declared total. Pulls any remainder off the socket → fsWriteStream, + // so an upload of any size streams to the file (rejected if it exceeds kUploadMax or free space). + void handleWriteFile(platform::TcpConnection& conn, const char* query, + const char* initialBody, size_t initialLen, size_t contentLen); + // File Manager: one directory's children as JSON (the /api/dir endpoint) — the source the lazy + // tree loads a node's children from. Single-level; `hidden=1` in the query includes dotfiles. + void serveDirListing(platform::TcpConnection& conn, const char* query); + void handleMakeDir(platform::TcpConnection& conn, const char* query); // POST /api/dir?path= + void handleRemoveEntry(platform::TcpConnection& conn, const char* query); // DELETE /api/dir?path= + // ----------------------------------------------------------------------- // JSON state // ----------------------------------------------------------------------- @@ -224,9 +258,9 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster { // ----------------------------------------------------------------------- void handleSetControl(platform::TcpConnection& conn, const char* body); - // Find a module anywhere in the scheduler's tree by its name. DFS, first match. + // Find a module anywhere in the scheduler's tree by its name — null-guards scheduler_ + // then delegates to Scheduler::firstByName (the one canonical tree-walk). MoonModule* findModuleByName(const char* name); - static MoonModule* findInTree(MoonModule* mod, const char* name); // ----------------------------------------------------------------------- // System metrics @@ -239,11 +273,9 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster { void serveWledState(platform::TcpConnection& conn); void serveWledStateInfo(platform::TcpConnection& conn); void handleWledState(platform::TcpConnection& conn, const char* body); - void applyWledState(const char* body); ///< `{on,bri}` → Drivers brightness (HTTP + WS) void pollWledStateFromWebSockets(); ///< read app's slider/toggle sent over /ws void writeWledInfoBody(JsonSink& sink, const char* name, const uint8_t mac[6]); void writeWledStateBody(JsonSink& sink); - uint8_t driversBrightness(); void writeModuleMetricsJson(JsonSink& sink, MoonModule* mod, bool& first); // ----------------------------------------------------------------------- @@ -260,6 +292,8 @@ class HttpServerModule : public MoonModule, public BinaryBroadcaster { /// to platform::http_fetch_to_ota which spawns a task and returns. Caller /// gets 202 immediately; progress streams via FirmwareUpdateModule controls. void handleFirmwareUrl(platform::TcpConnection& conn, const char* body); + void handleFirmwareUpload(platform::TcpConnection& conn, const char* initialBody, + size_t initialLen, size_t contentLen); // POST /api/firmware/upload // ----------------------------------------------------------------------- // WebSocket diff --git a/src/core/ImprovFrame.h b/src/core/ImprovFrame.h index 6bcd59fc..5e4ab47c 100644 --- a/src/core/ImprovFrame.h +++ b/src/core/ImprovFrame.h @@ -14,7 +14,7 @@ // - Keep the ESP32 task (platform_esp32.cpp) thin: feed bytes from the // UART driver into ImprovFrameParser, react to complete frames. // - Re-use the builder for both the ESP32 send path and the Python CLI -// spec (which reimplements the same framing in scripts/build/ +// spec (which reimplements the same framing in moondeck/build/ // improv_provision.py — same wire format, two languages). #include <cstdint> diff --git a/src/core/IrModule.h b/src/core/IrModule.h index f3183b5f..12791b4a 100644 --- a/src/core/IrModule.h +++ b/src/core/IrModule.h @@ -102,25 +102,31 @@ class IrModule : public MoonModule { void injectCodeForTest(uint32_t code) { processCode(code); } private: - // One action: a UI button + a learnable remote code, nudging (module, control) by delta, - // clamped to the control's own bounds. Adding an action is one row here. + // How an action changes its target control: a relative nudge (clamped to the control's bounds) + // or a boolean toggle (read the current value, write its inverse). A toggle can't be expressed + // as a delta — +1 on a 0/1 control clamps 0→1 but leaves 1→1 — so it's its own kind. + enum class ActionKind : uint8_t { Delta, Toggle }; + // One action: a UI button + a learnable remote code, changing (module, control). Adding an + // action is one row here. `delta` is used by Delta actions; Toggle ignores it. struct Action { const char* button; // the UI button name const char* codeCtrl; // the readonly control showing this action's learned code const char* module; const char* control; - int delta; + int delta; // Delta actions: the signed nudge; Toggle actions: unused (0) + ActionKind kind; }; static constexpr Action kActions[] = { - {"brightness up", "code brightness up", "Drivers", "brightness", +16}, - {"brightness down", "code brightness down", "Drivers", "brightness", -16}, - {"palette next", "code palette next", "Drivers", "palette", +1}, - {"palette prev", "code palette prev", "Drivers", "palette", -1}, + {"on/off", "code on/off", "Drivers", "on", 0, ActionKind::Toggle}, + {"brightness up", "code brightness up", "Drivers", "brightness", +16, ActionKind::Delta}, + {"brightness down", "code brightness down", "Drivers", "brightness", -16, ActionKind::Delta}, + {"palette next", "code palette next", "Drivers", "palette", +1, ActionKind::Delta}, + {"palette prev", "code palette prev", "Drivers", "palette", -1, ActionKind::Delta}, }; static constexpr uint8_t kActionCount = sizeof(kActions) / sizeof(kActions[0]); // learn select: index 0 = off, 1..N = arm learning for kActions[index-1]. static constexpr const char* kLearnOptions[] = { - "off", "brightness up", "brightness down", "palette next", "palette prev", + "off", "on/off", "brightness up", "brightness down", "palette next", "palette prev", }; // Handle a decoded code: in learn mode bind it to the armed action (and persist); otherwise @@ -171,6 +177,16 @@ class IrModule : public MoonModule { for (uint8_t i = 0; i < ctrls.count(); i++) { auto& c = ctrls[i]; if (std::strcmp(c.name, a.control) != 0) continue; + if (a.kind == ActionKind::Toggle) { + // Read the current bool (stored as a 1-byte value) and write its inverse. + const bool next = controlIntValue(c) == 0; + sched->setControl(a.module, a.control, next ? "{\"value\":true}" : "{\"value\":false}"); + std::snprintf(statusBuf_, sizeof(statusBuf_), "%s.%s → %s", + a.module, a.control, next ? "on" : "off"); + setStatus(statusBuf_); + markDirty(); + return; + } int next = controlIntValue(c) + a.delta; if (next < c.min) next = c.min; if (next > c.max) next = c.max; @@ -190,8 +206,9 @@ class IrModule : public MoonModule { else setStatus("ready"); } - // A 1-byte numeric control (Uint8 / Select) read as int via its descriptor pointer — covers - // every kActions target (brightness Uint8, palette Select both store a uint8_t). + // A 1-byte control (Uint8 / Select / Bool) read as int via its descriptor pointer — covers every + // kActions target: the brightness Uint8, the palette Select, and the on/off Bool (a bool is a + // 1-byte object, so reading it through the same uint8_t* yields 0/1 for the Toggle action). static int controlIntValue(const ControlDescriptor& c) { return c.ptr ? *static_cast<const uint8_t*>(c.ptr) : 0; } diff --git a/src/core/MoonModule.h b/src/core/MoonModule.h index a38fcb9c..af8a5b18 100644 --- a/src/core/MoonModule.h +++ b/src/core/MoonModule.h @@ -222,6 +222,13 @@ class MoonModule { /// Network, Filesystem) so the user can re-enable other modules through them. virtual bool respectsEnabled() const { return true; } + /// Whether this module appears in the UI (/api/state → nav card). Default true. A pure engine + /// with no user-facing controls returns false so it isn't shown as an empty card — e.g. + /// FilesystemModule (the persistence engine; its one status readout lives on FileManagerModule) + /// and HttpServerModule (the web server itself). The state serializer skips any module whose + /// appearsInUi() is false. + virtual bool appearsInUi() const { return true; } + /// Dirty flag — set by HttpServerModule when a control changes. FilesystemModule (or any /// consumer interested in "this module's state has been touched") observes it in loop1s() /// and clears it after persisting. @@ -249,6 +256,30 @@ class MoonModule { ControlList& controls() { return controls_; } const ControlList& controls() const { return controls_; } + /// Read one of this module's controls by name generically (no per-consumer control scan). Returns + /// `dflt` when the control is absent or the wrong type. The domain-neutral way for another module + /// (HttpServerModule's WLED shim, MqttModule) to read a target's control without a light include + /// or a hand-rolled scan — one implementation, so the absent-control default can't disagree + /// between callers. + bool readBool(const char* name, bool dflt) const { + for (uint8_t i = 0; i < controls_.count(); i++) { + const ControlDescriptor& c = controls_[i]; + if (c.ptr && c.type == ControlType::Bool && std::strcmp(c.name, name) == 0) + return *static_cast<const bool*>(c.ptr); + } + return dflt; + } + uint8_t readUint8(const char* name, uint8_t dflt) const { + for (uint8_t i = 0; i < controls_.count(); i++) { + const ControlDescriptor& c = controls_[i]; + // Uint8 covers both a plain uint8 (brightness) and a Select stored as uint8 (palette). + if (c.ptr && std::strcmp(c.name, name) == 0 && + (c.type == ControlType::Uint8 || c.type == ControlType::Select)) + return *static_cast<const uint8_t*>(c.ptr); + } + return dflt; + } + /// Role for type identification (no RTTI needed). virtual ModuleRole role() const { return ModuleRole::Generic; } diff --git a/src/core/MqttModule.cpp b/src/core/MqttModule.cpp new file mode 100644 index 00000000..3891d2d9 --- /dev/null +++ b/src/core/MqttModule.cpp @@ -0,0 +1,393 @@ +#include "core/MqttModule.h" + +#include "core/Scheduler.h" // setControl — the shared apply-core +#include "light/Palette.h" // Palettes::nearestForHue — a pure hue/sat→index CONVERSION with no + // light state or objects, the one narrow reach this core module makes + // into the light domain. PO-accepted: routing a HomeKit colour to a + // palette needs the palette set, which is inherently light-domain, and + // a format conversion is the least-coupling way to bridge it (the + // module still drives the palette via Scheduler::setControl, not a + // light object). Deliberate divergence from the plan's "no light + // include" line, made with the trade-off understood, not by precedent. + +#include <cstdio> +#include <cstdlib> +#include <cstring> + +namespace mm { + +namespace { +// A scratch send buffer big enough for any control packet we build (CONNECT with auth is the +// largest at well under 200 bytes). Sends go through sendPacket (non-blocking writeSome). +constexpr size_t kSendBufLen = 256; +} // namespace + +// The topic prefix, derived live from a STABLE hardware id: projectMM/<last6-of-MAC> (lowercase +// hex), e.g. projectMM/563cfe. Not stored (no buffer). The MAC is fixed for the chip's life, so a +// device RENAME never changes the topics — external integrations (Homebridge, Home Assistant) stay +// pinned. This is the WLED / Tasmota / HA-discovery convention: the MQTT identity is the hardware +// id, and the friendly *display* name is a separate concern (published on the `name` topic, read +// from deviceName()). The last 6 hex is the same short-id WLED uses (`wled/<last6>`). +void MqttModule::topicPrefix(char* out, size_t cap) const { + uint8_t mac[6] = {}; + platform::getMacAddress(mac); + std::snprintf(out, cap, "%s/%02x%02x%02x", prefixRoot_, mac[3], mac[4], mac[5]); +} + +// A full topic: <prefix>/<suffix>, e.g. projectMM/563cfe/on/set. +void MqttModule::buildTopic(char* out, size_t cap, const char* suffix) const { + char prefix[96]; + topicPrefix(prefix, sizeof(prefix)); + std::snprintf(out, cap, "%s/%s", prefix, suffix); +} + +void MqttModule::setup() { + setStatusLine(enabled() ? "idle" : "disabled"); + MoonModule::setup(); +} + +void MqttModule::onBuildControls() { + controls_.addText("broker", broker_, sizeof(broker_)); + controls_.addUint16("port", port_, 1, 65535); + controls_.addText("username", username_, sizeof(username_)); + controls_.addPassword("password", password_, sizeof(password_)); + controls_.addReadOnly("mqtt_status", statusStr_, sizeof(statusStr_)); + MoonModule::onBuildControls(); +} + +// A broker/port/credentials change re-homes the connection: drop the socket so loop1s reconnects +// with the new settings on the next tick. Scoped to THIS module's controls via onUpdate (not the +// whole-tree onBuildState sweep), so an unrelated change — a grid resize, a layout edit — never +// drops the MQTT connection. Live, no reboot. +void MqttModule::onUpdate(const char* controlName) { + if (std::strcmp(controlName, "broker") == 0 || std::strcmp(controlName, "port") == 0 || + std::strcmp(controlName, "username") == 0 || std::strcmp(controlName, "password") == 0) { + resetConnection(enabled() ? "reconnecting" : "disabled"); + } +} + +// Enable/disable transition. On disable we send a clean DISCONNECT + close (rather than leaving a +// dangling socket for the broker to time out) — loop1s stops being called once disabled, so this +// transition hook is the only place a disable can act. +void MqttModule::onEnabled(bool enabled) { + if (!enabled && conn_.valid()) { + uint8_t buf[4]; + const size_t n = buildMqttDisconnect(buf, sizeof(buf)); + if (n) sendPacket(buf, n); // best-effort courtesy DISCONNECT + } + resetConnection(enabled ? "idle" : "disabled"); +} + +// Send a whole MQTT packet without EVER blocking the render loop. Uses writeSome (non-blocking): a +// control packet is ≤256 B, far under the socket send buffer, so a healthy socket accepts it all in +// one call. A partial or zero write means the buffer is backing up (a zero-window / stalled broker) — +// return false so the caller resets the connection rather than spin-retrying forever, which is what +// the blocking write() would do inside loop1s (the hot-path violation this avoids). +bool MqttModule::sendPacket(const uint8_t* data, size_t len) { + if (len == 0) return true; + const int sent = conn_.writeSome(data, len); + return sent == static_cast<int>(len); // all-or-fail; a partial/0/-1 is a connection problem +} + +// Close the socket and return to Idle with a status line — the single reset path so every caller +// (reconfig, disable, timeout, protocol error, peer close) leaves the same clean state. +void MqttModule::resetConnection(const char* status) { + conn_.close(); + state_ = Conn::Idle; + havePublished_ = false; + setStatusLine(status); +} + +void MqttModule::loop1s() { + if constexpr (!platform::hasNetwork) { MoonModule::loop1s(); return; } + + if (!enabled() || broker_[0] == '\0') { + if (conn_.valid()) resetConnection(enabled() ? "idle" : "disabled"); + MoonModule::loop1s(); + return; + } + if (!platform::networkReady()) { MoonModule::loop1s(); return; } + + const uint32_t now = platform::millis(); + switch (state_) { + case Conn::Idle: { + // Backoff between connect attempts so a down broker isn't hammered every tick. A prior + // FAILURE (unreachable broker, bad hostname → a synchronous getaddrinfo each try) backs + // off harder to keep the recurring DNS stall rare. + const uint32_t backoff = lastConnectFailed_ ? kFailedBackoffMs : kReconnectBackoffMs; + if (now - lastConnectTry_ >= backoff || lastConnectTry_ == 0) { + lastConnectTry_ = now; + startConnect(); + } + break; + } + case Conn::ConnectingTcp: { + // Poll the non-blocking TCP connect — never blocks the tick. + const auto r = conn_.connectPoll(); + if (r == platform::TcpConnection::ConnectResult::Connected) sendConnectPacket(); + else if (r == platform::TcpConnection::ConnectResult::Failed) resetConnection("error: connect failed"); + else if (now - connectStartedMs_ >= kConnectTimeoutMs) resetConnection("error: connect timeout"); + break; + } + case Conn::Connecting: + // TCP up, CONNECT sent, waiting for CONNACK. A broker that accepts TCP but never CONNACKs + // (finding: the silent-broker wedge) is bounded by the same connect timeout. + serviceConnected(); + if (state_ == Conn::Connecting && now - connectStartedMs_ >= kConnectTimeoutMs) + resetConnection("error: no CONNACK"); + break; + case Conn::Connected: + serviceConnected(); + break; + } + MoonModule::loop1s(); +} + +// Begin a NON-BLOCKING TCP connect (getaddrinfo + connect kicked off, returns immediately). loop1s +// polls it in ConnectingTcp so the render loop never stalls on an unreachable broker. +void MqttModule::startConnect() { + setStatusLine("connecting"); + // Assume failure until a full connect succeeds (cleared in the CONNACK-accepted path). Every + // failure route resets to Idle without clearing this, so the next Idle uses the longer backoff. + lastConnectFailed_ = true; + if (!conn_.connectStart(broker_, port_)) { // immediate failure (DNS / socket) + resetConnection("error: connect failed"); + return; + } + connectStartedMs_ = platform::millis(); + state_ = Conn::ConnectingTcp; +} + +// TCP is up — send CONNECT and wait for CONNACK. +void MqttModule::sendConnectPacket() { + uint8_t buf[kSendBufLen]; + const char* user = username_[0] ? username_ : nullptr; + const char* pass = password_[0] ? password_ : nullptr; + // A stable, slash-free clientId (MQTT-3.1.3-5 allows only [0-9a-zA-Z], and a broker may reject a + // '/'): "projectMM-<last6-of-MAC>", alphanumeric + one hyphen. NOT topicPrefix() — that carries a + // slash. Same MAC identity as the topics, just without the path separator. + uint8_t mac[6] = {}; + platform::getMacAddress(mac); + char clientId[32]; + std::snprintf(clientId, sizeof(clientId), "projectMM-%02x%02x%02x", mac[3], mac[4], mac[5]); + const size_t n = buildMqttConnect(clientId, user, pass, kKeepaliveSec, buf, sizeof(buf)); + if (n == 0 || !sendPacket(buf, n)) { + resetConnection("error: connect send failed"); + return; + } + parser_ = MqttInboundParser{}; // fresh parser per connection + state_ = Conn::Connecting; // waiting for CONNACK + connectStartedMs_ = platform::millis(); + lastActivity_ = connectStartedMs_; + lastPingSent_ = connectStartedMs_; +} + +void MqttModule::serviceConnected() { + // Drain whatever the socket has (non-blocking). A bounded read per tick keeps this cheap. + uint8_t rx[256]; + for (int pass = 0; pass < 8; pass++) { + const int n = conn_.read(rx, sizeof(rx)); + if (n > 0) { + lastActivity_ = platform::millis(); + for (int i = 0; i < n; i++) handleInboundByte(rx[i]); + if (state_ == Conn::Idle) return; // handleInboundByte reset us (refused / malformed) + } else if (n == 0) { // peer closed + resetConnection("disconnected"); + return; + } else { + break; // -1 = nothing pending right now + } + } + + if (state_ == Conn::Connected) { + publishState(false); // emit any changed get topics + // Re-publish the friendly name if the device was renamed while connected (topics are stable, + // but the display label should follow). Cheap change-detect via a rolling signature — no + // stored name buffer. publishName is a no-op if unchanged. + maybeRepublishName(); + + // Keepalive: PINGREQ at keepalive/2. If the broker goes silent past ~keepalive*1.5, drop. + const uint32_t now = platform::millis(); + if (now - lastPingSent_ >= (kKeepaliveSec * 1000u) / 2) { + uint8_t ping[2]; + const size_t pn = buildMqttPingreq(ping, sizeof(ping)); + if (pn == 0 || !sendPacket(ping, pn)) { resetConnection("error: ping failed"); return; } + lastPingSent_ = now; + } + if (now - lastActivity_ >= kKeepaliveSec * 1500u) // 1.5× keepalive with no traffic + resetConnection("timeout"); + } +} + +void MqttModule::handleInboundByte(uint8_t byte) { + const MqttFeedResult r = parser_.feed(byte); + // A malformed / oversize packet desyncs the byte stream for the connection's life (MQTT 3.1.1 + // §4.8: a protocol violation MUST close the connection). Drop and reconnect rather than reinterpret + // mid-body garbage as fixed headers. + if (r == MqttFeedResult::Malformed) { resetConnection("error: bad packet"); return; } + if (r != MqttFeedResult::PacketReady) return; + + const uint8_t type = parser_.lastType(); + if (type == static_cast<uint8_t>(MqttPacketType::Connack)) { + // CONNACK body is [session-present][return-code] (§3.2). A short body is a protocol violation + // — treat as a failed connect, don't fall through and subscribe on a malformed accept. + if (parser_.bodyLen() < 2) { resetConnection("error: bad CONNACK"); return; } + // Non-zero return code = the broker refused (bad auth, unavailable, …). + if (parser_.body()[1] != 0) { + resetConnection("error: broker refused"); + return; + } + // Subscribe to <prefix>/+/set with three explicit filters (one SUBSCRIBE each — simple). + static const char* kSets[] = {"on/set", "brightness/set", "hsv/set"}; + char topic[128]; + for (const char* suffix : kSets) { + buildTopic(topic, sizeof(topic), suffix); + uint8_t buf[kSendBufLen]; + const size_t n = buildMqttSubscribe(nextPacketId_++, topic, buf, sizeof(buf)); + if (n == 0 || !sendPacket(buf, n)) { resetConnection("error: subscribe failed"); return; } + } + state_ = Conn::Connected; + lastConnectFailed_ = false; // full success → next reconnect uses the short backoff + setStatusLine("connected"); + havePublished_ = false; + publishName(); // retained friendly name so a hub shows the display name + publishState(true); // publish initial state so mqttthing shows it + } else if (type == static_cast<uint8_t>(MqttPacketType::Publish)) { + const char* topic = nullptr; const uint8_t* payload = nullptr; size_t plLen = 0; + if (parser_.publish(&topic, &payload, &plLen)) routePublish(topic, payload, plLen); + } + // PINGRESP / SUBACK: nothing to do beyond the activity timestamp already stamped. +} + +void MqttModule::routePublish(const char* topic, const uint8_t* payload, size_t payloadLen) { + // Match the topic suffix after our (derived) prefix. A short fixed payload is copied + // NUL-terminated so the parsers below (strcmp / atoi) are safe on the non-terminated socket slice. + char prefix[96]; + topicPrefix(prefix, sizeof(prefix)); + const size_t prefixLen = std::strlen(prefix); + if (std::strncmp(topic, prefix, prefixLen) != 0 || topic[prefixLen] != '/') return; + const char* suffix = topic + prefixLen + 1; + + char value[32]; + const size_t vlen = payloadLen < sizeof(value) - 1 ? payloadLen : sizeof(value) - 1; + std::memcpy(value, payload, vlen); + value[vlen] = '\0'; + + if (std::strcmp(suffix, "on/set") == 0) { + const bool on = (std::strcmp(value, "true") == 0 || std::strcmp(value, "1") == 0); + setControlValue("on", on ? "{\"value\":true}" : "{\"value\":false}"); + } else if (std::strcmp(suffix, "brightness/set") == 0) { + // mqttthing sends 0..100; rescale to 0..255. + int pct = std::atoi(value); + if (pct < 0) pct = 0; + if (pct > 100) pct = 100; + const int bri = (pct * 255) / 100; + char json[24]; + std::snprintf(json, sizeof(json), "{\"value\":%d}", bri); + setControlValue("brightness", json); + } else if (std::strcmp(suffix, "hsv/set") == 0) { + // "h,s,v" — hue 0..359, sat 0..100, val 0..100 (mqttthing HSV). Hue+sat pick the nearest + // palette; value maps to brightness so the color wheel's brightness ring still dims. + int h = 0, s = 0, v = -1; + std::sscanf(value, "%d,%d,%d", &h, &s, &v); + const uint8_t idx = Palettes::nearestForHue(static_cast<uint16_t>(h < 0 ? 0 : h), + static_cast<uint8_t>(s < 0 ? 0 : (s > 100 ? 255 : s * 255 / 100))); + char json[24]; + std::snprintf(json, sizeof(json), "{\"value\":%u}", static_cast<unsigned>(idx)); + setControlValue("palette", json); + if (v >= 0) { + int bri = (v > 100 ? 100 : v) * 255 / 100; + std::snprintf(json, sizeof(json), "{\"value\":%d}", bri); + setControlValue("brightness", json); + } + } +} + +// Publish the friendly display name (deviceName) on the retained `<prefix>/name` topic. The topic +// identity is the stable MAC (rename-proof); this is the separate human-facing label a hub reads for +// its accessory title (the WLED serverDescription / HA discovery `name` role). Retained so a hub +// connecting later still gets it. Published on connect + on a deviceName change. +// A cheap rolling signature of the current deviceName (djb2), so a rename can be detected without +// storing the name string. +static uint32_t nameSignature(const char* s) { + uint32_t h = 5381; + for (; s && *s; s++) h = h * 33u + static_cast<uint8_t>(*s); + return h; +} + +void MqttModule::publishName() { + if (state_ != Conn::Connected) return; + // Always the device's own name — SystemModule guarantees it non-empty and per-device unique + // (it falls back to MM-<last4MAC>, never a shared literal), so N devices never collide as one + // name in the hub. No systemModule_ (a wiring bug) → skip; don't invent a non-unique fallback. + const char* dn = systemModule_ ? systemModule_->deviceName() : nullptr; + if (!dn || !dn[0]) return; + char topic[128]; + buildTopic(topic, sizeof(topic), "name"); + uint8_t buf[kSendBufLen]; + const size_t n = buildMqttPublish(topic, reinterpret_cast<const uint8_t*>(dn), std::strlen(dn), + buf, sizeof(buf), /*retain=*/true); + // Stamp the signature only on a SUCCESSFUL send — else a failed name publish is retried next + // tick (maybeRepublishName sees the mismatch) rather than being lost until the next reconnect. + if (n && sendPacket(buf, n)) nameSig_ = nameSignature(dn); +} + +// Re-publish the name only if it changed since the last publish (a rename while connected). +void MqttModule::maybeRepublishName() { + const char* dn = systemModule_ ? systemModule_->deviceName() : nullptr; + if (dn && dn[0] && nameSignature(dn) != nameSig_) publishName(); +} + +void MqttModule::publishState(bool force) { + if (state_ != Conn::Connected) return; + Scheduler* s = Scheduler::instance(); + const bool on = driversOn(s); + const uint8_t bri = driversBrightness(s); + const uint8_t pal = driversPalette(s); + if (!force && havePublished_ && on == lastOn_ && bri == lastBri_ && pal == lastPalette_) return; + + char topic[128]; + uint8_t buf[kSendBufLen]; + + // A publish here is one of the three get-topics. On ANY send failure, reset the connection and + // DON'T commit last*/havePublished_ — so after the reconnect the change is republished, not lost + // (a committed-but-unsent state would leave the hub showing stale values forever). Same "stamp + // only on success" rule as publishName / the ping path. + auto publish = [&](const char* suffix, const char* payload) -> bool { + buildTopic(topic, sizeof(topic), suffix); + const size_t n = buildMqttPublish(topic, reinterpret_cast<const uint8_t*>(payload), + std::strlen(payload), buf, sizeof(buf)); + return n != 0 && sendPacket(buf, n); + }; + + char briStr[8]; + std::snprintf(briStr, sizeof(briStr), "%d", (bri * 100) / 255); + // hsv/get — the chosen palette's representative hue, full sat, value = brightness%. + char hsvStr[16]; + std::snprintf(hsvStr, sizeof(hsvStr), "%u,100,%d", + static_cast<unsigned>(Palettes::representativeHue(pal)), (bri * 100) / 255); + + if (!publish("on/get", on ? "true" : "false") || + !publish("brightness/get", briStr) || + !publish("hsv/get", hsvStr)) { + resetConnection("error: state publish failed"); + return; + } + + lastOn_ = on; lastBri_ = bri; lastPalette_ = pal; + havePublished_ = true; +} + +void MqttModule::setControlValue(const char* control, const char* valueJson) { + if (Scheduler* s = Scheduler::instance()) s->setControl("Drivers", control, valueJson); +} + +void MqttModule::feedForTest(const uint8_t* bytes, size_t len) { + for (size_t i = 0; i < len; i++) handleInboundByte(bytes[i]); +} + +void MqttModule::setStatusLine(const char* msg) { + std::snprintf(statusStr_, sizeof(statusStr_), "%s", msg); +} + +} // namespace mm diff --git a/src/core/MqttModule.h b/src/core/MqttModule.h new file mode 100644 index 00000000..f77ff1ac --- /dev/null +++ b/src/core/MqttModule.h @@ -0,0 +1,120 @@ +#pragma once + +#include "core/MoonModule.h" +#include "core/MqttPacket.h" +#include "core/SystemModule.h" +#include "platform/platform.h" + +#include <cstdint> + +namespace mm { + +/// MQTT client service: bridges the device's light controls (on / brightness / palette) to an MQTT +/// broker so home-automation hubs can drive it. The headline consumer is **Homebridge** (via the +/// `homebridge-mqttthing` "lightbulb" accessory), which publishes to `set` topics and reads `get` +/// topics — a bare on/off + brightness + colour surface. It is a network sub-service, a code-wired +/// child of NetworkModule alongside Improv/Devices, and it drives the shared apply-core exactly as +/// IR and the WLED bridge do: every command routes through `Scheduler::setControl("Drivers", …)`, +/// so MQTT adds a transport, not new control plumbing. +/// +/// **The client is our own.** MQTT 3.1.1 is a small, standard protocol (the same framing mosquitto +/// and mqttthing speak), so the wire format lives in a dependency-free, golden-vector-tested header +/// (MqttPacket.h) and this module owns only the socket lifecycle over `platform::TcpConnection` +/// (`connect` + the non-blocking `read`/`writeSome`). No library — the framing is pinned by tests +/// the way the Improv frames are. +/// +/// **Topics** (prefix `projectMM/<last6-of-MAC>` — a STABLE id, so a rename never repoints topics; +/// the friendly deviceName rides the separate retained `<prefix>/name` topic): +/// `<prefix>/on/set` ← "true"/"false" → Drivers.on +/// `<prefix>/on/get` → publish current on +/// `<prefix>/brightness/set` ← 0..100 (mqttthing) → *255/100 → Drivers.brightness +/// `<prefix>/brightness/get` → publish brightness*100/255 +/// `<prefix>/hsv/set` ← "h,s,v" → hue+sat → nearest palette → Drivers.palette +/// `<prefix>/hsv/get` → publish the chosen palette's representative "h,s,v" +/// +/// **Lifecycle** (all on loop1s(), off the render hot path — MQTT is slow control): connect lazily +/// once `networkReady() && enabled`, CONNECT → CONNACK → SUBSCRIBE to the `set` topics, PINGREQ every +/// keepalive/2, drain inbound bytes through MqttInboundParser and route PUBLISHes to Drivers, and +/// publish the `get` topics whenever the local value changes (and on connect, so mqttthing never +/// reads "No Response"). A dropped socket reconnects with a backoff. +/// +/// **Prior art:** the OASIS MQTT 3.1.1 standard + homebridge-mqttthing's topic conventions (see +/// docs/moonmodules/core/ui/ui.md#mqtt for the Homebridge accessory config). The MoonLight sibling +/// bridges the same on/off+brightness surface through a full framework MQTT client + HA discovery; +/// projectMM writes its own lean client over the platform socket primitive instead. +class MqttModule : public MoonModule { +public: + void setSystemModule(SystemModule* s) { systemModule_ = s; } + + void setup() override; + void onBuildControls() override; + void onUpdate(const char* controlName) override; // a broker/port/cred change re-homes the socket + void onEnabled(bool enabled) override; // enable/disable → connect / clean DISCONNECT + void loop1s() override; + + /// Feed inbound bytes as if they arrived from the broker socket — the entry the host unit tests + /// drive (there's no live broker in ctest). Mirrors IrModule::injectCodeForTest. + void feedForTest(const uint8_t* bytes, size_t len); + +private: + // Connection state machine — advanced by loop1s(). ConnectingTcp = a non-blocking TCP connect is + // in flight (polled, never blocks the tick); Connecting = TCP up, CONNECT sent, awaiting CONNACK. + enum class Conn : uint8_t { Idle, ConnectingTcp, Connecting, Connected }; + + void startConnect(); // begin a non-blocking TCP connect (ConnectingTcp) + void sendConnectPacket(); // TCP up → send CONNECT, go to Connecting + void serviceConnected(); // drain inbound, keepalive, publish-on-change + bool sendPacket(const uint8_t* data, size_t len); // non-blocking atomic send (true = fully sent) + void resetConnection(const char* status); // close socket + back to Idle with a status line + void handleInboundByte(uint8_t byte); // feed the parser, route a completed PUBLISH + void routePublish(const char* topic, const uint8_t* payload, size_t payloadLen); + void publishState(bool force); // publish get topics when local values changed + void publishName(); // publish the friendly deviceName on the retained name topic + void maybeRepublishName(); // re-publish the name if it changed (rename while connected) + void setControlValue(const char* control, const char* valueJson); // → Scheduler::setControl + void setStatusLine(const char* msg); + + SystemModule* systemModule_ = nullptr; + + // The topic prefix is DERIVED from a STABLE hardware id: projectMM/<last6-of-MAC>. Not stored (no + // buffer). The MAC is fixed for the chip's life, so a device rename never changes the topics — + // external integrations stay pinned (the WLED/Tasmota/HA convention). The friendly display name + // is a separate concern, published on the retained `name` topic (publishName). buildTopic writes + // <prefix>/<suffix>. + const char* prefixRoot_ = "projectMM"; + void topicPrefix(char* out, size_t cap) const; + void buildTopic(char* out, size_t cap, const char* suffix) const; + + // Controls (persisted). + char broker_[64] = ""; // hostname or IP of the broker + uint16_t port_ = 1883; + char username_[48] = ""; + char password_[48] = ""; + char statusStr_[64] = "disabled"; + + platform::TcpConnection conn_; + MqttInboundParser parser_; + Conn state_ = Conn::Idle; + uint32_t lastPingSent_ = 0; + uint32_t lastActivity_ = 0; // last inbound byte or successful send (for keepalive) + uint32_t lastConnectTry_ = 0; // backoff clock for reconnects + uint32_t connectStartedMs_ = 0; // when the current TCP-connect / CONNACK-wait began + uint32_t nameSig_ = 0; // signature of the last-published friendly name (rename detect) + uint16_t nextPacketId_ = 1; + + // Last-published state, so publishState only emits on change. + bool lastOn_ = false; + uint8_t lastBri_ = 0; + uint8_t lastPalette_ = 0xFF; // 0xFF = "never published" → force first publish + bool havePublished_ = false; + + bool lastConnectFailed_ = false; // widen the backoff after a failure (esp. a slow DNS lookup) + + static constexpr uint16_t kKeepaliveSec = 30; + static constexpr uint32_t kReconnectBackoffMs = 5000; // after a clean disconnect + static constexpr uint32_t kFailedBackoffMs = 30000; // after a failed attempt — a bad + // hostname re-runs a synchronous getaddrinfo each try (a brief tick stall), so back off harder + static constexpr uint32_t kConnectTimeoutMs = 8000; // overall TCP-connect + CONNACK-wait bound +}; + +} // namespace mm diff --git a/src/core/MqttPacket.h b/src/core/MqttPacket.h new file mode 100644 index 00000000..2a0dc1ec --- /dev/null +++ b/src/core/MqttPacket.h @@ -0,0 +1,306 @@ +#pragma once + +// MQTT 3.1.1 wire framing — pure C++, no ESP-IDF or stdlib-network deps. +// +// Wire format (OASIS MQTT Version 3.1.1, §2 Fixed Header, §3 Control Packets): +// [type<<4 | flags][remaining-length varint][variable header + payload] +// +// This header owns the framing + the handful of control packets a client light needs — CONNECT, +// PUBLISH (both directions), SUBSCRIBE, PINGREQ, DISCONNECT to send; CONNACK, SUBACK, PINGRESP, +// PUBLISH to receive. It is deliberately dependency-free so the packet math is host-unit-tested +// (test/unit/core/unit_MqttPacket.cpp) with golden byte vectors, exactly like ImprovFrame.h. The +// live socket lifecycle (connect/keepalive/reconnect) lives in MqttModule; this is only the bytes. +// +// Prior art: the protocol is the OASIS MQTT 3.1.1 standard (the same framing mosquitto, HiveMQ, and +// homebridge-mqttthing speak). Written fresh against our own buffers — no library. + +#include <cstdint> +#include <cstddef> +#include <cstring> + +namespace mm { + +// --8<-- [start:mqtt-constants] +// Control packet types — the high nibble of the fixed header's first byte (§2.2.1). The low nibble +// carries per-type flags; for the packets we send it is 0 except SUBSCRIBE (0x2) and PUBLISH (QoS in +// bits 1-2 — we send QoS0, so 0). +enum class MqttPacketType : uint8_t { + Connect = 0x1, + Connack = 0x2, + Publish = 0x3, + Suback = 0x9, + Subscribe = 0x8, + Pingreq = 0xC, + Pingresp = 0xD, + Disconnect = 0xE, +}; + +// The protocol-name + level a 3.1.1 CONNECT carries in its variable header (§3.1.2.1). +inline constexpr char kMqttProtocolName[4] = {'M','Q','T','T'}; +inline constexpr uint8_t kMqttProtocolLevel = 0x04; // 4 == MQTT 3.1.1 + +// CONNECT flag bits (§3.1.2.3). We use clean-session + optionally username/password. +inline constexpr uint8_t kMqttConnectCleanSession = 0x02; +inline constexpr uint8_t kMqttConnectPasswordFlag = 0x40; +inline constexpr uint8_t kMqttConnectUsernameFlag = 0x80; +// --8<-- [end:mqtt-constants] + +// --- Remaining-length varint (§2.2.3) --- +// 1-4 bytes, 7 data bits each, high bit = "more follows". The one genuinely fiddly bit of MQTT, so +// it's its own pair of pure functions, boundary-tested at 0 / 127 / 128 / 16383 / 16384. + +// Encode `value` (0 .. 268435455) into `out` (needs ≤4 bytes). Returns bytes written, or 0 if the +// value exceeds the 4-byte maximum. +inline size_t encodeRemainingLength(uint32_t value, uint8_t* out) { + if (value > 268435455u) return 0; // 0xFFFFFFF — the 4-byte ceiling + size_t n = 0; + do { + uint8_t byte = value & 0x7F; + value >>= 7; + if (value > 0) byte |= 0x80; // more bytes follow + out[n++] = byte; + } while (value > 0); + return n; +} + +// Decode a remaining-length varint from `in` (up to `inLen` bytes). On success writes the value to +// `*value` and the varint's byte count to `*consumed`, returns true. Returns false if the field is +// truncated (need more bytes) or malformed (5th continuation byte — a protocol violation). +inline bool decodeRemainingLength(const uint8_t* in, size_t inLen, + uint32_t* value, size_t* consumed) { + uint32_t result = 0; + uint32_t multiplier = 1; + size_t i = 0; + for (;;) { + if (i >= inLen) return false; // truncated — caller feeds more + if (i >= 4) return false; // >4 bytes → malformed varint + const uint8_t byte = in[i++]; + result += static_cast<uint32_t>(byte & 0x7F) * multiplier; + if ((byte & 0x80) == 0) break; // high bit clear → last byte + multiplier <<= 7; + } + *value = result; + *consumed = i; + return true; +} + +// --- Builders (§3) --- +// Each writes a complete packet into a caller-owned buffer and returns the byte count, or 0 on +// overflow / bad input. No allocation. A 16-bit-length-prefixed UTF-8 string field (§1.5.3) is the +// repeating shape, so it gets a small helper. + +// Append a 2-byte-big-endian-length-prefixed string. Returns the new write position, or 0 on +// overflow. `len == 0` writes just the 2-byte zero length (a valid empty string). +inline size_t mqttAppendString(uint8_t* out, size_t outLen, size_t pos, + const char* s, size_t len) { + if (pos + 2 + len > outLen) return 0; + out[pos++] = static_cast<uint8_t>((len >> 8) & 0xFF); + out[pos++] = static_cast<uint8_t>(len & 0xFF); + if (len > 0) { std::memcpy(out + pos, s, len); pos += len; } + return pos; +} + +// Assemble a fixed header (type/flags + remaining-length) followed by an already-built variable +// header+payload of `bodyLen` bytes sitting at `out + <fixed header size>`. Because the fixed +// header's size depends on the varint length, builders write the body into a scratch region first, +// then this prepends the header. To avoid a second buffer we compute the header, then memmove — but +// bodies here are small and known, so builders instead write the header first with a pre-computed +// length. This helper writes just the fixed header; callers pass the final bodyLen. +inline size_t mqttWriteFixedHeader(uint8_t* out, size_t outLen, + MqttPacketType type, uint8_t flags, uint32_t bodyLen) { + // Encode the varint into a local scratch first, so we validate 1 + rl <= outLen BEFORE writing + // anything into out — encodeRemainingLength is unbounded, and a 2–4-byte length into a 1-byte + // buffer would otherwise overrun. Remaining-length is at most 4 bytes (§2.2.3). + uint8_t rlBuf[4]; + const size_t rl = encodeRemainingLength(bodyLen, rlBuf); + if (rl == 0 || 1 + rl > outLen) return 0; + out[0] = static_cast<uint8_t>((static_cast<uint8_t>(type) << 4) | (flags & 0x0F)); + std::memcpy(out + 1, rlBuf, rl); + return 1 + rl; // total fixed-header size +} + +// CONNECT (§3.1). clientId is required; username/password optional (nullptr → omitted). keepalive is +// in seconds. Clean session always set (we don't resume state). Returns total bytes, 0 on overflow. +inline size_t buildMqttConnect(const char* clientId, + const char* username, const char* password, + uint16_t keepaliveSec, uint8_t* out, size_t outLen) { + if (!clientId) return 0; + // Treat an empty-string username as "no username" (a UI text control left blank is "", not null), + // and — MQTT-3.1.2-22 — the password flag MUST NOT be set without the username flag, so a password + // with no username is dropped (a compliant broker would reject the whole CONNECT otherwise). + if (username && !username[0]) username = nullptr; + if (!username) password = nullptr; + const size_t idLen = std::strlen(clientId); + const size_t userLen = username ? std::strlen(username) : 0; + const size_t passLen = password ? std::strlen(password) : 0; + + // Variable header: protocol name (6) + level (1) + connect flags (1) + keepalive (2) = 10. + // Payload: clientId, then username, then password — each length-prefixed. + uint8_t connectFlags = kMqttConnectCleanSession; + if (username) connectFlags |= kMqttConnectUsernameFlag; + if (password) connectFlags |= kMqttConnectPasswordFlag; + + const uint32_t bodyLen = static_cast<uint32_t>( + 2 + 4 + 1 + 1 + 2 + // proto name(len+"MQTT") + level + flags + keepalive + 2 + idLen + + (username ? 2 + userLen : 0) + + (password ? 2 + passLen : 0)); + + size_t pos = mqttWriteFixedHeader(out, outLen, MqttPacketType::Connect, 0, bodyLen); + if (pos == 0) return 0; + + // Variable header + pos = mqttAppendString(out, outLen, pos, kMqttProtocolName, 4); // "MQTT" + if (pos == 0) return 0; + if (pos + 4 > outLen) return 0; + out[pos++] = kMqttProtocolLevel; + out[pos++] = connectFlags; + out[pos++] = static_cast<uint8_t>((keepaliveSec >> 8) & 0xFF); + out[pos++] = static_cast<uint8_t>(keepaliveSec & 0xFF); + + // Payload + pos = mqttAppendString(out, outLen, pos, clientId, idLen); + if (pos == 0) return 0; + if (username) { pos = mqttAppendString(out, outLen, pos, username, userLen); if (pos == 0) return 0; } + if (password) { pos = mqttAppendString(out, outLen, pos, password, passLen); if (pos == 0) return 0; } + return pos; +} + +// PUBLISH (§3.3), QoS0 (no packet identifier), not duplicate. `retain` sets the RETAIN flag (bit 0 +// of the fixed-header flags nibble, §3.3.1.3) — a retained topic (the friendly `name`) is redelivered +// to a late-subscribing hub. Returns total bytes. +inline size_t buildMqttPublish(const char* topic, const uint8_t* payload, size_t payloadLen, + uint8_t* out, size_t outLen, bool retain = false) { + if (!topic) return 0; + const size_t topicLen = std::strlen(topic); + const uint32_t bodyLen = static_cast<uint32_t>(2 + topicLen + payloadLen); // topic (QoS0: no id) + size_t pos = mqttWriteFixedHeader(out, outLen, MqttPacketType::Publish, + retain ? 0x1 : 0x0, bodyLen); + if (pos == 0) return 0; + pos = mqttAppendString(out, outLen, pos, topic, topicLen); + if (pos == 0) return 0; + if (payloadLen > 0) { + if (pos + payloadLen > outLen) return 0; + std::memcpy(out + pos, payload, payloadLen); + pos += payloadLen; + } + return pos; +} + +// SUBSCRIBE (§3.8), one topic filter at QoS0. Flags nibble is 0x2 (reserved, mandated by spec). +// `packetId` must be non-zero. Returns total bytes. +inline size_t buildMqttSubscribe(uint16_t packetId, const char* topic, uint8_t* out, size_t outLen) { + if (!topic || packetId == 0) return 0; + const size_t topicLen = std::strlen(topic); + const uint32_t bodyLen = static_cast<uint32_t>(2 + 2 + topicLen + 1); // packetId + filter + QoS + size_t pos = mqttWriteFixedHeader(out, outLen, MqttPacketType::Subscribe, 0x2, bodyLen); + if (pos == 0) return 0; + if (pos + 2 > outLen) return 0; + out[pos++] = static_cast<uint8_t>((packetId >> 8) & 0xFF); + out[pos++] = static_cast<uint8_t>(packetId & 0xFF); + pos = mqttAppendString(out, outLen, pos, topic, topicLen); + if (pos == 0) return 0; + if (pos + 1 > outLen) return 0; + out[pos++] = 0x00; // requested QoS 0 + return pos; +} + +// PINGREQ (§3.12) and DISCONNECT (§3.14) — 2-byte packets (type + zero remaining-length). +inline size_t buildMqttPingreq(uint8_t* out, size_t outLen) { + return mqttWriteFixedHeader(out, outLen, MqttPacketType::Pingreq, 0, 0); +} +inline size_t buildMqttDisconnect(uint8_t* out, size_t outLen) { + return mqttWriteFixedHeader(out, outLen, MqttPacketType::Disconnect, 0, 0); +} + +// --- Inbound frame parser (byte-at-a-time state machine, ImprovFrameParser shape) --- +// The received side: the module feeds socket bytes here and inspects lastType() + the parsed body. +// A CONNACK's return code is read directly off the completed body (body()[1]); SUBACK/PINGRESP need +// no field beyond the type — so no standalone per-type parse helper is carried (the outbound build* +// helpers have callers; a receive-side raw-packet parser would not). +// Feeds off a non-blocking socket that hands over arbitrary byte runs; reassembles one control +// packet at a time so the receive path is host-testable with no socket. On a complete PUBLISH it +// exposes topic()/payload()/payloadLen(); for other types the caller reads lastType(). + +inline constexpr size_t kMqttMaxPacket = 512; // control packets a light exchanges are tiny + +enum class MqttFeedResult : uint8_t { + NeedMore, // mid-packet + PacketReady, // a complete packet is available via lastType() + (for PUBLISH) topic()/payload() + Malformed, // bad varint / oversize packet — parser resynced +}; + +class MqttInboundParser { +public: + // Feed one received byte. Returns NeedMore until a full control packet has been read. + MqttFeedResult feed(uint8_t byte) { + switch (state_) { + case State::FixedHeader: + type_ = static_cast<uint8_t>(byte >> 4); + flags_ = static_cast<uint8_t>(byte & 0x0F); + rlValue_ = 0; + rlMultiplier_ = 1; + rlBytes_ = 0; + state_ = State::RemainingLength; + return MqttFeedResult::NeedMore; + case State::RemainingLength: { + rlValue_ += static_cast<uint32_t>(byte & 0x7F) * rlMultiplier_; + rlMultiplier_ <<= 7; + if (++rlBytes_ > 4) { reset(); return MqttFeedResult::Malformed; } // >4 → malformed + if ((byte & 0x80) == 0) { // last length byte + if (rlValue_ > kMqttMaxPacket) { reset(); return MqttFeedResult::Malformed; } + bodyLen_ = rlValue_; + bodyPos_ = 0; + if (bodyLen_ == 0) { state_ = State::FixedHeader; return MqttFeedResult::PacketReady; } + state_ = State::Body; + } + return MqttFeedResult::NeedMore; + } + case State::Body: + body_[bodyPos_++] = byte; + if (bodyPos_ >= bodyLen_) { + state_ = State::FixedHeader; + return MqttFeedResult::PacketReady; + } + return MqttFeedResult::NeedMore; + } + return MqttFeedResult::NeedMore; // unreachable + } + + uint8_t lastType() const { return type_; } + uint8_t lastFlags() const { return flags_; } + const uint8_t* body() const { return body_; } + size_t bodyLen() const { return bodyLen_; } + + // For a completed PUBLISH: the topic (NUL-terminated in topicBuf_) and the payload slice. Returns + // false if the last packet wasn't a well-formed PUBLISH. QoS0 assumed (no packet-id in the body). + bool publish(const char** topic, const uint8_t** payload, size_t* payloadLen) { + if (type_ != static_cast<uint8_t>(MqttPacketType::Publish)) return false; + if (bodyLen_ < 2) return false; + const size_t tLen = static_cast<size_t>((body_[0] << 8) | body_[1]); + if (2 + tLen > bodyLen_) return false; + if (tLen >= sizeof(topicBuf_)) return false; + std::memcpy(topicBuf_, body_ + 2, tLen); + topicBuf_[tLen] = '\0'; + if (topic) *topic = topicBuf_; + if (payload) *payload = body_ + 2 + tLen; + if (payloadLen) *payloadLen = bodyLen_ - 2 - tLen; + return true; + } + +private: + void reset() { state_ = State::FixedHeader; } + enum class State : uint8_t { FixedHeader, RemainingLength, Body }; + State state_ = State::FixedHeader; + uint8_t type_ = 0; + uint8_t flags_ = 0; + uint32_t rlValue_ = 0; + uint32_t rlMultiplier_ = 1; + uint8_t rlBytes_ = 0; + size_t bodyLen_ = 0; + size_t bodyPos_ = 0; + uint8_t body_[kMqttMaxPacket] = {}; + char topicBuf_[128] = {}; +}; + +} // namespace mm diff --git a/src/core/NetworkModule.h b/src/core/NetworkModule.h index c63d9022..a9cb3d26 100644 --- a/src/core/NetworkModule.h +++ b/src/core/NetworkModule.h @@ -566,7 +566,15 @@ class NetworkModule : public MoonModule { // ethType_ is uint8_t (not int16_t like the pins) so it binds as a Select // dropdown via addSelect — the value is the option index, which matches the // EthPhyType enum order (None/LAN8720/IP101/W5500). - uint8_t ethType_ = static_cast<uint8_t>(platform::ethConfigDefault.phyType); + // + // Defaults to 0 (None) — Ethernet is OPT-IN per board, set explicitly by the + // deviceModels.json eth block (ethType: 1/2/3/4). A WiFi-only board (no eth block) + // must NOT try to bring up a PHY it doesn't have — that wasted RMII/SPI init is the + // bug this default avoids. The pins below stay seeded from the per-chip + // ethConfigDefault so a board that DOES opt in gets its chip's historical pins without + // re-listing them; only the PHY *selection* defaults off. Matches the installer UI, + // whose Ethernet pill is "active" (green) only when ethType is set (ethConfigured()). + uint8_t ethType_ = 0; // 0 = None; a board opts in via its catalog eth block // GPIO/address members are int8_t (one byte; -1 = unused). A GPIO never exceeds // ~54 on any ESP32-family chip, so int8 is ample — bound via addPin (Pin control // → number input). ethConfigDefault's fields are plain int; the values are all diff --git a/src/core/Scheduler.cpp b/src/core/Scheduler.cpp index 74b4ef1a..80c6ba73 100644 --- a/src/core/Scheduler.cpp +++ b/src/core/Scheduler.cpp @@ -200,6 +200,7 @@ void Scheduler::deduplicateNamesInTree() { } MoonModule* Scheduler::firstByName(const char* name) { + if (!name || name[0] == 0) return nullptr; // firstInTree strcmps name; a null would be UB for (uint8_t i = 0; i < moduleCount_; i++) { if (auto* m = firstInTree(modules_[i], name)) return m; } diff --git a/src/core/Scheduler.h b/src/core/Scheduler.h index c89c7480..d35438f0 100644 --- a/src/core/Scheduler.h +++ b/src/core/Scheduler.h @@ -135,4 +135,21 @@ class Scheduler { uint32_t lastTimingUpdate_ = 0; // 1-second window start }; +// --- Drivers master-state reads, shared by every consumer that mirrors the light state (the WLED +// /json shim, MQTT). Domain-neutral: they name the `Drivers` module + its controls and read via the +// generic MoonModule::read* helpers, so the absent-control defaults are defined ONCE here instead of +// per consumer (a device with no `on` control still reads as on; brightness/palette default to 0). --- +inline bool driversOn(Scheduler* s) { + MoonModule* d = s ? s->firstByName("Drivers") : nullptr; + return d ? d->readBool("on", true) : true; // absent → on +} +inline uint8_t driversBrightness(Scheduler* s) { + MoonModule* d = s ? s->firstByName("Drivers") : nullptr; + return d ? d->readUint8("brightness", 0) : 0; +} +inline uint8_t driversPalette(Scheduler* s) { + MoonModule* d = s ? s->firstByName("Drivers") : nullptr; + return d ? d->readUint8("palette", 0) : 0; +} + } // namespace mm diff --git a/src/core/SystemModule.h b/src/core/SystemModule.h index a2382d2a..40c1313b 100644 --- a/src/core/SystemModule.h +++ b/src/core/SystemModule.h @@ -91,16 +91,11 @@ class SystemModule : public MoonModule { mac[4], mac[5]); } - // Snprintf static display strings into the bound buffers. onBuildControls already - // bound these buffers by pointer; we fill them now and the UI picks up the content - // on the next WebSocket push. - std::snprintf(chipInfo_, sizeof(chipInfo_), "%s", platform::chipModel()); - std::snprintf(sdkInfo_, sizeof(sdkInfo_), "%s", platform::sdkVersion()); - // version / build / firmware (firmware identity, incl. the build date) moved to - // FirmwareUpdateModule — SystemModule keeps only the IDF version string (`sdk`). - std::snprintf(bootReasonStr_, sizeof(bootReasonStr_), "%s", platform::resetReason()); + // chip / sdk / mac / bootReason controls bind straight to the platform's static strings (no + // per-module copy — see onBuildControls). version / build / firmware (firmware identity, incl. + // the build date) moved to FirmwareUpdateModule — SystemModule keeps only the IDF `sdk`. if constexpr (platform::hasWifiCoprocessor) { - std::snprintf(coprocStr_, sizeof(coprocStr_), "%s", platform::coprocessorWifi()); + (void)platform::coprocessorWifi(); // prime the static (the control points at it) } if (chipFlashVal_ > 0) { @@ -163,18 +158,22 @@ class SystemModule : public MoonModule { controls_.addReadOnly("flash", flashStr_, sizeof(flashStr_)); } - // Static info - controls_.addReadOnly("chip", chipInfo_, sizeof(chipInfo_)); - controls_.addReadOnly("sdk", sdkInfo_, sizeof(sdkInfo_)); - controls_.addReadOnly("bootReason", bootReasonStr_, sizeof(bootReasonStr_)); + // Static info — the platform returns these as permanent static strings (a chip's MAC / model + // / SDK never change at runtime), so the ReadOnly control binds straight to the platform + // buffer. Nothing stored per-module: don't copy a static string into a member (minimise + // memory; don't store what's derivable). const_cast is safe — ReadOnly never writes. + // bufSize is unused for a ReadOnly (never written), so the default is fine. + controls_.addReadOnly("mac", const_cast<char*>(platform::macString())); // stable per-chip identity (tooling keys on it) + controls_.addReadOnly("chip", const_cast<char*>(platform::chipModel())); + controls_.addReadOnly("sdk", const_cast<char*>(platform::sdkVersion())); + controls_.addReadOnly("bootReason", const_cast<char*>(platform::resetReason())); // WiFi co-processor (P4 + on-board C6) firmware read-out. Gated at compile // time on hasWifiCoprocessor, so the whole control — and the snprintf/query // cost — vanishes on native-radio builds (classic/S3/desktop) and the // eth-only P4. Its value proves the C6 slave-firmware state ("C6 fw 2.12.9" // vs "not detected"). loop1s() refreshes it. if constexpr (platform::hasWifiCoprocessor) { - std::snprintf(coprocStr_, sizeof(coprocStr_), "%s", platform::coprocessorWifi()); - controls_.addReadOnly("wifiCoproc", coprocStr_, sizeof(coprocStr_)); + controls_.addReadOnly("wifiCoproc", const_cast<char*>(platform::coprocessorWifi())); } // Chain into children (user-added Peripherals). Per the override-and-chain @@ -230,9 +229,11 @@ class SystemModule : public MoonModule { // Refresh the WiFi co-processor status, so the displayed C6 firmware state // stays current if the link comes up after boot or the C6 is reflashed - // without a host reboot. Compiled out where there's no co-processor. + // without a host reboot. coprocessorWifi() re-queries the C6 and updates its own + // static buffer in place — the wifiCoproc control points straight at it, so we + // just call it to refresh (no per-module copy). Compiled out where there's no co-proc. if constexpr (platform::hasWifiCoprocessor) { - std::snprintf(coprocStr_, sizeof(coprocStr_), "%s", platform::coprocessorWifi()); + (void)platform::coprocessorWifi(); } // Chain to base so children get their loop1s() — a Peripheral formats @@ -291,10 +292,6 @@ class SystemModule : public MoonModule { uint32_t psramUsedVal_ = 0; // Static (set in setup) - char chipInfo_[16] = {}; - char sdkInfo_[24] = {}; - char bootReasonStr_[16] = {}; - char coprocStr_[24] = {}; // WiFi co-processor status, e.g. "C6 fw 2.12.9" / "not detected" uint32_t totalInternalVal_ = 0; uint32_t totalHeapVal_ = 0; char flashStr_[12] = {}; diff --git a/src/light/Palette.h b/src/light/Palette.h index c3b928df..ae11bf42 100644 --- a/src/light/Palette.h +++ b/src/light/Palette.h @@ -234,7 +234,76 @@ class Palettes { return p; } + // --- Representative colour of a palette (for the HomeKit color-wheel → palette mapping) --- + // + // HomeKit (via MQTT/Homebridge) has no "palette" concept but a native colour wheel, so we map a + // wheel colour to the nearest palette. Each palette's representative (hue, sat) is COMPUTED from + // its expanded entries — the textbook "dominant colour": average the 16 RGB entries, convert to + // HSV. No hand-maintained table (it auto-covers every built-in + any future one). A rainbow/ + // multi-hue palette averages toward grey → low saturation, which correctly matches the wheel's + // desaturated centre rather than any single hue. Off the hot path (called on an MQTT set / get). + + // Representative hue (0..359) of built-in `index`. See representativeHueSat(). + static uint16_t representativeHue(uint8_t index) { + uint16_t hue = 0, sat = 0; + representativeHueSat(index, hue, sat); + return hue; + } + + // The palette whose representative (hue, sat) is closest to the target, by 2D distance in + // (circular-hue, saturation) space. `hue` 0..359, `sat` 0..255. Low target saturation snaps to a + // desaturated palette (e.g. Rainbow); a vivid hue snaps to the matching single-hue palette. + static uint8_t nearestForHue(uint16_t hue, uint8_t sat) { + hue %= 360; // fold any caller's value into 0..359 — a >=360 hue would otherwise make the + // circular-distance fold below underflow and the squared distance overflow int32 + uint8_t best = 0; + uint32_t bestDist = 0xFFFFFFFF; + for (uint8_t i = 0; i < palettes::kCount; i++) { + uint16_t ph = 0, ps = 0; + representativeHueSat(i, ph, ps); + // Circular hue distance (0..180), scaled so hue and sat weigh comparably (hue 0..359 vs + // sat 0..255): fold the hue delta to ≤180, then to a 0..255-ish range. + uint16_t dh = static_cast<uint16_t>((hue > ph) ? (hue - ph) : (ph - hue)); + if (dh > 180) dh = static_cast<uint16_t>(360 - dh); + const int32_t hueDelta = (static_cast<int32_t>(dh) * 255) / 180; // 0..255 + const int32_t satDelta = static_cast<int32_t>(sat) - static_cast<int32_t>(ps); + const uint32_t dist = static_cast<uint32_t>(hueDelta * hueDelta + satDelta * satDelta); + if (dist < bestDist) { bestDist = dist; best = i; } + } + return best; + } + + // Compute a palette's representative hue (0..359) + saturation (0..255) from the average of its + // expanded RGB entries. Pure; no allocation. + static void representativeHueSat(uint8_t index, uint16_t& hueOut, uint16_t& satOut) { + const Palette p = fromBuiltin(index); + uint32_t rs = 0, gs = 0, bs = 0; + for (uint8_t i = 0; i < Palette::kEntries; i++) { + rs += p.entry[i].r; gs += p.entry[i].g; bs += p.entry[i].b; + } + const uint8_t r = static_cast<uint8_t>(rs / Palette::kEntries); + const uint8_t g = static_cast<uint8_t>(gs / Palette::kEntries); + const uint8_t b = static_cast<uint8_t>(bs / Palette::kEntries); + rgbToHueSat(r, g, b, hueOut, satOut); + } + private: + // Integer RGB → (hue 0..359, saturation 0..255). The textbook max/min/delta HSV derivation; V is + // unused (we only match on hue+sat). Achromatic (delta 0) → hue 0, sat 0. + static void rgbToHueSat(uint8_t r, uint8_t g, uint8_t b, uint16_t& hue, uint16_t& sat) { + const uint8_t mx = r > g ? (r > b ? r : b) : (g > b ? g : b); + const uint8_t mn = r < g ? (r < b ? r : b) : (g < b ? g : b); + const uint8_t delta = static_cast<uint8_t>(mx - mn); + if (delta == 0 || mx == 0) { hue = 0; sat = 0; return; } + sat = static_cast<uint16_t>((static_cast<uint32_t>(delta) * 255) / mx); + int32_t h; + if (mx == r) h = 60 * (static_cast<int32_t>(g) - b) / delta; + else if (mx == g) h = 120 + 60 * (static_cast<int32_t>(b) - r) / delta; + else h = 240 + 60 * (static_cast<int32_t>(r) - g) / delta; + if (h < 0) h += 360; + hue = static_cast<uint16_t>(h % 360); + } + // Default to a full rainbow (index 0): always colourful, so an effect renders visible output // before any palette is selected. setActive() (Drivers setup) overrides from the saved index. static inline Palette active_ = fromBuiltin(0); diff --git a/src/light/drivers/Drivers.h b/src/light/drivers/Drivers.h index 736aa92b..367ac9aa 100644 --- a/src/light/drivers/Drivers.h +++ b/src/light/drivers/Drivers.h @@ -61,10 +61,12 @@ class DriverBase : public MoonModule { /// doesn't trigger a structural rebuild. virtual void onCorrectionChanged() {} - /// Clear both shared status strings on teardown (frees the owned `failBuf_`). A - /// driver that overrides teardown() for its own peripheral cleanup chains to this - /// afterwards: `deinit(); DriverBase::teardown();`. - void teardown() override { clearFailBuf(); clearConfigErr(); } + /// Clear every shared status string on teardown — fail buffer, config error, and config + /// warning — so a stopped driver leaves nothing behind (frees the owned `failBuf_`; retracts + /// the warning the same "clear only MY status" way as the error). A driver that overrides + /// teardown() for its own peripheral cleanup chains to this afterwards: + /// `deinit(); DriverBase::teardown();`. + void teardown() override { clearFailBuf(); clearConfigErr(); setConfigWarn(nullptr); } protected: Layer* layer_ = nullptr; @@ -123,6 +125,7 @@ class DriverBase : public MoonModule { // rule). Preview-style drivers never touch these, so the cost is a couple of // null pointers they ignore. const char* configErr_ = nullptr; + const char* configWarn_ = nullptr; char* failBuf_ = nullptr; static constexpr size_t kFailBufLen = 48; @@ -139,6 +142,22 @@ class DriverBase : public MoonModule { } } + // A non-fatal config warning (a borrowed literal, like configErr_): the driver keeps + // running but flags something the user should see (e.g. a per-pin count clamped to the + // WS2812 ceiling). `warn` non-null sets it; null retracts a stale one — so a driver calls + // setConfigWarn(warn) unconditionally each parse and the warning tracks the live state. + // Same "clear only MY status" rule as configErr_. Factored here so the WS2812 drivers + // (Rmt / LCD / Parlio) don't each inline it — the No-duplication rule this block records. + void setConfigWarn(const char* warn) { + if (warn) { + configWarn_ = warn; + setStatus(warn, Severity::Warning); + } else if (configWarn_) { + if (status() == configWarn_) clearStatus(); + configWarn_ = nullptr; + } + } + // Lazily allocate the owned fail-message buffer (caller snprintf's into it then // setStatus(failBuf_)). Returns null if the allocation fails, in which case the // caller falls back to a literal status. @@ -220,6 +239,14 @@ class Drivers : public MoonModule { /// first boot can't brown out the board before the user sets a safe level. /// The user raises it via the brightness control once their supply is known. uint8_t brightness = 20; + /// Master power. `on=false` outputs black without touching `brightness`, so toggling back + /// to `on=true` restores the exact level. Implemented by scaling the correction LUT to zero + /// (effectiveBrightness()) — the same cold-path rebuild brightness uses, no hot-path branch. + /// This is the single power control every consumer drives: the UI toggle, IR's on/off action, + /// the WLED app / Home Assistant (`{"on":…}`), and MQTT/Homebridge all set THIS control through + /// `Scheduler::setControl` — define-once, reuse everywhere (the first slice of a global + /// lights-control surface). Default on so a freshly-flashed board lights up. + bool on = true; /// Physical wire format: channel order and whether the light is RGBW (index into /// `kLightPresetOptions`; options `RGB`, `RBG`, `GRB`, `GBR`, `BRG`, `BGR`, `RGBW`, /// `GRBW`). RGBW presets make each driver emit 4 channels per light with white @@ -254,7 +281,12 @@ class Drivers : public MoonModule { layer_ = layer; } + // The brightness actually fed to the LUT: 0 when powered off, else the set brightness. Keeping + // `on` and `brightness` independent means "off" never clobbers the level the user chose. + uint8_t effectiveBrightness() const { return on ? brightness : 0; } + void onBuildControls() override { + controls_.addBool("on", on); // master power — first so it renders at the top of the card controls_.addUint8("brightness", brightness, 0, 255); controls_.addSelect("lightPreset", lightPreset, kLightPresetOptions, kLightPresetCount); controls_.addPalette("palette", palette, mm::paletteOptions, mm::palettes::kCount); @@ -269,9 +301,10 @@ class Drivers : public MoonModule { Palettes::setActive(palette); // rebuild the active 16-entry lookup (cheap, off the hot path) return; } - if (std::strcmp(controlName, "brightness") == 0 || + if (std::strcmp(controlName, "on") == 0 || + std::strcmp(controlName, "brightness") == 0 || std::strcmp(controlName, "lightPreset") == 0) { - correction_.rebuild(brightness, static_cast<LightPreset>(lightPreset)); + correction_.rebuild(effectiveBrightness(), static_cast<LightPreset>(lightPreset)); // Propagate so physical drivers that maintain a correction-applied // buffer (today: ArtNet) can resize off the hot path. A brightness- // only change is a no-op for resizing (outChannels stays 3); the @@ -283,7 +316,7 @@ class Drivers : public MoonModule { } void setup() override { - correction_.rebuild(brightness, static_cast<LightPreset>(lightPreset)); + correction_.rebuild(effectiveBrightness(), static_cast<LightPreset>(lightPreset)); Palettes::setActive(palette); // seed the global active palette from the persisted index MoonModule::setup(); passBufferToDrivers(); diff --git a/src/light/drivers/ParallelLedDriver.h b/src/light/drivers/ParallelLedDriver.h index 8c99554d..2fb7daaf 100644 --- a/src/light/drivers/ParallelLedDriver.h +++ b/src/light/drivers/ParallelLedDriver.h @@ -227,11 +227,14 @@ class ParallelLedDriver : public DriverBase { if constexpr (Derived::kExactLaneCount) { if (!err && n != kMaxLanes) err = "LCD bus needs exactly 8 pins"; } + const char* warn = nullptr; if (!err) { // Distribute over this driver's window slice, not the whole buffer. + // assignCounts clamps each lane to kMaxWs2812LedsPerPin (drives that many + // rather than choking a whole grid onto one WS2812 line). const nrOfLightsType bufN = sourceBuffer_ ? sourceBuffer_->count() : 0; windowSlice(bufN, winStart_, winLen_); - err = assignCounts(ledsPerPin, n, winLen_, laneCounts_); + err = assignCounts(ledsPerPin, n, winLen_, laneCounts_, kMaxWs2812LedsPerPin, &warn); } if (err) { setConfigErr(err); @@ -247,6 +250,8 @@ class ParallelLedDriver : public DriverBase { const uint8_t outCh = correction_ ? correction_->outChannels : 0; frameBytes_ = frameBytesFor(maxLaneLights_, outCh); clearConfigErr(); + // A lane clamped to the WS2812 ceiling still drives — Warning, not error (see RmtLed). + setConfigWarn(warn); return true; } diff --git a/src/light/drivers/PinList.h b/src/light/drivers/PinList.h index c4144d7c..e20f024b 100644 --- a/src/light/drivers/PinList.h +++ b/src/light/drivers/PinList.h @@ -39,13 +39,46 @@ inline const char* parsePinList(const char* s, uint16_t* out, uint8_t maxPins, } } +// The per-pin light ceiling for a WS2812-class 1-wire protocol (RMT / LCD_CAM / Parlio +// all shift the same NRZ wire timing): one line clocks a fixed ~30 us/LED (24 bits × +// 1.25 us), so 2048 LEDs ≈ 61 ms ≈ 16 FPS — the floor before output turns to a slideshow. +// The WS2812 drivers pass this as assignCounts' maxPerPin. Clocked 2-wire types +// (APA102/SK9822 over SPI) have no such fixed-rate limit — they pass a far higher cap (or 0). +inline constexpr nrOfLightsType kMaxWs2812LedsPerPin = 2048; + // Fill counts[0..nPins) from "100,100,50" (may be empty or shorter than the // pin list; extra entries beyond nPins are ignored — a stale longer list // after pins shrank is not an error). Explicit counts are clamped so the // running sum never exceeds totalLights; the unassigned remainder splits // evenly over the unlisted pins, last pin takes the rounding remainder. +// +// `maxPerPin` is the driver's protocol ceiling on lights-per-data-line: a pin whose count +// exceeds it is clamped to it, so the driver drives the first maxPerPin (output stays lit) +// instead of choking on the rest. Per-protocol, so the driver passes its own — a WS2812-class +// 1-wire line (RMT / LCD_CAM / Parlio) clocks a fixed ~30 us/LED, so ~2048/pin is already +// ~16 FPS (kMaxWs2812LedsPerPin); a clocked 2-wire SPI type (APA102/SK9822) runs tens of MHz +// and does 10k+/pin, so it passes a far higher cap. Pass 0 for no ceiling. The intended way +// to output fewer lights is the driver's start/count window, not this safety cap. +// +// On a clamp, `*warn` (if given) is set to kClampedWarning — a Warning the caller shows while +// still running, distinct from the return value (which stays nullptr: clamping is not an +// error that idles the driver). Left null when nothing was clamped. +// +// Offsets accumulate from the CLAMPED counts, so clamping pin i also shifts every later pin's +// source slice down by the trimmed amount. For the headline case — a whole grid funneled onto +// ONE pin — this is exactly right (that one pin drives the first maxPerPin, nothing after it). +// For the pathological multi-pin-all-over-ceiling case (e.g. 3000 on each of 3 pins) the later +// strips then show a shifted window, not just a truncated one. Accepted, not fixed: it degrades +// rather than crashes (architecture.md Robustness), the misconfig is not one anyone wires on +// purpose, and preserving alignment would need a parallel unclamped-counts array — more state +// for a case the warning already flags. The window (start/count) is the real way to send fewer. +inline constexpr const char* kClampedWarning = + "some LEDs not driven: over per-pin max; add pins or use start/count"; + inline const char* assignCounts(const char* s, uint8_t nPins, - nrOfLightsType totalLights, nrOfLightsType* counts) { + nrOfLightsType totalLights, nrOfLightsType* counts, + nrOfLightsType maxPerPin = 0, const char** warn = nullptr) { + if (warn) *warn = nullptr; for (uint8_t i = 0; i < nPins; i++) counts[i] = 0; nrOfLightsType remaining = totalLights; uint8_t nExplicit = 0; @@ -71,6 +104,12 @@ inline const char* assignCounts(const char* s, uint8_t nPins, counts[nPins - 1] = static_cast<nrOfLightsType>( counts[nPins - 1] + (remaining - per * nRemaining)); } + if (maxPerPin > 0) + for (uint8_t i = 0; i < nPins; i++) + if (counts[i] > maxPerPin) { + counts[i] = maxPerPin; + if (warn) *warn = kClampedWarning; + } return nullptr; } diff --git a/src/light/drivers/RmtLedDriver.h b/src/light/drivers/RmtLedDriver.h index c6798c86..189f0c54 100644 --- a/src/light/drivers/RmtLedDriver.h +++ b/src/light/drivers/RmtLedDriver.h @@ -326,13 +326,16 @@ class RmtLedDriver : public DriverBase { bool parseConfig() { pinCount_ = 0; uint8_t n = 0; + const char* warn = nullptr; const char* err = parsePinList(pins, pinList_, maxPinsForTarget(), n); if (!err) { // Distribute over the driver's window slice, not the whole buffer, so // ledsPerPin's "rest" only fills this driver's [start, start+count). + // assignCounts clamps each pin to kMaxWs2812LedsPerPin (the driver drives + // that many rather than choking a whole grid onto one WS2812 line). const nrOfLightsType bufN = sourceBuffer_ ? sourceBuffer_->count() : 0; windowSlice(bufN, winStart_, winLen_); - err = assignCounts(ledsPerPin, n, winLen_, pinCounts_); + err = assignCounts(ledsPerPin, n, winLen_, pinCounts_, kMaxWs2812LedsPerPin, &warn); } if (err) { setConfigErr(err); @@ -348,6 +351,11 @@ class RmtLedDriver : public DriverBase { txLightCount_ = static_cast<nrOfLightsType>(txLightCount_ + pinCounts_[i]); } clearConfigErr(); + // assignCounts sets `warn` when it clamped a pin to the WS2812 ceiling — the output + // still runs (the first 2048/pin), so it's a Warning, not an idling error. Passing it + // (or null when nothing clamped) to setConfigWarn tracks the live state and retracts a + // stale warning once the user drops back under the ceiling. + setConfigWarn(warn); return true; } diff --git a/src/light/effects/RubiksCubeEffect.h b/src/light/effects/RubiksCubeEffect.h index 0ebfeaf0..f1b2a6d0 100644 --- a/src/light/effects/RubiksCubeEffect.h +++ b/src/light/effects/RubiksCubeEffect.h @@ -3,6 +3,7 @@ #include "light/effects/EffectBase.h" #include "light/layers/Layer.h" // layer()->buffer() #include "light/draw.h" // draw::pixel +#include "light/Palette.h" // colorFromPalette + the active palette (usePalette) #include "core/math8.h" // Random8 #include <array> @@ -36,11 +37,28 @@ class RubiksCubeEffect : public EffectBase { uint8_t turnsPerSecond = 2; // 0..20 uint8_t cubeSize = 3; // 1..8 (cube order) bool randomTurning = false; + bool usePalette = false; // off = the classic 6 Rubik's face colours; on = 6 samples + // of the system-wide palette (advised: a primary-ish palette) void onBuildControls() override { controls_.addUint8("turnsPerSecond", turnsPerSecond, 0, 20); controls_.addUint8("cubeSize", cubeSize, 1, 8); controls_.addBool("randomTurning", randomTurning); + controls_.addBool("usePalette", usePalette); + } + + // The 6 face colours drawCube paints with. Classic Rubik's set (red, dark-orange, blue, green, + // yellow, white) by default; with usePalette, 6 evenly-spaced samples of the system-wide active + // palette (0, 51, 102, … 255) so the cube recolours to whatever palette is chosen — advised: a + // primary-ish palette so the 6 faces stay distinct. + std::array<RGB, 6> faceColors() const { + if (!usePalette) + return {{{255, 0, 0}, {255, 140, 0}, {0, 0, 255}, {0, 128, 0}, {255, 255, 0}, {255, 255, 255}}}; + std::array<RGB, 6> pal{}; + const Palette& active = *Palettes::active(); + for (int i = 0; i < 6; i++) + pal[i] = colorFromPalette(active, static_cast<uint8_t>(i * 255 / 5)); + return pal; } // cubeSize / randomTurning changes re-scramble (MoonLight reinitialises on those controls). No @@ -80,7 +98,7 @@ class RubiksCubeEffect : public EffectBase { : unpackMove(moveList_[moveIndex_]); // Playback applies the inverse direction so the scramble unwinds toward solved. (cube_.*kRotateFuncs[move.face])(!move.direction, static_cast<uint8_t>(move.width + 1)); - cube_.drawCube(buf, dims, w, h, d); + cube_.drawCube(buf, dims, w, h, d, faceColors()); if (!randomTurning && moveIndex_ == 0) { step_ = now + 3000; // solved: hold for 3 s, then re-scramble @@ -205,7 +223,9 @@ class RubiksCubeEffect : public EffectBase { // Project the cube onto the LED volume: every in-bounds voxel is coloured by the outer face // it sits nearest. (MoonLight's drawCube, with the isMapped()-skip and sizeX++/etc dropped.) - void drawCube(Buffer& buf, Coord3D dims, lengthType sx, lengthType sy, lengthType sz) const { + // The 6 face colours are supplied by the caller (classic Rubik's set, or palette samples). + void drawCube(Buffer& buf, Coord3D dims, lengthType sx, lengthType sy, lengthType sz, + const std::array<RGB, 6>& COLOR_MAP) const { // This effect owns its background: drawCube writes only the SURFACE voxels (the loop has // no else for the interior), and a turn moves stickers to new positions, so without a // wipe the old pose's stickers linger and the cube accretes garbage — it never settles. @@ -220,10 +240,6 @@ class RubiksCubeEffect : public EffectBase { const int denX = 2 * sizeX, denY = 2 * sizeY, denZ = 2 * sizeZ; const int halfX = sizeX / 2, halfY = sizeY / 2, halfZ = sizeZ / 2; - // Red, DarkOrange, Blue, Green, Yellow, White. - const RGB COLOR_MAP[6] = { - {255, 0, 0}, {255, 140, 0}, {0, 0, 255}, {0, 128, 0}, {255, 255, 0}, {255, 255, 255}}; - for (int x = 0; x < sx; x++) for (int y = 0; y < sy; y++) for (int z = 0; z < sz; z++) { @@ -291,7 +307,7 @@ class RubiksCubeEffect : public EffectBase { (cube_.*kRotateFuncs[move.face])(move.direction, static_cast<uint8_t>(move.width + 1)); } moveIndex_ = static_cast<uint8_t>(cappedMoves - 1); - cube_.drawCube(buf, dims, w, h, d); + cube_.drawCube(buf, dims, w, h, d, faceColors()); } // Inline integer helpers (MoonLight's MIN/MAX/constrain). diff --git a/src/light/modifiers/MultiplyModifier.h b/src/light/modifiers/MultiplyModifier.h index c914819b..7d3d2fa9 100644 --- a/src/light/modifiers/MultiplyModifier.h +++ b/src/light/modifiers/MultiplyModifier.h @@ -22,10 +22,12 @@ class MultiplyModifier : public ModifierBase { public: const char* tags() const override { return "💫"; } // MoonLight origin - // Tiles per axis. 1 = no multiplication on that axis. + // Tiles per axis. 1 = no multiplication on that axis. All default to 2 (tile on every + // axis the layout has); on a 2D grid multiplyZ clamps to 1 (a no-op), so it only tiles + // Z on an actual 3D volume — consistent with X/Y rather than a silent exception. uint8_t multiplyX = 2; uint8_t multiplyY = 2; - uint8_t multiplyZ = 1; + uint8_t multiplyZ = 2; // Reflect alternate (odd-numbered) tiles on this axis. All default on — a // mirror on an axis the layout doesn't use (e.g. Z on a 2D grid) is a no-op, // so defaulting them true gives a kaleidoscope on whatever axes exist. diff --git a/src/main.cpp b/src/main.cpp index 5788118e..56493ac8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -98,8 +98,10 @@ #include "core/AudioModule.h" #include "core/I2cScanModule.h" #include "core/IrModule.h" +#include "core/FileManagerModule.h" #include "core/FirmwareUpdateModule.h" #include "core/ImprovProvisioningModule.h" +#include "core/MqttModule.h" #include "core/DevicesModule.h" #include "core/FilesystemModule.h" #include "core/ModuleFactory.h" @@ -207,16 +209,18 @@ static void registerModuleTypes() { #if defined(CONFIG_SOC_PARLIO_SUPPORTED) mm::ModuleFactory::registerType<mm::ParlioLedDriver>("ParlioLedDriver", "light/drivers/ParlioLedDriver.md"); #endif - mm::ModuleFactory::registerType<mm::HttpServerModule>("HttpServerModule", "core/HttpServerModule.md"); - mm::ModuleFactory::registerType<mm::SystemModule>("SystemModule", "core/SystemModule.md"); - mm::ModuleFactory::registerType<mm::AudioModule>("AudioModule", "core/AudioModule.md"); - mm::ModuleFactory::registerType<mm::I2cScanModule>("I2cScanModule", "core/I2cScanModule.md"); - mm::ModuleFactory::registerType<mm::IrModule>("IrModule", "core/IrModule.md"); - mm::ModuleFactory::registerType<mm::FirmwareUpdateModule>("FirmwareUpdateModule", "core/FirmwareUpdateModule.md"); - mm::ModuleFactory::registerType<mm::ImprovProvisioningModule>("ImprovProvisioningModule", "core/ImprovProvisioningModule.md"); - mm::ModuleFactory::registerType<mm::DevicesModule>("DevicesModule", "core/DevicesModule.md"); - mm::ModuleFactory::registerType<mm::NetworkModule>("NetworkModule", "core/NetworkModule.md"); - mm::ModuleFactory::registerType<mm::FilesystemModule>("FilesystemModule", "core/FilesystemModule.md"); + mm::ModuleFactory::registerType<mm::HttpServerModule>("HttpServerModule", "core/ui/ui.md"); + mm::ModuleFactory::registerType<mm::SystemModule>("SystemModule", "core/ui/ui.md#system"); + mm::ModuleFactory::registerType<mm::AudioModule>("AudioModule", "core/ui/ui.md#audio"); + mm::ModuleFactory::registerType<mm::I2cScanModule>("I2cScanModule", "core/ui/ui.md#i2c-scan"); + mm::ModuleFactory::registerType<mm::IrModule>("IrModule", "core/ui/ui.md#ir"); + mm::ModuleFactory::registerType<mm::FileManagerModule>("FileManagerModule", "core/ui/ui.md#file-manager"); + mm::ModuleFactory::registerType<mm::FirmwareUpdateModule>("FirmwareUpdateModule", "core/ui/ui.md#firmware-update"); + mm::ModuleFactory::registerType<mm::ImprovProvisioningModule>("ImprovProvisioningModule", "core/ui/ui.md#improv-provisioning"); + mm::ModuleFactory::registerType<mm::MqttModule>("MqttModule", "core/ui/ui.md#mqtt"); + mm::ModuleFactory::registerType<mm::DevicesModule>("DevicesModule", "core/ui/ui.md#devices"); + mm::ModuleFactory::registerType<mm::NetworkModule>("NetworkModule", "core/ui/ui.md#network"); + mm::ModuleFactory::registerType<mm::FilesystemModule>("FilesystemModule", "core/ui/ui.md#filesystem"); } static void printModuleMetrics(mm::MoonModule* mod, int depth) { @@ -263,6 +267,12 @@ void mm_main(volatile bool& keepRunning, uint16_t httpPort) { auto* filesystemModule = static_cast<mm::FilesystemModule*>(mm::ModuleFactory::create("FilesystemModule")); filesystemModule->setScheduler(&scheduler); + // File Manager — browse/manage the filesystem (a device-wide tool, boot-wired like the other + // system modules, not per-board). Distinct from FilesystemModule (the persistence engine). + // setName gives the card a clean "File Manager" label (the type stays FileManagerModule). + auto* fileManagerModule = static_cast<mm::FileManagerModule*>(mm::ModuleFactory::create("FileManagerModule")); + fileManagerModule->setName("File Manager"); + // System (deviceName needed by other modules) auto* systemModule = static_cast<mm::SystemModule*>(mm::ModuleFactory::create("SystemModule")); systemModule->setScheduler(&scheduler); @@ -325,6 +335,17 @@ void mm_main(volatile bool& keepRunning, uint16_t httpPort) { improvModule->markWiredByCode(); } + // MQTT service: a code-wired child of Network (like Improv), bridging the light controls to a + // broker for Homebridge/Home-Assistant. Built on every networked target (it uses TCP, so it + // works over WiFi or Ethernet); disabled until the user sets a broker. systemModule is injected + // for the default topic prefix (the device name). + mm::MqttModule* mqttModule = nullptr; + if constexpr (mm::platform::hasNetwork) { + mqttModule = static_cast<mm::MqttModule*>(mm::ModuleFactory::create("MqttModule")); + mqttModule->setSystemModule(systemModule); + mqttModule->markWiredByCode(); + } + // Layouts: top-level container; one or more layouts. Today one GridLayout, // which self-initialises to defaultGridSize (persistence overlays any saved // size before setup()). No boot-time dimensions threaded in here. @@ -396,15 +417,18 @@ void mm_main(volatile bool& keepRunning, uint16_t httpPort) { // Register top-level modules with scheduler (scheduler deletes on teardown). // Order matters: filesystem first (load hook runs before any module's setup), - // then system (deviceName), firmwareUpdate (status surface, no deps), network - // (hosts ImprovProvisioning as a child — same lifecycle, one less top-level - // entry, conceptually right since Improv only exists to feed Network credentials), - // light pipeline (Layouts → Layers → Drivers), then HTTP. The Scheduler walks - // roots in this order each tick; child propagation happens inside each root. + // then system (deviceName), fileManager (filesystem browser, reads the persistence + // engine's "last saved"), firmwareUpdate (status surface, no deps), network (hosts + // ImprovProvisioning and Mqtt as children — same lifecycle, one less top-level entry + // each; Improv only exists to feed Network credentials, and Mqtt only bridges once the + // network is up), light pipeline (Layouts → Layers → Drivers), then HTTP. The Scheduler + // walks roots in this order each tick; child propagation happens inside each root. scheduler.addModule(filesystemModule); scheduler.addModule(systemModule); + scheduler.addModule(fileManagerModule); scheduler.addModule(firmwareUpdateModule); if (improvModule) networkModule->addChild(improvModule); + if (mqttModule) networkModule->addChild(mqttModule); // Devices: discovers other devices on the LAN. Child of Network (discovery // depends on the network being up); wired-by-code so persistence preserves it // on devices whose saved Network.json predates the child (see DevicesModule.md). diff --git a/src/platform/desktop/platform_desktop.cpp b/src/platform/desktop/platform_desktop.cpp index 1a5dc347..5abab6e6 100644 --- a/src/platform/desktop/platform_desktop.cpp +++ b/src/platform/desktop/platform_desktop.cpp @@ -22,6 +22,7 @@ #include <sys/socket.h> #include <netinet/in.h> #include <arpa/inet.h> +#include <netdb.h> // getaddrinfo — hostname resolution for TcpConnection::connect #include <unistd.h> #include <fcntl.h> #include <sys/mman.h> // mmap/munmap for allocExec (executable pages) @@ -227,6 +228,17 @@ void getMacAddress(uint8_t mac[6]) { mac[3] = 0xEF; mac[4] = 0xCA; mac[5] = 0xFE; } +const char* macString() { + static char buf[18] = {}; + if (buf[0] == 0) { + uint8_t mac[6]; + getMacAddress(mac); + std::snprintf(buf, sizeof(buf), "%02X:%02X:%02X:%02X:%02X:%02X", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + } + return buf; +} + const char* chipModel() { return "desktop"; } @@ -376,6 +388,59 @@ bool fsWriteAtomic(const char* path, const char* data, size_t len) { return true; } +long fsSize(const char* path) { + std::error_code ec; + auto p = toFsPath(path); + if (!std::filesystem::is_regular_file(p, ec)) return -1; + const auto sz = std::filesystem::file_size(p, ec); + return ec ? -1 : static_cast<long>(sz); +} + +int fsReadAt(const char* path, long offset, char* buf, size_t len) { + if (!buf) return -1; + FILE* f = std::fopen(toFsPath(path).string().c_str(), "rb"); + if (!f) return -1; + if (std::fseek(f, offset, SEEK_SET) != 0) { std::fclose(f); return -1; } + const size_t n = std::fread(buf, 1, len, f); + std::fclose(f); + return static_cast<int>(n); // 0 at EOF +} + +bool fsWriteStream(const char* path, FsWriteSrc src, void* user) { + if (!src) return false; + auto target = toFsPath(path); + auto tmp = target; + tmp += ".tmp"; + + FILE* f = std::fopen(tmp.string().c_str(), "wb"); + if (!f) return false; + // Pull chunks from the source and write each straight through — fixed buffer, any file size. + // `abort` set by the source (a short/timed-out upload) means the data is incomplete → discard. + char chunk[1024]; + bool ok = true, abort = false; + for (;;) { + const size_t got = src(chunk, sizeof(chunk), user, &abort); + if (abort) { ok = false; break; } + if (got == 0) break; // clean end of stream + if (std::fwrite(chunk, 1, got, f) != got) { ok = false; break; } + } + std::fflush(f); +#ifdef _WIN32 + int fd = ::_fileno(f); + if (fd >= 0) ::_commit(fd); +#else + int fd = ::fileno(f); + if (fd >= 0) ::fsync(fd); +#endif + std::fclose(f); + + std::error_code ec; + if (!ok) { std::filesystem::remove(tmp, ec); return false; } + std::filesystem::rename(tmp, target, ec); + if (ec) { std::filesystem::remove(tmp, ec); return false; } + return true; +} + void fsList(const char* dir, FsListCb cb, void* user) { if (!cb) return; std::error_code ec; @@ -386,7 +451,10 @@ void fsList(const char* dir, FsListCb cb, void* user) { // path::filename().c_str() returns wchar_t* on Windows; the callback // wants char*. Round-trip through .string() to get a portable view. std::string name = entry.path().filename().string(); - cb(name.c_str(), entry.is_directory(ec), user); + const bool isDir = entry.is_directory(ec); + std::error_code sizeEc; + const auto sz = isDir ? 0u : static_cast<uint32_t>(std::filesystem::file_size(entry.path(), sizeEc)); + cb(name.c_str(), isDir, sizeEc ? 0u : sz, user); } } @@ -485,6 +553,14 @@ bool http_fetch_to_ota(const char* /*url*/, return false; } +bool otaWriteStream(FsWriteSrc /*src*/, void* /*user*/, size_t /*contentLen*/, + char* statusBuf, size_t statusBufLen, uint32_t* bytesReadOut) { + // No OTA partition on desktop — call sites guard with `if constexpr (mm::platform::hasOta)`. + if (statusBuf && statusBufLen > 0) std::snprintf(statusBuf, statusBufLen, "unsupported on desktop"); + if (bytesReadOut) *bytesReadOut = 0; + return false; +} + // Outbound HTTP request (plain HTTP, LAN, no TLS) — see platform.h. Blocking, bounded by a // receive/send timeout. Builds the request into a stack buffer, connects, sends, reads the // response, and returns the status code + the body (after the \r\n\r\n). Used by HueDriver @@ -769,6 +845,54 @@ int TcpConnection::writeSome(const uint8_t* data, size_t len) { } +bool TcpConnection::connectStart(const char* host, uint16_t port) { + if (!host || !host[0]) return false; + close(); + + // One bounded DNS lookup (getaddrinfo) up front — resolving is synchronous, but it's the one + // unavoidable blocking bit; the CONNECT itself then proceeds non-blocking and is polled. + char portStr[6]; + std::snprintf(portStr, sizeof(portStr), "%u", static_cast<unsigned>(port)); + addrinfo hints{}; + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + addrinfo* res = nullptr; + if (::getaddrinfo(host, portStr, &hints, &res) != 0 || !res) return false; + struct AiGuard { addrinfo* p; ~AiGuard() { if (p) ::freeaddrinfo(p); } } aiGuard{res}; + + int fd = open_sock(res->ai_family, res->ai_socktype, res->ai_protocol); + if (fd < 0) return false; + if (make_nonblocking(fd) != 0) { close_sock(fd); return false; } + int cr = ::connect(sock(fd), res->ai_addr, static_cast<int>(res->ai_addrlen)); +#ifdef _WIN32 + const bool inProgress = (cr != 0 && ::WSAGetLastError() == WSAEWOULDBLOCK); +#else + const bool inProgress = (cr != 0 && errno == EINPROGRESS); +#endif + if (cr != 0 && !inProgress) { close_sock(fd); return false; } // immediate hard failure + fd_ = fd; // in flight (or already connected) — connectPoll() resolves which + return true; +} + +TcpConnection::ConnectResult TcpConnection::connectPoll() { + if (fd_ < 0) return ConnectResult::Failed; + // Zero-timeout select: is the socket writable yet? Never blocks. + // Watch BOTH writability and the exception set: a completed connect signals writable on POSIX, + // but a FAILED (refused) non-blocking connect signals via the exception set on Winsock — checking + // only writefds there leaves a refused connect reading Pending until the caller's timeout. + fd_set wf; FD_ZERO(&wf); FD_SET(sock(fd_), &wf); + fd_set ef; FD_ZERO(&ef); FD_SET(sock(fd_), &ef); + timeval zero{}; // 0s / 0us + const int r = ::select(static_cast<int>(sock(fd_)) + 1, nullptr, &wf, &ef, &zero); + if (r == 0) return ConnectResult::Pending; // neither writable nor errored yet + if (r < 0) { close(); return ConnectResult::Failed; } + // SO_ERROR distinguishes a real connect from an errored one on both platforms. + int soerr = 0; socklen_t len = sizeof(soerr); + ::getsockopt(sock(fd_), SOL_SOCKET, SO_ERROR, reinterpret_cast<char*>(&soerr), &len); + if (soerr != 0) { close(); return ConnectResult::Failed; } + return ConnectResult::Connected; // socket stays non-blocking +} + void TcpConnection::close() { if (fd_ >= 0) { close_sock(fd_); diff --git a/src/platform/esp32/platform_esp32.cpp b/src/platform/esp32/platform_esp32.cpp index 55eaee14..d088f39e 100644 --- a/src/platform/esp32/platform_esp32.cpp +++ b/src/platform/esp32/platform_esp32.cpp @@ -67,6 +67,7 @@ #include "freertos/task.h" #include "lwip/sockets.h" #include "lwip/inet.h" +#include "lwip/netdb.h" // getaddrinfo — hostname resolution for TcpConnection::connect #include <atomic> #include <cstdarg> @@ -224,6 +225,19 @@ void getMacAddress(uint8_t mac[6]) { esp_efuse_mac_get_default(mac); } +const char* macString() { + // The base MAC is fixed for the chip's life, so format it once into a static buffer the caller + // can point at (no per-module copy). Not called before the first use, single-threaded init. + static char buf[18] = {}; + if (buf[0] == 0) { + uint8_t mac[6]; + esp_efuse_mac_get_default(mac); + std::snprintf(buf, sizeof(buf), "%02X:%02X:%02X:%02X:%02X:%02X", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + } + return buf; +} + const char* chipModel() { esp_chip_info_t info; esp_chip_info(&info); @@ -1381,6 +1395,45 @@ int TcpConnection::writeSome(const uint8_t* data, size_t len) { } +bool TcpConnection::connectStart(const char* host, uint16_t port) { + if (!host || !host[0]) return false; + close(); + + // One bounded DNS lookup up front (lwip_getaddrinfo is synchronous — the one unavoidable block); + // the CONNECT itself then proceeds non-blocking and is polled across ticks. + char portStr[6]; + std::snprintf(portStr, sizeof(portStr), "%u", static_cast<unsigned>(port)); + struct addrinfo hints = {}; + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo* res = nullptr; + if (lwip_getaddrinfo(host, portStr, &hints, &res) != 0 || !res) return false; + struct AiGuard { struct addrinfo* p; ~AiGuard() { if (p) lwip_freeaddrinfo(p); } } aiGuard{res}; + + int fd = lwip_socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if (fd < 0) return false; + const int flags = fcntl(fd, F_GETFL, 0); + fcntl(fd, F_SETFL, flags | O_NONBLOCK); + int cr = lwip_connect(fd, res->ai_addr, res->ai_addrlen); + const bool inProgress = (cr != 0 && errno == EINPROGRESS); + if (cr != 0 && !inProgress) { lwip_close(fd); return false; } // immediate hard failure + fd_ = fd; // in flight — connectPoll() resolves which + return true; +} + +TcpConnection::ConnectResult TcpConnection::connectPoll() { + if (fd_ < 0) return ConnectResult::Failed; + fd_set wf; FD_ZERO(&wf); FD_SET(fd_, &wf); + struct timeval zero = {}; // 0s / 0us — never blocks + const int r = lwip_select(fd_ + 1, nullptr, &wf, nullptr, &zero); + if (r == 0) return ConnectResult::Pending; + if (r < 0) { close(); return ConnectResult::Failed; } + int soerr = 0; socklen_t len = sizeof(soerr); + lwip_getsockopt(fd_, SOL_SOCKET, SO_ERROR, &soerr, &len); + if (soerr != 0) { close(); return ConnectResult::Failed; } + return ConnectResult::Connected; +} + void TcpConnection::close() { if (fd_ >= 0) { lwip_close(fd_); diff --git a/src/platform/esp32/platform_esp32_fs.cpp b/src/platform/esp32/platform_esp32_fs.cpp index 1079a7a7..1be06f1b 100644 --- a/src/platform/esp32/platform_esp32_fs.cpp +++ b/src/platform/esp32/platform_esp32_fs.cpp @@ -99,6 +99,11 @@ bool fsRemove(const char* path) { if (!fsMounted_) return false; char full[128]; if (!fsTranslate(path, full, sizeof(full))) return false; + // Match the desktop contract (std::filesystem::remove): delete a file OR an empty directory. + // On the LittleFS VFS ::remove() maps to unlink(), which fails on a directory — so stat first + // and route a directory to rmdir() (which itself fails cleanly if the directory isn't empty). + struct stat st; + if (::stat(full, &st) == 0 && S_ISDIR(st.st_mode)) return ::rmdir(full) == 0; return ::remove(full) == 0; } @@ -146,6 +151,55 @@ bool fsWriteAtomic(const char* path, const char* data, size_t len) { return true; } +int fsReadAt(const char* path, long offset, char* buf, size_t len) { + if (!fsMounted_ || !buf) return -1; + char full[128]; + if (!fsTranslate(path, full, sizeof(full))) return -1; + FILE* f = std::fopen(full, "rb"); + if (!f) return -1; + if (std::fseek(f, offset, SEEK_SET) != 0) { std::fclose(f); return -1; } + const size_t n = std::fread(buf, 1, len, f); + std::fclose(f); + return static_cast<int>(n); // 0 at EOF +} + +long fsSize(const char* path) { + if (!fsMounted_) return -1; + char full[128]; + if (!fsTranslate(path, full, sizeof(full))) return -1; + struct stat st; + if (::stat(full, &st) != 0 || S_ISDIR(st.st_mode)) return -1; + return static_cast<long>(st.st_size); +} + +bool fsWriteStream(const char* path, FsWriteSrc src, void* user) { + if (!fsMounted_ || !src) return false; + char full[128]; + char tmp[136]; + if (!fsTranslate(path, full, sizeof(full))) return false; + int n = std::snprintf(tmp, sizeof(tmp), "%s.tmp", full); + if (n < 0 || static_cast<size_t>(n) >= sizeof(tmp)) return false; + + FILE* f = std::fopen(tmp, "wb"); + if (!f) return false; + // Pull chunks from the source and write each straight through — fixed buffer, any file size. + // `abort` set by the source (a short/timed-out upload) means the data is incomplete → discard. + char chunk[1024]; + bool ok = true, abort = false; + for (;;) { + const size_t got = src(chunk, sizeof(chunk), user, &abort); + if (abort) { ok = false; break; } + if (got == 0) break; // clean end of stream + if (std::fwrite(chunk, 1, got, f) != got) { ok = false; break; } + } + std::fflush(f); + int fd = ::fileno(f); + if (fd >= 0) ::fsync(fd); + std::fclose(f); + if (!ok || ::rename(tmp, full) != 0) { ::remove(tmp); return false; } + return true; +} + void fsList(const char* dir, FsListCb cb, void* user) { if (!fsMounted_ || !cb) return; char full[128]; @@ -158,8 +212,10 @@ void fsList(const char* dir, FsListCb cb, void* user) { struct stat st; while ((ent = ::readdir(d)) != nullptr) { std::snprintf(childPath, sizeof(childPath), "%s/%s", full, ent->d_name); - bool isDir = stat(childPath, &st) == 0 && S_ISDIR(st.st_mode); - cb(ent->d_name, isDir, user); + const bool statOk = stat(childPath, &st) == 0; + const bool isDir = statOk && S_ISDIR(st.st_mode); + const uint32_t size = (statOk && !isDir) ? static_cast<uint32_t>(st.st_size) : 0; + cb(ent->d_name, isDir, size, user); } ::closedir(d); } diff --git a/src/platform/esp32/platform_esp32_ota.cpp b/src/platform/esp32/platform_esp32_ota.cpp index 01c9dd5b..3177c99d 100644 --- a/src/platform/esp32/platform_esp32_ota.cpp +++ b/src/platform/esp32/platform_esp32_ota.cpp @@ -52,9 +52,11 @@ void otaTask(void* arg) { *p->bytesReadOut = 0; *p->bytesTotalOut = 0; // unknown until esp_https_ota reports it - // `esp_crt_bundle_attach` enables the bundled-trust-anchor mode for TLS - // verification — the same mechanism Chrome/curl use for general HTTPS - // (api.github.com, objects.githubusercontent.com, …). No baked cert. + // `esp_crt_bundle_attach` enables the bundled-trust-anchor mode for TLS verification — the same + // mechanism Chrome/curl use for general HTTPS (api.github.com, objects.githubusercontent.com, …). + // No baked cert. It's attached unconditionally: for an https URL it verifies the server; for a + // plain-http LAN OTA (MoonDeck serving a local build) it goes unused, but its presence satisfies + // esp_https_ota_begin's "server verification enabled" check, so the fetch proceeds over plain TCP. esp_http_client_config_t http_config = {}; http_config.url = p->url; http_config.crt_bundle_attach = esp_crt_bundle_attach; @@ -185,4 +187,65 @@ bool http_fetch_to_ota(const char* url, return true; } +bool otaWriteStream(FsWriteSrc src, void* user, size_t contentLen, + char* statusBuf, size_t statusBufLen, uint32_t* bytesReadOut) { + if (!src || !statusBuf || statusBufLen == 0 || !bytesReadOut) return false; + auto setStatus = [&](const char* fmt, auto... a) { + std::snprintf(statusBuf, statusBufLen, fmt, a...); + }; + + const esp_partition_t* part = esp_ota_get_next_update_partition(nullptr); + if (!part) { setStatus("error: no OTA partition"); return false; } + + setStatus("flashing"); + esp_ota_handle_t handle = 0; + // OTA_SIZE_UNKNOWN: the upload streams, so we don't pre-declare the exact size (Content-Length + // is advisory for the UI); esp_ota_begin erases lazily as writes arrive. + esp_err_t err = esp_ota_begin(part, OTA_SIZE_UNKNOWN, &handle); + if (err != ESP_OK) { setStatus("error: ota begin %s", esp_err_to_name(err)); return false; } + + // Pull the upload body chunk-by-chunk and write each into the partition — the same producer + // callback fsWriteStream drives, here feeding esp_ota_write instead of a file. `abort` from the + // caller (an incomplete/timed-out upload) fails the OTA, and esp_ota_abort discards the partial. + static uint8_t buf[4096]; // static: keep it off this call's stack (4 KB is large for a task frame) + uint32_t written = 0; + for (;;) { + bool abort = false; + const size_t n = src(reinterpret_cast<char*>(buf), sizeof(buf), user, &abort); + if (abort) { + setStatus("error: upload aborted"); + esp_ota_abort(handle); + return false; + } + if (n == 0) break; // clean EOF — whole body delivered + err = esp_ota_write(handle, buf, n); + if (err != ESP_OK) { + setStatus("error: ota write %s", esp_err_to_name(err)); + esp_ota_abort(handle); + return false; + } + written += static_cast<uint32_t>(n); + *bytesReadOut = written; + } + // Guard a truncated upload: if the client sent fewer bytes than Content-Length, the image is + // incomplete — don't commit a half-image. (contentLen 0 = unknown; skip the check then.) + if (contentLen && written < contentLen) { + setStatus("error: incomplete upload (%u/%u)", + static_cast<unsigned>(written), static_cast<unsigned>(contentLen)); + esp_ota_abort(handle); + return false; + } + + err = esp_ota_end(handle); // validates the image (magic/checksum); consumes the handle + if (err != ESP_OK) { setStatus("error: ota end %s", esp_err_to_name(err)); return false; } + err = esp_ota_set_boot_partition(part); + if (err != ESP_OK) { setStatus("error: set boot %s", esp_err_to_name(err)); return false; } + + setStatus("rebooting"); + // Image committed + boot pointer flipped. Return to the caller so it can send its HTTP 200 + // BEFORE the reboot (the caller closes the socket + reboots, same sequence as /api/reboot) — + // that's what lets the browser see a clean "flashed" response instead of an aborted socket. + return true; +} + } // namespace mm::platform diff --git a/src/platform/platform.h b/src/platform/platform.h index 8f2ba45a..627a79ab 100644 --- a/src/platform/platform.h +++ b/src/platform/platform.h @@ -66,6 +66,10 @@ size_t totalInternalHeap(); // total internal heap capacity constexpr size_t HEAP_RESERVE = 32768; void getMacAddress(uint8_t mac[6]); +// The MAC as canonical "XX:XX:XX:XX:XX:XX", formatted once into a static buffer — a stable per-chip +// identity string a caller can point at without keeping its own copy. (chipModel/sdkVersion likewise +// return static strings; a ReadOnly control binds straight to these, storing nothing per-module.) +const char* macString(); const char* chipModel(); const char* sdkVersion(); @@ -108,9 +112,23 @@ bool fsMkdir(const char* path); // mkdir -p; no err bool fsExists(const char* path); bool fsRemove(const char* path); // file or empty dir int fsRead(const char* path, char* buf, size_t maxLen); // bytes read; -1 on error; null-terminated on success +long fsSize(const char* path); // file size in bytes; -1 if missing/not a file +int fsReadAt(const char* path, long offset, char* buf, size_t len); + // pread-style: read up to `len` bytes at `offset`; bytes read, 0 at EOF, -1 on error. NOT null-terminated. Lets a caller stream a file in fixed chunks. bool fsWriteAtomic(const char* path, const char* data, size_t len); // writes <path>.tmp, fsync, rename. Caller ensures parent dir exists. -using FsListCb = void(*)(const char* name, bool isDir, void* user); +// Streamed atomic write: open <path>.tmp, repeatedly call src(buf, cap, user, &abort) to pull up to +// `cap` bytes, fwrite each chunk, then fsync + rename into place. The source returns the byte count; +// 0 means *end*, BUT it distinguishes a clean end from a failed/incomplete one via `*abort`: a 0 +// with `*abort == false` is a clean EOF (commit), a 0 (or any return) with `*abort == true` is an +// error (a short/timed-out upload) → the temp file is DISCARDED, not renamed. Returns false on abort, +// a write failure, or a rename failure. Lets the HTTP layer stream an upload of any size with a fixed +// small buffer — the device never holds the whole file in RAM. Caller ensures the parent dir exists. +using FsWriteSrc = size_t(*)(char* buf, size_t cap, void* user, bool* abort); +bool fsWriteStream(const char* path, FsWriteSrc src, void* user); +// Per-entry callback for fsList: name, whether it's a directory, and its size in bytes +// (0 for a directory). One level, one call per child. +using FsListCb = void(*)(const char* name, bool isDir, uint32_t sizeBytes, void* user); void fsList(const char* dir, FsListCb cb, void* user); // single-level listing // Network (ESP32 only, stubs on desktop) @@ -209,6 +227,22 @@ bool http_fetch_to_ota(const char* url, char* statusBuf, size_t statusBufLen, uint32_t* bytesReadOut, uint32_t* bytesTotalOut); +// OTA — flash a firmware image STREAMED from `src` (no URL fetch; the caller pulls the bytes, +// e.g. straight off an HTTP upload body). Same producer callback shape as fsWriteStream: `src` +// fills up to `cap` bytes, returns the count (0 = clean EOF), and sets `*abort` to fail the OTA +// (an incomplete/timed-out upload). Runs esp_ota_begin → esp_ota_write per chunk → esp_ota_end + +// set_boot_partition, then RETURNS true (it does NOT reboot — the caller sends its HTTP 200 first, +// then reboots into the flashed image, the same order /api/reboot uses). SYNCHRONOUS (unlike +// http_fetch_to_ota, which runs on its own task): the caller is the HTTP request handler, which runs +// on the loop20ms tick INSIDE Scheduler::tick — so this blocks rendering for the flash duration. That +// is the accepted trade-off (a firmware upload is user-initiated and reboots the device on success), +// bounded by the same upload idle/hard limits; the caller needs the result to reply. +// `statusBuf` / `bytesReadOut` are updated in place (bytesTotal is the caller-supplied +// Content-Length, so no separate out-param). Desktop: returns false (no OTA partition); guard with +// `if constexpr (mm::platform::hasOta)`. Returns true iff the image flashed + boot pointer flipped. +bool otaWriteStream(FsWriteSrc src, void* user, size_t contentLen, + char* statusBuf, size_t statusBufLen, uint32_t* bytesReadOut); + // Synchronous outbound HTTP request to a LAN host — plain HTTP, no TLS (the Philips Hue v1 // API, which HueDriver drives, allows it). Connects to `host:port`, sends `method path` // with `reqBody` (NUL-terminated; "" for none — a Content-Length + JSON content-type are @@ -317,6 +351,17 @@ class TcpConnection { return *this; } + // Non-blocking outbound connect to host:port, for a client that must NOT stall the render loop + // (MQTT runs on loop1s inside Scheduler::tick). `connectStart` resolves `host` (a hostname via + // getaddrinfo — one bounded DNS lookup — or a dotted-quad IP) and kicks off a non-blocking + // connect, returning immediately; `connectPoll` checks the in-flight connect WITHOUT blocking and + // returns Pending / Connected / Failed. The caller polls across ticks and enforces its own overall + // timeout, then reads/writes via the non-blocking read()/writeSome(). Caller gates on + // networkReady(). Desktop + ESP32 (lwip); a fresh TcpConnection (no fd yet) is the receiver. + enum class ConnectResult : uint8_t { Pending, Connected, Failed }; + bool connectStart(const char* host, uint16_t port); // false = immediate failure (DNS/socket) + ConnectResult connectPoll(); // non-blocking; call after connectStart + bool valid() const { return fd_ >= 0; } int read(uint8_t* buf, size_t maxLen); // non-blocking: >0 data, 0 closed, -1 nothing bool write(const uint8_t* data, size_t len); // blocking — sends all bytes (HTTP responses must complete) diff --git a/src/ui/app.js b/src/ui/app.js index 61138bdb..8fd579e4 100644 --- a/src/ui/app.js +++ b/src/ui/app.js @@ -175,6 +175,17 @@ async function init() { preview.setupLayout(); } +// The message for a failed fetch Response: the server's own `{"error": …}` body (JSON, e.g. +// "not enough space (N free)") when present, else a bare `HTTP <status>`. Every /api/* handler +// returns errors as that JSON shape, so this is the one place the extraction lives. +async function errorMessage(res) { + try { + const j = await res.json(); + if (j && j.error) return j.error; + } catch (_) { /* non-JSON error body — fall through to the status code */ } + return `HTTP ${res.status}`; +} + async function sendControl(moduleName, controlName, value) { // Best-effort by design — failures are not retried here. Non-ok responses + // network errors are logged to console so a user with devtools open can see @@ -509,7 +520,7 @@ function createCard(mod, depth) { // Help link → the module's spec page on the rendered docs site, far right of // the row. docPath comes from /api/types (relative to docs/moonmodules/, e.g. - // "core/AudioModule.md" or "light/effects/effects.md#fire"); omitted if none. + // "core/ui/ui.md#audio" or "light/effects/effects.md#fire"); omitted if none. // The site is Material for MkDocs at moonmodules.org/projectMM/ (flat URLs, so // foo.md → foo.html; the MkDocs heading slugs match these #anchors), reached // via the same /projectMM/ subpath the installer uses. Convert only the `.md` @@ -540,7 +551,10 @@ function createCard(mod, depth) { // Network) never collapses its own controls, even though it accepts children // (System hosts peripherals). It's the card the user is looking at, so its // settings should be visible, not hidden behind a "controls" disclosure. - const hasVisibleControls = mod.controls && mod.controls.some(c => !c.hidden); + // Use the SAME predicate the row loop renders by (controlRendersGenerically), not a bare + // !c.hidden — else the disclosure could open for a module whose only "visible" controls render + // elsewhere (e.g. FileManager's filesystem/lastSaved render inside its own panel, not generically). + const hasVisibleControls = mod.controls && mod.controls.some(c => controlRendersGenerically(mod, c)); const wrapInDetails = depth > 0 && acceptsNewChildren(mod) && hasVisibleControls; const controlsHost = wrapInDetails ? (() => { const d = document.createElement("details"); @@ -569,12 +583,19 @@ function createCard(mod, depth) { if (mod.controls) { for (const ctrl of mod.controls) { - if (ctrl.hidden) continue; // plan-10 hidden flag (still respected) + if (!controlRendersGenerically(mod, ctrl)) continue; const row = createControl(mod.name, mod.type, ctrl); if (row) controlsHost.appendChild(row); } } + // File Manager: a modern expand/collapse folder tree + inline text editor. Browsing is UI-side + // over /api/dir; a create/delete is a POST/DELETE /api/dir?path= call (the path rides the request, + // no persisted control). Only the `show hidden` toggle renders as a raw control; the tree is the rest. + if (mod.type === "FileManagerModule") { + renderFileManager(mod, controlsHost); + } + // FirmwareUpdate card hosts the shared install picker. Mount once per // card-build. The picker reads SystemModule.firmware (already in // /api/state) to filter to OTA-compatible releases. On install, the @@ -607,16 +628,54 @@ function createCard(mod, depth) { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ url: binaryUrl }), }); - if (!res.ok) { - let msg = `HTTP ${res.status}`; - try { - const j = await res.json(); - if (j.error) msg = j.error; - } catch (_) { /* non-JSON error response */ } - throw new Error(msg); - } + if (!res.ok) throw new Error(await errorMessage(res)); }, }); + + // Install-from-file: pick a locally-built .bin and stream its bytes to /api/firmware/upload, + // which feeds platform::otaWriteStream (esp_ota_write). Complements the release picker above — + // the release path has the device fetch a URL; this path has the browser push the body, so a + // dev build that isn't a published release can be flashed straight over the network. The device + // reboots on success (no response body), so a completed POST that closes mid-flight is success. + const fileRow = document.createElement("div"); + fileRow.className = "fw-upload-row"; + const upBtn = document.createElement("button"); + upBtn.className = "fm-tool fm-tool--icon"; + upBtn.textContent = "↥"; // same upload glyph as the file-manager toolbar + upBtn.title = "Install from file — flash a firmware .bin from your computer over the network (OTA)"; + const upStatus = document.createElement("span"); + upStatus.className = "fw-upload-status"; + const upInput = document.createElement("input"); + upInput.type = "file"; + upInput.accept = ".bin,application/octet-stream"; + upInput.style.display = "none"; + upInput.addEventListener("change", async () => { + const file = (upInput.files || [])[0]; + upInput.value = ""; // reset so re-picking the same file re-fires change + if (!file) return; + upBtn.disabled = true; + upStatus.textContent = `uploading ${fmSize(file.size)}…`; + try { + const res = await fetch("/api/firmware/upload", { + method: "POST", headers: { "Content-Type": "application/octet-stream" }, body: file, + }); + // The device sends a 200 once the image is committed, THEN reboots — so res.ok is the + // real success signal. A 4xx/5xx carries the device's ota error in the JSON body. + if (res.ok) upStatus.textContent = "flashed — device rebooting"; + else throw new Error(await errorMessage(res)); + } catch (err) { + // A network error mid-upload is a genuine failure now (the device answers before it + // reboots), so surface it as such rather than assuming a reboot. + upStatus.textContent = "upload failed: " + err.message; + } finally { + upBtn.disabled = false; + } + }); + upBtn.addEventListener("click", () => upInput.click()); + fileRow.appendChild(upBtn); + fileRow.appendChild(upStatus); + fileRow.appendChild(upInput); + controlsHost.appendChild(fileRow); } // -- Children block + footer -- @@ -679,6 +738,10 @@ function armPressTwice(btn, onConfirm, opts = {}) { let savedText = ""; let savedTitle = ""; const disarm = () => { + // Only restore label/title if we actually armed — otherwise a mouseleave before the first + // click would write the initial empty savedText/savedTitle over the button, collapsing a + // text-sized button (e.g. "🗑 delete") to an empty sliver. + if (!armed) return; armed = false; btn.classList.remove("armed"); if (opts.armedText !== undefined) btn.textContent = savedText; @@ -873,6 +936,15 @@ function emojiTagsForMod(mod) { return emojiTagsFor(t).join(""); } +// Whether a control appears in the generic control list — false for controls the module marked +// `hidden`. A module that renders a control in its own panel instead (e.g. the File Manager's +// `filesystem` usage bar + `lastSaved` readout, drawn by renderFileManager) sets the hidden flag on +// the C++ side, so the generic list skips it and the value never shows twice. Used by BOTH render +// paths (renderCards's initial build + updateModuleControls's WS live-patch) so they agree. +function controlRendersGenerically(mod, ctrl) { + return !ctrl.hidden; +} + function createControl(moduleName, moduleType, ctrl) { const row = document.createElement("div"); row.className = "control-row"; @@ -1096,7 +1168,7 @@ function createControl(moduleName, moduleType, ctrl) { input.value = ctrl.value ?? ""; input.dataset.mid = moduleName; input.dataset.key = ctrl.name; - input.rows = 1; // default compact; CSS height + resize grip control size + input.rows = 2; // default 2 lines; CSS height + resize grip control size input.spellcheck = false; if (ctrl.readonly) { input.readOnly = true; @@ -1733,7 +1805,7 @@ function syncVisibleControls(mod) { // other's rows in a loop — tearing down (and closing) any open <select>. const host = card.querySelector(":scope > .card-controls-collapse") || card; - const wantNames = mod.controls.filter(c => !c.hidden).map(c => c.name); + const wantNames = mod.controls.filter(c => controlRendersGenerically(mod, c)).map(c => c.name); const haveRows = [...host.querySelectorAll(":scope > .control-row[data-key]")]; const haveNames = haveRows.map(r => r.dataset.key); if (wantNames.length === haveNames.length && wantNames.every((n, i) => n === haveNames[i])) { @@ -1749,7 +1821,7 @@ function syncVisibleControls(mod) { // first existing control row that should come AFTER this one; null → append // before the children block (insertBefore(node, null) appends to host's end, // but control rows precede .card-children which lives on the card, not here). - const visibleControls = mod.controls.filter(c => !c.hidden); + const visibleControls = mod.controls.filter(c => controlRendersGenerically(mod, c)); for (let i = 0; i < visibleControls.length; i++) { const name = visibleControls[i].name; if (host.querySelector(`:scope > .control-row[data-key="${cssEscape(name)}"]`)) continue; @@ -2573,6 +2645,547 @@ function setupUpdateBadge() { }); } +// --------------------------------------------------------------------------- +// 8b. File Manager view +// --------------------------------------------------------------------------- + +// Format a byte count as a short human size (matches the fs seam's uint32 sizes). +function fmSize(n) { + if (n < 1024) return n + " B"; + if (n < 1024 * 1024) return (n / 1024).toFixed(1) + " KB"; + return (n / (1024 * 1024)).toFixed(1) + " MB"; +} + +// Per-module File Manager UI state, kept across state refreshes (the DOM is rebuilt on refetch, but +// the tree's expand-state + selection are the user's, not the module's). Keyed by module name. +const fmStateByMod = {}; +function fmState(mod) { + return (fmStateByMod[mod.name] ||= { expanded: new Set(["/"]), selected: "/" }); +} + +// Fetch one directory's children (name/isDir/size) from /api/dir. `hidden` includes dotfiles. +async function fmFetchDir(absPath, hidden) { + const res = await fetch("/api/dir?path=" + encodeURIComponent(absPath) + (hidden ? "&hidden=1" : "")); + if (!res.ok) throw new Error(await errorMessage(res)); + const rows = await res.json(); + return Array.isArray(rows) ? rows : []; +} + +// Render the File Manager panel: a lazy expand/collapse folder tree (the standard VS Code / Explorer +// shape — an expanded folder's children are loaded from /api/dir), plus a toolbar (show +// hidden, new folder, delete on the selected node). Filesystem ops go through the module's controls +// (path/new folder/delete); browsing is pure UI over /api/dir, so the module stays minimal. +function renderFileManager(mod, host) { + const ctrl = (n) => (mod.controls || []).find(c => c.name === n); + const st = fmState(mod); + // The toggle is UI-owned: seed it from the persisted control on first render, then `st` is the + // source of truth. Reading it back from `mod` each render would revert the checkbox, since an + // in-panel re-render runs against the same (stale) state snapshot, before /api/state updates. + if (st.showHidden === undefined) st.showHidden = !!ctrl("show hidden")?.value; + const hidden = st.showHidden; + + // Render into a single stable panel that we REPLACE on every re-render (expand / collapse / + // select / op) — reusing it in place rather than appending, so the tree updates inline instead + // of stacking a fresh copy below the old one. + let panel = host.querySelector(":scope > .fm-panel"); + if (panel) panel.replaceChildren(); + else { panel = document.createElement("div"); panel.className = "fm-panel"; host.appendChild(panel); } + + // Show-hidden toggle: owned by the panel (not the generic control list) so flipping it can + // re-fetch the tree with the new `hidden` filter immediately, rather than waiting for a state + // refresh that never re-runs the /api/dir fetch. Reuses the same .switch pill markup every + // other bool control uses (common patterns first), so it reads consistently. + const hiddenRow = document.createElement("label"); + hiddenRow.className = "fm-hidden-toggle"; + hiddenRow.appendChild(document.createTextNode("show hidden")); + const sw = document.createElement("span"); + sw.className = "switch"; + const hiddenBox = document.createElement("input"); + hiddenBox.type = "checkbox"; + hiddenBox.checked = hidden; + hiddenBox.addEventListener("change", () => { + st.showHidden = hiddenBox.checked; // UI-owned source of truth + sendControl(mod.name, "show hidden", hiddenBox.checked); // persist (best-effort, no await) + renderFileManager(mod, host); // re-list with the new filter + }); + const track = document.createElement("span"); + track.className = "switch-track"; + sw.appendChild(hiddenBox); + sw.appendChild(track); + hiddenRow.appendChild(sw); + panel.appendChild(hiddenRow); + + // Select a path level (from a breadcrumb crumb): update the selection, tidy the tree to just the + // path, and re-render. "Reveal and collapse siblings" — keep the ancestor chain root→…→crumb + // open, fold every other branch AND the clicked node's own descendants. Selecting a directory + // drives where ＋ folder/＋ file create and what delete targets. + const selectPath = (absPath, isDir) => { + st.selected = absPath; + st.selectedIsDir = isDir; + // Rebuild `expanded` as the ancestor chain root→…→crumb, plus the clicked node itself when + // it's a directory (so you see one level into it) — every other branch and anything deeper + // folds. Root is always expanded (the tree shows its children). + const segs = absPath.split("/").filter(Boolean); // "/.config/foo" → [".config","foo"] + st.expanded = new Set(["/"]); + for (let i = 0; i < segs.length; i++) { + const p = "/" + segs.slice(0, i + 1).join("/"); + // Include every ancestor; include the clicked node only if it's a directory. + if (i < segs.length - 1 || isDir) st.expanded.add(p); + } + renderFileManager(mod, host); + }; + + // Breadcrumb of the selected path, on its OWN row above the toolbar (a deep path wraps freely + // without crowding the buttons — key on narrow displays). Click a crumb to jump there; `root` + // deselects back to /. + const crumbs = document.createElement("div"); + crumbs.className = "fm-crumbs"; + const mkCrumb = (label, absPath, isDir) => { + const b = document.createElement("button"); + b.className = "fm-crumb" + (absPath === st.selected ? " fm-crumb--here" : ""); + b.textContent = label; + b.addEventListener("click", () => selectPath(absPath, isDir)); + return b; + }; + // (Re)build the breadcrumb from the current selection, into the stable `crumbs` element — called + // on first render and by refreshSelectionControls() when a file click updates the selection in place. + const rebuildCrumbs = () => { + crumbs.replaceChildren(); + crumbs.appendChild(mkCrumb("root", "/", true)); // always present — the way back to / + const segs = st.selected.split("/").filter(Boolean); // "/.config/foo" → [".config","foo"] + segs.forEach((s, i) => { + crumbs.appendChild(document.createTextNode(" / ")); + // A crumb is a directory unless it's the last segment of a selected *file*. + const isLast = i === segs.length - 1; + const isDir = !isLast || st.selectedIsDir; + crumbs.appendChild(mkCrumb(s, "/" + segs.slice(0, i + 1).join("/"), isDir)); + }); + }; + rebuildCrumbs(); + panel.appendChild(crumbs); + + // Refresh the controls that depend on the current selection WITHOUT rebuilding the tree: the + // breadcrumb and the delete button's disabled state. Used by the in-place file-click path so a + // file's row survives for its dblclick. (delBtn's press-twice handler reads st.selected live.) + const refreshSelectionControls = () => { + rebuildCrumbs(); + delBtn.disabled = st.selected === "/"; + }; + + // Toolbar (own row below the breadcrumb): New folder / New file / Delete / Refresh. + const bar = document.createElement("div"); + bar.className = "fm-bar"; + + // A filesystem op is an HTTP call on /api/dir?path= (POST=mkdir, DELETE=remove) — the path + // rides the query, so nothing is stored/persisted on the device. On failure, surface the + // server's error; on success, re-render the tree from disk. + const runOp = async (op, targetPath) => { + const method = op === "delete" ? "DELETE" : "POST"; + try { + const res = await fetch("/api/dir?path=" + encodeURIComponent(targetPath), { method }); + if (!res.ok) alert(`${op} failed: ${await errorMessage(res)}`); + } catch (err) { // a network error (offline / reset) would otherwise be an unhandled rejection + alert(`${op} failed: ${err.message || err}`); + } + renderFileManager(mod, host); // rebuild the tree from /api/dir (fresh listing), success or handled failure + }; + + // A new file/folder is created inside the selected node if it's a folder, else next to it (in + // the selected file's parent). Shared by both create buttons. + const createBase = () => (st.selectedIsDir ? st.selected : fmParent(st.selected)); + + // Icon-only toolbar: each button shows a glyph; the `title` carries the word for the tooltip + + // screen readers (the standard icon-button pattern — an icon with an accessible label). + const newBtn = document.createElement("button"); + newBtn.className = "fm-tool fm-tool--icon"; + newBtn.textContent = "📁"; + newBtn.title = "New folder — create a folder inside the selected folder"; + newBtn.addEventListener("click", async () => { + const base = createBase(); + const name = (prompt("New folder name in " + base + ":") || "").trim(); + if (!name) return; // blank or whitespace-only → no-op + st.expanded.add(base); // reveal the new child + await runOp("new folder", joinFsPath(base, name)); + }); + bar.appendChild(newBtn); + + // New file: no module op needed — the /api/file POST creates a file at a path (empty body), the + // same endpoint the editor saves through. Then re-render the tree from disk. + const newFileBtn = document.createElement("button"); + newFileBtn.className = "fm-tool fm-tool--icon"; + newFileBtn.textContent = "📝"; + newFileBtn.title = "New file — create an empty file inside the selected folder"; + newFileBtn.addEventListener("click", async () => { + const base = createBase(); + const name = (prompt("New file name in " + base + ":") || "").trim(); + if (!name) return; // blank or whitespace-only → no-op + const filePath = joinFsPath(base, name); + try { + const res = await fetch("/api/file?path=" + encodeURIComponent(filePath), { + method: "POST", headers: { "Content-Type": "text/plain" }, body: "", + }); + if (!res.ok) throw new Error(await errorMessage(res)); + } catch (err) { + alert("create file failed: " + err.message); + return; + } + st.expanded.add(base); // reveal the new file + renderFileManager(mod, host); // re-list from disk + }); + bar.appendChild(newFileBtn); + + // Upload: pick desktop files and stream them into the selected folder via the same /api/file + // POST the drag-drop path uses (fmDropUpload) — a button for people who don't drag. A hidden + // <input type=file multiple> is the recognizable browser file-picker; clicking the button opens it. + const upBtn = document.createElement("button"); + upBtn.className = "fm-tool fm-tool--icon"; + upBtn.textContent = "↥"; // pairs with the per-row ↓ download glyph + upBtn.title = "Upload — upload files from your computer into the selected folder"; + const upInput = document.createElement("input"); + upInput.type = "file"; + upInput.multiple = true; + upInput.style.display = "none"; + upInput.addEventListener("change", async () => { + const files = Array.from(upInput.files || []); + upInput.value = ""; // reset so re-picking the same file re-fires change + if (!files.length) return; + const base = createBase(); + st.expanded.add(base); // reveal the destination folder + const skipped = await fmDropUpload(base, files); + renderFileManager(mod, host); // re-list from disk + if (skipped.length) alert("Not uploaded:\n" + skipped.join("\n")); + }); + upBtn.addEventListener("click", () => upInput.click()); + bar.appendChild(upBtn); + bar.appendChild(upInput); + + const delBtn = document.createElement("button"); + delBtn.className = "fm-tool fm-tool--icon fm-tool--danger"; + delBtn.textContent = "🗑"; + delBtn.title = "Delete — delete the selected file or empty folder"; + delBtn.disabled = st.selected === "/"; // never delete the root + armPressTwice(delBtn, () => runOp("delete", st.selected), { armedText: "✓" }); + bar.appendChild(delBtn); + + const refBtn = document.createElement("button"); + refBtn.className = "fm-tool fm-tool--icon"; + refBtn.textContent = "⟳"; + refBtn.title = "Refresh"; + refBtn.addEventListener("click", () => renderFileManager(mod, host)); + bar.appendChild(refBtn); + panel.appendChild(bar); + + // The tree. Root ("/") is always present and expanded; its children populate asynchronously. + const tree = document.createElement("div"); + tree.className = "fm-tree"; + // Dropping desktop files onto the tree's empty space uploads them into root. + fmMakeDropTarget(tree, "/", hidden, () => renderFileManager(mod, host), st); + panel.appendChild(tree); + + // Render one directory's children into `container` at `depth`, recursing into expanded folders. + const renderChildren = async (dirPath, container, depth) => { + let rows; + try { + rows = await fmFetchDir(dirPath, hidden); + } catch (err) { + const e = document.createElement("div"); + e.className = "fm-empty"; + e.textContent = "list failed: " + err.message; + container.appendChild(e); + return; + } + if (rows.length === 0) { + const e = document.createElement("div"); + e.className = "fm-empty"; + e.style.paddingLeft = (depth * 16 + 20) + "px"; + e.textContent = "empty"; + container.appendChild(e); + return; + } + // Folders first, then files; each alphabetical — the conventional file-manager sort. + rows.sort((a, b) => (b.isDir - a.isDir) || a.name.localeCompare(b.name)); + for (const entry of rows) { + const childPath = joinFsPath(dirPath, entry.name); + const isOpen = st.expanded.has(childPath); + + const rowEl = document.createElement("div"); + rowEl.className = "fm-row" + (childPath === st.selected ? " fm-row--sel" : ""); + rowEl.style.paddingLeft = (depth * 16 + 4) + "px"; + + // Chevron: only folders can expand; a file gets a spacer so names line up. + const chev = document.createElement("span"); + chev.className = "fm-chev"; + chev.textContent = entry.isDir ? (isOpen ? "▾" : "▸") : ""; + rowEl.appendChild(chev); + + const icon = document.createElement("span"); + icon.className = "fm-icon"; + icon.textContent = entry.isDir ? (isOpen ? "📂" : "📁") : "📄"; + rowEl.appendChild(icon); + + const name = document.createElement("span"); + name.className = "fm-name"; + name.textContent = entry.name; + rowEl.appendChild(name); + + const size = document.createElement("span"); + size.className = "fm-size"; + size.textContent = entry.isDir ? "" : fmSize(entry.size || 0); + rowEl.appendChild(size); + + // Per-file download (device → desktop): a plain <a download> on /api/file forces a save + // with the right name, every browser, any file type — the portable counterpart to the + // drag-drop upload (a true drag-*out* has no cross-browser API). Folders get no ⤓; + // folder-as-zip is backlogged (needs a bundled zip lib + recursion). + if (!entry.isDir) { + const dl = document.createElement("a"); + dl.className = "fm-dl"; + dl.textContent = "⤓"; + dl.title = "download"; + dl.href = "/api/file?path=" + encodeURIComponent(childPath); + dl.setAttribute("download", entry.name); + dl.addEventListener("click", (ev) => ev.stopPropagation()); // don't select/open + rowEl.appendChild(dl); + } + + // Single-click SELECTS (highlights, sets the op target for delete); a folder also + // toggles expand. DOUBLE-click OPENS a file in the editor — the VS Code / Finder / + // Explorer norm, and it keeps a distinct "selected but not opened" state for future + // rename / multi-select / context-menu features. + rowEl.addEventListener("click", (ev) => { + ev.stopPropagation(); + st.selected = childPath; + st.selectedIsDir = entry.isDir; + if (entry.isDir) { + // A folder click also expands/collapses — rows appear/disappear, so a full + // re-render is needed. + if (isOpen) st.expanded.delete(childPath); + else st.expanded.add(childPath); + renderFileManager(mod, host); + } else { + // A file click only moves the selection — update the highlight IN PLACE, never + // re-render. A re-render here would destroy this row mid-gesture, so the dblclick + // that follows a double-click would land on a fresh element and openFileEditor + // might not fire. Move the --sel class + refresh the selection-dependent controls. + for (const r of panel.querySelectorAll(".fm-row--sel")) r.classList.remove("fm-row--sel"); + rowEl.classList.add("fm-row--sel"); + refreshSelectionControls(); + } + }); + if (!entry.isDir) { + rowEl.addEventListener("dblclick", (ev) => { + ev.stopPropagation(); + openFileEditor(childPath, entry.size); // size lets the editor detect a truncated read + }); + } + + // Drag-drop upload target: dropping desktop files onto a FOLDER row uploads them into + // that folder (tier 1: text/config ≤8KB — see fmDropUpload). + if (entry.isDir) fmMakeDropTarget(rowEl, childPath, hidden, () => renderFileManager(mod, host), st); + container.appendChild(rowEl); + + // Recurse into an expanded folder (its own indented sub-container). + if (entry.isDir && isOpen) { + const sub = document.createElement("div"); + sub.className = "fm-subtree"; + container.appendChild(sub); + await renderChildren(childPath, sub, depth + 1); + } + } + }; + + renderChildren("/", tree, 0); + + // Filesystem usage bar below the tree — the File Manager's own `filesystem` control (used / + // total bytes from the platform). Absent (e.g. desktop fs total 0) → nothing shown. + const fsCtrl = fmFilesystemUsage(mod); + if (fsCtrl) { + const usage = document.createElement("div"); + usage.className = "fm-usage"; + const name = document.createElement("span"); + name.className = "fm-usage-name"; + name.textContent = "Used"; // the bar/value is used space out of total (see the trailing label) + const bar = document.createElement("progress"); + bar.value = Number(fsCtrl.value) || 0; + bar.max = Number(fsCtrl.total) || 1; + const lbl = document.createElement("span"); + lbl.className = "fm-usage-label"; + lbl.textContent = fmtProgressLabel(fsCtrl); + usage.appendChild(name); + usage.appendChild(bar); + usage.appendChild(lbl); + panel.appendChild(usage); + } + + // "last saved" readout below the usage bar — how long ago config was persisted. The value is + // owned by the FilesystemModule engine (non-UI); the File Manager displays it because this is + // where filesystem state is topical (same reasoning as the usage bar). + const savedCtrl = (mod?.controls || []).find(c => c.name === "lastSaved"); + if (savedCtrl) { + const row = document.createElement("div"); + row.className = "fm-lastsaved"; + row.textContent = `saved: ${savedCtrl.value ?? "never"}`; + panel.appendChild(row); + } +} + +// The File Manager's own `filesystem` usage progress control (used/total bytes), or null if the +// platform reports no partition. Rendered as the bar below the tree; skipped in the generic control +// loop so it appears only here. +function fmFilesystemUsage(mod) { + return (mod?.controls || []).find(c => c.name === "filesystem") || null; +} + +// Format a file's text for the editor, by extension. JSON is re-indented (2 spaces) so the persisted +// config files are readable; anything that doesn't parse is shown verbatim rather than erroring. +// Extension seam for later: MoonLive `.ml` source wants syntax *highlighting* (a colour layer over +// the textarea), not reformatting — that's a bigger editor change, added when MoonLive `.ml` files +// land on disk. +function fmPrettify(text, relPath) { + if (relPath.endsWith(".json")) { + try { return JSON.stringify(JSON.parse(text), null, 2); } catch (_) {} + } + return text; +} + +// The parent directory of an absolute path ("/a/b" → "/a", "/a" → "/"). +function fmParent(absPath) { + const cut = absPath.lastIndexOf("/"); + return cut <= 0 ? "/" : absPath.slice(0, cut); +} + +// Join a dir + name with one slash (UI-side path building for the editor's path= query). +function joinFsPath(dir, name) { + return dir.endsWith("/") ? dir + name : dir + "/" + name; +} + +// Drag-drop upload (desktop → device). The device streams the body straight to the file (any size, +// binary-safe), so the only client-side bound is a sanity cap matching the device's kUploadMax; a +// file over it is skipped with a visible note (no silent drop). A too-big-for-free-space file is +// also rejected device-side with a "not enough space" message. +const FM_UPLOAD_CAP = 256 * 1024; // matches HttpServerModule::kUploadMax + +// Wire an element as a drop target that uploads dropped files into `destDir`, then re-renders. +function fmMakeDropTarget(el, destDir, hidden, rerender, st) { + el.addEventListener("dragover", (e) => { + e.preventDefault(); + e.stopPropagation(); + el.classList.add("fm-row--drop"); + }); + el.addEventListener("dragleave", (e) => { + e.stopPropagation(); + el.classList.remove("fm-row--drop"); + }); + el.addEventListener("drop", async (e) => { + e.preventDefault(); + e.stopPropagation(); + el.classList.remove("fm-row--drop"); + const files = Array.from(e.dataTransfer?.files || []); + if (!files.length) return; + const skipped = await fmDropUpload(destDir, files); + if (destDir !== "/") st.expanded.add(destDir); // reveal where they landed + rerender(); + if (skipped.length) { + // Report what wasn't uploaded (over the size cap or a write error) rather than dropping + // it silently. The limit is derived from FM_UPLOAD_CAP so the text never drifts from it. + alert(`Not uploaded (over ${fmSize(FM_UPLOAD_CAP)} or failed):\n` + skipped.join("\n")); + } + }); +} + +// Upload each dropped file into destDir via /api/file. Returns the names skipped (too big / failed) +// so the caller can report them. The File blob is sent as the body directly — the browser streams +// its raw bytes (binary-safe), matching the device's streamed, byte-exact write. +async function fmDropUpload(destDir, files) { + const skipped = []; + for (const file of files) { + if (file.size > FM_UPLOAD_CAP) { skipped.push(file.name + " (" + fmSize(file.size) + ")"); continue; } + try { + const res = await fetch("/api/file?path=" + encodeURIComponent(joinFsPath(destDir, file.name)), { + method: "POST", headers: { "Content-Type": "application/octet-stream" }, body: file, + }); + if (!res.ok) throw new Error(await errorMessage(res)); // surfaces "not enough space (N free)" etc. + } catch (err) { + skipped.push(file.name + " (" + err.message + ")"); + } + } + return skipped; +} + +// Open a modal text editor for the file at `relPath`. Loads via GET /api/file (streamed whole, any +// size), saves via POST. A file that isn't valid text (a NUL byte, or UTF-8 decode damage) loads +// read-only so a lossy re-save can't corrupt it. Uses the native <dialog> — no bespoke overlay code. +async function openFileEditor(relPath, expectedSize) { + const dlg = document.createElement("dialog"); + dlg.className = "fm-editor"; + dlg.innerHTML = + '<form method="dialog" class="fm-editor-head">' + + ' <span class="fm-editor-path"></span>' + + ' <button value="close" class="fm-editor-x" title="close">✕</button>' + + '</form>' + + '<textarea class="fm-editor-body" spellcheck="false"></textarea>' + + '<div class="fm-editor-foot">' + + ' <span class="fm-editor-status"></span>' + + ' <button class="action-btn fm-editor-save">Save</button>' + + '</div>'; + dlg.querySelector(".fm-editor-path").textContent = relPath; + const body = dlg.querySelector(".fm-editor-body"); + const status = dlg.querySelector(".fm-editor-status"); + const saveBtn = dlg.querySelector(".fm-editor-save"); + document.body.appendChild(dlg); + dlg.addEventListener("close", () => dlg.remove()); + dlg.showModal(); + + try { + const res = await fetch("/api/file?path=" + encodeURIComponent(relPath)); + // Surface the server's own message (e.g. "not found") rather than a bare status code. + if (!res.ok) throw new Error(await errorMessage(res)); + const text = await res.text(); + // Truncation guard: serveFileContents streams the whole file but stops short on a read error + // (a filesystem fault mid-stream). Saving a short read back would overwrite the file with a + // truncated copy — so if the received byte count is under the size the listing reported, load + // read-only. TextEncoder gives the byte length (text.length is chars, not bytes). + if (typeof expectedSize === "number" && + new TextEncoder().encode(text).length < expectedSize) { + body.value = text; + body.readOnly = true; + saveBtn.disabled = true; + status.textContent = "truncated read — read-only (save would corrupt the file)"; + // The editor is text/config only: a <textarea> can't faithfully round-trip non-text bytes, + // so a re-save would corrupt the file. Treat it as binary — read-only, save disabled — if it + // has a NUL OR if res.text()'s UTF-8 decode left a replacement char (U+FFFD), which means the + // bytes weren't valid UTF-8 and are already lossy in the textarea. Use the per-row ⤓ to + // download such files intact. + } else if (text.indexOf("\0") !== -1 || text.indexOf("�") !== -1) { + body.value = text; + body.readOnly = true; + saveBtn.disabled = true; + status.textContent = "binary / non-text file — read-only"; + } else { + body.value = fmPrettify(text, relPath); + } + } catch (err) { + body.value = ""; + status.textContent = "load failed: " + err.message; + saveBtn.disabled = true; // never let a Save post an empty body over a file that failed to load + } + + saveBtn.addEventListener("click", async () => { + status.textContent = "saving…"; + try { + const res = await fetch("/api/file?path=" + encodeURIComponent(relPath), { + method: "POST", + headers: { "Content-Type": "text/plain" }, + body: body.value, + }); + if (!res.ok) throw new Error(await errorMessage(res)); + status.textContent = "saved"; + } catch (err) { + status.textContent = "save failed: " + err.message; + } + }); +} + // --------------------------------------------------------------------------- // 9. Boot // --------------------------------------------------------------------------- diff --git a/src/ui/style.css b/src/ui/style.css index c6a4a496..2a4fa34b 100644 --- a/src/ui/style.css +++ b/src/ui/style.css @@ -710,17 +710,17 @@ body { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; resize: vertical; /* draggable grip, vertical only */ min-height: 1.5em; - height: 1.9em; /* default ~one row; resize/typing grows it */ + height: 3.4em; /* default ~two rows; resize/typing grows it */ line-height: 1.4; width: 100%; white-space: pre; /* don't wrap long script lines */ overflow: auto; - /* A small solid corner triangle (no transparency) — a compact, prominent - resize affordance. */ + /* A solid corner triangle (no transparency) — a prominent, easy-to-grab resize + affordance (bigger than the browser's faint native handle). */ background-image: linear-gradient(135deg, transparent 50%, var(--accent) 50%); background-repeat: no-repeat; background-position: bottom 1px right 1px; - background-size: 7px 7px; + background-size: 12px 12px; } .control-row input[type="checkbox"] { accent-color: var(--accent); @@ -1028,3 +1028,130 @@ body { display: block; } } + +/* --------------------------------------------------------------------------- + File Manager — breadcrumb + entry list + inline editor (FileManagerModule). + Theme-aware via the shared --vars; no hardcoded colours. + --------------------------------------------------------------------------- */ +.fm-panel { margin-top: 8px; } + +/* Show-hidden toggle — the text label + the shared .switch pill, above the toolbar. */ +.fm-hidden-toggle { + display: flex; align-items: center; gap: 8px; margin-bottom: 8px; + font-size: 0.85rem; color: var(--fg-muted); cursor: pointer; user-select: none; +} + +/* Breadcrumb `root / .config / foo` on its own row — each segment a button; wraps freely on narrow + displays without pushing the toolbar buttons around. */ +.fm-crumbs { + display: flex; flex-wrap: wrap; align-items: center; + font-family: ui-monospace, monospace; font-size: 0.85rem; color: var(--fg-muted); + margin-bottom: 8px; +} +/* Toolbar row: the folder/file/delete/refresh buttons; wraps if the card is very narrow. */ +.fm-bar { + display: flex; flex-wrap: wrap; align-items: center; gap: 8px; margin-bottom: 8px; +} +/* Install-from-file row under the Firmware card: the .bin picker button + its live status. */ +.fw-upload-row { + display: flex; flex-wrap: wrap; align-items: center; gap: 8px; margin-top: 8px; +} +.fw-upload-status { color: var(--muted); font-size: 0.85em; } +.fm-crumb { + background: none; border: none; color: var(--accent); cursor: pointer; + padding: 2px 4px; border-radius: 4px; font: inherit; +} +.fm-crumb:hover { background: var(--card-bg-2); } +.fm-crumb--here { color: var(--fg); font-weight: 600; cursor: default; } +.fm-crumb--here:hover { background: none; } +.fm-tool { + flex: 0 0 auto; background: var(--card-bg-2); border: 1px solid var(--border); + color: var(--fg); cursor: pointer; padding: 4px 10px; border-radius: 6px; + font: inherit; font-size: 0.85rem; +} +.fm-tool:hover { background: var(--card-bg-1); } +.fm-tool:disabled { opacity: 0.4; cursor: default; } +.fm-tool--danger.armed { background: var(--red); border-color: var(--red); color: #fff; } +/* Icon-only toolbar buttons: square, centered glyph, a touch larger than the text size so the + icons read cleanly. The `title` still carries the label (tooltip + a11y). */ +.fm-tool--icon { + width: 34px; height: 34px; padding: 0; + display: inline-flex; align-items: center; justify-content: center; + font-size: 1.05rem; line-height: 1; +} + +/* Tree: an indented list of rows; each folder's children nest in a .fm-subtree. */ +.fm-tree { + border: 1px solid var(--border); border-radius: 8px; overflow: hidden; + background: var(--card-bg-0); padding: 4px 0; +} +.fm-row { + display: flex; align-items: center; gap: 6px; + padding: 4px 8px; cursor: pointer; user-select: none; +} +.fm-row:hover { background: var(--card-bg-1); } +.fm-row--sel { background: var(--card-bg-2); } +.fm-row--sel:hover { background: var(--card-bg-2); } +.fm-chev { + flex: 0 0 auto; width: 14px; text-align: center; + color: var(--fg-muted); font-size: 0.8rem; +} +.fm-icon { flex: 0 0 auto; font-size: 1rem; } +.fm-name { + flex: 1 1 auto; overflow: hidden; + text-overflow: ellipsis; white-space: nowrap; +} +.fm-row--sel .fm-name { font-weight: 500; } +.fm-size { + flex: 0 0 auto; color: var(--fg-muted); font-size: 0.8rem; + font-variant-numeric: tabular-nums; +} +/* Per-file download link (⤓) — dim until row hover, like a secondary affordance. */ +.fm-dl { + flex: 0 0 auto; color: var(--fg-muted); text-decoration: none; + font-size: 0.95rem; padding: 0 4px; opacity: 0.5; +} +.fm-row:hover .fm-dl { opacity: 1; } +.fm-dl:hover { color: var(--accent); } +/* Drag-drop upload target highlight (a folder row or the tree while a desktop file hovers). */ +.fm-row--drop, .fm-tree.fm-row--drop { + outline: 2px dashed var(--accent); outline-offset: -2px; + background: color-mix(in srgb, var(--accent) 12%, transparent); +} +.fm-empty { padding: 6px 8px; color: var(--fg-muted); font-size: 0.85rem; font-style: italic; } + +/* Filesystem usage bar below the tree (used / total space). Owned by FilesystemModule, shown here. */ +.fm-usage { + display: flex; align-items: center; gap: 8px; margin-top: 8px; + font-size: 0.8rem; color: var(--fg-muted); +} +.fm-usage-name { flex: 0 0 auto; } +.fm-usage progress { flex: 1 1 auto; height: 8px; } +.fm-usage-label { flex: 0 0 auto; font-variant-numeric: tabular-nums; } + +/* Inline editor — native <dialog>, sized to a comfortable text pane. */ +.fm-editor { + width: min(720px, 92vw); max-height: 82vh; padding: 0; + border: 1px solid var(--border); border-radius: 10px; + background: var(--card-bg-0); color: var(--fg); + display: flex; flex-direction: column; +} +.fm-editor::backdrop { background: rgba(0,0,0,0.5); } +.fm-editor-head { + display: flex; align-items: center; justify-content: space-between; + padding: 12px 14px; border-bottom: 1px solid var(--border); +} +.fm-editor-path { font-family: ui-monospace, monospace; font-size: 0.9rem; color: var(--fg-muted); } +.fm-editor-x { background: none; border: none; color: var(--fg-muted); cursor: pointer; font-size: 1rem; } +.fm-editor-x:hover { color: var(--fg); } +.fm-editor-body { + flex: 1 1 auto; min-height: 240px; margin: 0; padding: 12px 14px; + border: none; resize: vertical; font-family: ui-monospace, monospace; + font-size: 0.85rem; line-height: 1.5; background: var(--bg-1); color: var(--fg); +} +.fm-editor-body:focus { outline: none; } +.fm-editor-foot { + display: flex; align-items: center; justify-content: space-between; + padding: 10px 14px; border-top: 1px solid var(--border); +} +.fm-editor-status { color: var(--fg-muted); font-size: 0.85rem; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f74bd43a..d46c45bb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -10,6 +10,10 @@ add_executable(mm_tests unit/core/unit_crc.cpp unit/core/unit_AudioModule_sync.cpp unit/core/unit_IrModule.cpp + unit/core/unit_TcpConnect.cpp + unit/core/unit_MqttPacket.cpp + unit/core/unit_MqttModule.cpp + unit/core/unit_FileManagerModule.cpp unit/core/unit_Control_apply_absent_key.cpp unit/core/unit_Control_list.cpp unit/core/unit_DeviceIdentify.cpp diff --git a/test/js/improv-frame.test.mjs b/test/js/improv-frame.test.mjs index b5185098..d2d522dc 100644 --- a/test/js/improv-frame.test.mjs +++ b/test/js/improv-frame.test.mjs @@ -1,5 +1,5 @@ // Improv frame-contract tests — pin the wire format the device C++ -// (src/core/ImprovFrame.h), Python (scripts/build/improv_provision.py), and the +// (src/core/ImprovFrame.h), Python (moondeck/build/improv_provision.py), and the // installer JS (web-installer/improv-frame.js) must all agree on byte-for-byte. // The golden vectors here are asserted identically in test/python/test_improv_frame.py // so the JS and Python builders can't drift; they're hand-verified against the C++ diff --git a/test/js/installer-flash-baud.test.mjs b/test/js/installer-flash-baud.test.mjs new file mode 100644 index 00000000..fad9f144 --- /dev/null +++ b/test/js/installer-flash-baud.test.mjs @@ -0,0 +1,65 @@ +// Flash-baud contract for the web installer. A board opts into a non-default flash +// baud via deviceModels.json `flashBaud` — up for a verified-fast bridge (the S31's +// 921600, CLI-only), down for a flaky one (a LOLIN-style CH340 pinned to 460800). The +// installer's orchestrator resolves it per board (catalogFlashBaud) and hands it to +// esptool-js; the CLI (flash_esp32.py) resolves the same field. This test pins that the +// wiring stays honest so the two flash paths can't drift from the catalog data. +// +// Run: `node --test test/js`. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); + +const orchestrator = readFileSync( + join(ROOT, "web-installer", "install-orchestrator.js"), "utf8" +); +const boards = JSON.parse( + readFileSync(join(ROOT, "web-installer", "deviceModels.json"), "utf8") +); + +// The standard esptool bauds a board may declare — keep in step with +// check_devices.py FLASH_BAUDS and flash_esp32.py. +const VALID_BAUDS = new Set([115200, 230400, 460800, 921600]); + +test("the orchestrator drives esptool-js from the catalog flashBaud, not a hardcoded rate", () => { + // The esploader baudrate must come from the resolved variable, so a board's + // flashBaud actually reaches the flash. A literal `baudrate: 460800` would mean + // the catalog value is ignored — pin against that regression. + assert.match( + orchestrator, + /const\s+flashBaud\s*=\s*await\s+catalogFlashBaud\(/, + "start() must resolve flashBaud via catalogFlashBaud(board)" + ); + assert.match( + orchestrator, + /baudrate:\s*flashBaud\b/, + "the ESPLoader must be constructed with baudrate: flashBaud (not a literal)" + ); +}); + +test("catalogFlashBaud falls back to 460800 on any lookup failure", () => { + // The resolver must be best-effort: an unknown board / fetch error / parse error + // returns the safe default rather than blocking or guessing a fast rate. + const m = orchestrator.match(/const\s+DEFAULT_FLASH_BAUD\s*=\s*(\d+)/); + assert.ok(m, "orchestrator must define DEFAULT_FLASH_BAUD"); + assert.equal(Number(m[1]), 460800, "the safe default flash baud must be 460800"); +}); + +test("every board's flashBaud (when set) is a standard esptool rate", () => { + // A typo'd baud would stringify wrong for esptool / be rejected by the driver. + // The Python check_devices gate enforces this too; pinning it here keeps the JS + // consumer honest against the same catalog. + for (const b of boards) { + if (b.flashBaud === undefined) continue; + assert.ok( + Number.isInteger(b.flashBaud) && VALID_BAUDS.has(b.flashBaud), + `board "${b.name}" flashBaud must be one of ${[...VALID_BAUDS].sort((x, y) => x - y).join(", ")}, ` + + `got ${JSON.stringify(b.flashBaud)}` + ); + } +}); diff --git a/test/python/test_build_esp32_s31.py b/test/python/test_build_esp32_s31.py index 68ef23d7..7be1d762 100644 --- a/test/python/test_build_esp32_s31.py +++ b/test/python/test_build_esp32_s31.py @@ -13,7 +13,7 @@ would build the S31 firmware for the WRONG target (esp32s3). We re-implement the same precedence rule here and pin that esp32s31 resolves to esp32s31. -Imports the real dicts from scripts/build/build_esp32.py (no ESP-IDF needed). +Imports the real dicts from moondeck/build/build_esp32.py (no ESP-IDF needed). Run: `uv run --with pytest pytest test/python -q`. """ @@ -21,7 +21,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / "scripts" / "build")) +sys.path.insert(0, str(ROOT / "moondeck" / "build")) from build_esp32 import ( # noqa: E402 FIRMWARES, diff --git a/test/python/test_build_esp32_venv_select.py b/test/python/test_build_esp32_venv_select.py index 5e613ff1..2ab79fc5 100644 --- a/test/python/test_build_esp32_venv_select.py +++ b/test/python/test_build_esp32_venv_select.py @@ -9,7 +9,7 @@ selection follows the *target* IDF version, so it's a function of what we build, not of what was last sourced. -Imports the real function from scripts/build/build_esp32.py (no ESP-IDF needed). +Imports the real function from moondeck/build/build_esp32.py (no ESP-IDF needed). Run: `uv run --with pytest pytest test/python -q`. """ @@ -20,7 +20,7 @@ import pytest ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(ROOT / "scripts" / "build")) +sys.path.insert(0, str(ROOT / "moondeck" / "build")) import build_esp32 # noqa: E402 diff --git a/test/python/test_check_specs_drift.py b/test/python/test_check_specs_drift.py index 11d3713a..3cd81bca 100644 --- a/test/python/test_check_specs_drift.py +++ b/test/python/test_check_specs_drift.py @@ -10,7 +10,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(ROOT / "scripts" / "check")) +sys.path.insert(0, str(ROOT / "moondeck" / "check")) from check_specs import _check_range_drift, _check_author_url_drift # noqa: E402 diff --git a/test/python/test_compute_version.py b/test/python/test_compute_version.py index 515b6321..b0a04955 100644 --- a/test/python/test_compute_version.py +++ b/test/python/test_compute_version.py @@ -14,7 +14,7 @@ import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "scripts" / "build")) +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "moondeck" / "build")) import compute_version as cv # noqa: E402 diff --git a/test/python/test_flash_baud.py b/test/python/test_flash_baud.py new file mode 100644 index 00000000..d1842b9a --- /dev/null +++ b/test/python/test_flash_baud.py @@ -0,0 +1,54 @@ +"""flash_esp32.py baud resolution (_catalog_flash_baud). + +MoonDeck / CLI flashing defaults FAST (921600 — the DIY bench assumes a modern USB +bridge). A deviceModel with a flaky bridge pins its own lower `flashBaud` in the catalog +(the LOLIN D32's CH340 → 460800). The catch: `flashBaud` keys off the deviceModel, but +several models share a firmware — so resolving by firmware alone lets one board's opt-down +leak to its siblings (the LOLIN's 460800 wrongly slowing the Dig-Uno, both on `esp32`). +Resolving by the EXACT deviceModel (which MoonDeck maps from the port) fixes that. + +Pins the resolution so those two real bugs (fast default; sibling leakage) can't regress. +Run: `uv run --with pytest pytest test/python -q`. +""" + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "moondeck" / "build")) + +from flash_esp32 import _catalog_flash_baud, DEFAULT_FLASH_BAUD # noqa: E402 + + +def test_default_is_fast(): + # The DIY-bench default is 921600 (~2x faster than the installer's 460800). + assert DEFAULT_FLASH_BAUD == 921600 + + +def test_exact_model_with_optdown_is_slow(): + # A board that pins a lower flashBaud (LOLIN D32's flaky CH340) gets it exactly. + assert _catalog_flash_baud("esp32", "LOLIN D32") == 460800 + + +def test_exact_model_without_flashbaud_gets_fast_default_not_sibling(): + # THE FIX: a known model with no flashBaud of its own gets the fast default — it must + # NOT inherit a firmware-sibling's opt-down. The Dig-Uno shares `esp32` with the LOLIN + # but has a fine bridge, so it flashes at 921600, not the LOLIN's 460800. + assert _catalog_flash_baud("esp32", "QuinLED Dig-Uno V3") == 921600 + + +def test_firmware_only_falls_back_to_lowest_sibling(): + # No deviceModel known (a plain --firmware flash): take the lowest flashBaud among + # models sharing the firmware, so an opt-down still protects an unidentified board. + assert _catalog_flash_baud("esp32") == 460800 # LOLIN's 460800 protects + assert _catalog_flash_baud("esp32-eth") == 921600 # no sibling opt-down → fast + + +def test_unknown_model_falls_through_to_firmware(): + # A device_model not in the catalog behaves like no model given (firmware resolution). + assert _catalog_flash_baud("esp32", "Nonexistent Model") == 460800 + + +def test_unknown_firmware_is_fast_default(): + assert _catalog_flash_baud("nonexistent-fw") == DEFAULT_FLASH_BAUD + assert _catalog_flash_baud("nonexistent-fw", "Nonexistent Model") == DEFAULT_FLASH_BAUD diff --git a/test/python/test_improv_frame.py b/test/python/test_improv_frame.py index d327ea81..401db26b 100644 --- a/test/python/test_improv_frame.py +++ b/test/python/test_improv_frame.py @@ -4,7 +4,7 @@ """Improv frame-contract tests (Python side). Pins the wire format the device C++ (src/core/ImprovFrame.h), the installer JS -(web-installer/improv-frame.js), and this Python builder (scripts/build/improv_provision.py) +(web-installer/improv-frame.js), and this Python builder (moondeck/build/improv_provision.py) must all agree on byte-for-byte. The G1 golden vector below is the SAME one asserted in test/js/improv-frame.test.mjs, so the JS and Python envelope builders can't drift; it's hand-verified against the C++ sum-mod-256 checksum too. @@ -20,8 +20,8 @@ import sys from pathlib import Path -# improv_provision.py lives in scripts/build and imports a sibling (host_wifi). -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "scripts" / "build")) +# improv_provision.py lives in moondeck/build and imports a sibling (host_wifi). +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "moondeck" / "build")) import improv_provision as ip # noqa: E402 diff --git a/test/python/test_installer_manifests.py b/test/python/test_installer_manifests.py index 561ff019..099ae151 100644 --- a/test/python/test_installer_manifests.py +++ b/test/python/test_installer_manifests.py @@ -29,7 +29,7 @@ ROOT = Path(__file__).resolve().parent.parent.parent FIRMWARES_JSON = ROOT / "web-installer" / "firmwares.json" -GENERATE = ROOT / "scripts" / "build" / "generate_manifest.py" +GENERATE = ROOT / "moondeck" / "build" / "generate_manifest.py" # The exact globs the deploy's `gh release download` stages onto Pages (release.yml, # "Stage cumulative release content"). A manifest part must match one of these, or the diff --git a/test/python/test_mkdocs_slug.py b/test/python/test_mkdocs_slug.py index f8e348ea..ea3257df 100644 --- a/test/python/test_mkdocs_slug.py +++ b/test/python/test_mkdocs_slug.py @@ -21,7 +21,7 @@ from markdown.extensions.toc import slugify # noqa: E402 ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(ROOT / "scripts" / "docs")) +sys.path.insert(0, str(ROOT / "moondeck" / "docs")) from mkdocs_hooks import _slug # noqa: E402 diff --git a/test/python/test_moondeck_mac_match.py b/test/python/test_moondeck_mac_match.py new file mode 100644 index 00000000..eef6454f --- /dev/null +++ b/test/python/test_moondeck_mac_match.py @@ -0,0 +1,62 @@ +"""MoonDeck breadcrumb MAC matching (_mac_matches). + +After a flash, flash_esp32.py records esptool's raw efuse MAC in the breadcrumb, and +discover links the just-flashed serial port to the probed device whose reported MAC +matches. On most chips the device reports that same efuse MAC, so an exact compare +works. But the ESP32-S31's `esp_efuse_mac_get_default()` returns the EUI-64 encoding +of the efuse MAC (FF:FE inserted after the OUI) truncated to 6 bytes — so the device +reports 30:ED:A0:FF:FE:F3 for an efuse MAC of 30:ED:A0:F3:D4:68, and a raw compare +never links the port. _mac_matches accepts both forms. + +Regression: without the EUI-64 tolerance the S31's port silently never linked in +moondeck.json after a flash. Run: `uv run --with pytest pytest test/python -q`. +""" + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "moondeck")) + +from moondeck import _mac_matches # noqa: E402 + + +def test_exact_match_case_insensitive(): + assert _mac_matches("AA:BB:CC:DD:EE:FF", "aa:bb:cc:dd:ee:ff") + assert _mac_matches("aa:bb:cc:dd:ee:ff", "AA:BB:CC:DD:EE:FF") + + +def test_s31_eui64_truncated_derivation_matches(): + # The real S31 pair seen on the bench: esptool efuse MAC vs the device's + # esp_efuse_mac_get_default() report. OUI (30:ED:A0) kept, FF:FE inserted, + # then truncated to 6 bytes — so only the first post-OUI efuse byte (F3) survives. + assert _mac_matches("30:ED:A0:F3:D4:68", "30:ED:A0:FF:FE:F3") + + +def test_unrelated_macs_do_not_match(): + assert not _mac_matches("30:ED:A0:F3:D4:68", "11:22:33:44:55:66") + # Same OUI but a different chip — OUI overlap must NOT be treated as a match. + assert not _mac_matches("30:ED:A0:F3:D4:68", "30:ED:A0:AA:BB:CC") + + +def test_derivation_is_directional_efuse_to_device(): + # The breadcrumb always holds the efuse MAC; the device holds the derived one. + # Matching is computed forward (efuse -> EUI-64-trunc). The reverse string in the + # breadcrumb slot (the already-derived MAC) has no efuse to reconstruct, so it only + # matches itself exactly — pinning that we don't over-accept. + assert _mac_matches("30:ED:A0:FF:FE:F3", "30:ED:A0:FF:FE:F3") # exact still fine + assert not _mac_matches("30:ED:A0:FF:FE:F3", "30:ED:A0:F3:D4:68") # can't go backward + + +def test_empty_never_matches(): + assert not _mac_matches("", "30:ED:A0:FF:FE:F3") + assert not _mac_matches("30:ED:A0:F3:D4:68", "") + assert not _mac_matches(None, None) + + +def test_malformed_only_matches_itself_never_derives(): + # A too-short breadcrumb can't drive the EUI-64 derivation (needs 6 octets), so it + # only ever matches an identical string — never a derived one. Exact-match still + # holds for identical inputs (harmless: same source), but no derivation is attempted. + assert _mac_matches("30:ED:A0", "30:ED:A0") # identical → exact branch + assert not _mac_matches("30:ED:A0", "30:ED:A0:FF:FE") # can't derive from 3 octets diff --git a/test/python/test_verify_version.py b/test/python/test_verify_version.py index 573f4103..5d46a233 100644 --- a/test/python/test_verify_version.py +++ b/test/python/test_verify_version.py @@ -17,7 +17,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -SCRIPT = ROOT / "scripts" / "build" / "verify_version.py" +SCRIPT = ROOT / "moondeck" / "ci" / "verify_version.py" def run(tag, version, tmp_path): @@ -28,7 +28,7 @@ def run(tag, version, tmp_path): # (it resolves ROOT from __file__), so patch by running a tiny wrapper that swaps the path. code = ( f"import runpy, sys; " - f"import scripts.build.verify_version as v; " + f"import moondeck.ci.verify_version as v; " f"v.LIBRARY_JSON = __import__('pathlib').Path(r'{lib}'); " f"sys.argv = ['verify_version', '--tag', r'{tag}']; " f"sys.exit(v.main())" diff --git a/test/scenario_runner.cpp b/test/scenario_runner.cpp index 378618a6..724572d1 100644 --- a/test/scenario_runner.cpp +++ b/test/scenario_runner.cpp @@ -781,7 +781,7 @@ static int runScenario(const char* path) { // The floor dominates below ~1ms tick (the realistic case). // esp32-* — bounded RTOS but lwIP/EMAC jitter, 10% pct + 5us floor. // KEEP IN SYNC: the live runner re-declares the same defaults at - // scripts/scenario/run_live_scenario.py contract-block handler — + // moondeck/scenario/run_live_scenario.py contract-block handler — // tuning one without the other silently desyncs the two tiers. const bool isPc = std::strncmp(hostTarget(), "pc-", 3) == 0; double tickTolPct = exp.has("tick_tolerance_pct") ? exp["tick_tolerance_pct"].num diff --git a/test/scenarios/core/scenario_MoonModule_control_change.json b/test/scenarios/core/scenario_MoonModule_control_change.json index 0f4bb254..c4c1d50b 100644 --- a/test/scenarios/core/scenario_MoonModule_control_change.json +++ b/test/scenarios/core/scenario_MoonModule_control_change.json @@ -171,20 +171,20 @@ }, "esp32": { "tick_us": [ - 253513, + 1293, 253513 ], "free_heap": [ 90488, - 90488 + 125144 ], "max_alloc_block": [ 49152, - 49152 + 110592 ], "at": [ "2026-06-02", - "2026-06-02" + "2026-07-05" ] }, "pc-windows": { @@ -304,20 +304,20 @@ }, "esp32": { "tick_us": [ - 209451, + 1075, 209451 ], "free_heap": [ 90356, - 90356 + 125136 ], "max_alloc_block": [ 49152, - 49152 + 110592 ], "at": [ "2026-06-02", - "2026-06-02" + "2026-07-05" ] }, "pc-windows": { @@ -437,20 +437,20 @@ }, "esp32": { "tick_us": [ - 226151, + 1104, 226151 ], "free_heap": [ 90052, - 90052 + 120044 ], "max_alloc_block": [ 49152, - 49152 + 106496 ], "at": [ "2026-06-02", - "2026-06-02" + "2026-07-05" ] }, "pc-windows": { @@ -578,20 +578,20 @@ }, "esp32": { "tick_us": [ - 229353, + 1202, 229353 ], "free_heap": [ 90344, - 90344 + 125128 ], "max_alloc_block": [ 49152, - 49152 + 110592 ], "at": [ "2026-06-02", - "2026-06-02" + "2026-07-05" ] }, "pc-windows": { diff --git a/test/scenarios/core/scenario_NetworkModule_mdns_toggle.json b/test/scenarios/core/scenario_NetworkModule_mdns_toggle.json index 5e795b12..556ccba5 100644 --- a/test/scenarios/core/scenario_NetworkModule_mdns_toggle.json +++ b/test/scenarios/core/scenario_NetworkModule_mdns_toggle.json @@ -69,20 +69,20 @@ }, "esp32": { "tick_us": [ - 229859, + 161, 229859 ], "free_heap": [ 90056, - 90056 + 129340 ], "max_alloc_block": [ 49152, - 49152 + 110592 ], "at": [ "2026-06-02", - "2026-06-02" + "2026-07-05" ] }, "esp32p4-eth": { @@ -160,20 +160,20 @@ }, "esp32": { "tick_us": [ - 274639, + 141, 274639 ], "free_heap": [ 89796, - 89796 + 129688 ], "max_alloc_block": [ 49152, - 49152 + 110592 ], "at": [ "2026-06-02", - "2026-06-02" + "2026-07-05" ] }, "esp32p4-eth": { @@ -251,20 +251,20 @@ }, "esp32": { "tick_us": [ - 234016, + 148, 234016 ], "free_heap": [ 84848, - 84848 + 127100 ], "max_alloc_block": [ 49152, - 49152 + 110592 ], "at": [ "2026-06-02", - "2026-06-02" + "2026-07-05" ] }, "esp32p4-eth": { diff --git a/test/scenarios/light/scenario_Audio_mutation.json b/test/scenarios/light/scenario_Audio_mutation.json index ba6dc0df..b5cd094d 100644 --- a/test/scenarios/light/scenario_Audio_mutation.json +++ b/test/scenarios/light/scenario_Audio_mutation.json @@ -88,7 +88,7 @@ "pc-macos": { "tick_us": [ 8, - 64 + 98 ], "free_heap": [ 0, @@ -100,7 +100,7 @@ ], "at": [ "2026-06-12", - "2026-07-01" + "2026-07-05" ] } } @@ -127,7 +127,7 @@ "pc-macos": { "tick_us": [ 8, - 63 + 269 ], "free_heap": [ 0, @@ -139,7 +139,7 @@ ], "at": [ "2026-06-12", - "2026-07-01" + "2026-07-05" ] } } diff --git a/test/unit/core/unit_FileManagerModule.cpp b/test/unit/core/unit_FileManagerModule.cpp new file mode 100644 index 00000000..66743fb8 --- /dev/null +++ b/test/unit/core/unit_FileManagerModule.cpp @@ -0,0 +1,206 @@ +// @module FileManagerModule + +// Drives the file-manager create/delete ops against the real platform::fs* seam, isolated to a temp +// dir via fsSetRoot (the FilesystemModule-test pattern). The mkdir/delete ops are HTTP endpoints +// (POST/DELETE /api/dir?path=) whose handlers do parseFilePath(query) → fsMkdir/fsRemove; the HTTP +// framing needs a socket fixture (backlogged), so here we exercise the same seam contract the +// handler runs on — the create/delete behaviour + robustness (non-empty-dir / '..' traversal) — plus +// HttpServerModule::parseFilePath directly (it's pure string→string, so it needs no socket). + +#include "doctest.h" +#include "core/FileManagerModule.h" +#include "core/HttpServerModule.h" // parseFilePath — the shared filesystem-path guard +#include "platform/platform.h" + +#include <cstdio> +#include <cstring> +#include <filesystem> +#include <string> + +using namespace mm; + +namespace { + +// Write a seed file, failing the test cleanly if the handle can't be opened (never fputs/fclose a +// null FILE*, which would crash the whole binary). +void writeFile(const std::string& path, const char* contents) { + FILE* f = std::fopen(path.c_str(), "w"); + REQUIRE(f != nullptr); + std::fputs(contents, f); + std::fclose(f); +} + +// A file-manager on a fresh temp filesystem root. Seeds a known layout the ops act against. +struct Rig { + char root[256]; + FileManagerModule fm; + Rig() { + // A monotonic per-process counter, not millis() — several Rigs construct within the same + // millisecond in one test run, so a time-based name could collide. + static unsigned counter = 0; + // temp_directory_path() is the portable temp root (/tmp on POSIX, %TEMP% on Windows), not a + // hardcoded "/tmp"; the counter keeps each Rig's dir unique within the run. + std::snprintf(root, sizeof(root), "%s/mm_fm_test_%u", + std::filesystem::temp_directory_path().string().c_str(), counter++); + std::filesystem::remove_all(root); + std::filesystem::create_directories(std::string(root) + "/.config"); + writeFile(std::string(root) + "/.config/Drivers.json", "{\"brightness\":20}"); + writeFile(std::string(root) + "/readme.txt", "hello"); + platform::fsSetRoot(root); + fm.setTypeName("FileManagerModule"); + fm.onBuildControls(); + fm.setup(); + } + // Restore the DEFAULT root (fsSetRoot("") → "build"), not ".", so a later test in the same + // binary starts from the same baseline this Rig assumed, never a leaked "." repo-root. + ~Rig() { platform::fsSetRoot(""); std::filesystem::remove_all(root); } + + bool onDisk(const char* rel) const { + return std::filesystem::exists(std::string(root) + rel); + } +}; + +// The mkdir op (POST /api/dir?path=), sans HTTP: the handler is fsMkdir on the guarded path. +// Returns the seam result. fsSetRoot has already rooted the temp dir, so a plain rel path maps in. +bool mkdirOp(const char* rel) { return platform::fsMkdir(rel); } +// The delete op (DELETE /api/dir?path=): fsRemove a file or EMPTY dir. +bool deleteOp(const char* rel) { return platform::fsRemove(rel); } + +} // namespace + +TEST_CASE("FileManager: mkdir creates a dir at the target path; delete removes it") { + Rig r; + CHECK(mkdirOp("/newdir")); + CHECK(r.onDisk("/newdir")); + CHECK(deleteOp("/newdir")); + CHECK(!r.onDisk("/newdir")); +} + +TEST_CASE("FileManager: mkdir nested under an existing dir") { + Rig r; + CHECK(mkdirOp("/.config/sub")); + CHECK(r.onDisk("/.config/sub")); +} + +TEST_CASE("FileManager: delete of a non-empty folder is rejected, not a crash") { + Rig r; + CHECK(!deleteOp("/.config")); // .config holds Drivers.json → not empty → fails cleanly + CHECK(r.onDisk("/.config")); // still there +} + +TEST_CASE("FileManager: delete removes a file too") { + Rig r; + CHECK(deleteOp("/readme.txt")); + CHECK(!r.onDisk("/readme.txt")); +} + +TEST_CASE("FileManager: a '..' traversal never escapes root (the seam's confinement)") { + Rig r; + // fsSetRoot confines the seam to the temp root; even a raw '..' can't create outside it. The HTTP + // handler additionally rejects '..' up front in parseFilePath before reaching the seam (below). + (void)mkdirOp("/../escape"); + CHECK(!std::filesystem::exists(std::string(r.root) + "/../escape")); // nothing outside root +} + +// parseFilePath is the single path guard every filesystem HTTP entry (read/write/dir/mkdir/delete) +// runs on — pure string→string, so it's tested directly here without a socket. It decodes the +// `path=` query value (%XX + '+'), roots a relative path at the mount, and rejects a missing/empty +// path, a `..` traversal (raw OR percent-encoded), and an overlong (buffer-filling) value. +TEST_CASE("HttpServer::parseFilePath accepts a valid path and roots a relative one") { + char out[64]; + // Absolute path passes through as-is. + CHECK(HttpServerModule::parseFilePath("path=/foo/bar.json", out, sizeof(out))); + CHECK(std::strcmp(out, "/foo/bar.json") == 0); + // Relative path is rooted at the mount ('/'). + CHECK(HttpServerModule::parseFilePath("path=readme.txt", out, sizeof(out))); + CHECK(std::strcmp(out, "/readme.txt") == 0); + // %XX + '+' decode: "%2Ffoo bar" → "/foo bar". + CHECK(HttpServerModule::parseFilePath("path=%2Ffoo+bar", out, sizeof(out))); + CHECK(std::strcmp(out, "/foo bar") == 0); + // The path stops at the '&' delimiter (further query params are ignored). + CHECK(HttpServerModule::parseFilePath("path=/a.json&hidden=1", out, sizeof(out))); + CHECK(std::strcmp(out, "/a.json") == 0); +} + +TEST_CASE("HttpServer::parseFilePath rejects traversal, empty, missing, and overlong") { + char out[64]; + // Raw '..' anywhere → reject. + CHECK_FALSE(HttpServerModule::parseFilePath("path=/../etc/passwd", out, sizeof(out))); + CHECK_FALSE(HttpServerModule::parseFilePath("path=foo/../../bar", out, sizeof(out))); + // Percent-encoded '..' (%2e%2e) decodes to '..' BEFORE the check → also reject. + CHECK_FALSE(HttpServerModule::parseFilePath("path=%2e%2e/secret", out, sizeof(out))); + // Missing path= key, and an empty value → reject. + CHECK_FALSE(HttpServerModule::parseFilePath("hidden=1", out, sizeof(out))); + CHECK_FALSE(HttpServerModule::parseFilePath("path=", out, sizeof(out))); + CHECK_FALSE(HttpServerModule::parseFilePath(nullptr, out, sizeof(out))); + // Overlong value (fills the buffer before the terminator) → reject, not a truncated prefix. + char small[8]; + CHECK_FALSE(HttpServerModule::parseFilePath("path=/this/is/way/too/long", small, sizeof(small))); +} + +// The /api/file upload streams the body to fsWriteStream (any size, binary-safe) and downloads via +// fsReadAt (positional, chunked). These pin the seam primitives that path relies on. (The HTTP +// framing itself needs a socket fixture — backlogged; here we exercise the platform contracts.) + +// A multi-chunk source (larger than the src callback's cap) with a NUL writes in full and reads back +// byte-for-byte — the streamed-upload contract. +namespace { +struct SpanSrc { const char* p; size_t left; }; +size_t spanPull(char* out, size_t cap, void* user, bool* abort) { + (void)abort; // this source always ends cleanly (no early close / timeout to signal) + auto* s = static_cast<SpanSrc*>(user); + const size_t n = s->left < cap ? s->left : cap; + std::memcpy(out, s->p, n); + s->p += n; s->left -= n; + return n; // 0 when exhausted → clean EOF +} +} // namespace + +TEST_CASE("FileManager: fsWriteStream writes a multi-chunk NUL-containing payload in full") { + Rig r; + // 3000 bytes (spans several 1KB internal chunks) with a NUL planted mid-stream. + std::string payload(3000, 'X'); + payload[1500] = '\0'; + payload[2999] = 'Z'; + SpanSrc src{payload.data(), payload.size()}; + REQUIRE(platform::fsWriteStream("/blob.bin", &spanPull, &src)); + CHECK(platform::fsSize("/blob.bin") == static_cast<long>(payload.size())); + + // Read it back positionally across a chunk boundary — the streamed-download contract. + char win[64] = {}; + const int got = platform::fsReadAt("/blob.bin", 1480, win, 40); // straddles the NUL at 1500 + CHECK(got == 40); + CHECK(win[20] == '\0'); // the planted NUL, offset 1500 + CHECK(std::memcmp(win, payload.data() + 1480, 40) == 0); +} + +// A source that reports a short read (mid-stream failure) still commits atomically — here a clean +// end just yields the bytes delivered; there's no partial/torn file (temp → rename). +TEST_CASE("FileManager: fsWriteStream commits atomically (no torn file)") { + Rig r; + const char data[] = {'a', 'b', '\0', 'c', 'd'}; + SpanSrc src{data, sizeof(data)}; + REQUIRE(platform::fsWriteStream("/atomic.bin", &spanPull, &src)); + char back[16] = {}; + const int n = platform::fsRead("/atomic.bin", back, sizeof(back)); + CHECK(n == static_cast<int>(sizeof(data))); + CHECK(std::memcmp(back, data, sizeof(data)) == 0); + // No leftover temp file beside it. + CHECK(!std::filesystem::exists(std::string(r.root) + "/atomic.bin.tmp")); +} + +// A source that aborts mid-stream (an incomplete/timed-out upload) must NOT commit — fsWriteStream +// discards the temp and returns false, so a truncated body never lands as a real file. +TEST_CASE("FileManager: fsWriteStream discards on abort (incomplete upload)") { + Rig r; + struct AbortSrc { int calls = 0; }; + AbortSrc a; + auto src = [](char* out, size_t cap, void* user, bool* abort) -> size_t { + auto* s = static_cast<AbortSrc*>(user); + if (s->calls++ == 0) { const size_t n = cap < 3 ? cap : 3; std::memcpy(out, "abc", n); return n; } + *abort = true; return 0; // second call: the rest never arrived + }; + CHECK(!platform::fsWriteStream("/partial.bin", src, &a)); // aborted → false + CHECK(platform::fsSize("/partial.bin") < 0); // no file committed + CHECK(!std::filesystem::exists(std::string(r.root) + "/partial.bin.tmp")); // temp discarded +} diff --git a/test/unit/core/unit_HttpServerModule_apply.cpp b/test/unit/core/unit_HttpServerModule_apply.cpp index 698902d0..a4df0476 100644 --- a/test/unit/core/unit_HttpServerModule_apply.cpp +++ b/test/unit/core/unit_HttpServerModule_apply.cpp @@ -50,6 +50,17 @@ struct Tag : public mm::MoonModule { } }; +// A stand-in for the real Drivers module the WLED shim targets: an `on` Bool + a `brightness` +// Uint8, the two controls applyWledState drives. Named "Drivers" so findModuleByName resolves it. +struct FakeDrivers : public mm::MoonModule { + bool on = true; + uint8_t brightness = 20; + void onBuildControls() override { + controls_.addBool("on", on); + controls_.addUint8("brightness", brightness, 0, 255); + } +}; + // Build a tree: scheduler root "Root" (a Box) with HttpServerModule wired to it. // Returns via out-params so each case starts clean. Caller owns teardown via the // scheduler. @@ -59,6 +70,7 @@ void registerTestTypes() { mm::ModuleFactory::registerType<Knob>("Knob"); mm::ModuleFactory::registerType<Box>("Box"); mm::ModuleFactory::registerType<Tag>("Tag"); + mm::ModuleFactory::registerType<FakeDrivers>("Drivers"); done = true; } @@ -226,3 +238,43 @@ TEST_CASE("apply-core: a control validator rejects bad input on the set/APPLY_OP s.deleteTree(root); } + +// The WLED shim's `{on,bri}` apply drives the real `on` + `brightness` controls independently: +// turning off must NOT clobber the brightness value (the whole point of the shared `on` control, +// replacing the old bri=0 fudge). Home Assistant + the WLED app both post through this path. +TEST_CASE("apply-core: applyWledState sets on + bri independently (no brightness clobber)") { + registerTestTypes(); + mm::Scheduler s; + auto* root = new Box(); + root->setName("Root"); + s.addModule(root); + auto* drivers = new FakeDrivers(); + drivers->setName("Drivers"); + drivers->onBuildControls(); // register on + brightness so applyWledState can find them + s.addModule(drivers); + mm::HttpServerModule http; + http.setScheduler(&s); + + // Set a real brightness first. + http.applyWledState("{\"bri\":200}"); + CHECK(drivers->brightness == 200); + CHECK(drivers->on == true); + + // Turn OFF → on flips false, brightness value is PRESERVED (the fudge would have zeroed it). + http.applyWledState("{\"on\":false}"); + CHECK(drivers->on == false); + CHECK(drivers->brightness == 200); // preserved — this is the subtraction's payoff + + // Turn back ON → on true, brightness still the level the user had. + http.applyWledState("{\"on\":true}"); + CHECK(drivers->on == true); + CHECK(drivers->brightness == 200); + + // Combined {on,bri} applies both. + http.applyWledState("{\"on\":true,\"bri\":50}"); + CHECK(drivers->on == true); + CHECK(drivers->brightness == 50); + + s.deleteTree(root); + s.deleteTree(drivers); +} diff --git a/test/unit/core/unit_IrModule.cpp b/test/unit/core/unit_IrModule.cpp index be49aaec..aab2cd9a 100644 --- a/test/unit/core/unit_IrModule.cpp +++ b/test/unit/core/unit_IrModule.cpp @@ -19,12 +19,15 @@ using namespace mm; namespace { -// Stands in for Drivers: a brightness Uint8 (0–255) and a palette Select (0–3). Named "Drivers" -// so IrModule's kActions ("Drivers"/"brightness", "Drivers"/"palette") resolve to it. +// Stands in for Drivers: an `on` Bool, a brightness Uint8 (0–255) and a palette Select (0–3). Named +// "Drivers" so IrModule's kActions ("Drivers"/"on", "Drivers"/"brightness", "Drivers"/"palette") +// resolve to it. struct FakeDrivers : public MoonModule { + bool on = true; uint8_t brightness = 100; uint8_t palette = 1; void onBuildControls() override { + controls_.addBool("on", on); controls_.addUint8("brightness", brightness, 0, 255); // A Select's max is (optionCount - 1); addSelect binds min 0 / max count-1. static const char* kPalettes[] = {"A", "B", "C", "D"}; @@ -46,9 +49,9 @@ struct Rig { scheduler.setup(); // binds controls + sets Scheduler::instance() } ~Rig() { scheduler.teardown(); } - // Learn `code` to the action at `learnIndex` (1-based: 1=brightness up, 2=down, 3=palette - // next, 4=prev), then that code drives the action on later inject. Mirrors the on-device flow: - // arm the learn select → the next received code binds → subsequent codes fire the action. + // Learn `code` to the action at `learnIndex` (1-based: 1=on/off, 2=brightness up, 3=down, + // 4=palette next, 5=prev), then that code drives the action on later inject. Mirrors the on-device + // flow: arm the learn select → the next received code binds → subsequent codes fire the action. void learn(int learnIndex, uint32_t code) { char v[24]; std::snprintf(v, sizeof(v), "{\"value\":%d}", learnIndex); Scheduler::instance()->setControl("Ir", "learn", v); @@ -62,8 +65,8 @@ struct Rig { TEST_CASE("IrModule: a learned code adjusts Drivers brightness by the step") { Rig r; r.drivers->brightness = 100; - r.learn(1, 0xB1); // bind code 0xB1 → brightness up - r.learn(2, 0xB2); // bind code 0xB2 → brightness down + r.learn(2, 0xB1); // bind code 0xB1 → brightness up + r.learn(3, 0xB2); // bind code 0xB2 → brightness down r.fire(0xB1); CHECK(r.drivers->brightness == 116); // +16 r.fire(0xB2); @@ -71,12 +74,26 @@ TEST_CASE("IrModule: a learned code adjusts Drivers brightness by the step") { CHECK(r.drivers->brightness == 84); // 116 - 32 } +TEST_CASE("IrModule: a learned on/off code toggles the Drivers on control") { + Rig r; + r.drivers->on = true; + r.learn(1, 0xA0); // bind code 0xA0 → on/off (learn index 1 = first action) + r.fire(0xA0); + CHECK(r.drivers->on == false); // toggled off + r.fire(0xA0); + CHECK(r.drivers->on == true); // toggled back on + r.fire(0xA0); + CHECK(r.drivers->on == false); // and off again — a genuine toggle, not a saturating nudge + // The status names the new state, not a numeric value. + CHECK(std::strcmp(r.ir->status(), "Drivers.on → off") == 0); +} + TEST_CASE("IrModule: firing a learned code reports what it changed via status") { Rig r; r.drivers->brightness = 100; // Before setup drives anything, the pin is unset in the rig → readiness warns to set it. CHECK(std::strstr(r.ir->status(), "set pin") != nullptr); - r.learn(1, 0xB1); + r.learn(2, 0xB1); r.fire(0xB1); // The action acknowledges the change it made, naming the target control + new value. CHECK(std::strcmp(r.ir->status(), "Drivers.brightness → 116") == 0); @@ -95,8 +112,8 @@ TEST_CASE("IrModule: pin state drives readiness status") { TEST_CASE("IrModule: a learned brightness code clamps at 0 and 255") { Rig r; - r.learn(1, 0xB1); // brightness up - r.learn(2, 0xB2); // brightness down + r.learn(2, 0xB1); // brightness up + r.learn(3, 0xB2); // brightness down r.drivers->brightness = 250; r.fire(0xB1); CHECK(r.drivers->brightness == 255); // 250+16 clamps to 255, not wrap @@ -107,8 +124,8 @@ TEST_CASE("IrModule: a learned brightness code clamps at 0 and 255") { TEST_CASE("IrModule: learned palette codes step the Select and clamp at the ends") { Rig r; - r.learn(3, 0xB3); // palette next - r.learn(4, 0xB4); // palette prev + r.learn(4, 0xB3); // palette next + r.learn(5, 0xB4); // palette prev r.drivers->palette = 1; r.fire(0xB3); CHECK(r.drivers->palette == 2); @@ -125,8 +142,8 @@ TEST_CASE("IrModule: learned palette codes step the Select and clamp at the ends TEST_CASE("IrModule: learn binds a code to an action, then that code drives it") { Rig r; r.drivers->brightness = 100; - // Arm learning for "brightness up" (learn select index 1 = first action). - Scheduler::instance()->setControl("Ir", "learn", "{\"value\":1}"); + // Arm learning for "brightness up" (learn select index 2; index 1 is on/off). + Scheduler::instance()->setControl("Ir", "learn", "{\"value\":2}"); // A code arrives → it binds to "brightness up" and learning disarms; brightness unchanged yet. r.ir->injectCodeForTest(0xFA057F80); CHECK(r.drivers->brightness == 100); @@ -151,8 +168,8 @@ TEST_CASE("IrModule: two codes bind to two actions independently") { Rig r; r.drivers->brightness = 100; r.drivers->palette = 1; - r.learn(1, 0x11111111); // brightness up - r.learn(3, 0x22222222); // palette next + r.learn(2, 0x11111111); // brightness up + r.learn(4, 0x22222222); // palette next // Each code drives only its own action. r.fire(0x11111111); CHECK(r.drivers->brightness == 116); @@ -174,7 +191,7 @@ TEST_CASE("IrModule: an unassigned code is a no-op, not a crash") { TEST_CASE("IrModule: a learned code whose target module is gone is a no-op, reported") { Rig r; const uint8_t before = r.drivers->brightness; - r.learn(1, 0xC0DE); // bind 0xC0DE → brightness up (targets "Drivers") + r.learn(2, 0xC0DE); // bind 0xC0DE → brightness up (targets "Drivers") // Take the target out of reach: rename it so firstByName("Drivers") returns null. r.drivers->setName("NotDrivers"); r.fire(0xC0DE); // the action fires but its module is missing diff --git a/test/unit/core/unit_MoonModule.cpp b/test/unit/core/unit_MoonModule.cpp index 08ec6f24..98b941b3 100644 --- a/test/unit/core/unit_MoonModule.cpp +++ b/test/unit/core/unit_MoonModule.cpp @@ -24,6 +24,13 @@ class TestModule : public mm::MoonModule { } }; +// A module that hides from the UI (the FilesystemModule / HttpServerModule pattern): the state +// serializer skips any module whose appearsInUi() is false. +class HiddenModule : public mm::MoonModule { +public: + bool appearsInUi() const override { return false; } +}; + } // namespace // setup() and teardown() each fire exactly when called and update their respective state flags. @@ -195,3 +202,43 @@ TEST_CASE("Bool control binding") { mod.enabled = false; CHECK(*static_cast<bool*>(ctrl.ptr) == false); } + +// appearsInUi() defaults to true (every ordinary module shows in the UI) and is overridable to +// false so infrastructure modules (FilesystemModule, HttpServerModule) can hide — the flag the +// state serializer reads to skip a module. A base MoonModule and a control-bearing one both +// appear; only a module that overrides to false hides. +TEST_CASE("MoonModule appearsInUi defaults true, overridable false") { + TestModule visible; + CHECK(visible.appearsInUi()); // ordinary module: shown + + HiddenModule hidden; + CHECK_FALSE(hidden.appearsInUi()); // infrastructure module: skipped by the serializer + + // Through the base-class pointer the virtual still dispatches to the override — the path the + // serializer actually takes (it holds MoonModule*). + mm::MoonModule* asBase = &hidden; + CHECK_FALSE(asBase->appearsInUi()); +} + +// readBool/readUint8 — the shared generic control reader (reviewer #8): one implementation so the +// absent-control default can't disagree between callers (HttpServerModule + MqttModule both read +// Drivers.on through this). Returns the bound value; returns the caller's default when absent/wrong-type. +TEST_CASE("MoonModule readBool/readUint8 return the value, or the default when absent") { + TestModule mod; + mod.onBuildControls(); // binds brightness(Uint8), speed(Uint8), enabled(Bool) + + // Present controls read their live value. + CHECK(mod.readUint8("brightness", 99) == 128); // TestModule brightness default + CHECK(mod.readBool("enabled", false) == true); + + // The live field updates flow through (the reader dereferences the bound pointer). + mod.brightness = 42; + CHECK(mod.readUint8("brightness", 99) == 42); + mod.enabled = false; + CHECK(mod.readBool("enabled", true) == false); + + // Absent control → the caller's default, not a hard-coded one (the bug this shared reader fixes). + CHECK(mod.readBool("on", true) == true); // no "on" control → default true + CHECK(mod.readBool("on", false) == false); // ...and the OTHER default, from the same call + CHECK(mod.readUint8("missing", 7) == 7); +} diff --git a/test/unit/core/unit_MqttModule.cpp b/test/unit/core/unit_MqttModule.cpp new file mode 100644 index 00000000..65c75856 --- /dev/null +++ b/test/unit/core/unit_MqttModule.cpp @@ -0,0 +1,159 @@ +// @module MqttModule +// @also Scheduler + +// Pins MqttModule's inbound routing: a PUBLISH arriving on a <prefix>/…/set topic drives the +// matching Drivers control through the shared Scheduler::setControl primitive — the same seam IR and +// the WLED bridge use. The socket is not involved: feedForTest() injects raw MQTT bytes (built with +// the tested MqttPacket builders) exactly as the broker would deliver them, so the routing is +// provable with no broker (mirrors IrModule::injectCodeForTest). A FakeDrivers stands in for the +// real Drivers with the on / brightness / palette controls MQTT targets. + +#include "doctest.h" +#include "core/MqttModule.h" +#include "core/MqttPacket.h" +#include "core/Scheduler.h" +#include "core/MoonModule.h" +#include "core/SystemModule.h" + +#include <cstdint> +#include <cstring> +#include <vector> + +using namespace mm; + +namespace { + +// Stands in for Drivers: on (Bool), brightness (Uint8 0-255), palette (Select). Named "Drivers" so +// MqttModule's setControl("Drivers", …) resolves to it. +struct FakeDrivers : public MoonModule { + bool on = true; + uint8_t brightness = 100; + uint8_t palette = 0; + void onBuildControls() override { + controls_.addBool("on", on); + controls_.addUint8("brightness", brightness, 0, 255); + // A Uint8 palette with the real built-in range (0..255 is a superset of the ~60 built-ins), + // so a nearest-palette index the MQTT map returns isn't clamped away by an artificially small + // Select — the real Drivers.palette binds 0..kCount-1. + controls_.addUint8("palette", palette, 0, 255); + } +}; + +// Build a scheduler with FakeDrivers + a SystemModule + an MqttModule, run setup so +// Scheduler::instance() is live and controls are bound. The topic prefix is STABLE + MAC-derived +// (projectMM/<last6-of-MAC>), NOT from deviceName — so it's rename-proof. On desktop the fake MAC is +// DE:AD:BE:EF:CA:FE (platform_desktop.cpp), so last-6 = "efcafe". +struct Rig { + static constexpr const char* kPrefix = "projectMM/efcafe"; // desktop fake MAC last-6 + Scheduler scheduler; + FakeDrivers* drivers = new FakeDrivers(); + SystemModule* system = new SystemModule(); + MqttModule* mqtt = new MqttModule(); + Rig() { + drivers->setName("Drivers"); + system->setName("System"); + mqtt->setName("Mqtt"); + mqtt->setSystemModule(system); // for the published friendly-name (not the topic identity) + scheduler.addModule(drivers); + scheduler.addModule(system); + scheduler.addModule(mqtt); + scheduler.setup(); // binds controls, sets Scheduler::instance() + } + ~Rig() { scheduler.teardown(); } + + // Deliver a PUBLISH to `suffix` (under the derived prefix, e.g. "on/set") with a string payload, + // as the broker socket would. Pass a leading "/" to send an ABSOLUTE topic (for the wrong-prefix + // test); otherwise the derived prefix is prepended. + void publish(const char* suffix, const char* payload) { + char topic[128]; + if (suffix[0] == '/') std::snprintf(topic, sizeof(topic), "%s", suffix + 1); // absolute + else std::snprintf(topic, sizeof(topic), "%s/%s", kPrefix, suffix); + uint8_t buf[160]; + const size_t n = buildMqttPublish(topic, reinterpret_cast<const uint8_t*>(payload), + std::strlen(payload), buf, sizeof(buf)); + REQUIRE(n > 0); + mqtt->feedForTest(buf, n); + } +}; + +} // namespace + +TEST_CASE("MqttModule: on/set drives Drivers.on") { + Rig r; + r.drivers->on = true; + r.publish("on/set", "false"); + CHECK(r.drivers->on == false); + r.publish("on/set", "true"); + CHECK(r.drivers->on == true); + // "1"/"0" are accepted too (mqttthing integerValue mode). + r.publish("on/set", "0"); + CHECK(r.drivers->on == false); + r.publish("on/set", "1"); + CHECK(r.drivers->on == true); +} + +TEST_CASE("MqttModule: brightness/set rescales 0-100 to 0-255") { + Rig r; + r.publish("brightness/set", "0"); + CHECK(r.drivers->brightness == 0); + r.publish("brightness/set", "100"); + CHECK(r.drivers->brightness == 255); + r.publish("brightness/set", "50"); + CHECK(r.drivers->brightness == 127); // 50*255/100 + // Out-of-range clamps, not wraps. + r.publish("brightness/set", "250"); + CHECK(r.drivers->brightness == 255); +} + +TEST_CASE("MqttModule: hsv/set maps a hue to the nearest palette + value to brightness") { + Rig r; + // A blue-ish hue at full saturation should pick a blue-family palette (a non-zero index — not + // Rainbow at 0). We assert it moved off the default and that value drove brightness. + r.drivers->palette = 0; + r.publish("hsv/set", "210,100,40"); // blue, sat 100%, value 40% + CHECK(r.drivers->palette != 0); // snapped to some blue-family palette + CHECK(r.drivers->brightness == (40 * 255) / 100); // value → brightness +} + +TEST_CASE("MqttModule: a PUBLISH on an unrelated topic is ignored, not a crash") { + Rig r; + const uint8_t beforeBri = r.drivers->brightness; + const bool beforeOn = r.drivers->on; + r.publish("unknown/set", "whatever"); // no matching suffix + r.publish("/otherdevice/on/set", "false"); // wrong prefix + CHECK(r.drivers->brightness == beforeBri); + CHECK(r.drivers->on == beforeOn); +} + +TEST_CASE("MqttModule: a PUBLISH split across feeds still routes (fragment reassembly)") { + Rig r; + r.drivers->on = true; + uint8_t buf[128]; + const char* payload = "false"; + const size_t n = buildMqttPublish("projectMM/efcafe/on/set", reinterpret_cast<const uint8_t*>(payload), + std::strlen(payload), buf, sizeof(buf)); + REQUIRE(n > 0); + // Feed one byte at a time — the parser holds partial state until the packet completes. + for (size_t i = 0; i < n; i++) r.mqtt->feedForTest(&buf[i], 1); + CHECK(r.drivers->on == false); +} + +// Regression (reviewer): the topic identity is the STABLE MAC (projectMM/<last6>), NOT the device +// name — so renaming the device must NOT change which topics the module listens on. A command on the +// MAC-based topic keeps working after a rename; a command on a name-based topic never matched. +TEST_CASE("MqttModule: topic identity is MAC-stable, not affected by a device rename") { + Rig r; + r.drivers->on = true; + // Command on the MAC topic works. + r.publish("on/set", "false"); + CHECK(r.drivers->on == false); + // Rename the device — topics must stay on the MAC prefix. + Scheduler::instance()->setControl("System", "deviceName", "{\"value\":\"LivingRoom\"}"); + r.drivers->on = true; + r.publish("on/set", "false"); // still the MAC prefix (Rig::kPrefix) + CHECK(r.drivers->on == false); // rename didn't break routing + // A command on a name-derived topic never matches (proves identity isn't the name). + r.drivers->on = true; + r.publish("/projectMM/LivingRoom/on/set", "false"); // absolute, name-based + CHECK(r.drivers->on == true); // ignored — not our (MAC) prefix +} diff --git a/test/unit/core/unit_MqttPacket.cpp b/test/unit/core/unit_MqttPacket.cpp new file mode 100644 index 00000000..2fdeb14d --- /dev/null +++ b/test/unit/core/unit_MqttPacket.cpp @@ -0,0 +1,248 @@ +// @module MqttPacket + +// Pins the MQTT 3.1.1 wire framing (src/core/MqttPacket.h) with golden byte vectors + round-trips — +// the compatibility contract with any broker (mosquitto / homebridge-mqttthing), same rigor as the +// Improv frame golden vectors. Covers: the remaining-length varint at its boundaries; byte-exact +// CONNECT / PUBLISH / SUBSCRIBE / PINGREQ; CONNACK / SUBACK parse; and the byte-at-a-time inbound +// parser reassembling a PUBLISH across arbitrary read() boundaries (fragmentation). + +#include "doctest.h" +#include "core/MqttPacket.h" + +#include <cstdint> +#include <cstring> +#include <vector> + +using namespace mm; + +// --- Remaining-length varint (§2.2.3) — the fiddly bit, boundary-tested --- +TEST_CASE("MqttPacket: remaining-length varint encodes/decodes at the boundaries") { + struct Case { uint32_t value; std::vector<uint8_t> bytes; }; + const Case cases[] = { + {0, {0x00}}, + {127, {0x7F}}, + {128, {0x80, 0x01}}, + {16383, {0xFF, 0x7F}}, + {16384, {0x80, 0x80, 0x01}}, + {268435455, {0xFF, 0xFF, 0xFF, 0x7F}}, // the 4-byte maximum + }; + for (const auto& c : cases) { + uint8_t out[4] = {}; + const size_t n = encodeRemainingLength(c.value, out); + REQUIRE(n == c.bytes.size()); + CHECK(std::memcmp(out, c.bytes.data(), n) == 0); + + uint32_t decoded = 0; size_t consumed = 0; + REQUIRE(decodeRemainingLength(out, n, &decoded, &consumed)); + CHECK(decoded == c.value); + CHECK(consumed == n); + } + // Over the 4-byte max → encode refuses. + uint8_t out[4]; + CHECK(encodeRemainingLength(268435456u, out) == 0); + // A 5th continuation byte → decode reports malformed. + const uint8_t bad[] = {0x80, 0x80, 0x80, 0x80, 0x01}; + uint32_t v; size_t c; + CHECK_FALSE(decodeRemainingLength(bad, sizeof(bad), &v, &c)); + // Truncated (high bit set, no more bytes) → decode needs more. + const uint8_t trunc[] = {0x80}; + CHECK_FALSE(decodeRemainingLength(trunc, sizeof(trunc), &v, &c)); +} + +// mqttWriteFixedHeader must validate space for the WHOLE header (type byte + varint) before it +// writes any of it — a bodyLen needing a 2-byte varint into a 2-byte buffer must return 0 without +// touching out[2] (the byte past the buffer). Regression guard for a fixed-header overrun. +TEST_CASE("MqttPacket: fixed-header write refuses a buffer too small for its varint") { + uint8_t guarded[3] = {0xAA, 0xAA, 0xAA}; // sentinel; only [0..1] are "the buffer" + // bodyLen 128 → remaining-length is 2 bytes (0x80 0x01), so the full header is 3 bytes and + // does not fit in 2. Must reject, and must NOT have written the type byte or scribbled past. + CHECK(mqttWriteFixedHeader(guarded, 2, MqttPacketType::Publish, 0, 128) == 0); + CHECK(guarded[2] == 0xAA); // the out-of-bounds byte is untouched + // A 3-byte buffer fits exactly: type + 2 varint bytes. + uint8_t ok[3] = {}; + REQUIRE(mqttWriteFixedHeader(ok, sizeof(ok), MqttPacketType::Publish, 0, 128) == 3); + CHECK(ok[1] == 0x80); + CHECK(ok[2] == 0x01); +} + +// --- CONNECT golden vector (§3.1) --- +TEST_CASE("MqttPacket: CONNECT is byte-exact (clean session, no auth)") { + uint8_t out[64] = {}; + const size_t n = buildMqttConnect("mm", nullptr, nullptr, 60, out, sizeof(out)); + // Expected: fixed header 0x10, remaining-length 14, then: + // proto: 00 04 'M' 'Q' 'T' 'T' + // level: 04 + // flags: 02 (clean session, no user/pass) + // keepalive: 00 3C (60) + // clientId: 00 02 'm' 'm' + const uint8_t expected[] = { + 0x10, 0x0E, + 0x00, 0x04, 'M', 'Q', 'T', 'T', + 0x04, + 0x02, + 0x00, 0x3C, + 0x00, 0x02, 'm', 'm', + }; + REQUIRE(n == sizeof(expected)); + CHECK(std::memcmp(out, expected, n) == 0); +} + +TEST_CASE("MqttPacket: CONNECT with username + password sets the flags + payload") { + uint8_t out[64] = {}; + const size_t n = buildMqttConnect("c", "u", "p", 15, out, sizeof(out)); + REQUIRE(n > 0); + CHECK(out[0] == 0x10); // CONNECT + // connect flags byte is at offset 9 (after fixed header[2] + proto[6] + level[1]). + CHECK(out[9] == (kMqttConnectCleanSession | kMqttConnectUsernameFlag | kMqttConnectPasswordFlag)); + // The username "u" and password "p" appear length-prefixed after the clientId "c". + // Tail: ...00 01 'c' 00 01 'u' 00 01 'p' + const uint8_t tail[] = {0x00, 0x01, 'c', 0x00, 0x01, 'u', 0x00, 0x01, 'p'}; + CHECK(std::memcmp(out + n - sizeof(tail), tail, sizeof(tail)) == 0); +} + +// Regression (reviewer #10 / MQTT-3.1.2-22): a password without a username must NOT set the password +// flag — a compliant broker rejects that CONNECT. The builder drops the password when username empty. +TEST_CASE("MqttPacket: CONNECT drops a password when there is no username") { + uint8_t out[64] = {}; + const size_t n = buildMqttConnect("c", nullptr, "secret", 15, out, sizeof(out)); + REQUIRE(n > 0); + // Flags byte at offset 9: clean-session only — neither username nor password flag set. + CHECK(out[9] == kMqttConnectCleanSession); + // The payload is just the clientId (no username/password fields appended). + const uint8_t tail[] = {0x00, 0x01, 'c'}; + CHECK(std::memcmp(out + n - sizeof(tail), tail, sizeof(tail)) == 0); + // Same when username is an empty string (not just nullptr). + const size_t n2 = buildMqttConnect("c", "", "secret", 15, out, sizeof(out)); + REQUIRE(n2 > 0); + CHECK(out[9] == kMqttConnectCleanSession); +} + +// --- PUBLISH golden vector (§3.3), QoS0 --- +TEST_CASE("MqttPacket: PUBLISH is byte-exact (QoS0)") { + uint8_t out[64] = {}; + const uint8_t payload[] = {'O', 'N'}; + const size_t n = buildMqttPublish("a/b", payload, sizeof(payload), out, sizeof(out)); + // 0x30 PUBLISH, remaining-length 7, topic 00 03 'a' '/' 'b', payload 'O' 'N'. + const uint8_t expected[] = { + 0x30, 0x07, + 0x00, 0x03, 'a', '/', 'b', + 'O', 'N', + }; + REQUIRE(n == sizeof(expected)); + CHECK(std::memcmp(out, expected, n) == 0); +} + +// The retain flag (§3.3.1.3) sets bit 0 of the fixed-header type nibble — used for the friendly +// `name` topic so a late-subscribing hub still receives it. +TEST_CASE("MqttPacket: PUBLISH retain flag sets the fixed-header bit") { + uint8_t out[32] = {}; + const uint8_t payload[] = {'x'}; + const size_t n = buildMqttPublish("t", payload, 1, out, sizeof(out), /*retain=*/true); + REQUIRE(n > 0); + CHECK(out[0] == 0x31); // PUBLISH (0x3<<4) | retain (0x1) + // Without retain, bit 0 is clear. + const size_t n2 = buildMqttPublish("t", payload, 1, out, sizeof(out), /*retain=*/false); + REQUIRE(n2 > 0); + CHECK(out[0] == 0x30); +} + +// --- SUBSCRIBE golden vector (§3.8) --- +TEST_CASE("MqttPacket: SUBSCRIBE is byte-exact (one filter, QoS0)") { + uint8_t out[64] = {}; + const size_t n = buildMqttSubscribe(1, "a/b", out, sizeof(out)); + // 0x82 (SUBSCRIBE + reserved flag 0x2), remaining-length 8, packetId 00 01, + // filter 00 03 'a' '/' 'b', requested QoS 00. + const uint8_t expected[] = { + 0x82, 0x08, + 0x00, 0x01, + 0x00, 0x03, 'a', '/', 'b', + 0x00, + }; + REQUIRE(n == sizeof(expected)); + CHECK(std::memcmp(out, expected, n) == 0); + // packetId 0 is rejected (spec: must be non-zero). + CHECK(buildMqttSubscribe(0, "a/b", out, sizeof(out)) == 0); +} + +// --- PINGREQ / DISCONNECT --- +TEST_CASE("MqttPacket: PINGREQ and DISCONNECT are 2-byte packets") { + uint8_t out[4] = {}; + REQUIRE(buildMqttPingreq(out, sizeof(out)) == 2); + CHECK(out[0] == 0xC0); + CHECK(out[1] == 0x00); + REQUIRE(buildMqttDisconnect(out, sizeof(out)) == 2); + CHECK(out[0] == 0xE0); + CHECK(out[1] == 0x00); +} + +// --- Inbound parser: CONNACK return code read off the completed body (how the module reads it) --- +TEST_CASE("MqttPacket: inbound parser exposes a CONNACK's return code in its body") { + const uint8_t accepted[] = {0x20, 0x02, 0x00, 0x00}; // session-present 0, return 0 (accepted) + MqttInboundParser p; + MqttFeedResult result = MqttFeedResult::NeedMore; + for (uint8_t b : accepted) result = p.feed(b); + REQUIRE(result == MqttFeedResult::PacketReady); + CHECK(p.lastType() == static_cast<uint8_t>(MqttPacketType::Connack)); + REQUIRE(p.bodyLen() == 2); + CHECK(p.body()[1] == 0); // return code — the module checks this byte for accept/refuse +} + +// --- Inbound parser: round-trip a PUBLISH built by our own builder --- +TEST_CASE("MqttPacket: inbound parser reassembles a PUBLISH (build→parse round-trip)") { + uint8_t pkt[64] = {}; + const uint8_t payload[] = {'h', 'i'}; + const size_t n = buildMqttPublish("proj/on/set", payload, sizeof(payload), pkt, sizeof(pkt)); + REQUIRE(n > 0); + + MqttInboundParser parser; + MqttFeedResult result = MqttFeedResult::NeedMore; + for (size_t i = 0; i < n; i++) result = parser.feed(pkt[i]); + REQUIRE(result == MqttFeedResult::PacketReady); + CHECK(parser.lastType() == static_cast<uint8_t>(MqttPacketType::Publish)); + + const char* topic = nullptr; const uint8_t* pl = nullptr; size_t plLen = 0; + REQUIRE(parser.publish(&topic, &pl, &plLen)); + CHECK(std::strcmp(topic, "proj/on/set") == 0); + REQUIRE(plLen == 2); + CHECK(pl[0] == 'h'); + CHECK(pl[1] == 'i'); +} + +// --- Inbound parser: fragmentation — a packet split across feeds still reassembles --- +TEST_CASE("MqttPacket: inbound parser reassembles across read() boundaries") { + uint8_t pkt[64] = {}; + const uint8_t payload[] = {'x'}; + const size_t n = buildMqttPublish("t", payload, sizeof(payload), pkt, sizeof(pkt)); + REQUIRE(n > 0); + + // Feed one byte at a time — every byte but the last returns NeedMore, proving the state machine + // holds partial state across calls (the socket hands over arbitrary runs). + MqttInboundParser parser; + for (size_t i = 0; i + 1 < n; i++) { + CHECK(parser.feed(pkt[i]) == MqttFeedResult::NeedMore); + } + CHECK(parser.feed(pkt[n - 1]) == MqttFeedResult::PacketReady); + const char* topic = nullptr; + REQUIRE(parser.publish(&topic, nullptr, nullptr)); + CHECK(std::strcmp(topic, "t") == 0); +} + +// --- Inbound parser: a multi-byte remaining-length (128+ body) is handled --- +TEST_CASE("MqttPacket: inbound parser handles a 2-byte remaining-length") { + // A PUBLISH whose body is 130 bytes forces a 2-byte remaining-length varint. + uint8_t big[200] = {}; + uint8_t payload[126]; + for (auto& b : payload) b = 'z'; + const size_t n = buildMqttPublish("tt", payload, sizeof(payload), big, sizeof(big)); + REQUIRE(n > 0); + CHECK((big[1] & 0x80) != 0); // first remaining-length byte has continuation bit → multi-byte + + MqttInboundParser parser; + MqttFeedResult result = MqttFeedResult::NeedMore; + for (size_t i = 0; i < n; i++) result = parser.feed(big[i]); + CHECK(result == MqttFeedResult::PacketReady); + const char* topic = nullptr; const uint8_t* pl = nullptr; size_t plLen = 0; + REQUIRE(parser.publish(&topic, &pl, &plLen)); + CHECK(std::strcmp(topic, "tt") == 0); + CHECK(plLen == sizeof(payload)); +} diff --git a/test/unit/core/unit_NetworkModule_ethernet.cpp b/test/unit/core/unit_NetworkModule_ethernet.cpp index e6803791..cc469b94 100644 --- a/test/unit/core/unit_NetworkModule_ethernet.cpp +++ b/test/unit/core/unit_NetworkModule_ethernet.cpp @@ -45,6 +45,17 @@ TEST_CASE("Desktop ethConfigDefault is ethNone (no Ethernet)") { CHECK(mm::platform::ethConfigDefault.phyType == mm::platform::ethNone); } +// Ethernet is OPT-IN: NetworkModule's `ethType` control defaults to ethNone(0), so a board whose +// deviceModels.json entry has no `ethType` brings up NO PHY (a WiFi-only board like Shelly, or the +// QuinLED Dig-Uno/Quad with optional-only eth, must not waste an RMII/SPI init on a PHY it lacks). +// A board with real Ethernet sets `ethType` explicitly in its catalog eth block. On ESP32 the +// control default seeds ethType_ = 0; the platform's ethConfigDefault (whose phyType is the chip's +// historical PHY) still seeds the PINS so an opt-in board gets them without re-listing — but the +// PHY *selection* is off until the catalog turns it on. ethNone==0 is what makes "unset → off" work. +TEST_CASE("Ethernet is opt-in: ethNone is the zero value (unset ethType → no PHY)") { + CHECK(mm::platform::ethNone == 0); // an absent/zero ethType control resolves to None +} + // The platform seam must accept any runtime config and never bring Ethernet up on // desktop — ethInit() returns false so NetworkModule cascades to WiFi/AP. Pushing a // fully-populated W5500 config and an RMII config both leave ethInit() false and diff --git a/test/unit/core/unit_TcpConnect.cpp b/test/unit/core/unit_TcpConnect.cpp new file mode 100644 index 00000000..ab4528d8 --- /dev/null +++ b/test/unit/core/unit_TcpConnect.cpp @@ -0,0 +1,56 @@ +// @module platform + +// Pins platform::TcpConnection::connectStart/connectPoll — the NON-BLOCKING outbound TCP client +// primitive MqttModule uses to reach a broker without stalling Scheduler::tick. connectStart resolves +// a host (name or dotted-quad via getaddrinfo) and kicks off a non-blocking connect; connectPoll +// reports Pending/Connected/Failed without blocking. Driven over loopback against a real TcpServer +// (deterministic on desktop): a listening port connects + the server accepts; a dead port / bad host +// fails cleanly (no hang, no crash). + +#include "doctest.h" +#include "platform/platform.h" + +#include <cstdint> + +using namespace mm; + +// connectStart/connectPoll — the NON-BLOCKING outbound connect MqttModule uses so it never stalls +// Scheduler::tick (reviewer #2). connectStart returns immediately; connectPoll reports +// Pending/Connected/Failed without blocking. Over loopback the connect completes within a few polls. +TEST_CASE("platform::TcpConnection::connectStart/connectPoll is non-blocking") { + using CR = platform::TcpConnection::ConnectResult; + platform::TcpServer server; + uint16_t port = 0; + for (uint16_t p = 34700; p < 34740; p++) { if (server.open(p)) { port = p; break; } } + REQUIRE(port != 0); + + platform::TcpConnection client; + REQUIRE(client.connectStart("127.0.0.1", port)); // returns immediately, connect in flight + // Poll until connected, accepting on the server side and yielding a little between polls so the + // loopback TCP handshake can complete (connectPoll never blocks, so a tight spin would just burn + // 200 microseconds before the kernel finishes the handshake). + platform::TcpConnection accepted; + CR r = CR::Pending; + for (int i = 0; i < 200 && r == CR::Pending; i++) { + if (!accepted.valid()) accepted = server.accept(); + r = client.connectPoll(); + if (r == CR::Pending) platform::delayMs(1); + } + CHECK(r == CR::Connected); + CHECK(client.valid()); + + for (int i = 0; i < 100 && !accepted.valid(); i++) { accepted = server.accept(); platform::delayMs(1); } + CHECK(accepted.valid()); + + client.close(); + server.close(); +} + +// connectStart on a bad/empty host fails immediately (no hang); connectPoll on an unstarted +// connection reports Failed rather than blocking. +TEST_CASE("platform::TcpConnection::connectStart rejects a bad host; poll on no-fd is Failed") { + platform::TcpConnection client; + CHECK_FALSE(client.connectStart("", 1883)); + CHECK_FALSE(client.connectStart(nullptr, 1883)); + CHECK(client.connectPoll() == platform::TcpConnection::ConnectResult::Failed); +} diff --git a/test/unit/light/unit_Drivers_container.cpp b/test/unit/light/unit_Drivers_container.cpp index bb259afb..36f8831f 100644 --- a/test/unit/light/unit_Drivers_container.cpp +++ b/test/unit/light/unit_Drivers_container.cpp @@ -26,6 +26,44 @@ class CountingDriver : public mm::DriverBase { } // namespace +// A driver that captures the Correction* it's handed, so a test can read the resulting LUT. +class CorrectionCapturingDriver : public mm::DriverBase { +public: + void setSourceBuffer(mm::Buffer*) override {} + void loop() override {} + void setCorrection(const mm::Correction* c) override { correction = c; } + const mm::Correction* correction = nullptr; +}; + +// The `on` control is master power: on=false scales the correction LUT to zero (output black) while +// PRESERVING the brightness value, so on=true restores the exact level. It rides the same cheap LUT +// rebuild as brightness (no pipeline realloc). This pins the shared power control IR/MQTT/WLED drive. +TEST_CASE("Drivers::on gates the correction LUT without clobbering brightness") { + mm::Drivers drivers; + CorrectionCapturingDriver drv; + drivers.addChild(&drv); + drivers.setup(); // seeds correction_ from on(true)+brightness, hands it to drv + REQUIRE(drv.correction != nullptr); + + drivers.brightness = 200; + drivers.on = true; + drivers.onUpdate("brightness"); // rebuild LUT at full power + CHECK(drivers.effectiveBrightness() == 200); + CHECK(drv.correction->briLut[255] == 200); // (255 * 200) / 255 == 200 + + // Turn off → LUT scales to black, but the brightness value is untouched. + drivers.on = false; + drivers.onUpdate("on"); + CHECK(drivers.brightness == 200); // value preserved + CHECK(drivers.effectiveBrightness() == 0); + CHECK(drv.correction->briLut[255] == 0); // output black + + // Turn back on → the exact level returns, no stored-value juggling. + drivers.on = true; + drivers.onUpdate("on"); + CHECK(drv.correction->briLut[255] == 200); +} + // Disabled child drivers don't tick: toggling `enabled` flips whether that driver's loop() runs. TEST_CASE("Drivers::loop() skips disabled child drivers") { mm::Drivers drivers; diff --git a/test/unit/light/unit_Palette.cpp b/test/unit/light/unit_Palette.cpp index a67bb487..d414a133 100644 --- a/test/unit/light/unit_Palette.cpp +++ b/test/unit/light/unit_Palette.cpp @@ -76,3 +76,50 @@ TEST_CASE("Palettes::active swaps the global palette on setActive") { // Restore the default active palette — it's a global other effects' tests read. mm::Palettes::setActive(0); } + +// The HomeKit-colour-wheel → palette mapping (MQTT/Homebridge). Each palette's representative +// (hue, sat) is computed from its expanded entries; nearestForHue picks the closest. A vivid hue +// snaps to that hue's palette family; a low-saturation target snaps to the desaturated Rainbow. +TEST_CASE("Palettes::nearestForHue maps a colour to the closest palette") { + // A vivid red hue lands on a red/orange-family palette (Party≈13° / Lava≈24° are the reds), + // never on the all-hue Rainbow (index 0, which has ~0 saturation). + const uint8_t redIdx = mm::Palettes::nearestForHue(5, 255); + CHECK(redIdx != 0); + uint16_t rh = 0, rs = 0; mm::Palettes::representativeHueSat(redIdx, rh, rs); + CHECK(rh < 45); // the picked palette is genuinely red/orange + + // A vivid blue hue lands on a blue-family palette (Ocean≈195° / Fierce Ice≈213°). + const uint8_t blueIdx = mm::Palettes::nearestForHue(210, 255); + CHECK(blueIdx != 0); + uint16_t bh = 0, bs = 0; mm::Palettes::representativeHueSat(blueIdx, bh, bs); + CHECK(bh > 150); + CHECK(bh < 260); // genuinely blue, not green or red + + // A vivid green hue lands on a green-family palette (Forest≈111°). + const uint8_t greenIdx = mm::Palettes::nearestForHue(120, 255); + uint16_t gh = 0, gs = 0; mm::Palettes::representativeHueSat(greenIdx, gh, gs); + CHECK(gh > 60); + CHECK(gh < 180); + + // Very low saturation (the wheel's desaturated centre) → the low-sat Rainbow (index 0). + CHECK(mm::Palettes::nearestForHue(0, 5) == 0); + + // Hue wraps: 359° is adjacent to 0°, so it picks the same red family as ~0°. + const uint8_t wrapIdx = mm::Palettes::nearestForHue(359, 255); + uint16_t wh = 0, ws = 0; mm::Palettes::representativeHueSat(wrapIdx, wh, ws); + CHECK((wh < 45 || wh > 315)); // red/orange either side of the 0/360 seam +} + +// Regression (reviewer #4): a hue >= 360 (a broker client can send "400,…" on hsv/set) must not +// overflow the int32 squared-distance math — nearestForHue folds any input into 0..359 up front. +// 400 % 360 == 40 (orange), 720 % 360 == 0 (red), so both resolve to a valid, sensible index. +TEST_CASE("Palettes::nearestForHue folds an out-of-range hue instead of overflowing") { + const uint8_t i400 = mm::Palettes::nearestForHue(400, 255); // == 40° + const uint8_t i40 = mm::Palettes::nearestForHue(40, 255); + CHECK(i400 == i40); // 400 folds to 40 → same palette + const uint8_t i720 = mm::Palettes::nearestForHue(720, 255); // == 0° + const uint8_t i0 = mm::Palettes::nearestForHue(0, 255); + CHECK(i720 == i0); // 720 folds to 0 → same palette + // A large value doesn't crash / return garbage — any index < kCount is acceptable. + CHECK(mm::Palettes::nearestForHue(65535, 200) < mm::palettes::kCount); +} diff --git a/test/unit/light/unit_RmtLedDriver_pins.cpp b/test/unit/light/unit_RmtLedDriver_pins.cpp index 730fd047..20f4aba1 100644 --- a/test/unit/light/unit_RmtLedDriver_pins.cpp +++ b/test/unit/light/unit_RmtLedDriver_pins.cpp @@ -124,6 +124,46 @@ TEST_CASE("assignCounts clamps so the sum never exceeds the buffer") { CHECK(counts[0] == 250); } +TEST_CASE("assignCounts clamps a pin to the WS2812 ceiling and warns (drives 2048, not zero)") { + // A 128×128 grid (16384 lights) all on one pin is ~490 ms/frame (FPS 1) on a WS2812 + // line. The driver drives the first kMaxWs2812LedsPerPin (output stays lit) — count + // clamped, not zeroed — and `warn` is set (not nullptr) so the driver shows a Warning + // while still running. The return stays nullptr (a clamp is not an idling error). + mm::nrOfLightsType counts[8] = {}; + const char* warn = nullptr; + CHECK(mm::assignCounts("", 1, 16384, counts, mm::kMaxWs2812LedsPerPin, &warn) == nullptr); + CHECK(counts[0] == mm::kMaxWs2812LedsPerPin); + CHECK(warn == mm::kClampedWarning); // warned, but not an error + + // Without a cap (a clocked-SPI type), the same 16384 passes unclamped, no warning. + warn = mm::kClampedWarning; // ensure it's reset to null on the no-clamp path + CHECK(mm::assignCounts("", 1, 16384, counts, /*maxPerPin=*/0, &warn) == nullptr); + CHECK(counts[0] == 16384); + CHECK(warn == nullptr); + + // The ceiling is per-pin, not total: 4096 over two pins (2048 each) fits, no warning. + CHECK(mm::assignCounts("", 2, 4096, counts, mm::kMaxWs2812LedsPerPin, &warn) == nullptr); + CHECK(counts[0] == mm::kMaxWs2812LedsPerPin); + CHECK(counts[1] == mm::kMaxWs2812LedsPerPin); + CHECK(warn == nullptr); + + // An explicit SMALL count is the user's choice, not a clamp — must NOT warn. + CHECK(mm::assignCounts("100", 1, 1000, counts, mm::kMaxWs2812LedsPerPin, &warn) == nullptr); + CHECK(counts[0] == 100); + CHECK(warn == nullptr); + + // An explicit OVERSIZED count on pin 0 clamps + warns, while the unlisted pins still + // split the correct remainder. Buffer 6000 over 3 pins, "5000" explicit on pin 0: + // pin 0 wants 5000 → clamped to 2048 (warn); remaining 6000-5000=1000 splits over the + // 2 unlisted pins (500 each). The clamp does NOT redistribute the trimmed 2952 — the + // remainder is computed from the REQUESTED count, matching the driver's slice offsets. + CHECK(mm::assignCounts("5000", 3, 6000, counts, mm::kMaxWs2812LedsPerPin, &warn) == nullptr); + CHECK(counts[0] == mm::kMaxWs2812LedsPerPin); // clamped from 5000 + CHECK(counts[1] == 500); // (6000 - 5000) / 2 + CHECK(counts[2] == 500); + CHECK(warn == mm::kClampedWarning); +} + TEST_CASE("assignCounts handles a zero-light buffer (0×0×0 grid) as all-zero") { mm::nrOfLightsType counts[8] = {0xFF, 0xFF, 0xFF}; CHECK(mm::assignCounts("", 3, 0, counts) == nullptr); diff --git a/web-installer/README.md b/web-installer/README.md index 6a0b59e0..d3d5aa4b 100644 --- a/web-installer/README.md +++ b/web-installer/README.md @@ -69,7 +69,7 @@ labelled summary with a thumbnail: A flat JSON array of catalog entries. Each entry is the single source of truth for one piece of hardware and what to set up on it at install time. Two clients consume it identically — the web installer (`install-orchestrator.js`, which emits -the entry's units as APPLY_OP ops over serial) and MoonDeck (`scripts/moondeck.py`, +the entry's units as APPLY_OP ops over serial) and MoonDeck (`moondeck/moondeck.py`, which POSTs them over HTTP REST on the LAN) — so **adding another module-with-controls unit needs no client change** (both walk the same entry; only the transport differs). @@ -140,6 +140,7 @@ declares only what is actually on that board. | `url` | no | product-page link the picker shows next to the board (the vendor's own page, e.g. `https://quinled.info/quinled-dig2go/`). A remote URL is fine here — it's a click-through link, not an asset the installer fetches | | `supported` | no | short capability labels the firmware drives on this board *today* (e.g. `["LEDs", "WiFi", "Ethernet"]`), grounded in the modules the entry actually adds. Rendered as solid chips on the picker card | | `planned` | no | short labels for peripherals the board physically has but no module drives *yet* (e.g. `["IR receiver", "Onboard button"]`) — the backlog seed for future spec + test work. Rendered as dashed "(soon)" chips on the picker card | +| `flashBaud` | no | esptool flash baud for this board, overriding the audience default (installer 460800, CLI/MoonDeck 921600). Set it only when a board's USB bridge needs a non-default rate — e.g. a flaky CH340 pinned to `460800`. One of the standard rates (`115200`/`230400`/`460800`/`921600`); `check_devices.py` validates it | | `modules` | yes | the list of module-with-controls units that set the board up | Each `modules[]` unit is `{ type, id, parent_id?, controls? }`: @@ -239,7 +240,7 @@ same two operations per module: `add_module {type, id, parent_id}` (== "create a module, then configure it." (A scenario keeps a separate `props` block for in-process C++ construction wiring — `setLayouts`/`setChannelsPerLight`/grid dims that aren't control writes — which the catalog never needs, so the catalog -unit carries only `controls`.) `scripts/scenario/run_live_scenario.py` already +unit carries only `controls`.) `moondeck/scenario/run_live_scenario.py` already runs these ops over HTTP against a live device, the same channel the installer uses. ## What's *not* in this directory @@ -271,7 +272,7 @@ the install button can actually flash. Quickest. In MoonDeck: **PC tab → Preview Installer**. Or from the CLI: ```bash -uv run scripts/run/preview_installer.py +uv run moondeck/run/preview_installer.py # open http://localhost:8000/ in Chrome / Edge / Opera ``` @@ -309,7 +310,7 @@ for F in esp32 esp32-eth esp32s3-n16r8; do TMP=$(mktemp -d) gh run download "$RUN_ID" -n "esp32-$F" -D "$TMP" cp "$TMP"/*.bin "$DIST/releases/$TAG/" - python3 scripts/build/generate_manifest.py \ + python3 moondeck/build/generate_manifest.py \ --firmware "$F" --version "$V" \ --release-url . \ --flasher-args "$TMP/flasher-$F.json" \ diff --git a/web-installer/deviceModels.json b/web-installer/deviceModels.json index 93f9807d..b44de44c 100644 --- a/web-installer/deviceModels.json +++ b/web-installer/deviceModels.json @@ -49,6 +49,7 @@ "firmwares": [ "esp32" ], + "flashBaud": 460800, "image": "assets/boards/lolin-d32.jpg", "url": "https://www.wemos.cc/en/latest/d32/d32.html", "supported": [ @@ -284,7 +285,8 @@ "ethClockGpio": 17, "ethClockExtIn": false, "ethMdcGpio": 23, - "ethMdioGpio": 18 + "ethMdioGpio": 18, + "ethRstGpio": -1 } } ] @@ -701,6 +703,19 @@ "type": "NetworkReceiveEffect", "id": "NetworkReceive", "parent_id": "Layer" + }, + { + "type": "NetworkModule", + "id": "Network", + "controls": { + "ethType": 2, + "ethPhyAddr": 1, + "ethMdcGpio": 31, + "ethMdioGpio": 52, + "ethRstGpio": 51, + "ethClockGpio": 50, + "ethClockExtIn": true + } } ] }, @@ -942,7 +957,7 @@ ] }, { - "name": "Espressif ESP32-S31 Function-CoreBoard-1", + "name": "Espressif ESP32-S31 CoreBoard", "chip": "ESP32-S31", "firmwares": [ "esp32s31" @@ -968,7 +983,7 @@ "type": "System", "id": "System", "controls": { - "deviceModel": "Espressif ESP32-S31 Function-CoreBoard-1" + "deviceModel": "Espressif ESP32-S31 CoreBoard" } }, { @@ -982,7 +997,13 @@ { "type": "NetworkModule", "id": "Network", - "controls": {} + "controls": { + "ethType": 4, + "ethPhyAddr": -1, + "ethMdcGpio": 5, + "ethMdioGpio": 6, + "ethRstGpio": 7 + } }, { "type": "AudioModule", @@ -1004,5 +1025,80 @@ } } ] + }, + { + "name": "Shelly", + "chip": "ESP32", + "firmwares": [ + "esp32" + ], + "flashBaud": 460800, + "image": "assets/boards/shelly.jpg", + "supported": [ + "LEDs", + "WiFi", + "MQTT", + "Hue" + ], + "modules": [ + { + "type": "System", + "id": "System", + "controls": { + "deviceModel": "Shelly", + "deviceName": "Shelly" + } + }, + { + "type": "GridLayout", + "id": "Grid", + "parent_id": "Layouts", + "controls": { + "width": 1, + "height": 24 + } + }, + { + "type": "Layer", + "id": "Layer", + "parent_id": "Layers" + }, + { + "type": "SolidEffect", + "id": "Solid", + "parent_id": "Layer" + }, + { + "type": "Drivers", + "id": "Drivers", + "controls": { + "lightPreset": 0 + } + }, + { + "type": "RmtLedDriver", + "id": "RmtLed", + "parent_id": "Drivers", + "controls": { + "pins": "2" + } + }, + { + "type": "HueDriver", + "id": "Hue", + "parent_id": "Drivers" + }, + { + "type": "NetworkModule", + "id": "Network", + "controls": { + "txPowerSetting": 8 + } + }, + { + "type": "MqttModule", + "id": "Mqtt" + } + ] } ] diff --git a/web-installer/improv-frame.js b/web-installer/improv-frame.js index 9051df88..eb83374d 100644 --- a/web-installer/improv-frame.js +++ b/web-installer/improv-frame.js @@ -5,7 +5,7 @@ // SDK). The orchestrator imports these for the actual send; test/js imports them // to pin the byte layout. This is the SAME wire format three implementations must // agree on: device C++ (src/core/ImprovFrame.h + platform_esp32_improv.cpp), -// Python (scripts/build/improv_provision.py), and this file. The frame-contract +// Python (moondeck/build/improv_provision.py), and this file. The frame-contract // tests (test/js, test/python) assert a shared golden vector so they can't drift. // // Frame layout (matches src/core/ImprovFrame.h): diff --git a/web-installer/install-orchestrator.js b/web-installer/install-orchestrator.js index 30297a00..6cfaee64 100644 --- a/web-installer/install-orchestrator.js +++ b/web-installer/install-orchestrator.js @@ -73,7 +73,7 @@ import { planConfigOps } from "./config-ops.js"; // esptool-js with undefined data. // // Manifest shape (single build per file — projectMM generates one per -// firmware variant; see scripts/build/generate_manifest.py): +// firmware variant; see moondeck/build/generate_manifest.py): // { name, version, builds: [{ chipFamily, parts: [{ path, offset }] }] } async function fetchManifest(manifestUrl) { // Resolve the manifest URL against the document so a relative path like @@ -191,21 +191,47 @@ async function pushDefaultsOverSerial(port, board, applyDefaults, trackProgress, return await sendConfigOverSerial(port, board, onLog); } -// Push a device-model's whole catalog config to the device over serial. Walks the SAME -// deviceModels.json entry the HTTP path used (see planConfigOps) but emits APPLY_OP ops -// instead of HTTP requests — so the defaults apply during provisioning with no HTTP and -// no browser handoff. Returns true if the entry was found + pushed, false if none. -async function sendConfigOverSerial(port, board, onLog) { - let entry; +// The installer's default esptool-js flash baud: SAFE, because the installer serves unknown +// walk-up hardware (a cheap CH340 clone, a bad cable). A well-tested compromise across CH340 / +// CP2102 / FT232 bridges. A board overrides it in deviceModels.json via `flashBaud` — today +// only down, for a bridge known to stall faster (the LOLIN D32's CH340 → 460800, already the +// default here, so a no-op in the installer but a real opt-down for the CLI's fast default). +// Native-USB boards (P4) ignore baud. (The CLI / MoonDeck path defaults FAST instead — see +// flash_esp32.py; same catalog field, opposite default, different audience.) +const DEFAULT_FLASH_BAUD = 460800; + +// Fetch a board's deviceModels.json catalog entry by name, or null (unknown board, or any +// fetch/parse failure — callers decide how to fall back). The single catalog read both the +// flash-baud lookup and the serial-config push share, so they can't diverge on how the +// catalog is fetched or matched. +async function fetchCatalogEntry(board, onLog) { + if (!board) return null; try { const res = await fetch("./deviceModels.json", { signal: AbortSignal.timeout(5000) }); - if (!res.ok) return false; + if (!res.ok) return null; const catalog = await res.json(); - entry = Array.isArray(catalog) ? catalog.find(b => b && b.name === board) : null; + return Array.isArray(catalog) ? catalog.find(b => b && b.name === board) || null : null; } catch (e) { - if (onLog) onLog(`[orchestrator] serial config: catalog fetch failed (${e && e.message || e})`); - return false; + if (onLog) onLog(`[orchestrator] catalog lookup failed for ${board} (${e && e.message || e})`); + return null; } +} + +// Resolve a board's flash baud from the catalog: the entry's `flashBaud` when set, +// else DEFAULT_FLASH_BAUD. Best-effort — any fetch/parse failure or unknown board +// falls back to the safe default, so a catalog hiccup never blocks a flash. Mirrors +// the CLI's flash_esp32.py `_catalog_flash_baud`, keeping both flash paths in step. +async function catalogFlashBaud(board, onLog) { + const entry = await fetchCatalogEntry(board, onLog); + return entry && Number.isInteger(entry.flashBaud) ? entry.flashBaud : DEFAULT_FLASH_BAUD; +} + +// Push a device-model's whole catalog config to the device over serial. Walks the SAME +// deviceModels.json entry the HTTP path used (see planConfigOps) but emits APPLY_OP ops +// instead of HTTP requests — so the defaults apply during provisioning with no HTTP and +// no browser handoff. Returns true if the entry was found + pushed, false if none. +async function sendConfigOverSerial(port, board, onLog) { + const entry = await fetchCatalogEntry(board, onLog); if (!entry) return false; for (const op of planConfigOps(entry)) { await sendApplyOpFrame(port, op); @@ -593,6 +619,26 @@ export const installer = { ({ port, transport, esploader } = _detected); const chipName = _detected.chipName; _detected = null; + // detect() left the loader connected at its baudrate (before the board was + // known). If this board's catalog flashBaud differs, re-negotiate now, so a + // reused detect() connection honours the override like the fresh-connect branch. + const flashBaud = await catalogFlashBaud(board, onLog); + const priorBaud = esploader.baudrate; + if (flashBaud !== priorBaud) { + esploader.baudrate = flashBaud; + if (onLog) onLog(`[orchestrator] flash baud: ${flashBaud} (re-negotiated on reused connection)`); + try { + await esploader.changeBaud(); + } catch (e) { + // A failed re-negotiation isn't fatal: best case the port is still at the + // prior baud (the change command failed), so restore that and flash at it + // rather than aborting. If the command succeeded but the re-open failed the + // device is at the new baud and the flash will fail fast + visibly — still + // better than aborting outright, and no worse than the pre-existing path. + esploader.baudrate = priorBaud; + if (onLog) onLog(`[orchestrator] baud re-negotiation failed (${e && e.message || e}); flashing at ${priorBaud}`); + } + } trackProgress("connect-flash", { chipName }); } else { let probeFailed = false; @@ -639,14 +685,15 @@ export const installer = { trackProgress("connect-flash"); transport = new Transport(port, false); - // 460800 baud is ESP Web Tools' default — a well-tested compromise - // that works across CH340 / CP2102 / FT232 USB-serial chips and - // cheap cables. 921600 is faster but caused FLASH_DEFL_BEGIN - // timeouts on the LOLIN D32's CH340. Bootloader sync runs at - // 115200 first (romBaudrate); esptool-js negotiates up after. + // Flash baud: the board's catalog `flashBaud` if set, else the safe 460800 + // default (a LOLIN-style CH340 pins 460800; native-USB boards ignore it). + // Bootloader sync runs at 115200 first (romBaudrate); esptool-js negotiates + // up to `baudrate` after. + const flashBaud = await catalogFlashBaud(board, onLog); + if (onLog) onLog(`[orchestrator] flash baud: ${flashBaud}`); esploader = new ESPLoader({ transport, - baudrate: 460800, + baudrate: flashBaud, romBaudrate: 115200, terminal: espTerminal, });