From a181ad51160db734152fe97e716e3bb5207f4e08 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Mon, 24 Aug 2026 03:04:36 +0900 Subject: [PATCH 1/7] docs: design trusted firmware publishing --- docs/designs/firmware-supply-chain.md | 108 ++++++++++++++++++++++++ docs/plans/firmware-publisher.md | 116 ++++++++++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 docs/designs/firmware-supply-chain.md create mode 100644 docs/plans/firmware-publisher.md diff --git a/docs/designs/firmware-supply-chain.md b/docs/designs/firmware-supply-chain.md new file mode 100644 index 0000000..9d9948c --- /dev/null +++ b/docs/designs/firmware-supply-chain.md @@ -0,0 +1,108 @@ +# Firmware Supply Chain Design + +Status: approved on 2026-08-24 + +## Goal + +Hana Cloud accepts reviewable Arduino sketch sources from same-repository and +fork pull requests, then publishes an upload-ready Intel HEX whose bytes are +provably derived from those sources. HanBeon downloads the HEX and does not +need Arduino CLI on the user's computer. + +## Trust boundary + +Contributors may edit only authoring inputs: + +- `sources/boards/.json` +- `boards/.ino` +- optional `boards/.png` + +The following are generated outputs and pull-request validation rejects direct +changes to them: + +- `boards/.hex` +- `boards/.json` +- the `boards` array in `registry.json` + +Application profiles and the `apps` array remain human-managed. The protected +`main` branch requires pull requests and validation, blocks deletion and force +pushes, and grants an always-on bypass only to the repository's GitHub Actions +App so the trusted publisher can add generated output. + +## Authoring input + +`sources/boards/.json` contains schema version, stable board ID, display +name, sketch path and FQBN, USB detection metadata, wiring, and optional image +metadata. It contains no generated path, size, or hash. A repository-owned +toolchain lock maps the FQBN platform to Arduino AVR Core 1.8.8 and pins +Arduino CLI 1.5.1. + +## Published contract + +The publisher creates a schema-version 2 board manifest with this firmware +shape: + +```json +{ + "path": "boards/arduino-uno-r3.hex", + "format": "intel-hex", + "size": 12345, + "sha256": "64 lowercase hex characters", + "fqbn": "arduino:avr:uno", + "source": { + "path": "boards/arduino-uno-r3.ino", + "sha256": "64 lowercase hex characters" + }, + "toolchain": { + "arduinoCli": "1.5.1", + "platform": "arduino:avr@1.8.8" + } +} +``` + +The publisher emits the normal application HEX, never the bootloader-inclusive +HEX. It stores byte size and SHA-256 for download limits and integrity checks. +The root registry receives the generated manifest hash and increments revision +only when its published contents change. + +## Data flow + +Pull-request validation runs with read-only permissions. It rejects generated +output edits, validates authoring data, and clean-compiles every affected +sketch twice with the pinned toolchain to prove reproducible output. + +After an accepted source change reaches `main`, the trusted publisher checks +out the latest `main`, performs the same two clean builds, compares the HEX +bytes, regenerates all derived files, and commits them atomically as +`github-actions[bot]`. A bot push made with `GITHUB_TOKEN` does not recursively +start another publisher run. Concurrent publishers serialize and regenerate +from the latest `main` before pushing. + +Between the source merge and generated commit, existing boards remain on their +previous valid HEX and newly added boards remain absent from `registry.json`. +No client is pointed at an incomplete artifact. + +## Failure handling + +- Compile failure, nondeterministic output, malformed Intel HEX, missing output, + unsafe path, schema failure, or hash mismatch stops publication. +- A failed publisher never changes `registry.json` or any public manifest. +- A non-fast-forward push causes a fresh generation attempt from current + `origin/main`; retries are bounded. +- The workflow uses exact toolchain and action versions and minimal token + permissions. + +## Verification + +Unit tests cover path validation, generated-file policy, canonical manifests, +hash/size calculation, registry revision behavior, and normal-versus-bootloader +artifact selection. Integration verification compiles the current Uno sketch +twice, compares bytes, checks Intel HEX syntax, and confirms regeneration leaves +the repository clean. The initial generated commit must contain a HEX whose +hash matches both the public manifest and a fresh pinned-toolchain build. + +## Non-goals + +This wave does not implement firmware upload in HanBeon, identify boards at +runtime, or bundle an uploader executable. Those remain in the separate +HanBeon uploader pull request. diff --git a/docs/plans/firmware-publisher.md b/docs/plans/firmware-publisher.md new file mode 100644 index 0000000..ce1f99e --- /dev/null +++ b/docs/plans/firmware-publisher.md @@ -0,0 +1,116 @@ +# Hana Cloud Firmware Publisher Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Compile Hana Cloud Arduino sources into trusted Intel HEX artifacts and publish their manifests and registry hashes automatically. + +**Architecture:** Fork pull requests can change authoring inputs but never generated firmware. A read-only validator compiles proposed sketches, while a post-merge GitHub Actions publisher with the sole branch-protection bypass performs two clean builds and atomically commits generated HEX, manifests, and registry board metadata. + +**Tech Stack:** Python 3 standard library, Arduino CLI, Arduino AVR Core, JSON Schema 2020-12, GitHub Actions. + +**Spec:** `docs/designs/firmware-supply-chain.md` + +## Global Constraints + +- `docs/superpowers` must not be created or tracked. +- Fork pull-request jobs have read-only repository permissions. +- Arduino CLI and every platform core use exact versions from `.github/firmware-toolchain.json`. +- Only the normal application Intel HEX is published; `with_bootloader.hex` is forbidden. +- Generated JSON is UTF-8, two-space indented, key-order stable, and ends with one LF. +- Generated HEX is capped at 2 MiB and hashed as exact repository bytes. +- Human pull requests cannot change `boards/*.hex`, `boards/*.json`, or `registry.json#/boards`. +- HanBeon uploader code is outside this plan. + +--- + +### Task 1: Authoring schema and deterministic metadata renderer + +**Files:** +- Create: `sources/boards/arduino-uno-r3.json` +- Create: `schemas/board-source.schema.json` +- Create: `.github/firmware-toolchain.json` +- Create: `scripts/firmware.py` +- Create: `tests/test_firmware.py` + +**Interfaces:** +- Consumes: source descriptors and toolchain lock JSON. +- Produces: `load_sources(root: Path)`, `render_manifest(source, firmware)`, and `render_registry(registry, published_boards)`. + +- [ ] **Step 1: Write failing unit tests** for unsafe paths, missing source fields, canonical schema-version 2 manifest output, registry app preservation, and revision increments only on board changes. +- [ ] **Step 2: Run `python -m unittest -v tests.test_firmware`** and confirm failures are caused by missing production functions. +- [ ] **Step 3: Implement strict dataclasses, validators, canonical JSON, manifest rendering, and registry rendering** in `scripts/firmware.py`. +- [ ] **Step 4: Run `python -m unittest -v tests.test_firmware`** and confirm all Task 1 tests pass. +- [ ] **Step 5: Commit** with `feat: define firmware publishing inputs`. + +### Task 2: Reproducible Arduino compiler and generated-file policy + +**Files:** +- Modify: `scripts/firmware.py` +- Modify: `tests/test_firmware.py` +- Modify: `.gitattributes` + +**Interfaces:** +- Consumes: `BoardSource`, Arduino CLI path, and a temporary build root. +- Produces: `compile_reproducible(root: Path, source: BoardSource, cli: str) -> bytes`, `is_intel_hex(data: bytes) -> bool`, and `validate_pr_changes(root: Path, base: str) -> None`. + +- [ ] **Step 1: Write failing tests** proving normal `.ino.hex` selection, rejection of bootloader-only output, byte mismatch rejection, Intel HEX validation, and generated path/registry-board edit rejection. +- [ ] **Step 2: Run the focused unit tests** and verify expected failures. +- [ ] **Step 3: Implement staged sketch directories, two `arduino-cli compile --clean` invocations, exact byte comparison, size limits, Intel HEX checks, and semantic PR diff policy.** +- [ ] **Step 4: Run the full unit suite** and verify it passes. +- [ ] **Step 5: Commit** with `feat: build reproducible Arduino firmware`. + +### Task 3: Published schema and current Uno artifact verification + +**Files:** +- Modify: `schemas/board.schema.json` +- Modify: `README.md` +- Generated after merge: `boards/arduino-uno-r3.hex` +- Generated after merge: `boards/arduino-uno-r3.json` +- Generated after merge: `registry.json` + +**Interfaces:** +- Consumes: the current `boards/arduino-uno-r3.ino` and pinned toolchain. +- Produces: schema-version 2 public manifest and verified Intel HEX bytes. + +- [ ] **Step 1: Add failing schema/renderer fixtures** for `format`, `size`, source provenance, and toolchain provenance. +- [ ] **Step 2: Run tests and confirm the legacy public schema fails the new fixtures.** +- [ ] **Step 3: Update the public schema and documentation without manually adding generated output.** +- [ ] **Step 4: Install the locked toolchain in a temporary directory, compile the Uno sketch twice, and compare the exact HEX bytes.** +- [ ] **Step 5: Run generation in check/temporary mode and verify the prospective manifest and registry hashes.** +- [ ] **Step 6: Commit** with `docs: define compiled firmware contract`. + +### Task 4: Fork-safe validation and trusted publisher workflows + +**Files:** +- Create: `.github/workflows/validate.yml` +- Create: `.github/workflows/publish-firmware.yml` +- Create: `.github/CODEOWNERS` +- Modify: `scripts/firmware.py` +- Modify: `tests/test_firmware.py` + +**Interfaces:** +- Consumes: pull-request base SHA or protected `main` head. +- Produces: required `validate` check and an atomic `github-actions[bot]` generated commit. + +- [ ] **Step 1: Write failing tests** for no-op generation, bounded stale-main retry decisions, and generated commit contents. +- [ ] **Step 2: Run tests and verify expected failures.** +- [ ] **Step 3: Implement `validate-pr`, `compile`, `generate`, and `verify` CLI commands.** +- [ ] **Step 4: Add read-only fork PR validation and serialized `main` publisher workflows with exact action/toolchain versions and minimal permissions.** +- [ ] **Step 5: Run unit tests, workflow syntax checks, descriptor validation, and a clean local generation verification.** +- [ ] **Step 6: Commit** with `ci: publish compiled firmware from trusted sources`. + +### Task 5: Merge, bot publication, and repository enforcement + +**Files:** +- No additional source files unless verification reveals a defect. + +**Interfaces:** +- Consumes: merged workflow and GitHub Actions App integration. +- Produces: published Uno HEX on `main` and an active main-branch ruleset. + +- [ ] **Step 1: Run all local verification**: unit tests, Python compilation, JSON parsing/schema checks, two-build HEX comparison, and `git diff --check`. +- [ ] **Step 2: Open the Hana Cloud pull request and wait for the fork-safe validation check.** +- [ ] **Step 3: Merge the PR using its exact head SHA.** +- [ ] **Step 4: Observe the publisher bot commit and verify HEX size/hash, manifest hash, registry revision, and fresh-build byte equality.** +- [ ] **Step 5: Create an active `main` ruleset requiring pull requests and `validate`, blocking deletion/force pushes, and granting always-on bypass only to the GitHub Actions App.** +- [ ] **Step 6: Audit the ruleset and repository tree, then record final SHAs and URLs.** From ce630ee3fdc4bceb07b4b0d823e21a79e90d28ac Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Mon, 24 Aug 2026 03:08:29 +0900 Subject: [PATCH 2/7] feat: define firmware publishing inputs --- .github/firmware-toolchain.json | 7 + schemas/board-source.schema.json | 53 ++++++ scripts/firmware.py | 266 +++++++++++++++++++++++++++++ sources/boards/arduino-uno-r3.json | 60 +++++++ tests/__init__.py | 1 + tests/test_firmware.py | 170 ++++++++++++++++++ 6 files changed, 557 insertions(+) create mode 100644 .github/firmware-toolchain.json create mode 100644 schemas/board-source.schema.json create mode 100644 scripts/firmware.py create mode 100644 sources/boards/arduino-uno-r3.json create mode 100644 tests/__init__.py create mode 100644 tests/test_firmware.py diff --git a/.github/firmware-toolchain.json b/.github/firmware-toolchain.json new file mode 100644 index 0000000..33074b4 --- /dev/null +++ b/.github/firmware-toolchain.json @@ -0,0 +1,7 @@ +{ + "schemaVersion": 1, + "arduinoCli": "1.5.1", + "platforms": { + "arduino:avr": "1.8.8" + } +} diff --git a/schemas/board-source.schema.json b/schemas/board-source.schema.json new file mode 100644 index 0000000..09a6ecf --- /dev/null +++ b/schemas/board-source.schema.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/dev-five-git/hana-cloud/main/schemas/board-source.schema.json", + "title": "Hana Board Authoring Source", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "id", "name", "sketch", "detect", "wiring"], + "properties": { + "schemaVersion": { "const": 1 }, + "id": { "type": "string", "pattern": "^[a-z0-9]+(?:[.-][a-z0-9]+)*$" }, + "name": { "type": "string", "minLength": 1, "maxLength": 100 }, + "sketch": { + "type": "object", + "additionalProperties": false, + "required": ["path", "fqbn"], + "properties": { + "path": { "type": "string", "pattern": "^boards/[a-z0-9]+(?:-[a-z0-9]+)*\\.ino$" }, + "fqbn": { "type": "string", "pattern": "^[A-Za-z0-9_-]+:[A-Za-z0-9_-]+:[A-Za-z0-9_-]+$" } + } + }, + "detect": { + "type": "object", + "additionalProperties": false, + "required": ["usb"], + "properties": { + "usb": { "type": "array", "minItems": 1, "items": { "$ref": "registry.schema.json#/$defs/usbMatch" } } + } + }, + "wiring": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["from", "to"], + "properties": { + "from": { "type": "string", "minLength": 1, "maxLength": 40 }, + "to": { "type": "string", "minLength": 1, "maxLength": 100 }, + "note": { "type": "string", "minLength": 1, "maxLength": 200 } + } + } + }, + "image": { + "type": "object", + "additionalProperties": false, + "required": ["path", "alt"], + "properties": { + "path": { "type": "string", "pattern": "^boards/[a-z0-9]+(?:-[a-z0-9]+)*\\.png$" }, + "alt": { "type": "string", "minLength": 1, "maxLength": 300 } + } + } + } +} diff --git a/scripts/firmware.py b/scripts/firmware.py new file mode 100644 index 0000000..64873e4 --- /dev/null +++ b/scripts/firmware.py @@ -0,0 +1,266 @@ +"""Build and validate generated Hana Cloud firmware artifacts.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +ID_PATTERN = re.compile(r"^[a-z0-9]+(?:[.-][a-z0-9]+)*$") +SLUG_PATTERN = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +FQBN_PATTERN = re.compile(r"^[A-Za-z0-9_-]+:[A-Za-z0-9_-]+:[A-Za-z0-9_-]+$") +USB_ID_PATTERN = re.compile(r"^[0-9a-f]{4}$") +CONFIDENCE_VALUES = {"exact", "likely", "ambiguous"} + + +def canonical_json(value: Any) -> bytes: + return (json.dumps(value, ensure_ascii=False, indent=2) + "\n").encode("utf-8") + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _object(value: Any, field: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValueError(f"{field} must be an object") + return value + + +def _array(value: Any, field: str, *, nonempty: bool = False) -> list[Any]: + if not isinstance(value, list) or (nonempty and not value): + suffix = " a non-empty array" if nonempty else " an array" + raise ValueError(f"{field} must be{suffix}") + return value + + +def _keys(value: dict[str, Any], required: set[str], optional: set[str], field: str) -> None: + missing = required - value.keys() + unknown = value.keys() - required - optional + if missing: + raise ValueError(f"{field} is missing fields: {', '.join(sorted(missing))}") + if unknown: + raise ValueError(f"{field} has unknown fields: {', '.join(sorted(unknown))}") + + +def _text(value: Any, field: str, maximum: int) -> str: + if not isinstance(value, str) or not 1 <= len(value) <= maximum: + raise ValueError(f"{field} must contain 1 to {maximum} characters") + return value + + +def _aliases(value: Any, field: str) -> list[str]: + aliases = _array(value, field) + rendered = [_text(alias, field, 100) for alias in aliases] + if len(set(rendered)) != len(rendered): + raise ValueError(f"{field} contains duplicate aliases") + return rendered + + +def _validate_detect(value: Any) -> dict[str, Any]: + detect = _object(value, "detect") + _keys(detect, {"usb"}, set(), "detect") + usb = _array(detect["usb"], "detect.usb", nonempty=True) + rendered = [] + for index, raw_matcher in enumerate(usb): + field = f"detect.usb[{index}]" + matcher = _object(raw_matcher, field) + _keys( + matcher, + {"vid", "pid", "confidence"}, + {"manufacturerAliases", "productAliases"}, + field, + ) + vid = matcher["vid"] + pid = matcher["pid"] + confidence = matcher["confidence"] + if not isinstance(vid, str) or not USB_ID_PATTERN.fullmatch(vid): + raise ValueError(f"{field}.vid must be four lowercase hex characters") + if not isinstance(pid, str) or not USB_ID_PATTERN.fullmatch(pid): + raise ValueError(f"{field}.pid must be four lowercase hex characters") + if confidence not in CONFIDENCE_VALUES: + raise ValueError(f"{field}.confidence is unsupported") + item: dict[str, Any] = {"vid": vid, "pid": pid, "confidence": confidence} + if "manufacturerAliases" in matcher: + item["manufacturerAliases"] = _aliases( + matcher["manufacturerAliases"], f"{field}.manufacturerAliases" + ) + if "productAliases" in matcher: + item["productAliases"] = _aliases( + matcher["productAliases"], f"{field}.productAliases" + ) + rendered.append(item) + return {"usb": rendered} + + +def _validate_wiring(value: Any) -> list[dict[str, str]]: + wiring = _array(value, "wiring", nonempty=True) + rendered = [] + for index, raw_connection in enumerate(wiring): + field = f"wiring[{index}]" + connection = _object(raw_connection, field) + _keys(connection, {"from", "to"}, {"note"}, field) + item = { + "from": _text(connection["from"], f"{field}.from", 40), + "to": _text(connection["to"], f"{field}.to", 100), + } + if "note" in connection: + item["note"] = _text(connection["note"], f"{field}.note", 200) + rendered.append(item) + return rendered + + +@dataclass(frozen=True) +class BoardSource: + slug: str + id: str + name: str + sketch_path: str + fqbn: str + detect: dict[str, Any] + wiring: list[dict[str, str]] + image: dict[str, str] | None = None + + @classmethod + def from_document(cls, slug: str, raw: Any) -> "BoardSource": + if not SLUG_PATTERN.fullmatch(slug): + raise ValueError("source filename slug is invalid") + document = _object(raw, "source") + _keys( + document, + {"schemaVersion", "id", "name", "sketch", "detect", "wiring"}, + {"image"}, + "source", + ) + if document["schemaVersion"] != 1: + raise ValueError("source.schemaVersion must be 1") + board_id = document["id"] + if not isinstance(board_id, str) or not ID_PATTERN.fullmatch(board_id): + raise ValueError("source.id is invalid") + + sketch = _object(document["sketch"], "sketch") + _keys(sketch, {"path", "fqbn"}, set(), "sketch") + expected_sketch_path = f"boards/{slug}.ino" + if sketch["path"] != expected_sketch_path: + raise ValueError(f"sketch.path must be {expected_sketch_path}") + fqbn = sketch["fqbn"] + if not isinstance(fqbn, str) or not FQBN_PATTERN.fullmatch(fqbn): + raise ValueError("sketch.fqbn is invalid") + + image = None + if "image" in document: + raw_image = _object(document["image"], "image") + _keys(raw_image, {"path", "alt"}, set(), "image") + expected_image_path = f"boards/{slug}.png" + if raw_image["path"] != expected_image_path: + raise ValueError(f"image.path must be {expected_image_path}") + image = { + "path": expected_image_path, + "alt": _text(raw_image["alt"], "image.alt", 300), + } + + return cls( + slug=slug, + id=board_id, + name=_text(document["name"], "source.name", 100), + sketch_path=expected_sketch_path, + fqbn=fqbn, + detect=_validate_detect(document["detect"]), + wiring=_validate_wiring(document["wiring"]), + image=image, + ) + + +@dataclass(frozen=True) +class FirmwareArtifact: + path: str + data: bytes + source_sha256: str + arduino_cli: str + platform: str + image_sha256: str | None = None + + @property + def sha256(self) -> str: + return sha256_bytes(self.data) + + @property + def size(self) -> int: + return len(self.data) + + +def load_sources(root: Path) -> list[BoardSource]: + source_dir = root / "sources" / "boards" + if not source_dir.is_dir(): + raise ValueError("sources/boards directory is missing") + sources = [] + ids: set[str] = set() + for path in sorted(source_dir.glob("*.json")): + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"cannot read {path.relative_to(root).as_posix()}: {error}") from error + source = BoardSource.from_document(path.stem, raw) + if source.id in ids: + raise ValueError(f"duplicate source id: {source.id}") + ids.add(source.id) + sources.append(source) + if not sources: + raise ValueError("at least one board source is required") + return sources + + +def render_manifest(source: BoardSource, artifact: FirmwareArtifact) -> dict[str, Any]: + expected_path = f"boards/{source.slug}.hex" + if artifact.path != expected_path: + raise ValueError(f"firmware path must be {expected_path}") + firmware = { + "path": artifact.path, + "format": "intel-hex", + "size": artifact.size, + "sha256": artifact.sha256, + "fqbn": source.fqbn, + "source": { + "path": source.sketch_path, + "sha256": artifact.source_sha256, + }, + "toolchain": { + "arduinoCli": artifact.arduino_cli, + "platform": artifact.platform, + }, + } + manifest: dict[str, Any] = { + "schemaVersion": 2, + "id": source.id, + "firmware": firmware, + "wiring": copy.deepcopy(source.wiring), + } + if source.image is not None: + if artifact.image_sha256 is None: + raise ValueError("image sha256 is required when an image is configured") + manifest["image"] = { + "path": source.image["path"], + "sha256": artifact.image_sha256, + "alt": source.image["alt"], + } + return manifest + + +def render_registry( + registry: dict[str, Any], published_boards: list[dict[str, Any]] +) -> dict[str, Any]: + rendered = copy.deepcopy(registry) + previous = rendered.get("boards") + boards = copy.deepcopy(published_boards) + if previous != boards: + revision = rendered.get("revision") + if not isinstance(revision, int) or isinstance(revision, bool) or revision < 1: + raise ValueError("registry revision must be a positive integer") + rendered["revision"] = revision + 1 + rendered["boards"] = boards + return rendered diff --git a/sources/boards/arduino-uno-r3.json b/sources/boards/arduino-uno-r3.json new file mode 100644 index 0000000..1286038 --- /dev/null +++ b/sources/boards/arduino-uno-r3.json @@ -0,0 +1,60 @@ +{ + "schemaVersion": 1, + "id": "arduino.uno-r3", + "name": "Arduino Uno R3", + "sketch": { + "path": "boards/arduino-uno-r3.ino", + "fqbn": "arduino:avr:uno" + }, + "detect": { + "usb": [ + { + "vid": "2341", + "pid": "0043", + "confidence": "exact", + "manufacturerAliases": ["Arduino", "Arduino LLC", "Arduino (www.arduino.cc)"], + "productAliases": ["Arduino Uno", "Arduino Uno R3", "Arduino Uno Rev3"] + }, + { + "vid": "2341", + "pid": "0001", + "confidence": "exact", + "manufacturerAliases": ["Arduino", "Arduino LLC", "Arduino (www.arduino.cc)"], + "productAliases": ["Arduino Uno", "Arduino Uno R3", "Arduino Uno Rev3"] + }, + { + "vid": "2a03", + "pid": "0043", + "confidence": "exact", + "manufacturerAliases": ["Arduino", "Arduino SRL", "Arduino (www.arduino.cc)"], + "productAliases": ["Arduino Uno", "Arduino Uno R3", "Arduino Uno Rev3"] + }, + { + "vid": "2341", + "pid": "0243", + "confidence": "exact", + "manufacturerAliases": ["Arduino", "Arduino LLC", "Arduino (www.arduino.cc)"], + "productAliases": ["Arduino Uno", "Arduino Uno R3", "Arduino Uno Rev3"] + }, + { + "vid": "2341", + "pid": "006a", + "confidence": "exact", + "manufacturerAliases": ["Arduino", "Arduino LLC", "Arduino (www.arduino.cc)"], + "productAliases": ["Arduino Uno", "Arduino Uno R3", "Arduino Uno Rev3"] + } + ] + }, + "wiring": [ + { + "from": "D2", + "to": "순간 누름 스위치 NO 단자", + "note": "스위치 COM 단자는 GND에 연결" + }, + { + "from": "D9", + "to": "LED 양극", + "note": "220~330Ω 직렬 저항을 사용하고 LED 음극은 GND에 연결" + } + ] +} diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..dda4dfd --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Hana Cloud validation tests.""" diff --git a/tests/test_firmware.py b/tests/test_firmware.py new file mode 100644 index 0000000..7d4e3af --- /dev/null +++ b/tests/test_firmware.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import copy +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import firmware # noqa: E402 + + +PROFILE_HASH = "739fdb18d93b143ffe6598c26177be73794f12052ad6018eca24d1912cbf22a8" +SOURCE_HASH = "57e7e3011111cfc98c740c32be386b85fab304edd1e458c090fad9d303ed8f2d" +HEX_HASH = "9e2df0a1190a1205c098889c455e5b76c4df18b5ccac2b7605da1575f05b64c5" + + +def source_document() -> dict: + return { + "schemaVersion": 1, + "id": "arduino.uno-r3", + "name": "Arduino Uno R3", + "sketch": { + "path": "boards/arduino-uno-r3.ino", + "fqbn": "arduino:avr:uno", + }, + "detect": { + "usb": [ + { + "vid": "2341", + "pid": "0043", + "confidence": "exact", + "manufacturerAliases": ["Arduino LLC"], + "productAliases": ["Arduino Uno R3"], + } + ] + }, + "wiring": [ + { + "from": "D2", + "to": "순간 누름 스위치 NO 단자", + "note": "스위치 COM 단자는 GND에 연결", + } + ], + } + + +def board_source() -> firmware.BoardSource: + return firmware.BoardSource.from_document("arduino-uno-r3", source_document()) + + +class BoardSourceTests(unittest.TestCase): + def test_rejects_a_sketch_path_that_escapes_the_board_directory(self) -> None: + document = source_document() + document["sketch"]["path"] = "boards/../outside.ino" + + with self.assertRaisesRegex(ValueError, "sketch.path"): + firmware.BoardSource.from_document("arduino-uno-r3", document) + + def test_rejects_a_human_supplied_generated_hash(self) -> None: + document = source_document() + document["sketch"]["sha256"] = "0" * 64 + + with self.assertRaisesRegex(ValueError, "unknown"): + firmware.BoardSource.from_document("arduino-uno-r3", document) + + +class RenderingTests(unittest.TestCase): + def test_renders_the_literal_public_v2_manifest_contract(self) -> None: + artifact = firmware.FirmwareArtifact( + path="boards/arduino-uno-r3.hex", + data=b":00000001FF\n", + source_sha256=SOURCE_HASH, + arduino_cli="1.5.1", + platform="arduino:avr@1.8.8", + ) + + manifest = firmware.render_manifest(board_source(), artifact) + + self.assertEqual( + manifest, + { + "schemaVersion": 2, + "id": "arduino.uno-r3", + "firmware": { + "path": "boards/arduino-uno-r3.hex", + "format": "intel-hex", + "size": 12, + "sha256": HEX_HASH, + "fqbn": "arduino:avr:uno", + "source": { + "path": "boards/arduino-uno-r3.ino", + "sha256": SOURCE_HASH, + }, + "toolchain": { + "arduinoCli": "1.5.1", + "platform": "arduino:avr@1.8.8", + }, + }, + "wiring": [ + { + "from": "D2", + "to": "순간 누름 스위치 NO 단자", + "note": "스위치 COM 단자는 GND에 연결", + } + ], + }, + ) + + def test_registry_preserves_apps_and_increments_for_changed_boards(self) -> None: + registry = { + "schemaVersion": 1, + "revision": 2, + "apps": [ + { + "id": "pdf-viewer", + "path": "apps/pdf-viewer.json", + "sha256": PROFILE_HASH, + } + ], + "boards": [], + } + board = { + "id": "arduino.uno-r3", + "name": "Arduino Uno R3", + "manifest": "boards/arduino-uno-r3.json", + "sha256": SOURCE_HASH, + "detect": source_document()["detect"], + } + + rendered = firmware.render_registry(registry, [board]) + + self.assertEqual(rendered["apps"], registry["apps"]) + self.assertEqual(rendered["boards"], [board]) + self.assertEqual(rendered["revision"], 3) + self.assertEqual(registry["revision"], 2) + self.assertEqual(registry["boards"], []) + + def test_registry_keeps_revision_when_published_boards_are_unchanged(self) -> None: + board = { + "id": "arduino.uno-r3", + "name": "Arduino Uno R3", + "manifest": "boards/arduino-uno-r3.json", + "sha256": SOURCE_HASH, + "detect": source_document()["detect"], + } + registry = { + "schemaVersion": 1, + "revision": 9, + "apps": [], + "boards": [copy.deepcopy(board)], + } + + rendered = firmware.render_registry(registry, [board]) + + self.assertEqual(rendered["revision"], 9) + + def test_canonical_json_uses_utf8_two_spaces_and_one_final_lf(self) -> None: + rendered = firmware.canonical_json({"label": "한번", "nested": {"value": 1}}) + + self.assertEqual( + rendered, + b'{\n "label": "\xed\x95\x9c\xeb\xb2\x88",\n "nested": {\n "value": 1\n }\n}\n', + ) + + +if __name__ == "__main__": + unittest.main() From fa648681c9323892915563f4a7fc92bde57c5aa0 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Mon, 24 Aug 2026 03:10:26 +0900 Subject: [PATCH 3/7] feat: build reproducible Arduino firmware --- .gitattributes | 1 + scripts/firmware.py | 113 ++++++++++++++++++++++++++++++++++++++++- tests/test_firmware.py | 99 ++++++++++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 8a8a65a..0885718 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,6 @@ * text=auto eol=lf *.ino text eol=lf +*.hex -text *.json text eol=lf *.md text eol=lf diff --git a/scripts/firmware.py b/scripts/firmware.py index 64873e4..53356f7 100644 --- a/scripts/firmware.py +++ b/scripts/firmware.py @@ -6,9 +6,12 @@ import hashlib import json import re +import shutil +import subprocess +import tempfile from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Callable ID_PATTERN = re.compile(r"^[a-z0-9]+(?:[.-][a-z0-9]+)*$") @@ -16,6 +19,7 @@ FQBN_PATTERN = re.compile(r"^[A-Za-z0-9_-]+:[A-Za-z0-9_-]+:[A-Za-z0-9_-]+$") USB_ID_PATTERN = re.compile(r"^[0-9a-f]{4}$") CONFIDENCE_VALUES = {"exact", "likely", "ambiguous"} +FIRMWARE_LIMIT = 2 * 1024 * 1024 def canonical_json(value: Any) -> bytes: @@ -264,3 +268,110 @@ def render_registry( rendered["revision"] = revision + 1 rendered["boards"] = boards return rendered + + +CompileRunner = Callable[[str, str, Path, Path], None] + + +def _run_arduino_cli(cli: str, fqbn: str, staged_sketch: Path, output_dir: Path) -> None: + subprocess.run( + [ + cli, + "compile", + "--clean", + "--fqbn", + fqbn, + "--output-dir", + str(output_dir), + str(staged_sketch), + ], + check=True, + ) + + +def validate_intel_hex(data: bytes) -> None: + if not data or len(data) > FIRMWARE_LIMIT: + raise ValueError(f"firmware must contain 1 to {FIRMWARE_LIMIT} bytes") + try: + lines = data.decode("ascii").splitlines() + except UnicodeDecodeError as error: + raise ValueError("firmware is not ASCII Intel HEX") from error + if not lines: + raise ValueError("firmware has no Intel HEX records") + + saw_end_of_file = False + for line_number, line in enumerate(lines, start=1): + if saw_end_of_file: + raise ValueError("Intel HEX contains a record after end-of-file") + if not line.startswith(":") or len(line) == 1 or len(line[1:]) % 2: + raise ValueError(f"Intel HEX record {line_number} has invalid syntax") + try: + record = bytes.fromhex(line[1:]) + except ValueError as error: + raise ValueError(f"Intel HEX record {line_number} is not hexadecimal") from error + if len(record) < 5 or len(record) != record[0] + 5: + raise ValueError(f"Intel HEX record {line_number} has an invalid byte count") + if sum(record) & 0xFF: + raise ValueError(f"Intel HEX record {line_number} has an invalid checksum") + record_type = record[3] + if record_type not in {0, 1, 2, 3, 4, 5}: + raise ValueError(f"Intel HEX record {line_number} has an unsupported type") + if record_type == 1: + if record[0] != 0 or record[1:3] != b"\x00\x00": + raise ValueError("Intel HEX end-of-file record is malformed") + saw_end_of_file = True + if not saw_end_of_file: + raise ValueError("Intel HEX end-of-file record is missing") + + +def _read_normal_hex(output_dir: Path, slug: str) -> bytes: + path = output_dir / f"{slug}.ino.hex" + if not path.is_file(): + raise ValueError(f"compiler did not emit the normal application HEX for {slug}") + data = path.read_bytes() + validate_intel_hex(data) + return data + + +def compile_reproducible( + root: Path, + source: BoardSource, + cli: str, + *, + runner: CompileRunner = _run_arduino_cli, +) -> bytes: + source_path = root / source.sketch_path + if not source_path.is_file(): + raise ValueError(f"sketch is missing: {source.sketch_path}") + + with tempfile.TemporaryDirectory(prefix=f"hana-{source.slug}-") as directory: + build_root = Path(directory) + results = [] + for attempt in (1, 2): + staged_sketch = build_root / f"sketch-{attempt}" / source.slug + staged_sketch.mkdir(parents=True) + shutil.copyfile(source_path, staged_sketch / f"{source.slug}.ino") + output_dir = build_root / f"output-{attempt}" + runner(cli, source.fqbn, staged_sketch, output_dir) + results.append(_read_normal_hex(output_dir, source.slug)) + + if results[0] != results[1]: + raise ValueError(f"compiler output is not reproducible for {source.id}") + return results[0] + + +def validate_changed_paths( + changed_paths: list[str], + base_registry: dict[str, Any] | None, + current_registry: dict[str, Any] | None, +) -> None: + for raw_path in changed_paths: + path = raw_path.replace("\\", "/") + if path.startswith("boards/") and (path.endswith(".hex") or path.endswith(".json")): + raise ValueError(f"generated file cannot be changed in a pull request: {path}") + + if "registry.json" in {path.replace("\\", "/") for path in changed_paths}: + if not isinstance(base_registry, dict) or not isinstance(current_registry, dict): + raise ValueError("registry.json comparison requires both revisions") + if base_registry.get("boards") != current_registry.get("boards"): + raise ValueError("registry.json boards are generated and cannot be changed directly") diff --git a/tests/test_firmware.py b/tests/test_firmware.py index 7d4e3af..d4e7325 100644 --- a/tests/test_firmware.py +++ b/tests/test_firmware.py @@ -2,6 +2,7 @@ import copy import sys +import tempfile import unittest from pathlib import Path @@ -166,5 +167,103 @@ def test_canonical_json_uses_utf8_two_spaces_and_one_final_lf(self) -> None: ) +class CompilerTests(unittest.TestCase): + VALID_HEX = b":0100000001FE\n:00000001FF\n" + + def compile_in_temporary_root(self, outputs: list[dict[str, bytes]]) -> bytes: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + sketch = root / "boards" / "arduino-uno-r3.ino" + sketch.parent.mkdir(parents=True) + sketch.write_text("void setup() {}\nvoid loop() {}\n", encoding="utf-8") + calls = 0 + + def runner( + cli: str, + fqbn: str, + staged_sketch: Path, + output_dir: Path, + ) -> None: + nonlocal calls + self.assertEqual(cli, "arduino-cli") + self.assertEqual(fqbn, "arduino:avr:uno") + self.assertEqual( + (staged_sketch / "arduino-uno-r3.ino").read_text(encoding="utf-8"), + "void setup() {}\nvoid loop() {}\n", + ) + output_dir.mkdir(parents=True, exist_ok=True) + for name, data in outputs[calls].items(): + (output_dir / name).write_bytes(data) + calls += 1 + + result = firmware.compile_reproducible( + root, + board_source(), + "arduino-cli", + runner=runner, + ) + self.assertEqual(calls, 2) + return result + + def test_selects_the_normal_hex_and_never_the_bootloader_image(self) -> None: + outputs = [ + { + "arduino-uno-r3.ino.hex": self.VALID_HEX, + "arduino-uno-r3.ino.with_bootloader.hex": b"bootloader", + }, + { + "arduino-uno-r3.ino.hex": self.VALID_HEX, + "arduino-uno-r3.ino.with_bootloader.hex": b"different bootloader", + }, + ] + + result = self.compile_in_temporary_root(outputs) + + self.assertEqual(result, self.VALID_HEX) + + def test_rejects_builds_that_only_emit_a_bootloader_image(self) -> None: + outputs = [ + {"arduino-uno-r3.ino.with_bootloader.hex": self.VALID_HEX}, + {"arduino-uno-r3.ino.with_bootloader.hex": self.VALID_HEX}, + ] + + with self.assertRaisesRegex(ValueError, "normal application HEX"): + self.compile_in_temporary_root(outputs) + + def test_rejects_nondeterministic_compiler_output(self) -> None: + outputs = [ + {"arduino-uno-r3.ino.hex": self.VALID_HEX}, + {"arduino-uno-r3.ino.hex": b":00000001FF\n"}, + ] + + with self.assertRaisesRegex(ValueError, "not reproducible"): + self.compile_in_temporary_root(outputs) + + def test_rejects_an_intel_hex_record_with_a_bad_checksum(self) -> None: + with self.assertRaisesRegex(ValueError, "checksum"): + firmware.validate_intel_hex(b":0100000001FD\n:00000001FF\n") + + +class PullRequestPolicyTests(unittest.TestCase): + def test_rejects_direct_generated_file_changes(self) -> None: + for path in ["boards/arduino-uno-r3.hex", "boards/arduino-uno-r3.json"]: + with self.subTest(path=path), self.assertRaisesRegex(ValueError, "generated"): + firmware.validate_changed_paths([path], None, None) + + def test_rejects_a_direct_registry_board_change(self) -> None: + base = {"apps": [], "boards": []} + current = {"apps": [], "boards": [{"id": "arduino.uno-r3"}]} + + with self.assertRaisesRegex(ValueError, "registry.json boards"): + firmware.validate_changed_paths(["registry.json"], base, current) + + def test_allows_app_registry_changes_when_boards_are_unchanged(self) -> None: + boards = [{"id": "arduino.uno-r3"}] + base = {"apps": [], "boards": copy.deepcopy(boards)} + current = {"apps": [{"id": "pdf-viewer"}], "boards": copy.deepcopy(boards)} + + firmware.validate_changed_paths(["registry.json", "apps/pdf-viewer.json"], base, current) + + if __name__ == "__main__": unittest.main() From a511b66cf1be934f8d896445f23465b365c385d3 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Mon, 24 Aug 2026 03:15:59 +0900 Subject: [PATCH 4/7] docs: define compiled firmware contract --- README.md | 47 +++++++++++++++----- requirements-dev.txt | 1 + schemas/board.schema.json | 94 ++++++++++++++++++++++++++++++++------- tests/test_firmware.py | 21 +++++++++ 4 files changed, 136 insertions(+), 27 deletions(-) create mode 100644 requirements-dev.txt diff --git a/README.md b/README.md index d0cab00..2d08cad 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,27 @@ # Hana Cloud Hana Cloud는 한번(HanBeon)이 사용하는 응용 프로그램 프로필과 외부 보드 -자료를 배포하는 공개 데이터 저장소입니다. 실행 코드와 업로더 구현은 두지 않고, -검토 가능한 JSON·펌웨어·이미지만 관리합니다. +자료를 배포하는 공개 데이터 저장소입니다. 클라이언트 실행 코드와 업로더 구현은 +두지 않고, 검토 가능한 JSON·펌웨어·이미지와 이를 검증·생성하는 CI만 관리합니다. ## 저장소 구조 ```text .gitattributes registry.json +.github/ + firmware-toolchain.json + workflows/ apps/ music-app.json pdf-viewer.json boards/ + arduino-uno-r3.hex arduino-uno-r3.json arduino-uno-r3.ino arduino-uno-r3.png +sources/boards/ + arduino-uno-r3.json contracts/ normalization-examples.json schemas/ @@ -121,13 +127,22 @@ schemas/ ```json { - "schemaVersion": 1, + "schemaVersion": 2, "id": "arduino.uno-r3", "firmware": { - "path": "boards/arduino-uno-r3.ino", - "format": "arduino-sketch", + "path": "boards/arduino-uno-r3.hex", + "format": "intel-hex", + "size": 11211, + "sha256": "64자리 소문자 SHA-256", "fqbn": "arduino:avr:uno", - "sha256": "64자리 소문자 SHA-256" + "source": { + "path": "boards/arduino-uno-r3.ino", + "sha256": "64자리 소문자 SHA-256" + }, + "toolchain": { + "arduinoCli": "1.5.1", + "platform": "arduino:avr@1.8.8" + } }, "wiring": [ { @@ -151,6 +166,10 @@ schemas/ - `id`는 `registry.json` 보드 항목과 정확히 같아야 합니다. - `firmware.path`와 선택적인 `image.path`는 같은 보드 basename을 사용합니다. +- `firmware`는 Arduino CLI가 필요 없는 일반 업로드용 Intel HEX입니다. bootloader를 + 포함한 HEX는 배포하지 않습니다. +- `firmware.source`와 `firmware.toolchain`은 소스와 바이너리의 대응을 감사하기 위한 + provenance이며 CI가 계산합니다. - 이미지를 제공하면 스크린 리더용 `alt` 설명이 반드시 있어야 합니다. - 클라이언트는 manifest, 펌웨어, 이미지의 해시를 모두 확인한 뒤에만 로컬 경로를 업로더 인터페이스에 넘깁니다. @@ -206,8 +225,8 @@ Uno의 VID/PID 목록과 USB 문자열은 Arduino의 검증하고 last-known-good 캐시에 원자적으로 저장합니다. 4. 새 프로필 적용 시 `미리보기 프로필 인식 완료 · 버튼 2개 추가`처럼 한 번만 알립니다. 다운로드 중이거나 실패한 상태를 300ms 폴링마다 반복 표시하지 않습니다. -5. 네트워크·검증 실패 시 마지막 정상 캐시를 유지하고, 캐시도 없으면 한번에 내장된 - 기본 프리셋 또는 기본 4칸으로 동작합니다. +5. 네트워크·검증 실패 시 마지막 정상 캐시를 유지하고, 캐시도 없으면 기본 4칸으로 + 동작합니다. 현재 HanBeon 펌웨어의 `HANBEON_UNO_V1` handshake는 펌웨어 설치가 끝난 보드와 런타임 연결을 맺는 기존 프로토콜로만 유지합니다. 최초 보드 식별이나 레지스트리 @@ -238,15 +257,21 @@ UI는 플랫폼 조건문을 갖지 않습니다. 단위 테스트를 추가합니다. 네트워크는 포함하지 않습니다. 3. **App profiles** — 인덱스 갱신, 프로필 검증·캐시·적용, 인식 완료 메시지를 추가합니다. -4. **Board catalog** — 보드 자료 검증·캐시, 배선 안내 UI, 다른 작업자가 구현하는 - 업로더에 넘길 안정적인 인터페이스를 추가합니다. 업로드 구현은 포함하지 않습니다. +4. **Board catalog** — 보드 자료 검증·캐시, 배선 안내 UI, 컴파일된 HEX를 다른 + 작업자가 구현하는 업로더에 넘길 안정적인 인터페이스를 추가합니다. 업로드 구현은 + 포함하지 않습니다. 5. **Desktop releases** — Changepacks가 만든 draft release에 Windows, macOS, Linux Tauri 번들을 올리고 모든 빌드 성공 후 release를 공개합니다. ## 변경 규칙 - 데이터 변경은 PR로만 받습니다. -- `registry.json`과 대상 파일은 같은 PR에서 함께 갱신합니다. +- 보드 기여자는 `sources/boards/*.json`, 대응하는 `.ino`, 선택적인 `.png`만 + 수정합니다. `.hex`, 공개 board manifest, `registry.json`의 boards 항목은 사람이 + 수정할 수 없습니다. +- 보드 소스 PR이 병합되면 고정된 Arduino toolchain을 사용하는 GitHub Actions가 + 스케치를 두 번 clean build하고 동일한 일반 HEX만 후속 커밋으로 게시합니다. +- 앱 프로필 변경은 `registry.json`의 apps 항목과 대상 파일을 같은 PR에서 갱신합니다. - 기존 `id`의 의미를 바꾸지 않습니다. 호환되지 않는 변경은 새 `id` 또는 새 `schemaVersion`을 사용합니다. - 저작권이나 재배포 권한을 확인할 수 없는 펌웨어와 이미지는 추가하지 않습니다. diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..3ee1ea1 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +jsonschema==4.26.0 diff --git a/schemas/board.schema.json b/schemas/board.schema.json index 8233245..8e35db3 100644 --- a/schemas/board.schema.json +++ b/schemas/board.schema.json @@ -6,28 +6,93 @@ "additionalProperties": false, "required": ["schemaVersion", "id", "firmware", "wiring"], "properties": { - "schemaVersion": { "const": 1 }, + "schemaVersion": { "enum": [1, 2] }, + "id": { "$ref": "#/$defs/id" }, + "firmware": { "type": "object" }, + "wiring": { "$ref": "#/$defs/wiring" }, + "image": { "$ref": "#/$defs/image" } + }, + "allOf": [ + { + "if": { + "properties": { "schemaVersion": { "const": 1 } }, + "required": ["schemaVersion"] + }, + "then": { "properties": { "firmware": { "$ref": "#/$defs/legacyFirmware" } } } + }, + { + "if": { + "properties": { "schemaVersion": { "const": 2 } }, + "required": ["schemaVersion"] + }, + "then": { "properties": { "firmware": { "$ref": "#/$defs/compiledFirmware" } } } + } + ], + "$defs": { "id": { "type": "string", "pattern": "^[a-z0-9]+(?:[.-][a-z0-9]+)*$" }, - "firmware": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "fqbn": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]+:[A-Za-z0-9_-]+:[A-Za-z0-9_-]+$" + }, + "sketchPath": { + "type": "string", + "pattern": "^boards/[a-z0-9]+(?:-[a-z0-9]+)*\\.ino$" + }, + "hexPath": { + "type": "string", + "pattern": "^boards/[a-z0-9]+(?:-[a-z0-9]+)*\\.hex$" + }, + "legacyFirmware": { "type": "object", "additionalProperties": false, "required": ["path", "format", "fqbn", "sha256"], "properties": { - "path": { - "type": "string", - "pattern": "^boards/[a-z0-9]+(?:-[a-z0-9]+)*\\.ino$" - }, + "path": { "$ref": "#/$defs/sketchPath" }, "format": { "const": "arduino-sketch" }, - "fqbn": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]+:[A-Za-z0-9_-]+:[A-Za-z0-9_-]+$" + "fqbn": { "$ref": "#/$defs/fqbn" }, + "sha256": { "$ref": "#/$defs/sha256" } + } + }, + "compiledFirmware": { + "type": "object", + "additionalProperties": false, + "required": ["path", "format", "size", "sha256", "fqbn", "source", "toolchain"], + "properties": { + "path": { "$ref": "#/$defs/hexPath" }, + "format": { "const": "intel-hex" }, + "size": { "type": "integer", "minimum": 1, "maximum": 2097152 }, + "sha256": { "$ref": "#/$defs/sha256" }, + "fqbn": { "$ref": "#/$defs/fqbn" }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256"], + "properties": { + "path": { "$ref": "#/$defs/sketchPath" }, + "sha256": { "$ref": "#/$defs/sha256" } + } }, - "sha256": { - "type": "string", - "pattern": "^[0-9a-f]{64}$" + "toolchain": { + "type": "object", + "additionalProperties": false, + "required": ["arduinoCli", "platform"], + "properties": { + "arduinoCli": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "platform": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]+:[A-Za-z0-9_-]+@[0-9]+\\.[0-9]+\\.[0-9]+$" + } + } } } }, @@ -54,10 +119,7 @@ "type": "string", "pattern": "^boards/[a-z0-9]+(?:-[a-z0-9]+)*\\.png$" }, - "sha256": { - "type": "string", - "pattern": "^[0-9a-f]{64}$" - }, + "sha256": { "$ref": "#/$defs/sha256" }, "alt": { "type": "string", "minLength": 1, "maxLength": 300 } } } diff --git a/tests/test_firmware.py b/tests/test_firmware.py index d4e7325..2dfbef4 100644 --- a/tests/test_firmware.py +++ b/tests/test_firmware.py @@ -1,11 +1,14 @@ from __future__ import annotations import copy +import json import sys import tempfile import unittest from pathlib import Path +import jsonschema + ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) @@ -265,5 +268,23 @@ def test_allows_app_registry_changes_when_boards_are_unchanged(self) -> None: firmware.validate_changed_paths(["registry.json", "apps/pdf-viewer.json"], base, current) +class PublishedSchemaTests(unittest.TestCase): + def test_public_schema_accepts_the_compiled_firmware_contract(self) -> None: + schema = json.loads((ROOT / "schemas" / "board.schema.json").read_text(encoding="utf-8")) + artifact = firmware.FirmwareArtifact( + path="boards/arduino-uno-r3.hex", + data=b":00000001FF\n", + source_sha256=SOURCE_HASH, + arduino_cli="1.5.1", + platform="arduino:avr@1.8.8", + ) + manifest = firmware.render_manifest(board_source(), artifact) + validator = jsonschema.Draft202012Validator(schema) + + errors = [error.message for error in validator.iter_errors(manifest)] + + self.assertEqual(errors, []) + + if __name__ == "__main__": unittest.main() From ce14b2cf37729945805796bc0b4bfef61f10d308 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Mon, 24 Aug 2026 03:35:19 +0900 Subject: [PATCH 5/7] ci: publish verified Arduino firmware --- .github/CODEOWNERS | 6 + .github/workflows/guard-generated-files.yml | 37 +++ .github/workflows/publish-firmware.yml | 85 ++++++ .github/workflows/validate.yml | 50 ++++ .gitignore | 5 + scripts/firmware.py | 300 ++++++++++++++++++++ tests/test_firmware.py | 195 +++++++++++++ 7 files changed, 678 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/guard-generated-files.yml create mode 100644 .github/workflows/publish-firmware.yml create mode 100644 .github/workflows/validate.yml create mode 100644 .gitignore diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..b587b5f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +/.github/ @owjs3901 +/scripts/ @owjs3901 +/schemas/ @owjs3901 +/boards/*.hex @owjs3901 +/boards/*.json @owjs3901 +/registry.json @owjs3901 diff --git a/.github/workflows/guard-generated-files.yml b/.github/workflows/guard-generated-files.yml new file mode 100644 index 0000000..11dc13f --- /dev/null +++ b/.github/workflows/guard-generated-files.yml @@ -0,0 +1,37 @@ +name: Generated File Guard + +on: + pull_request_target: + +permissions: + contents: read + +jobs: + guard: + name: guard-generated-files + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out the trusted base revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.event.pull_request.base.sha }} + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.14" + + - name: Inspect the pull request without checking out or executing its code + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + shell: bash + run: | + set -euo pipefail + git fetch --no-tags origin "refs/pull/${PR_NUMBER}/merge" + head_sha="$(git rev-parse FETCH_HEAD)" + python scripts/firmware.py --root . validate-pr \ + --base "$BASE_SHA" \ + --head "$head_sha" diff --git a/.github/workflows/publish-firmware.yml b/.github/workflows/publish-firmware.yml new file mode 100644 index 0000000..7e9a0eb --- /dev/null +++ b/.github/workflows/publish-firmware.yml @@ -0,0 +1,85 @@ +name: Publish Firmware + +on: + push: + branches: + - main + paths: + - ".github/firmware-toolchain.json" + - ".github/workflows/publish-firmware.yml" + - "boards/*.ino" + - "boards/*.png" + - "schemas/**" + - "scripts/firmware.py" + - "sources/boards/**" + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: hana-firmware-publisher + cancel-in-progress: false + +jobs: + publish: + name: publish + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out protected main + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: main + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.14" + cache: pip + + - name: Set up Arduino CLI + uses: arduino/setup-arduino-cli@81d310742121c928ea9c8bbd407b4217b432ae02 # v2.0.0 + with: + version: "1.5.1" + + - name: Install validation dependencies + run: python -m pip install --requirement requirements-dev.txt + + - name: Install locked Arduino platform + run: | + arduino-cli core update-index + arduino-cli core install arduino:avr@1.8.8 + + - name: Test publisher + run: python -m unittest -v tests.test_firmware + + - name: Generate and publish verified firmware + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + for attempt in 1 2 3; do + git fetch origin main + git reset --hard origin/main + python scripts/firmware.py --root . generate --arduino-cli arduino-cli + + if [[ -z "$(git status --porcelain -- boards registry.json)" ]]; then + echo "Published firmware is already current." + exit 0 + fi + + python scripts/firmware.py --root . verify --arduino-cli arduino-cli + git add -- boards registry.json + git commit -m "chore(firmware): publish compiled board artifacts" + if git push origin HEAD:main; then + exit 0 + fi + echo "main advanced during publication; retrying (${attempt}/3)" >&2 + done + + echo "main kept advancing; firmware publication aborted" >&2 + exit 1 diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..b2d5aa9 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,50 @@ +name: Validate + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + name: validate + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.14" + cache: pip + + - name: Set up Arduino CLI + uses: arduino/setup-arduino-cli@81d310742121c928ea9c8bbd407b4217b432ae02 # v2.0.0 + with: + version: "1.5.1" + + - name: Install validation dependencies + run: python -m pip install --requirement requirements-dev.txt + + - name: Install locked Arduino platform + run: | + arduino-cli core update-index + arduino-cli core install arduino:avr@1.8.8 + + - name: Run unit tests + run: python -m unittest -v tests.test_firmware + + - name: Reject human changes to generated board data + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: python scripts/firmware.py --root . validate-pr --base "$BASE_SHA" + + - name: Compile every board twice + run: python scripts/firmware.py --root . compile --arduino-cli arduino-cli diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..62e9b68 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.py[cod] + +# Local planning artifacts must never be published in Hana Cloud. +docs/superpowers/ diff --git a/scripts/firmware.py b/scripts/firmware.py index 53356f7..94634cd 100644 --- a/scripts/firmware.py +++ b/scripts/firmware.py @@ -2,12 +2,15 @@ from __future__ import annotations +import argparse import copy import hashlib import json +import os import re import shutil import subprocess +import sys import tempfile from dataclasses import dataclass from pathlib import Path @@ -17,6 +20,8 @@ ID_PATTERN = re.compile(r"^[a-z0-9]+(?:[.-][a-z0-9]+)*$") SLUG_PATTERN = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") FQBN_PATTERN = re.compile(r"^[A-Za-z0-9_-]+:[A-Za-z0-9_-]+:[A-Za-z0-9_-]+$") +PLATFORM_PATTERN = re.compile(r"^[A-Za-z0-9_-]+:[A-Za-z0-9_-]+$") +VERSION_PATTERN = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") USB_ID_PATTERN = re.compile(r"^[0-9a-f]{4}$") CONFIDENCE_VALUES = {"exact", "likely", "ambiguous"} FIRMWARE_LIMIT = 2 * 1024 * 1024 @@ -198,6 +203,45 @@ def size(self) -> int: return len(self.data) +@dataclass(frozen=True) +class ToolchainLock: + arduino_cli: str + platforms: dict[str, str] + + @classmethod + def load(cls, root: Path) -> "ToolchainLock": + path = root / ".github" / "firmware-toolchain.json" + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"cannot read .github/firmware-toolchain.json: {error}") from error + lock = _object(document, "toolchain") + _keys(lock, {"schemaVersion", "arduinoCli", "platforms"}, set(), "toolchain") + if lock["schemaVersion"] != 1: + raise ValueError("toolchain.schemaVersion must be 1") + cli_version = lock["arduinoCli"] + if not isinstance(cli_version, str) or not VERSION_PATTERN.fullmatch(cli_version): + raise ValueError("toolchain.arduinoCli must be an exact semantic version") + raw_platforms = _object(lock["platforms"], "toolchain.platforms") + if not raw_platforms: + raise ValueError("toolchain.platforms must not be empty") + platforms = {} + for platform, version in raw_platforms.items(): + if not isinstance(platform, str) or not PLATFORM_PATTERN.fullmatch(platform): + raise ValueError(f"toolchain platform is invalid: {platform}") + if not isinstance(version, str) or not VERSION_PATTERN.fullmatch(version): + raise ValueError(f"toolchain platform version is invalid: {platform}") + platforms[platform] = version + return cls(arduino_cli=cli_version, platforms=platforms) + + def platform_for(self, fqbn: str) -> str: + platform = ":".join(fqbn.split(":", 2)[:2]) + version = self.platforms.get(platform) + if version is None: + raise ValueError(f"FQBN platform is not locked: {platform}") + return f"{platform}@{version}" + + def load_sources(root: Path) -> list[BoardSource]: source_dir = root / "sources" / "boards" if not source_dir.is_dir(): @@ -375,3 +419,259 @@ def validate_changed_paths( raise ValueError("registry.json comparison requires both revisions") if base_registry.get("boards") != current_registry.get("boards"): raise ValueError("registry.json boards are generated and cannot be changed directly") + + +GenerationCompiler = Callable[[Path, BoardSource, str], bytes] + + +def _read_json(path: Path, field: str) -> dict[str, Any]: + try: + return _object(json.loads(path.read_text(encoding="utf-8")), field) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"cannot read {field}: {error}") from error + + +def _atomic_write(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) + try: + with os.fdopen(descriptor, "wb") as temporary: + temporary.write(data) + temporary.flush() + os.fsync(temporary.fileno()) + os.replace(temporary_name, path) + except BaseException: + try: + os.unlink(temporary_name) + except FileNotFoundError: + pass + raise + + +def build_publication( + root: Path, + cli: str, + *, + compiler: GenerationCompiler = compile_reproducible, +) -> dict[str, bytes]: + sources = load_sources(root) + toolchain = ToolchainLock.load(root) + registry = _read_json(root / "registry.json", "registry.json") + generated: dict[str, bytes] = {} + published_boards = [] + + for source in sources: + firmware_data = compiler(root, source, cli) + validate_intel_hex(firmware_data) + sketch_data = (root / source.sketch_path).read_bytes() + image_sha256 = None + if source.image is not None: + image_path = root / source.image["path"] + if not image_path.is_file(): + raise ValueError(f"image is missing: {source.image['path']}") + image_sha256 = sha256_bytes(image_path.read_bytes()) + artifact = FirmwareArtifact( + path=f"boards/{source.slug}.hex", + data=firmware_data, + source_sha256=sha256_bytes(sketch_data), + arduino_cli=toolchain.arduino_cli, + platform=toolchain.platform_for(source.fqbn), + image_sha256=image_sha256, + ) + manifest_path = f"boards/{source.slug}.json" + manifest_data = canonical_json(render_manifest(source, artifact)) + generated[artifact.path] = firmware_data + generated[manifest_path] = manifest_data + published_boards.append( + { + "id": source.id, + "name": source.name, + "manifest": manifest_path, + "sha256": sha256_bytes(manifest_data), + "detect": copy.deepcopy(source.detect), + } + ) + + generated["registry.json"] = canonical_json(render_registry(registry, published_boards)) + return generated + + +def generate( + root: Path, + cli: str, + *, + compiler: GenerationCompiler = compile_reproducible, +) -> list[str]: + generated = build_publication(root, cli, compiler=compiler) + changed = [] + for relative_path, data in generated.items(): + path = root / relative_path + try: + current = path.read_bytes() + except FileNotFoundError: + current = None + if current != data: + changed.append(relative_path) + + expected_board_paths = {path for path in generated if path.startswith("boards/")} + existing_board_paths = { + path.relative_to(root).as_posix() + for pattern in ("*.hex", "*.json") + for path in (root / "boards").glob(pattern) + } + stale = sorted(existing_board_paths - expected_board_paths) + changed.extend(stale) + + for relative_path in changed: + if relative_path in generated: + _atomic_write(root / relative_path, generated[relative_path]) + else: + (root / relative_path).unlink() + return changed + + +def verify( + root: Path, + cli: str, + *, + compiler: GenerationCompiler = compile_reproducible, +) -> None: + expected = build_publication(root, cli, compiler=compiler) + mismatches = [] + for relative_path, data in expected.items(): + try: + current = (root / relative_path).read_bytes() + except FileNotFoundError: + current = None + if current != data: + mismatches.append(relative_path) + expected_board_paths = {path for path in expected if path.startswith("boards/")} + existing_board_paths = { + path.relative_to(root).as_posix() + for pattern in ("*.hex", "*.json") + for path in (root / "boards").glob(pattern) + } + mismatches.extend(sorted(existing_board_paths - expected_board_paths)) + if mismatches: + raise ValueError(f"generated files are out of date: {', '.join(mismatches)}") + + +def _resolve_commit(root: Path, revision: str, field: str) -> str: + if revision != "HEAD" and not re.fullmatch(r"[0-9a-fA-F]{40,64}", revision): + raise ValueError(f"{field} must be a full commit SHA") + try: + resolved = subprocess.check_output( + ["git", "rev-parse", "--verify", f"{revision}^{{commit}}"], + cwd=root, + text=True, + encoding="utf-8", + ).strip() + except subprocess.CalledProcessError as error: + raise ValueError(f"cannot resolve {field}") from error + if not re.fullmatch(r"[0-9a-fA-F]{40,64}", resolved): + raise ValueError(f"resolved {field} is not a full commit SHA") + return resolved + + +def _read_git_json(root: Path, revision: str, path: str, field: str) -> dict[str, Any]: + try: + document = subprocess.check_output( + ["git", "show", f"{revision}:{path}"], + cwd=root, + text=True, + encoding="utf-8", + ) + return _object(json.loads(document), field) + except (subprocess.CalledProcessError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"cannot read {field}") from error + + +def validate_pr(root: Path, base: str, *, head: str = "HEAD") -> None: + if not re.fullmatch(r"[0-9a-fA-F]{40,64}", base): + raise ValueError("pull request base must be a full commit SHA") + base_commit = _resolve_commit(root, base, "pull request base") + head_commit = _resolve_commit(root, head, "pull request head") + try: + changed_output = subprocess.check_output( + [ + "git", + "diff", + "--name-only", + "--diff-filter=ACDMRT", + f"{base_commit}...{head_commit}", + ], + cwd=root, + text=True, + encoding="utf-8", + ) + except subprocess.CalledProcessError as error: + raise ValueError("cannot inspect pull request changes") from error + changed_paths = [line for line in changed_output.splitlines() if line] + base_registry = None + current_registry = None + if "registry.json" in {path.replace("\\", "/") for path in changed_paths}: + base_registry = _read_git_json(root, base_commit, "registry.json", "base registry.json") + current_registry = _read_git_json(root, head_commit, "registry.json", "head registry.json") + validate_changed_paths(changed_paths, base_registry, current_registry) + + +def compile_all( + root: Path, + cli: str, + *, + compiler: GenerationCompiler = compile_reproducible, +) -> list[tuple[str, int, str]]: + toolchain = ToolchainLock.load(root) + results = [] + for source in load_sources(root): + toolchain.platform_for(source.fqbn) + data = compiler(root, source, cli) + validate_intel_hex(data) + results.append((source.id, len(data), sha256_bytes(data))) + return results + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(".")) + subparsers = parser.add_subparsers(dest="command", required=True) + + validate_parser = subparsers.add_parser("validate-pr") + validate_parser.add_argument("--base", required=True) + validate_parser.add_argument("--head", default="HEAD") + + for name in ("compile", "generate", "verify"): + command_parser = subparsers.add_parser(name) + command_parser.add_argument("--arduino-cli", default="arduino-cli") + return parser + + +def main(argv: list[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + root = arguments.root.resolve() + try: + if arguments.command == "validate-pr": + validate_pr(root, arguments.base, head=arguments.head) + print("pull request generated-file policy: valid") + elif arguments.command == "compile": + for board_id, size, digest in compile_all(root, arguments.arduino_cli): + print(f"compiled {board_id}: {size} bytes sha256={digest}") + elif arguments.command == "generate": + changed = generate(root, arguments.arduino_cli) + if changed: + print("generated: " + ", ".join(changed)) + else: + print("generated firmware is already current") + elif arguments.command == "verify": + verify(root, arguments.arduino_cli) + print("generated firmware matches a fresh reproducible build") + else: + raise AssertionError(f"unsupported command: {arguments.command}") + except (OSError, ValueError, subprocess.CalledProcessError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_firmware.py b/tests/test_firmware.py index 2dfbef4..d966535 100644 --- a/tests/test_firmware.py +++ b/tests/test_firmware.py @@ -2,6 +2,7 @@ import copy import json +import subprocess import sys import tempfile import unittest @@ -286,5 +287,199 @@ def test_public_schema_accepts_the_compiled_firmware_contract(self) -> None: self.assertEqual(errors, []) +class PublicationTests(unittest.TestCase): + VALID_HEX = b":0100000001FE\n:00000001FF\n" + + def create_root(self, directory: str) -> Path: + root = Path(directory) + (root / "sources" / "boards").mkdir(parents=True) + (root / "boards").mkdir() + (root / ".github").mkdir() + (root / "sources" / "boards" / "arduino-uno-r3.json").write_text( + json.dumps(source_document(), ensure_ascii=False), + encoding="utf-8", + ) + (root / "boards" / "arduino-uno-r3.ino").write_text( + "void setup() {}\nvoid loop() {}\n", + encoding="utf-8", + ) + (root / ".github" / "firmware-toolchain.json").write_text( + json.dumps( + { + "schemaVersion": 1, + "arduinoCli": "1.5.1", + "platforms": {"arduino:avr": "1.8.8"}, + } + ), + encoding="utf-8", + ) + (root / "registry.json").write_text( + json.dumps( + { + "schemaVersion": 1, + "revision": 2, + "apps": [{"id": "pdf-viewer"}], + "boards": [], + } + ), + encoding="utf-8", + ) + return root + + def test_generation_writes_one_atomic_board_set_then_becomes_a_noop(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.create_root(directory) + compiles = 0 + + def compiler(path: Path, source: firmware.BoardSource, cli: str) -> bytes: + nonlocal compiles + self.assertEqual(path, root) + self.assertEqual(source.id, "arduino.uno-r3") + self.assertEqual(cli, "arduino-cli") + compiles += 1 + return self.VALID_HEX + + first_changes = firmware.generate(root, "arduino-cli", compiler=compiler) + second_changes = firmware.generate(root, "arduino-cli", compiler=compiler) + + self.assertEqual( + first_changes, + [ + "boards/arduino-uno-r3.hex", + "boards/arduino-uno-r3.json", + "registry.json", + ], + ) + self.assertEqual(second_changes, []) + self.assertEqual(compiles, 2) + self.assertEqual( + (root / "boards" / "arduino-uno-r3.hex").read_bytes(), + self.VALID_HEX, + ) + manifest_bytes = (root / "boards" / "arduino-uno-r3.json").read_bytes() + manifest = json.loads(manifest_bytes) + registry = json.loads((root / "registry.json").read_bytes()) + self.assertEqual(manifest["firmware"]["format"], "intel-hex") + self.assertEqual(registry["apps"], [{"id": "pdf-viewer"}]) + self.assertEqual(registry["revision"], 3) + self.assertEqual(registry["boards"][0]["sha256"], firmware.sha256_bytes(manifest_bytes)) + + def test_generation_failure_leaves_every_published_file_untouched(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.create_root(directory) + old_manifest = b'{"legacy":true}\n' + (root / "boards" / "arduino-uno-r3.json").write_bytes(old_manifest) + old_registry = (root / "registry.json").read_bytes() + + def failed_compiler(path: Path, source: firmware.BoardSource, cli: str) -> bytes: + raise ValueError("compile failed") + + with self.assertRaisesRegex(ValueError, "compile failed"): + firmware.generate(root, "arduino-cli", compiler=failed_compiler) + + self.assertEqual( + (root / "boards" / "arduino-uno-r3.json").read_bytes(), + old_manifest, + ) + self.assertEqual((root / "registry.json").read_bytes(), old_registry) + self.assertFalse((root / "boards" / "arduino-uno-r3.hex").exists()) + + def test_toolchain_rejects_an_unlocked_fqbn_platform(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.create_root(directory) + document = source_document() + document["sketch"]["fqbn"] = "vendor:other:board" + (root / "sources" / "boards" / "arduino-uno-r3.json").write_text( + json.dumps(document), + encoding="utf-8", + ) + + with self.assertRaisesRegex(ValueError, "not locked"): + firmware.generate(root, "arduino-cli", compiler=lambda *_: self.VALID_HEX) + + def test_generation_removes_artifacts_for_a_deleted_board_source(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.create_root(directory) + second = source_document() + second["id"] = "arduino.other" + second["name"] = "Arduino Other" + second["sketch"]["path"] = "boards/arduino-other.ino" + (root / "sources" / "boards" / "arduino-other.json").write_text( + json.dumps(second), + encoding="utf-8", + ) + (root / "boards" / "arduino-other.ino").write_text( + "void setup() {}\nvoid loop() {}\n", + encoding="utf-8", + ) + compiler = lambda *_: self.VALID_HEX + firmware.generate(root, "arduino-cli", compiler=compiler) + (root / "sources" / "boards" / "arduino-other.json").unlink() + (root / "boards" / "arduino-other.ino").unlink() + + changes = firmware.generate(root, "arduino-cli", compiler=compiler) + + self.assertIn("boards/arduino-other.hex", changes) + self.assertIn("boards/arduino-other.json", changes) + self.assertFalse((root / "boards" / "arduino-other.hex").exists()) + self.assertFalse((root / "boards" / "arduino-other.json").exists()) + + def test_verify_detects_a_tampered_published_hex(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.create_root(directory) + compiler = lambda *_: self.VALID_HEX + firmware.generate(root, "arduino-cli", compiler=compiler) + (root / "boards" / "arduino-uno-r3.hex").write_bytes(b":00000001FF\n") + + with self.assertRaisesRegex(ValueError, "out of date"): + firmware.verify(root, "arduino-cli", compiler=compiler) + + def test_validate_pr_compares_registry_boards_against_the_real_git_base(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.create_root(directory) + + def git(*arguments: str) -> None: + subprocess.run( + ["git", "-c", "commit.gpgSign=false", *arguments], + cwd=root, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + git("init") + git("config", "user.name", "Hana Test") + git("config", "user.email", "hana-test@example.invalid") + git("add", ".") + git("commit", "-m", "base") + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + + registry = json.loads((root / "registry.json").read_text(encoding="utf-8")) + registry["apps"] = [{"id": "music-app"}] + (root / "registry.json").write_text(json.dumps(registry), encoding="utf-8") + git("add", "registry.json") + git("commit", "-m", "change apps") + + # The trusted pull_request_target guard must inspect the requested + # Git object, never an untrusted or dirty working tree checkout. + app_head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True + ).strip() + registry["boards"] = [{"id": "working-tree-only-tamper"}] + (root / "registry.json").write_text(json.dumps(registry), encoding="utf-8") + firmware.validate_pr(root, base, head=app_head) + + git("restore", "registry.json") + firmware.validate_pr(root, base) + + registry["boards"] = [{"id": "arduino.uno-r3"}] + (root / "registry.json").write_text(json.dumps(registry), encoding="utf-8") + git("add", "registry.json") + git("commit", "-m", "change boards") + + with self.assertRaisesRegex(ValueError, "registry.json boards"): + firmware.validate_pr(root, base) + + if __name__ == "__main__": unittest.main() From 1f7d1dd461b1467773747ed51d7842e319ded200 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Mon, 24 Aug 2026 03:42:08 +0900 Subject: [PATCH 6/7] fix: harden firmware provenance checks --- scripts/firmware.py | 74 +++++++++++++++++++++++++++++++++------- tests/test_firmware.py | 77 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 12 deletions(-) diff --git a/scripts/firmware.py b/scripts/firmware.py index 94634cd..4202f9e 100644 --- a/scripts/firmware.py +++ b/scripts/firmware.py @@ -35,6 +35,26 @@ def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() +def _reject_symlink_components(root: Path, path: Path, field: str) -> None: + try: + relative = path.relative_to(root) + except ValueError as error: + raise ValueError(f"{field} escapes the repository root") from error + candidate = root + for part in relative.parts: + candidate /= part + if candidate.is_symlink(): + rendered = relative.as_posix() + raise ValueError(f"{field} cannot use a symbolic link: {rendered}") + + +def _require_regular_file(root: Path, path: Path, field: str) -> Path: + _reject_symlink_components(root, path, field) + if not path.is_file(): + raise ValueError(f"{field} is missing: {path.relative_to(root).as_posix()}") + return path + + def _object(value: Any, field: str) -> dict[str, Any]: if not isinstance(value, dict): raise ValueError(f"{field} must be an object") @@ -211,6 +231,7 @@ class ToolchainLock: @classmethod def load(cls, root: Path) -> "ToolchainLock": path = root / ".github" / "firmware-toolchain.json" + _require_regular_file(root, path, "toolchain lock") try: document = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError) as error: @@ -244,11 +265,13 @@ def platform_for(self, fqbn: str) -> str: def load_sources(root: Path) -> list[BoardSource]: source_dir = root / "sources" / "boards" + _reject_symlink_components(root, source_dir, "board source directory") if not source_dir.is_dir(): raise ValueError("sources/boards directory is missing") sources = [] ids: set[str] = set() for path in sorted(source_dir.glob("*.json")): + _require_regular_file(root, path, "board source descriptor") try: raw = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError) as error: @@ -256,6 +279,9 @@ def load_sources(root: Path) -> list[BoardSource]: source = BoardSource.from_document(path.stem, raw) if source.id in ids: raise ValueError(f"duplicate source id: {source.id}") + _require_regular_file(root, root / source.sketch_path, "board sketch") + if source.image is not None: + _require_regular_file(root, root / source.image["path"], "board image") ids.add(source.id) sources.append(source) if not sources: @@ -370,8 +396,10 @@ def validate_intel_hex(data: bytes) -> None: def _read_normal_hex(output_dir: Path, slug: str) -> bytes: path = output_dir / f"{slug}.ino.hex" - if not path.is_file(): - raise ValueError(f"compiler did not emit the normal application HEX for {slug}") + try: + _require_regular_file(output_dir, path, "compiler output") + except ValueError as error: + raise ValueError(f"compiler did not emit a safe normal application HEX for {slug}") from error data = path.read_bytes() validate_intel_hex(data) return data @@ -384,9 +412,9 @@ def compile_reproducible( *, runner: CompileRunner = _run_arduino_cli, ) -> bytes: - source_path = root / source.sketch_path - if not source_path.is_file(): - raise ValueError(f"sketch is missing: {source.sketch_path}") + source_path = _require_regular_file( + root, root / source.sketch_path, "board sketch" + ) with tempfile.TemporaryDirectory(prefix=f"hana-{source.slug}-") as directory: build_root = Path(directory) @@ -456,19 +484,23 @@ def build_publication( ) -> dict[str, bytes]: sources = load_sources(root) toolchain = ToolchainLock.load(root) - registry = _read_json(root / "registry.json", "registry.json") + registry_path = _require_regular_file(root, root / "registry.json", "registry.json") + registry = _read_json(registry_path, "registry.json") generated: dict[str, bytes] = {} published_boards = [] for source in sources: firmware_data = compiler(root, source, cli) validate_intel_hex(firmware_data) - sketch_data = (root / source.sketch_path).read_bytes() + sketch_path = _require_regular_file( + root, root / source.sketch_path, "board sketch" + ) + sketch_data = sketch_path.read_bytes() image_sha256 = None if source.image is not None: - image_path = root / source.image["path"] - if not image_path.is_file(): - raise ValueError(f"image is missing: {source.image['path']}") + image_path = _require_regular_file( + root, root / source.image["path"], "board image" + ) image_sha256 = sha256_bytes(image_path.read_bytes()) artifact = FirmwareArtifact( path=f"boards/{source.slug}.hex", @@ -506,6 +538,7 @@ def generate( changed = [] for relative_path, data in generated.items(): path = root / relative_path + _reject_symlink_components(root, path, "generated output") try: current = path.read_bytes() except FileNotFoundError: @@ -523,6 +556,7 @@ def generate( changed.extend(stale) for relative_path in changed: + _reject_symlink_components(root, root / relative_path, "generated output") if relative_path in generated: _atomic_write(root / relative_path, generated[relative_path]) else: @@ -539,6 +573,7 @@ def verify( expected = build_publication(root, cli, compiler=compiler) mismatches = [] for relative_path, data in expected.items(): + _reject_symlink_components(root, root / relative_path, "generated output") try: current = (root / relative_path).read_bytes() except FileNotFoundError: @@ -596,7 +631,10 @@ def validate_pr(root: Path, base: str, *, head: str = "HEAD") -> None: [ "git", "diff", - "--name-only", + "--name-status", + "-z", + "--find-renames", + "--find-copies", "--diff-filter=ACDMRT", f"{base_commit}...{head_commit}", ], @@ -606,7 +644,19 @@ def validate_pr(root: Path, base: str, *, head: str = "HEAD") -> None: ) except subprocess.CalledProcessError as error: raise ValueError("cannot inspect pull request changes") from error - changed_paths = [line for line in changed_output.splitlines() if line] + fields = changed_output.split("\0") + if fields and fields[-1] == "": + fields.pop() + changed_paths = [] + index = 0 + while index < len(fields): + status = fields[index] + index += 1 + path_count = 2 if status.startswith(("R", "C")) else 1 + if not status or index + path_count > len(fields): + raise ValueError("git returned malformed changed-path data") + changed_paths.extend(fields[index : index + path_count]) + index += path_count base_registry = None current_registry = None if "registry.json" in {path.replace("\\", "/") for path in changed_paths}: diff --git a/tests/test_firmware.py b/tests/test_firmware.py index d966535..c557e50 100644 --- a/tests/test_firmware.py +++ b/tests/test_firmware.py @@ -364,6 +364,47 @@ def compiler(path: Path, source: firmware.BoardSource, cli: str) -> bytes: self.assertEqual(registry["revision"], 3) self.assertEqual(registry["boards"][0]["sha256"], firmware.sha256_bytes(manifest_bytes)) + def test_rejects_a_symlinked_source_descriptor(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.create_root(directory) + descriptor = root / "sources" / "boards" / "arduino-uno-r3.json" + target = root / "descriptor-target.json" + target.write_bytes(descriptor.read_bytes()) + descriptor.unlink() + descriptor.symlink_to(target) + + with self.assertRaisesRegex(ValueError, "symbolic link"): + firmware.load_sources(root) + + def test_rejects_a_symlinked_sketch_before_compilation(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.create_root(directory) + sketch = root / "boards" / "arduino-uno-r3.ino" + target = root / "boards" / "other.ino" + target.write_bytes(sketch.read_bytes()) + sketch.unlink() + sketch.symlink_to(target) + + with self.assertRaisesRegex(ValueError, "symbolic link"): + firmware.generate(root, "arduino-cli", compiler=lambda *_: self.VALID_HEX) + + def test_rejects_a_symlinked_board_image_before_hashing(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.create_root(directory) + descriptor = root / "sources" / "boards" / "arduino-uno-r3.json" + document = json.loads(descriptor.read_text(encoding="utf-8")) + document["image"] = { + "path": "boards/arduino-uno-r3.png", + "alt": "Arduino Uno R3 wiring", + } + descriptor.write_text(json.dumps(document), encoding="utf-8") + target = root / "boards" / "other.png" + target.write_bytes(b"not-a-real-image") + (root / "boards" / "arduino-uno-r3.png").symlink_to(target) + + with self.assertRaisesRegex(ValueError, "symbolic link"): + firmware.generate(root, "arduino-cli", compiler=lambda *_: self.VALID_HEX) + def test_generation_failure_leaves_every_published_file_untouched(self) -> None: with tempfile.TemporaryDirectory() as directory: root = self.create_root(directory) @@ -480,6 +521,42 @@ def git(*arguments: str) -> None: with self.assertRaisesRegex(ValueError, "registry.json boards"): firmware.validate_pr(root, base) + def test_validate_pr_rejects_renaming_a_generated_file_out_of_boards(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.create_root(directory) + (root / "boards" / "arduino-uno-r3.json").write_text( + json.dumps({"schemaVersion": 1}), encoding="utf-8" + ) + + def git(*arguments: str) -> None: + subprocess.run( + ["git", "-c", "commit.gpgSign=false", *arguments], + cwd=root, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + git("init") + git("config", "user.name", "Hana Test") + git("config", "user.email", "hana-test@example.invalid") + git("add", ".") + git("commit", "-m", "base") + base = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True + ).strip() + + (root / "archive").mkdir() + git( + "mv", + "boards/arduino-uno-r3.json", + "archive/arduino-uno-r3-manifest.txt", + ) + git("commit", "-m", "rename generated manifest") + + with self.assertRaisesRegex(ValueError, "generated file"): + firmware.validate_pr(root, base) + if __name__ == "__main__": unittest.main() From e06137d8d24921e30370e5280cbbcdbe9fa2ca86 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Mon, 24 Aug 2026 03:45:57 +0900 Subject: [PATCH 7/7] fix: key pip cache from dev requirements --- .github/workflows/publish-firmware.yml | 1 + .github/workflows/validate.yml | 1 + tests/test_firmware.py | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/.github/workflows/publish-firmware.yml b/.github/workflows/publish-firmware.yml index 7e9a0eb..26035f9 100644 --- a/.github/workflows/publish-firmware.yml +++ b/.github/workflows/publish-firmware.yml @@ -38,6 +38,7 @@ jobs: with: python-version: "3.14" cache: pip + cache-dependency-path: requirements-dev.txt - name: Set up Arduino CLI uses: arduino/setup-arduino-cli@81d310742121c928ea9c8bbd407b4217b432ae02 # v2.0.0 diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index b2d5aa9..82b323d 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -23,6 +23,7 @@ jobs: with: python-version: "3.14" cache: pip + cache-dependency-path: requirements-dev.txt - name: Set up Arduino CLI uses: arduino/setup-arduino-cli@81d310742121c928ea9c8bbd407b4217b432ae02 # v2.0.0 diff --git a/tests/test_firmware.py b/tests/test_firmware.py index c557e50..2c9ff52 100644 --- a/tests/test_firmware.py +++ b/tests/test_firmware.py @@ -269,6 +269,14 @@ def test_allows_app_registry_changes_when_boards_are_unchanged(self) -> None: firmware.validate_changed_paths(["registry.json", "apps/pdf-viewer.json"], base, current) +class WorkflowContractTests(unittest.TestCase): + def test_pip_cache_tracks_the_actual_dev_requirements_file(self) -> None: + for name in ("validate.yml", "publish-firmware.yml"): + workflow = (ROOT / ".github" / "workflows" / name).read_text(encoding="utf-8") + self.assertIn("cache: pip", workflow, name) + self.assertIn("cache-dependency-path: requirements-dev.txt", workflow, name) + + class PublishedSchemaTests(unittest.TestCase): def test_public_schema_accepts_the_compiled_firmware_contract(self) -> None: schema = json.loads((ROOT / "schemas" / "board.schema.json").read_text(encoding="utf-8"))