From fd7ee39366c7f535ac5ed1a05066e1dd0c4096b5 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 18 Aug 2026 08:50:43 +0000 Subject: [PATCH 1/2] docs: publish prism recipe 2.1 --- README.md | 19 +++++++++++----- docs/README.md | 4 ++-- docs/api.md | 2 +- docs/getting-started.md | 30 ++++++++++++++++--------- docs/prism.md | 45 ++++++++++++++++++++++++------------- docs/scoring.md | 13 +++++++++++ docs/troubleshooting.md | 10 +++++---- examples/baseline/README.md | 10 ++++----- top-model/README.md | 2 +- 9 files changed, 89 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 086cb5e..94fe018 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,9 @@ PRISM is a research challenge on a pinned [NeMo AutoModel](https://github.com/NVIDIA-NeMo/Automodel) base: you fork the operator pin, edit under that tree, and submit a **unified git diff**. The operator applies your patch fail-closed, then re-executes training on a -miner-funded Lium GPU pod against a pinned FineWeb-Edu shard. Score is pure -**bits-per-byte** (bpb, lower is better). There is **no** miner Docker image, +miner-funded Lium GPU pod against a pinned FineWeb-Edu shard. The live leaf is +the equal-weight **G2 benchmark accuracy** lattice; the v3 harness also records +the complete G1–G8 battery. There is **no** miner Docker image, no CVM, no on-chain write from miners — HTTP submit only. | | | @@ -35,8 +36,8 @@ no CVM, no on-chain write from miners — HTTP submit only. | Production gateway | `https://chain.joinbase.ai` | | Staging gateway | `http://staging.api.joinbase.ai` | | Submit path | `/challenge/prism/v1/submissions` | -| Recipe | **2.0.0** — AutoModel pin + patch (`automodel@v0.5.0`) | -| Live GPU | Miner-funded Lium — pass `X-Lium-Api-Key` | +| Recipe | **2.1.0** — AutoModel patch + attested dual cap | +| Live GPU | Miner-funded four-GPU RTX 5090 pod — pass `X-Lium-Api-Key` | This repository holds **miner documentation and examples only**. Control-plane source lives in [BaseIntelligence/base](https://github.com/BaseIntelligence/base). @@ -48,7 +49,13 @@ source lives in [BaseIntelligence/base](https://github.com/BaseIntelligence/base 3. Checkout that AutoModel commit → edit → `git diff > automodel.patch`. 4. Pack `automodel.base` + `automodel.patch` (+ optional `prism.toml`) and submit with your hotkey + **`X-Lium-Api-Key`** — see [Submit](docs/submit.md). -5. Poll events until `terminated`, then check your bpb — see [API](docs/api.md). +5. Poll events until `terminated`, then inspect G2 + battery metrics — see [API](docs/api.md). + +Recipe 2.1's CUDA 13 pod includes Transformer Engine/NVFP4 and a compiler +toolchain. Add `requirements.txt` or `pyproject.toml` at the AutoModel repo +root in your patch for a network-on install before the offline train/eval. +Training must consume `ctx["train_stream"]`; for DDP, rank 0 owns and +scatters each global batch so FLOPs/tokens/bytes remain attested. ```bash export GATEWAY=https://chain.joinbase.ai @@ -68,7 +75,7 @@ curl -sS -X POST "$GATEWAY/challenge/prism/v1/submissions" \ ## The three things miners get wrong 1. **Legacy 1.x ZIPs** — `architecture.py` + `training.py` (or training-only - `arch_id`) return `400 unsupported_layout` / `recipe_version` on live 2.0. + `arch_id`) return `400 unsupported_layout` / `recipe_version` on live 2.1. Ship `automodel.base` + `automodel.patch` only. 2. **Wrong pin / stale diff** — `automodel.base` must equal live `automodel_pin_id` (`automodel@v0.5.0`); regenerate the patch against the diff --git a/docs/README.md b/docs/README.md index 21aa4d6..4906939 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,13 +1,13 @@ # PRISM miner docs -Live recipe is **2.0.0**: submit an AutoModel pin id + unified git diff +Live recipe is **2.1.0**: submit an AutoModel pin id + unified git diff (`automodel.base` + `automodel.patch`), not a free-form two-script ZIP. | Page | What it covers | |------|----------------| | [Getting started](getting-started.md) | Fork pin → edit → `git diff` → pack ZIP | | [Submit](submit.md) | ZIP/JSON, BYOK Lium key, gating, precheck, retries | -| [Scoring & competition](scoring.md) | bpb lattice, patch anti-copy, causal ban, top-model | +| [Scoring & competition](scoring.md) | G2 lattice, G1–G8 battery, emission, anti-copy | | [API](api.md) | Routes, statuses, diff + telemetry | | [Troubleshooting](troubleshooting.md) | `unsupported_layout`, pin/patch failures, Lium | | [Full guide](prism.md) | Complete miner guide (mirrors BASE `docs/external-miner/prism.md`) | diff --git a/docs/api.md b/docs/api.md index 311ade7..a81a442 100644 --- a/docs/api.md +++ b/docs/api.md @@ -11,7 +11,7 @@ Replace `{GATEWAY}` with `https://chain.joinbase.ai` (prod) or |-------|-------------------| | `POST /challenge/prism/v1/submissions` | Submit AutoModel ZIP / JSON (`automodel.base` + `automodel.patch`) | | `POST /challenge/prism/v1/submissions/precheck` | Advisory copy/layout gate (3/coldkey/UTC day; no queue/pod) | -| `GET /challenge/prism/v1/submissions/{id}` | Detail + bpb + review/similarity/agentic records | +| `GET /challenge/prism/v1/submissions/{id}` | Detail + G2/battery/bpb + review/similarity/agentic records | | `GET /challenge/prism/v1/submissions/{id}/diff` | Unified diff + diffstat / classification (recipe ≥ 2.0) | | `GET /challenge/prism/v1/submissions/{id}/events` | Stage timeline | | `POST /challenge/prism/v1/submissions/{id}/retry` | Requeue an infra-failed row (within recovery window) | diff --git a/docs/getting-started.md b/docs/getting-started.md index cb3e011..c031837 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,9 +1,9 @@ # Getting started -## The contract (recipe v2.0.0) +## The contract (recipe v2.1.0) You do **not** ship a free-form `architecture.py` / `training.py` project. -Live recipe **2.0.0** accepts only a pin id plus your unified diff against that +Live recipe **2.1.0** accepts only a pin id plus your unified diff against that pin: ```text @@ -25,9 +25,14 @@ prism.toml # optional — entry / model-config knobs 5. Write `automodel.base` as a single line equal to `automodel_pin_id`, pack the ZIP, and `POST /v1/submissions` with your hotkey + **`X-Lium-Api-Key`**. -Models must stay **≤ 350M parameters**. The pod has **no network** -(`unshare --net`) beyond the operator-owned dataset pull — do not call Hub -downloads from miner code. +Models must stay **≤ 1B parameters**. The CUDA 13 pod exposes four RTX 5090 +GPUs by default and includes Transformer Engine/NVFP4. Add a repo-root +`requirements.txt` or `pyproject.toml` in your patch for a network-on install +phase; model train/eval then runs offline under `unshare --net`. + +Training must consume `ctx["train_stream"]`, which owns the attested FLOPs, +token/byte counters, and hard caps. For DDP, rank 0 consumes and scatters each +global batch; independent per-worker dataset streams fail the v3 contract. **Legacy recipe 1.x is rejected on live.** Two-script ZIPs (`architecture.py` + `training.py`), 1.3 source-tree ZIPs, and training-only @@ -48,9 +53,10 @@ The key is held in master memory for that submission and may also land in a logged) so a control-plane restart can still stop your pod. Missing key on live → `400 missing_lium_api_key`. -If the challenge process restarts mid-run, your submission is marked failed -promptly with `control_plane_restart` / `harness_detached`. Stop the Lium pod -if it is still billing, then resubmit with `X-Lium-Api-Key`. Poll +If the challenge process restarts mid-run, healthy pods resume from the +durable short-TTL payer seal; unrecoverable runs surface +`control_plane_restart` / `harness_detached`. Stop only a pod tied to one of +those failed rows, then retry with `X-Lium-Api-Key`. Poll `GET /v1/submissions/{id}/events` and `GET /v1/submissions/{id}/logs?since=`. ## Telemetry hooks (still required) @@ -77,9 +83,11 @@ eval as `ChallengeInternal` — never a miner score. Always confirm live values | Cap | Value | |-----|-------| -| Train wall clock | 6.0 h per submission (`train_hours_cap`) | +| Attested compute | `3.0e18` FLOPs (`train_flops_cap`) | +| Train wall clock | 5.0 h anti-DoS bound (`train_hours_cap`) | | Hard step cap | 20 000 (`max_train_steps`) | -| Model parameters | ≤ **350 000 000** (`max_params`) | +| Model parameters | ≤ **1 000 000 000** (`max_params`) | +| Minimum voluntary spend | `0.5`; step/wall/FLOPs stops are exempt | Trust live `GET /v1/recipe` (`version`, `automodel_*`, `pin_hex`, caps) over any marketing chart. @@ -90,7 +98,7 @@ marketing chart. curl -sS "$GATEWAY/challenge/prism/v1/recipe" ``` -Live recipe **2.0.0** advertises `version: "2.0.0"` and AutoModel pin fields +Live recipe **2.1.0** advertises `version: "2.1.0"` and AutoModel pin fields (`automodel_pin_id` = `automodel@v0.5.0`, `automodel_repo_url`, `automodel_git_ref`, `automodel_git_commit`, `automodel_content_sha256`). diff --git a/docs/prism.md b/docs/prism.md index a0a0818..2d2880d 100644 --- a/docs/prism.md +++ b/docs/prism.md @@ -3,8 +3,8 @@ # Prism challenge — HTTP AutoModel patch submit **challenge_id:** `prism` -**scoring_version:** `2` live (bpb-only; LLM review is an anti-cheat gate, not a grader). **v3 (opt-in, shadow-by-default):** composite scoring runs alongside — your run is also measured on the G1–G8 battery; see *v3 scoring* below. -**recipe_version:** `2.0.0` (pinned [NeMo AutoModel](https://github.com/NVIDIA-NeMo/Automodel) base + miner unified diff; legacy 1.x layouts rejected on live) +**scoring_version:** `4` live (equal-weight G2 public accuracies; LLM review is an anti-cheat gate, not a grader). **v3 harness:** every run also measures the complete G1–G8 battery; composite remains governance-gated. +**recipe_version:** `2.1.0` (AutoModel diff + 4-GPU CUDA 13/TE pod + attested dual cap; legacy 1.x layouts rejected) **Path:** HTTP only — **no Phala/CVM** Normative docs (BASE monorepo): @@ -14,7 +14,7 @@ Normative docs (BASE monorepo): ## What you submit A **ZIP** (preferred) — or JSON with the same members / `zip_base64` — that is -**not** a free-form `architecture.py` / `training.py` project. Recipe **2.0.0** +**not** a free-form `architecture.py` / `training.py` project. Recipe **2.1.0** accepts only an AutoModel pin id plus your git diff against that pin: ```text @@ -36,9 +36,14 @@ prism.toml # optional — entry / model-config knobs 5. Write `automodel.base` as a single line equal to `automodel_pin_id`, pack the ZIP, and `POST /v1/submissions` with your hotkey + **`X-Lium-Api-Key`**. -Models must stay **≤ 350M parameters**. The pod has **no network** -(`unshare --net`) beyond the operator-owned dataset pull — do not call Hub -downloads from miner code. +Models must stay **≤ 1B parameters**. The CUDA 13 pod exposes four RTX 5090 +GPUs and includes Transformer Engine/NVFP4. Add `requirements.txt` or +`pyproject.toml` at the repo root for the pre-sandbox install phase; model +train/eval is offline. + +Training must consume global batches from `ctx["train_stream"]`. For DDP, +rank 0 owns and scatters each batch; independent worker streams bypass +attested FLOPs/G6 and are rejected. **Legacy recipe 1.x rejected on live.** Two-script ZIPs (`architecture.py` + `training.py`), 1.3 source-tree ZIPs, and training-only @@ -110,12 +115,13 @@ Inspect recipe + AutoModel pin before coding: curl -sS "$BASE_GATEWAY/challenge/prism/v1/recipe" ``` -Live recipe **2.0.0** advertises `version: "2.0.0"` and AutoModel pin fields +Live recipe **2.1.0** advertises `version: "2.1.0"` and AutoModel pin fields (`automodel_pin_id` = `automodel@v0.5.0`, `automodel_repo_url`, `automodel_git_ref`, `automodel_git_commit`, `automodel_content_sha256`), -plus caps such as `train_hours_cap: 6.0`, `max_train_steps: 20000`, -`max_params: 350000000`, FineWeb dataset pin, and `pin_hex` (sha over the -versioned descriptor). Trust `/v1/recipe`, not marketing chart labels. +plus `train_flops_cap: 3.0e18`, `train_hours_cap: 5.0`, +`min_spend_fraction: 0.5` (voluntary only; protocol caps exempt), +`max_train_steps: 20000`, `max_params: 1000000000`, the FineWeb pin, and +`pin_hex`. Trust `/v1/recipe`, not marketing chart labels. `POST /v1/submissions` is idempotent by `submission_id` (hash of **pin id ‖ `0x00` ‖ patch bytes**). @@ -186,7 +192,8 @@ submission and never rents a Lium pod. ## Scoring (summary) -Final leaf score is pure bits-per-byte (bpb) on the lattice `[0, SCORE_MAX]`. +Final leaf score is the equal-weight mean of available G2 public accuracies, +mapped onto `[0, SCORE_MAX]`. Bits/token bpb remains telemetry/G1 input. The shared **agentic** gate (AST + metrics/receipt) hard-zeros `cheat` / `suspicious`. Cheap LLM similarity hard-zeros `Copied`, and `Suspicious` only when confidence `≥ 0.9` with non-generic evidence (below that — e.g. 0.7 citing @@ -199,7 +206,7 @@ quality is coherence-only, not a grader. Public gallery/leaderboard show champions only. **Competition (temporary):** emission uses **your own best training score only** — architecture-owner credit (rewarding arch owners when others train -well on their code) is **disabled** for now so the best-BPB trainer keeps +well on their code) is **disabled** for now so the best-scoring trainer keeps Prism's weights. Emission remains **winner-take-all**: only the single highest own score that epoch receives Prism's share (50% of the subnet); ties break by lexicographically smallest hotkey. Scores first land in the leaf @@ -212,7 +219,7 @@ published to [`BaseIntelligence/prism`](https://github.com/BaseIntelligence/prism) `top-model/`. -## v3 scoring (shadow-by-default) +## v3 scoring (measured, composite default-off) Recipe ≥ 1.3.0 harnesses run a **two-phase pod flow**: your code trains (`phase=train`), checkpoints, and only then does the operator stage private @@ -224,6 +231,11 @@ inference efficiency (G7), and training stability/µP (G8). Everything the battery reports is organizer-measured (**Zone A**, `org.*`) and is computed inside the harness — your code never emits it. +Recipe 2.1 emits the whole anchored surface: G1 prose/math/fresh crawl, v3 +byte/compute G6, measured-or-censored 32k G7 + reasoning throughput, and G8 +µP. Unsupported/OOM telemetry fails closed to explicit worst-case values +rather than disappearing. + **G5 is pretrain-only (recipe ≥ 1.4.0).** The long-context group scores a **base LM**, not an instruction-tuned chat model: completion-style / few-shot base prompts, short exact-match or multiple-choice logprob — @@ -252,10 +264,11 @@ terminal-loss band) and the cross-miner cohort, and land a stored verdict (`ok` / `flagged` / `quarantined`) — verdicts are evidence, never an auto-zero. Malformed or over-cap envelopes reject `422` and store nothing. -While `PRISM_SCORING_MODE=shadow` (default) the leaf score stays pure bpb, -bit-identical to v2. After the reference baselines are measured and the +While `PRISM_SCORING_MODE=benchmarks` (default) the leaf is the G2 accuracy +lattice. `shadow` is the legacy bpb mode. After references are measured and the anchor set is pre-registered, governance may flip to `composite`: group -scores are anchor-normalized, gate-filtered (`g3 ≥ 0.25`, `g8 ≥ 0.5`, +scores are anchor-normalized, gate-filtered (G3 hard floor currently disarmed; +`g8 ≥ 0.5`, budget + CI gates), combined as a weighted geometric mean, and ranked by the bootstrap lower-confidence bound (`lattice = round(SCORE_MAX × max(0, C − 1.645·SE))`). Inspect the anchor diff --git a/docs/scoring.md b/docs/scoring.md index 3084bdb..c093bd8 100644 --- a/docs/scoring.md +++ b/docs/scoring.md @@ -14,6 +14,13 @@ game the rank. LLM reviews remain **gates, not graders**. Legacy: `PRISM_SCORING_MODE=shadow` restores pure bits/token bpb (v2); `composite` uses the full G1–G8 lattice when anchors are ready. +Recipe 2.1 emits every v3 anchored metric: G1 code/prose/math/fresh-crawl, +byte/compute G6, measured-or-censored 32k G7 plus reasoning throughput, and +G8 stability + µP LR transfer. Missing hardware capability emits a worst-case +censored value rather than silently dropping the key. The G3 hard floor stays +disarmed. v3 anchors remain uncalibrated placeholders, so live is not flipped +to composite. + ## Anti-copy (patch / delta) Copying another miner's **patch** (or an equivalent touched-file rewrite of @@ -50,6 +57,12 @@ Prism's weights. Emission remains **winner-take-all**: only the single highest own score that epoch receives Prism's share (50% of the subnet); ties break by lexicographically smallest hotkey. +Implemented opt-ins remain **off by default**: `top3` keeps decayed credits for +the first three ranks; owner credit may split at most half of the winning leaf +with the registered architecture owner; `sig` uses same-slice private +significance evidence and otherwise burns/holds. Live remains `wta`, owner +credit `0`, and non-significance-gated until an announced governance change. + Scores first land in the leaf set emitted at the first chain-epoch boundary **after** your run finalizes. Positive scores then keep participating in later epochs' competition sets until a better valid score supersedes them (WTA still diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 5ca410e..adfea91 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2,14 +2,16 @@ | Symptom | Likely cause | What to check | |---------|--------------|---------------| -| Rejected submit | Recipe contract | `GET /v1/recipe` (`automodel_pin_id` + caps); follow recipe **2.0** | -| `400 unsupported_layout` | Legacy 1.x ZIP or missing AutoModel members | Ship `automodel.base` + `automodel.patch` (+ optional `prism.toml`). Two-script / source-tree / `arch_id` layouts are rejected on live 2.0 | +| Rejected submit | Recipe contract | `GET /v1/recipe` (`automodel_pin_id` + caps); follow recipe **2.1** | +| `400 unsupported_layout` | Legacy 1.x ZIP or missing AutoModel members | Ship `automodel.base` + `automodel.patch` (+ optional `prism.toml`). Two-script / source-tree / `arch_id` layouts are rejected on live | | `400 recipe_version` | Payload implies recipe 1.x while live advertises ≥ 2.0 | Re-pack as AutoModel patch ZIP; do not send `architecture.py`/`training.py` | | Patch apply failure / conflict | Diff not against the live pin, or stale rebase | Checkout exact `automodel_git_commit` from `/v1/recipe`; regenerate `git diff `; ensure `automodel.base` == `automodel_pin_id` | | Wrong / unknown pin id | `automodel.base` ≠ recipe `automodel_pin_id` | Copy `automodel_pin_id` (live: `automodel@v0.5.0`) byte-identical from `/v1/recipe` | | Binary / path-escape / oversized patch | Fail-closed apply rules | Text-only unified diff; no path escape outside allowlisted roots; keep diff within intake budgets | | Tokenizer / hub errors on pod | No network; Hub download from miner code | Stay offline; use pin/harness tokenizer paths — do not `from_pretrained("")` | -| `CAP_EXCEEDED` / Score 0 | Model > 350M params | Terminal — resize model config in your patch; not auto-retried | +| `CAP_EXCEEDED` / Score 0 | Model > 1B params | Terminal — resize model config in your patch; not auto-retried | +| v3 train fails with zero accounted tokens | Trainer bypassed `ctx["train_stream"]` | Rank 0 must consume/scatter harness batches; do not create an independent DDP dataset | +| FLOPs probe batch reduced | Full-row probe OOM | Expected safe degradation; inspect `probe_rows_reduced`, then reduce train geometry if the one-row probe still fails | | `missing_telemetry_hooks` | Patch removed / bypassed harness telemetry | Keep `prism_telemetry.report` (+ optional `finish_evaluation`) under the AutoModel train entry | | Score 0 after review | `Copied` / high-confidence `Suspicious` (≥0.9, non-trope) | Similarity on **your delta**; rewrite unique hunks; tropes alone are not plagiarism | | `similar: true` on precheck | Would hit intake copy gate | Change the patch vs prior champions; starting from the operator pin is fine | @@ -18,7 +20,7 @@ | `400 missing_lium_api_key` | Live path needs miner-funded Lium | Pass `X-Lium-Api-Key` (your Lium account); see [Submit](submit.md) | | **409 `not_failed`** on `/retry` | Row is not `failed` | `/retry` only recovers **failed** rows. Re-POSTing the identical ZIP is `already-queued` (no new GPU). After infra failure: `POST .../retry` with **`X-Lium-Api-Key`** (hotkey/Bearer alone is not enough) | | **400 `missing_lium_api_key`** on `/retry` | Need another GPU rent | Pass `X-Lium-Api-Key` on live (same header as submit) | -| Non-5090 / slow tok/s vs peers | Marketplace drew another SKU | Prism hard-pins **1× RTX 5090**; non-5090 is rejected at rent (no silent score normalize) | +| Wrong GPU width / non-5090 | Marketplace mismatch | Recipe 2.1 requests **4× RTX 5090** by default; non-5090 and undersized offers are rejected | | `403 hotkey_not_in_metagraph` | Hotkey not registered | Check the hex (64 lowercase, no `0x`) | | `409 submission_gated` | 1-max slot already used | One accepted patch per hotkey; identical pin+patch is idempotent | | `503 metagraph_unavailable` | Snapshot lag after a fresh registration | Retry in a couple of minutes | diff --git a/examples/baseline/README.md b/examples/baseline/README.md index d2ce870..0331dfc 100644 --- a/examples/baseline/README.md +++ b/examples/baseline/README.md @@ -1,15 +1,15 @@ -# Historical recipe 1.x baseline (not accepted on live 2.0) +# Historical recipe 1.x baseline (not accepted on live) -> **Live recipe is 2.0.0.** Submitting `architecture.py` + `training.py` returns +> **Live recipe is 2.1.0.** Submitting `architecture.py` + `training.py` returns > `400 unsupported_layout` / `recipe_version` on production. Use the AutoModel > patch workflow instead — see [Getting started](../../docs/getting-started.md) > and the [full guide](../../docs/prism.md). This directory keeps the old two-script baseline for reference only (tiny GPT-style causal transformer + AdamW loop with telemetry hooks). It is **not** -a valid live submission under recipe 2.0. +a valid live submission under recipe 2.1. -## Live path (recipe 2.0) +## Live path (recipe 2.1) 1. `GET /v1/recipe` → copy `automodel_pin_id` (`automodel@v0.5.0`) and `automodel_git_commit`. @@ -24,5 +24,5 @@ a valid live submission under recipe 2.0. | `architecture.py` | `build_model(ctx)` → TinyGPT (historical) | | `training.py` | `train(model, ctx)` → AdamW + `prism_telemetry` hooks (historical) | -Telemetry hooks remain required under the AutoModel train entry on live 2.0 — +Telemetry hooks remain required under the AutoModel train entry on live 2.1 — patches that remove them fail with `missing_telemetry_hooks`. diff --git a/top-model/README.md b/top-model/README.md index a597c75..b912e22 100644 --- a/top-model/README.md +++ b/top-model/README.md @@ -1,6 +1,6 @@ # PRISM top model -Published by the Base master on every new global-best bpb. This +Published by the Base master on every new global-best live score. This directory always mirrors the current champion; history lives in git. | field | value | From d7373f293439b2db6e3145414f7d3cecc6d2b4cb Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:00:57 +0000 Subject: [PATCH 2/2] ci: validate miner docs repository --- .github/workflows/ci.yml | 210 +++++++++------------------------------ 1 file changed, 46 insertions(+), 164 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb9a62a..ad5532c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,23 +4,12 @@ on: push: branches: - "**" - tags: - - "v*.*.*" pull_request: workflow_dispatch: - inputs: - confirm_publish: - description: "Type true to publish the Docker images to GHCR" - required: true - default: "false" permissions: contents: read -env: - SERVICE_IMAGE_NAME: ghcr.io/baseintelligence/prism - EVALUATOR_IMAGE_NAME: ghcr.io/baseintelligence/prism-evaluator - jobs: lint: runs-on: ubuntu-latest @@ -29,169 +18,62 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install dependencies - run: python -m pip install -e ".[dev]" - - name: Ruff lint - run: ruff check . - - name: Mypy type check - run: mypy + - name: Compile example Python + run: python -m compileall -q examples top-model + - name: Check documentation hygiene + run: | + python - <<'PY' + from pathlib import Path - test: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install dependencies - run: python -m pip install -e ".[dev]" - - name: Pytest with coverage - # Exclude the flaky multi-rank gloo collectives from the publish-gating suite: a collective - # hang must never run to GitHub's ~6h job ceiling and silently stall image publication. They - # run separately in distributed-gloo-tests (non-gating). timeout-minutes is a final backstop. - run: pytest -m "not distributed_gloo" --cov=prism_challenge --cov-report=term-missing --cov-fail-under=80 + markdown = [Path("README.md"), *Path("docs").glob("*.md")] + assert markdown + for path in markdown: + text = path.read_text(encoding="utf-8") + assert "\x00" not in text, path + assert "recipe_version: 2.0.0" not in text, path + assert "**2.1.0**" in Path("README.md").read_text(encoding="utf-8") + PY - distributed-gloo-tests: - # The world_size=4 gloo collective reliably times out on CPU-only (2-core ubuntu-latest) - # runners, so `continue-on-error: true` keeps this non-gating job from failing the workflow. - # It is also non-gating by DEPENDENCY (NOT in docker-build/docker-publish `needs`, so it can - # never block image publication), and this job's timeout-minutes + the per-test - # @pytest.mark.timeout(300) backstop bound any collective hang so it fails fast instead of - # burning a ~6h job. To get real regression signal, run these on a self-hosted - # multi-core/GPU runner. - continue-on-error: true + test: runs-on: ubuntu-latest - timeout-minutes: 20 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install dependencies - run: python -m pip install -e ".[dev]" - - name: Pytest distributed gloo (non-gating) - run: pytest -m "distributed_gloo" -p no:cacheprovider + - name: Validate miner example contracts + run: | + python - <<'PY' + import ast + from pathlib import Path - docker-build: - needs: - - lint - - test - runs-on: ubuntu-latest - strategy: - matrix: - include: - - image: ghcr.io/baseintelligence/prism - target: service - - image: ghcr.io/baseintelligence/prism-evaluator - target: evaluator - steps: - - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 - - name: Build ${{ matrix.image }} image - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile - target: ${{ matrix.target }} - push: false - tags: ${{ matrix.image }}:ci-${{ github.sha }} + for directory in (Path("examples/baseline"), Path("top-model")): + expected = { + "architecture.py": "build_model", + "training.py": "train", + } + for filename, function in expected.items(): + path = directory / filename + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + functions = { + node.name + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + assert function in functions, f"{path} must define {function}()" + PY - docker-publish: - if: >- - github.event_name != 'pull_request' && - (github.ref == 'refs/heads/main' || - startsWith(github.ref, 'refs/tags/v') || - (github.event_name == 'workflow_dispatch' && inputs.confirm_publish == 'true')) - needs: - - docker-build + distributed-gloo-tests: + # Legacy check id retained for branch protection; this repository no + # longer contains the private distributed evaluator. runs-on: ubuntu-latest - permissions: - contents: read - packages: write steps: - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 - - name: Log in to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Generate service Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.SERVICE_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{raw}} - type=sha,prefix=sha- - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} - - name: Build and publish service image - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile - target: service - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - name: Generate evaluator Docker metadata - id: evaluator-meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.EVALUATOR_IMAGE_NAME }} - tags: | - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{raw}} - type=sha,prefix=sha- - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} - - name: Build and publish evaluator image - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile - target: evaluator - push: true - tags: ${{ steps.evaluator-meta.outputs.tags }} - labels: ${{ steps.evaluator-meta.outputs.labels }} - - github-release: - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - needs: - - docker-publish - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Prepare release metadata - id: release - run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" - - name: Create GitHub release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ github.ref_name }} - name: Prism ${{ steps.release.outputs.version }} - generate_release_notes: true - append_body: true - draft: false - prerelease: ${{ contains(github.ref_name, '-') }} - make_latest: ${{ !contains(github.ref_name, '-') }} - body: | - ## Container Images - - - `ghcr.io/baseintelligence/prism:${{ steps.release.outputs.version }}` - - `ghcr.io/baseintelligence/prism:${{ github.ref_name }}` - - `ghcr.io/baseintelligence/prism:sha-${{ github.sha }}` - - `ghcr.io/baseintelligence/prism-evaluator:${{ steps.release.outputs.version }}` - - `ghcr.io/baseintelligence/prism-evaluator:${{ github.ref_name }}` - - `ghcr.io/baseintelligence/prism-evaluator:sha-${{ github.sha }}` - - ## Deployment Notes - - BASE master deployments should pin the SemVer image tag plus the - immutable `@sha256` digest. The `latest` tag is published only from - `main`, not from release tags. + - name: Assert public repository surface + run: | + test -f README.md + test -f docs/getting-started.md + test -f examples/baseline/architecture.py + test ! -e crates + test ! -e bins + test ! -e deploy