diff --git a/.gitattributes b/.gitattributes index 5614fda..1a837a3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -11,6 +11,7 @@ LICENSE text eol=lf # Preserve byte-level evidence identities on every checkout. research/results/experiment-001/** -text -figures/figure-manifest.json -text +research/results/autopolicy-001/** -text +figures/*manifest.json -text figures/*.png -text requirements/verification-*.txt -text diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 9b77ec9..b29b932 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -13,7 +13,7 @@ body: attributes: label: Version or commit description: Provide the release version or Git commit SHA you tested. - placeholder: "0.1.2 or abc1234" + placeholder: "0.2.0 or abc1234" validations: required: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8a29c2..a3a6689 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,11 @@ jobs: --inventory-only research/results/experiment-001 + - name: Validate AutoPolicy evidence + run: >- + python scripts/validate_autopolicy_release.py + research/results/autopolicy-001 + - name: Run tests run: python -m pytest --basetemp .pytest-ci @@ -58,4 +63,4 @@ jobs: - name: Smoke-test installed wheel run: >- python scripts/smoke_installed_distribution.py - dist/cliffquant-0.1.2-py3-none-any.whl + dist/cliffquant-0.2.0-py3-none-any.whl diff --git a/CITATION.cff b/CITATION.cff index 6e2669e..80e1547 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,15 +5,17 @@ type: software authors: - given-names: "Muhammad Labeeb Aryan" family-names: "bin Mohd Lokman" -version: 0.1.2 +version: 0.2.0 license: Apache-2.0 repository-code: "https://github.com/Labeeb2339/cliffquant" url: "https://github.com/Labeeb2339/cliffquant" abstract: >- Exact minimax FP16 scale selection for a frozen multi-environment diagonal - quantization proxy, checked against exhaustive reference enumeration. + quantization proxy, plus exact logical-budget policy planning over recorded + mixed-precision candidates. keywords: - quantization - W4A16 - model compression + - mixed precision - reproducible research diff --git a/README.md b/README.md index 59c7b6b..856a485 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,54 @@ calibration environment separately, and optimizes the worst-environment scale objective exactly. The RecurQuant diagnostic is context for this decision, not part of CliffQuant's Experiment 001 evidence. +## AutoPolicy 001: exact logical-budget planning + +AutoPolicy turns the frozen Experiment 001 measurements into an auditable +mixed-precision plan. It records W4 CliffQuant, W8 AbsMax, and unquantized BF16 +source-reference candidates for all 150 target matrices. The published robust +view selects one precision per transformer block and exactly minimizes the +recorded worst-environment additive proxy under a logical target-payload +budget. + +![AutoPolicy logical-byte frontier](figures/autopolicy-logical-byte-frontier.png) + +| Logical bits per target weight | W4 blocks | W8 blocks | BF16 blocks | Worst-environment proxy | +|---:|---:|---:|---:|---:| +| `4.125` | 24 | 0 | 0 | `1078.217405` | +| `7.087` | 6 | 18 | 0 | `218.811198` | +| `8.125` | 0 | 24 | 0 | `4.745117` | +| `16.000` | 0 | 0 | 24 | `0.000000` | + +At the representative `440,721,408`-byte budget, the exact plan uses +`440,303,616` logical bytes and selects 6 W4 plus 18 W8 blocks. The full +150-matrix robust frontier was not tractable with the current exact dynamic +program, so the robust publication view makes a transformer-block constraint +explicit instead of hiding an approximation. Matrix-level exact planning +remains available for the scalar `sum-groupwise-max` objective. + +AutoPolicy does not yet export or benchmark a runnable mixed-bit checkpoint. +Its bytes are logical target payload, not checkpoint size or runtime memory, +and its proxy values are not NLL or accuracy predictions. + +[Read the method](research/AUTOPOLICY_001_METHOD.md) +| +[Inspect the measured result](research/AUTOPOLICY_001_RESULT.md) +| +[Review the prior-art boundary](research/AUTOPOLICY_PRIOR_ART.md) +| +[Verify the portable evidence](research/results/autopolicy-001) + +From a repository checkout: + +```bash +cliffquant autopolicy verify \ + --candidates research/results/autopolicy-001/candidates.json + +cliffquant autopolicy verify \ + --candidates research/results/autopolicy-001/candidates-transformer-blocks.json \ + --plan research/results/autopolicy-001/plan-blocks-7bpp.json +``` + ## Verification - `10,000 / 10,000` deterministic solver cases matched exhaustive FP16 @@ -202,7 +250,7 @@ python -m pytest python -m build ``` -Regenerate all three figures from the finalized proxy and NLL artifacts: +Regenerate the three proxy/NLL figures from the finalized artifacts: ```bash python scripts/generate_figures.py \ @@ -215,6 +263,20 @@ The generator verifies the input checksum manifests before parsing results and writes its own provenance to [`figures/figure-manifest.json`](figures/figure-manifest.json). +Regenerate the AutoPolicy frontier and its fourth figure from the checked-in +transformer-block candidate table: + +```bash +python scripts/generate_autopolicy_frontier.py \ + --candidates research/results/autopolicy-001/candidates-transformer-blocks.json \ + --output-json research/results/autopolicy-001/frontier.json \ + --output-figure figures/autopolicy-logical-byte-frontier.png \ + --output-manifest figures/autopolicy-logical-byte-frontier.manifest.json +``` + +That generator re-solves every recorded budget and writes a separate manifest +binding the graph and frontier to the exact candidate table and generator. + Validate the exact checked-in inventory on any supported development platform: ```bash diff --git a/figures/autopolicy-logical-byte-frontier.manifest.json b/figures/autopolicy-logical-byte-frontier.manifest.json new file mode 100644 index 0000000..85a18ac --- /dev/null +++ b/figures/autopolicy-logical-byte-frontier.manifest.json @@ -0,0 +1 @@ +{"generator":{"file":"scripts/generate_autopolicy_frontier.py","matplotlib":"3.10.8","sha256":"835354cdb00d519e9f4ae2a1699398a5b77115474e2ad369d4586c571e4a68df"},"input":{"candidate_fingerprint_sha256":"5af77e3afec3ac75339da56c171ae9ab37939cfe8ab0e4b0b878f370862031ca","file":"../research/results/autopolicy-001/candidates-transformer-blocks.json","sha256":"248268b088e8f82aa2125fa254f0ada7d2d120943199322bcef4f95beb633b3b","size_bytes":41267},"outputs":{"figure":{"file":"autopolicy-logical-byte-frontier.png","height_px":1332,"sha256":"664a469e7c50150458685ca1b7c0380d15e2adad5e87371abbf9a3ff4b66b8cb","size_bytes":124523,"width_px":2052},"frontier":{"file":"../research/results/autopolicy-001/frontier.json","sha256":"147945284fc3794143af87dc7c92db5fd98aa25eabe8b7c42f950f877ee3d869","size_bytes":5466}},"schema":"cliffquant.autopolicy-figure-provenance.v1"} diff --git a/figures/autopolicy-logical-byte-frontier.png b/figures/autopolicy-logical-byte-frontier.png new file mode 100644 index 0000000..a13c9c3 Binary files /dev/null and b/figures/autopolicy-logical-byte-frontier.png differ diff --git a/pyproject.toml b/pyproject.toml index bb1b384..5cf8b73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "cliffquant" -version = "0.1.2" -description = "Exact minimax FP16 scale selection for a multi-environment diagonal proxy" +version = "0.2.0" +description = "Exact multi-environment quantization scale and logical-budget policy research" readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" @@ -21,6 +21,10 @@ classifiers = [ ] [project.optional-dependencies] +autopolicy-experiment = [ + "ml-dtypes>=0.5.1", + "safetensors>=0.8.0", +] dev = [ "build>=1.2", "packaging>=26.2", @@ -31,6 +35,9 @@ figures = [ "matplotlib==3.10.8", ] +[project.scripts] +cliffquant = "cliffquant.cli:main" + [tool.setuptools.packages.find] where = ["src"] diff --git a/research/AUTOPOLICY_001_METHOD.md b/research/AUTOPOLICY_001_METHOD.md new file mode 100644 index 0000000..9f097d4 --- /dev/null +++ b/research/AUTOPOLICY_001_METHOD.md @@ -0,0 +1,129 @@ +# AutoPolicy 001 method + +## Purpose + +AutoPolicy 001 is a development measurement built from CliffQuant Experiment +001. It asks a narrower question than a mixed-bit model benchmark: + +> Given a recorded W4, W8, and unquantized BF16 candidate for every frozen +> target matrix, which discrete assignment minimizes a recorded +> multi-environment reconstruction proxy under a logical target-payload budget? + +This is not a preregistered confirmation experiment. Its role is to validate the +planner, artifact chain, byte model, and robust objective before any mixed-bit +checkpoint is exported. + +## Frozen inputs + +- Base model: `Qwen/Qwen3.5-0.8B-Base` at revision + `dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68`. +- The 150 language-tower matrices and 128-weight grouping already frozen by + Experiment 001. +- The four separately normalized calibration environments already frozen by + Experiment 001: general, code, math, and multilingual. +- The validated CliffQuant W4 scale run and its exact stored FP16 scales. + +The builder refuses model, calibration, module-inventory, or scale-run identity +drift. + +## Candidate bank + +Each matrix receives exactly three choices: + +| Candidate | Codes | Stored scale | Distortion | +|---|---:|---|---| +| `w4_cliffquant` | signed INT4 | one FP16 scale per 128 weights | direct re-evaluation of the validated exact CliffQuant scales | +| `w8_absmax` | signed INT8 | one FP16 scale per 128 weights | sign-aware no-clipping AbsMax target, rounded to the nearest positive finite FP16 scale | +| `dense16` | unquantized BF16 source reference | none | zero reconstruction distortion by definition | + +W8 uses round-to-nearest, ties-to-even, followed by clipping to `[-128, 127]`. +It is an AbsMax control, not an exact W8 scale optimizer. + +For every candidate, the builder records: + +- the sum of groupwise worst-environment normalized diagonal squared error; and +- the module-level squared-error total for each environment separately. + +## Allocation granularity + +The measured candidate bank contains all 150 target matrices. AutoPolicy exposes +two exact views over that table: + +- matrix-level allocation for the additive `sum-groupwise-max` scalar + objective; and +- a 24-transformer-block view for the four-environment robust-minimax + objective, where all matrices in a selected block use the same candidate. + +The block view sums the recorded matrix distortions and logical bytes without +changing them. It makes the exact multidimensional frontier tractable while +keeping the precision choice deployable at a clear architecture boundary. The +published AutoPolicy 001 frontier uses this block view. The full 150-matrix +robust search remains available as a fail-closed research path, but it is not +presented as the practical default. + +## Robust allocation objective + +For unit \(u\), candidate \(c\), environment \(e\), recorded distortion +\(d_{u,c,e}\), and logical byte cost \(m_{u,c}\), the robust planner solves + +$$ +\min_x \max_e \sum_{u,c} d_{u,c,e}x_{u,c} +$$ + +subject to + +$$ +\sum_c x_{u,c}=1\quad\forall u,\qquad +\sum_{u,c}m_{u,c}x_{u,c}\le B,\qquad +x_{u,c}\in\{0,1\}. +$$ + +For the published robust frontier, \(u\) is a transformer block. The +implementation keeps a componentwise byte/environment Pareto frontier. If the +exact search exceeds its configured state, transition, or Pareto-comparison +work limit, it aborts instead of silently returning an approximation. Objective +ties prefer the lower sum over environments, then fewer bytes, then +lexicographically smaller candidate IDs in sorted unit order. + +## Logical byte model + +For an integer candidate, the logical payload is: + +$$ +\left\lceil\frac{\text{weights}\times\text{bits}}{8}\right\rceil ++ 2\times\text{groups}. +$$ + +The unquantized BF16 source reference uses two bytes per target weight. + +The model deliberately excludes biases, zero buffers, `g_idx`, packing padding, +non-target tensors, metadata, filesystem overhead, activations, KV cache, and +runtime workspace. Therefore the budget is called **logical target-payload +bytes**, not checkpoint size or runtime memory. + +## Verification gates + +- Every recomputed W4 group objective must match the validated stored objective + within a strict float64 numerical tolerance. +- W8 vectorized results are tested against an independent scalar oracle. +- Candidate JSON and every W8 array archive are content-addressed. +- Candidate and plan JSON use strict duplicate-key rejection, finite-number + validation, canonical encoding, and self-fingerprints. +- Plan verification reloads the exact candidate file and recomputes the exact + allocation. +- Budgets below the all-W4 logical payload must fail closed. +- The all-W4 and all-BF16 endpoints must select their respective uniform + policies. + +## Claim boundary + +AutoPolicy 001 can support only this claim: + +> At the selected allocation granularity, the solver returns the exact optimum +> of the recorded discrete proxy table under the recorded logical-byte budget +> when its exact frontier completes. + +It does not establish an optimum for NLL, accuracy, checkpoint size, runtime +memory, latency, energy, or hardware throughput. It does not produce a runnable +mixed-bit checkpoint. Those require a separate exporter and held-out +confirmation. diff --git a/research/AUTOPOLICY_001_RESULT.md b/research/AUTOPOLICY_001_RESULT.md new file mode 100644 index 0000000..2186aea --- /dev/null +++ b/research/AUTOPOLICY_001_RESULT.md @@ -0,0 +1,105 @@ +# AutoPolicy 001 result + +## Outcome + +The frozen candidate build completed all 150 target matrices across the four +Experiment 001 calibration environments. Its 150 W8 archives passed the +content-bound archive verifier. The robust publication view aggregates those +unchanged measurements into 24 transformer-block choices and solves the exact +recorded minimax proxy under each logical target-payload budget. + +This is a planning result. No mixed-bit checkpoint or held-out mixed-policy NLL +is claimed. + +## Measured frontier + +| Selected logical bits per target weight | W4 blocks | W8 blocks | BF16 blocks | Worst-environment proxy | +|---:|---:|---:|---:|---:| +| `4.125` | 24 | 0 | 0 | `1078.217405` | +| `5.606` | 15 | 9 | 0 | `557.697827` | +| `7.087` | 6 | 18 | 0 | `218.811198` | +| `8.125` | 0 | 24 | 0 | `4.745117` | +| `10.019` | 0 | 18 | 6 | `3.296567` | +| `12.985` | 0 | 9 | 15 | `1.447354` | +| `16.000` | 0 | 0 | 24 | `0.000000` | + +The checked-in frontier contains ten points. At the representative +`440,721,408`-byte budget, AutoPolicy selects 6 W4 blocks and 18 W8 blocks, +uses `440,303,616` bytes (`7.087025` logical bits per target weight), and +reduces the recorded worst-environment proxy by `79.706%` relative to the +all-W4 endpoint. This percentage is not a predicted NLL or accuracy change. + +![AutoPolicy logical-byte frontier](../figures/autopolicy-logical-byte-frontier.png) + +## Negative result and correction + +The first exact 150-matrix robust search was not practically usable. Before +remaining-minimum feasibility pruning, even the all-W4 endpoint explored +incomplete high-byte prefixes and exceeded a two-minute run. After that bug was +fixed, the all-W4 endpoint completed with a one-state frontier and 150 +transitions. + +The nontrivial 150-matrix, four-environment frontier still grew too quickly for +the current multidimensional dynamic program. A capped all-W8-budget attempt +did not complete within 60 seconds. AutoPolicy therefore does not hide an +approximation or advertise that path as production-ready. + +The published robust view instead makes the deployment constraint explicit: +one candidate is selected per transformer block. Aggregating the same matrix +measurements produced 24 units. On the development machine, the all-W8 budget +then solved in about `0.12` seconds with a peak frontier of 122 states and +3,712 evaluated transitions. The result is exact for that block-constrained +problem. + +## Logical payload endpoints + +| Uniform policy | Logical bytes | Logical bits per target weight | +|---|---:|---:| +| W4 CliffQuant | `256,278,528` | `4.125` | +| W8 AbsMax | `504,791,040` | `8.125` | +| Unquantized BF16 source reference | `994,050,048` | `16.000` | + +These values cover target weight codes and stored group scales only. They are +not checkpoint sizes or runtime-memory measurements. + +## Portable evidence + +The directory +[`research/results/autopolicy-001`](results/autopolicy-001) contains: + +- the canonical 150-matrix candidate table; +- all 150 content-addressed W8 measurement archives; +- the content-bound 24-block candidate view; +- the ten-point robust frontier; +- the representative `7.087`-bit plan; and +- a deterministic 154-file release manifest. + +Key identities: + +- matrix candidate SHA256: + `308db237ade8bae8346a9c86c937ebde0d1c430d59e479ff90797c3892f9c9d7`; +- block candidate SHA256: + `248268b088e8f82aa2125fa254f0ada7d2d120943199322bcef4f95beb633b3b`; +- representative plan SHA256: + `1e0fe3afec4864f565cec59dd8010f038536705b482b401259d99dd73472c490`; + and +- release manifest SHA256: + `b4308c2392a6895e6992cba92d68db0bb5a0b3786f11d66cec897af06d9cdce3`. + +Revalidate the portable evidence from a repository checkout: + +```bash +python scripts/validate_autopolicy_release.py \ + research/results/autopolicy-001 +``` + +## Claim boundary + +AutoPolicy 001 supports one narrow result: at the selected allocation +granularity, it returns the exact optimum of the recorded additive proxy table +under the recorded logical-byte budget when the configured exact search +completes. + +It does not establish novelty, downstream quality, a globally optimal +quantized model, checkpoint size, runtime memory, latency, energy, or hardware +throughput. diff --git a/research/AUTOPOLICY_PRIOR_ART.md b/research/AUTOPOLICY_PRIOR_ART.md new file mode 100644 index 0000000..d7016c3 --- /dev/null +++ b/research/AUTOPOLICY_PRIOR_ART.md @@ -0,0 +1,50 @@ +# AutoPolicy prior-art boundary + +Automatic budget-aware mixed-precision quantization is established prior art. +AutoPolicy must not be presented as the first MCKP, dynamic-programming, or +mixed-bit allocator. + +The closest direct collision found in the audit is +[MixQuant](https://arxiv.org/abs/2607.23047), posted on 25 July 2026. It already +uses per-module distortion, an MCKP surrogate, and one calibration process to +serve multiple budgets; its deployment allocator is greedy and is compared +against an ILP solver. + +Other relevant primary sources include: + +- [GAMMA](https://arxiv.org/abs/2605.18475), which uses a + quantizer-agnostic teacher-forced reconstruction proxy, integer programming, + exact budget constraints, and reusable scores for arbitrary budgets; +- [HAWQ-V2](https://arxiv.org/abs/1911.03852), which uses Hessian sensitivity + and Pareto-based mixed precision; +- [HAWQ-V3](https://arxiv.org/abs/2011.10680), which uses hardware-aware + integer programming; +- [LLM-MQ](https://neurips2023-enlsp.github.io/papers/paper_4.pdf), which uses + first-order sensitivity and integer programming for low-bit LLM allocation; +- [SqueezeLLM](https://proceedings.mlr.press/v235/kim24f.html), which uses + Fisher-based sensitivity; +- [CLADO](https://arxiv.org/abs/2307.05657) and + [CoopQ](https://aclanthology.org/2026.findings-acl.373/), which model + cross-layer interactions and expose the limits of an additive proxy; +- [ScaleBITS](https://arxiv.org/abs/2602.17698), + [AMQ](https://aclanthology.org/2025.emnlp-main.1799/), and + [LieQ](https://aclanthology.org/2026.findings-acl.771/), which automate + mixed-precision selection; and +- [PiSO](https://arxiv.org/abs/2606.10890), which establishes exact + breakpoint-based scale optimization as prior art. + +The current implementation distinction under test is narrower: + +- candidates scored across separately normalized environments; +- an exact transformer-block robust minimax allocation over the recorded + environment vectors; +- fail-closed frontier limits; +- content-bound candidate and plan artifacts; and +- explicit logical-byte semantics. + +That combination may be useful and may be distinctive, but this audit is not +proof of novelty. + +In particular, AutoPolicy does not claim the first exact budget constraint, the +first reusable any-budget score table, or the first post-training global +allocator. diff --git a/research/PRIOR_ART.md b/research/PRIOR_ART.md index 28c7133..18021fe 100644 --- a/research/PRIOR_ART.md +++ b/research/PRIOR_ART.md @@ -1,7 +1,7 @@ # Prior-art boundary This is a bounded primary-source and public-code collision audit through -2026-07-26, not a proof of novelty. +2026-07-30, not a proof of novelty. The closest work I found is [PiSO](https://arxiv.org/abs/2606.10890), which already gives an exact @@ -12,6 +12,12 @@ first groupwise W4 scale search. Other close references include: +- [MixQuant](https://arxiv.org/abs/2607.23047), which formulates per-module + mixed-bit allocation as a multiple-choice knapsack problem using + context-marginal distortion and one calibration for multiple budgets; +- [GAMMA](https://arxiv.org/abs/2605.18475), which uses teacher-forced + reconstruction scoring and integer programming for exact-budget, + quantizer-agnostic mixed precision with reusable any-budget scores; - [NeUQI](https://arxiv.org/abs/2505.17595), which derives efficient scale and zero-point initialization for uniform low-bit quantization; - [DASH-Q](https://arxiv.org/abs/2604.13806), which uses stable diagonal @@ -48,6 +54,12 @@ Other close references include: - [the calibration-data study](https://arxiv.org/abs/2311.09755), which shows that PTQ quality can vary substantially with calibration data. +For AutoPolicy specifically, automatic budget-aware mixed precision is already +well established. The separate +[AutoPolicy prior-art boundary](AUTOPOLICY_PRIOR_ART.md) records the direct +allocation collisions and excludes first-MCKP, first-mixed-bit, or first +automatic-policy claims. + Generic worst-group optimization also predates this work; for example, [Group DRO](https://arxiv.org/abs/1911.08731) optimizes worst-group loss under distribution shifts. CliffQuant therefore does not claim minimax or diff --git a/research/results/autopolicy-001/candidates-transformer-blocks.json b/research/results/autopolicy-001/candidates-transformer-blocks.json new file mode 100644 index 0000000..5129049 --- /dev/null +++ b/research/results/autopolicy-001/candidates-transformer-blocks.json @@ -0,0 +1 @@ +{"builder":{"name":"cliffquant-autopolicy-transformer-blocks-v1","parent_profile":"experiment-001"},"candidate_definitions":{"dense16":{"bits":16,"group_size":null,"scale_selection":null},"w4_cliffquant":{"bits":4,"group_size":128,"scale_selection":"cliffquant_exact_minimax_fp16"},"w8_absmax":{"bits":8,"group_size":128,"scale_selection":"absmax_signed_range_nearest_fp16"}},"distortion_metric":{"additive_across_units":true,"downstream_metric":false,"groupwise_scalar":"sum_groupwise_max_environment_error","name":"normalized_diagonal_squared_error_with_separate_environment_totals","robust_policy":"max_environment_sum_module_error"},"environments":["general","code","math","multilingual"],"fingerprint_sha256":"5af77e3afec3ac75339da56c171ae9ab37939cfe8ab0e4b0b878f370862031ca","profile":"experiment-001-transformer-blocks","schema":"cliffquant.autopolicy-candidates.v1","scope":{"allocation_granularity":"transformer-block","allocation_unit_count":24,"budget_scope":"logical-target-weight-and-scale-payload","byte_model":"logical-codes-plus-fp16-scales-v1","checkpoint_export":false,"group_count":3883008,"group_size":128,"module_count":150,"weight_count":497025024},"source":{"measurement_source":{"base_model":{"config":{"file":"config.json","sha256":"b90b86f35c8e6925ef74ee04d0e758f0a845c83a42089ad82bbaa948de9b4204","size_bytes":2907},"fingerprint_sha256":"4c3f8c6ddda21c6e82eaad04eb9c962dc9bfab5ea727b05e4d3d42bfacc8c66a","id":"Qwen/Qwen3.5-0.8B-Base","index":{"file":"model.safetensors.index.json","sha256":"ce9a885efdf27d3664fdef5d512ad365216f1074051ef840c7cd8e5431495d0a","size_bytes":50900},"revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68","schema":"cliffquant.model-snapshot.v1","shards":[{"file":"model.safetensors-00001-of-00001.safetensors","sha256":"c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c","size_bytes":1746942600}],"tensor_count":488},"calibration":{"archive_sha256":"550f0b1b8a12b97435017a749fb8a6ffeae4db1bbe28f80d0257d78793103e3b","metadata_sha256":"b279ad598d7cf989ede0749f9f7fe3a455aa7de98d204155cfd9f25c09a821bf"},"w4_scale_run":{"manifest_sha256":"1292f4102dad57b070071c00d92ba43cd096b64b6434b3b9a3b3b41b73db3cf5","policy":"cliffquant_minimax"}},"parent_candidate_bundle":{"file_sha256":"308db237ade8bae8346a9c86c937ebde0d1c430d59e479ff90797c3892f9c9d7","fingerprint_sha256":"2ce53865d6835a08d1427060841eeff9b5184a9ff54f682d2854cf0de5e928a1"}},"status":"complete","totals":{"dense16":994050048,"w4_cliffquant":256278528,"w8_absmax":504791040},"units":[{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":43.282561075313254,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":39.947756527271565,"general":41.10817388548579,"math":40.06221048583258,"multilingual":41.47776684960404}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.192127736960688,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.178407193929284,"general":0.17780254488112018,"math":0.17807111519188398,"multilingual":0.17818084357312136}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.0.linear_attn.in_proj_qkv","model.language_model.layers.0.linear_attn.in_proj_z","model.language_model.layers.0.linear_attn.out_proj","model.language_model.layers.0.mlp.down_proj","model.language_model.layers.0.mlp.gate_proj","model.language_model.layers.0.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.0","weight_key":"aggregate:model.language_model.layers.0"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":42.78883696181707,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":36.72158240929327,"general":39.37249291831897,"math":37.12750031118767,"multilingual":39.944639172393536}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.21086223245262753,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.18495440111395225,"general":0.18388617670437699,"math":0.18506303223492204,"multilingual":0.183423292525368}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.1.linear_attn.in_proj_qkv","model.language_model.layers.1.linear_attn.in_proj_z","model.language_model.layers.1.linear_attn.out_proj","model.language_model.layers.1.mlp.down_proj","model.language_model.layers.1.mlp.gate_proj","model.language_model.layers.1.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.1","weight_key":"aggregate:model.language_model.layers.1"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":34.51141407619181,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":32.23956846806719,"general":32.38782799391179,"math":32.4397092139694,"multilingual":31.616124897334913}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.163885916454398,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.1502895354492825,"general":0.15013290743632743,"math":0.15051396820441135,"multilingual":0.14993214083567402}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.10.linear_attn.in_proj_qkv","model.language_model.layers.10.linear_attn.in_proj_z","model.language_model.layers.10.linear_attn.out_proj","model.language_model.layers.10.mlp.down_proj","model.language_model.layers.10.mlp.gate_proj","model.language_model.layers.10.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.10","weight_key":"aggregate:model.language_model.layers.10"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":36700160,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":36700160,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":25.185320889059447,"logical_bytes":9461760,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":9175040,"logical_scale_bytes":286720,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":23.596446510296335,"general":23.05210346435023,"math":23.697008137973402,"multilingual":22.83710272957728}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.11565204063910206,"logical_bytes":18636800,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":18350080,"logical_scale_bytes":286720,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.10548036945668297,"general":0.10568110096665992,"math":0.10550820278648768,"multilingual":0.10537885974384027}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.11.mlp.down_proj","model.language_model.layers.11.mlp.gate_proj","model.language_model.layers.11.mlp.up_proj","model.language_model.layers.11.self_attn.k_proj","model.language_model.layers.11.self_attn.o_proj","model.language_model.layers.11.self_attn.q_proj","model.language_model.layers.11.self_attn.v_proj"]},"group_count":143360,"shape":[1,18350080],"unit_id":"model.language_model.layers.11","weight_key":"aggregate:model.language_model.layers.11"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":42.71606711127728,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":39.78930590849279,"general":39.385833097815265,"math":39.97563770965499,"multilingual":38.727090165528054}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.19529844451454723,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.17649239081570822,"general":0.17616282060782523,"math":0.17655649774045262,"multilingual":0.17584536761899935}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.12.linear_attn.in_proj_qkv","model.language_model.layers.12.linear_attn.in_proj_z","model.language_model.layers.12.linear_attn.out_proj","model.language_model.layers.12.mlp.down_proj","model.language_model.layers.12.mlp.gate_proj","model.language_model.layers.12.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.12","weight_key":"aggregate:model.language_model.layers.12"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":40.56518406738631,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":37.19322837144629,"general":37.05667502337464,"math":37.68929450378773,"multilingual":36.2630640493215}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.18793605687195258,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.1665154558338638,"general":0.16627665257554394,"math":0.16654181561964,"multilingual":0.16572653829000372}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.13.linear_attn.in_proj_qkv","model.language_model.layers.13.linear_attn.in_proj_z","model.language_model.layers.13.linear_attn.out_proj","model.language_model.layers.13.mlp.down_proj","model.language_model.layers.13.mlp.gate_proj","model.language_model.layers.13.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.13","weight_key":"aggregate:model.language_model.layers.13"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":46.44125814629203,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":42.513768311378236,"general":42.6781410123057,"math":42.978297554122676,"multilingual":42.31408319119204}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.21104760702934738,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.18976715640353925,"general":0.1900199039067453,"math":0.18991298313214555,"multilingual":0.1899482725758803}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.14.linear_attn.in_proj_qkv","model.language_model.layers.14.linear_attn.in_proj_z","model.language_model.layers.14.linear_attn.out_proj","model.language_model.layers.14.mlp.down_proj","model.language_model.layers.14.mlp.gate_proj","model.language_model.layers.14.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.14","weight_key":"aggregate:model.language_model.layers.14"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":36700160,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":36700160,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":36.86705317861505,"logical_bytes":9461760,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":9175040,"logical_scale_bytes":286720,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":33.68459702779998,"general":32.59268056230606,"math":33.743556431702714,"multilingual":32.71570249194803}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.16840169626451063,"logical_bytes":18636800,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":18350080,"logical_scale_bytes":286720,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.14867993292875367,"general":0.1484981361021411,"math":0.1484445494734388,"multilingual":0.14841667770216282}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.15.mlp.down_proj","model.language_model.layers.15.mlp.gate_proj","model.language_model.layers.15.mlp.up_proj","model.language_model.layers.15.self_attn.k_proj","model.language_model.layers.15.self_attn.o_proj","model.language_model.layers.15.self_attn.q_proj","model.language_model.layers.15.self_attn.v_proj"]},"group_count":143360,"shape":[1,18350080],"unit_id":"model.language_model.layers.15","weight_key":"aggregate:model.language_model.layers.15"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":62.75419932994689,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":57.795815539201186,"general":58.14530518895762,"math":58.361305735925825,"multilingual":57.56329763421197}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.2746044028266674,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.24512567832913013,"general":0.24582893676652415,"math":0.2448794315603755,"multilingual":0.24542502725279672}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.16.linear_attn.in_proj_qkv","model.language_model.layers.16.linear_attn.in_proj_z","model.language_model.layers.16.linear_attn.out_proj","model.language_model.layers.16.mlp.down_proj","model.language_model.layers.16.mlp.gate_proj","model.language_model.layers.16.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.16","weight_key":"aggregate:model.language_model.layers.16"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":68.34665549437334,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":63.2480015830409,"general":64.19536882283226,"math":64.10194580568617,"multilingual":63.3458609838759}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.2999322491072222,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.2694490105563215,"general":0.2691229207968563,"math":0.2694495825115983,"multilingual":0.26811286351782593}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.17.linear_attn.in_proj_qkv","model.language_model.layers.17.linear_attn.in_proj_z","model.language_model.layers.17.linear_attn.out_proj","model.language_model.layers.17.mlp.down_proj","model.language_model.layers.17.mlp.gate_proj","model.language_model.layers.17.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.17","weight_key":"aggregate:model.language_model.layers.17"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":73.82954085158357,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":68.46147660298006,"general":69.54279188340745,"math":69.18570220624989,"multilingual":68.90322525830389}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.31937459999545087,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.29016712207898404,"general":0.29007626379054025,"math":0.29015572875640117,"multilingual":0.29043141706537134}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.18.linear_attn.in_proj_qkv","model.language_model.layers.18.linear_attn.in_proj_z","model.language_model.layers.18.linear_attn.out_proj","model.language_model.layers.18.mlp.down_proj","model.language_model.layers.18.mlp.gate_proj","model.language_model.layers.18.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.18","weight_key":"aggregate:model.language_model.layers.18"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":36700160,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":36700160,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":44.85899132694809,"logical_bytes":9461760,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":9175040,"logical_scale_bytes":286720,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":41.958790396598936,"general":41.0806579188413,"math":41.6781138390426,"multilingual":40.48967928272186}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.20174269953347285,"logical_bytes":18636800,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":18350080,"logical_scale_bytes":286720,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.1824341415983154,"general":0.1826164841079321,"math":0.1822392814165307,"multilingual":0.1828030095091704}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.19.mlp.down_proj","model.language_model.layers.19.mlp.gate_proj","model.language_model.layers.19.mlp.up_proj","model.language_model.layers.19.self_attn.k_proj","model.language_model.layers.19.self_attn.o_proj","model.language_model.layers.19.self_attn.q_proj","model.language_model.layers.19.self_attn.v_proj"]},"group_count":143360,"shape":[1,18350080],"unit_id":"model.language_model.layers.19","weight_key":"aggregate:model.language_model.layers.19"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":46.99384853461139,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":42.19830712442004,"general":44.07900185242275,"math":42.490641823918125,"multilingual":44.0386667128427}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.2274950433253654,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.20668343150186472,"general":0.2040345099304245,"math":0.20552167001511545,"multilingual":0.2038436712306959}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.2.linear_attn.in_proj_qkv","model.language_model.layers.2.linear_attn.in_proj_z","model.language_model.layers.2.linear_attn.out_proj","model.language_model.layers.2.mlp.down_proj","model.language_model.layers.2.mlp.gate_proj","model.language_model.layers.2.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.2","weight_key":"aggregate:model.language_model.layers.2"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":78.36596846229084,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":72.30336484463741,"general":73.5999626995297,"math":72.90415477233864,"multilingual":72.2421100826393}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.3402973741919436,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.3081734140292231,"general":0.3075080811856807,"math":0.30762807839240913,"multilingual":0.30714733169465314}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.20.linear_attn.in_proj_qkv","model.language_model.layers.20.linear_attn.in_proj_z","model.language_model.layers.20.linear_attn.out_proj","model.language_model.layers.20.mlp.down_proj","model.language_model.layers.20.mlp.gate_proj","model.language_model.layers.20.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.20","weight_key":"aggregate:model.language_model.layers.20"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":73.85020689006495,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":67.54520268693129,"general":68.92072992255969,"math":68.40455010803974,"multilingual":67.45437297345971}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.3306891783509614,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.2917292472034507,"general":0.2903544938014927,"math":0.2910770497467224,"multilingual":0.2913214797947965}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.21.linear_attn.in_proj_qkv","model.language_model.layers.21.linear_attn.in_proj_z","model.language_model.layers.21.linear_attn.out_proj","model.language_model.layers.21.mlp.down_proj","model.language_model.layers.21.mlp.gate_proj","model.language_model.layers.21.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.21","weight_key":"aggregate:model.language_model.layers.21"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":67.0530580826273,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":61.24217583916581,"general":62.19878876389374,"math":61.77807887819306,"multilingual":60.43976871721338}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.3068957780737304,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.2697817608293571,"general":0.2692656529435287,"math":0.2699774342356807,"multilingual":0.26815594927928527}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.22.linear_attn.in_proj_qkv","model.language_model.layers.22.linear_attn.in_proj_z","model.language_model.layers.22.linear_attn.out_proj","model.language_model.layers.22.mlp.down_proj","model.language_model.layers.22.mlp.gate_proj","model.language_model.layers.22.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.22","weight_key":"aggregate:model.language_model.layers.22"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":36700160,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":36700160,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":52.28308288473292,"logical_bytes":9461760,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":9175040,"logical_scale_bytes":286720,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":47.775789815717495,"general":48.38603212937909,"math":48.02571134426992,"multilingual":47.67009083923562}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.2375637159465251,"logical_bytes":18636800,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":18350080,"logical_scale_bytes":286720,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.21206419257710032,"general":0.21226135536290253,"math":0.21166186810368728,"multilingual":0.21216775197103072}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.23.mlp.down_proj","model.language_model.layers.23.mlp.gate_proj","model.language_model.layers.23.mlp.up_proj","model.language_model.layers.23.self_attn.k_proj","model.language_model.layers.23.self_attn.o_proj","model.language_model.layers.23.self_attn.q_proj","model.language_model.layers.23.self_attn.v_proj"]},"group_count":143360,"shape":[1,18350080],"unit_id":"model.language_model.layers.23","weight_key":"aggregate:model.language_model.layers.23"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":36700160,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":36700160,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":38.90126267230832,"logical_bytes":9461760,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":9175040,"logical_scale_bytes":286720,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":35.75313533851458,"general":36.67981405796299,"math":35.78822281558812,"multilingual":36.33617933720989}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.1797148477789056,"logical_bytes":18636800,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":18350080,"logical_scale_bytes":286720,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.1639824507395815,"general":0.1628384005115084,"math":0.16359783485267793,"multilingual":0.16324963340318943}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.3.mlp.down_proj","model.language_model.layers.3.mlp.gate_proj","model.language_model.layers.3.mlp.up_proj","model.language_model.layers.3.self_attn.k_proj","model.language_model.layers.3.self_attn.o_proj","model.language_model.layers.3.self_attn.q_proj","model.language_model.layers.3.self_attn.v_proj"]},"group_count":143360,"shape":[1,18350080],"unit_id":"model.language_model.layers.3","weight_key":"aggregate:model.language_model.layers.3"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":51.53371275990263,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":47.428718024512335,"general":48.84682896629219,"math":47.45986860126982,"multilingual":47.81516862568552}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.23502131413244448,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.2134016749098121,"general":0.21200356840894483,"math":0.21301811614764649,"multilingual":0.21270177804490775}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.4.linear_attn.in_proj_qkv","model.language_model.layers.4.linear_attn.in_proj_z","model.language_model.layers.4.linear_attn.out_proj","model.language_model.layers.4.mlp.down_proj","model.language_model.layers.4.mlp.gate_proj","model.language_model.layers.4.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.4","weight_key":"aggregate:model.language_model.layers.4"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":45.835059618457045,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":42.24099028359443,"general":43.39927247624052,"math":42.321084561529396,"multilingual":42.56722670373307}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.21092024111711224,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.192602435261214,"general":0.19142194365236187,"math":0.19214150138315403,"multilingual":0.19214444344772924}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.5.linear_attn.in_proj_qkv","model.language_model.layers.5.linear_attn.in_proj_z","model.language_model.layers.5.linear_attn.out_proj","model.language_model.layers.5.mlp.down_proj","model.language_model.layers.5.mlp.gate_proj","model.language_model.layers.5.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.5","weight_key":"aggregate:model.language_model.layers.5"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":40.8312270330258,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":37.79422164839062,"general":38.70048035830526,"math":37.71904440521914,"multilingual":37.62514050674895}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.19331728751040747,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.17712555803424532,"general":0.1765479962325378,"math":0.17710643852599195,"multilingual":0.1771008587603008}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.6.linear_attn.in_proj_qkv","model.language_model.layers.6.linear_attn.in_proj_z","model.language_model.layers.6.linear_attn.out_proj","model.language_model.layers.6.mlp.down_proj","model.language_model.layers.6.mlp.gate_proj","model.language_model.layers.6.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.6","weight_key":"aggregate:model.language_model.layers.6"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":36700160,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":36700160,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":26.914651858816992,"logical_bytes":9461760,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":9175040,"logical_scale_bytes":286720,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":25.14611616666656,"general":25.26365705677328,"math":25.0694593900882,"multilingual":24.743623369832044}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.12381094371776867,"logical_bytes":18636800,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":18350080,"logical_scale_bytes":286720,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.11372133265152089,"general":0.11343941309416472,"math":0.11372363344725844,"multilingual":0.11346231304530209}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.7.mlp.down_proj","model.language_model.layers.7.mlp.gate_proj","model.language_model.layers.7.mlp.up_proj","model.language_model.layers.7.self_attn.k_proj","model.language_model.layers.7.self_attn.o_proj","model.language_model.layers.7.self_attn.q_proj","model.language_model.layers.7.self_attn.v_proj"]},"group_count":143360,"shape":[1,18350080],"unit_id":"model.language_model.layers.7","weight_key":"aggregate:model.language_model.layers.7"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":36.95964829495392,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":34.73108924426697,"general":34.73477954606793,"math":34.632834409101946,"multilingual":33.83252916856728}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.17447653880193326,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.15984043355702293,"general":0.15907236051287002,"math":0.15962832378560218,"multilingual":0.15909027757122324}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.8.linear_attn.in_proj_qkv","model.language_model.layers.8.linear_attn.in_proj_z","model.language_model.layers.8.linear_attn.out_proj","model.language_model.layers.8.mlp.down_proj","model.language_model.layers.8.mlp.gate_proj","model.language_model.layers.8.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.8","weight_key":"aggregate:model.language_model.layers.8"},{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":42991616,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":42991616,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":35.022223661227294,"logical_bytes":11083776,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":10747904,"logical_scale_bytes":335872,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":32.75632689990168,"general":32.81000578099903,"math":32.84243747137766,"multilingual":32.04841196544964}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.16184605641411154,"logical_bytes":21831680,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":21495808,"logical_scale_bytes":335872,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.1482491284316698,"general":0.14782733755981514,"math":0.14816623426841785,"multilingual":0.1476474784242824}}],"evidence":{"aggregation":"transformer-block","member_unit_ids":["model.language_model.layers.9.linear_attn.in_proj_qkv","model.language_model.layers.9.linear_attn.in_proj_z","model.language_model.layers.9.linear_attn.out_proj","model.language_model.layers.9.mlp.down_proj","model.language_model.layers.9.mlp.gate_proj","model.language_model.layers.9.mlp.up_proj"]},"group_count":167936,"shape":[1,21495808],"unit_id":"model.language_model.layers.9","weight_key":"aggregate:model.language_model.layers.9"}]} diff --git a/research/results/autopolicy-001/candidates.json b/research/results/autopolicy-001/candidates.json new file mode 100644 index 0000000..6e2ea91 --- /dev/null +++ b/research/results/autopolicy-001/candidates.json @@ -0,0 +1 @@ +{"builder":{"name":"cliffquant-autopolicy-experiment001-v1","row_chunk_size":8,"runtime":{"device":"cpu","implementation":"CPython","machine":"AMD64","numpy":"2.4.6","numpy_build":{"blas":{"found":true,"name":"scipy-openblas","openblas configuration":"OpenBLAS 0.3.31.188.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY SkylakeX MAX_THREADS=24","version":"0.3.31.188.0"},"host":{"cpu":"x86_64","endian":"little","family":"x86_64","system":"windows"},"lapack":{"found":true,"name":"scipy-openblas","openblas configuration":"OpenBLAS 0.3.31.188.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY SkylakeX MAX_THREADS=24","version":"0.3.31.188.0"},"simd":{"baseline":["X86_V2"],"found":["X86_V3"]}},"packages":{"accelerate":null,"datasets":null,"huggingface-hub":null,"numpy":"2.4.6","tokenizers":null,"torch":null,"transformers":null},"platform":"Windows-10-10.0.26200-SP0","python":"3.11.15","python_executable":"python.exe"},"source_sha256":"597043bb0cc3b988592b24852858cd7e9e491b7840190372db3b07d1e4a387d6"},"candidate_definitions":{"dense16":{"bits":16,"group_size":null,"scale_selection":null},"w4_cliffquant":{"bits":4,"group_size":128,"scale_selection":"cliffquant_exact_minimax_fp16"},"w8_absmax":{"bits":8,"group_size":128,"scale_selection":"absmax_signed_range_nearest_fp16"}},"distortion_metric":{"additive_across_units":true,"downstream_metric":false,"groupwise_scalar":"sum_groupwise_max_environment_error","name":"normalized_diagonal_squared_error_with_separate_environment_totals","robust_policy":"max_environment_sum_module_error"},"environments":["general","code","math","multilingual"],"fingerprint_sha256":"2ce53865d6835a08d1427060841eeff9b5184a9ff54f682d2854cf0de5e928a1","profile":"experiment-001","schema":"cliffquant.autopolicy-candidates.v1","scope":{"budget_scope":"logical-target-weight-and-scale-payload","byte_model":"logical-codes-plus-fp16-scales-v1","checkpoint_export":false,"group_count":3883008,"group_size":128,"module_count":150,"weight_count":497025024},"source":{"base_model":{"config":{"file":"config.json","sha256":"b90b86f35c8e6925ef74ee04d0e758f0a845c83a42089ad82bbaa948de9b4204","size_bytes":2907},"fingerprint_sha256":"4c3f8c6ddda21c6e82eaad04eb9c962dc9bfab5ea727b05e4d3d42bfacc8c66a","id":"Qwen/Qwen3.5-0.8B-Base","index":{"file":"model.safetensors.index.json","sha256":"ce9a885efdf27d3664fdef5d512ad365216f1074051ef840c7cd8e5431495d0a","size_bytes":50900},"revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68","schema":"cliffquant.model-snapshot.v1","shards":[{"file":"model.safetensors-00001-of-00001.safetensors","sha256":"c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c","size_bytes":1746942600}],"tensor_count":488},"calibration":{"archive_sha256":"550f0b1b8a12b97435017a749fb8a6ffeae4db1bbe28f80d0257d78793103e3b","metadata_sha256":"b279ad598d7cf989ede0749f9f7fe3a455aa7de98d204155cfd9f25c09a821bf"},"w4_scale_run":{"manifest_sha256":"1292f4102dad57b070071c00d92ba43cd096b64b6434b3b9a3b3b41b73db3cf5","policy":"cliffquant_minimax"}},"status":"complete","totals":{"dense16":994050048,"w4_cliffquant":256278528,"w8_absmax":504791040},"units":[{"candidates":[{"candidate_id":"dense16","groupwise_max_distortion":0.0,"logical_bytes":12582912,"metadata":{"bits":16,"group_size":null,"logical_code_bytes":12582912,"logical_scale_bytes":0,"scale_selection":null},"per_environment_distortion":{"code":0.0,"general":0.0,"math":0.0,"multilingual":0.0}},{"candidate_id":"w4_cliffquant","groupwise_max_distortion":21.746162383876694,"logical_bytes":3244032,"metadata":{"bits":4,"group_size":128,"logical_code_bytes":3145728,"logical_scale_bytes":98304,"scale_selection":"cliffquant_exact_minimax_fp16"},"per_environment_distortion":{"code":21.04591658953203,"general":21.139329469270272,"math":21.01980013527177,"multilingual":21.20285629776141}},{"candidate_id":"w8_absmax","groupwise_max_distortion":0.08843108037847568,"logical_bytes":6389760,"metadata":{"bits":8,"group_size":128,"logical_code_bytes":6291456,"logical_scale_bytes":98304,"scale_selection":"absmax_signed_range_nearest_fp16"},"per_environment_distortion":{"code":0.08630889600518088,"general":0.08589130108269077,"math":0.0860742583934555,"multilingual":0.08618110337032736}}],"evidence":{"w4_scale_archive_sha256":"0a216551b82e92a487ce9f2bd690fa69dd2ae4009e7859d79cabf641ae2a0083","w4_scale_metadata_sha256":"fd942fa3b9f7c86a2bb66e5297d64e9c7435b13b4544d5566611daf893afd224","w8_archive":{"arrays":{"groupwise_max_objectives":{"dtype":" argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidates", type=Path, required=True) + parser.add_argument("--output-json", type=Path, required=True) + parser.add_argument("--output-figure", type=Path, required=True) + parser.add_argument("--output-manifest", type=Path, required=True) + parser.add_argument("--max-frontier-states", type=int, default=250_000) + return parser + + +def _budgets(bundle: dict) -> list[int]: + w4 = int(bundle["totals"][W4_CANDIDATE]) + w8 = int(bundle["totals"][W8_CANDIDATE]) + dense = int(bundle["totals"][DENSE16_CANDIDATE]) + interpolated = { + round(w4 + fraction * (dense - w4)) + for fraction in (0.0, 0.125, 0.25, 0.375, 0.5, 0.625, 0.75, 0.875, 1.0) + } + return sorted({w4, w8, dense, *interpolated}) + + +def generate( + candidate_path: Path, + *, + max_frontier_states: int, +) -> dict: + bundle = load_candidate_bundle(candidate_path) + points = [] + for budget in _budgets(bundle): + plan = solve_candidate_bundle( + bundle, + budget_bytes=budget, + objective=ROBUST_OBJECTIVE, + max_frontier_states=max_frontier_states, + ) + counts = Counter(decision.candidate_id for decision in plan.decisions) + points.append( + { + "budget_bytes": budget, + "used_bytes": plan.used_bytes, + "remaining_bytes": plan.remaining_bytes, + "logical_bits_per_target_weight": ( + plan.used_bytes * 8 / bundle["scope"]["weight_count"] + ), + "worst_environment_distortion": plan.worst_environment_distortion, + "per_environment_distortion": { + environment: value + for environment, value in zip( + plan.environments, + plan.per_environment_distortion, + strict=True, + ) + }, + "selection_counts": { + candidate_id: counts.get(candidate_id, 0) + for candidate_id in ( + W4_CANDIDATE, + W8_CANDIDATE, + DENSE16_CANDIDATE, + ) + }, + "transitions_evaluated": plan.transitions_evaluated, + "peak_frontier_states": plan.peak_frontier_states, + } + ) + baseline = points[0]["worst_environment_distortion"] + for point in points: + point["worst_environment_distortion_reduction_vs_all_w4"] = ( + 0.0 if baseline == 0.0 else 1.0 - point["worst_environment_distortion"] / baseline + ) + return { + "schema": FRONTIER_SCHEMA, + "status": "complete", + "candidate_sha256": sha256_file(candidate_path), + "candidate_fingerprint_sha256": bundle["fingerprint_sha256"], + "candidate_profile": bundle["profile"], + "objective": ROBUST_OBJECTIVE, + "budget_scope": bundle["scope"]["budget_scope"], + "allocation_granularity": bundle["scope"].get("allocation_granularity", "matrix"), + "allocation_unit_count": len(bundle["units"]), + "checkpoint_export": False, + "points": points, + } + + +def validate_frontier(payload: dict) -> None: + """Validate the portable AutoPolicy frontier summary.""" + + expected_keys = { + "schema", + "status", + "candidate_sha256", + "candidate_fingerprint_sha256", + "candidate_profile", + "objective", + "budget_scope", + "allocation_granularity", + "allocation_unit_count", + "checkpoint_export", + "points", + } + if set(payload) != expected_keys: + raise ValueError("frontier keys do not match the v1 schema") + if payload["schema"] != FRONTIER_SCHEMA or payload["status"] != "complete": + raise ValueError("unsupported or incomplete AutoPolicy frontier") + if payload["objective"] != ROBUST_OBJECTIVE: + raise ValueError("frontier objective must be robust-minimax") + if payload["checkpoint_export"] is not False: + raise ValueError("frontier must not claim checkpoint export") + unit_count = payload["allocation_unit_count"] + if type(unit_count) is not int or unit_count <= 0: + raise ValueError("allocation_unit_count must be a positive integer") + points = payload["points"] + if not isinstance(points, list) or len(points) < 2: + raise ValueError("frontier must contain at least two points") + previous_budget = -1 + for point in points: + budget = point.get("budget_bytes") + used = point.get("used_bytes") + if type(budget) is not int or type(used) is not int or not 0 < used <= budget: + raise ValueError("frontier point has invalid logical-byte values") + if budget <= previous_budget: + raise ValueError("frontier budgets must be strictly increasing") + previous_budget = budget + proxy = point.get("worst_environment_distortion") + if ( + isinstance(proxy, bool) + or not isinstance(proxy, (int, float)) + or not math.isfinite(float(proxy)) + or proxy < 0 + ): + raise ValueError("frontier point has invalid distortion") + counts = point.get("selection_counts") + if not isinstance(counts, dict) or sum(counts.values()) != unit_count: + raise ValueError("frontier selection counts do not match allocation units") + + +def _atomic_replace(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as handle: + temporary_path = Path(handle.name) + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_path, path) + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + + +def _plot(payload: dict, output: Path) -> None: + try: + import matplotlib.pyplot as plt + except ImportError as exc: + raise RuntimeError("install CliffQuant with the 'figures' extra") from exc + + points = payload["points"] + x = [point["used_bytes"] / (1024**2) for point in points] + y = [point["worst_environment_distortion"] for point in points] + counts = { + candidate: [point["selection_counts"][candidate] for point in points] + for candidate in (W4_CANDIDATE, W8_CANDIDATE, DENSE16_CANDIDATE) + } + + plt.style.use("dark_background") + figure, axes = plt.subplots( + 2, + 1, + figsize=(11.4, 7.4), + sharex=True, + gridspec_kw={"height_ratios": [2.0, 1.0]}, + ) + figure.patch.set_facecolor("#081015") + for axis in axes: + axis.set_facecolor("#081015") + axis.grid(color="#29424b", alpha=0.38, linewidth=0.8) + axis.spines[["top", "right"]].set_visible(False) + + axes[0].plot(x, y, color="#7ce7d8", marker="o", linewidth=2.2, markersize=5) + axes[0].set_ylabel("Worst-environment proxy") + axes[0].set_yscale("symlog", linthresh=1.0, linscale=0.7) + + bottom = [0] * len(points) + colors = { + W4_CANDIDATE: "#377f86", + W8_CANDIDATE: "#efb65b", + DENSE16_CANDIDATE: "#d66bb1", + } + labels = { + W4_CANDIDATE: "W4 CliffQuant", + W8_CANDIDATE: "W8 AbsMax", + DENSE16_CANDIDATE: "BF16 reference", + } + for candidate in (W4_CANDIDATE, W8_CANDIDATE, DENSE16_CANDIDATE): + axes[1].bar( + x, + counts[candidate], + bottom=bottom, + width=max(x) * 0.025, + color=colors[candidate], + label=labels[candidate], + ) + bottom = [ + previous + current for previous, current in zip(bottom, counts[candidate], strict=True) + ] + axes[1].set_xlabel("Selected logical target payload (MiB)") + unit_label = ( + "Transformer blocks" + if payload["allocation_granularity"] == "transformer-block" + else "Allocation units" + ) + axes[1].set_ylabel(unit_label) + axes[1].legend( + frameon=False, + ncol=3, + loc="upper center", + bbox_to_anchor=(0.5, 1.16), + ) + figure.suptitle( + "CliffQuant AutoPolicy 001", + x=0.09, + y=0.975, + ha="left", + fontsize=17, + weight="bold", + ) + figure.text( + 0.09, + 0.93, + "Exact robust transformer-block allocation on measured W4, W8, and BF16 candidates", + color="#9eb8c1", + fontsize=10, + ) + figure.text( + 0.09, + 0.018, + "Planning proxy; not checkpoint size, runtime memory, latency, or downstream quality.", + color="#8da6af", + fontsize=8, + ) + figure.subplots_adjust(left=0.09, right=0.98, top=0.88, bottom=0.11, hspace=0.2) + output.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + dir=output.parent, + prefix=f".{output.name}.", + suffix=".png", + delete=False, + ) as handle: + temporary_path = Path(handle.name) + figure.savefig( + temporary_path, + dpi=180, + facecolor=figure.get_facecolor(), + format="png", + ) + os.replace(temporary_path, output) + finally: + plt.close(figure) + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + + +def _png_dimensions(path: Path) -> tuple[int, int]: + header = path.read_bytes()[:24] + if len(header) != 24 or header[:8] != b"\x89PNG\r\n\x1a\n": + raise ValueError(f"not a PNG file: {path}") + return struct.unpack(">II", header[16:24]) + + +def _json_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def _relative_file(path: Path, *, manifest_path: Path) -> str: + return Path(os.path.relpath(path.resolve(), manifest_path.parent.resolve())).as_posix() + + +def build_provenance_manifest( + *, + candidate_path: Path, + frontier_path: Path, + figure_path: Path, + manifest_path: Path, + candidate_fingerprint_sha256: str, +) -> dict: + """Bind the checked-in AutoPolicy graph to its generator and exact inputs.""" + + import matplotlib + + width, height = _png_dimensions(figure_path) + generator_path = Path(__file__).resolve() + return { + "schema": FIGURE_PROVENANCE_SCHEMA, + "generator": { + "file": "scripts/generate_autopolicy_frontier.py", + "matplotlib": matplotlib.__version__, + "sha256": sha256_file(generator_path), + }, + "input": { + "candidate_fingerprint_sha256": candidate_fingerprint_sha256, + "file": _relative_file(candidate_path, manifest_path=manifest_path), + "sha256": sha256_file(candidate_path), + "size_bytes": candidate_path.stat().st_size, + }, + "outputs": { + "figure": { + "file": _relative_file(figure_path, manifest_path=manifest_path), + "height_px": height, + "sha256": sha256_file(figure_path), + "size_bytes": figure_path.stat().st_size, + "width_px": width, + }, + "frontier": { + "file": _relative_file(frontier_path, manifest_path=manifest_path), + "sha256": sha256_file(frontier_path), + "size_bytes": frontier_path.stat().st_size, + }, + }, + } + + +def verify_provenance_manifest(manifest_path: Path) -> dict: + """Verify every path, size, and digest recorded for the AutoPolicy graph.""" + + try: + payload = json.loads( + manifest_path.read_text(encoding="utf-8"), + object_pairs_hook=_json_object, + parse_constant=lambda token: (_ for _ in ()).throw( + ValueError(f"non-finite JSON number in {manifest_path}: {token}") + ), + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError(f"invalid AutoPolicy figure provenance: {manifest_path}") from exc + if not isinstance(payload, dict) or payload.get("schema") != FIGURE_PROVENANCE_SCHEMA: + raise ValueError("unsupported AutoPolicy figure provenance manifest") + root = manifest_path.parent.resolve() + records = [ + payload.get("input"), + payload.get("outputs", {}).get("figure"), + payload.get("outputs", {}).get("frontier"), + ] + for record in records: + if not isinstance(record, dict): + raise ValueError("AutoPolicy figure provenance record is missing") + relative = record.get("file") + expected_size = record.get("size_bytes") + expected_sha256 = record.get("sha256") + if ( + not isinstance(relative, str) + or Path(relative).is_absolute() + or type(expected_size) is not int + or expected_size < 0 + or not isinstance(expected_sha256, str) + or len(expected_sha256) != 64 + ): + raise ValueError("invalid AutoPolicy figure provenance record") + path = (root / relative).resolve() + if not path.is_file(): + raise ValueError(f"missing AutoPolicy figure provenance file: {relative}") + if path.stat().st_size != expected_size or sha256_file(path) != expected_sha256: + raise ValueError(f"AutoPolicy figure provenance mismatch: {relative}") + figure = payload["outputs"]["figure"] + width, height = _png_dimensions((root / figure["file"]).resolve()) + if figure.get("width_px") != width or figure.get("height_px") != height: + raise ValueError("AutoPolicy figure dimensions do not match provenance") + + generator = payload.get("generator") + if not isinstance(generator, dict): + raise ValueError("AutoPolicy figure generator provenance is missing") + generator_path = Path(__file__).resolve() + if generator.get("file") != "scripts/generate_autopolicy_frontier.py" or generator.get( + "sha256" + ) != sha256_file(generator_path): + raise ValueError("AutoPolicy figure generator does not match provenance") + return payload + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + payload = generate( + args.candidates, + max_frontier_states=args.max_frontier_states, + ) + validate_frontier(payload) + _atomic_replace(args.output_json, canonical_json_bytes(payload)) + _plot(payload, args.output_figure) + manifest = build_provenance_manifest( + candidate_path=args.candidates, + frontier_path=args.output_json, + figure_path=args.output_figure, + manifest_path=args.output_manifest, + candidate_fingerprint_sha256=payload["candidate_fingerprint_sha256"], + ) + _atomic_replace(args.output_manifest, canonical_json_bytes(manifest)) + verify_provenance_manifest(args.output_manifest) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke_installed_distribution.py b/scripts/smoke_installed_distribution.py index 4a25e61..e40dfbf 100644 --- a/scripts/smoke_installed_distribution.py +++ b/scripts/smoke_installed_distribution.py @@ -16,7 +16,7 @@ def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() parser.add_argument("artifact", type=Path) - parser.add_argument("--expected-version", default="0.1.2") + parser.add_argument("--expected-version", default="0.2.0") return parser @@ -96,6 +96,78 @@ def main(argv: Sequence[str] | None = None) -> int: "'size_bytes':17663}" ) _run([str(installed_python), "-I", "-c", import_check], cwd=root, env=env) + + installed_cli = ( + environment_dir / "Scripts" / "cliffquant.exe" + if os.name == "nt" + else environment_dir / "bin" / "cliffquant" + ) + _run([str(installed_cli), "--version"], cwd=root, env=env) + _run([str(installed_cli), "autopolicy", "--help"], cwd=root, env=env) + + autopolicy_dir = root / "autopolicy" + autopolicy_dir.mkdir() + candidate_path = autopolicy_dir / "candidates.json" + plan_path = autopolicy_dir / "plan.json" + fixture_code = ( + "from pathlib import Path; " + "from cliffquant.autopolicy_artifacts import " + "CANDIDATE_SCHEMA,write_candidate_bundle; " + f"path=Path({str(candidate_path)!r}); " + "write_candidate_bundle(path,{" + "'schema':CANDIDATE_SCHEMA,'status':'complete','profile':'wheel-smoke'," + "'scope':{'budget_scope':'logical-target-weight-and-scale-payload'," + "'checkpoint_export':False}," + "'source':{'kind':'synthetic'},'environments':['general','code']," + "'distortion_metric':{'name':'synthetic','additive_across_units':True," + "'downstream_metric':False}," + "'candidate_definitions':{'dense16':{},'w4':{}}," + "'units':[{'unit_id':'layer.0','weight_key':'layer.0.weight'," + "'shape':[8,128],'group_count':8,'candidates':[" + "{'candidate_id':'dense16','logical_bytes':2048," + "'groupwise_max_distortion':0.0," + "'per_environment_distortion':{'general':0.0,'code':0.0}," + "'metadata':{'bits':16}}," + "{'candidate_id':'w4','logical_bytes':528," + "'groupwise_max_distortion':2.0," + "'per_environment_distortion':{'general':2.0,'code':1.0}," + "'metadata':{'bits':4}}]," + "'evidence':{'kind':'synthetic'}}]," + "'totals':{'dense16':2048,'w4':528}," + "'builder':{'name':'wheel-smoke'}})" + ) + _run([str(installed_python), "-I", "-c", fixture_code], cwd=root, env=env) + _run( + [ + str(installed_cli), + "autopolicy", + "solve", + "--candidates", + str(candidate_path), + "--budget-bytes", + "528", + "--output", + str(plan_path), + ], + cwd=root, + env=env, + ) + _run( + [ + str(installed_cli), + "autopolicy", + "verify", + "--candidates", + str(candidate_path), + "--plan", + str(plan_path), + ], + cwd=root, + env=env, + ) + if not plan_path.is_file(): + raise ValueError("installed AutoPolicy CLI did not create a plan") + _run( [ str(installed_python), diff --git a/scripts/validate_autopolicy_release.py b/scripts/validate_autopolicy_release.py new file mode 100644 index 0000000..d8831fe --- /dev/null +++ b/scripts/validate_autopolicy_release.py @@ -0,0 +1,380 @@ +"""Build or validate the portable AutoPolicy 001 evidence manifest.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import tempfile +from collections import Counter +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +from cliffquant.autopolicy_artifacts import ( + ROBUST_OBJECTIVE, + aggregate_experiment001_transformer_blocks, + load_candidate_bundle, + solve_candidate_bundle, + verify_plan, +) +from cliffquant.autopolicy_experiment001 import ( + DENSE16_CANDIDATE, + W4_CANDIDATE, + W8_CANDIDATE, + verify_experiment001_evidence, +) +from cliffquant.provenance import canonical_json_bytes, sha256_bytes, sha256_file + +SCHEMA = "cliffquant.autopolicy-release.v1" +MANIFEST_NAME = "manifest.json" +MANIFEST_HASH_NAME = "manifest.sha256.json" +DATA_FILES = ( + "candidates.json", + "candidates-transformer-blocks.json", + "frontier.json", + "plan-blocks-7bpp.json", +) +MAX_FRONTIER_STATES = 250_000 + + +def _json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def _load_json(path: Path) -> dict[str, Any]: + try: + payload = json.loads( + path.read_text(encoding="utf-8"), + object_pairs_hook=_json_object, + parse_constant=lambda token: (_ for _ in ()).throw( + ValueError(f"non-finite JSON number in {path}: {token}") + ), + ) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid JSON: {path}") from exc + if not isinstance(payload, dict): + raise ValueError(f"JSON artifact must contain an object: {path}") + return payload + + +def _checked_file(root: Path, relative: str) -> Path: + value = Path(relative) + if not relative or value.is_absolute() or any(part in {"", ".", ".."} for part in value.parts): + raise ValueError(f"unsafe release path: {relative!r}") + candidate = (root / value).resolve() + if not candidate.is_relative_to(root) or not candidate.is_file() or candidate.is_symlink(): + raise ValueError(f"release member is missing, unsafe, or not regular: {relative}") + return candidate + + +def _validate_grouped_candidates( + matrix_path: Path, + matrix: Mapping[str, Any], + grouped: Mapping[str, Any], +) -> None: + expected = aggregate_experiment001_transformer_blocks(matrix_path, matrix) + if canonical_json_bytes(expected) != canonical_json_bytes(grouped): + raise ValueError( + "transformer-block candidates do not match exact aggregation " + "of the matrix candidate bundle" + ) + + +def _frontier_budgets(grouped: Mapping[str, Any]) -> list[int]: + w4 = int(grouped["totals"][W4_CANDIDATE]) + w8 = int(grouped["totals"][W8_CANDIDATE]) + dense = int(grouped["totals"][DENSE16_CANDIDATE]) + interpolated = { + round(w4 + fraction * (dense - w4)) + for fraction in (0.0, 0.125, 0.25, 0.375, 0.5, 0.625, 0.75, 0.875, 1.0) + } + return sorted({w4, w8, dense, *interpolated}) + + +def _recompute_frontier(grouped_path: Path, grouped: Mapping[str, Any]) -> dict[str, Any]: + points: list[dict[str, Any]] = [] + for budget in _frontier_budgets(grouped): + plan = solve_candidate_bundle( + grouped, + budget_bytes=budget, + objective=ROBUST_OBJECTIVE, + max_frontier_states=MAX_FRONTIER_STATES, + ) + counts = Counter(decision.candidate_id for decision in plan.decisions) + points.append( + { + "budget_bytes": budget, + "used_bytes": plan.used_bytes, + "remaining_bytes": plan.remaining_bytes, + "logical_bits_per_target_weight": ( + plan.used_bytes * 8 / grouped["scope"]["weight_count"] + ), + "worst_environment_distortion": plan.worst_environment_distortion, + "per_environment_distortion": { + environment: value + for environment, value in zip( + plan.environments, + plan.per_environment_distortion, + strict=True, + ) + }, + "selection_counts": { + candidate_id: counts.get(candidate_id, 0) + for candidate_id in ( + W4_CANDIDATE, + W8_CANDIDATE, + DENSE16_CANDIDATE, + ) + }, + "transitions_evaluated": plan.transitions_evaluated, + "peak_frontier_states": plan.peak_frontier_states, + } + ) + baseline = points[0]["worst_environment_distortion"] + for point in points: + point["worst_environment_distortion_reduction_vs_all_w4"] = ( + 0.0 if baseline == 0.0 else 1.0 - point["worst_environment_distortion"] / baseline + ) + return { + "schema": "cliffquant.autopolicy-frontier.v1", + "status": "complete", + "candidate_sha256": sha256_file(grouped_path), + "candidate_fingerprint_sha256": grouped["fingerprint_sha256"], + "candidate_profile": grouped["profile"], + "objective": ROBUST_OBJECTIVE, + "budget_scope": grouped["scope"]["budget_scope"], + "allocation_granularity": grouped["scope"].get("allocation_granularity", "matrix"), + "allocation_unit_count": len(grouped["units"]), + "checkpoint_export": False, + "points": points, + } + + +def _validate_frontier( + payload: Mapping[str, Any], + *, + grouped_path: Path, + grouped: Mapping[str, Any], +) -> None: + if ( + payload.get("schema") != "cliffquant.autopolicy-frontier.v1" + or payload.get("status") != "complete" + or payload.get("objective") != ROBUST_OBJECTIVE + or payload.get("checkpoint_export") is not False + ): + raise ValueError("unsupported or incomplete AutoPolicy frontier") + if payload.get("candidate_sha256") != sha256_file(grouped_path): + raise ValueError("frontier candidate hash mismatch") + points = payload.get("points") + if not isinstance(points, list) or len(points) < 2: + raise ValueError("frontier points are missing") + previous_budget = -1 + for point in points: + if not isinstance(point, dict): + raise TypeError("frontier points must be objects") + budget = point.get("budget_bytes") + used = point.get("used_bytes") + distortion = point.get("worst_environment_distortion") + if ( + type(budget) is not int + or type(used) is not int + or not 0 < used <= budget + or budget <= previous_budget + ): + raise ValueError("frontier byte values are invalid") + if ( + isinstance(distortion, bool) + or not isinstance(distortion, (int, float)) + or not math.isfinite(float(distortion)) + or distortion < 0 + ): + raise ValueError("frontier distortion is invalid") + previous_budget = budget + expected = _recompute_frontier(grouped_path, grouped) + if canonical_json_bytes(expected) != canonical_json_bytes(payload): + raise ValueError("AutoPolicy frontier does not match exact recomputation") + + +def _expected_inventory(root: Path, candidate: Mapping[str, Any]) -> tuple[str, ...]: + archives = tuple(unit["evidence"]["w8_archive"]["file"] for unit in candidate["units"]) + expected = tuple(sorted((*DATA_FILES, *archives))) + actual = tuple( + sorted( + path.relative_to(root).as_posix() + for path in root.rglob("*") + if path.is_file() and path.name not in {MANIFEST_NAME, MANIFEST_HASH_NAME} + ) + ) + if actual != expected: + missing = sorted(set(expected) - set(actual)) + extra = sorted(set(actual) - set(expected)) + raise ValueError(f"release inventory mismatch; missing={missing}, extra={extra}") + return expected + + +def build_manifest(root: str | Path) -> dict[str, Any]: + """Validate every portable artifact and return its deterministic manifest.""" + + release_root = Path(root).resolve() + if not release_root.is_dir(): + raise FileNotFoundError(f"release directory does not exist: {release_root}") + paths = {name: _checked_file(release_root, name) for name in DATA_FILES} + candidate = load_candidate_bundle(paths["candidates.json"]) + grouped = load_candidate_bundle(paths["candidates-transformer-blocks.json"]) + if candidate["profile"] != "experiment-001": + raise ValueError("matrix candidate profile is not Experiment 001") + parent = grouped["source"].get("parent_candidate_bundle", {}) + if parent.get("file_sha256") != sha256_file(paths["candidates.json"]): + raise ValueError("grouped candidates are not bound to the matrix candidate file") + if parent.get("fingerprint_sha256") != candidate["fingerprint_sha256"]: + raise ValueError("grouped candidate parent fingerprint mismatch") + _validate_grouped_candidates(paths["candidates.json"], candidate, grouped) + + evidence = verify_experiment001_evidence(paths["candidates.json"], candidate) + frontier = _load_json(paths["frontier.json"]) + _validate_frontier( + frontier, + grouped_path=paths["candidates-transformer-blocks.json"], + grouped=grouped, + ) + plan = verify_plan( + paths["candidates-transformer-blocks.json"], + paths["plan-blocks-7bpp.json"], + ) + expected = _expected_inventory(release_root, candidate) + files = [ + { + "path": relative, + "size_bytes": _checked_file(release_root, relative).stat().st_size, + "sha256": sha256_file(_checked_file(release_root, relative)), + } + for relative in expected + ] + return { + "schema": SCHEMA, + "status": "pass", + "profile": "autopolicy-001", + "candidate": { + "file_sha256": sha256_file(paths["candidates.json"]), + "fingerprint_sha256": candidate["fingerprint_sha256"], + "module_count": len(candidate["units"]), + "evidence_verification": evidence, + }, + "transformer_block_view": { + "file_sha256": sha256_file(paths["candidates-transformer-blocks.json"]), + "fingerprint_sha256": grouped["fingerprint_sha256"], + "allocation_unit_count": len(grouped["units"]), + }, + "frontier": { + "file_sha256": sha256_file(paths["frontier.json"]), + "point_count": len(frontier["points"]), + "objective": frontier["objective"], + }, + "representative_plan": plan, + "claim_boundary": { + "exact": ( + "recorded additive proxy allocation at the recorded granularity " + "under logical target-payload bytes" + ), + "not_claimed": [ + "mixed-bit checkpoint export", + "downstream quality", + "checkpoint size", + "runtime memory", + "latency", + "novelty", + ], + }, + "files": files, + } + + +def _atomic_create(path: Path, payload: bytes) -> None: + if path.exists(): + raise FileExistsError(f"refusing to overwrite existing file: {path}") + temporary: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as handle: + temporary = Path(handle.name) + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + if path.exists(): + raise FileExistsError(f"refusing to overwrite existing file: {path}") + os.replace(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def write_manifest(root: str | Path) -> dict[str, Any]: + release_root = Path(root).resolve() + manifest = build_manifest(release_root) + payload = canonical_json_bytes(manifest) + manifest_path = release_root / MANIFEST_NAME + _atomic_create(manifest_path, payload) + sidecar = { + "schema": "cliffquant.autopolicy-release-manifest-hash.v1", + "file": MANIFEST_NAME, + "sha256": sha256_bytes(payload), + } + _atomic_create(release_root / MANIFEST_HASH_NAME, canonical_json_bytes(sidecar)) + return manifest + + +def check_manifest(root: str | Path) -> dict[str, Any]: + release_root = Path(root).resolve() + manifest_path = _checked_file(release_root, MANIFEST_NAME) + sidecar_path = _checked_file(release_root, MANIFEST_HASH_NAME) + recorded = _load_json(manifest_path) + expected = build_manifest(release_root) + if recorded != expected: + raise ValueError("AutoPolicy release manifest content mismatch") + sidecar = _load_json(sidecar_path) + if ( + sidecar.get("schema") != "cliffquant.autopolicy-release-manifest-hash.v1" + or sidecar.get("file") != MANIFEST_NAME + or sidecar.get("sha256") != sha256_file(manifest_path) + ): + raise ValueError("AutoPolicy release manifest sidecar mismatch") + return recorded + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + parser.add_argument("--write", action="store_true") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + result = write_manifest(args.root) if args.write else check_manifest(args.root) + print( + json.dumps( + { + "status": result["status"], + "schema": result["schema"], + "files": len(result["files"]), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/cliffquant/__init__.py b/src/cliffquant/__init__.py index ffface2..9191e6a 100644 --- a/src/cliffquant/__init__.py +++ b/src/cliffquant/__init__.py @@ -1,6 +1,20 @@ """Correctness-first exhaustive FP16 scale selection.""" from cliffquant.approximate import solve_approximate +from cliffquant.autopolicy import ( + AutoPolicyPlan, + FrontierLimitError, + InfeasiblePolicyError, + PolicyCandidate, + PolicyDecision, + QuantizableUnit, + RobustAutoPolicyPlan, + RobustPolicyCandidate, + RobustPolicyDecision, + RobustQuantizableUnit, + solve_autopolicy, + solve_robust_autopolicy, +) from cliffquant.breakpoint import ( BreakpointSearchProvenance, BreakpointSolveResult, @@ -31,9 +45,19 @@ "FP16_POSITIVE_FINITE_COUNT", "ApproximateSearchProvenance", "ApproximateSolveResult", + "AutoPolicyPlan", "BreakpointSearchProvenance", "BreakpointSolveResult", + "FrontierLimitError", + "InfeasiblePolicyError", + "PolicyCandidate", + "PolicyDecision", + "QuantizableUnit", "QuantizerSpec", + "RobustAutoPolicyPlan", + "RobustPolicyCandidate", + "RobustPolicyDecision", + "RobustQuantizableUnit", "ScaleEvaluation", "SolveResult", "bits_to_scale", @@ -44,8 +68,10 @@ "quantize_codes", "scale_to_bits", "solve_approximate", + "solve_autopolicy", "solve_breakpoint_exact", "solve_exact", + "solve_robust_autopolicy", ] -__version__ = "0.1.2" +__version__ = "0.2.0" diff --git a/src/cliffquant/autopolicy.py b/src/cliffquant/autopolicy.py new file mode 100644 index 0000000..c0beff8 --- /dev/null +++ b/src/cliffquant/autopolicy.py @@ -0,0 +1,783 @@ +"""Exact byte-budget policy search over discrete per-unit candidates. + +The optimization problem is a multiple-choice knapsack with a separable proxy: + + minimize sum(candidate.distortion for each selected candidate) + subject to + sum(candidate.byte_size for each selected candidate) <= budget + exactly one candidate is selected for every quantizable unit + +``distortion`` is deliberately generic. It can be a calibration loss increase, +weighted reconstruction error, or another non-negative scalar, provided every +candidate was scored on the same scale. The solver makes no claim about +interactions between units; it exactly solves the supplied additive proxy. +""" + +from __future__ import annotations + +import json +import math +from collections.abc import Iterable +from dataclasses import dataclass +from numbers import Real +from typing import Any + +import numpy as np + +DEFAULT_MAX_TRANSITIONS = 1_000_000 +DEFAULT_MAX_PARETO_COMPARISONS = 25_000_000 + + +class InfeasiblePolicyError(ValueError): + """Raised when no one-candidate-per-unit plan fits the byte budget.""" + + +class FrontierLimitError(RuntimeError): + """Raised instead of silently approximating an oversized exact search.""" + + +def _validate_identifier(value: object, *, name: str) -> str: + if type(value) is not str: + raise TypeError(f"{name} must be a string") + if not value: + raise ValueError(f"{name} must not be empty") + if value != value.strip(): + raise ValueError(f"{name} must not have leading or trailing whitespace") + return value + + +@dataclass(frozen=True, slots=True) +class PolicyCandidate: + """One measured precision/configuration choice for a quantizable unit.""" + + candidate_id: str + byte_size: int + distortion: float + + def __post_init__(self) -> None: + _validate_identifier(self.candidate_id, name="candidate_id") + if type(self.byte_size) is not int: + raise TypeError("byte_size must be an integer, not a boolean or float") + if self.byte_size <= 0: + raise ValueError("byte_size must be positive") + if isinstance(self.distortion, bool) or not isinstance(self.distortion, Real): + raise TypeError("distortion must be a real number, not a boolean") + normalized_distortion = float(self.distortion) + if not math.isfinite(normalized_distortion): + raise ValueError("distortion must be finite") + if normalized_distortion < 0.0: + raise ValueError("distortion must be non-negative") + object.__setattr__(self, "distortion", normalized_distortion) + + +@dataclass(frozen=True, slots=True) +class QuantizableUnit: + """A named unit and every candidate the solver may choose for it.""" + + unit_id: str + candidates: tuple[PolicyCandidate, ...] + + def __post_init__(self) -> None: + _validate_identifier(self.unit_id, name="unit_id") + if isinstance(self.candidates, (str, bytes)): + raise TypeError("candidates must be an iterable of PolicyCandidate objects") + try: + candidates = tuple(self.candidates) + except TypeError as exc: + raise TypeError("candidates must be an iterable of PolicyCandidate objects") from exc + if not candidates: + raise ValueError("candidates must not be empty") + if any(not isinstance(candidate, PolicyCandidate) for candidate in candidates): + raise TypeError("candidates must contain only PolicyCandidate objects") + + candidate_ids = [candidate.candidate_id for candidate in candidates] + if len(set(candidate_ids)) != len(candidate_ids): + raise ValueError(f"candidate_id values must be unique within unit {self.unit_id!r}") + + object.__setattr__( + self, + "candidates", + tuple(sorted(candidates, key=lambda candidate: candidate.candidate_id)), + ) + + +@dataclass(frozen=True, slots=True) +class PolicyDecision: + """The selected candidate for one quantizable unit.""" + + unit_id: str + candidate_id: str + byte_size: int + distortion: float + + def to_dict(self) -> dict[str, str | int | float]: + """Return a JSON-compatible decision record.""" + + return { + "unit_id": self.unit_id, + "candidate_id": self.candidate_id, + "byte_size": self.byte_size, + "distortion": self.distortion, + } + + +@dataclass(frozen=True, slots=True) +class AutoPolicyPlan: + """Auditable result of the exact byte-budget search.""" + + budget_bytes: int + used_bytes: int + remaining_bytes: int + aggregate_distortion: float + decisions: tuple[PolicyDecision, ...] + transitions_evaluated: int + peak_frontier_states: int + + def to_dict(self) -> dict[str, Any]: + """Return a result made only of JSON-compatible primitives.""" + + return { + "schema_version": 1, + "algorithm": "exact_pareto_frontier", + "objective": "minimize_sum_distortion", + "constraint": "select_exactly_one_candidate_per_unit_under_byte_budget", + "tie_breaking": [ + "lower_aggregate_distortion", + "lower_used_bytes", + "lexicographically_smaller_candidate_ids_by_sorted_unit_id", + ], + "budget_bytes": self.budget_bytes, + "used_bytes": self.used_bytes, + "remaining_bytes": self.remaining_bytes, + "aggregate_distortion": self.aggregate_distortion, + "unit_count": len(self.decisions), + "transitions_evaluated": self.transitions_evaluated, + "peak_frontier_states": self.peak_frontier_states, + "decisions": [decision.to_dict() for decision in self.decisions], + } + + def to_json(self) -> str: + """Return a canonical compact JSON representation of this plan.""" + + return json.dumps( + self.to_dict(), + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + + +@dataclass(frozen=True, slots=True) +class RobustPolicyCandidate: + """One candidate scored separately in every frozen environment.""" + + candidate_id: str + byte_size: int + per_environment_distortion: tuple[float, ...] + + def __post_init__(self) -> None: + _validate_identifier(self.candidate_id, name="candidate_id") + if type(self.byte_size) is not int: + raise TypeError("byte_size must be an integer, not a boolean or float") + if self.byte_size <= 0: + raise ValueError("byte_size must be positive") + if isinstance(self.per_environment_distortion, (str, bytes)): + raise TypeError("per_environment_distortion must be an iterable of real numbers") + try: + values = tuple(self.per_environment_distortion) + except TypeError as exc: + raise TypeError( + "per_environment_distortion must be an iterable of real numbers" + ) from exc + if not values: + raise ValueError("per_environment_distortion must not be empty") + normalized: list[float] = [] + for value in values: + if isinstance(value, bool) or not isinstance(value, Real): + raise TypeError("per_environment_distortion must contain only real numbers") + number = float(value) + if not math.isfinite(number): + raise ValueError("per_environment_distortion must contain only finite values") + if number < 0.0: + raise ValueError("per_environment_distortion must be non-negative") + normalized.append(number) + object.__setattr__(self, "per_environment_distortion", tuple(normalized)) + + +@dataclass(frozen=True, slots=True) +class RobustQuantizableUnit: + """A unit whose candidates retain separate environment scores.""" + + unit_id: str + candidates: tuple[RobustPolicyCandidate, ...] + + def __post_init__(self) -> None: + _validate_identifier(self.unit_id, name="unit_id") + if isinstance(self.candidates, (str, bytes)): + raise TypeError("candidates must be an iterable of RobustPolicyCandidate objects") + try: + candidates = tuple(self.candidates) + except TypeError as exc: + raise TypeError( + "candidates must be an iterable of RobustPolicyCandidate objects" + ) from exc + if not candidates: + raise ValueError("candidates must not be empty") + if any(not isinstance(candidate, RobustPolicyCandidate) for candidate in candidates): + raise TypeError("candidates must contain only RobustPolicyCandidate objects") + widths = {len(candidate.per_environment_distortion) for candidate in candidates} + if len(widths) != 1: + raise ValueError("all candidates in a unit must use the same environment count") + candidate_ids = [candidate.candidate_id for candidate in candidates] + if len(set(candidate_ids)) != len(candidate_ids): + raise ValueError(f"candidate_id values must be unique within unit {self.unit_id!r}") + object.__setattr__( + self, + "candidates", + tuple(sorted(candidates, key=lambda candidate: candidate.candidate_id)), + ) + + +@dataclass(frozen=True, slots=True) +class RobustPolicyDecision: + """The selected robust candidate for one quantizable unit.""" + + unit_id: str + candidate_id: str + byte_size: int + per_environment_distortion: tuple[float, ...] + + def to_dict(self, environments: tuple[str, ...]) -> dict[str, Any]: + """Return one JSON-compatible decision record.""" + + return { + "unit_id": self.unit_id, + "candidate_id": self.candidate_id, + "byte_size": self.byte_size, + "per_environment_distortion": { + environment: value + for environment, value in zip( + environments, + self.per_environment_distortion, + strict=True, + ) + }, + } + + +@dataclass(frozen=True, slots=True) +class RobustAutoPolicyPlan: + """Exact minimax plan over separately retained environment totals.""" + + environments: tuple[str, ...] + budget_bytes: int + used_bytes: int + remaining_bytes: int + per_environment_distortion: tuple[float, ...] + worst_environment_distortion: float + decisions: tuple[RobustPolicyDecision, ...] + transitions_evaluated: int + peak_frontier_states: int + + def to_dict(self) -> dict[str, Any]: + """Return a self-describing result made of JSON primitives.""" + + return { + "schema_version": 1, + "algorithm": "exact_multienvironment_pareto_frontier", + "objective": "minimize_max_environment_sum_distortion", + "constraint": "select_exactly_one_candidate_per_unit_under_byte_budget", + "tie_breaking": [ + "lower_worst_environment_distortion", + "lower_sum_environment_distortion", + "lower_used_bytes", + "lexicographically_smaller_candidate_ids_by_sorted_unit_id", + ], + "environments": list(self.environments), + "budget_bytes": self.budget_bytes, + "used_bytes": self.used_bytes, + "remaining_bytes": self.remaining_bytes, + "per_environment_distortion": { + environment: value + for environment, value in zip( + self.environments, + self.per_environment_distortion, + strict=True, + ) + }, + "worst_environment_distortion": self.worst_environment_distortion, + "unit_count": len(self.decisions), + "transitions_evaluated": self.transitions_evaluated, + "peak_frontier_states": self.peak_frontier_states, + "decisions": [decision.to_dict(self.environments) for decision in self.decisions], + } + + def to_json(self) -> str: + """Return a canonical compact JSON representation of this plan.""" + + return json.dumps( + self.to_dict(), + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + + +@dataclass(frozen=True, slots=True) +class _SearchState: + used_bytes: int + aggregate_distortion: float + candidates: tuple[PolicyCandidate, ...] + + @property + def candidate_ids(self) -> tuple[str, ...]: + return tuple(candidate.candidate_id for candidate in self.candidates) + + +@dataclass(frozen=True, slots=True) +class _RobustSearchState: + used_bytes: int + per_environment_distortion: tuple[float, ...] + candidates: tuple[RobustPolicyCandidate, ...] + + @property + def candidate_ids(self) -> tuple[str, ...]: + return tuple(candidate.candidate_id for candidate in self.candidates) + + +def _validate_units(units: Iterable[QuantizableUnit]) -> tuple[QuantizableUnit, ...]: + if isinstance(units, (str, bytes)): + raise TypeError("units must be an iterable of QuantizableUnit objects") + try: + normalized_units = tuple(units) + except TypeError as exc: + raise TypeError("units must be an iterable of QuantizableUnit objects") from exc + if not normalized_units: + raise ValueError("units must not be empty") + if any(not isinstance(unit, QuantizableUnit) for unit in normalized_units): + raise TypeError("units must contain only QuantizableUnit objects") + + unit_ids = [unit.unit_id for unit in normalized_units] + if len(set(unit_ids)) != len(unit_ids): + raise ValueError("unit_id values must be unique") + return tuple(sorted(normalized_units, key=lambda unit: unit.unit_id)) + + +def _validate_robust_units( + units: Iterable[RobustQuantizableUnit], + *, + environment_count: int, +) -> tuple[RobustQuantizableUnit, ...]: + if isinstance(units, (str, bytes)): + raise TypeError("units must be an iterable of RobustQuantizableUnit objects") + try: + normalized_units = tuple(units) + except TypeError as exc: + raise TypeError("units must be an iterable of RobustQuantizableUnit objects") from exc + if not normalized_units: + raise ValueError("units must not be empty") + if any(not isinstance(unit, RobustQuantizableUnit) for unit in normalized_units): + raise TypeError("units must contain only RobustQuantizableUnit objects") + unit_ids = [unit.unit_id for unit in normalized_units] + if len(set(unit_ids)) != len(unit_ids): + raise ValueError("unit_id values must be unique") + for unit in normalized_units: + if len(unit.candidates[0].per_environment_distortion) != environment_count: + raise ValueError( + f"candidate environment count disagrees with environments for {unit.unit_id!r}" + ) + return tuple(sorted(normalized_units, key=lambda unit: unit.unit_id)) + + +def _validate_max_frontier_states(max_frontier_states: int) -> int: + if type(max_frontier_states) is not int: + raise TypeError("max_frontier_states must be an integer") + if max_frontier_states <= 0: + raise ValueError("max_frontier_states must be positive") + return max_frontier_states + + +def _validate_max_transitions(max_transitions: int) -> int: + if type(max_transitions) is not int: + raise TypeError("max_transitions must be an integer") + if max_transitions <= 0: + raise ValueError("max_transitions must be positive") + return max_transitions + + +def _validate_max_pareto_comparisons(max_pareto_comparisons: int) -> int: + if type(max_pareto_comparisons) is not int: + raise TypeError("max_pareto_comparisons must be an integer") + if max_pareto_comparisons <= 0: + raise ValueError("max_pareto_comparisons must be positive") + return max_pareto_comparisons + + +def _checked_fsum(values: Iterable[float], *, label: str) -> float: + try: + total = math.fsum(values) + except OverflowError as exc: + raise ArithmeticError(f"{label} overflowed") from exc + if not math.isfinite(total): + raise ArithmeticError(f"{label} overflowed") + return total + + +def _suffix_minimum_bytes( + units: tuple[QuantizableUnit, ...] | tuple[RobustQuantizableUnit, ...], +) -> tuple[int, ...]: + """Return the minimum bytes required from each unit index onward.""" + + suffix = [0] * (len(units) + 1) + for index in range(len(units) - 1, -1, -1): + suffix[index] = suffix[index + 1] + min( + candidate.byte_size for candidate in units[index].candidates + ) + return tuple(suffix) + + +def _prefer_same_size(candidate: _SearchState, incumbent: _SearchState) -> bool: + if candidate.aggregate_distortion != incumbent.aggregate_distortion: + return candidate.aggregate_distortion < incumbent.aggregate_distortion + return candidate.candidate_ids < incumbent.candidate_ids + + +def _pareto_prune(states_by_bytes: dict[int, _SearchState]) -> dict[int, _SearchState]: + """Drop states dominated by one using no more bytes and no more distortion.""" + + frontier: dict[int, _SearchState] = {} + best_distortion = float("inf") + for used_bytes in sorted(states_by_bytes): + state = states_by_bytes[used_bytes] + if state.aggregate_distortion < best_distortion: + frontier[used_bytes] = state + best_distortion = state.aggregate_distortion + return frontier + + +def solve_autopolicy( + units: Iterable[QuantizableUnit], + *, + budget_bytes: int, + max_frontier_states: int = 250_000, + max_transitions: int = DEFAULT_MAX_TRANSITIONS, +) -> AutoPolicyPlan: + """Return the exact minimum-distortion plan under ``budget_bytes``. + + Units and candidates are canonicalized by identifier. Objective ties prefer + fewer bytes, then the lexicographically smaller candidate-ID tuple in + sorted unit-ID order. The Pareto-frontier dynamic program is exact for the + additive proxy while avoiding an array whose size scales with the raw byte + budget. + """ + + if type(budget_bytes) is not int: + raise TypeError("budget_bytes must be an integer, not a boolean or float") + if budget_bytes < 0: + raise ValueError("budget_bytes must be non-negative") + _validate_max_frontier_states(max_frontier_states) + _validate_max_transitions(max_transitions) + normalized_units = _validate_units(units) + + suffix_minimum_bytes = _suffix_minimum_bytes(normalized_units) + minimum_required_bytes = suffix_minimum_bytes[0] + if minimum_required_bytes > budget_bytes: + raise InfeasiblePolicyError( + f"budget_bytes={budget_bytes} is infeasible; " + f"at least {minimum_required_bytes} bytes are required" + ) + + frontier = { + 0: _SearchState( + used_bytes=0, + aggregate_distortion=0.0, + candidates=(), + ) + } + transitions_evaluated = 0 + peak_frontier_states = 1 + + for unit_index, unit in enumerate(normalized_units): + next_states_by_bytes: dict[int, _SearchState] = {} + for state in (frontier[key] for key in sorted(frontier)): + for candidate in unit.candidates: + used_bytes = state.used_bytes + candidate.byte_size + if used_bytes + suffix_minimum_bytes[unit_index + 1] > budget_bytes: + continue + if transitions_evaluated >= max_transitions: + raise FrontierLimitError( + "exact AutoPolicy search exceeded " + f"max_transitions={max_transitions}; no approximate plan was returned" + ) + aggregate_distortion = _checked_fsum( + (state.aggregate_distortion, candidate.distortion), + label="aggregate distortion", + ) + transitions_evaluated += 1 + next_state = _SearchState( + used_bytes=used_bytes, + aggregate_distortion=aggregate_distortion, + candidates=(*state.candidates, candidate), + ) + incumbent = next_states_by_bytes.get(used_bytes) + if incumbent is None or _prefer_same_size(next_state, incumbent): + next_states_by_bytes[used_bytes] = next_state + + if not next_states_by_bytes: # pragma: no cover - guarded by the minimum-size check + raise InfeasiblePolicyError("no complete policy fits the byte budget") + frontier = _pareto_prune(next_states_by_bytes) + if len(frontier) > max_frontier_states: + raise FrontierLimitError( + "exact AutoPolicy frontier exceeded max_frontier_states=" + f"{max_frontier_states}; no approximate plan was returned" + ) + peak_frontier_states = max(peak_frontier_states, len(frontier)) + + winner = min( + frontier.values(), + key=lambda state: ( + state.aggregate_distortion, + state.used_bytes, + state.candidate_ids, + ), + ) + decisions = tuple( + PolicyDecision( + unit_id=unit.unit_id, + candidate_id=candidate.candidate_id, + byte_size=candidate.byte_size, + distortion=candidate.distortion, + ) + for unit, candidate in zip(normalized_units, winner.candidates, strict=True) + ) + return AutoPolicyPlan( + budget_bytes=budget_bytes, + used_bytes=winner.used_bytes, + remaining_bytes=budget_bytes - winner.used_bytes, + aggregate_distortion=winner.aggregate_distortion, + decisions=decisions, + transitions_evaluated=transitions_evaluated, + peak_frontier_states=peak_frontier_states, + ) + + +def _robust_state_dominates( + left: _RobustSearchState, + right: _RobustSearchState, +) -> bool: + if left.used_bytes > right.used_bytes: + return False + no_worse = all( + left_value <= right_value + for left_value, right_value in zip( + left.per_environment_distortion, + right.per_environment_distortion, + strict=True, + ) + ) + if not no_worse: + return False + strictly_better = left.used_bytes < right.used_bytes or any( + left_value < right_value + for left_value, right_value in zip( + left.per_environment_distortion, + right.per_environment_distortion, + strict=True, + ) + ) + if strictly_better: + return True + return left.candidate_ids <= right.candidate_ids + + +def _robust_pareto_prune( + states: Iterable[_RobustSearchState], +) -> list[_RobustSearchState]: + """Return the exact componentwise byte/environment Pareto frontier.""" + + best_for_point: dict[tuple[int, tuple[float, ...]], _RobustSearchState] = {} + for state in states: + key = (state.used_bytes, state.per_environment_distortion) + incumbent = best_for_point.get(key) + if incumbent is None or state.candidate_ids < incumbent.candidate_ids: + best_for_point[key] = state + ordered = sorted( + best_for_point.values(), + key=lambda state: ( + state.used_bytes, + state.per_environment_distortion, + state.candidate_ids, + ), + ) + if len(ordered) <= 1: + return ordered + + byte_sizes = np.asarray([state.used_bytes for state in ordered], dtype=np.int64) + costs = np.asarray( + [state.per_environment_distortion for state in ordered], + dtype=np.float64, + ) + dominated = np.zeros(len(ordered), dtype=np.bool_) + chunk_size = 128 + for start in range(0, len(ordered), chunk_size): + stop = min(start + chunk_size, len(ordered)) + right_bytes = byte_sizes[start:stop] + right_costs = costs[start:stop] + no_worse = (byte_sizes[:, None] <= right_bytes[None, :]) & np.all( + costs[:, None, :] <= right_costs[None, :, :], + axis=2, + ) + strictly_better = (byte_sizes[:, None] < right_bytes[None, :]) | np.any( + costs[:, None, :] < right_costs[None, :, :], + axis=2, + ) + dominated[start:stop] = np.any(no_worse & strictly_better, axis=0) + return [ + state + for state, is_dominated in zip( + ordered, + dominated.tolist(), + strict=True, + ) + if not is_dominated + ] + + +def solve_robust_autopolicy( + units: Iterable[RobustQuantizableUnit], + *, + environments: Iterable[str], + budget_bytes: int, + max_frontier_states: int = 250_000, + max_transitions: int = DEFAULT_MAX_TRANSITIONS, + max_pareto_comparisons: int = DEFAULT_MAX_PARETO_COMPARISONS, +) -> RobustAutoPolicyPlan: + """Exactly minimize the worst summed environment distortion under a budget. + + This solves the supplied discrete proxy only. It assumes per-unit + distortions are additive and does not claim to model cross-unit interactions + or downstream language-model loss. + """ + + if type(budget_bytes) is not int: + raise TypeError("budget_bytes must be an integer, not a boolean or float") + if budget_bytes < 0: + raise ValueError("budget_bytes must be non-negative") + _validate_max_frontier_states(max_frontier_states) + _validate_max_transitions(max_transitions) + _validate_max_pareto_comparisons(max_pareto_comparisons) + if isinstance(environments, (str, bytes)): + raise TypeError("environments must be an iterable of strings") + try: + normalized_environments = tuple(environments) + except TypeError as exc: + raise TypeError("environments must be an iterable of strings") from exc + if not normalized_environments: + raise ValueError("environments must not be empty") + for environment in normalized_environments: + _validate_identifier(environment, name="environment") + if len(set(normalized_environments)) != len(normalized_environments): + raise ValueError("environments must be unique") + + normalized_units = _validate_robust_units( + units, + environment_count=len(normalized_environments), + ) + suffix_minimum_bytes = _suffix_minimum_bytes(normalized_units) + minimum_required_bytes = suffix_minimum_bytes[0] + if minimum_required_bytes > budget_bytes: + raise InfeasiblePolicyError( + f"budget_bytes={budget_bytes} is infeasible; " + f"at least {minimum_required_bytes} bytes are required" + ) + + frontier = [ + _RobustSearchState( + used_bytes=0, + per_environment_distortion=(0.0,) * len(normalized_environments), + candidates=(), + ) + ] + transitions_evaluated = 0 + peak_frontier_states = 1 + for unit_index, unit in enumerate(normalized_units): + next_states: list[_RobustSearchState] = [] + for state in frontier: + for candidate in unit.candidates: + used_bytes = state.used_bytes + candidate.byte_size + if used_bytes + suffix_minimum_bytes[unit_index + 1] > budget_bytes: + continue + if transitions_evaluated >= max_transitions: + raise FrontierLimitError( + "exact robust AutoPolicy search exceeded " + f"max_transitions={max_transitions}; no approximate plan was returned" + ) + distortions = tuple( + _checked_fsum( + (left, right), + label="aggregate environment distortion", + ) + for left, right in zip( + state.per_environment_distortion, + candidate.per_environment_distortion, + strict=True, + ) + ) + transitions_evaluated += 1 + next_states.append( + _RobustSearchState( + used_bytes=used_bytes, + per_environment_distortion=distortions, + candidates=(*state.candidates, candidate), + ) + ) + if not next_states: # pragma: no cover - guarded by the minimum-size check + raise InfeasiblePolicyError("no complete policy fits the byte budget") + comparison_count = len(next_states) ** 2 + if comparison_count > max_pareto_comparisons: + raise FrontierLimitError( + "exact robust AutoPolicy Pareto prune requires " + f"{comparison_count} pairwise comparisons, exceeding " + f"max_pareto_comparisons={max_pareto_comparisons}; " + "no approximate plan was returned" + ) + frontier = _robust_pareto_prune(next_states) + if len(frontier) > max_frontier_states: + raise FrontierLimitError( + "exact robust AutoPolicy frontier exceeded max_frontier_states=" + f"{max_frontier_states}; no approximate plan was returned" + ) + peak_frontier_states = max(peak_frontier_states, len(frontier)) + + winner = min( + frontier, + key=lambda state: ( + max(state.per_environment_distortion), + _checked_fsum( + state.per_environment_distortion, + label="sum environment distortion", + ), + state.used_bytes, + state.candidate_ids, + ), + ) + decisions = tuple( + RobustPolicyDecision( + unit_id=unit.unit_id, + candidate_id=candidate.candidate_id, + byte_size=candidate.byte_size, + per_environment_distortion=candidate.per_environment_distortion, + ) + for unit, candidate in zip(normalized_units, winner.candidates, strict=True) + ) + return RobustAutoPolicyPlan( + environments=normalized_environments, + budget_bytes=budget_bytes, + used_bytes=winner.used_bytes, + remaining_bytes=budget_bytes - winner.used_bytes, + per_environment_distortion=winner.per_environment_distortion, + worst_environment_distortion=max(winner.per_environment_distortion), + decisions=decisions, + transitions_evaluated=transitions_evaluated, + peak_frontier_states=peak_frontier_states, + ) diff --git a/src/cliffquant/autopolicy_artifacts.py b/src/cliffquant/autopolicy_artifacts.py new file mode 100644 index 0000000..05d35b6 --- /dev/null +++ b/src/cliffquant/autopolicy_artifacts.py @@ -0,0 +1,702 @@ +"""Strict, content-bound artifacts for CliffQuant AutoPolicy planning.""" + +from __future__ import annotations + +import json +import math +import os +import re +from collections import defaultdict +from collections.abc import Mapping +from copy import deepcopy +from pathlib import Path +from typing import Any + +from .autopolicy import ( + DEFAULT_MAX_PARETO_COMPARISONS, + DEFAULT_MAX_TRANSITIONS, + AutoPolicyPlan, + PolicyCandidate, + QuantizableUnit, + RobustAutoPolicyPlan, + RobustPolicyCandidate, + RobustQuantizableUnit, + solve_autopolicy, + solve_robust_autopolicy, +) +from .provenance import canonical_json_bytes, sha256_bytes, sha256_file + +CANDIDATE_SCHEMA = "cliffquant.autopolicy-candidates.v1" +PLAN_SCHEMA = "cliffquant.autopolicy-plan.v1" +ROBUST_OBJECTIVE = "robust-minimax" +SCALAR_OBJECTIVE = "sum-groupwise-max" +OBJECTIVES = (ROBUST_OBJECTIVE, SCALAR_OBJECTIVE) +TRANSFORMER_BLOCK_PROFILE = "experiment-001-transformer-blocks" +_SHA256 = re.compile(r"[0-9a-f]{64}") +_TRANSFORMER_BLOCK = re.compile(r"^(model\.language_model\.layers\.\d+)\.") + + +def _load_json_object(path: str | Path) -> dict[str, Any]: + source = Path(path) + + def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ValueError(f"duplicate JSON key in {source}: {key}") + value[key] = item + return value + + try: + payload = json.loads( + source.read_text(encoding="utf-8"), + object_pairs_hook=reject_duplicates, + parse_constant=lambda token: (_ for _ in ()).throw( + ValueError(f"non-finite JSON number in {source}: {token}") + ), + ) + except FileNotFoundError: + raise + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError(f"invalid JSON artifact: {source}") from exc + if not isinstance(payload, dict): + raise ValueError(f"JSON artifact must contain an object: {source}") + return payload + + +def _without_fingerprint(payload: Mapping[str, Any]) -> dict[str, Any]: + return {key: value for key, value in payload.items() if key != "fingerprint_sha256"} + + +def payload_fingerprint(payload: Mapping[str, Any]) -> str: + """Hash a payload without its self-describing fingerprint field.""" + + return sha256_bytes(canonical_json_bytes(_without_fingerprint(payload))) + + +def bind_fingerprint(payload: Mapping[str, Any]) -> dict[str, Any]: + """Return a copy carrying a deterministic top-level fingerprint.""" + + result = _without_fingerprint(payload) + result["fingerprint_sha256"] = payload_fingerprint(result) + return result + + +def _atomic_write_new(path: Path, payload: bytes) -> None: + if path.exists(): + raise FileExistsError(f"refusing to overwrite existing artifact: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + try: + with temporary.open("xb") as handle: + handle.write(payload) + os.replace(temporary, path) + finally: + if temporary.exists(): + temporary.unlink() + + +def write_candidate_bundle(path: str | Path, payload: Mapping[str, Any]) -> dict[str, Any]: + """Validate, fingerprint, and atomically create a candidate bundle.""" + + bound = bind_fingerprint(payload) + validate_candidate_bundle(bound) + _atomic_write_new(Path(path), canonical_json_bytes(bound)) + return bound + + +def _require_exact_keys(value: Mapping[str, Any], expected: set[str], *, label: str) -> None: + actual = set(value) + if actual != expected: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + raise ValueError(f"{label} keys mismatch; missing={missing}, extra={extra}") + + +def _require_nonempty_string(value: object, *, label: str) -> str: + if type(value) is not str or not value or value != value.strip(): + raise ValueError(f"{label} must be a non-empty trimmed string") + return value + + +def _validate_candidate_entry( + entry: Mapping[str, Any], + *, + environments: tuple[str, ...], + unit_id: str, + definition: Mapping[str, Any], + shape: tuple[int, int], + group_count: int, +) -> None: + _require_exact_keys( + entry, + { + "candidate_id", + "logical_bytes", + "groupwise_max_distortion", + "per_environment_distortion", + "metadata", + }, + label=f"{unit_id} candidate", + ) + candidate_id = _require_nonempty_string( + entry.get("candidate_id"), + label=f"{unit_id} candidate_id", + ) + PolicyCandidate( + candidate_id, + entry.get("logical_bytes"), # type: ignore[arg-type] + entry.get("groupwise_max_distortion"), # type: ignore[arg-type] + ) + raw_per_environment = entry.get("per_environment_distortion") + if not isinstance(raw_per_environment, dict): + raise TypeError(f"{unit_id}/{candidate_id} per_environment_distortion must be an object") + if set(raw_per_environment) != set(environments): + raise ValueError(f"{unit_id}/{candidate_id} environments must exactly match the bundle") + RobustPolicyCandidate( + candidate_id, + entry.get("logical_bytes"), # type: ignore[arg-type] + tuple(raw_per_environment[environment] for environment in environments), + ) + metadata = entry.get("metadata") + if not isinstance(metadata, dict): + raise TypeError(f"{unit_id}/{candidate_id} metadata must be an object") + for key in ("bits", "group_size", "scale_selection"): + if key in definition and metadata.get(key) != definition[key]: + raise ValueError(f"{unit_id}/{candidate_id} metadata disagrees on {key}") + + has_code_bytes = "logical_code_bytes" in metadata + has_scale_bytes = "logical_scale_bytes" in metadata + if has_code_bytes != has_scale_bytes: + raise ValueError(f"{unit_id}/{candidate_id} must record both logical byte components") + if has_code_bytes: + code_bytes = metadata["logical_code_bytes"] + scale_bytes = metadata["logical_scale_bytes"] + if ( + type(code_bytes) is not int + or code_bytes < 0 + or type(scale_bytes) is not int + or scale_bytes < 0 + ): + raise ValueError(f"{unit_id}/{candidate_id} logical byte components are invalid") + if code_bytes + scale_bytes != entry["logical_bytes"]: + raise ValueError(f"{unit_id}/{candidate_id} logical byte components do not sum") + bits = metadata.get("bits") + if type(bits) is not int or bits not in {4, 8, 16}: + raise ValueError(f"{unit_id}/{candidate_id} bits metadata is invalid") + expected_code_bytes = (shape[0] * shape[1] * bits + 7) // 8 + if code_bytes != expected_code_bytes: + raise ValueError(f"{unit_id}/{candidate_id} logical code bytes mismatch") + expected_scale_bytes = 0 if bits == 16 else group_count * 2 + if scale_bytes != expected_scale_bytes: + raise ValueError(f"{unit_id}/{candidate_id} logical scale bytes mismatch") + + +def validate_candidate_bundle(payload: Mapping[str, Any]) -> None: + """Fail closed unless a candidate bundle satisfies the v1 contract.""" + + _require_exact_keys( + payload, + { + "schema", + "status", + "profile", + "scope", + "source", + "environments", + "distortion_metric", + "candidate_definitions", + "units", + "totals", + "builder", + "fingerprint_sha256", + }, + label="candidate bundle", + ) + if payload.get("schema") != CANDIDATE_SCHEMA or payload.get("status") != "complete": + raise ValueError("unsupported or incomplete AutoPolicy candidate bundle") + _require_nonempty_string(payload.get("profile"), label="profile") + supplied_fingerprint = payload.get("fingerprint_sha256") + if not isinstance(supplied_fingerprint, str) or not _SHA256.fullmatch(supplied_fingerprint): + raise ValueError("candidate bundle fingerprint is invalid") + if supplied_fingerprint != payload_fingerprint(payload): + raise ValueError("candidate bundle fingerprint mismatch") + + raw_environments = payload.get("environments") + if not isinstance(raw_environments, list) or not raw_environments: + raise ValueError("environments must be a non-empty list") + environments = tuple( + _require_nonempty_string(value, label="environment") for value in raw_environments + ) + if len(set(environments)) != len(environments): + raise ValueError("environments must be unique") + for label in ("scope", "source", "distortion_metric", "candidate_definitions", "builder"): + if not isinstance(payload.get(label), dict): + raise TypeError(f"{label} must be an object") + scope = payload["scope"] + _require_nonempty_string(scope.get("budget_scope"), label="scope budget_scope") + if type(scope.get("checkpoint_export")) is not bool: + raise TypeError("scope checkpoint_export must be a boolean") + if scope["checkpoint_export"] is not False: + raise ValueError("v1 candidate bundles cannot claim checkpoint export") + distortion_metric = payload["distortion_metric"] + if distortion_metric.get("additive_across_units") is not True: + raise ValueError("distortion_metric additive_across_units must be true") + candidate_definitions = payload["candidate_definitions"] + if not candidate_definitions: + raise ValueError("candidate_definitions must not be empty") + expected_candidate_ids = set(candidate_definitions) + for candidate_id, definition in candidate_definitions.items(): + _require_nonempty_string(candidate_id, label="candidate definition id") + if not isinstance(definition, dict): + raise TypeError(f"candidate definition {candidate_id!r} must be an object") + + raw_units = payload.get("units") + if not isinstance(raw_units, list) or not raw_units: + raise ValueError("units must be a non-empty list") + unit_ids: list[str] = [] + for raw_unit in raw_units: + if not isinstance(raw_unit, dict): + raise TypeError("unit entries must be objects") + _require_exact_keys( + raw_unit, + { + "unit_id", + "weight_key", + "shape", + "group_count", + "candidates", + "evidence", + }, + label="unit", + ) + unit_id = _require_nonempty_string(raw_unit.get("unit_id"), label="unit_id") + unit_ids.append(unit_id) + _require_nonempty_string(raw_unit.get("weight_key"), label=f"{unit_id} weight_key") + shape = raw_unit.get("shape") + if ( + not isinstance(shape, list) + or len(shape) != 2 + or any(type(value) is not int or value <= 0 for value in shape) + ): + raise ValueError(f"{unit_id} shape must contain two positive integers") + if type(raw_unit.get("group_count")) is not int or raw_unit["group_count"] <= 0: + raise ValueError(f"{unit_id} group_count must be positive") + candidates = raw_unit.get("candidates") + if not isinstance(candidates, list) or not candidates: + raise ValueError(f"{unit_id} candidates must be a non-empty list") + candidate_ids = [ + candidate.get("candidate_id") for candidate in candidates if isinstance(candidate, dict) + ] + if ( + len(candidate_ids) != len(candidates) + or len(candidate_ids) != len(expected_candidate_ids) + or len(set(candidate_ids)) != len(candidate_ids) + or set(candidate_ids) != expected_candidate_ids + ): + raise ValueError(f"{unit_id} candidate inventory disagrees with definitions") + for candidate in candidates: + if not isinstance(candidate, dict): + raise TypeError(f"{unit_id} candidates must be objects") + _validate_candidate_entry( + candidate, + environments=environments, + unit_id=unit_id, + definition=candidate_definitions[candidate["candidate_id"]], + shape=(shape[0], shape[1]), + group_count=raw_unit["group_count"], + ) + if not isinstance(raw_unit.get("evidence"), dict): + raise TypeError(f"{unit_id} evidence must be an object") + if unit_ids != sorted(unit_ids) or len(set(unit_ids)) != len(unit_ids): + raise ValueError("units must have unique identifiers in sorted order") + totals = payload.get("totals") + if not isinstance(totals, dict): + raise TypeError("totals must be an object") + if set(totals) != expected_candidate_ids: + raise ValueError("totals candidate inventory disagrees with definitions") + for candidate_id in sorted(expected_candidate_ids): + if type(totals[candidate_id]) is not int or totals[candidate_id] <= 0: + raise ValueError(f"totals value for {candidate_id} must be a positive integer") + expected_total = sum( + next( + candidate["logical_bytes"] + for candidate in unit["candidates"] + if candidate["candidate_id"] == candidate_id + ) + for unit in raw_units + ) + if totals[candidate_id] != expected_total: + raise ValueError(f"totals mismatch for {candidate_id}") + + +def load_candidate_bundle(path: str | Path) -> dict[str, Any]: + """Load and validate a canonical candidate bundle.""" + + source = Path(path) + payload = _load_json_object(source) + validate_candidate_bundle(payload) + if source.read_bytes() != canonical_json_bytes(payload): + raise ValueError("candidate bundle is not in canonical JSON form") + return payload + + +def aggregate_experiment001_transformer_blocks( + candidate_path: str | Path, + payload: Mapping[str, Any], +) -> dict[str, Any]: + """Aggregate measured matrix candidates into transformer-block choices.""" + + validate_candidate_bundle(payload) + if payload["profile"] != "experiment-001": + raise ValueError("transformer-block aggregation requires the experiment-001 profile") + + grouped: dict[str, list[Mapping[str, Any]]] = defaultdict(list) + for unit in payload["units"]: + match = _TRANSFORMER_BLOCK.match(unit["unit_id"]) + if match is None: + raise ValueError( + f"unit is not inside a recognized transformer block: {unit['unit_id']}" + ) + grouped[match.group(1)].append(unit) + if not grouped: # pragma: no cover - guarded by candidate validation + raise ValueError("candidate bundle contains no transformer blocks") + + environments = tuple(payload["environments"]) + candidate_ids = tuple(sorted(payload["candidate_definitions"])) + units: list[dict[str, Any]] = [] + for block_id in sorted(grouped): + members = sorted(grouped[block_id], key=lambda unit: unit["unit_id"]) + candidates = [] + for candidate_id in candidate_ids: + selected = [ + next( + candidate + for candidate in member["candidates"] + if candidate["candidate_id"] == candidate_id + ) + for member in members + ] + metadata_keys = set(selected[0]["metadata"]) + if any(set(candidate["metadata"]) != metadata_keys for candidate in selected[1:]): + raise ValueError(f"candidate metadata keys disagree within {block_id}") + metadata: dict[str, Any] = {} + for key in sorted(metadata_keys): + values = [candidate["metadata"][key] for candidate in selected] + if key in {"logical_code_bytes", "logical_scale_bytes"}: + if any(type(value) is not int or value < 0 for value in values): + raise ValueError(f"candidate byte metadata is invalid within {block_id}") + metadata[key] = sum(values) + elif all(value == values[0] for value in values[1:]): + metadata[key] = deepcopy(values[0]) + else: + raise ValueError( + f"candidate metadata is inconsistent within {block_id}: " + f"{candidate_id}.{key}" + ) + candidates.append( + { + "candidate_id": candidate_id, + "logical_bytes": sum(candidate["logical_bytes"] for candidate in selected), + "groupwise_max_distortion": math.fsum( + candidate["groupwise_max_distortion"] for candidate in selected + ), + "per_environment_distortion": { + environment: math.fsum( + candidate["per_environment_distortion"][environment] + for candidate in selected + ) + for environment in environments + }, + "metadata": metadata, + } + ) + weight_count = sum(member["shape"][0] * member["shape"][1] for member in members) + units.append( + { + "unit_id": block_id, + "weight_key": f"aggregate:{block_id}", + "shape": [1, weight_count], + "group_count": sum(member["group_count"] for member in members), + "candidates": candidates, + "evidence": { + "aggregation": "transformer-block", + "member_unit_ids": [member["unit_id"] for member in members], + }, + } + ) + + scope = deepcopy(payload["scope"]) + scope["module_count"] = len(payload["units"]) + scope["allocation_unit_count"] = len(units) + scope["allocation_granularity"] = "transformer-block" + aggregated = { + "schema": CANDIDATE_SCHEMA, + "status": "complete", + "profile": TRANSFORMER_BLOCK_PROFILE, + "scope": scope, + "source": { + "parent_candidate_bundle": { + "file_sha256": sha256_file(candidate_path), + "fingerprint_sha256": payload["fingerprint_sha256"], + }, + "measurement_source": deepcopy(payload["source"]), + }, + "environments": list(environments), + "distortion_metric": deepcopy(payload["distortion_metric"]), + "candidate_definitions": deepcopy(payload["candidate_definitions"]), + "units": units, + "totals": deepcopy(payload["totals"]), + "builder": { + "name": "cliffquant-autopolicy-transformer-blocks-v1", + "parent_profile": payload["profile"], + }, + } + bound = bind_fingerprint(aggregated) + validate_candidate_bundle(bound) + return bound + + +def _candidate_lookup(payload: Mapping[str, Any]) -> dict[tuple[str, str], Mapping[str, Any]]: + return { + (unit["unit_id"], candidate["candidate_id"]): candidate + for unit in payload["units"] + for candidate in unit["candidates"] + } + + +def scalar_units_from_bundle(payload: Mapping[str, Any]) -> tuple[QuantizableUnit, ...]: + """Convert a validated bundle to scalar additive solver contracts.""" + + validate_candidate_bundle(payload) + return tuple( + QuantizableUnit( + unit["unit_id"], + tuple( + PolicyCandidate( + candidate["candidate_id"], + candidate["logical_bytes"], + candidate["groupwise_max_distortion"], + ) + for candidate in unit["candidates"] + ), + ) + for unit in payload["units"] + ) + + +def robust_units_from_bundle(payload: Mapping[str, Any]) -> tuple[RobustQuantizableUnit, ...]: + """Convert a validated bundle to separately scored environment contracts.""" + + validate_candidate_bundle(payload) + environments = tuple(payload["environments"]) + return tuple( + RobustQuantizableUnit( + unit["unit_id"], + tuple( + RobustPolicyCandidate( + candidate["candidate_id"], + candidate["logical_bytes"], + tuple( + candidate["per_environment_distortion"][environment] + for environment in environments + ), + ) + for candidate in unit["candidates"] + ), + ) + for unit in payload["units"] + ) + + +def solve_candidate_bundle( + payload: Mapping[str, Any], + *, + budget_bytes: int, + objective: str = ROBUST_OBJECTIVE, + max_frontier_states: int = 250_000, + max_transitions: int = DEFAULT_MAX_TRANSITIONS, + max_pareto_comparisons: int = DEFAULT_MAX_PARETO_COMPARISONS, +) -> AutoPolicyPlan | RobustAutoPolicyPlan: + """Solve a validated bundle with one of the explicit v1 objectives.""" + + validate_candidate_bundle(payload) + if objective == ROBUST_OBJECTIVE: + return solve_robust_autopolicy( + robust_units_from_bundle(payload), + environments=tuple(payload["environments"]), + budget_bytes=budget_bytes, + max_frontier_states=max_frontier_states, + max_transitions=max_transitions, + max_pareto_comparisons=max_pareto_comparisons, + ) + if objective == SCALAR_OBJECTIVE: + return solve_autopolicy( + scalar_units_from_bundle(payload), + budget_bytes=budget_bytes, + max_frontier_states=max_frontier_states, + max_transitions=max_transitions, + ) + raise ValueError(f"objective must be one of {OBJECTIVES}") + + +def build_plan_payload( + candidate_path: str | Path, + candidate_bundle: Mapping[str, Any], + *, + objective: str, + optimization: AutoPolicyPlan | RobustAutoPolicyPlan, +) -> dict[str, Any]: + """Bind a solver result to the exact candidate evidence it used.""" + + validate_candidate_bundle(candidate_bundle) + lookup = _candidate_lookup(candidate_bundle) + enriched_decisions = [] + for decision in optimization.decisions: + candidate = lookup[(decision.unit_id, decision.candidate_id)] + enriched_decisions.append( + { + "unit_id": decision.unit_id, + "candidate_id": decision.candidate_id, + "logical_bytes": decision.byte_size, + "metadata": candidate["metadata"], + "evidence": next( + unit["evidence"] + for unit in candidate_bundle["units"] + if unit["unit_id"] == decision.unit_id + ), + } + ) + payload = { + "schema": PLAN_SCHEMA, + "status": "complete", + "candidate_bundle": { + "file_sha256": sha256_file(candidate_path), + "fingerprint_sha256": candidate_bundle["fingerprint_sha256"], + }, + "profile": candidate_bundle["profile"], + "budget_scope": candidate_bundle["scope"]["budget_scope"], + "objective": objective, + "optimization": optimization.to_dict(), + "selected_candidates": enriched_decisions, + "claim_boundary": { + "exact": ("selection for the recorded candidate distortions and logical byte sizes"), + "not_claimed": [ + "downstream-accuracy optimum", + "checkpoint size", + "runtime memory", + "latency improvement", + "checkpoint exportability", + ], + }, + } + return bind_fingerprint(payload) + + +def write_plan(path: str | Path, payload: Mapping[str, Any]) -> dict[str, Any]: + """Validate and atomically create a plan artifact.""" + + validate_plan(payload) + _atomic_write_new(Path(path), canonical_json_bytes(payload)) + return dict(payload) + + +def validate_plan(payload: Mapping[str, Any]) -> None: + """Validate the structural and self-hash portion of a plan.""" + + _require_exact_keys( + payload, + { + "schema", + "status", + "candidate_bundle", + "profile", + "budget_scope", + "objective", + "optimization", + "selected_candidates", + "claim_boundary", + "fingerprint_sha256", + }, + label="AutoPolicy plan", + ) + if payload.get("schema") != PLAN_SCHEMA or payload.get("status") != "complete": + raise ValueError("unsupported or incomplete AutoPolicy plan") + if payload.get("objective") not in OBJECTIVES: + raise ValueError(f"plan objective must be one of {OBJECTIVES}") + if payload.get("fingerprint_sha256") != payload_fingerprint(payload): + raise ValueError("AutoPolicy plan fingerprint mismatch") + descriptor = payload.get("candidate_bundle") + if not isinstance(descriptor, dict): + raise TypeError("candidate_bundle must be an object") + _require_exact_keys( + descriptor, + {"file_sha256", "fingerprint_sha256"}, + label="candidate_bundle descriptor", + ) + if any( + not isinstance(value, str) or not _SHA256.fullmatch(value) for value in descriptor.values() + ): + raise ValueError("candidate bundle descriptor contains an invalid hash") + if not isinstance(payload.get("optimization"), dict): + raise TypeError("optimization must be an object") + if not isinstance(payload.get("selected_candidates"), list): + raise TypeError("selected_candidates must be a list") + if not isinstance(payload.get("claim_boundary"), dict): + raise TypeError("claim_boundary must be an object") + + +def load_plan(path: str | Path) -> dict[str, Any]: + """Load and validate a canonical plan artifact.""" + + source = Path(path) + payload = _load_json_object(source) + validate_plan(payload) + if source.read_bytes() != canonical_json_bytes(payload): + raise ValueError("AutoPolicy plan is not in canonical JSON form") + return payload + + +def verify_plan( + candidate_path: str | Path, + plan_path: str | Path, + *, + max_frontier_states: int = 250_000, + max_transitions: int = DEFAULT_MAX_TRANSITIONS, + max_pareto_comparisons: int = DEFAULT_MAX_PARETO_COMPARISONS, +) -> dict[str, Any]: + """Recompute a plan from its bound candidate bundle and compare exactly.""" + + candidate_bundle = load_candidate_bundle(candidate_path) + plan = load_plan(plan_path) + descriptor = plan["candidate_bundle"] + if descriptor["file_sha256"] != sha256_file(candidate_path): + raise ValueError("plan candidate file hash mismatch") + if descriptor["fingerprint_sha256"] != candidate_bundle["fingerprint_sha256"]: + raise ValueError("plan candidate fingerprint mismatch") + optimization = plan["optimization"] + budget = optimization.get("budget_bytes") + recomputed = solve_candidate_bundle( + candidate_bundle, + budget_bytes=budget, + objective=plan["objective"], + max_frontier_states=max_frontier_states, + max_transitions=max_transitions, + max_pareto_comparisons=max_pareto_comparisons, + ) + expected = build_plan_payload( + candidate_path, + candidate_bundle, + objective=plan["objective"], + optimization=recomputed, + ) + if expected != plan: + raise ValueError("AutoPolicy plan does not match exact recomputation") + return { + "status": "pass", + "schema": "cliffquant.autopolicy-verification.v1", + "candidate_sha256": sha256_file(candidate_path), + "plan_sha256": sha256_file(plan_path), + "objective": plan["objective"], + "budget_bytes": budget, + } diff --git a/src/cliffquant/autopolicy_cli.py b/src/cliffquant/autopolicy_cli.py new file mode 100644 index 0000000..c7a92a9 --- /dev/null +++ b/src/cliffquant/autopolicy_cli.py @@ -0,0 +1,220 @@ +"""Command-line interface for measured CliffQuant AutoPolicy planning.""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +from .autopolicy import DEFAULT_MAX_PARETO_COMPARISONS, DEFAULT_MAX_TRANSITIONS +from .autopolicy_artifacts import ( + OBJECTIVES, + ROBUST_OBJECTIVE, + aggregate_experiment001_transformer_blocks, + build_plan_payload, + load_candidate_bundle, + solve_candidate_bundle, + verify_plan, + write_candidate_bundle, + write_plan, +) +from .autopolicy_experiment001 import ( + build_experiment001_candidates, + verify_experiment001_evidence, +) +from .provenance import sha256_file + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="cliffquant autopolicy", + description=( + "Build, solve, and verify measured mixed-precision policy tables. " + "This planner does not export a runnable mixed-bit checkpoint." + ), + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + build = subparsers.add_parser( + "build-experiment-001", + help="measure W4/W8/dense candidates from the frozen Experiment 001 artifacts", + ) + build.add_argument("--base-model", type=Path, required=True) + build.add_argument("--calibration-metadata", type=Path, required=True) + build.add_argument("--w4-scale-run", type=Path, required=True) + build.add_argument("--output", type=Path, required=True) + build.add_argument("--row-chunk-size", type=int, default=8) + + aggregate = subparsers.add_parser( + "aggregate-blocks", + help="derive content-bound transformer-block candidates from Experiment 001", + ) + aggregate.add_argument("--candidates", type=Path, required=True) + aggregate.add_argument("--output", type=Path, required=True) + + solve = subparsers.add_parser( + "solve", + help="select one recorded candidate per unit under an exact logical-byte budget", + ) + solve.add_argument("--candidates", type=Path, required=True) + solve.add_argument("--budget-bytes", type=int, required=True) + solve.add_argument("--objective", choices=OBJECTIVES, default=ROBUST_OBJECTIVE) + solve.add_argument("--max-frontier-states", type=int, default=250_000) + solve.add_argument("--max-transitions", type=int, default=DEFAULT_MAX_TRANSITIONS) + solve.add_argument( + "--max-pareto-comparisons", + type=int, + default=DEFAULT_MAX_PARETO_COMPARISONS, + ) + solve.add_argument("--output", type=Path, required=True) + + verify = subparsers.add_parser( + "verify", + help="verify candidate evidence and optionally recompute a bound plan", + ) + verify.add_argument("--candidates", type=Path, required=True) + verify.add_argument("--plan", type=Path) + verify.add_argument("--max-frontier-states", type=int, default=250_000) + verify.add_argument("--max-transitions", type=int, default=DEFAULT_MAX_TRANSITIONS) + verify.add_argument( + "--max-pareto-comparisons", + type=int, + default=DEFAULT_MAX_PARETO_COMPARISONS, + ) + return parser + + +def _print(payload: Any) -> None: + print(json.dumps(payload, indent=2, sort_keys=True, allow_nan=False)) + + +def _build(args: argparse.Namespace) -> dict[str, Any]: + def progress(completed: int, total: int, module_name: str) -> None: + print(f"[{completed:03d}/{total:03d}] {module_name}", file=sys.stderr, flush=True) + + bundle = build_experiment001_candidates( + base_model_dir=args.base_model, + calibration_metadata=args.calibration_metadata, + w4_scale_run=args.w4_scale_run, + output_dir=args.output, + row_chunk_size=args.row_chunk_size, + progress=progress, + ) + candidate_path = args.output.resolve() / "candidates.json" + return { + "status": "complete", + "schema": bundle["schema"], + "candidate_file": str(candidate_path), + "candidate_sha256": sha256_file(candidate_path), + "candidate_fingerprint_sha256": bundle["fingerprint_sha256"], + "module_count": bundle["scope"]["module_count"], + "totals": bundle["totals"], + "checkpoint_export": False, + } + + +def _solve(args: argparse.Namespace) -> dict[str, Any]: + bundle = load_candidate_bundle(args.candidates) + optimization = solve_candidate_bundle( + bundle, + budget_bytes=args.budget_bytes, + objective=args.objective, + max_frontier_states=args.max_frontier_states, + max_transitions=args.max_transitions, + max_pareto_comparisons=args.max_pareto_comparisons, + ) + plan = build_plan_payload( + args.candidates, + bundle, + objective=args.objective, + optimization=optimization, + ) + write_plan(args.output, plan) + return { + "status": "complete", + "plan_file": str(args.output.resolve()), + "plan_sha256": sha256_file(args.output), + "plan_fingerprint_sha256": plan["fingerprint_sha256"], + "objective": args.objective, + "budget_scope": plan["budget_scope"], + "optimization": optimization.to_dict(), + "checkpoint_export": False, + } + + +def _aggregate(args: argparse.Namespace) -> dict[str, Any]: + bundle = load_candidate_bundle(args.candidates) + aggregated = aggregate_experiment001_transformer_blocks( + args.candidates, + bundle, + ) + written = write_candidate_bundle(args.output, aggregated) + return { + "status": "complete", + "schema": written["schema"], + "profile": written["profile"], + "candidate_file": str(args.output.resolve()), + "candidate_sha256": sha256_file(args.output), + "candidate_fingerprint_sha256": written["fingerprint_sha256"], + "allocation_unit_count": written["scope"]["allocation_unit_count"], + "allocation_granularity": written["scope"]["allocation_granularity"], + } + + +def _verify(args: argparse.Namespace) -> dict[str, Any]: + bundle = load_candidate_bundle(args.candidates) + result: dict[str, Any] = { + "status": "pass", + "candidate_schema": bundle["schema"], + "candidate_sha256": sha256_file(args.candidates), + "candidate_fingerprint_sha256": bundle["fingerprint_sha256"], + } + if bundle["profile"] == "experiment-001": + result["candidate_evidence"] = verify_experiment001_evidence( + args.candidates, + bundle, + ) + if args.plan is not None: + result["plan"] = verify_plan( + args.candidates, + args.plan, + max_frontier_states=args.max_frontier_states, + max_transitions=args.max_transitions, + max_pareto_comparisons=args.max_pareto_comparisons, + ) + return result + + +def main(argv: Sequence[str] | None = None) -> int: + parser = _parser() + args = parser.parse_args(argv) + try: + if args.command == "build-experiment-001": + result = _build(args) + elif args.command == "aggregate-blocks": + result = _aggregate(args) + elif args.command == "solve": + result = _solve(args) + elif args.command == "verify": + result = _verify(args) + else: # pragma: no cover - guarded by argparse + parser.error(f"unsupported command: {args.command}") + except ( + ArithmeticError, + FileExistsError, + FileNotFoundError, + RuntimeError, + TypeError, + ValueError, + ) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + _print(result) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/cliffquant/autopolicy_experiment001.py b/src/cliffquant/autopolicy_experiment001.py new file mode 100644 index 0000000..bce85f8 --- /dev/null +++ b/src/cliffquant/autopolicy_experiment001.py @@ -0,0 +1,622 @@ +"""Measured AutoPolicy candidates from CliffQuant's frozen Experiment 001.""" + +from __future__ import annotations + +import json +import os +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Any + +import numpy as np +from numpy.typing import NDArray + +from .autopolicy_artifacts import CANDIDATE_SCHEMA, write_candidate_bundle +from .fp16_grid import positive_finite_fp16_scales +from .full_model_artifacts import ( + BASE_MODEL_ID, + BASE_MODEL_REVISION, + GROUP_SIZE, + POLICY_CLIFFQUANT, + CalibrationDiagonals, + FrozenModuleSpec, + ScaleRun, + load_calibration_diagonals, + load_scale_run, + resolve_hf_snapshot_file, +) +from .model_snapshot import describe_model_snapshot +from .provenance import ( + canonical_array_sha256, + canonical_json_bytes, + runtime_metadata, + sha256_bytes, + sha256_file, +) + +W4_CANDIDATE = "w4_cliffquant" +W8_CANDIDATE = "w8_absmax" +DENSE16_CANDIDATE = "dense16" +BYTE_MODEL = "logical-codes-plus-fp16-scales-v1" +BUILDER_VERSION = "cliffquant-autopolicy-experiment001-v1" + + +def logical_payload_bytes( + *, + out_features: int, + in_features: int, + bits: int, + group_size: int = GROUP_SIZE, +) -> dict[str, int]: + """Return theoretical code/scale bytes for one symmetric weight matrix.""" + + for value, label in ( + (out_features, "out_features"), + (in_features, "in_features"), + (group_size, "group_size"), + ): + if type(value) is not int or value <= 0: + raise ValueError(f"{label} must be a positive integer") + if type(bits) is not int or bits not in {4, 8, 16}: + raise ValueError("bits must be one of 4, 8, or 16") + if bits in {4, 8} and in_features % group_size: + raise ValueError("quantized in_features must be divisible by group_size") + + weight_count = out_features * in_features + if bits == 16: + return { + "codes": weight_count * 2, + "scales": 0, + "total": weight_count * 2, + } + code_bytes = (weight_count * bits + 7) // 8 + scale_bytes = out_features * (in_features // group_size) * 2 + return { + "codes": code_bytes, + "scales": scale_bytes, + "total": code_bytes + scale_bytes, + } + + +def _nearest_positive_fp16_bits(targets: NDArray[np.float64]) -> NDArray[np.uint16]: + if targets.dtype.kind != "f" or not np.all(np.isfinite(targets)): + raise ValueError("scale targets must be finite floating point") + if np.any(targets < 0.0): + raise ValueError("scale targets must be non-negative") + grid = positive_finite_fp16_scales() + insertion = np.searchsorted(grid, targets, side="left") + upper = np.minimum(insertion, grid.size - 1) + lower = np.maximum(insertion - 1, 0) + lower_distance = targets - grid[lower] + upper_distance = grid[upper] - targets + choose_lower = (insertion == grid.size) | (lower_distance <= upper_distance) + indices = np.where((insertion > 0) & choose_lower, lower, upper) + return np.ascontiguousarray(indices + 1, dtype=np.uint16) + + +def _validate_rows( + weights: NDArray[np.generic], + curvature: NDArray[np.generic], + *, + group_size: int, +) -> tuple[NDArray[np.float64], NDArray[np.float64]]: + weight_array = np.ascontiguousarray(np.asarray(weights), dtype=np.float64) + curvature_array = np.ascontiguousarray(np.asarray(curvature), dtype=np.float64) + if weight_array.ndim != 2 or weight_array.shape[0] == 0: + raise ValueError("weights must be a non-empty rank-two array") + if curvature_array.ndim != 2 or curvature_array.shape[0] == 0: + raise ValueError("curvature must be a non-empty rank-two array") + if curvature_array.shape[1] != weight_array.shape[1]: + raise ValueError("curvature feature width must match weights") + if weight_array.shape[1] % group_size: + raise ValueError("weight width must be divisible by group_size") + if not np.all(np.isfinite(weight_array)): + raise ValueError("weights must be finite") + if not np.all(np.isfinite(curvature_array)) or np.any(curvature_array < 0.0): + raise ValueError("curvature must be finite and non-negative") + return weight_array, curvature_array + + +def _evaluate_scale_rows( + weights: NDArray[np.generic], + curvature: NDArray[np.generic], + scale_bits: NDArray[np.generic], + *, + qmin: int, + qmax: int, + group_size: int = GROUP_SIZE, +) -> tuple[NDArray[np.float64], NDArray[np.float64]]: + weight_array, curvature_array = _validate_rows( + weights, + curvature, + group_size=group_size, + ) + rows = weight_array.shape[0] + groups_per_row = weight_array.shape[1] // group_size + bits = np.ascontiguousarray(np.asarray(scale_bits), dtype=np.uint16) + if bits.shape != (rows, groups_per_row): + raise ValueError("scale_bits shape does not match grouped weights") + if np.any(bits == 0) or np.any(bits > 0x7BFF): + raise ValueError("scale_bits must encode positive finite FP16 values") + scales = bits.astype("erg", + squared_error, + grouped_curvature, + optimize=True, + ) + groupwise_max = np.max(environment_group_losses, axis=0) + per_environment = np.sum(environment_group_losses, axis=(1, 2), dtype=np.float64) + return ( + np.ascontiguousarray(groupwise_max, dtype=np.float64), + np.ascontiguousarray(per_environment, dtype=np.float64), + ) + + +def vectorized_w8_absmax_rows( + weights: NDArray[np.generic], + curvature: NDArray[np.generic], + *, + group_size: int = GROUP_SIZE, +) -> tuple[NDArray[np.uint16], NDArray[np.float64], NDArray[np.float64]]: + """Evaluate sign-aware W8 AbsMax using stored FP16 scales.""" + + weight_array, curvature_array = _validate_rows( + weights, + curvature, + group_size=group_size, + ) + rows = weight_array.shape[0] + groups_per_row = weight_array.shape[1] // group_size + grouped = weight_array.reshape(rows, groups_per_row, group_size) + positive_limit = np.max(np.where(grouped > 0.0, grouped, 0.0), axis=-1) / 127.0 + negative_limit = np.max(np.where(grouped < 0.0, -grouped, 0.0), axis=-1) / 128.0 + scale_bits = _nearest_positive_fp16_bits(np.maximum(positive_limit, negative_limit)) + objectives, per_environment = _evaluate_scale_rows( + weight_array, + curvature_array, + scale_bits, + qmin=-128, + qmax=127, + group_size=group_size, + ) + return scale_bits, objectives, per_environment + + +class NumpySafetensorsWeightSource: + """Read BF16/FP16 model rows without importing PyTorch.""" + + def __init__(self, snapshot_dir: str | Path) -> None: + try: + import ml_dtypes # noqa: F401 + from safetensors import safe_open # noqa: F401 + except ImportError as exc: + raise RuntimeError("install CliffQuant with the 'autopolicy-experiment' extra") from exc + + self.root = Path(snapshot_dir).resolve() + if not self.root.is_dir(): + raise FileNotFoundError(f"base model snapshot does not exist: {self.root}") + if self.root.name != BASE_MODEL_REVISION: + raise ValueError( + "base model directory must be the frozen Hugging Face snapshot " + f"{BASE_MODEL_REVISION}" + ) + self._provenance = describe_model_snapshot( + self.root, + model_id=BASE_MODEL_ID, + revision=BASE_MODEL_REVISION, + ) + index_path = resolve_hf_snapshot_file( + self.root, + "model.safetensors.index.json", + label="safetensors index", + ) + index = json.loads(index_path.read_text(encoding="utf-8")) + raw_map = index.get("weight_map") + if not isinstance(raw_map, dict) or not raw_map: + raise ValueError("safetensors index has no weight_map") + if not all( + isinstance(key, str) and isinstance(value, str) for key, value in raw_map.items() + ): + raise TypeError("safetensors weight_map must map strings to strings") + self._weight_map = dict(raw_map) + self._shards = { + name: resolve_hf_snapshot_file(self.root, name, label="safetensors shard") + for name in sorted(set(self._weight_map.values())) + } + + @property + def provenance(self) -> Mapping[str, Any]: + return self._provenance + + @property + def keys(self) -> frozenset[str]: + return frozenset(self._weight_map) + + def read_rows(self, key: str, start: int, stop: int) -> NDArray[np.float64]: + if key not in self._weight_map: + raise KeyError(f"tensor is absent from pinned base checkpoint: {key}") + if type(start) is not int or type(stop) is not int or start < 0 or stop <= start: + raise ValueError("row bounds must be increasing non-negative integers") + import ml_dtypes # noqa: F401 + from safetensors import safe_open + + path = self._shards[self._weight_map[key]] + with safe_open(path, framework="np") as handle: + tensor_slice = handle.get_slice(key) + shape = tuple(int(value) for value in tensor_slice.get_shape()) + if len(shape) != 2 or stop > shape[0]: + raise ValueError(f"invalid row slice {start}:{stop} for {key} with shape {shape}") + value = tensor_slice[start:stop] + return np.ascontiguousarray(np.asarray(value), dtype=np.float64) + + +def _atomic_npz_new(path: Path, arrays: Mapping[str, NDArray[np.generic]]) -> None: + if path.exists(): + raise FileExistsError(f"refusing to overwrite existing artifact: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + try: + with temporary.open("xb") as handle: + np.savez_compressed(handle, **arrays) + os.replace(temporary, path) + finally: + if temporary.exists(): + temporary.unlink() + + +def _module_slug(module: FrozenModuleSpec) -> str: + return f"module-{module.layer:02d}-{module.branch.replace('.', '-')}" + + +def _source_identity_matches( + source: NumpySafetensorsWeightSource, + calibration: CalibrationDiagonals, + scale_run: ScaleRun, +) -> None: + scale_source = scale_run.metadata.get("base_model", {}).get("source") + if source.provenance != scale_source: + raise ValueError("base snapshot identity disagrees with the W4 scale run") + calibration_source = (calibration.source or {}).get("model") + if source.provenance != calibration_source: + raise ValueError("base snapshot identity disagrees with calibration diagonals") + if calibration.modules != scale_run.modules: + raise ValueError("calibration and W4 scale-run module inventories disagree") + if scale_run.policy != POLICY_CLIFFQUANT: + raise ValueError("W4 scale run must use the frozen CliffQuant minimax policy") + + +def _candidate_entry( + candidate_id: str, + payload: Mapping[str, int], + groupwise_max_distortion: float, + per_environment: NDArray[np.float64], + *, + environments: tuple[str, ...], + metadata: Mapping[str, Any], +) -> dict[str, Any]: + return { + "candidate_id": candidate_id, + "logical_bytes": payload["total"], + "groupwise_max_distortion": float(groupwise_max_distortion), + "per_environment_distortion": { + environment: float(value) + for environment, value in zip(environments, per_environment, strict=True) + }, + "metadata": { + **dict(metadata), + "logical_code_bytes": payload["codes"], + "logical_scale_bytes": payload["scales"], + }, + } + + +def _score_module( + output_root: Path, + *, + source: NumpySafetensorsWeightSource, + calibration: CalibrationDiagonals, + scale_run: ScaleRun, + module: FrozenModuleSpec, + row_chunk_size: int, +) -> dict[str, Any]: + curvature = calibration.normalized[module.name] + w4_artifact = scale_run.artifacts[module.name] + groups_per_row = module.in_features // GROUP_SIZE + w8_scale_bits = np.empty((module.out_features, groups_per_row), dtype=np.uint16) + w8_objectives = np.empty((module.out_features, groups_per_row), dtype=np.float64) + w4_per_environment = np.zeros(len(calibration.environments), dtype=np.float64) + w8_per_environment = np.zeros(len(calibration.environments), dtype=np.float64) + + for start in range(0, module.out_features, row_chunk_size): + stop = min(start + row_chunk_size, module.out_features) + weights = source.read_rows(module.weight_key, start, stop) + w4_objectives, w4_environment = _evaluate_scale_rows( + weights, + curvature, + w4_artifact.scale_bits[start:stop], + qmin=-8, + qmax=7, + ) + expected_w4 = w4_artifact.objectives[start:stop] + if not np.allclose(w4_objectives, expected_w4, rtol=2e-12, atol=1e-18): + maximum_delta = float(np.max(np.abs(w4_objectives - expected_w4))) + raise ValueError( + f"recomputed W4 objectives disagree for {module.name}; " + f"max_abs_delta={maximum_delta}" + ) + bits, w8_group_objectives, w8_environment = vectorized_w8_absmax_rows( + weights, + curvature, + ) + w8_scale_bits[start:stop] = bits + w8_objectives[start:stop] = w8_group_objectives + w4_per_environment += w4_environment + w8_per_environment += w8_environment + + module_path = output_root / "modules" / f"{_module_slug(module)}.w8.npz" + arrays: dict[str, NDArray[np.generic]] = { + "groupwise_max_objectives": w8_objectives, + "scale_bits": w8_scale_bits, + } + _atomic_npz_new(module_path, arrays) + relative_archive = module_path.relative_to(output_root).as_posix() + w4_bytes = logical_payload_bytes( + out_features=module.out_features, + in_features=module.in_features, + bits=4, + ) + w8_bytes = logical_payload_bytes( + out_features=module.out_features, + in_features=module.in_features, + bits=8, + ) + dense_bytes = logical_payload_bytes( + out_features=module.out_features, + in_features=module.in_features, + bits=16, + ) + zeros = np.zeros(len(calibration.environments), dtype=np.float64) + return { + "unit_id": module.name, + "weight_key": module.weight_key, + "shape": [module.out_features, module.in_features], + "group_count": module.out_features * groups_per_row, + "candidates": [ + _candidate_entry( + DENSE16_CANDIDATE, + dense_bytes, + 0.0, + zeros, + environments=calibration.environments, + metadata={ + "bits": 16, + "group_size": None, + "scale_selection": None, + }, + ), + _candidate_entry( + W4_CANDIDATE, + w4_bytes, + float(np.sum(w4_artifact.objectives, dtype=np.float64)), + w4_per_environment, + environments=calibration.environments, + metadata={ + "bits": 4, + "group_size": GROUP_SIZE, + "scale_selection": "cliffquant_exact_minimax_fp16", + }, + ), + _candidate_entry( + W8_CANDIDATE, + w8_bytes, + float(np.sum(w8_objectives, dtype=np.float64)), + w8_per_environment, + environments=calibration.environments, + metadata={ + "bits": 8, + "group_size": GROUP_SIZE, + "scale_selection": "absmax_signed_range_nearest_fp16", + }, + ), + ], + "evidence": { + "w4_scale_archive_sha256": w4_artifact.archive_sha256, + "w4_scale_metadata_sha256": w4_artifact.metadata_sha256, + "w8_archive": { + "file": relative_archive, + "sha256": sha256_file(module_path), + "arrays": { + name: { + "dtype": np.ascontiguousarray(value).dtype.str, + "shape": list(value.shape), + "sha256": canonical_array_sha256(np.ascontiguousarray(value)), + } + for name, value in arrays.items() + }, + }, + }, + } + + +def build_experiment001_candidates( + *, + base_model_dir: str | Path, + calibration_metadata: str | Path, + w4_scale_run: str | Path, + output_dir: str | Path, + row_chunk_size: int = 8, + progress: Callable[[int, int, str], None] | None = None, +) -> dict[str, Any]: + """Build the measured W4/W8/dense candidate table for Experiment 001.""" + + if type(row_chunk_size) is not int or row_chunk_size <= 0: + raise ValueError("row_chunk_size must be a positive integer") + output_root = Path(output_dir).resolve() + candidate_path = output_root / "candidates.json" + if candidate_path.exists() or (output_root / "modules").exists(): + raise FileExistsError( + f"refusing to reuse a non-empty AutoPolicy output directory: {output_root}" + ) + + scale_run = load_scale_run(w4_scale_run, strict_inventory=True) + calibration = load_calibration_diagonals( + calibration_metadata, + strict_inventory=True, + corpus_replay=scale_run.metadata.get("corpus_replay"), + ) + source = NumpySafetensorsWeightSource(base_model_dir) + _source_identity_matches(source, calibration, scale_run) + for module in calibration.modules: + if module.weight_key not in source.keys: + raise ValueError(f"base checkpoint is missing {module.weight_key}") + + units: list[dict[str, Any]] = [] + total = len(calibration.modules) + for index, module in enumerate(calibration.modules, start=1): + units.append( + _score_module( + output_root, + source=source, + calibration=calibration, + scale_run=scale_run, + module=module, + row_chunk_size=row_chunk_size, + ) + ) + if progress is not None: + progress(index, total, module.name) + units.sort(key=lambda unit: unit["unit_id"]) + + totals = { + candidate_id: sum( + next( + candidate["logical_bytes"] + for candidate in unit["candidates"] + if candidate["candidate_id"] == candidate_id + ) + for unit in units + ) + for candidate_id in (W4_CANDIDATE, W8_CANDIDATE, DENSE16_CANDIDATE) + } + weight_count = sum(module.out_features * module.in_features for module in calibration.modules) + group_count = sum(unit["group_count"] for unit in units) + payload = { + "schema": CANDIDATE_SCHEMA, + "status": "complete", + "profile": "experiment-001", + "scope": { + "module_count": len(units), + "weight_count": weight_count, + "group_count": group_count, + "group_size": GROUP_SIZE, + "byte_model": BYTE_MODEL, + "budget_scope": "logical-target-weight-and-scale-payload", + "checkpoint_export": False, + }, + "source": { + "base_model": dict(source.provenance), + "calibration": { + "metadata_sha256": calibration.metadata_sha256, + "archive_sha256": calibration.archive_sha256, + }, + "w4_scale_run": { + "manifest_sha256": scale_run.manifest_sha256, + "policy": scale_run.policy, + }, + }, + "environments": list(calibration.environments), + "distortion_metric": { + "name": ("normalized_diagonal_squared_error_with_separate_environment_totals"), + "groupwise_scalar": "sum_groupwise_max_environment_error", + "robust_policy": "max_environment_sum_module_error", + "additive_across_units": True, + "downstream_metric": False, + }, + "candidate_definitions": { + W4_CANDIDATE: { + "bits": 4, + "group_size": GROUP_SIZE, + "scale_selection": "cliffquant_exact_minimax_fp16", + }, + W8_CANDIDATE: { + "bits": 8, + "group_size": GROUP_SIZE, + "scale_selection": "absmax_signed_range_nearest_fp16", + }, + DENSE16_CANDIDATE: { + "bits": 16, + "group_size": None, + "scale_selection": None, + }, + }, + "units": units, + "totals": totals, + "builder": { + "name": BUILDER_VERSION, + "row_chunk_size": row_chunk_size, + "runtime": runtime_metadata(device="cpu"), + "source_sha256": sha256_file(__file__), + }, + } + return write_candidate_bundle(candidate_path, payload) + + +def verify_experiment001_evidence( + candidate_path: str | Path, + payload: Mapping[str, Any], +) -> dict[str, Any]: + """Verify every local W8 archive bound by a candidate bundle.""" + + root = Path(candidate_path).resolve().parent + checked = 0 + for unit in payload["units"]: + descriptor = unit["evidence"]["w8_archive"] + relative = descriptor.get("file") + if not isinstance(relative, str): + raise TypeError("W8 archive file must be a string") + archive = (root / Path(relative)).resolve() + try: + archive.relative_to(root) + except ValueError as exc: + raise ValueError("W8 archive path escapes the candidate bundle") from exc + if sha256_file(archive) != descriptor.get("sha256"): + raise ValueError(f"W8 archive hash mismatch for {unit['unit_id']}") + with np.load(archive, allow_pickle=False) as arrays: + if set(arrays.files) != {"groupwise_max_objectives", "scale_bits"}: + raise ValueError(f"unexpected W8 arrays for {unit['unit_id']}") + for name in arrays.files: + value = np.ascontiguousarray(arrays[name]) + expected = descriptor["arrays"].get(name) + if ( + not isinstance(expected, dict) + or expected.get("dtype") != value.dtype.str + or expected.get("shape") != list(value.shape) + or expected.get("sha256") != canonical_array_sha256(value) + ): + raise ValueError(f"W8 array evidence mismatch for {unit['unit_id']}/{name}") + checked += 1 + result = { + "schema": "cliffquant.autopolicy-evidence-verification.v1", + "status": "pass", + "candidate_sha256": sha256_file(candidate_path), + "candidate_fingerprint_sha256": payload["fingerprint_sha256"], + "w8_archives_checked": checked, + } + result["sha256"] = sha256_bytes(canonical_json_bytes(result)) + return result diff --git a/src/cliffquant/cli.py b/src/cliffquant/cli.py new file mode 100644 index 0000000..7ae5529 --- /dev/null +++ b/src/cliffquant/cli.py @@ -0,0 +1,35 @@ +"""Top-level CliffQuant command-line interface.""" + +from __future__ import annotations + +import argparse +import sys +from collections.abc import Sequence + +from . import __version__ + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="cliffquant", + description="Correctness-first quantization research tools.", + ) + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + parser.add_argument("command", choices=("autopolicy",)) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = list(sys.argv[1:] if argv is None else argv) + if arguments and arguments[0] == "autopolicy": + from .autopolicy_cli import main as autopolicy_main + + return autopolicy_main(arguments[1:]) + parser = _parser() + args = parser.parse_args(arguments) + parser.error(f"unsupported command: {args.command}") # pragma: no cover + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_autopolicy.py b/tests/test_autopolicy.py new file mode 100644 index 0000000..ee38c25 --- /dev/null +++ b/tests/test_autopolicy.py @@ -0,0 +1,546 @@ +from __future__ import annotations + +import itertools +import json +import random + +import pytest + +from cliffquant.autopolicy import ( + FrontierLimitError, + InfeasiblePolicyError, + PolicyCandidate, + QuantizableUnit, + RobustPolicyCandidate, + RobustQuantizableUnit, + solve_autopolicy, + solve_robust_autopolicy, +) + + +def _candidate(candidate_id: str, byte_size: int, distortion: float) -> PolicyCandidate: + return PolicyCandidate( + candidate_id=candidate_id, + byte_size=byte_size, + distortion=distortion, + ) + + +def _exhaustive_oracle( + units: list[QuantizableUnit], + budget_bytes: int, +) -> tuple[float, int, tuple[str, ...]]: + canonical_units = sorted(units, key=lambda unit: unit.unit_id) + feasible: list[tuple[float, int, tuple[str, ...]]] = [] + for combination in itertools.product(*(unit.candidates for unit in canonical_units)): + used_bytes = sum(candidate.byte_size for candidate in combination) + if used_bytes > budget_bytes: + continue + aggregate_distortion = 0.0 + for candidate in combination: + aggregate_distortion += candidate.distortion + feasible.append( + ( + aggregate_distortion, + used_bytes, + tuple(candidate.candidate_id for candidate in combination), + ) + ) + if not feasible: + raise InfeasiblePolicyError + return min(feasible) + + +def test_known_budget_tradeoff_selects_exact_optimum() -> None: + units = [ + QuantizableUnit( + "block.0", + ( + _candidate("int4", 4, 8.0), + _candidate("int8", 8, 1.0), + _candidate("fp16", 16, 0.0), + ), + ), + QuantizableUnit( + "block.1", + ( + _candidate("int4", 4, 5.0), + _candidate("int8", 8, 2.0), + _candidate("fp16", 16, 0.0), + ), + ), + ] + + result = solve_autopolicy(units, budget_bytes=16) + + assert [(item.unit_id, item.candidate_id) for item in result.decisions] == [ + ("block.0", "int8"), + ("block.1", "int8"), + ] + assert result.used_bytes == 16 + assert result.remaining_bytes == 0 + assert result.aggregate_distortion == 3.0 + + +@pytest.mark.parametrize("seed", range(30)) +def test_random_small_problems_match_exhaustive_oracle(seed: int) -> None: + rng = random.Random(seed) + units: list[QuantizableUnit] = [] + for unit_index in range(rng.randint(1, 4)): + candidates = tuple( + _candidate( + f"choice-{candidate_index}", + rng.randint(1, 12), + rng.randint(0, 100) / 10.0, + ) + for candidate_index in range(rng.randint(2, 4)) + ) + units.append(QuantizableUnit(f"unit-{unit_index}", candidates)) + + minimum = sum(min(item.byte_size for item in unit.candidates) for unit in units) + maximum = sum(max(item.byte_size for item in unit.candidates) for unit in units) + budget = rng.randint(minimum, maximum) + oracle = _exhaustive_oracle(units, budget) + + result = solve_autopolicy(reversed(units), budget_bytes=budget) + actual = ( + result.aggregate_distortion, + result.used_bytes, + tuple(item.candidate_id for item in result.decisions), + ) + + assert actual == oracle + assert result.used_bytes <= budget + assert len(result.decisions) == len(units) + + +def test_input_order_does_not_change_plan_or_serialization() -> None: + first = QuantizableUnit( + "a", + ( + _candidate("z", 3, 1.0), + _candidate("a", 3, 1.0), + ), + ) + second = QuantizableUnit( + "b", + ( + _candidate("high", 5, 0.0), + _candidate("low", 2, 2.0), + ), + ) + + forward = solve_autopolicy([first, second], budget_bytes=8) + reversed_plan = solve_autopolicy( + [ + QuantizableUnit("b", tuple(reversed(second.candidates))), + QuantizableUnit("a", tuple(reversed(first.candidates))), + ], + budget_bytes=8, + ) + + assert forward == reversed_plan + assert forward.to_json() == reversed_plan.to_json() + + +def test_ties_prefer_lower_bytes_then_lexicographic_candidate_ids() -> None: + lower_bytes = QuantizableUnit( + "unit", + ( + _candidate("large", 9, 1.0), + _candidate("small", 4, 1.0), + ), + ) + same_bytes = QuantizableUnit( + "other", + ( + _candidate("z-choice", 3, 2.0), + _candidate("a-choice", 3, 2.0), + ), + ) + + result = solve_autopolicy([lower_bytes, same_bytes], budget_bytes=20) + + assert [(item.unit_id, item.candidate_id) for item in result.decisions] == [ + ("other", "a-choice"), + ("unit", "small"), + ] + assert result.used_bytes == 7 + + +def test_relaxing_budget_never_worsens_exact_objective() -> None: + units = [ + QuantizableUnit( + f"unit-{index}", + ( + _candidate("int4", 4, float(10 - index)), + _candidate("int8", 8, float(3 - index / 10)), + _candidate("fp16", 16, 0.0), + ), + ) + for index in range(3) + ] + + objectives = [ + solve_autopolicy(units, budget_bytes=budget).aggregate_distortion + for budget in range(12, 49) + ] + + assert all(later <= earlier for earlier, later in itertools.pairwise(objectives)) + + +def test_dominated_candidate_does_not_change_plan() -> None: + baseline_unit = QuantizableUnit( + "unit", + ( + _candidate("compact", 4, 2.0), + _candidate("accurate", 8, 0.5), + ), + ) + with_dominated = QuantizableUnit( + "unit", + ( + *baseline_unit.candidates, + _candidate("dominated", 9, 3.0), + ), + ) + + baseline = solve_autopolicy([baseline_unit], budget_bytes=9) + augmented = solve_autopolicy([with_dominated], budget_bytes=9) + + assert baseline.aggregate_distortion == augmented.aggregate_distortion + assert baseline.used_bytes == augmented.used_bytes + assert baseline.decisions[0].candidate_id == augmented.decisions[0].candidate_id + + +def test_result_is_json_serializable_and_self_describing() -> None: + unit = QuantizableUnit("layer.0", (_candidate("w4", 128, 0.25),)) + + result = solve_autopolicy([unit], budget_bytes=256) + payload = result.to_dict() + round_tripped = json.loads(json.dumps(payload, allow_nan=False)) + + assert round_tripped["schema_version"] == 1 + assert round_tripped["algorithm"] == "exact_pareto_frontier" + assert round_tripped["objective"] == "minimize_sum_distortion" + assert round_tripped["budget_bytes"] == 256 + assert round_tripped["used_bytes"] == 128 + assert round_tripped["remaining_bytes"] == 128 + assert round_tripped["decisions"] == [ + { + "unit_id": "layer.0", + "candidate_id": "w4", + "byte_size": 128, + "distortion": 0.25, + } + ] + assert json.loads(result.to_json()) == round_tripped + + +def test_infeasible_budget_fails_closed_with_minimum_requirement() -> None: + units = [ + QuantizableUnit("a", (_candidate("only", 5, 0.0),)), + QuantizableUnit("b", (_candidate("only", 7, 0.0),)), + ] + + with pytest.raises( + InfeasiblePolicyError, + match=r"budget_bytes=11 is infeasible; at least 12 bytes are required", + ): + solve_autopolicy(units, budget_bytes=11) + + +@pytest.mark.parametrize("invalid_budget", [True, 1.5, "10", None]) +def test_invalid_budget_type_is_rejected(invalid_budget: object) -> None: + unit = QuantizableUnit("unit", (_candidate("only", 1, 0.0),)) + + with pytest.raises(TypeError, match="budget_bytes must be an integer"): + solve_autopolicy([unit], budget_bytes=invalid_budget) # type: ignore[arg-type] + + +def test_negative_budget_and_empty_problem_are_rejected() -> None: + unit = QuantizableUnit("unit", (_candidate("only", 1, 0.0),)) + + with pytest.raises(ValueError, match="budget_bytes must be non-negative"): + solve_autopolicy([unit], budget_bytes=-1) + with pytest.raises(ValueError, match="units must not be empty"): + solve_autopolicy([], budget_bytes=1) + + +def test_duplicate_unit_and_candidate_identifiers_are_rejected() -> None: + candidate = _candidate("same", 1, 0.0) + + with pytest.raises(ValueError, match="candidate_id values must be unique"): + QuantizableUnit("unit", (candidate, candidate)) + + units = [ + QuantizableUnit("same", (_candidate("a", 1, 0.0),)), + QuantizableUnit("same", (_candidate("b", 1, 0.0),)), + ] + with pytest.raises(ValueError, match="unit_id values must be unique"): + solve_autopolicy(units, budget_bytes=2) + + +@pytest.mark.parametrize( + ("candidate_id", "byte_size", "distortion", "exception"), + [ + ("", 1, 0.0, ValueError), + (" padded", 1, 0.0, ValueError), + ("valid", True, 0.0, TypeError), + ("valid", 0, 0.0, ValueError), + ("valid", -1, 0.0, ValueError), + ("valid", 1, True, TypeError), + ("valid", 1, -0.1, ValueError), + ("valid", 1, float("nan"), ValueError), + ("valid", 1, float("inf"), ValueError), + ], +) +def test_invalid_candidates_are_rejected( + candidate_id: str, + byte_size: int, + distortion: float, + exception: type[Exception], +) -> None: + with pytest.raises(exception): + PolicyCandidate(candidate_id, byte_size, distortion) + + +def test_invalid_units_are_rejected() -> None: + candidate = _candidate("only", 1, 0.0) + + with pytest.raises(ValueError, match="unit_id must not be empty"): + QuantizableUnit("", (candidate,)) + with pytest.raises(ValueError, match="candidates must not be empty"): + QuantizableUnit("unit", ()) + with pytest.raises(TypeError, match="candidates must contain only"): + QuantizableUnit("unit", (candidate, object())) # type: ignore[arg-type] + with pytest.raises(TypeError, match="units must contain only"): + solve_autopolicy([QuantizableUnit("unit", (candidate,)), object()], budget_bytes=2) + + +def _robust_candidate( + candidate_id: str, + byte_size: int, + *distortions: float, +) -> RobustPolicyCandidate: + return RobustPolicyCandidate(candidate_id, byte_size, distortions) + + +def _robust_exhaustive_oracle( + units: list[RobustQuantizableUnit], + environments: tuple[str, ...], + budget_bytes: int, +) -> tuple[float, float, int, tuple[str, ...], tuple[float, ...]]: + canonical_units = sorted(units, key=lambda unit: unit.unit_id) + feasible = [] + for combination in itertools.product(*(unit.candidates for unit in canonical_units)): + used_bytes = sum(candidate.byte_size for candidate in combination) + if used_bytes > budget_bytes: + continue + totals = tuple( + sum(candidate.per_environment_distortion[index] for candidate in combination) + for index in range(len(environments)) + ) + feasible.append( + ( + max(totals), + sum(totals), + used_bytes, + tuple(candidate.candidate_id for candidate in combination), + totals, + ) + ) + if not feasible: + raise InfeasiblePolicyError + return min(feasible) + + +@pytest.mark.parametrize("seed", range(20)) +def test_robust_random_problems_match_exhaustive_oracle(seed: int) -> None: + rng = random.Random(10_000 + seed) + environments = ("general", "code", "math") + units: list[RobustQuantizableUnit] = [] + for unit_index in range(rng.randint(1, 4)): + units.append( + RobustQuantizableUnit( + f"unit-{unit_index}", + tuple( + _robust_candidate( + f"choice-{candidate_index}", + rng.randint(1, 10), + *(rng.randint(0, 100) / 10.0 for _ in environments), + ) + for candidate_index in range(rng.randint(2, 4)) + ), + ) + ) + minimum = sum(min(item.byte_size for item in unit.candidates) for unit in units) + maximum = sum(max(item.byte_size for item in unit.candidates) for unit in units) + budget = rng.randint(minimum, maximum) + oracle = _robust_exhaustive_oracle(units, environments, budget) + + result = solve_robust_autopolicy( + reversed(units), + environments=environments, + budget_bytes=budget, + ) + actual = ( + result.worst_environment_distortion, + sum(result.per_environment_distortion), + result.used_bytes, + tuple(item.candidate_id for item in result.decisions), + result.per_environment_distortion, + ) + + assert actual == oracle + assert json.loads(result.to_json()) == result.to_dict() + + +def test_robust_minimax_can_differ_from_sum_of_per_unit_maxima() -> None: + units = [ + RobustQuantizableUnit( + "a", + ( + _robust_candidate("balanced", 4, 4.0, 4.0), + _robust_candidate("general-heavy", 4, 7.0, 0.0), + ), + ), + RobustQuantizableUnit( + "b", + ( + _robust_candidate("balanced", 4, 4.0, 4.0), + _robust_candidate("code-heavy", 4, 0.0, 7.0), + ), + ), + ] + + result = solve_robust_autopolicy( + units, + environments=("general", "code"), + budget_bytes=8, + ) + + assert [decision.candidate_id for decision in result.decisions] == [ + "general-heavy", + "code-heavy", + ] + assert result.per_environment_distortion == (7.0, 7.0) + assert result.worst_environment_distortion == 7.0 + + +def test_minimum_budget_prunes_incomplete_high_byte_prefixes() -> None: + units = [ + RobustQuantizableUnit( + f"unit-{index:03d}", + ( + _robust_candidate("minimum", 1, 1.0, 1.0), + _robust_candidate("larger", 2, 0.0, 0.0), + ), + ) + for index in range(150) + ] + + result = solve_robust_autopolicy( + units, + environments=("general", "code"), + budget_bytes=150, + max_frontier_states=1, + ) + + assert result.used_bytes == 150 + assert result.peak_frontier_states == 1 + assert result.transitions_evaluated == 150 + assert {decision.candidate_id for decision in result.decisions} == {"minimum"} + + +def test_search_transition_limit_fails_closed() -> None: + units = [ + RobustQuantizableUnit( + f"unit-{index}", + ( + _robust_candidate("a", 1, 0.0, 1.0), + _robust_candidate("b", 1, 1.0, 0.0), + ), + ) + for index in range(2) + ] + + with pytest.raises(FrontierLimitError, match=r"max_transitions=2"): + solve_robust_autopolicy( + units, + environments=("one", "two"), + budget_bytes=2, + max_transitions=2, + ) + + +def test_robust_pareto_work_limit_fails_closed_before_pruning() -> None: + unit = RobustQuantizableUnit( + "unit", + ( + _robust_candidate("a", 1, 0.0, 1.0), + _robust_candidate("b", 1, 1.0, 0.0), + ), + ) + + with pytest.raises(FrontierLimitError, match=r"max_pareto_comparisons=1"): + solve_robust_autopolicy( + [unit], + environments=("one", "two"), + budget_bytes=1, + max_pareto_comparisons=1, + ) + + +def test_robust_secondary_sum_overflow_fails_closed() -> None: + unit = RobustQuantizableUnit( + "unit", + ( + _robust_candidate("a", 1, 1e308, 1e308), + _robust_candidate("b", 2, 1e308, 9e307), + ), + ) + + with pytest.raises(ArithmeticError, match="sum environment distortion overflowed"): + solve_robust_autopolicy( + [unit], + environments=("one", "two"), + budget_bytes=2, + ) + + +def test_robust_validation_and_frontier_limit_fail_closed() -> None: + with pytest.raises(ValueError, match="same environment count"): + RobustQuantizableUnit( + "unit", + ( + _robust_candidate("a", 1, 0.0), + _robust_candidate("b", 2, 0.0, 0.0), + ), + ) + unit = RobustQuantizableUnit( + "unit", + ( + _robust_candidate("a", 1, 0.0, 1.0), + _robust_candidate("b", 2, 1.0, 0.0), + ), + ) + with pytest.raises(ValueError, match="environment count disagrees"): + solve_robust_autopolicy( + [unit], + environments=("only-one",), + budget_bytes=2, + ) + with pytest.raises(ValueError, match="environments must be unique"): + solve_robust_autopolicy( + [unit], + environments=("same", "same"), + budget_bytes=2, + ) + with pytest.raises(FrontierLimitError, match="no approximate plan was returned"): + solve_robust_autopolicy( + [unit], + environments=("one", "two"), + budget_bytes=2, + max_frontier_states=1, + ) diff --git a/tests/test_autopolicy_artifacts.py b/tests/test_autopolicy_artifacts.py new file mode 100644 index 0000000..15abb9d --- /dev/null +++ b/tests/test_autopolicy_artifacts.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from cliffquant.autopolicy_artifacts import ( + CANDIDATE_SCHEMA, + ROBUST_OBJECTIVE, + TRANSFORMER_BLOCK_PROFILE, + aggregate_experiment001_transformer_blocks, + bind_fingerprint, + build_plan_payload, + load_candidate_bundle, + load_plan, + solve_candidate_bundle, + verify_plan, + write_candidate_bundle, + write_plan, +) +from cliffquant.provenance import canonical_json_bytes + + +def _sample_bundle() -> dict[str, object]: + units = [] + for index in range(2): + units.append( + { + "unit_id": f"model.language_model.layers.{index}.mlp.up_proj", + "weight_key": f"model.language_model.layers.{index}.mlp.up_proj.weight", + "shape": [8, 128], + "group_count": 8, + "candidates": [ + { + "candidate_id": "dense16", + "logical_bytes": 2048, + "groupwise_max_distortion": 0.0, + "per_environment_distortion": { + "general": 0.0, + "code": 0.0, + }, + "metadata": {"bits": 16, "group_size": None}, + }, + { + "candidate_id": "w4_cliffquant", + "logical_bytes": 528, + "groupwise_max_distortion": 5.0 + index, + "per_environment_distortion": { + "general": 4.0 + index, + "code": 3.0 + index, + }, + "metadata": {"bits": 4, "group_size": 128}, + }, + { + "candidate_id": "w8_absmax", + "logical_bytes": 1040, + "groupwise_max_distortion": 1.0 + index, + "per_environment_distortion": { + "general": 0.8 + index, + "code": 0.7 + index, + }, + "metadata": {"bits": 8, "group_size": 128}, + }, + ], + "evidence": {"kind": "unit-test"}, + } + ) + return { + "schema": CANDIDATE_SCHEMA, + "status": "complete", + "profile": "unit-test", + "scope": { + "budget_scope": "logical-target-weight-and-scale-payload", + "checkpoint_export": False, + }, + "source": {"kind": "synthetic"}, + "environments": ["general", "code"], + "distortion_metric": { + "name": "unit-test", + "additive_across_units": True, + "downstream_metric": False, + }, + "candidate_definitions": {"dense16": {}, "w4_cliffquant": {}, "w8_absmax": {}}, + "units": units, + "totals": { + "dense16": 4096, + "w4_cliffquant": 1056, + "w8_absmax": 2080, + }, + "builder": {"name": "unit-test"}, + } + + +def test_candidate_and_plan_round_trip_with_exact_recomputation(tmp_path: Path) -> None: + candidate_path = tmp_path / "candidates.json" + candidate_bundle = write_candidate_bundle(candidate_path, _sample_bundle()) + + loaded = load_candidate_bundle(candidate_path) + assert loaded == candidate_bundle + optimization = solve_candidate_bundle( + loaded, + budget_bytes=2080, + objective=ROBUST_OBJECTIVE, + ) + plan = build_plan_payload( + candidate_path, + loaded, + objective=ROBUST_OBJECTIVE, + optimization=optimization, + ) + plan_path = tmp_path / "plan.json" + write_plan(plan_path, plan) + + assert load_plan(plan_path) == plan + verification = verify_plan(candidate_path, plan_path) + assert verification["status"] == "pass" + assert verification["objective"] == ROBUST_OBJECTIVE + assert verification["budget_bytes"] == 2080 + + +def test_writers_refuse_to_overwrite(tmp_path: Path) -> None: + candidate_path = tmp_path / "candidates.json" + write_candidate_bundle(candidate_path, _sample_bundle()) + + with pytest.raises(FileExistsError, match="refusing to overwrite"): + write_candidate_bundle(candidate_path, _sample_bundle()) + + +def test_candidate_loader_rejects_duplicate_keys_and_noncanonical_json( + tmp_path: Path, +) -> None: + duplicate = tmp_path / "duplicate.json" + duplicate.write_text('{"schema":"a","schema":"b"}', encoding="utf-8") + with pytest.raises(ValueError, match="duplicate JSON key"): + load_candidate_bundle(duplicate) + + noncanonical = tmp_path / "noncanonical.json" + bound = bind_fingerprint(_sample_bundle()) + noncanonical.write_text(json.dumps(bound, indent=2), encoding="utf-8") + with pytest.raises(ValueError, match="not in canonical JSON form"): + load_candidate_bundle(noncanonical) + + +def test_candidate_fingerprint_and_plan_binding_detect_tampering(tmp_path: Path) -> None: + candidate_path = tmp_path / "candidates.json" + bundle = write_candidate_bundle(candidate_path, _sample_bundle()) + tampered = dict(bundle) + tampered["profile"] = "tampered" + candidate_path.write_bytes(canonical_json_bytes(tampered)) + + with pytest.raises(ValueError, match="fingerprint mismatch"): + load_candidate_bundle(candidate_path) + + +def test_plan_verifier_rejects_changed_candidate_file(tmp_path: Path) -> None: + candidate_path = tmp_path / "candidates.json" + bundle = write_candidate_bundle(candidate_path, _sample_bundle()) + optimization = solve_candidate_bundle(bundle, budget_bytes=2080) + plan_path = tmp_path / "plan.json" + write_plan( + plan_path, + build_plan_payload( + candidate_path, + bundle, + objective=ROBUST_OBJECTIVE, + optimization=optimization, + ), + ) + + replacement = _sample_bundle() + replacement["profile"] = "replacement" + candidate_path.unlink() + write_candidate_bundle(candidate_path, replacement) + + with pytest.raises(ValueError, match="candidate file hash mismatch"): + verify_plan(candidate_path, plan_path) + + +def test_candidate_environment_inventory_is_part_of_contract(tmp_path: Path) -> None: + payload = _sample_bundle() + first_candidate = payload["units"][0]["candidates"][0] # type: ignore[index] + first_candidate["per_environment_distortion"] = { # type: ignore[index] + "general": 0.0, + "unexpected": 0.0, + } + + with pytest.raises(ValueError, match="environments must exactly match"): + write_candidate_bundle(tmp_path / "bad.json", payload) + + +def test_candidate_bundle_rejects_duplicate_candidates_and_nonadditive_metrics( + tmp_path: Path, +) -> None: + duplicate = _sample_bundle() + duplicate["units"][0]["candidates"].append( # type: ignore[index,union-attr] + dict(duplicate["units"][0]["candidates"][0]) # type: ignore[index,arg-type] + ) + with pytest.raises(ValueError, match="candidate inventory disagrees"): + write_candidate_bundle(tmp_path / "duplicate.json", duplicate) + + nonadditive = _sample_bundle() + nonadditive["distortion_metric"]["additive_across_units"] = False # type: ignore[index] + with pytest.raises(ValueError, match="additive_across_units must be true"): + write_candidate_bundle(tmp_path / "nonadditive.json", nonadditive) + + +def test_candidate_bundle_totals_are_strict_positive_integers(tmp_path: Path) -> None: + payload = _sample_bundle() + payload["totals"]["dense16"] = 4096.0 # type: ignore[index] + + with pytest.raises(ValueError, match="must be a positive integer"): + write_candidate_bundle(tmp_path / "float-total.json", payload) + + +def test_candidate_bundle_rejects_export_claim_and_byte_component_drift( + tmp_path: Path, +) -> None: + exported = _sample_bundle() + exported["scope"]["checkpoint_export"] = True # type: ignore[index] + with pytest.raises(ValueError, match="cannot claim checkpoint export"): + write_candidate_bundle(tmp_path / "exported.json", exported) + + drifted = _sample_bundle() + candidate = drifted["units"][0]["candidates"][0] # type: ignore[index] + candidate["metadata"].update( # type: ignore[index,union-attr] + { + "logical_code_bytes": 2047, + "logical_scale_bytes": 0, + } + ) + with pytest.raises(ValueError, match="logical byte components do not sum"): + write_candidate_bundle(tmp_path / "byte-drift.json", drifted) + + +def test_transformer_block_aggregation_is_content_bound_and_conserves_totals( + tmp_path: Path, +) -> None: + candidate_path = tmp_path / "candidates.json" + source = _sample_bundle() + source["profile"] = "experiment-001" + bundle = write_candidate_bundle(candidate_path, source) + + aggregated = aggregate_experiment001_transformer_blocks(candidate_path, bundle) + + assert aggregated["profile"] == TRANSFORMER_BLOCK_PROFILE + assert aggregated["scope"]["allocation_granularity"] == "transformer-block" + assert aggregated["scope"]["allocation_unit_count"] == 2 + assert aggregated["totals"] == bundle["totals"] + assert ( + aggregated["source"]["parent_candidate_bundle"]["fingerprint_sha256"] + == (bundle["fingerprint_sha256"]) + ) + assert [unit["unit_id"] for unit in aggregated["units"]] == [ + "model.language_model.layers.0", + "model.language_model.layers.1", + ] diff --git a/tests/test_autopolicy_cli.py b/tests/test_autopolicy_cli.py new file mode 100644 index 0000000..b538ab3 --- /dev/null +++ b/tests/test_autopolicy_cli.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from cliffquant.autopolicy_artifacts import CANDIDATE_SCHEMA, write_candidate_bundle +from cliffquant.autopolicy_cli import main as autopolicy_main +from cliffquant.cli import main as cli_main + + +def _bundle() -> dict[str, object]: + return { + "schema": CANDIDATE_SCHEMA, + "status": "complete", + "profile": "cli-test", + "scope": { + "budget_scope": "logical-target-weight-and-scale-payload", + "checkpoint_export": False, + }, + "source": {"kind": "synthetic"}, + "environments": ["general", "code"], + "distortion_metric": { + "name": "synthetic", + "additive_across_units": True, + "downstream_metric": False, + }, + "candidate_definitions": {"dense16": {}, "w4": {}}, + "units": [ + { + "unit_id": "layer.0", + "weight_key": "layer.0.weight", + "shape": [8, 128], + "group_count": 8, + "candidates": [ + { + "candidate_id": "dense16", + "logical_bytes": 2048, + "groupwise_max_distortion": 0.0, + "per_environment_distortion": { + "general": 0.0, + "code": 0.0, + }, + "metadata": {"bits": 16}, + }, + { + "candidate_id": "w4", + "logical_bytes": 528, + "groupwise_max_distortion": 2.0, + "per_environment_distortion": { + "general": 2.0, + "code": 1.0, + }, + "metadata": {"bits": 4}, + }, + ], + "evidence": {"kind": "synthetic"}, + } + ], + "totals": {"dense16": 2048, "w4": 528}, + "builder": {"name": "cli-test"}, + } + + +def test_top_level_and_autopolicy_help(capsys) -> None: + try: + cli_main(["--help"]) + except SystemExit as exc: + assert exc.code == 0 + top_help = capsys.readouterr().out + assert "autopolicy" in top_help + + try: + cli_main(["autopolicy", "--help"]) + except SystemExit as exc: + assert exc.code == 0 + autopolicy_help = capsys.readouterr().out + assert "build-experiment-001" in autopolicy_help + assert "aggregate-blocks" in autopolicy_help + assert "does not export a runnable mixed-bit checkpoint" in autopolicy_help + + +def test_solve_and_verify_cli_round_trip(tmp_path: Path, capsys) -> None: + candidates = tmp_path / "candidates.json" + plan = tmp_path / "plan.json" + write_candidate_bundle(candidates, _bundle()) + + exit_code = autopolicy_main( + [ + "solve", + "--candidates", + str(candidates), + "--budget-bytes", + "528", + "--output", + str(plan), + ] + ) + assert exit_code == 0 + solve_output = json.loads(capsys.readouterr().out) + assert solve_output["checkpoint_export"] is False + assert solve_output["optimization"]["used_bytes"] == 528 + assert solve_output["optimization"]["decisions"][0]["candidate_id"] == "w4" + + exit_code = autopolicy_main( + [ + "verify", + "--candidates", + str(candidates), + "--plan", + str(plan), + ] + ) + assert exit_code == 0 + verify_output = json.loads(capsys.readouterr().out) + assert verify_output["status"] == "pass" + assert verify_output["plan"]["status"] == "pass" + + +def test_cli_reports_infeasible_budget_without_writing_plan( + tmp_path: Path, + capsys, +) -> None: + candidates = tmp_path / "candidates.json" + plan = tmp_path / "plan.json" + write_candidate_bundle(candidates, _bundle()) + + exit_code = autopolicy_main( + [ + "solve", + "--candidates", + str(candidates), + "--budget-bytes", + "527", + "--output", + str(plan), + ] + ) + + assert exit_code == 2 + assert "infeasible" in capsys.readouterr().err + assert not plan.exists() + + +def test_cli_refuses_to_overwrite_plan(tmp_path: Path, capsys) -> None: + candidates = tmp_path / "candidates.json" + plan = tmp_path / "plan.json" + write_candidate_bundle(candidates, _bundle()) + arguments = [ + "solve", + "--candidates", + str(candidates), + "--budget-bytes", + "528", + "--output", + str(plan), + ] + + assert autopolicy_main(arguments) == 0 + capsys.readouterr() + assert autopolicy_main(arguments) == 2 + assert "refusing to overwrite" in capsys.readouterr().err + + +def test_aggregate_blocks_cli_writes_content_bound_candidate_view( + tmp_path: Path, + capsys, +) -> None: + candidates = tmp_path / "candidates.json" + grouped = tmp_path / "grouped.json" + payload = _bundle() + payload["profile"] = "experiment-001" + payload["units"][0]["unit_id"] = "model.language_model.layers.0.mlp.up_proj" # type: ignore[index] + payload["units"][0]["weight_key"] = ( # type: ignore[index] + "model.language_model.layers.0.mlp.up_proj.weight" + ) + write_candidate_bundle(candidates, payload) + + exit_code = autopolicy_main( + [ + "aggregate-blocks", + "--candidates", + str(candidates), + "--output", + str(grouped), + ] + ) + + assert exit_code == 0 + output = json.loads(capsys.readouterr().out) + assert output["profile"] == "experiment-001-transformer-blocks" + assert output["allocation_unit_count"] == 1 + assert grouped.exists() diff --git a/tests/test_autopolicy_experiment001.py b/tests/test_autopolicy_experiment001.py new file mode 100644 index 0000000..4ca94ad --- /dev/null +++ b/tests/test_autopolicy_experiment001.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +import itertools +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +import cliffquant.autopolicy_experiment001 as experiment001 +from cliffquant.autopolicy_experiment001 import ( + _evaluate_scale_rows, + build_experiment001_candidates, + logical_payload_bytes, + vectorized_w8_absmax_rows, + verify_experiment001_evidence, +) +from cliffquant.fp16_grid import positive_finite_fp16_scales +from cliffquant.full_model_artifacts import POLICY_CLIFFQUANT + + +def _scalar_w8_oracle( + weights: np.ndarray, + curvature: np.ndarray, + *, + group_size: int, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + grid = positive_finite_fp16_scales() + rows = weights.shape[0] + groups_per_row = weights.shape[1] // group_size + bits = np.empty((rows, groups_per_row), dtype=np.uint16) + objectives = np.empty((rows, groups_per_row), dtype=np.float64) + environment_totals = np.zeros(curvature.shape[0], dtype=np.float64) + for row, group in itertools.product(range(rows), range(groups_per_row)): + start = group * group_size + stop = start + group_size + values = np.asarray(weights[row, start:stop], dtype=np.float64) + positive = values[values > 0] + negative = values[values < 0] + target = max( + float(np.max(positive)) / 127.0 if positive.size else 0.0, + float(-np.min(negative)) / 128.0 if negative.size else 0.0, + ) + distances = np.abs(grid - target) + scale_index = int(np.flatnonzero(distances == np.min(distances))[0]) + scale = float(grid[scale_index]) + bits[row, group] = scale_index + 1 + codes = np.clip(np.rint(values / scale), -128, 127) + squared_error = np.square(values - scale * codes) + losses = np.asarray( + [ + np.sum( + curvature[environment, start:stop] * squared_error, + dtype=np.float64, + ) + for environment in range(curvature.shape[0]) + ] + ) + objectives[row, group] = float(np.max(losses)) + environment_totals += losses + return bits, objectives, environment_totals + + +def test_logical_payload_byte_formula() -> None: + assert logical_payload_bytes(out_features=8, in_features=128, bits=4) == { + "codes": 512, + "scales": 16, + "total": 528, + } + assert logical_payload_bytes(out_features=8, in_features=128, bits=8) == { + "codes": 1024, + "scales": 16, + "total": 1040, + } + assert logical_payload_bytes(out_features=8, in_features=128, bits=16) == { + "codes": 2048, + "scales": 0, + "total": 2048, + } + + +@pytest.mark.parametrize("seed", range(12)) +def test_vectorized_w8_matches_independent_scalar_oracle(seed: int) -> None: + rng = np.random.default_rng(seed) + weights = rng.normal(size=(3, 16)).astype(np.float64) + curvature = rng.uniform(0.01, 2.0, size=(4, 16)).astype(np.float64) + + expected = _scalar_w8_oracle(weights, curvature, group_size=8) + actual = vectorized_w8_absmax_rows(weights, curvature, group_size=8) + + assert np.array_equal(actual[0], expected[0]) + assert np.allclose(actual[1], expected[1], rtol=1e-14, atol=1e-18) + assert np.allclose(actual[2], expected[2], rtol=1e-14, atol=1e-18) + + +@pytest.mark.parametrize( + "values", + [ + np.zeros(8), + np.arange(1.0, 9.0), + -np.arange(1.0, 9.0), + np.asarray([-128.0, -1.0, 0.0, 1.0, 127.0, 3.5, -2.5, 0.25]), + ], +) +def test_w8_edge_sign_patterns_are_finite(values: np.ndarray) -> None: + bits, objectives, totals = vectorized_w8_absmax_rows( + values.reshape(1, 8), + np.ones((2, 8)), + group_size=8, + ) + + assert bits.shape == (1, 1) + assert 0 < int(bits[0, 0]) <= 0x7BFF + assert np.all(np.isfinite(objectives)) + assert np.all(objectives >= 0.0) + assert np.all(np.isfinite(totals)) + + +def test_chunking_does_not_change_w8_scores() -> None: + rng = np.random.default_rng(2339) + weights = rng.normal(size=(7, 16)) + curvature = rng.uniform(0.1, 1.5, size=(3, 16)) + whole = vectorized_w8_absmax_rows(weights, curvature, group_size=8) + chunks = [ + vectorized_w8_absmax_rows(weights[start:stop], curvature, group_size=8) + for start, stop in ((0, 2), (2, 5), (5, 7)) + ] + + assert np.array_equal(np.concatenate([item[0] for item in chunks]), whole[0]) + assert np.allclose(np.concatenate([item[1] for item in chunks]), whole[1]) + assert np.allclose(sum((item[2] for item in chunks), np.zeros(3)), whole[2]) + + +def test_direct_scale_evaluation_uses_requested_integer_range() -> None: + weights = np.asarray([[7.6, -8.6, 0.5, -0.5, 1.5, -1.5, 0.0, 2.0]]) + curvature = np.ones((2, 8)) + one_bits = np.asarray([[0x3C00]], dtype=np.uint16) + + w4, _ = _evaluate_scale_rows( + weights, + curvature, + one_bits, + qmin=-8, + qmax=7, + group_size=8, + ) + w8, _ = _evaluate_scale_rows( + weights, + curvature, + one_bits, + qmin=-128, + qmax=127, + group_size=8, + ) + + assert float(w8[0, 0]) < float(w4[0, 0]) + + +@pytest.mark.parametrize( + ("weights", "curvature", "message"), + [ + (np.ones((1, 7)), np.ones((2, 7)), "divisible"), + (np.ones((1, 8)), np.ones((2, 7)), "feature width"), + (np.asarray([[float("nan")] * 8]), np.ones((2, 8)), "weights must be finite"), + (np.ones((1, 8)), -np.ones((2, 8)), "curvature"), + ], +) +def test_w8_rejects_invalid_inputs( + weights: np.ndarray, + curvature: np.ndarray, + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + vectorized_w8_absmax_rows(weights, curvature, group_size=8) + + +def test_candidate_builder_and_evidence_verifier_end_to_end_with_frozen_fakes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + rng = np.random.default_rng(2339) + weights = rng.uniform(-0.75, 0.75, size=(2, 128)).astype(np.float64) + environments = ("general", "code") + curvature = rng.uniform(0.1, 1.2, size=(2, 128)).astype(np.float64) + scale_bits = np.full((2, 1), 0x3800, dtype=np.uint16) + w4_objectives, _ = _evaluate_scale_rows( + weights, + curvature, + scale_bits, + qmin=-8, + qmax=7, + ) + module = SimpleNamespace( + name="model.language_model.layers.0.mlp.up_proj", + weight_key="model.language_model.layers.0.mlp.up_proj.weight", + out_features=2, + in_features=128, + layer=0, + branch="mlp.up_proj", + ) + modules = (module,) + provenance = {"model_id": "frozen-test", "revision": "frozen-revision"} + calibration = SimpleNamespace( + environments=environments, + modules=modules, + normalized={module.name: curvature}, + source={"model": provenance}, + metadata_sha256="1" * 64, + archive_sha256="2" * 64, + ) + scale_run = SimpleNamespace( + modules=modules, + policy=POLICY_CLIFFQUANT, + metadata={ + "base_model": {"source": provenance}, + "corpus_replay": {"status": "pass"}, + }, + artifacts={ + module.name: SimpleNamespace( + scale_bits=scale_bits, + objectives=w4_objectives, + archive_sha256="3" * 64, + metadata_sha256="4" * 64, + ) + }, + manifest_sha256="5" * 64, + ) + + class FakeWeightSource: + def __init__(self, _snapshot: str | Path) -> None: + self.provenance = provenance + self.keys = frozenset({module.weight_key}) + + def read_rows(self, key: str, start: int, stop: int) -> np.ndarray: + assert key == module.weight_key + return weights[start:stop] + + monkeypatch.setattr(experiment001, "load_scale_run", lambda *_args, **_kwargs: scale_run) + monkeypatch.setattr( + experiment001, + "load_calibration_diagonals", + lambda *_args, **_kwargs: calibration, + ) + monkeypatch.setattr(experiment001, "NumpySafetensorsWeightSource", FakeWeightSource) + + output = tmp_path / "autopolicy" + bundle = build_experiment001_candidates( + base_model_dir=tmp_path / "snapshot", + calibration_metadata=tmp_path / "calibration.json", + w4_scale_run=tmp_path / "scale-run", + output_dir=output, + row_chunk_size=1, + ) + verification = verify_experiment001_evidence(output / "candidates.json", bundle) + + assert bundle["scope"]["module_count"] == 1 + assert bundle["scope"]["weight_count"] == 256 + assert bundle["totals"] == { + "w4_cliffquant": 132, + "w8_absmax": 260, + "dense16": 512, + } + assert verification["status"] == "pass" + assert verification["w8_archives_checked"] == 1 + assert (output / "modules" / "module-00-mlp-up_proj.w8.npz").is_file() + + with pytest.raises(FileExistsError, match="refusing to reuse"): + build_experiment001_candidates( + base_model_dir=tmp_path / "snapshot", + calibration_metadata=tmp_path / "calibration.json", + w4_scale_run=tmp_path / "scale-run", + output_dir=output, + ) diff --git a/tests/test_autopolicy_frontier.py b/tests/test_autopolicy_frontier.py new file mode 100644 index 0000000..ddbbf83 --- /dev/null +++ b/tests/test_autopolicy_frontier.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path + +import pytest + +from cliffquant.autopolicy_artifacts import CANDIDATE_SCHEMA, write_candidate_bundle +from scripts.generate_autopolicy_frontier import ( + build_provenance_manifest, + generate, + validate_frontier, + verify_provenance_manifest, +) + + +def _grouped_bundle() -> dict[str, object]: + units = [] + for index in range(2): + units.append( + { + "unit_id": f"model.language_model.layers.{index}", + "weight_key": f"aggregate:model.language_model.layers.{index}", + "shape": [1, 400], + "group_count": 4, + "candidates": [ + { + "candidate_id": "dense16", + "logical_bytes": 200, + "groupwise_max_distortion": 0.0, + "per_environment_distortion": {"general": 0.0, "code": 0.0}, + "metadata": {"bits": 16}, + }, + { + "candidate_id": "w4_cliffquant", + "logical_bytes": 50, + "groupwise_max_distortion": 5.0 + index, + "per_environment_distortion": { + "general": 5.0 + index, + "code": 4.0 + index, + }, + "metadata": {"bits": 4}, + }, + { + "candidate_id": "w8_absmax", + "logical_bytes": 100, + "groupwise_max_distortion": 1.0 + index, + "per_environment_distortion": { + "general": 1.0 + index, + "code": 0.8 + index, + }, + "metadata": {"bits": 8}, + }, + ], + "evidence": {"aggregation": "transformer-block"}, + } + ) + return { + "schema": CANDIDATE_SCHEMA, + "status": "complete", + "profile": "experiment-001-transformer-blocks", + "scope": { + "weight_count": 800, + "budget_scope": "logical-target-weight-and-scale-payload", + "checkpoint_export": False, + "allocation_granularity": "transformer-block", + "allocation_unit_count": 2, + }, + "source": {"kind": "synthetic"}, + "environments": ["general", "code"], + "distortion_metric": { + "name": "synthetic", + "additive_across_units": True, + "downstream_metric": False, + }, + "candidate_definitions": { + "dense16": {}, + "w4_cliffquant": {}, + "w8_absmax": {}, + }, + "units": units, + "totals": { + "dense16": 400, + "w4_cliffquant": 100, + "w8_absmax": 200, + }, + "builder": {"name": "unit-test"}, + } + + +def test_frontier_generation_is_deterministic_and_self_consistent(tmp_path: Path) -> None: + candidate_path = tmp_path / "candidates.json" + write_candidate_bundle(candidate_path, _grouped_bundle()) + + first = generate(candidate_path, max_frontier_states=1_000) + second = generate(candidate_path, max_frontier_states=1_000) + + assert first == second + validate_frontier(first) + assert first["allocation_granularity"] == "transformer-block" + assert first["points"][0]["selection_counts"]["w4_cliffquant"] == 2 + assert first["points"][-1]["selection_counts"]["dense16"] == 2 + + +def test_frontier_validator_rejects_selection_count_drift(tmp_path: Path) -> None: + candidate_path = tmp_path / "candidates.json" + write_candidate_bundle(candidate_path, _grouped_bundle()) + payload = generate(candidate_path, max_frontier_states=1_000) + tampered = deepcopy(payload) + tampered["points"][0]["selection_counts"]["w4_cliffquant"] = 1 + + with pytest.raises(ValueError, match="selection counts"): + validate_frontier(tampered) + + +def test_figure_provenance_binds_input_frontier_figure_and_generator(tmp_path: Path) -> None: + candidate_path = tmp_path / "candidates.json" + frontier_path = tmp_path / "frontier.json" + figure_path = tmp_path / "frontier.png" + manifest_path = tmp_path / "frontier.manifest.json" + write_candidate_bundle(candidate_path, _grouped_bundle()) + payload = generate(candidate_path, max_frontier_states=1_000) + frontier_path.write_text("frontier\n", encoding="utf-8") + figure_path.write_bytes( + b"\x89PNG\r\n\x1a\n" + + b"\x00\x00\x00\rIHDR" + + (640).to_bytes(4, "big") + + (480).to_bytes(4, "big") + ) + manifest = build_provenance_manifest( + candidate_path=candidate_path, + frontier_path=frontier_path, + figure_path=figure_path, + manifest_path=manifest_path, + candidate_fingerprint_sha256=payload["candidate_fingerprint_sha256"], + ) + from cliffquant.provenance import canonical_json_bytes + + manifest_path.write_bytes(canonical_json_bytes(manifest)) + + assert verify_provenance_manifest(manifest_path) == manifest + + frontier_path.write_text("tampered\n", encoding="utf-8") + with pytest.raises(ValueError, match="provenance mismatch"): + verify_provenance_manifest(manifest_path) + + +def test_checked_in_figure_provenance_revalidates() -> None: + root = Path(__file__).resolve().parents[1] + manifest_path = root / "figures" / "autopolicy-logical-byte-frontier.manifest.json" + if not manifest_path.is_file(): + pytest.skip("checked-in AutoPolicy figure provenance is not present") + + payload = verify_provenance_manifest(manifest_path) + assert payload["outputs"]["figure"]["width_px"] == 2052 diff --git a/tests/test_autopolicy_release.py b/tests/test_autopolicy_release.py new file mode 100644 index 0000000..6f79697 --- /dev/null +++ b/tests/test_autopolicy_release.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path + +import pytest + +from cliffquant.autopolicy_artifacts import bind_fingerprint, load_candidate_bundle +from scripts.validate_autopolicy_release import ( + _validate_frontier, + _validate_grouped_candidates, + check_manifest, +) + +RELEASE_ROOT = Path(__file__).resolve().parents[1] / "research" / "results" / "autopolicy-001" + +pytestmark = pytest.mark.skipif( + not (RELEASE_ROOT / "manifest.json").is_file(), + reason="portable AutoPolicy evidence is not included in this source distribution", +) + + +def test_checked_in_autopolicy_release_revalidates() -> None: + manifest = check_manifest(RELEASE_ROOT) + + assert manifest["status"] == "pass" + assert manifest["candidate"]["module_count"] == 150 + assert manifest["candidate"]["evidence_verification"]["w8_archives_checked"] == 150 + assert manifest["transformer_block_view"]["allocation_unit_count"] == 24 + assert manifest["frontier"]["point_count"] == 10 + assert manifest["representative_plan"]["status"] == "pass" + + +def test_grouped_candidates_fail_if_not_exact_matrix_aggregation() -> None: + matrix_path = RELEASE_ROOT / "candidates.json" + matrix = load_candidate_bundle(matrix_path) + grouped = load_candidate_bundle(RELEASE_ROOT / "candidates-transformer-blocks.json") + tampered = deepcopy(grouped) + candidate = tampered["units"][0]["candidates"][0] + environment = max( + candidate["per_environment_distortion"], + key=candidate["per_environment_distortion"].get, + ) + candidate["per_environment_distortion"][environment] += 1.0 + candidate["groupwise_max_distortion"] = max(candidate["per_environment_distortion"].values()) + tampered = bind_fingerprint(tampered) + + with pytest.raises(ValueError, match="exact aggregation"): + _validate_grouped_candidates(matrix_path, matrix, tampered) + + +def test_frontier_fails_if_any_published_point_is_not_exact() -> None: + grouped_path = RELEASE_ROOT / "candidates-transformer-blocks.json" + grouped = load_candidate_bundle(grouped_path) + frontier = json.loads((RELEASE_ROOT / "frontier.json").read_text(encoding="utf-8")) + tampered = deepcopy(frontier) + tampered["points"][1]["worst_environment_distortion"] += 1.0 + + with pytest.raises(ValueError, match="exact recomputation"): + _validate_frontier(tampered, grouped_path=grouped_path, grouped=grouped)