diff --git a/.github/workflows/thesis-facts-append.yml b/.github/workflows/thesis-facts-append.yml index 73c6cca..2685fbd 100644 --- a/.github/workflows/thesis-facts-append.yml +++ b/.github/workflows/thesis-facts-append.yml @@ -18,13 +18,67 @@ on: pull_request: branches: - codex/thesis-ledger-facts + # When this definition is installed on the repository default branch, this + # base-owned event can be the trust root for a required PR workflow. The + # ordinary pull_request workflow below still tests the merge ref, but its YAML + # is candidate-controlled and cannot by itself prevent a PR from stubbing the + # job that invokes the detached base gate. No paths filter here either: a + # required check that a paths filter can skip deadlocks on "Expected" for + # out-of-scope PRs. + pull_request_target: + branches: + - codex/thesis-ledger-facts push: branches: - codex/thesis-ledger-facts +permissions: + contents: read + jobs: + trusted-base-append-gate: + name: Trusted base append gate + if: github.event_name == 'pull_request_target' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Judge the PR merge tree with the base gate + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + [[ "$BASE_SHA" =~ ^[0-9a-f]{40,64}$ ]] + [[ "$MERGE_SHA" =~ ^[0-9a-f]{40,64}$ ]] + [[ "$PR_NUMBER" =~ ^[0-9]+$ ]] + + repository_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" + candidate="$RUNNER_TEMP/thesis-facts-candidate" + base_gate="$RUNNER_TEMP/thesis-facts-base-gate" + + git clone --no-checkout "$repository_url" "$candidate" + git -C "$candidate" fetch --no-tags origin \ + "+refs/pull/${PR_NUMBER}/merge:refs/remotes/origin/pr-merge" + git -C "$candidate" checkout --detach refs/remotes/origin/pr-merge + test "$(git -C "$candidate" rev-parse HEAD)" = "$MERGE_SHA" + git -C "$candidate" cat-file -e "${BASE_SHA}^{commit}" + + git clone --no-checkout "$repository_url" "$base_gate" + git -C "$base_gate" fetch --no-tags origin "$BASE_SHA" + git -C "$base_gate" checkout --detach "$BASE_SHA" + test "$(git -C "$base_gate" rev-parse HEAD)" = "$BASE_SHA" + + cd "$RUNNER_TEMP" + PYTHONPATH="$base_gate/scripts" \ + PYTHONNOUSERSITE=1 \ + python3 "$base_gate/scripts/check_thesis_facts_append.py" \ + --root "$candidate" \ + --base-ref "$BASE_SHA" + append-gate: name: Append gate + if: github.event_name != 'pull_request_target' runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -38,9 +92,22 @@ jobs: set -euo pipefail if [ "${{ github.event_name }}" = "pull_request" ]; then base_sha="${{ github.event.pull_request.base.sha }}" - git cat-file -e "${base_sha}^{commit}" - python3 scripts/check_thesis_facts_append.py \ - --base-ref "$base_sha" + base_gate="$RUNNER_TEMP/thesis-facts-base-gate" + + git -C "$GITHUB_WORKSPACE" cat-file -e "${base_sha}^{commit}" + git clone --no-checkout \ + "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" \ + "$base_gate" + git -C "$base_gate" fetch --no-tags --depth 1 origin "$base_sha" + git -C "$base_gate" checkout --detach "$base_sha" + test "$(git -C "$base_gate" rev-parse HEAD)" = "$base_sha" + + cd "$RUNNER_TEMP" + PYTHONPATH="$base_gate/scripts" \ + PYTHONNOUSERSITE=1 \ + python3 "$base_gate/scripts/check_thesis_facts_append.py" \ + --root "$GITHUB_WORKSPACE" \ + --base-ref "$base_sha" else python3 scripts/check_thesis_facts_append.py fi diff --git a/pyproject.toml b/pyproject.toml index cb05188..ff835bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ authors = [ ] dependencies = [ + "cryptography>=46.0.3", "sqlmodel>=0.0.22", "supabase>=2.0.0", # Supabase Python client "pyyaml>=6.0", diff --git a/releases/README.md b/releases/README.md index 054788d..3156bcf 100644 --- a/releases/README.md +++ b/releases/README.md @@ -2,9 +2,11 @@ In the gated append flow, each proposed state of `ledger/official_observations.jsonl` is committed by a canonical release manifest -and witnessed by two RFC 3161 timestamp authorities. The manifests form an +and witnessed by two RFC 3161 timestamp authorities. The producer also signs the +manifest's exact bytes with the pinned Ed25519 release key. The manifests form an append-only hash chain. The receipts let a verifier show that the exact manifest -bytes existed no later than each receipt's `genTime`. +bytes existed no later than each receipt's `genTime`; the producer signature +shows that the pinned release key signed those bytes. ## Files @@ -15,17 +17,22 @@ the manifest file's SHA-256 digest, a release consists of exactly these files: - `releases/manifests/.json` - `releases/manifests/.freetsa.tsr` - `releases/manifests/.digicert.tsr` +- `releases/manifests/.producer.sig` For example, the receipt for manifest `0007-0123456789abcdef.json` is named `0007-0123456789abcdef.freetsa.tsr`, not `0007-0123456789abcdef.json.freetsa.tsr`. Receipts are DER-encoded RFC 3161 responses whose SHA-256 message imprint covers the manifest file's exact bytes. Those bytes are canonical JSON produced by `scripts/canonical_json.py`, followed -by one newline. +by one newline. The producer sibling is a raw, DER-free 64-byte Ed25519 signature +over those same exact bytes. The pinned TSA verification chains are committed as PEM files under -`releases/anchors/`. Verification does not contact either TSA or any other -network service. +`releases/anchors/`. The producer public key is committed there as +`producer-ed25519.pub`; the verifier pins the SHA-256 digest of its DER +SubjectPublicKeyInfo to +`4a90eff40455ce0d853d4bab1608efbdae1efaf8c06054ead6e396c5b0c4846e`. +Verification does not contact either TSA or any other network service. ## Manifest schema @@ -57,7 +64,9 @@ Genesis has index zero, a null previous hash, and a null append block. Every later release increments the index by one, hashes the preceding manifest file, increases the line count, and binds both the row-count delta and the exact byte suffix in its append block. Both receipts must verify against their separately -pinned anchor chain and cover that release's exact manifest bytes. +pinned anchor chain and cover that release's exact manifest bytes. Every release, +including genesis, must also carry a valid signature from the pinned producer +key. ## Offline verification @@ -67,11 +76,15 @@ Clone the repository at the state you want to inspect and run: python3 scripts/verify_release_chain.py --full ``` -The verifier needs only Python, OpenSSL, the committed manifests and receipts, -the committed TSA anchors, and the ledger files. It checks canonical bytes, +The verifier needs Python, OpenSSL, the committed manifests, signatures and +receipts, the committed anchors, and the ledger files. The project environment's +`cryptography` dependency provides portable Ed25519 verification; when that +package is unavailable, the verifier falls back to OpenSSL 3.0 or newer because +Ed25519 is a one-shot `pkeyutl -rawin` operation. It checks canonical bytes, filenames, contiguous indices, previous-manifest links, state and append -commitments, both timestamp receipts, and timestamp ordering. Any mismatch exits -nonzero with an error identifying the failed invariant. +commitments, the producer signature, both timestamp receipts, and timestamp +ordering. Any mismatch exits nonzero with an error identifying the failed +invariant. Retain a trusted checkpoint outside this repository, at minimum the full SHA-256 digest of a previously accepted head manifest, and compare it with later clones. @@ -83,11 +96,12 @@ do not publish a uniqueness-enforcing append-only ledger for this repository. ## Security properties and limits Under the configured proposal gate, every ledger append arrives with the next -manifest and both receipts. This binds the proposed ledger state, immutable -prefix, previous release, row-count change, and exact appended byte suffix into a -witnessed chain. Rewriting a checkpointed manifest or producing an unwitnessed or -internally inconsistent state is therefore detectable by any verifier holding -that checkpoint. +manifest, both receipts, and its producer signature. This binds the proposed +ledger state, immutable prefix, previous release, row-count change, and exact +appended byte suffix into a witnessed and producer-authenticated chain. Rewriting +a checkpointed manifest or producing an unwitnessed, unsigned, or internally +inconsistent state is therefore detectable by any verifier holding that +checkpoint. An RFC 3161 `genTime` establishes that the manifest existed no later than that time. It does **not** timestamp GitHub's merge or prove when the organization @@ -95,25 +109,39 @@ accepted the proposal. In the intended pull-request flow the receipts are create before the gate can pass and therefore before merge; their times do not upper-bound the later acceptance time. -This mechanism provides tamper evidence, not multi-party authorization or admin -non-repudiation. Governance remains within one GitHub organization: +The Ed25519 signature proves that the pinned producer key signed the manifest. It +does not prove that the manifest's claims are correct, that the ledger append was +properly reviewed, or that GitHub accepted it at a particular time. The overall +mechanism provides tamper evidence and producer identity, not multi-party +authorization. Governance remains within one GitHub organization: - It does not require approval by an independent institution or a second party. - An unwitnessed or inconsistent admin direct push turns verification and CI red, but repository controls cannot prevent every admin bypass. -- An admin direct push that includes a valid next manifest and both valid receipts - is neither prevented nor distinguishable by offline cryptographic verification - from an append merged through the intended pull-request path. -- The verifier, workflows, and anchors live in the repository they verify. Running - the verifier from a clone assumes those security files were not weakened in the - same rewrite; independent verification should pin or audit them separately as - well as retaining a manifest checkpoint. +- An admin direct push that includes a valid next manifest, receipts, and producer + signature is neither prevented nor distinguishable by offline cryptographic + verification from an append merged through the intended pull-request path. +- Pull requests are split into data and gate classes. A data pull request cannot + change the verifier, cutter, canonicalizer, append workflow, or anchors. The + ordinary pull-request job runs the base commit's copies of those files against + the proposed merge tree, but its workflow definition is candidate-controlled + and is therefore test feedback rather than a complete trust root. The + `Trusted base append gate` uses `pull_request_target` without executing + candidate code. GitHub loads that event's workflow from the repository default + branch, so this workflow must also be installed there, or bound as an + organization ruleset's required workflow, before that check is active. Do not + rely only on a name-based required status: candidate workflow YAML can emit a + job with the same display name. A gate-only pull request can change the judge + only for later data pull requests. Review controls and external checkpoints + therefore remain necessary: a later gate change or an admin rewrite can still + weaken future enforcement. - The manifest chain hashes manifests, not receipt files. A receipt can be replaced by a newer valid receipt over the same manifest without changing later manifest hashes unless its exact bytes or digest were retained externally. -- The production anchors are fixed. There is no in-schema anchor-rotation - protocol, so a TSA chain change requires an explicit verifier/schema migration - that continues to preserve verification of the old chain. +- The production anchors and producer key are fixed. There is no in-schema key- or + anchor-rotation protocol, so a producer-key or TSA-chain change requires an + explicit verifier/schema migration that continues to preserve verification of + the old chain. - Four-digit indices cap this filename format at release 9999; the current schema does not define a rollover. diff --git a/releases/anchors/producer-ed25519.pub b/releases/anchors/producer-ed25519.pub new file mode 100644 index 0000000..575aa6e --- /dev/null +++ b/releases/anchors/producer-ed25519.pub @@ -0,0 +1,3 @@ +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEAQ2V5V3Sz8sLp0K/JfxDX25nE5AnW8/6uIbJZWsZzrKI= +-----END PUBLIC KEY----- diff --git a/releases/manifests/0000-307cedbc91de43be.producer.sig b/releases/manifests/0000-307cedbc91de43be.producer.sig new file mode 100644 index 0000000..ae0c1c0 --- /dev/null +++ b/releases/manifests/0000-307cedbc91de43be.producer.sig @@ -0,0 +1 @@ +MDՋn'zrO3ZiVFy&>1gӸ ڰo֖:2Lʀ㔽J \ No newline at end of file diff --git a/scripts/check_thesis_facts_append.py b/scripts/check_thesis_facts_append.py index 08e8711..df1a4a1 100644 --- a/scripts/check_thesis_facts_append.py +++ b/scripts/check_thesis_facts_append.py @@ -18,8 +18,9 @@ the later row's ``assertionVersion.supersedes`` must name the version ID of the row it replaces. - after witnessed genesis, every byte append carries exactly one next canonical - release manifest and two independently anchored RFC 3161 receipts; all prior - release files remain byte-immutable against the PR's base commit. + release manifest, its producer signature, and two independently anchored + RFC 3161 receipts; all prior release files remain byte-immutable against the + PR's base commit. Usage: python3 scripts/check_thesis_facts_append.py [--base-ref REF] @@ -33,6 +34,7 @@ import argparse import hashlib import json +import os import pathlib import re import subprocess @@ -43,6 +45,7 @@ from verify_release_chain import ( ANCHORS, MANIFEST_RE, + PRODUCER_PUBLIC_KEY_FILENAME, ReleaseChainError, git_blob_bytes, git_file_entry, @@ -51,15 +54,34 @@ verify_release_history_immutable, ) -ROOT = pathlib.Path(__file__).resolve().parents[1] +CODE_ROOT = pathlib.Path(__file__).resolve().parents[1] +ROOT = CODE_ROOT LEDGER_PATH = ROOT / "ledger" / "official_observations.jsonl" PREFIX_PATH = ROOT / "ledger" / "immutable_prefix.json" RELEASE_MANIFEST_PREFIX = "releases/manifests/" GENESIS_SUPPORT_FILES = { "releases/README.md", *(f"releases/anchors/{spec.filename}" for spec in ANCHORS.values()), + f"releases/anchors/{PRODUCER_PUBLIC_KEY_FILENAME}", } +GATE_SURFACE = frozenset( + { + "scripts/check_thesis_facts_append.py", + "scripts/verify_release_chain.py", + "scripts/canonical_json.py", + "scripts/cut_release_manifest.py", + ".github/workflows/thesis-facts-append.yml", + "releases/anchors/**", + } +) +DATA_SURFACE = frozenset( + { + "ledger/**", + "releases/manifests/**", + } +) + ASSERTION_CONTENT_KEYS = ( "source_record_id", "value", @@ -77,6 +99,106 @@ class AppendError(ValueError): """The proposed ledger change violates an append invariant.""" +def _set_root(root: pathlib.Path) -> None: + """Select the candidate worktree without changing the trusted code root. + + Pull-request CI executes this module from a detached checkout of the base + commit, while ``--root`` points at the checked-out PR merge tree. Imports + and production anchors therefore remain rooted at immutable ``CODE_ROOT``; + only candidate data paths and git comparisons use ``ROOT``. + """ + + global ROOT, LEDGER_PATH, PREFIX_PATH + ROOT = root.resolve() + LEDGER_PATH = ROOT / "ledger" / "official_observations.jsonl" + PREFIX_PATH = ROOT / "ledger" / "immutable_prefix.json" + + +def _git_output(arguments: list[str]) -> bytes: + try: + completed = subprocess.run( + ["git", *arguments], + cwd=ROOT, + check=False, + capture_output=True, + ) + except FileNotFoundError as exc: + raise AppendError("git is required for --base-ref verification") from exc + if completed.returncode != 0: + diagnostic = completed.stderr.decode( + "utf-8", errors="replace" + ).strip() + raise AppendError( + f"git {' '.join(arguments)} failed: {diagnostic}" + ) + return completed.stdout + + +def _resolve_base_commit(base_ref: str) -> str: + completed = _git_output( + ["rev-parse", "--verify", "--end-of-options", f"{base_ref}^{{commit}}"] + ) + commit = completed.decode("ascii").strip() + _git_output(["merge-base", "--is-ancestor", commit, "HEAD"]) + return commit + + +def _nul_paths(payload: bytes) -> set[str]: + return {os.fsdecode(path) for path in payload.split(b"\0") if path} + + +def _matches_surface(path: str, surface: frozenset[str]) -> bool: + for pattern in surface: + if pattern.endswith("/**"): + if path.startswith(pattern[:-2]): + return True + elif path == pattern: + return True + return False + + +def check_surface_separation(base_ref: str) -> tuple[set[str], set[str]]: + """Return data/gate changes and reject a proposal that combines them.""" + + commit = _resolve_base_commit(base_ref) + changed = _nul_paths( + _git_output( + [ + "diff", + "--name-only", + "-z", + "--no-renames", + "--no-ext-diff", + "--no-textconv", + commit, + "--", + ] + ) + ) + # ``git diff`` excludes untracked files. Tests mint release siblings before + # staging them, and a newly added anchor must still classify as gate code. + changed.update( + _nul_paths( + _git_output( + ["ls-files", "--others", "--exclude-standard", "-z", "--"] + ) + ) + ) + data_changes = { + path for path in changed if _matches_surface(path, DATA_SURFACE) + } + gate_changes = { + path for path in changed if _matches_surface(path, GATE_SURFACE) + } + if data_changes and gate_changes: + raise AppendError( + "mixed data/gate proposal is forbidden: DATA_SURFACE changes=" + f"{sorted(data_changes)}; GATE_SURFACE changes=" + f"{sorted(gate_changes)}; split them into separate pull requests" + ) + return data_changes, gate_changes + + def _lines(text: str) -> list[str]: return [line for line in text.split("\n") if line.strip()] @@ -379,7 +501,7 @@ def _release_triple( *, allowed_support_files: set[str] | None = None, ) -> pathlib.Path: - """Require exactly one new manifest and its two mapped receipt files.""" + """Require exactly one manifest, producer signature, and two receipts.""" manifest_files = [ relative @@ -405,14 +527,16 @@ def _release_triple( expected = { manifest_relative, *(f"{RELEASE_MANIFEST_PREFIX}{stem}.{tsa}.tsr" for tsa in ANCHORS), + f"{RELEASE_MANIFEST_PREFIX}{stem}.producer.sig", } allowed = expected | (allowed_support_files or set()) if new_files != expected and not ( expected <= new_files and new_files <= allowed ): raise AppendError( - "release proposal must add its manifest and exactly the freetsa " - "and digicert receipts with no other releases/ changes; " + "release proposal must add its manifest, producer signature, and " + "exactly the freetsa and digicert receipts with no other releases/ " + "changes; " f"missing={sorted(expected - new_files)}, " f"extra={sorted(new_files - allowed)}" ) @@ -437,14 +561,15 @@ def check_release_proposal( base_ref: str, *, anchor_dir: pathlib.Path | None = None, + enforce_production_pins: bool | None = None, ) -> int | None: """Verify the base chain and the one allowed candidate transition. A pre-genesis base keeps legacy append proposals valid only while they do - not touch ``releases/``. Genesis may add the two prescribed anchors and - README alongside its exact manifest/receipt triple. Once genesis exists, + not touch ``releases/``. Genesis may add the prescribed anchors and README + alongside its exact manifest/signature/receipt bundle. Once genesis exists, all base release files are byte- and mode-immutable and a ledger byte - append must carry exactly one next release triple. + append must carry exactly one next release bundle. """ try: @@ -466,15 +591,17 @@ def check_release_proposal( candidate_bytes = LEDGER_PATH.read_bytes() appended_bytes = _check_exact_byte_append(base_bytes, candidate_bytes) ledger_changed = bool(appended_bytes) - enforce_production_pins = anchor_dir is None + if enforce_production_pins is None: + enforce_production_pins = anchor_dir is None if not base_has_chain: if not candidate_has_chain: if new_files: raise AppendError( "legacy pre-genesis proposal must not change releases/; " - "add a complete genesis manifest and both receipts or no " - f"release files at all (changed={sorted(new_files)})" + "add a complete genesis manifest, producer signature, and " + "both receipts or no release files at all " + f"(changed={sorted(new_files)})" ) return None _release_triple( @@ -539,20 +666,24 @@ def check_release_proposal( def check_release_chain_without_base( - *, anchor_dir: pathlib.Path | None = None + *, + anchor_dir: pathlib.Path | None = None, + enforce_production_pins: bool | None = None, ) -> int | None: """On push, verify any initialized chain against working-tree state.""" manifest_directory = ROOT / "releases" / "manifests" if not manifest_directory.is_dir() or not any(manifest_directory.iterdir()): return None + if enforce_production_pins is None: + enforce_production_pins = anchor_dir is None try: verification = verify_release_chain( ROOT, anchor_dir=anchor_dir, require_chain=True, verify_state=True, - enforce_production_pins=anchor_dir is None, + enforce_production_pins=enforce_production_pins, ) except ReleaseChainError as exc: raise AppendError(str(exc)) from exc @@ -562,6 +693,12 @@ def check_release_chain_without_base( def main() -> int: parser = argparse.ArgumentParser() + parser.add_argument( + "--root", + type=pathlib.Path, + default=CODE_ROOT, + help="candidate worktree root (defaults to the checker's repository)", + ) parser.add_argument( "--base-ref", help="enforce an append-only diff against this git ref", @@ -572,8 +709,19 @@ def main() -> int: help=argparse.SUPPRESS, ) args = parser.parse_args() - text = LEDGER_PATH.read_text(encoding="utf-8") try: + _set_root(args.root) + if args.base_ref: + _data_changes, gate_changes = check_surface_separation(args.base_ref) + if gate_changes: + print( + "thesis-facts append check OK: gate-only proposal; " + "DATA_SURFACE unchanged; GATE_SURFACE changes=" + f"{sorted(gate_changes)}" + ) + return 0 + + text = LEDGER_PATH.read_text(encoding="utf-8") reject_non_append_bytes(text) lines = _lines(text) prefix = check_prefix(lines) @@ -588,14 +736,24 @@ def main() -> int: binding_boundary = check_prefix_anchored_to_base(args.base_ref, prefix) appended = check_append_only(args.base_ref, lines) check_rows(lines, binding_boundary) + # On the PR path, CODE_ROOT is the detached base checkout. Production + # verification must use those immutable anchors and the base verifier's + # pins, never files supplied by the candidate worktree. The hidden test + # override remains unpinned and continues to use generated test anchors. + production_pins = args.release_anchor_dir is None + anchor_dir = args.release_anchor_dir or ( + CODE_ROOT / "releases" / "anchors" + ) release_index = ( check_release_proposal( args.base_ref, - anchor_dir=args.release_anchor_dir, + anchor_dir=anchor_dir, + enforce_production_pins=production_pins, ) if args.base_ref else check_release_chain_without_base( - anchor_dir=args.release_anchor_dir + anchor_dir=anchor_dir, + enforce_production_pins=production_pins, ) ) except AppendError as exc: diff --git a/scripts/cut_release_manifest.py b/scripts/cut_release_manifest.py index 9351ce1..50c7a94 100644 --- a/scripts/cut_release_manifest.py +++ b/scripts/cut_release_manifest.py @@ -4,8 +4,9 @@ The cutter accepts a pending append after an already witnessed HEAD: it first verifies the existing release chain without requiring that HEAD equal the working-tree ledger, then proves that the old HEAD is the exact byte prefix of -the current ledger. A networked cut stages and fully verifies both RFC 3161 -responses before any release file is created. +the current ledger. A complete cut signs the exact manifest bytes, obtains both +RFC 3161 responses, and stages and verifies all four siblings before any release +file is created. """ from __future__ import annotations @@ -29,6 +30,7 @@ from canonical_json import canonical_bytes from verify_release_chain import ( + ANCHORS, DEFAULT_CLOCK_SKEW_SECONDS, LEDGER_RELATIVE, MANIFEST_RELATIVE, @@ -40,10 +42,14 @@ jsonl_line_offsets, load_manifest, manifest_filename, + producer_signature_path_for_manifest, receipt_paths_for_manifest, sha256_bytes, validate_manifest_schema, + verify_producer_signature, + verify_producer_signature_bytes, verify_release_chain, + verify_release_receipts, ) MAX_TOKEN_BYTES = 1024 * 1024 @@ -330,6 +336,65 @@ def _build_manifest( return manifest, canonical_bytes(manifest) + b"\n" +def _sign_manifest( + manifest: bytes, + signing_key: pathlib.Path, + timeout_seconds: float, +) -> bytes: + """Sign exact manifest bytes without reading private-key material in Python.""" + + try: + key_metadata = signing_key.lstat() + except FileNotFoundError as exc: + raise ReleaseCutError("producer signing key path does not exist") from exc + if not stat.S_ISREG(key_metadata.st_mode): + raise ReleaseCutError("producer signing key path is not a regular file") + + with tempfile.TemporaryDirectory(prefix="thesis-release-signature-") as name: + temporary = pathlib.Path(name) + manifest_path = temporary / "manifest.json" + signature_path = temporary / "producer.sig" + manifest_path.write_bytes(manifest) + try: + completed = subprocess.run( + [ + "openssl", + "pkeyutl", + "-sign", + "-inkey", + str(signing_key), + "-rawin", + "-in", + str(manifest_path), + "-out", + str(signature_path), + ], + check=False, + capture_output=True, + text=True, + timeout=timeout_seconds, + env={**os.environ, "LC_ALL": "C", "OPENSSL_CONF": "/dev/null"}, + ) + except FileNotFoundError as exc: + raise ReleaseCutError( + "OpenSSL 3 is required for Ed25519 producer signing" + ) from exc + except subprocess.TimeoutExpired as exc: + raise ReleaseCutError("OpenSSL producer signing timed out") from exc + if completed.returncode != 0: + diagnostic = (completed.stderr or completed.stdout).strip() + raise ReleaseCutError( + "OpenSSL producer signing failed: " + f"{diagnostic[-1000:] or 'no diagnostic'}" + ) + signature = _regular_bytes(signature_path, "producer signature") + if len(signature) != 64: + raise ReleaseCutError( + "OpenSSL producer signing did not emit a 64-byte raw Ed25519 signature" + ) + return signature + + def _build_timestamp_query(manifest: bytes, timeout_seconds: float) -> bytes: with tempfile.TemporaryDirectory(prefix="thesis-release-query-") as name: temporary = pathlib.Path(name) @@ -412,6 +477,7 @@ def _write_staged_tree( filename: str, manifest: bytes, receipts: Mapping[str, bytes], + producer_signature: bytes | None, ) -> None: ledger_path = stage / LEDGER_RELATIVE prefix_path = stage / PREFIX_RELATIVE @@ -426,10 +492,20 @@ def _write_staged_tree( (manifest_dir / receipt_path.name).write_bytes( _regular_bytes(receipt_path, f"existing {tsa} receipt") ) + (manifest_dir / record.producer_signature_path.name).write_bytes( + _regular_bytes( + record.producer_signature_path, + "existing producer signature", + ) + ) candidate = manifest_dir / filename candidate.write_bytes(manifest) for tsa, token in receipts.items(): receipt_paths_for_manifest(candidate)[tsa].write_bytes(token) + if producer_signature is not None: + producer_signature_path_for_manifest(candidate).write_bytes( + producer_signature + ) def _verify_staged_release( @@ -439,6 +515,7 @@ def _verify_staged_release( filename: str, manifest: bytes, receipts: Mapping[str, bytes], + producer_signature: bytes | None, *, anchor_dir: pathlib.Path, enforce_production_pins: bool, @@ -454,15 +531,45 @@ def _verify_staged_release( filename, manifest, receipts, + producer_signature, ) - verify_release_chain( - stage, - anchor_dir=anchor_dir, - require_chain=True, - verify_state=True, - enforce_production_pins=enforce_production_pins, - clock_skew_seconds=clock_skew_seconds, - ) + candidate = stage / MANIFEST_RELATIVE / filename + if set(receipts) == set(ANCHORS) and producer_signature is not None: + verify_release_chain( + stage, + anchor_dir=anchor_dir, + require_chain=True, + verify_state=True, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + ) + return + + loaded, loaded_raw, digest = load_manifest(candidate) + if loaded_raw != manifest: + raise ReleaseCutError("staged manifest differs from constructed bytes") + if receipts: + if set(receipts) != set(ANCHORS): + raise ReleaseCutError("staged release has an incomplete receipt set") + staged_receipts = receipt_paths_for_manifest(candidate) + verify_release_receipts( + loaded, + digest, + staged_receipts, + anchor_dir=anchor_dir, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + previous_times=( + existing.head.receipt_times if existing.head is not None else None + ), + ) + if producer_signature is not None: + verify_producer_signature( + loaded_raw, + producer_signature_path_for_manifest(candidate), + anchor_dir=anchor_dir, + enforce_production_pin=enforce_production_pins, + ) def _check_output_ancestors(root: pathlib.Path) -> None: @@ -483,108 +590,298 @@ def _check_targets_absent(paths: list[pathlib.Path]) -> None: raise ReleaseCutError(f"refusing to overwrite release file: {path}") -def _prepare_output_directory( - root: pathlib.Path, -) -> tuple[pathlib.Path, list[pathlib.Path]]: - created: list[pathlib.Path] = [] - try: - for directory in (root / "releases", root / MANIFEST_RELATIVE): - try: - os.mkdir(directory, 0o755) - created.append(directory) - except FileExistsError: - try: - metadata = directory.lstat() - except FileNotFoundError as exc: - raise ReleaseCutError( - f"release output directory changed concurrently: {directory}" - ) from exc - if not stat.S_ISDIR(metadata.st_mode): - raise ReleaseCutError( - f"release output parent is not a real directory: {directory}" - ) - except Exception: - _remove_empty_directories(created) - raise - return root / MANIFEST_RELATIVE, created - - -def _remove_empty_directories(directories: list[pathlib.Path]) -> None: - for directory in reversed(directories): - try: - directory.rmdir() - except OSError: - pass - - def _exclusive_batch_write( root: pathlib.Path, payloads: Mapping[pathlib.Path, bytes], *, + reserved_paths: list[pathlib.Path], verify_written: Callable[[], None], ) -> None: if not payloads: raise ReleaseCutError("no release files were provided for writing") - parents = {path.parent for path in payloads} + reserved = list(reserved_paths) + if not reserved: + raise ReleaseCutError("no release paths were reserved for writing") + parents = {path.parent for path in [*payloads, *reserved]} if parents != {root / MANIFEST_RELATIVE}: raise ReleaseCutError("release outputs escaped releases/manifests") + reserved_by_name = {path.name: path for path in reserved} + if len(reserved_by_name) != len(reserved): + raise ReleaseCutError("duplicate release paths were reserved for writing") + if not set(payloads).issubset(set(reserved)): + raise ReleaseCutError("release payloads were not included in reserved paths") + payload_names = {path.name for path in payloads} + omitted_names = set(reserved_by_name) - payload_names _check_output_ancestors(root) - _check_targets_absent(list(payloads)) - directory, created_directories = _prepare_output_directory(root) + _check_targets_absent(reserved) directory_flags = os.O_RDONLY if hasattr(os, "O_DIRECTORY"): directory_flags |= os.O_DIRECTORY if hasattr(os, "O_NOFOLLOW"): directory_flags |= os.O_NOFOLLOW + root_fd: int | None = None + releases_fd: int | None = None directory_fd: int | None = None - created_names: list[str] = [] + root_identity: tuple[int, int] | None = None + releases_identity: tuple[int, int] | None = None + directory_identity: tuple[int, int] | None = None + created_releases = False + created_manifests = False + created_entries: list[tuple[str, int, int]] = [] succeeded = False + + def identity(metadata: os.stat_result) -> tuple[int, int]: + return metadata.st_dev, metadata.st_ino + + def open_child_directory(parent_fd: int, name: str) -> int: + try: + descriptor = os.open(name, directory_flags, dir_fd=parent_fd) + except OSError as exc: + raise ReleaseCutError( + f"release output component is not a stable real directory: {name}" + ) from exc + metadata = os.fstat(descriptor) + if not stat.S_ISDIR(metadata.st_mode): + os.close(descriptor) + raise ReleaseCutError( + f"release output component is not a real directory: {name}" + ) + return descriptor + + def make_child_directory(parent_fd: int, name: str) -> bool: + try: + os.mkdir(name, 0o755, dir_fd=parent_fd) + except FileExistsError: + return False + except OSError as exc: + raise ReleaseCutError( + f"cannot create release output directory component: {name}" + ) from exc + return True + + def stat_entry(name: str) -> os.stat_result | None: + assert directory_fd is not None + try: + return os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + except FileNotFoundError: + return None + + def assert_names_absent(names: set[str], phase: str) -> None: + for name in sorted(names): + if stat_entry(name) is not None: + raise ReleaseCutError( + "reserved release file appeared concurrently " + f"{phase}: {reserved_by_name[name]}" + ) + + def assert_output_path_matches(phase: str) -> None: + assert root_identity is not None + assert releases_identity is not None + assert directory_identity is not None + check_root_fd: int | None = None + check_releases_fd: int | None = None + check_directory_fd: int | None = None + try: + try: + check_root_fd = os.open(root, directory_flags) + except OSError as exc: + raise ReleaseCutError( + f"repository root path changed {phase}: {root}" + ) from exc + if identity(os.fstat(check_root_fd)) != root_identity: + raise ReleaseCutError( + f"repository root path was replaced {phase}: {root}" + ) + check_releases_fd = open_child_directory(check_root_fd, "releases") + if identity(os.fstat(check_releases_fd)) != releases_identity: + raise ReleaseCutError( + f"release output parent was replaced {phase}: {root / 'releases'}" + ) + check_directory_fd = open_child_directory( + check_releases_fd, + MANIFEST_RELATIVE.name, + ) + if identity(os.fstat(check_directory_fd)) != directory_identity: + raise ReleaseCutError( + "release manifest directory was replaced " + f"{phase}: {root / MANIFEST_RELATIVE}" + ) + finally: + for descriptor in ( + check_directory_fd, + check_releases_fd, + check_root_fd, + ): + if descriptor is not None: + os.close(descriptor) + + def assert_created_entries_match(phase: str) -> None: + for name, expected_device, expected_inode in created_entries: + descriptor: int | None = None + try: + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(name, flags, dir_fd=directory_fd) + except OSError as exc: + raise ReleaseCutError( + f"created release file disappeared {phase}: {name}" + ) from exc + metadata = os.fstat(descriptor) + if identity(metadata) != (expected_device, expected_inode): + raise ReleaseCutError( + "created release file was replaced concurrently " + f"{phase}: {name}" + ) + expected_payload = payloads[reserved_by_name[name]] + with os.fdopen(descriptor, "rb") as stream: + descriptor = None + actual_payload = stream.read(len(expected_payload) + 1) + if actual_payload != expected_payload: + raise ReleaseCutError( + f"created release file bytes changed {phase}: {name}" + ) + current = stat_entry(name) + if current is None or identity(current) != ( + expected_device, + expected_inode, + ): + raise ReleaseCutError( + "created release file was replaced concurrently " + f"{phase}: {name}" + ) + finally: + if descriptor is not None: + os.close(descriptor) + + def remove_created_directory_if_owned( + parent_fd: int, + name: str, + expected_identity: tuple[int, int] | None, + ) -> None: + if expected_identity is None: + return + try: + current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + except OSError: + return + if identity(current) != expected_identity: + return + try: + os.rmdir(name, dir_fd=parent_fd) + except OSError: + pass + try: - directory_fd = os.open(directory, directory_flags) + try: + root_fd = os.open(root, directory_flags) + except OSError as exc: + raise ReleaseCutError( + f"repository root is not a stable real directory: {root}" + ) from exc + root_identity = identity(os.fstat(root_fd)) + created_releases = make_child_directory(root_fd, "releases") + releases_fd = open_child_directory(root_fd, "releases") + releases_identity = identity(os.fstat(releases_fd)) + created_manifests = make_child_directory( + releases_fd, + MANIFEST_RELATIVE.name, + ) + directory_fd = open_child_directory( + releases_fd, + MANIFEST_RELATIVE.name, + ) + directory_identity = identity(os.fstat(directory_fd)) + assert_output_path_matches("before batch write") + assert_names_absent(set(reserved_by_name), "before batch write") for path, payload in payloads.items(): - try: - os.stat(path.name, dir_fd=directory_fd, follow_symlinks=False) - except FileNotFoundError: - pass - else: + if stat_entry(path.name) is not None: raise ReleaseCutError(f"refusing to overwrite release file: {path}") flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW descriptor = os.open(path.name, flags, 0o644, dir_fd=directory_fd) - created_names.append(path.name) - with os.fdopen(descriptor, "wb") as stream: - stream.write(payload) - stream.flush() - os.fsync(stream.fileno()) + try: + metadata = os.fstat(descriptor) + created_entries.append( + (path.name, metadata.st_dev, metadata.st_ino) + ) + stream = os.fdopen(descriptor, "wb") + descriptor = -1 + with stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + finally: + if descriptor >= 0: + os.close(descriptor) os.fsync(directory_fd) + assert_output_path_matches("before verification") + assert_names_absent(omitted_names, "before verification") + assert_created_entries_match("before verification") verify_written() + assert_output_path_matches("after verification") + assert_names_absent(omitted_names, "after verification") + assert_created_entries_match("after verification") succeeded = True except BaseException as exc: - cleanup_failures: list[str] = [] + cleanup_conflicts: list[str] = [] if directory_fd is not None: - for name in reversed(created_names): + for name, expected_device, expected_inode in reversed(created_entries): + try: + current = stat_entry(name) + except OSError as cleanup_error: + cleanup_conflicts.append( + f"{name} (cannot inspect: {cleanup_error})" + ) + continue + if current is None: + continue + if (current.st_dev, current.st_ino) != ( + expected_device, + expected_inode, + ): + cleanup_conflicts.append( + f"{name} (concurrent replacement preserved)" + ) + continue try: os.unlink(name, dir_fd=directory_fd) except FileNotFoundError: pass - except OSError: - cleanup_failures.append(name) + except OSError as cleanup_error: + cleanup_conflicts.append(f"{name} ({cleanup_error})") try: os.fsync(directory_fd) except OSError: pass - if cleanup_failures: + if cleanup_conflicts: raise ReleaseCutError( - "release write failed and rollback could not remove: " - f"{cleanup_failures}; original error: {exc}" + "release write failed and rollback could not remove all reserved " + "paths due to cleanup conflicts; concurrent files were preserved: " + f"{cleanup_conflicts}; original error: {exc}" ) from exc raise finally: if directory_fd is not None: os.close(directory_fd) - if not succeeded: - _remove_empty_directories(created_directories) + if not succeeded and created_manifests and releases_fd is not None: + remove_created_directory_if_owned( + releases_fd, + MANIFEST_RELATIVE.name, + directory_identity, + ) + if releases_fd is not None: + os.close(releases_fd) + if not succeeded and created_releases and root_fd is not None: + remove_created_directory_if_owned( + root_fd, + "releases", + releases_identity, + ) + if root_fd is not None: + os.close(root_fd) def _snapshot_existing_files( @@ -595,6 +892,10 @@ def _snapshot_existing_files( snapshot[record.path] = record.raw for tsa, path in record.receipt_paths.items(): snapshot[path] = _regular_bytes(path, f"existing {tsa} receipt") + snapshot[record.producer_signature_path] = _regular_bytes( + record.producer_signature_path, + "existing producer signature", + ) return snapshot @@ -624,6 +925,8 @@ def cut_release_manifest( repo: str | None = None, branch: str | None = None, no_tsa: bool = False, + signing_key: pathlib.Path | None = None, + no_sign: bool = False, anchor_dir: pathlib.Path | None = None, timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, clock_skew_seconds: int = DEFAULT_CLOCK_SKEW_SECONDS, @@ -634,6 +937,12 @@ def cut_release_manifest( if type(no_tsa) is not bool: raise ReleaseCutError("no_tsa must be a boolean") + if type(no_sign) is not bool: + raise ReleaseCutError("no_sign must be a boolean") + if no_sign and signing_key is not None: + raise ReleaseCutError("signing_key and no_sign are mutually exclusive") + if not no_sign and signing_key is None: + raise ReleaseCutError("signing_key is required unless no_sign is set") if type(clock_skew_seconds) is not int or clock_skew_seconds < 0: raise ReleaseCutError("clock_skew_seconds must be a non-negative integer") timeout = _validate_timeout(timeout_seconds) @@ -649,6 +958,9 @@ def cut_release_manifest( else selected_root / "releases" / "anchors" ) enforce_production_pins = anchor_dir is None + selected_signing_key = ( + pathlib.Path(signing_key).absolute() if signing_key is not None else None + ) existing = verify_release_chain( selected_root, @@ -678,45 +990,36 @@ def cut_release_manifest( filename = manifest_filename(_manifest["releaseIndex"], raw) manifest_path = selected_root / MANIFEST_RELATIVE / filename receipt_paths = receipt_paths_for_manifest(manifest_path) - all_targets = [manifest_path, *receipt_paths.values()] + producer_signature_path = producer_signature_path_for_manifest(manifest_path) + all_targets = [ + manifest_path, + *receipt_paths.values(), + producer_signature_path, + ] _check_output_ancestors(selected_root) _check_targets_absent(all_targets) history = _snapshot_existing_files(existing) - if no_tsa: - - def verify_manifest_only() -> None: - loaded, loaded_raw, _digest = load_manifest(manifest_path) - if loaded != _manifest or loaded_raw != raw: - raise ReleaseCutError("written manifest differs from constructed bytes") - _assert_unchanged( - ledger_path, - prefix_path, - ledger, - immutable_prefix, - history, - ) - - _assert_unchanged( - ledger_path, - prefix_path, - ledger, - immutable_prefix, - history, + producer_signature: bytes | None = None + if selected_signing_key is not None: + producer_signature = _sign_manifest(raw, selected_signing_key, timeout) + verify_producer_signature_bytes( + raw, + producer_signature, + anchor_dir=selected_anchors, + enforce_production_pin=enforce_production_pins, + label=producer_signature_path.name, ) - _exclusive_batch_write( - selected_root, - {manifest_path: raw}, - verify_written=verify_manifest_only, + + receipts: dict[str, bytes] = {} + if not no_tsa: + query = _build_timestamp_query(raw, timeout) + receipts = _request_receipts( + query, + requester=request_timestamp if requester is None else requester, + timeout_seconds=timeout, ) - return manifest_path - query = _build_timestamp_query(raw, timeout) - receipts = _request_receipts( - query, - requester=request_timestamp if requester is None else requester, - timeout_seconds=timeout, - ) _verify_staged_release( existing, ledger, @@ -724,14 +1027,17 @@ def verify_manifest_only() -> None: filename, raw, receipts, + producer_signature, anchor_dir=selected_anchors, enforce_production_pins=enforce_production_pins, clock_skew_seconds=clock_skew_seconds, ) - payloads = { - manifest_path: raw, - **{receipt_paths[tsa]: token for tsa, token in receipts.items()}, - } + payloads: dict[pathlib.Path, bytes] = {manifest_path: raw} + payloads.update( + {receipt_paths[tsa]: token for tsa, token in receipts.items()} + ) + if producer_signature is not None: + payloads[producer_signature_path] = producer_signature def verify_committed_release() -> None: _assert_unchanged( @@ -741,18 +1047,52 @@ def verify_committed_release() -> None: immutable_prefix, history, ) - verify_release_chain( - selected_root, - anchor_dir=selected_anchors, - require_chain=True, - verify_state=True, - enforce_production_pins=enforce_production_pins, - clock_skew_seconds=clock_skew_seconds, - ) + if set(receipts) == set(ANCHORS) and producer_signature is not None: + verify_release_chain( + selected_root, + anchor_dir=selected_anchors, + require_chain=True, + verify_state=True, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + ) + return + + loaded, loaded_raw, digest = load_manifest(manifest_path) + if loaded != _manifest or loaded_raw != raw: + raise ReleaseCutError("written manifest differs from constructed bytes") + if receipts: + verify_release_receipts( + loaded, + digest, + receipt_paths, + anchor_dir=selected_anchors, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + previous_times=( + existing.head.receipt_times if existing.head is not None else None + ), + ) + if producer_signature is not None: + verify_producer_signature( + loaded_raw, + producer_signature_path, + anchor_dir=selected_anchors, + enforce_production_pin=enforce_production_pins, + ) + + _assert_unchanged( + ledger_path, + prefix_path, + ledger, + immutable_prefix, + history, + ) _exclusive_batch_write( selected_root, payloads, + reserved_paths=all_targets, verify_written=verify_committed_release, ) return manifest_path @@ -771,14 +1111,25 @@ def main() -> int: parser.add_argument( "--anchor-dir", type=pathlib.Path, - help="override TSA anchors (intended for offline tests)", + help="override TSA and producer anchors (intended for offline tests)", ) parser.add_argument("--repo", help="override producer.repo provenance") parser.add_argument("--branch", help="override producer.branch provenance") + signing = parser.add_mutually_exclusive_group(required=True) + signing.add_argument( + "--signing-key", + type=pathlib.Path, + help="Ed25519 private-key PEM path passed directly to OpenSSL", + ) + signing.add_argument( + "--no-sign", + action="store_true", + help="omit the producer signature for an explicitly partial release", + ) parser.add_argument( "--no-tsa", action="store_true", - help="write only the manifest; do not request or write receipts", + help="do not request or write TSA receipts", ) parser.add_argument( "--timeout-seconds", @@ -797,6 +1148,8 @@ def main() -> int: repo=args.repo, branch=args.branch, no_tsa=args.no_tsa, + signing_key=args.signing_key, + no_sign=args.no_sign, anchor_dir=args.anchor_dir, timeout_seconds=args.timeout_seconds, clock_skew_seconds=args.clock_skew_seconds, @@ -804,8 +1157,15 @@ def main() -> int: except (OSError, ReleaseChainError, ReleaseCutError, ValueError) as exc: print(f"release cut failed: {exc}", file=sys.stderr) return 1 - if args.no_tsa: - print(f"release manifest written without TSA receipts: {manifest_path}") + if args.no_tsa and args.no_sign: + print( + "release manifest written without TSA receipts or producer signature: " + f"{manifest_path}" + ) + elif args.no_tsa: + print(f"producer-signed release written without TSA receipts: {manifest_path}") + elif args.no_sign: + print(f"timestamped release written without producer signature: {manifest_path}") else: print(f"witnessed release written: {manifest_path}") return 0 diff --git a/scripts/verify_release_chain.py b/scripts/verify_release_chain.py index 03c2d6c..31c66d1 100644 --- a/scripts/verify_release_chain.py +++ b/scripts/verify_release_chain.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 """Offline verification for the witnessed thesis-ledger release chain. -The verifier treats manifest and receipt bytes as an append-only journal. It -does not trust manifest provenance or timestamps supplied by the producer: -each manifest is canonical and content-addressed, every state and append digest -is recomputed from the current append-only JSONL, and both RFC 3161 receipts are -verified against separate, committed trust anchors. +The verifier treats manifest, signature, and receipt bytes as an append-only +journal. It does not trust manifest provenance or timestamps supplied by the +producer: each manifest is canonical and content-addressed, every state and +append digest is recomputed from the current append-only JSONL, every manifest +has a valid signature from the pinned producer key, and both RFC 3161 receipts +are verified against separate, committed trust anchors. """ from __future__ import annotations @@ -25,6 +26,19 @@ from canonical_json import canonical_bytes +try: + from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + from cryptography.hazmat.primitives.serialization import ( + Encoding, + PublicFormat, + load_pem_public_key, + ) +except ImportError: # Bare pre-sync CI falls back to the OpenSSL 3 CLI below. + CRYPTOGRAPHY_AVAILABLE = False +else: + CRYPTOGRAPHY_AVAILABLE = True + ROOT = pathlib.Path(__file__).resolve().parents[1] MANIFEST_RELATIVE = pathlib.PurePosixPath("releases/manifests") LEDGER_RELATIVE = pathlib.PurePosixPath("ledger/official_observations.jsonl") @@ -40,6 +54,14 @@ r"(?P[0-9]{4}-[0-9a-f]{16})" r"\.(?Pfreetsa|digicert)\.tsr\Z" ) +PRODUCER_SIGNATURE_RE = re.compile( + r"(?P[0-9]{4}-[0-9a-f]{16})\.producer\.sig\Z" +) +PRODUCER_PUBLIC_KEY_FILENAME = "producer-ed25519.pub" +PRODUCER_SPKI_SHA256 = ( + "4a90eff40455ce0d853d4bab1608efbdae1efaf8c06054ead6e396c5b0c4846e" +) +PRODUCER_SIGNATURE_BYTES = 64 STRICT_UTC_RE = re.compile( r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:" r"[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z\Z" @@ -108,6 +130,7 @@ class ReleaseRecord: manifest: dict[str, Any] receipt_paths: dict[str, pathlib.Path] receipt_times: dict[str, datetime] + producer_signature_path: pathlib.Path @property def release_index(self) -> int: @@ -315,9 +338,13 @@ def receipt_paths_for_manifest(path: pathlib.Path) -> dict[str, pathlib.Path]: return {tsa: path.with_name(f"{stem}.{tsa}.tsr") for tsa in ANCHORS} +def producer_signature_path_for_manifest(path: pathlib.Path) -> pathlib.Path: + return path.with_name(f"{path.stem}.producer.sig") + + def _enumerate_manifest_files( root: pathlib.Path, -) -> list[tuple[pathlib.Path, dict[str, pathlib.Path]]]: +) -> list[tuple[pathlib.Path, dict[str, pathlib.Path], pathlib.Path]]: directory = root / MANIFEST_RELATIVE if not directory.exists(): return [] @@ -328,6 +355,7 @@ def _enumerate_manifest_files( manifests: dict[str, pathlib.Path] = {} receipts: dict[str, dict[str, pathlib.Path]] = {} + producer_signatures: dict[str, pathlib.Path] = {} for entry in directory.iterdir(): if entry.is_symlink() or not entry.is_file(): raise ReleaseChainError( @@ -343,6 +371,10 @@ def _enumerate_manifest_files( tsa = receipt_match.group("tsa") receipts.setdefault(stem, {})[tsa] = entry continue + signature_match = PRODUCER_SIGNATURE_RE.fullmatch(entry.name) + if signature_match is not None: + producer_signatures[signature_match.group("stem")] = entry + continue raise ReleaseChainError( f"unknown file in closed release manifest directory: {entry.name}" ) @@ -352,7 +384,15 @@ def _enumerate_manifest_files( raise ReleaseChainError( f"orphan release receipts for manifest stems: {orphan_receipts}" ) - result: list[tuple[pathlib.Path, dict[str, pathlib.Path]]] = [] + orphan_signatures = sorted(set(producer_signatures) - set(manifests)) + if orphan_signatures: + raise ReleaseChainError( + "orphan producer signatures for manifest stems: " + f"{orphan_signatures}" + ) + result: list[ + tuple[pathlib.Path, dict[str, pathlib.Path], pathlib.Path] + ] = [] seen_indices: dict[int, str] = {} for stem, path in manifests.items(): match = MANIFEST_RE.fullmatch(path.name) @@ -369,7 +409,13 @@ def _enumerate_manifest_files( f"manifest {path.name} must have exactly freetsa and digicert " f"receipts; found={sorted(actual_receipts)}" ) - result.append((path, actual_receipts)) + producer_signature = producer_signatures.get(stem) + if producer_signature is None: + raise ReleaseChainError( + f"manifest {path.name} is missing its producer signature " + f"{stem}.producer.sig" + ) + result.append((path, actual_receipts, producer_signature)) return sorted( result, key=lambda item: int(MANIFEST_RE.fullmatch(item[0].name).group("index")), @@ -479,6 +525,180 @@ def _openssl_binary( return completed.stdout +def _producer_openssl_binary( + arguments: list[str], + *, + environment: dict[str, str], + label: str, +) -> bytes: + try: + completed = subprocess.run( + ["openssl", *arguments], + check=False, + capture_output=True, + env=environment, + ) + except FileNotFoundError as exc: + raise ReleaseChainError( + "producer signature verification requires cryptography or OpenSSL 3" + ) from exc + if completed.returncode != 0: + diagnostic = (completed.stderr or completed.stdout).decode( + "utf-8", errors="replace" + ) + raise ReleaseChainError( + f"OpenSSL producer {label} failed (exit {completed.returncode}): " + f"{diagnostic.strip()[-1000:]}" + ) + return completed.stdout + + +def _verify_producer_signature_with_openssl( + manifest: bytes, + signature: bytes, + public_key_pem: bytes, + *, + enforce_production_pin: bool, + label: str, +) -> None: + with tempfile.TemporaryDirectory(prefix="thesis-release-producer-") as name: + temporary = pathlib.Path(name) + empty_ca_dir = temporary / "empty-ca" + empty_ca_dir.mkdir() + environment = _openssl_environment(empty_ca_dir) + manifest_path = temporary / "manifest.json" + signature_path = temporary / "producer.sig" + public_key_path = temporary / PRODUCER_PUBLIC_KEY_FILENAME + manifest_path.write_bytes(manifest) + signature_path.write_bytes(signature) + public_key_path.write_bytes(public_key_pem) + + spki_der = _producer_openssl_binary( + [ + "pkey", + "-pubin", + "-in", + str(public_key_path), + "-outform", + "DER", + ], + environment=environment, + label=f"public-key decoding for {label}", + ) + if enforce_production_pin: + spki_sha256 = sha256_bytes(spki_der) + if spki_sha256 != PRODUCER_SPKI_SHA256: + raise ReleaseChainError( + "producer public-key SPKI is not code-pinned: " + f"{spki_sha256}" + ) + + try: + _producer_openssl_binary( + [ + "pkeyutl", + "-verify", + "-pubin", + "-inkey", + str(public_key_path), + "-rawin", + "-in", + str(manifest_path), + "-sigfile", + str(signature_path), + ], + environment=environment, + label=f"Ed25519 signature verification for {label}", + ) + except ReleaseChainError as exc: + raise ReleaseChainError( + f"producer Ed25519 signature verification failed for {label}" + ) from exc + + +def verify_producer_signature_bytes( + manifest: bytes, + signature: bytes, + *, + anchor_dir: pathlib.Path, + enforce_production_pin: bool, + label: str, +) -> None: + """Verify one raw Ed25519 signature over exact manifest bytes.""" + + if type(manifest) is not bytes: + raise ReleaseChainError("producer-signed manifest payload must be bytes") + if type(signature) is not bytes or len(signature) != PRODUCER_SIGNATURE_BYTES: + actual = len(signature) if isinstance(signature, bytes) else "non-bytes" + raise ReleaseChainError( + f"producer signature for {label} must be exactly " + f"{PRODUCER_SIGNATURE_BYTES} raw bytes; found={actual}" + ) + public_key_path = anchor_dir / PRODUCER_PUBLIC_KEY_FILENAME + if public_key_path.is_symlink() or not public_key_path.is_file(): + raise ReleaseChainError( + f"missing or non-regular producer public key: {public_key_path}" + ) + public_key_pem = public_key_path.read_bytes() + + if not CRYPTOGRAPHY_AVAILABLE: + _verify_producer_signature_with_openssl( + manifest, + signature, + public_key_pem, + enforce_production_pin=enforce_production_pin, + label=label, + ) + return + + try: + public_key = load_pem_public_key(public_key_pem) + except (TypeError, ValueError, UnsupportedAlgorithm) as exc: + raise ReleaseChainError( + f"cannot decode producer Ed25519 public key: {public_key_path}" + ) from exc + if not isinstance(public_key, Ed25519PublicKey): + raise ReleaseChainError( + f"producer public key is not Ed25519: {public_key_path}" + ) + spki_der = public_key.public_bytes( + Encoding.DER, + PublicFormat.SubjectPublicKeyInfo, + ) + if enforce_production_pin: + spki_sha256 = sha256_bytes(spki_der) + if spki_sha256 != PRODUCER_SPKI_SHA256: + raise ReleaseChainError( + f"producer public-key SPKI is not code-pinned: {spki_sha256}" + ) + try: + public_key.verify(signature, manifest) + except InvalidSignature as exc: + raise ReleaseChainError( + f"producer Ed25519 signature verification failed for {label}" + ) from exc + + +def verify_producer_signature( + manifest: bytes, + signature_path: pathlib.Path, + *, + anchor_dir: pathlib.Path, + enforce_production_pin: bool, +) -> None: + if signature_path.is_symlink() or not signature_path.is_file(): + raise ReleaseChainError( + f"missing or non-regular producer signature: {signature_path}" + ) + verify_producer_signature_bytes( + manifest, + signature_path.read_bytes(), + anchor_dir=anchor_dir, + enforce_production_pin=enforce_production_pin, + label=signature_path.name, + ) + + def _verify_production_signer( receipt: pathlib.Path, anchor: pathlib.Path, @@ -671,6 +891,60 @@ def verify_receipt( return gen_time +def verify_release_receipts( + manifest: dict[str, Any], + manifest_digest: str, + receipt_paths: dict[str, pathlib.Path], + *, + anchor_dir: pathlib.Path, + enforce_production_pins: bool, + clock_skew_seconds: int, + previous_times: dict[str, datetime] | None = None, + now: datetime | None = None, +) -> dict[str, datetime]: + """Verify both receipts and their chronology for one manifest.""" + + if set(receipt_paths) != set(ANCHORS): + raise ReleaseChainError( + "release must have exactly freetsa and digicert receipt paths" + ) + receipt_times = { + tsa: verify_receipt( + manifest_digest, + receipt_path, + tsa, + anchor_dir=anchor_dir, + enforce_production_pins=enforce_production_pins, + now=now, + ) + for tsa, receipt_path in receipt_paths.items() + } + created_at = parse_created_at(manifest["createdAtUtc"]) + earliest_allowed = created_at - timedelta(seconds=clock_skew_seconds) + release_index = manifest["releaseIndex"] + for tsa, gen_time in receipt_times.items(): + if gen_time < earliest_allowed: + raise ReleaseChainError( + f"release {release_index} {tsa} genTime " + f"{gen_time.isoformat()} impossibly precedes createdAtUtc " + f"{created_at.isoformat()}" + ) + if previous_times is not None: + lower_bound = max(previous_times.values()) - timedelta( + seconds=clock_skew_seconds + ) + current_earliest = min(receipt_times.values()) + if current_earliest < lower_bound: + raise ReleaseChainError( + f"release {release_index} receipt chronology regresses: " + f"earliest current genTime {current_earliest.isoformat()} " + f"precedes latest prior genTime " + f"{max(previous_times.values()).isoformat()} beyond " + f"{clock_skew_seconds}s skew" + ) + return receipt_times + + def jsonl_line_offsets(payload: bytes, label: str) -> list[int]: """Return exact byte offsets after each non-empty LF-terminated row.""" @@ -793,7 +1067,7 @@ def verify_release_chain( clock_skew_seconds: int = DEFAULT_CLOCK_SKEW_SECONDS, now: datetime | None = None, ) -> ChainVerification: - """Verify all manifests, receipts, links, chronology, and state bytes.""" + """Verify all manifests, signatures, receipts, links, and state bytes.""" root = root.resolve() default_anchor_dir = root / "releases" / "anchors" @@ -813,7 +1087,9 @@ def verify_release_chain( previous_hash: str | None = None previous_times: dict[str, datetime] | None = None verification_now = now or datetime.now(timezone.utc) - for expected_index, (path, receipt_paths) in enumerate(enumerated): + for expected_index, (path, receipt_paths, producer_signature_path) in enumerate( + enumerated + ): manifest, raw, digest = load_manifest(path) filename_match = MANIFEST_RE.fullmatch(path.name) assert filename_match is not None @@ -837,6 +1113,12 @@ def verify_release_chain( f"release {expected_index} previousManifestSha256 does not " "match the previous manifest file bytes" ) + verify_producer_signature( + raw, + producer_signature_path, + anchor_dir=selected_anchors, + enforce_production_pin=enforce_production_pins, + ) if records: previous_line_count = records[-1].manifest["state"]["lineCount"] line_count = manifest["state"]["lineCount"] @@ -859,39 +1141,16 @@ def verify_release_chain( f"{row_delta}" ) - receipt_times = { - tsa: verify_receipt( - digest, - receipt_path, - tsa, - anchor_dir=selected_anchors, - enforce_production_pins=enforce_production_pins, - now=verification_now, - ) - for tsa, receipt_path in receipt_paths.items() - } - created_at = parse_created_at(manifest["createdAtUtc"]) - earliest_allowed = created_at - timedelta(seconds=clock_skew_seconds) - for tsa, gen_time in receipt_times.items(): - if gen_time < earliest_allowed: - raise ReleaseChainError( - f"release {expected_index} {tsa} genTime " - f"{gen_time.isoformat()} impossibly precedes createdAtUtc " - f"{created_at.isoformat()}" - ) - if previous_times is not None: - lower_bound = max(previous_times.values()) - timedelta( - seconds=clock_skew_seconds - ) - current_earliest = min(receipt_times.values()) - if current_earliest < lower_bound: - raise ReleaseChainError( - f"release {expected_index} receipt chronology regresses: " - f"earliest current genTime {current_earliest.isoformat()} " - f"precedes latest prior genTime " - f"{max(previous_times.values()).isoformat()} beyond " - f"{clock_skew_seconds}s skew" - ) + receipt_times = verify_release_receipts( + manifest, + digest, + receipt_paths, + anchor_dir=selected_anchors, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + previous_times=previous_times, + now=verification_now, + ) records.append( ReleaseRecord( @@ -901,6 +1160,7 @@ def verify_release_chain( manifest=manifest, receipt_paths=receipt_paths, receipt_times=receipt_times, + producer_signature_path=producer_signature_path, ) ) previous_hash = digest diff --git a/tests/test_release_chain.py b/tests/test_release_chain.py index 2ee1c5a..a49c655 100644 --- a/tests/test_release_chain.py +++ b/tests/test_release_chain.py @@ -34,16 +34,23 @@ from check_thesis_facts_append import ( # noqa: E402 expected_assertion_version_id, ) +import cut_release_manifest as cutter_module # noqa: E402 from cut_release_manifest import ( # noqa: E402 TSA_ENDPOINTS, ReleaseCutError, + _build_manifest, cut_release_manifest, ) from verify_release_chain import ( # noqa: E402 + ChainVerification, + PRODUCER_PUBLIC_KEY_FILENAME, + PRODUCER_SPKI_SHA256, ReleaseChainError, load_manifest, manifest_filename, + producer_signature_path_for_manifest, validate_manifest_schema, + verify_producer_signature_bytes, ) @@ -56,6 +63,10 @@ class ReleaseEnvironment: def anchors(self) -> Path: return self.tsa / "anchors" + @property + def producer_signing_key(self) -> Path: + return self.tsa / "producer-ed25519-private.pem" + def _run( command: list[str], @@ -117,6 +128,60 @@ def _created_at() -> str: return value.isoformat(timespec="seconds").replace("+00:00", "Z") +def _generate_producer_keypair( + directory: Path, + *, + public_key: Path | None = None, +) -> Path: + directory.mkdir(parents=True, exist_ok=True) + private_key = directory / "producer-ed25519-private.pem" + _run( + [ + "openssl", + "genpkey", + "-algorithm", + "ED25519", + "-out", + str(private_key), + ], + cwd=directory, + ) + if public_key is not None: + public_key.parent.mkdir(parents=True, exist_ok=True) + _run( + [ + "openssl", + "pkey", + "-in", + str(private_key), + "-pubout", + "-out", + str(public_key), + ], + cwd=directory, + ) + return private_key + + +def _sign_producer_payload(payload: Path, signature: Path, key: Path) -> None: + _run( + [ + "openssl", + "pkeyutl", + "-sign", + "-inkey", + str(key), + "-rawin", + "-in", + str(payload), + "-out", + str(signature), + ], + cwd=payload.parent, + ) + assert len(signature.read_bytes()) == 64 + + def _release_manifest( repo: Path, *, @@ -205,6 +270,9 @@ def _write_release( *, signers: dict[str, str] | None = None, signed_payloads: dict[str, Path] | None = None, + producer_key: Path | None = None, + producer_signed_payload: Path | None = None, + include_producer_signature: bool = True, ) -> Path: raw = canonical_bytes(manifest) + b"\n" directory = environment.repo / "releases" / "manifests" @@ -216,6 +284,12 @@ def _write_release( payload = (signed_payloads or {}).get(slot, path) signer = (signers or {}).get(slot, slot) _mint_receipt(payload, receipt, tsa=environment.tsa, signer=signer) + if include_producer_signature: + _sign_producer_payload( + producer_signed_payload or path, + producer_signature_path_for_manifest(path), + producer_key or environment.producer_signing_key, + ) return path @@ -334,6 +408,10 @@ def pregenesis_template(tmp_path_factory: pytest.TempPathFactory) -> ReleaseEnvi repo.mkdir() tsa = root / "release_tsa" shutil.copytree(TSA_FIXTURE, tsa) + _generate_producer_keypair( + tsa, + public_key=tsa / "anchors" / PRODUCER_PUBLIC_KEY_FILENAME, + ) _initialize_repo(repo) return ReleaseEnvironment(repo=repo, tsa=tsa) @@ -614,6 +692,121 @@ def test_verifier_cli_full_chain_passes( assert "release chain OK: 2 releases" in completed.stdout +@pytest.mark.parametrize("index", [0, 1]) +def test_verifier_requires_producer_signature_for_every_release( + full_chain_environment: ReleaseEnvironment, + index: int, +): + manifest = next( + (full_chain_environment.repo / "releases" / "manifests").glob( + f"{index:04d}-*.json" + ) + ) + producer_signature_path_for_manifest(manifest).unlink() + + completed = _run_verifier(full_chain_environment) + + assert completed.returncode == 1 + assert "missing its producer signature" in completed.stderr + + +def test_verifier_rejects_wrong_producer_key_signature( + full_chain_environment: ReleaseEnvironment, +): + head = next( + (full_chain_environment.repo / "releases" / "manifests").glob( + "0001-*.json" + ) + ) + wrong_key = _generate_producer_keypair(full_chain_environment.tsa / "wrong-key") + _sign_producer_payload( + head, + producer_signature_path_for_manifest(head), + wrong_key, + ) + + completed = _run_verifier(full_chain_environment) + + assert completed.returncode == 1 + assert "producer Ed25519 signature verification failed" in completed.stderr + + +def test_verifier_rejects_producer_signature_over_different_bytes( + full_chain_environment: ReleaseEnvironment, +): + head = next( + (full_chain_environment.repo / "releases" / "manifests").glob( + "0001-*.json" + ) + ) + different = full_chain_environment.tsa / "different-producer-payload.json" + different.write_bytes(b'{"different":true}\n') + _sign_producer_payload( + different, + producer_signature_path_for_manifest(head), + full_chain_environment.producer_signing_key, + ) + + completed = _run_verifier(full_chain_environment) + + assert completed.returncode == 1 + assert "producer Ed25519 signature verification failed" in completed.stderr + + +def test_verifier_rejects_non_raw_producer_signature_size( + full_chain_environment: ReleaseEnvironment, +): + head = next( + (full_chain_environment.repo / "releases" / "manifests").glob( + "0001-*.json" + ) + ) + producer_signature_path_for_manifest(head).write_bytes(b"not-64-bytes") + + completed = _run_verifier(full_chain_environment) + + assert completed.returncode == 1 + assert "must be exactly 64 raw bytes" in completed.stderr + + +def test_verifier_rejects_unpinned_producer_public_key( + tmp_path: Path, +): + anchors = tmp_path / "anchors" + _generate_producer_keypair( + tmp_path / "wrong-production-key", + public_key=anchors / PRODUCER_PUBLIC_KEY_FILENAME, + ) + + with pytest.raises(ReleaseChainError, match="SPKI is not code-pinned"): + verify_producer_signature_bytes( + b'{"manifest":true}\n', + b"\0" * 64, + anchor_dir=anchors, + enforce_production_pin=True, + label="tampered-key.producer.sig", + ) + + +def test_committed_producer_public_key_matches_spki_pin(): + completed = subprocess.run( + [ + "openssl", + "pkey", + "-pubin", + "-in", + str(ROOT / "releases" / "anchors" / PRODUCER_PUBLIC_KEY_FILENAME), + "-outform", + "DER", + ], + cwd=ROOT, + check=True, + capture_output=True, + ) + + assert _sha256(completed.stdout) == PRODUCER_SPKI_SHA256 + + def test_cutter_no_tsa_writes_only_canonical_genesis( pregenesis_environment: ReleaseEnvironment, ): @@ -622,6 +815,7 @@ def test_cutter_no_tsa_writes_only_canonical_genesis( repo="PolicyEngine/ledger", branch="release-chain-test", no_tsa=True, + no_sign=True, anchor_dir=pregenesis_environment.anchors, ) @@ -633,6 +827,7 @@ def test_cutter_no_tsa_writes_only_canonical_genesis( assert digest == _sha256(raw) assert not path.with_name(f"{path.stem}.freetsa.tsr").exists() assert not path.with_name(f"{path.stem}.digicert.tsr").exists() + assert not producer_signature_path_for_manifest(path).exists() def test_cutter_builds_and_verifies_exact_next_append( @@ -647,6 +842,7 @@ def test_cutter_builds_and_verifies_exact_next_append( chain_environment.repo, repo="PolicyEngine/ledger", branch="release-chain-test", + signing_key=chain_environment.producer_signing_key, anchor_dir=chain_environment.anchors, requester=_local_timestamp_requester(chain_environment), ) @@ -657,6 +853,7 @@ def test_cutter_builds_and_verifies_exact_next_append( assert manifest["releaseIndex"] == 1 assert manifest["append"]["appendedRowCount"] == 1 assert manifest["append"]["appendedBytesSha256"] == _sha256(suffix) + assert len(producer_signature_path_for_manifest(path).read_bytes()) == 64 def test_cutter_rejects_wrong_second_tsa_without_partial_files( @@ -667,6 +864,7 @@ def test_cutter_rejects_wrong_second_tsa_without_partial_files( pregenesis_environment.repo, repo="PolicyEngine/ledger", branch="release-chain-test", + signing_key=pregenesis_environment.producer_signing_key, anchor_dir=pregenesis_environment.anchors, requester=_local_timestamp_requester( pregenesis_environment, @@ -680,6 +878,532 @@ def test_cutter_rejects_wrong_second_tsa_without_partial_files( assert not manifest_directory.exists() or not any(manifest_directory.iterdir()) +def test_cutter_no_sign_writes_receipt_only_release( + pregenesis_environment: ReleaseEnvironment, +): + path = cut_release_manifest( + pregenesis_environment.repo, + repo="PolicyEngine/ledger", + branch="release-chain-test", + no_sign=True, + anchor_dir=pregenesis_environment.anchors, + requester=_local_timestamp_requester(pregenesis_environment), + ) + + assert path.is_file() + assert path.with_name(f"{path.stem}.freetsa.tsr").is_file() + assert path.with_name(f"{path.stem}.digicert.tsr").is_file() + assert not producer_signature_path_for_manifest(path).exists() + + +def test_cutter_signed_no_tsa_writes_signature_only_release( + pregenesis_environment: ReleaseEnvironment, +): + path = cut_release_manifest( + pregenesis_environment.repo, + repo="PolicyEngine/ledger", + branch="release-chain-test", + no_tsa=True, + signing_key=pregenesis_environment.producer_signing_key, + anchor_dir=pregenesis_environment.anchors, + ) + + _manifest, raw, _digest = load_manifest(path) + signature_path = producer_signature_path_for_manifest(path) + assert signature_path.is_file() + assert len(signature_path.read_bytes()) == 64 + assert not path.with_name(f"{path.stem}.freetsa.tsr").exists() + assert not path.with_name(f"{path.stem}.digicert.tsr").exists() + verify_producer_signature_bytes( + raw, + signature_path.read_bytes(), + anchor_dir=pregenesis_environment.anchors, + enforce_production_pin=False, + label=signature_path.name, + ) + + +def test_cutter_self_verification_rejects_wrong_signing_key_without_outputs( + pregenesis_environment: ReleaseEnvironment, +): + wrong_key = _generate_producer_keypair( + pregenesis_environment.tsa / "wrong-cutter-key" + ) + + with pytest.raises(ReleaseChainError, match="producer Ed25519 signature"): + cut_release_manifest( + pregenesis_environment.repo, + repo="PolicyEngine/ledger", + branch="release-chain-test", + no_tsa=True, + signing_key=wrong_key, + anchor_dir=pregenesis_environment.anchors, + ) + + manifest_directory = ( + pregenesis_environment.repo / "releases" / "manifests" + ) + assert not manifest_directory.exists() or not any(manifest_directory.iterdir()) + + +def test_cutter_refuses_concurrent_signature_overwrite( + pregenesis_environment: ReleaseEnvironment, +): + now = datetime.now(timezone.utc) - timedelta(seconds=1) + ledger = ( + pregenesis_environment.repo + / "ledger" + / "official_observations.jsonl" + ).read_bytes() + immutable_prefix = ( + pregenesis_environment.repo / "ledger" / "immutable_prefix.json" + ).read_bytes() + producer = { + "repo": "PolicyEngine/ledger", + "branch": "release-chain-test", + } + manifest, raw = _build_manifest( + ledger, + immutable_prefix, + ChainVerification(()), + producer, + now=now, + ) + filename = manifest_filename(manifest["releaseIndex"], raw) + manifest_path = ( + pregenesis_environment.repo / "releases" / "manifests" / filename + ) + signature_path = producer_signature_path_for_manifest(manifest_path) + sentinel = b"concurrent-writer-owned-this-path" + local_request = _local_timestamp_requester(pregenesis_environment) + injected = False + + def racing_request(endpoint: str, query: bytes, timeout_seconds: float) -> bytes: + nonlocal injected + if not injected: + signature_path.parent.mkdir(parents=True) + signature_path.write_bytes(sentinel) + injected = True + return local_request(endpoint, query, timeout_seconds) + + with pytest.raises(ReleaseCutError, match="refusing to overwrite"): + cut_release_manifest( + pregenesis_environment.repo, + repo=producer["repo"], + branch=producer["branch"], + signing_key=pregenesis_environment.producer_signing_key, + anchor_dir=pregenesis_environment.anchors, + requester=racing_request, + now=now, + ) + + assert signature_path.read_bytes() == sentinel + assert not manifest_path.exists() + for receipt in ( + manifest_path.with_name(f"{manifest_path.stem}.freetsa.tsr"), + manifest_path.with_name(f"{manifest_path.stem}.digicert.tsr"), + ): + assert not receipt.exists() + + +def test_cutter_no_sign_reserves_omitted_signature_during_tsa_requests( + pregenesis_environment: ReleaseEnvironment, +): + now = datetime.now(timezone.utc) - timedelta(seconds=1) + ledger = ( + pregenesis_environment.repo + / "ledger" + / "official_observations.jsonl" + ).read_bytes() + immutable_prefix = ( + pregenesis_environment.repo / "ledger" / "immutable_prefix.json" + ).read_bytes() + producer = { + "repo": "PolicyEngine/ledger", + "branch": "release-chain-test", + } + manifest, raw = _build_manifest( + ledger, + immutable_prefix, + ChainVerification(()), + producer, + now=now, + ) + filename = manifest_filename(manifest["releaseIndex"], raw) + manifest_path = ( + pregenesis_environment.repo / "releases" / "manifests" / filename + ) + signature_path = producer_signature_path_for_manifest(manifest_path) + sentinel = b"concurrent-omitted-signature" + local_request = _local_timestamp_requester(pregenesis_environment) + injected = False + + def racing_request(endpoint: str, query: bytes, timeout_seconds: float) -> bytes: + nonlocal injected + if not injected: + signature_path.parent.mkdir(parents=True) + signature_path.write_bytes(sentinel) + injected = True + return local_request(endpoint, query, timeout_seconds) + + with pytest.raises(ReleaseCutError, match="refusing to overwrite"): + cut_release_manifest( + pregenesis_environment.repo, + repo=producer["repo"], + branch=producer["branch"], + no_sign=True, + anchor_dir=pregenesis_environment.anchors, + requester=racing_request, + now=now, + ) + + assert signature_path.read_bytes() == sentinel + assert not manifest_path.exists() + assert not manifest_path.with_name(f"{manifest_path.stem}.freetsa.tsr").exists() + assert not manifest_path.with_name(f"{manifest_path.stem}.digicert.tsr").exists() + + +def test_cutter_no_tsa_reserves_omitted_receipts_during_signing( + pregenesis_environment: ReleaseEnvironment, + monkeypatch: pytest.MonkeyPatch, +): + now = datetime.now(timezone.utc) - timedelta(seconds=1) + ledger = ( + pregenesis_environment.repo + / "ledger" + / "official_observations.jsonl" + ).read_bytes() + immutable_prefix = ( + pregenesis_environment.repo / "ledger" / "immutable_prefix.json" + ).read_bytes() + producer = { + "repo": "PolicyEngine/ledger", + "branch": "release-chain-test", + } + manifest, raw = _build_manifest( + ledger, + immutable_prefix, + ChainVerification(()), + producer, + now=now, + ) + filename = manifest_filename(manifest["releaseIndex"], raw) + manifest_path = ( + pregenesis_environment.repo / "releases" / "manifests" / filename + ) + receipt_path = manifest_path.with_name(f"{manifest_path.stem}.freetsa.tsr") + signature_path = producer_signature_path_for_manifest(manifest_path) + sentinel = b"concurrent-omitted-receipt" + real_sign = cutter_module._sign_manifest + + def racing_sign(*args, **kwargs): + signature = real_sign(*args, **kwargs) + receipt_path.parent.mkdir(parents=True) + receipt_path.write_bytes(sentinel) + return signature + + monkeypatch.setattr(cutter_module, "_sign_manifest", racing_sign) + + with pytest.raises(ReleaseCutError, match="refusing to overwrite"): + cut_release_manifest( + pregenesis_environment.repo, + repo=producer["repo"], + branch=producer["branch"], + no_tsa=True, + signing_key=pregenesis_environment.producer_signing_key, + anchor_dir=pregenesis_environment.anchors, + now=now, + ) + + assert receipt_path.read_bytes() == sentinel + assert not manifest_path.exists() + assert not signature_path.exists() + assert not manifest_path.with_name(f"{manifest_path.stem}.digicert.tsr").exists() + + +def test_cutter_rolls_back_signature_with_batch_on_postwrite_failure( + pregenesis_environment: ReleaseEnvironment, + monkeypatch: pytest.MonkeyPatch, +): + real_verify = cutter_module.verify_release_chain + calls = 0 + + def fail_committed_verification(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 3: + raise ReleaseChainError("forced post-write verification failure") + return real_verify(*args, **kwargs) + + monkeypatch.setattr( + cutter_module, + "verify_release_chain", + fail_committed_verification, + ) + + with pytest.raises(ReleaseChainError, match="forced post-write"): + cut_release_manifest( + pregenesis_environment.repo, + repo="PolicyEngine/ledger", + branch="release-chain-test", + signing_key=pregenesis_environment.producer_signing_key, + anchor_dir=pregenesis_environment.anchors, + requester=_local_timestamp_requester(pregenesis_environment), + ) + + manifest_directory = ( + pregenesis_environment.repo / "releases" / "manifests" + ) + assert calls == 3 + assert not manifest_directory.exists() or not any(manifest_directory.iterdir()) + + +def test_cutter_rollback_preserves_concurrent_replacement( + pregenesis_environment: ReleaseEnvironment, + monkeypatch: pytest.MonkeyPatch, +): + real_verify = cutter_module.verify_release_chain + calls = 0 + replacement_path: Path | None = None + replacement = b"concurrent-replacement-must-survive" + + def replace_then_fail(*args, **kwargs): + nonlocal calls, replacement_path + calls += 1 + if calls == 3: + manifest_directory = ( + pregenesis_environment.repo / "releases" / "manifests" + ) + replacement_path = next(manifest_directory.glob("*.producer.sig")) + replacement_path.unlink() + replacement_path.write_bytes(replacement) + raise ReleaseChainError("forced verification after replacement") + return real_verify(*args, **kwargs) + + monkeypatch.setattr( + cutter_module, + "verify_release_chain", + replace_then_fail, + ) + + with pytest.raises(ReleaseCutError, match="rollback could not remove"): + cut_release_manifest( + pregenesis_environment.repo, + repo="PolicyEngine/ledger", + branch="release-chain-test", + signing_key=pregenesis_environment.producer_signing_key, + anchor_dir=pregenesis_environment.anchors, + requester=_local_timestamp_requester(pregenesis_environment), + ) + + assert calls == 3 + assert replacement_path is not None + assert replacement_path.read_bytes() == replacement + remaining = list(replacement_path.parent.iterdir()) + assert remaining == [replacement_path] + + +def test_batch_write_refuses_intermediate_parent_symlink_swap( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + root = tmp_path / "repo" + root.mkdir() + external_releases = tmp_path / "external" / "releases" + (external_releases / "manifests").mkdir(parents=True) + displaced_releases = root / "releases-owned-by-cutter" + target = root / "releases" / "manifests" / "candidate.json" + real_mkdir = cutter_module.os.mkdir + swapped = False + + def racing_mkdir(path, mode=0o777, *, dir_fd=None): + nonlocal swapped + result = real_mkdir(path, mode, dir_fd=dir_fd) + if path == "releases" and dir_fd is not None and not swapped: + (root / "releases").rename(displaced_releases) + (root / "releases").symlink_to( + external_releases, + target_is_directory=True, + ) + swapped = True + return result + + monkeypatch.setattr(cutter_module.os, "mkdir", racing_mkdir) + + with pytest.raises(ReleaseCutError, match="stable real directory"): + cutter_module._exclusive_batch_write( + root, + {target: b"must-stay-inside-repository"}, + reserved_paths=[target], + verify_written=lambda: None, + ) + + assert swapped + assert (root / "releases").is_symlink() + assert not (external_releases / "manifests" / target.name).exists() + assert not (displaced_releases / "manifests" / target.name).exists() + + +def test_batch_write_detects_parent_swap_during_verification(tmp_path: Path): + root = tmp_path / "repo" + manifest_directory = root / "releases" / "manifests" + manifest_directory.mkdir(parents=True) + external_releases = tmp_path / "external" / "releases" + (external_releases / "manifests").mkdir(parents=True) + displaced_releases = root / "releases-displaced-during-verification" + target = manifest_directory / "candidate.json" + + def swap_parent() -> None: + (root / "releases").rename(displaced_releases) + (root / "releases").symlink_to( + external_releases, + target_is_directory=True, + ) + + with pytest.raises(ReleaseCutError, match="stable real directory"): + cutter_module._exclusive_batch_write( + root, + {target: b"verified-candidate"}, + reserved_paths=[target], + verify_written=swap_parent, + ) + + assert (root / "releases").is_symlink() + assert not (external_releases / "manifests" / target.name).exists() + assert not ( + displaced_releases / "manifests" / target.name + ).exists() + + +def test_batch_rollback_preserves_replacement_manifest_directory(tmp_path: Path): + root = tmp_path / "repo" + root.mkdir() + manifest_directory = root / "releases" / "manifests" + displaced_directory = root / "releases" / "manifests-owned-by-cutter" + target = manifest_directory / "candidate.json" + + def replace_manifest_directory() -> None: + manifest_directory.rename(displaced_directory) + manifest_directory.mkdir() + + with pytest.raises(ReleaseCutError, match="manifest directory was replaced"): + cutter_module._exclusive_batch_write( + root, + {target: b"owned-candidate"}, + reserved_paths=[target], + verify_written=replace_manifest_directory, + ) + + assert manifest_directory.is_dir() + assert not any(manifest_directory.iterdir()) + assert displaced_directory.is_dir() + assert not (displaced_directory / target.name).exists() + + +def test_batch_rollback_preserves_replacement_releases_directory(tmp_path: Path): + root = tmp_path / "repo" + root.mkdir() + manifest_directory = root / "releases" / "manifests" + displaced_releases = root / "releases-owned-by-cutter" + target = manifest_directory / "candidate.json" + + def replace_releases_directory() -> None: + (root / "releases").rename(displaced_releases) + (root / "releases").mkdir() + + with pytest.raises(ReleaseCutError, match="output parent was replaced"): + cutter_module._exclusive_batch_write( + root, + {target: b"owned-candidate"}, + reserved_paths=[target], + verify_written=replace_releases_directory, + ) + + replacement_releases = root / "releases" + assert replacement_releases.is_dir() + assert not any(replacement_releases.iterdir()) + assert displaced_releases.is_dir() + assert not (displaced_releases / "manifests" / target.name).exists() + + +@pytest.mark.parametrize( + "arguments", + [ + [], + ["--signing-key", "test-private.pem", "--no-sign"], + ], +) +def test_cutter_cli_requires_exactly_one_signing_mode( + arguments: list[str], + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(sys, "argv", ["cut_release_manifest.py", *arguments]) + + with pytest.raises(SystemExit) as raised: + cutter_module.main() + + assert raised.value.code == 2 + + +@pytest.mark.parametrize( + ("signing_arguments", "expected"), + [ + ( + ["--signing-key", "runtime-test-private.pem", "--no-tsa"], + { + "signing_key": Path("runtime-test-private.pem"), + "no_sign": False, + "no_tsa": True, + "message": "producer-signed release written without TSA receipts", + }, + ), + ( + ["--no-sign"], + { + "signing_key": None, + "no_sign": True, + "no_tsa": False, + "message": "timestamped release written without producer signature", + }, + ), + ], +) +def test_cutter_cli_forwards_explicit_signing_mode_and_reports_output( + signing_arguments: list[str], + expected: dict[str, Any], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +): + captured: dict[str, Any] = {} + returned = tmp_path / "releases" / "manifests" / "0000-test.json" + + def fake_cut(root: Path, **kwargs: Any) -> Path: + captured["root"] = root + captured.update(kwargs) + return returned + + monkeypatch.setattr(cutter_module, "cut_release_manifest", fake_cut) + monkeypatch.setattr( + sys, + "argv", + [ + "cut_release_manifest.py", + "--root", + str(tmp_path), + *signing_arguments, + ], + ) + + assert cutter_module.main() == 0 + + assert captured["root"] == tmp_path + assert captured["signing_key"] == expected["signing_key"] + assert captured["no_sign"] is expected["no_sign"] + assert captured["no_tsa"] is expected["no_tsa"] + assert expected["message"] in capsys.readouterr().out + + @pytest.mark.parametrize( "tamper", [ @@ -732,6 +1456,7 @@ def test_verifier_cli_catches_manifest_and_receipt_tampering( head, head.with_name(f"{head.stem}.freetsa.tsr"), head.with_name(f"{head.stem}.digicert.tsr"), + producer_signature_path_for_manifest(head), ): path.unlink() if tamper == "wrong-previous": diff --git a/tests/test_thesis_append_adversarial.py b/tests/test_thesis_append_adversarial.py index 4dcf298..30b404b 100644 --- a/tests/test_thesis_append_adversarial.py +++ b/tests/test_thesis_append_adversarial.py @@ -13,6 +13,7 @@ import copy import hashlib import json +import os import subprocess import sys from pathlib import Path @@ -175,6 +176,185 @@ def _extend_manifest_to_all_lines(lines: list[str]) -> dict: return manifest +def _initialize_surface_fixture(path: Path) -> tuple[list[str], str]: + lines = _read_lines() + manifest = json.loads(PREFIX_PATH.read_text(encoding="utf-8")) + _write_checker_fixture(path, "\n".join(lines) + "\n", manifest) + return lines, _init_fixture_repo(path) + + +def _append_surface_fixture_row(path: Path, lines: list[str], identity: str) -> None: + row = _appended_row( + json.loads(lines[-1]), + value_delta=1, + source_record_id=identity, + ) + (path / "ledger" / "official_observations.jsonl").write_text( + "\n".join([*lines, _json_line(row)]) + "\n", + encoding="utf-8", + ) + + +def _add_untracked_gate_anchor(path: Path) -> Path: + anchor = path / "releases" / "anchors" / "test-producer.pub" + anchor.parent.mkdir(parents=True, exist_ok=True) + anchor.write_text("test public key only\n", encoding="utf-8") + return anchor + + +def test_two_class_data_only_proposal_passes(tmp_path): + lines, base = _initialize_surface_fixture(tmp_path) + _append_surface_fixture_row( + tmp_path, + lines, + "verification.surface.data-only", + ) + + completed = _run_checker(tmp_path, base) + + assert completed.returncode == 0, completed.stdout + completed.stderr + assert "+1 appended vs base" in completed.stdout + + +def test_two_class_gate_only_proposal_passes(tmp_path): + _lines, base = _initialize_surface_fixture(tmp_path) + _add_untracked_gate_anchor(tmp_path) + + completed = _run_checker(tmp_path, base) + + assert completed.returncode == 0, completed.stdout + completed.stderr + assert "gate-only proposal" in completed.stdout + assert "DATA_SURFACE unchanged" in completed.stdout + assert "releases/anchors/test-producer.pub" in completed.stdout + + +def test_two_class_mixed_proposal_is_rejected_before_data_checks(tmp_path): + lines, base = _initialize_surface_fixture(tmp_path) + _append_surface_fixture_row( + tmp_path, + lines, + "verification.surface.mixed", + ) + _add_untracked_gate_anchor(tmp_path) + + completed = _run_checker(tmp_path, base) + + assert completed.returncode == 1, completed.stdout + completed.stderr + assert ( + "mixed data/gate proposal is forbidden: DATA_SURFACE changes=" + "['ledger/official_observations.jsonl']; GATE_SURFACE changes=" + "['releases/anchors/test-producer.pub']; split them into separate " + "pull requests" + ) in completed.stderr + + +@pytest.mark.parametrize( + "dependency", + ["canonical_json.py", "verify_release_chain.py"], +) +def test_base_gate_uses_base_script_and_dependency_imports( + tmp_path, + dependency: str, +): + candidate = tmp_path / "candidate" + candidate.mkdir() + _lines, base = _initialize_surface_fixture(candidate) + + base_gate = tmp_path / "base-gate" + subprocess.run( + ["git", "clone", "--no-checkout", str(candidate), str(base_gate)], + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + [ + "git", + "-C", + str(base_gate), + "fetch", + "--no-tags", + "--depth", + "1", + "origin", + base, + ], + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + ["git", "-C", str(base_gate), "checkout", "--detach", base], + check=True, + capture_output=True, + text=True, + ) + + candidate_gate_marker = tmp_path / "candidate-gate-executed" + candidate_dependency_marker = tmp_path / "candidate-dependency-imported" + (candidate / "scripts" / "check_thesis_facts_append.py").write_text( + "from pathlib import Path\n" + f"Path({str(candidate_gate_marker)!r}).write_text('executed')\n" + "raise SystemExit(97)\n", + encoding="utf-8", + ) + (candidate / "scripts" / dependency).write_text( + "from pathlib import Path\n" + f"Path({str(candidate_dependency_marker)!r}).write_text('imported')\n" + "raise RuntimeError('candidate dependency imported')\n", + encoding="utf-8", + ) + + environment = os.environ.copy() + environment["PYTHONPATH"] = str(base_gate / "scripts") + environment["PYTHONNOUSERSITE"] = "1" + completed = subprocess.run( + [ + sys.executable, + str(base_gate / "scripts" / "check_thesis_facts_append.py"), + "--root", + str(candidate), + "--base-ref", + base, + ], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + ) + + assert completed.returncode == 0, completed.stdout + completed.stderr + assert "gate-only proposal" in completed.stdout + assert not candidate_gate_marker.exists() + assert not candidate_dependency_marker.exists() + + +def test_workflow_has_a_base_owned_trusted_pr_gate(): + workflow = ( + ROOT / ".github" / "workflows" / "thesis-facts-append.yml" + ).read_text(encoding="utf-8") + + # A pull_request workflow is loaded from the candidate merge ref, so its + # detached base-script step can itself be replaced. Once installed on the + # default branch, the base-owned target event is not candidate-controlled; + # it fetches the merge ref as inert data and executes only base modules. + for required in ( + "\n pull_request_target:\n", + "permissions:\n contents: read\n", + "trusted-base-append-gate:", + "name: Trusted base append gate", + '[[ "$BASE_SHA" =~ ^[0-9a-f]{40,64}$ ]]', + '[[ "$MERGE_SHA" =~ ^[0-9a-f]{40,64}$ ]]', + '[[ "$PR_NUMBER" =~ ^[0-9]+$ ]]', + '"+refs/pull/${PR_NUMBER}/merge:refs/remotes/origin/pr-merge"', + 'PYTHONPATH="$base_gate/scripts"', + 'python3 "$base_gate/scripts/check_thesis_facts_append.py"', + '--root "$candidate"', + '--base-ref "$BASE_SHA"', + ): + assert required in workflow + + def test_base_check_rejects_an_existing_line_rewrite(): lines = _read_lines() rewritten = json.loads(lines[-1]) diff --git a/uv.lock b/uv.lock index a88cf35..3a7c2b8 100644 --- a/uv.lock +++ b/uv.lock @@ -285,6 +285,7 @@ name = "policyengine-arch-data" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "cryptography" }, { name = "httpx" }, { name = "microplex" }, { name = "numpy" }, @@ -316,6 +317,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "cryptography", specifier = ">=46.0.3" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "microplex", specifier = ">=0.1.0" }, { name = "numpy", specifier = ">=1.24.0" },