diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52fbd20 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# Files whose exact Git and working-tree bytes are hashed by the release chain. +ledger/official_observations.jsonl text eol=lf +ledger/immutable_prefix.json text eol=lf +releases/manifests/*.json text eol=lf +releases/anchors/*.pem text eol=lf + +# RFC 3161 timestamp responses are DER binary. +releases/manifests/*.tsr binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da65e34..9647692 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,12 @@ jobs: - name: Check out repository uses: actions/checkout@v4 + - name: Verify witnessed release chain + if: >- + github.event_name == 'push' && + github.ref == 'refs/heads/codex/thesis-ledger-facts' + run: python3 scripts/verify_release_chain.py --full + - name: Install uv uses: astral-sh/setup-uv@v5 @@ -65,8 +71,12 @@ jobs: tests/test_hierarchical_pipeline.py tests/test_supabase_client.py tests/test_policyengine_ledger.py + tests/test_release_chain.py + tests/test_thesis_append_adversarial.py scripts/check_thesis_facts_append.py scripts/canonical_json.py + scripts/cut_release_manifest.py + scripts/verify_release_chain.py - name: Test Arch surface run: > @@ -90,6 +100,8 @@ jobs: tests/test_hierarchical_pipeline.py tests/test_supabase_client.py tests/test_policyengine_ledger.py + tests/test_release_chain.py + tests/test_thesis_append_adversarial.py -q - name: Build target input database diff --git a/.github/workflows/thesis-facts-append.yml b/.github/workflows/thesis-facts-append.yml index a04a602..cff2c1e 100644 --- a/.github/workflows/thesis-facts-append.yml +++ b/.github/workflows/thesis-facts-append.yml @@ -3,8 +3,9 @@ name: Thesis facts append gate # Deterministic review for every change to the observation ledger. Resolver # appends arrive as pull requests targeting codex/thesis-ledger-facts; this # gate enforces the immutable frozen prefix, an append-only diff against the -# PR base, per-row schema and binding requirements, and explicit supersede -# semantics for corrections. Direct pushes get the same full-file checks. +# PR base, the witnessed release chain, per-row schema and binding requirements, +# and explicit supersede semantics for corrections. Direct pushes get the same +# full-file checks. on: pull_request: @@ -12,14 +13,23 @@ on: - codex/thesis-ledger-facts paths: - "ledger/**" + - "releases/**" - "scripts/check_thesis_facts_append.py" - "scripts/canonical_json.py" + - "scripts/cut_release_manifest.py" + - "scripts/verify_release_chain.py" - ".github/workflows/thesis-facts-append.yml" push: branches: - codex/thesis-ledger-facts paths: - "ledger/**" + - "releases/**" + - "scripts/check_thesis_facts_append.py" + - "scripts/canonical_json.py" + - "scripts/cut_release_manifest.py" + - "scripts/verify_release_chain.py" + - ".github/workflows/thesis-facts-append.yml" jobs: append-gate: @@ -35,10 +45,11 @@ jobs: - name: Enforce append-only ledger invariants run: | set -euo pipefail - if [ -n "${{ github.base_ref }}" ]; then - git fetch origin "${{ github.base_ref }}" --depth=1 + if [ "${{ github.event_name }}" = "pull_request" ]; then + base_sha="${{ github.event.pull_request.base.sha }}" + git cat-file -e "${base_sha}^{commit}" python3 scripts/check_thesis_facts_append.py \ - --base-ref "origin/${{ github.base_ref }}" + --base-ref "$base_sha" else python3 scripts/check_thesis_facts_append.py fi @@ -55,4 +66,9 @@ jobs: run: uv sync --locked --all-extras - name: Ledger observation invariants - run: uv run pytest tests/test_policyengine_ledger.py -q + run: > + uv run pytest + tests/test_policyengine_ledger.py + tests/test_release_chain.py + tests/test_thesis_append_adversarial.py + -q diff --git a/ledger/README.md b/ledger/README.md index 7bba850..8886627 100644 --- a/ledger/README.md +++ b/ledger/README.md @@ -7,3 +7,8 @@ forecast distributions, agent traces, or forecast scores. official observations that downstream systems can use as resolution facts. Each row should keep `source_record_id` stable and source-specific, because downstream prediction systems resolve against that ID. + +Proposed ledger states in the gated append flow are recorded in chained, RFC +3161-witnessed release manifests. See +[`../releases/README.md`](../releases/README.md) for the manifest schema, offline +verification procedure, and security limitations. diff --git a/releases/README.md b/releases/README.md new file mode 100644 index 0000000..054788d --- /dev/null +++ b/releases/README.md @@ -0,0 +1,121 @@ +# Witnessed ledger releases + +In the gated append flow, each proposed state of +`ledger/official_observations.jsonl` is committed by a canonical release manifest +and witnessed by two RFC 3161 timestamp authorities. The manifests form an +append-only hash chain. The receipts let a verifier show that the exact manifest +bytes existed no later than each receipt's `genTime`. + +## Files + +For a stem `-`, where `index` is the zero-padded four-digit +release index and `hash16` is the first 16 lowercase hexadecimal characters of +the manifest file's SHA-256 digest, a release consists of exactly these files: + +- `releases/manifests/.json` +- `releases/manifests/.freetsa.tsr` +- `releases/manifests/.digicert.tsr` + +For example, the receipt for manifest `0007-0123456789abcdef.json` is named +`0007-0123456789abcdef.freetsa.tsr`, not +`0007-0123456789abcdef.json.freetsa.tsr`. Receipts are DER-encoded RFC 3161 +responses whose SHA-256 message imprint covers the manifest file's exact bytes. +Those bytes are canonical JSON produced by `scripts/canonical_json.py`, followed +by one newline. + +The pinned TSA verification chains are committed as PEM files under +`releases/anchors/`. Verification does not contact either TSA or any other +network service. + +## Manifest schema + +Every manifest uses the closed-world `thesis_ledger_release_v1` schema. Unknown +keys are invalid at every level, and counts are JSON integers (booleans are not +accepted as integers). The required root members are: + +- `schemaVersion`: the literal `"thesis_ledger_release_v1"`. +- `releaseIndex`: a contiguous integer beginning at zero. +- `previousManifestSha256`: `null` for genesis; otherwise the full lowercase + SHA-256 digest of the previous manifest file's exact bytes. +- `state`: an object containing only: + - `path`: the literal `"ledger/official_observations.jsonl"`. + - `jsonlSha256`: the lowercase SHA-256 digest of the ledger bytes represented + by this release. + - `lineCount`: the number of JSONL rows represented by this release. + - `immutablePrefixSha256`: the lowercase SHA-256 digest of the exact bytes of + `ledger/immutable_prefix.json`. +- `append`: `null` for genesis; otherwise an object containing only: + - `previousLineCount`: the preceding manifest's `state.lineCount`. + - `appendedRowCount`: the number of newly appended JSONL rows. + - `appendedBytesSha256`: the lowercase SHA-256 digest of the exact byte suffix + added after the preceding state. +- `createdAtUtc`: a strict UTC timestamp ending in `Z`. +- `producer`: an object containing only the free-form provenance strings `repo` + and `branch`. These strings are recorded claims, not trusted authorization. + +Genesis has index zero, a null previous hash, and a null append block. Every +later release increments the index by one, hashes the preceding manifest file, +increases the line count, and binds both the row-count delta and the exact byte +suffix in its append block. Both receipts must verify against their separately +pinned anchor chain and cover that release's exact manifest bytes. + +## Offline verification + +Clone the repository at the state you want to inspect and run: + +```console +python3 scripts/verify_release_chain.py --full +``` + +The verifier needs only Python, OpenSSL, the committed manifests and receipts, +the committed TSA anchors, and the ledger files. It checks canonical bytes, +filenames, contiguous indices, previous-manifest links, state and append +commitments, both timestamp receipts, and timestamp ordering. Any mismatch exits +nonzero with an error identifying the failed invariant. + +Retain a trusted checkpoint outside this repository, at minimum the full SHA-256 +digest of a previously accepted head manifest, and compare it with later clones. +Internal verification proves that a clone is self-consistent; by itself it cannot +distinguish the original history from a complete, freshly witnessed replacement +fork. RFC 3161 timestamp authorities attest timestamps for submitted digests but +do not publish a uniqueness-enforcing append-only ledger for this repository. + +## Security properties and limits + +Under the configured proposal gate, every ledger append arrives with the next +manifest and both receipts. This binds the proposed ledger state, immutable +prefix, previous release, row-count change, and exact appended byte suffix into a +witnessed chain. Rewriting a checkpointed manifest or producing an unwitnessed or +internally inconsistent state is therefore detectable by any verifier holding +that checkpoint. + +An RFC 3161 `genTime` establishes that the manifest existed no later than that +time. It does **not** timestamp GitHub's merge or prove when the organization +accepted the proposal. In the intended pull-request flow the receipts are created +before the gate can pass and therefore before merge; their times do not upper-bound +the later acceptance time. + +This mechanism provides tamper evidence, not multi-party authorization or admin +non-repudiation. Governance remains within one GitHub organization: + +- It does not require approval by an independent institution or a second party. +- An unwitnessed or inconsistent admin direct push turns verification and CI red, + but repository controls cannot prevent every admin bypass. +- An admin direct push that includes a valid next manifest and both valid receipts + is neither prevented nor distinguishable by offline cryptographic verification + from an append merged through the intended pull-request path. +- The verifier, workflows, and anchors live in the repository they verify. Running + the verifier from a clone assumes those security files were not weakened in the + same rewrite; independent verification should pin or audit them separately as + well as retaining a manifest checkpoint. +- The manifest chain hashes manifests, not receipt files. A receipt can be + replaced by a newer valid receipt over the same manifest without changing later + manifest hashes unless its exact bytes or digest were retained externally. +- The production anchors are fixed. There is no in-schema anchor-rotation + protocol, so a TSA chain change requires an explicit verifier/schema migration + that continues to preserve verification of the old chain. +- Four-digit indices cap this filename format at release 9999; the current schema + does not define a rollover. + +These limitations are why a retained external checkpoint is essential and why +the current design should not be described as cross-institution governance. diff --git a/releases/anchors/digicert-trusted-root-g4.pem b/releases/anchors/digicert-trusted-root-g4.pem new file mode 100644 index 0000000..4214227 --- /dev/null +++ b/releases/anchors/digicert-trusted-root-g4.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFkDCCA3igAwIBAgIQBZsbV56OITLiOQe9p3d1XDANBgkqhkiG9w0BAQwFADBi +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3Qg +RzQwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBiMQswCQYDVQQGEwJV +UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu +Y29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQC/5pBzaN675F1KPDAiMGkz7MKnJS7JIT3y +ithZwuEppz1Yq3aaza57G4QNxDAf8xukOBbrVsaXbR2rsnnyyhHS5F/WBTxSD1If +xp4VpX6+n6lXFllVcq9ok3DCsrp1mWpzMpTREEQQLt+C8weE5nQ7bXHiLQwb7iDV +ySAdYyktzuxeTsiT+CFhmzTrBcZe7FsavOvJz82sNEBfsXpm7nfISKhmV1efVFiO +DCu3T6cw2Vbuyntd463JT17lNecxy9qTXtyOj4DatpGYQJB5w3jHtrHEtWoYOAMQ +jdjUN6QuBX2I9YI+EJFwq1WCQTLX2wRzKm6RAXwhTNS8rhsDdV14Ztk6MUSaM0C/ +CNdaSaTC5qmgZ92kJ7yhTzm1EVgX9yRcRo9k98FpiHaYdj1ZXUJ2h4mXaXpI8OCi +EhtmmnTK3kse5w5jrubU75KSOp493ADkRSWJtppEGSt+wJS00mFt6zPZxd9LBADM +fRyVw4/3IbKyEbe7f/LVjHAsQWCqsWMYRJUadmJ+9oCw++hkpjPRiQfhvbfmQ6QY +uKZ3AeEPlAwhHbJUKSWJbOUOUlFHdL4mrLZBdd56rF+NP8m800ERElvlEFDrMcXK +chYiCd98THU/Y+whX8QgUWtvsauGi0/C1kVfnSD8oR7FwI+isX4KJpn15GkvmB0t +9dmpsh3lGwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB +hjAdBgNVHQ4EFgQU7NfjgtJxXWRM3y5nP+e6mK4cD08wDQYJKoZIhvcNAQEMBQAD +ggIBALth2X2pbL4XxJEbw6GiAI3jZGgPVs93rnD5/ZpKmbnJeFwMDF/k5hQpVgs2 +SV1EY+CtnJYYZhsjDT156W1r1lT40jzBQ0CuHVD1UvyQO7uYmWlrx8GnqGikJ9yd ++SeuMIW59mdNOj6PWTkiU0TryF0Dyu1Qen1iIQqAyHNm0aAFYF/opbSnr6j3bTWc +fFqK1qI4mfN4i/RN0iAL3gTujJtHgXINwBQy7zBZLq7gcfJW5GqXb5JQbZaNaHqa +sjYUegbyJLkJEVDXCLG4iXqEI2FCKeWjzaIgQdfRnGTZ6iahixTXTBmyUEFxPT9N +cCOGDErcgdLMMpSEDQgJlxxPwO5rIHQw0uA5NBCFIRUBCOhVMt5xSdkoF1BN5r5N +0XWs0Mr7QbhDparTwwVETyw2m+L64kW4I1NsBm9nVX9GtUw/bihaeSbSpKhil9Ie +4u1Ki7wb/UdKDd9nZn6yW0HQO+T0O/QEY+nvwlQAUaCKKsnOeMzV6ocEGLPOr0mI +r/OSmbaz5mEP0oUA51Aa5BuVnRmhuZyxm7EAHu/QD09CbMkKvO5D+jpxpchNJqU1 +/YldvIViHTLSoCtU7ZpXwdv6EM8Zt4tKG48BtieVU+i2iW1bvGjUI+iLUaJW+fCm +gKDWHrO8Dw9TdSmq6hN35N6MgSGtBxBHEa2HPQfRdbzP82Z+ +-----END CERTIFICATE----- diff --git a/releases/anchors/freetsa-root-2016.pem b/releases/anchors/freetsa-root-2016.pem new file mode 100644 index 0000000..c144895 --- /dev/null +++ b/releases/anchors/freetsa-root-2016.pem @@ -0,0 +1,45 @@ +-----BEGIN CERTIFICATE----- +MIIH/zCCBeegAwIBAgIJAMHphhYNqOmAMA0GCSqGSIb3DQEBDQUAMIGVMREwDwYD +VQQKEwhGcmVlIFRTQTEQMA4GA1UECxMHUm9vdCBDQTEYMBYGA1UEAxMPd3d3LmZy +ZWV0c2Eub3JnMSIwIAYJKoZIhvcNAQkBFhNidXNpbGV6YXNAZ21haWwuY29tMRIw +EAYDVQQHEwlXdWVyemJ1cmcxDzANBgNVBAgTBkJheWVybjELMAkGA1UEBhMCREUw +HhcNMTYwMzEzMDE1MjEzWhcNNDEwMzA3MDE1MjEzWjCBlTERMA8GA1UEChMIRnJl +ZSBUU0ExEDAOBgNVBAsTB1Jvb3QgQ0ExGDAWBgNVBAMTD3d3dy5mcmVldHNhLm9y +ZzEiMCAGCSqGSIb3DQEJARYTYnVzaWxlemFzQGdtYWlsLmNvbTESMBAGA1UEBxMJ +V3VlcnpidXJnMQ8wDQYDVQQIEwZCYXllcm4xCzAJBgNVBAYTAkRFMIICIjANBgkq +hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtgKODjAy8REQ2WTNqUudAnjhlCrpE6ql +mQfNppeTmVvZrH4zutn+NwTaHAGpjSGv4/WRpZ1wZ3BRZ5mPUBZyLgq0YrIfQ5Fx +0s/MRZPzc1r3lKWrMR9sAQx4mN4z11xFEO529L0dFJjPF9MD8Gpd2feWzGyptlel +b+PqT+++fOa2oY0+NaMM7l/xcNHPOaMz0/2olk0i22hbKeVhvokPCqhFhzsuhKsm +q4Of/o+t6dI7sx5h0nPMm4gGSRhfq+z6BTRgCrqQG2FOLoVFgt6iIm/BnNffUr7V +DYd3zZmIwFOj/H3DKHoGik/xK3E82YA2ZulVOFRW/zj4ApjPa5OFbpIkd0pmzxzd +EcL479hSA9dFiyVmSxPtY5ze1P+BE9bMU1PScpRzw8MHFXxyKqW13Qv7LWw4sbk3 +SciB7GACbQiVGzgkvXG6y85HOuvWNvC5GLSiyP9GlPB0V68tbxz4JVTRdw/Xn/XT +FNzRBM3cq8lBOAVt/PAX5+uFcv1S9wFE8YjaBfWCP1jdBil+c4e+0tdywT2oJmYB +BF/kEt1wmGwMmHunNEuQNzh1FtJY54hbUfiWi38mASE7xMtMhfj/C4SvapiDN837 +gYaPfs8x3KZxbX7C3YAsFnJinlwAUss1fdKar8Q/YVs7H/nU4c4Ixxxz4f67fcVq +M2ITKentbCMCAwEAAaOCAk4wggJKMAwGA1UdEwQFMAMBAf8wDgYDVR0PAQH/BAQD +AgHGMB0GA1UdDgQWBBT6VQ2MNGZRQ0z357OnbJWveuaklzCBygYDVR0jBIHCMIG/ +gBT6VQ2MNGZRQ0z357OnbJWveuakl6GBm6SBmDCBlTERMA8GA1UEChMIRnJlZSBU +U0ExEDAOBgNVBAsTB1Jvb3QgQ0ExGDAWBgNVBAMTD3d3dy5mcmVldHNhLm9yZzEi +MCAGCSqGSIb3DQEJARYTYnVzaWxlemFzQGdtYWlsLmNvbTESMBAGA1UEBxMJV3Vl +cnpidXJnMQ8wDQYDVQQIEwZCYXllcm4xCzAJBgNVBAYTAkRFggkAwemGFg2o6YAw +MwYDVR0fBCwwKjAooCagJIYiaHR0cDovL3d3dy5mcmVldHNhLm9yZy9yb290X2Nh +LmNybDCBzwYDVR0gBIHHMIHEMIHBBgorBgEEAYHyJAEBMIGyMDMGCCsGAQUFBwIB +FidodHRwOi8vd3d3LmZyZWV0c2Eub3JnL2ZyZWV0c2FfY3BzLmh0bWwwMgYIKwYB +BQUHAgEWJmh0dHA6Ly93d3cuZnJlZXRzYS5vcmcvZnJlZXRzYV9jcHMucGRmMEcG +CCsGAQUFBwICMDsaOUZyZWVUU0EgdHJ1c3RlZCB0aW1lc3RhbXBpbmcgU29mdHdh +cmUgYXMgYSBTZXJ2aWNlIChTYWFTKTA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUH +MAGGG2h0dHA6Ly93d3cuZnJlZXRzYS5vcmc6MjU2MDANBgkqhkiG9w0BAQ0FAAOC +AgEAaK9+v5OFYu9M6ztYC+L69sw1omdyli89lZAfpWMMh9CRmJhM6KBqM/ipwoLt +nxyxGsbCPhcQjuTvzm+ylN6VwTMmIlVyVSLKYZcdSjt/eCUN+41K7sD7GVmxZBAF +ILnBDmTGJmLkrU0KuuIpj8lI/E6Z6NnmuP2+RAQSHsfBQi6sssnXMo4HOW5gtPO7 +gDrUpVXID++1P4XndkoKn7Svw5n0zS9fv1hxBcYIHPPQUze2u30bAQt0n0iIyRLz +aWuhtpAtd7ffwEbASgzB7E+NGF4tpV37e8KiA2xiGSRqT5ndu28fgpOY87gD3ArZ +DctZvvTCfHdAS5kEO3gnGGeZEVLDmfEsv8TGJa3AljVa5E40IQDsUXpQLi8G+UC4 +1DWZu8EVT4rnYaCw1VX7ShOR1PNCCvjb8S8tfdudd9zhU3gEB0rxdeTy1tVbNLXW +99y90xcwr1ZIDUwM/xQ/noO8FRhm0LoPC73Ef+J4ZBdrvWwauF3zJe33d4ibxEcb +8/pz5WzFkeixYM2nsHhqHsBKw7JPouKNXRnl5IAE1eFmqDyC7G/VT7OF669xM6hb +Ut5G21JE4cNK6NNucS+fzg1JPX0+3VhsYZjj7D5uljRvQXrJ8iHgr/M6j2oLHvTA +I2MLdq2qjZFDOCXsxBxJpbmLGBx9ow6ZerlUxzws2AWv2pk= +-----END CERTIFICATE----- diff --git a/releases/manifests/0000-307cedbc91de43be.digicert.tsr b/releases/manifests/0000-307cedbc91de43be.digicert.tsr new file mode 100644 index 0000000..c12d825 Binary files /dev/null and b/releases/manifests/0000-307cedbc91de43be.digicert.tsr differ diff --git a/releases/manifests/0000-307cedbc91de43be.freetsa.tsr b/releases/manifests/0000-307cedbc91de43be.freetsa.tsr new file mode 100644 index 0000000..acf3892 Binary files /dev/null and b/releases/manifests/0000-307cedbc91de43be.freetsa.tsr differ diff --git a/releases/manifests/0000-307cedbc91de43be.json b/releases/manifests/0000-307cedbc91de43be.json new file mode 100644 index 0000000..2362599 --- /dev/null +++ b/releases/manifests/0000-307cedbc91de43be.json @@ -0,0 +1 @@ +{"append":null,"createdAtUtc":"2026-07-11T14:13:39Z","previousManifestSha256":null,"producer":{"branch":"witnessed-journal","repo":"PolicyEngine/ledger"},"releaseIndex":0,"schemaVersion":"thesis_ledger_release_v1","state":{"immutablePrefixSha256":"db5d69575db0adb505b1b46b0496befb852315b3ff8c7690313fe77319578c0b","jsonlSha256":"590b71bed9a7fc0354f2449c5e533ed20b508fb3554e395b1bdd283b945f0ea5","lineCount":143,"path":"ledger/official_observations.jsonl"}} diff --git a/scripts/check_thesis_facts_append.py b/scripts/check_thesis_facts_append.py index 38b2456..08e8711 100644 --- a/scripts/check_thesis_facts_append.py +++ b/scripts/check_thesis_facts_append.py @@ -17,6 +17,9 @@ - a duplicate ``source_record_id`` is legal only as an explicit correction: the later row's ``assertionVersion.supersedes`` must name the version ID of the row it replaces. +- after witnessed genesis, every byte append carries exactly one next canonical + release manifest and two independently anchored RFC 3161 receipts; all prior + release files remain byte-immutable against the PR's base commit. Usage: python3 scripts/check_thesis_facts_append.py [--base-ref REF] @@ -37,10 +40,25 @@ from typing import Any from canonical_json import canonical_sha256 +from verify_release_chain import ( + ANCHORS, + MANIFEST_RE, + ReleaseChainError, + git_blob_bytes, + git_file_entry, + verify_base_release_chain, + verify_release_chain, + verify_release_history_immutable, +) ROOT = pathlib.Path(__file__).resolve().parents[1] LEDGER_PATH = ROOT / "ledger" / "official_observations.jsonl" PREFIX_PATH = ROOT / "ledger" / "immutable_prefix.json" +RELEASE_MANIFEST_PREFIX = "releases/manifests/" +GENESIS_SUPPORT_FILES = { + "releases/README.md", + *(f"releases/anchors/{spec.filename}" for spec in ANCHORS.values()), +} ASSERTION_CONTENT_KEYS = ( "source_record_id", @@ -355,12 +373,204 @@ def check_prefix_anchored_to_base(base_ref: str, candidate_prefix: dict[str, Any return int(base_prefix["prefixLineCount"]) +def _release_triple( + new_files: set[str], + expected_index: int, + *, + allowed_support_files: set[str] | None = None, +) -> pathlib.Path: + """Require exactly one new manifest and its two mapped receipt files.""" + + manifest_files = [ + relative + for relative in new_files + if relative.startswith(RELEASE_MANIFEST_PREFIX) + and MANIFEST_RE.fullmatch(pathlib.PurePosixPath(relative).name) + ] + if len(manifest_files) != 1: + raise AppendError( + f"release proposal must add exactly one manifest for index " + f"{expected_index}; found {sorted(manifest_files)}" + ) + manifest_relative = manifest_files[0] + manifest_name = pathlib.PurePosixPath(manifest_relative).name + match = MANIFEST_RE.fullmatch(manifest_name) + assert match is not None + if int(match.group("index")) != expected_index: + raise AppendError( + f"release proposal index must be {expected_index}, not " + f"{int(match.group('index'))}" + ) + stem = pathlib.PurePosixPath(manifest_name).stem + expected = { + manifest_relative, + *(f"{RELEASE_MANIFEST_PREFIX}{stem}.{tsa}.tsr" for tsa in ANCHORS), + } + allowed = expected | (allowed_support_files or set()) + if new_files != expected and not ( + expected <= new_files and new_files <= allowed + ): + raise AppendError( + "release proposal must add its manifest and exactly the freetsa " + "and digicert receipts with no other releases/ changes; " + f"missing={sorted(expected - new_files)}, " + f"extra={sorted(new_files - allowed)}" + ) + return ROOT / pathlib.PurePosixPath(manifest_relative) + + +def _base_ledger_bytes(commit: str) -> bytes: + relative = LEDGER_PATH.relative_to(ROOT).as_posix() + return git_blob_bytes(ROOT, git_file_entry(ROOT, commit, relative)) + + +def _check_exact_byte_append(base_bytes: bytes, candidate_bytes: bytes) -> bytes: + if not candidate_bytes.startswith(base_bytes): + raise AppendError( + "change is not an exact byte append to the base JSONL; existing " + "bytes, including line endings, are immutable" + ) + return candidate_bytes[len(base_bytes) :] + + +def check_release_proposal( + base_ref: str, + *, + anchor_dir: pathlib.Path | None = None, +) -> int | None: + """Verify the base chain and the one allowed candidate transition. + + A pre-genesis base keeps legacy append proposals valid only while they do + not touch ``releases/``. Genesis may add the two prescribed anchors and + README alongside its exact manifest/receipt triple. Once genesis exists, + all base release files are byte- and mode-immutable and a ledger byte + append must carry exactly one next release triple. + """ + + try: + commit, new_files, base_release_entries = ( + verify_release_history_immutable(ROOT, base_ref) + ) + except ReleaseChainError as exc: + raise AppendError(str(exc)) from exc + + base_has_chain = any( + relative.startswith(RELEASE_MANIFEST_PREFIX) + for relative in base_release_entries + ) + candidate_has_chain = any( + path.is_file() + for path in (ROOT / "releases" / "manifests").glob("*.json") + ) if (ROOT / "releases" / "manifests").is_dir() else False + base_bytes = _base_ledger_bytes(commit) + candidate_bytes = LEDGER_PATH.read_bytes() + appended_bytes = _check_exact_byte_append(base_bytes, candidate_bytes) + ledger_changed = bool(appended_bytes) + enforce_production_pins = anchor_dir is None + + if not base_has_chain: + if not candidate_has_chain: + if new_files: + raise AppendError( + "legacy pre-genesis proposal must not change releases/; " + "add a complete genesis manifest and both receipts or no " + f"release files at all (changed={sorted(new_files)})" + ) + return None + _release_triple( + new_files, + 0, + allowed_support_files=GENESIS_SUPPORT_FILES, + ) + try: + verification = verify_release_chain( + ROOT, + anchor_dir=anchor_dir, + require_chain=True, + verify_state=True, + enforce_production_pins=enforce_production_pins, + ) + except ReleaseChainError as exc: + raise AppendError(str(exc)) from exc + if len(verification.releases) != 1: + raise AppendError( + "genesis proposal must create exactly one release at index 0" + ) + return 0 + + try: + base_verification = verify_base_release_chain( + ROOT, + commit, + base_release_entries, + anchor_dir=anchor_dir, + enforce_production_pins=enforce_production_pins, + ) + except ReleaseChainError as exc: + raise AppendError(f"base release chain is invalid: {exc}") from exc + assert base_verification.head is not None + expected_index = base_verification.head.release_index + 1 + + if ledger_changed: + _release_triple(new_files, expected_index) + elif new_files: + raise AppendError( + "release-only proposal is forbidden after genesis; a next release " + "must witness an actual ledger byte append" + ) + + try: + candidate_verification = verify_release_chain( + ROOT, + anchor_dir=anchor_dir, + require_chain=True, + verify_state=True, + enforce_production_pins=enforce_production_pins, + ) + except ReleaseChainError as exc: + raise AppendError(str(exc)) from exc + expected_length = len(base_verification.releases) + (1 if ledger_changed else 0) + if len(candidate_verification.releases) != expected_length: + raise AppendError( + f"release chain length must be {expected_length} for this proposal; " + f"found {len(candidate_verification.releases)}" + ) + return candidate_verification.head.release_index + + +def check_release_chain_without_base( + *, anchor_dir: pathlib.Path | None = None +) -> int | None: + """On push, verify any initialized chain against working-tree state.""" + + manifest_directory = ROOT / "releases" / "manifests" + if not manifest_directory.is_dir() or not any(manifest_directory.iterdir()): + return None + try: + verification = verify_release_chain( + ROOT, + anchor_dir=anchor_dir, + require_chain=True, + verify_state=True, + enforce_production_pins=anchor_dir is None, + ) + except ReleaseChainError as exc: + raise AppendError(str(exc)) from exc + assert verification.head is not None + return verification.head.release_index + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument( "--base-ref", help="enforce an append-only diff against this git ref", ) + parser.add_argument( + "--release-anchor-dir", + type=pathlib.Path, + help=argparse.SUPPRESS, + ) args = parser.parse_args() text = LEDGER_PATH.read_text(encoding="utf-8") try: @@ -378,13 +588,26 @@ def main() -> int: binding_boundary = check_prefix_anchored_to_base(args.base_ref, prefix) appended = check_append_only(args.base_ref, lines) check_rows(lines, binding_boundary) + release_index = ( + check_release_proposal( + args.base_ref, + anchor_dir=args.release_anchor_dir, + ) + if args.base_ref + else check_release_chain_without_base( + anchor_dir=args.release_anchor_dir + ) + ) except AppendError as exc: print(f"thesis-facts append check failed: {exc}", file=sys.stderr) return 1 suffix = f", +{appended} appended vs base" if appended is not None else "" + release_suffix = ( + f", release {release_index}" if release_index is not None else "" + ) print( f"thesis-facts append check OK: {len(lines)} rows, immutable prefix " - f"{prefix['prefixLineCount']}{suffix}" + f"{prefix['prefixLineCount']}{suffix}{release_suffix}" ) return 0 diff --git a/scripts/cut_release_manifest.py b/scripts/cut_release_manifest.py new file mode 100644 index 0000000..9351ce1 --- /dev/null +++ b/scripts/cut_release_manifest.py @@ -0,0 +1,815 @@ +#!/usr/bin/env python3 +"""Cut the next witnessed thesis-ledger release for the working tree. + +The cutter accepts a pending append after an already witnessed HEAD: it first +verifies the existing release chain without requiring that HEAD equal the +working-tree ledger, then proves that the old HEAD is the exact byte prefix of +the current ledger. A networked cut stages and fully verifies both RFC 3161 +responses before any release file is created. +""" + +from __future__ import annotations + +import argparse +import http.client +import math +import os +import pathlib +import re +import stat +import subprocess +import sys +import tempfile +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable, Mapping +from datetime import datetime, timezone +from typing import Any + +from canonical_json import canonical_bytes +from verify_release_chain import ( + DEFAULT_CLOCK_SKEW_SECONDS, + LEDGER_RELATIVE, + MANIFEST_RELATIVE, + PREFIX_RELATIVE, + ROOT, + SCHEMA_VERSION, + ChainVerification, + ReleaseChainError, + jsonl_line_offsets, + load_manifest, + manifest_filename, + receipt_paths_for_manifest, + sha256_bytes, + validate_manifest_schema, + verify_release_chain, +) + +MAX_TOKEN_BYTES = 1024 * 1024 +DEFAULT_TIMEOUT_SECONDS = 45.0 +TSA_ENDPOINTS = { + "freetsa": "https://freetsa.org/tsr", + # DigiCert's documented RFC 3161 endpoint is plain HTTP (their TLS + # endpoint does not answer timestamp queries); the token itself is a + # self-authenticating signature, verified against pinned anchors. + "digicert": "http://timestamp.digicert.com", +} +Requester = Callable[[str, bytes, float], bytes] + + +class ReleaseCutError(RuntimeError): + """The next release could not be constructed or safely written.""" + + +def _validate_timeout(value: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ReleaseCutError("timeout_seconds must be a finite positive number") + result = float(value) + if not math.isfinite(result) or result <= 0: + raise ReleaseCutError("timeout_seconds must be a finite positive number") + return result + + +def request_timestamp(endpoint: str, query: bytes, timeout_seconds: float) -> bytes: + """POST one DER RFC 3161 query and return a size-bounded response.""" + + if endpoint not in TSA_ENDPOINTS.values(): + raise ReleaseCutError(f"refusing unapproved TSA endpoint: {endpoint!r}") + if type(query) is not bytes or not query: + raise ReleaseCutError("RFC 3161 query must be non-empty bytes") + timeout = _validate_timeout(timeout_seconds) + request = urllib.request.Request( + endpoint, + data=query, + headers={ + "Content-Type": "application/timestamp-query", + "User-Agent": "Thesis-Ledger-Release-Witness/1", + }, + method="POST", + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + token = response.read(MAX_TOKEN_BYTES + 1) + if not token: + raise ReleaseCutError(f"TSA returned an empty response: {endpoint}") + if len(token) > MAX_TOKEN_BYTES: + raise ReleaseCutError( + f"TSA response exceeds the one-megabyte limit: {endpoint}" + ) + return token + + +def _run_git(root: pathlib.Path, arguments: list[str], label: str) -> str: + try: + completed = subprocess.run( + ["git", "-C", str(root), *arguments], + check=False, + capture_output=True, + text=True, + ) + except FileNotFoundError as exc: + raise ReleaseCutError( + f"git is required to auto-detect producer {label}" + ) from exc + if completed.returncode != 0: + diagnostic = (completed.stderr or completed.stdout).strip() + raise ReleaseCutError( + f"cannot auto-detect producer {label}: {diagnostic or 'git command failed'}" + ) + output = completed.stdout.strip() + if not output or "\n" in output or "\r" in output: + raise ReleaseCutError(f"cannot auto-detect a unique non-empty producer {label}") + return output + + +def _validate_provenance(value: str, label: str) -> str: + if type(value) is not str or not value or value != value.strip(): + raise ReleaseCutError( + f"producer {label} must be a non-empty string without outer whitespace" + ) + if len(value) > 1024 or any( + ord(char) < 0x20 or ord(char) == 0x7F for char in value + ): + raise ReleaseCutError( + f"producer {label} contains control characters or is too long" + ) + return value + + +_SCP_REMOTE_RE = re.compile(r"(?:[^/@:]+@)?(?P[^/:]+):(?P[^?#]+)\Z") + + +def _normalize_remote(remote: str) -> str: + """Return credential-free repository provenance from a Git remote URL.""" + + remote = _validate_provenance(remote, "remote URL") + host: str + path: str + if "://" in remote: + parsed = urllib.parse.urlsplit(remote) + if parsed.scheme not in {"https", "ssh", "git"}: + raise ReleaseCutError( + "cannot safely normalize the origin URL; pass --repo explicitly" + ) + if parsed.query or parsed.fragment or not parsed.hostname: + raise ReleaseCutError( + "cannot safely normalize the origin URL; pass --repo explicitly" + ) + host = parsed.hostname + try: + port = parsed.port + except ValueError as exc: + raise ReleaseCutError( + "cannot safely normalize the origin URL; pass --repo explicitly" + ) from exc + if port is not None: + host = f"{host}:{port}" + path = parsed.path + else: + match = _SCP_REMOTE_RE.fullmatch(remote) + if match is None: + raise ReleaseCutError( + "origin is not a network repository URL; pass --repo explicitly" + ) + host = match.group("host") + path = match.group("path") + + pieces = path.strip("/").split("/") + if pieces and pieces[-1].endswith(".git"): + pieces[-1] = pieces[-1][:-4] + if not pieces or any(piece in {"", ".", ".."} for piece in pieces): + raise ReleaseCutError( + "cannot safely normalize the origin URL; pass --repo explicitly" + ) + if host.lower() in {"github.com", "www.github.com"}: + if len(pieces) != 2: + raise ReleaseCutError( + "GitHub origin is not an owner/repository URL; pass --repo explicitly" + ) + return _validate_provenance("/".join(pieces), "repo") + return _validate_provenance(f"{host}/{'/'.join(pieces)}", "repo") + + +def _git_root(root: pathlib.Path) -> None: + detected = pathlib.Path( + _run_git(root, ["rev-parse", "--show-toplevel"], "Git worktree") + ).resolve() + if detected != root: + raise ReleaseCutError( + f"--root must be the Git worktree root ({detected}), not {root}" + ) + + +def detect_producer( + root: pathlib.Path, + *, + repo: str | None, + branch: str | None, +) -> dict[str, str]: + """Resolve producer claims without invoking a shell or recording credentials.""" + + if repo is not None: + selected_repo = _validate_provenance(repo, "repo") + else: + _git_root(root) + selected_repo = _normalize_remote( + _run_git(root, ["remote", "get-url", "origin"], "repo") + ) + + if branch is not None: + selected_branch = _validate_provenance(branch, "branch") + else: + _git_root(root) + selected_branch = _validate_provenance( + _run_git( + root, + ["symbolic-ref", "--quiet", "--short", "HEAD"], + "branch (detached HEAD; pass --branch explicitly)", + ), + "branch", + ) + return {"repo": selected_repo, "branch": selected_branch} + + +def _regular_bytes(path: pathlib.Path, label: str) -> bytes: + try: + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags) + except FileNotFoundError as exc: + raise ReleaseCutError(f"required {label} is missing: {path}") from exc + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise ReleaseCutError(f"required {label} is not a regular file: {path}") + with os.fdopen(descriptor, "rb") as stream: + descriptor = -1 + return stream.read() + finally: + if descriptor >= 0: + os.close(descriptor) + + +def _created_at_utc(now: datetime | None) -> str: + current = now or datetime.now(timezone.utc) + if ( + not isinstance(current, datetime) + or current.tzinfo is None + or current.utcoffset() is None + ): + raise ReleaseCutError("now must be a timezone-aware datetime") + try: + current_utc = current.astimezone(timezone.utc) + except (OverflowError, ValueError) as exc: + raise ReleaseCutError("now cannot be represented as UTC") from exc + return current_utc.isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _build_manifest( + ledger: bytes, + immutable_prefix: bytes, + existing: ChainVerification, + producer: dict[str, str], + *, + now: datetime | None, +) -> tuple[dict[str, Any], bytes]: + offsets = jsonl_line_offsets(ledger, LEDGER_RELATIVE.as_posix()) + line_count = len(offsets) - 1 + head = existing.head + if head is None: + release_index = 0 + previous_hash = None + append = None + else: + previous_count = head.manifest["state"]["lineCount"] + if previous_count > line_count: + raise ReleaseCutError( + f"working-tree ledger has {line_count} rows, fewer than witnessed " + f"HEAD's {previous_count} rows" + ) + witnessed_prefix = ledger[: offsets[previous_count]] + witnessed_digest = sha256_bytes(witnessed_prefix) + if witnessed_digest != head.manifest["state"]["jsonlSha256"]: + raise ReleaseCutError( + "working-tree ledger does not begin with the exact byte state " + "committed by the witnessed HEAD" + ) + immutable_digest = sha256_bytes(immutable_prefix) + if immutable_digest != head.manifest["state"]["immutablePrefixSha256"]: + raise ReleaseCutError( + "ledger/immutable_prefix.json differs from the witnessed HEAD" + ) + if line_count == previous_count: + raise ReleaseCutError( + "working-tree ledger has no pending rows after the witnessed HEAD" + ) + release_index = head.release_index + 1 + previous_hash = head.sha256 + suffix = ledger[offsets[previous_count] :] + append = { + "previousLineCount": previous_count, + "appendedRowCount": line_count - previous_count, + "appendedBytesSha256": sha256_bytes(suffix), + } + + manifest: dict[str, Any] = { + "schemaVersion": SCHEMA_VERSION, + "releaseIndex": release_index, + "previousManifestSha256": previous_hash, + "state": { + "path": LEDGER_RELATIVE.as_posix(), + "jsonlSha256": sha256_bytes(ledger), + "lineCount": line_count, + "immutablePrefixSha256": sha256_bytes(immutable_prefix), + }, + "append": append, + "createdAtUtc": _created_at_utc(now), + "producer": producer, + } + validate_manifest_schema(manifest) + return manifest, canonical_bytes(manifest) + b"\n" + + +def _build_timestamp_query(manifest: bytes, timeout_seconds: float) -> bytes: + with tempfile.TemporaryDirectory(prefix="thesis-release-query-") as name: + temporary = pathlib.Path(name) + manifest_path = temporary / "manifest.json" + query_path = temporary / "request.tsq" + manifest_path.write_bytes(manifest) + try: + completed = subprocess.run( + [ + "openssl", + "ts", + "-query", + "-config", + "/dev/null", + "-data", + str(manifest_path), + "-sha256", + "-cert", + "-out", + str(query_path), + ], + check=False, + capture_output=True, + text=True, + timeout=timeout_seconds, + env={**os.environ, "LC_ALL": "C", "OPENSSL_CONF": "/dev/null"}, + ) + except FileNotFoundError as exc: + raise ReleaseCutError( + "openssl is required to construct the RFC 3161 query" + ) from exc + except subprocess.TimeoutExpired as exc: + raise ReleaseCutError( + "OpenSSL timestamp-query construction timed out" + ) from exc + if completed.returncode != 0: + diagnostic = (completed.stderr or completed.stdout).strip() + raise ReleaseCutError( + "OpenSSL timestamp-query construction failed: " + f"{diagnostic[-1000:] or 'no diagnostic'}" + ) + query = _regular_bytes(query_path, "RFC 3161 query") + if not query or len(query) > MAX_TOKEN_BYTES: + raise ReleaseCutError("OpenSSL produced an invalid-sized RFC 3161 query") + return query + + +def _request_receipts( + query: bytes, + *, + requester: Requester, + timeout_seconds: float, +) -> dict[str, bytes]: + receipts: dict[str, bytes] = {} + for tsa, endpoint in TSA_ENDPOINTS.items(): + try: + token = requester(endpoint, query, timeout_seconds) + except ( + http.client.HTTPException, + OSError, + ReleaseCutError, + urllib.error.URLError, + ) as exc: + raise ReleaseCutError(f"{tsa} timestamp request failed: {exc}") from exc + except Exception as exc: + raise ReleaseCutError(f"{tsa} timestamp request failed: {exc}") from exc + if type(token) is not bytes or not token: + raise ReleaseCutError(f"{tsa} TSA must return non-empty bytes") + if len(token) > MAX_TOKEN_BYTES: + raise ReleaseCutError(f"{tsa} TSA response exceeds the one-megabyte limit") + receipts[tsa] = token + return receipts + + +def _write_staged_tree( + stage: pathlib.Path, + existing: ChainVerification, + ledger: bytes, + immutable_prefix: bytes, + filename: str, + manifest: bytes, + receipts: Mapping[str, bytes], +) -> None: + ledger_path = stage / LEDGER_RELATIVE + prefix_path = stage / PREFIX_RELATIVE + manifest_dir = stage / MANIFEST_RELATIVE + ledger_path.parent.mkdir(parents=True) + manifest_dir.mkdir(parents=True) + ledger_path.write_bytes(ledger) + prefix_path.write_bytes(immutable_prefix) + for record in existing.releases: + (manifest_dir / record.path.name).write_bytes(record.raw) + for tsa, receipt_path in record.receipt_paths.items(): + (manifest_dir / receipt_path.name).write_bytes( + _regular_bytes(receipt_path, f"existing {tsa} receipt") + ) + candidate = manifest_dir / filename + candidate.write_bytes(manifest) + for tsa, token in receipts.items(): + receipt_paths_for_manifest(candidate)[tsa].write_bytes(token) + + +def _verify_staged_release( + existing: ChainVerification, + ledger: bytes, + immutable_prefix: bytes, + filename: str, + manifest: bytes, + receipts: Mapping[str, bytes], + *, + anchor_dir: pathlib.Path, + enforce_production_pins: bool, + clock_skew_seconds: int, +) -> None: + with tempfile.TemporaryDirectory(prefix="thesis-release-stage-") as name: + stage = pathlib.Path(name) + _write_staged_tree( + stage, + existing, + ledger, + immutable_prefix, + filename, + manifest, + receipts, + ) + verify_release_chain( + stage, + anchor_dir=anchor_dir, + require_chain=True, + verify_state=True, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + ) + + +def _check_output_ancestors(root: pathlib.Path) -> None: + for directory in (root / "releases", root / MANIFEST_RELATIVE): + try: + metadata = directory.lstat() + except FileNotFoundError: + continue + if not stat.S_ISDIR(metadata.st_mode): + raise ReleaseCutError( + f"release output parent is not a real directory: {directory}" + ) + + +def _check_targets_absent(paths: list[pathlib.Path]) -> None: + for path in paths: + if os.path.lexists(path): + raise ReleaseCutError(f"refusing to overwrite release file: {path}") + + +def _prepare_output_directory( + root: pathlib.Path, +) -> tuple[pathlib.Path, list[pathlib.Path]]: + created: list[pathlib.Path] = [] + try: + for directory in (root / "releases", root / MANIFEST_RELATIVE): + try: + os.mkdir(directory, 0o755) + created.append(directory) + except FileExistsError: + try: + metadata = directory.lstat() + except FileNotFoundError as exc: + raise ReleaseCutError( + f"release output directory changed concurrently: {directory}" + ) from exc + if not stat.S_ISDIR(metadata.st_mode): + raise ReleaseCutError( + f"release output parent is not a real directory: {directory}" + ) + except Exception: + _remove_empty_directories(created) + raise + return root / MANIFEST_RELATIVE, created + + +def _remove_empty_directories(directories: list[pathlib.Path]) -> None: + for directory in reversed(directories): + try: + directory.rmdir() + except OSError: + pass + + +def _exclusive_batch_write( + root: pathlib.Path, + payloads: Mapping[pathlib.Path, bytes], + *, + verify_written: Callable[[], None], +) -> None: + if not payloads: + raise ReleaseCutError("no release files were provided for writing") + parents = {path.parent for path in payloads} + if parents != {root / MANIFEST_RELATIVE}: + raise ReleaseCutError("release outputs escaped releases/manifests") + _check_output_ancestors(root) + _check_targets_absent(list(payloads)) + directory, created_directories = _prepare_output_directory(root) + directory_flags = os.O_RDONLY + if hasattr(os, "O_DIRECTORY"): + directory_flags |= os.O_DIRECTORY + if hasattr(os, "O_NOFOLLOW"): + directory_flags |= os.O_NOFOLLOW + directory_fd: int | None = None + created_names: list[str] = [] + succeeded = False + try: + directory_fd = os.open(directory, directory_flags) + for path, payload in payloads.items(): + try: + os.stat(path.name, dir_fd=directory_fd, follow_symlinks=False) + except FileNotFoundError: + pass + else: + raise ReleaseCutError(f"refusing to overwrite release file: {path}") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path.name, flags, 0o644, dir_fd=directory_fd) + created_names.append(path.name) + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.fsync(directory_fd) + verify_written() + succeeded = True + except BaseException as exc: + cleanup_failures: list[str] = [] + if directory_fd is not None: + for name in reversed(created_names): + try: + os.unlink(name, dir_fd=directory_fd) + except FileNotFoundError: + pass + except OSError: + cleanup_failures.append(name) + try: + os.fsync(directory_fd) + except OSError: + pass + if cleanup_failures: + raise ReleaseCutError( + "release write failed and rollback could not remove: " + f"{cleanup_failures}; original error: {exc}" + ) from exc + raise + finally: + if directory_fd is not None: + os.close(directory_fd) + if not succeeded: + _remove_empty_directories(created_directories) + + +def _snapshot_existing_files( + verification: ChainVerification, +) -> dict[pathlib.Path, bytes]: + snapshot: dict[pathlib.Path, bytes] = {} + for record in verification.releases: + snapshot[record.path] = record.raw + for tsa, path in record.receipt_paths.items(): + snapshot[path] = _regular_bytes(path, f"existing {tsa} receipt") + return snapshot + + +def _assert_unchanged( + ledger_path: pathlib.Path, + prefix_path: pathlib.Path, + ledger: bytes, + immutable_prefix: bytes, + history: Mapping[pathlib.Path, bytes], +) -> None: + if _regular_bytes(ledger_path, "ledger JSONL") != ledger: + raise ReleaseCutError("ledger changed while the release was being cut") + if _regular_bytes(prefix_path, "immutable-prefix manifest") != immutable_prefix: + raise ReleaseCutError( + "immutable-prefix manifest changed while the release was being cut" + ) + for path, expected in history.items(): + if _regular_bytes(path, "existing release file") != expected: + raise ReleaseCutError( + f"existing release file changed while cutting the release: {path}" + ) + + +def cut_release_manifest( + root: pathlib.Path = ROOT, + *, + repo: str | None = None, + branch: str | None = None, + no_tsa: bool = False, + anchor_dir: pathlib.Path | None = None, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + clock_skew_seconds: int = DEFAULT_CLOCK_SKEW_SECONDS, + requester: Requester | None = None, + now: datetime | None = None, +) -> pathlib.Path: + """Build and safely write the next release, returning its manifest path.""" + + if type(no_tsa) is not bool: + raise ReleaseCutError("no_tsa must be a boolean") + if type(clock_skew_seconds) is not int or clock_skew_seconds < 0: + raise ReleaseCutError("clock_skew_seconds must be a non-negative integer") + timeout = _validate_timeout(timeout_seconds) + try: + selected_root = pathlib.Path(root).resolve(strict=True) + except FileNotFoundError as exc: + raise ReleaseCutError(f"repository root does not exist: {root}") from exc + if not selected_root.is_dir(): + raise ReleaseCutError(f"repository root is not a directory: {selected_root}") + selected_anchors = ( + pathlib.Path(anchor_dir).resolve() + if anchor_dir is not None + else selected_root / "releases" / "anchors" + ) + enforce_production_pins = anchor_dir is None + + existing = verify_release_chain( + selected_root, + anchor_dir=selected_anchors, + require_chain=False, + verify_state=True, + allow_pending_append=True, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + ) + ledger_path = selected_root / LEDGER_RELATIVE + prefix_path = selected_root / PREFIX_RELATIVE + ledger = _regular_bytes(ledger_path, "ledger JSONL") + immutable_prefix = _regular_bytes(prefix_path, "immutable-prefix manifest") + producer = detect_producer( + selected_root, + repo=repo, + branch=branch, + ) + _manifest, raw = _build_manifest( + ledger, + immutable_prefix, + existing, + producer, + now=now, + ) + filename = manifest_filename(_manifest["releaseIndex"], raw) + manifest_path = selected_root / MANIFEST_RELATIVE / filename + receipt_paths = receipt_paths_for_manifest(manifest_path) + all_targets = [manifest_path, *receipt_paths.values()] + _check_output_ancestors(selected_root) + _check_targets_absent(all_targets) + history = _snapshot_existing_files(existing) + + if no_tsa: + + def verify_manifest_only() -> None: + loaded, loaded_raw, _digest = load_manifest(manifest_path) + if loaded != _manifest or loaded_raw != raw: + raise ReleaseCutError("written manifest differs from constructed bytes") + _assert_unchanged( + ledger_path, + prefix_path, + ledger, + immutable_prefix, + history, + ) + + _assert_unchanged( + ledger_path, + prefix_path, + ledger, + immutable_prefix, + history, + ) + _exclusive_batch_write( + selected_root, + {manifest_path: raw}, + verify_written=verify_manifest_only, + ) + return manifest_path + + query = _build_timestamp_query(raw, timeout) + receipts = _request_receipts( + query, + requester=request_timestamp if requester is None else requester, + timeout_seconds=timeout, + ) + _verify_staged_release( + existing, + ledger, + immutable_prefix, + filename, + raw, + receipts, + anchor_dir=selected_anchors, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + ) + payloads = { + manifest_path: raw, + **{receipt_paths[tsa]: token for tsa, token in receipts.items()}, + } + + def verify_committed_release() -> None: + _assert_unchanged( + ledger_path, + prefix_path, + ledger, + immutable_prefix, + history, + ) + verify_release_chain( + selected_root, + anchor_dir=selected_anchors, + require_chain=True, + verify_state=True, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + ) + + _exclusive_batch_write( + selected_root, + payloads, + verify_written=verify_committed_release, + ) + return manifest_path + + +def main() -> int: + parser = argparse.ArgumentParser( + description="cut the next witnessed thesis-ledger release manifest" + ) + parser.add_argument( + "--root", + type=pathlib.Path, + default=ROOT, + help="repository root (default: this script's repository)", + ) + parser.add_argument( + "--anchor-dir", + type=pathlib.Path, + help="override TSA anchors (intended for offline tests)", + ) + parser.add_argument("--repo", help="override producer.repo provenance") + parser.add_argument("--branch", help="override producer.branch provenance") + parser.add_argument( + "--no-tsa", + action="store_true", + help="write only the manifest; do not request or write receipts", + ) + parser.add_argument( + "--timeout-seconds", + type=float, + default=DEFAULT_TIMEOUT_SECONDS, + ) + parser.add_argument( + "--clock-skew-seconds", + type=int, + default=DEFAULT_CLOCK_SKEW_SECONDS, + ) + args = parser.parse_args() + try: + manifest_path = cut_release_manifest( + args.root, + repo=args.repo, + branch=args.branch, + no_tsa=args.no_tsa, + anchor_dir=args.anchor_dir, + timeout_seconds=args.timeout_seconds, + clock_skew_seconds=args.clock_skew_seconds, + ) + except (OSError, ReleaseChainError, ReleaseCutError, ValueError) as exc: + print(f"release cut failed: {exc}", file=sys.stderr) + return 1 + if args.no_tsa: + print(f"release manifest written without TSA receipts: {manifest_path}") + else: + print(f"witnessed release written: {manifest_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_release_chain.py b/scripts/verify_release_chain.py new file mode 100644 index 0000000..03c2d6c --- /dev/null +++ b/scripts/verify_release_chain.py @@ -0,0 +1,1171 @@ +#!/usr/bin/env python3 +"""Offline verification for the witnessed thesis-ledger release chain. + +The verifier treats manifest and receipt bytes as an append-only journal. It +does not trust manifest provenance or timestamps supplied by the producer: +each manifest is canonical and content-addressed, every state and append digest +is recomputed from the current append-only JSONL, and both RFC 3161 receipts are +verified against separate, committed trust anchors. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import re +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any + +from canonical_json import canonical_bytes + +ROOT = pathlib.Path(__file__).resolve().parents[1] +MANIFEST_RELATIVE = pathlib.PurePosixPath("releases/manifests") +LEDGER_RELATIVE = pathlib.PurePosixPath("ledger/official_observations.jsonl") +PREFIX_RELATIVE = pathlib.PurePosixPath("ledger/immutable_prefix.json") +STATE_PATH = LEDGER_RELATIVE.as_posix() +SCHEMA_VERSION = "thesis_ledger_release_v1" +MAX_RELEASE_INDEX = 9_999 +DEFAULT_CLOCK_SKEW_SECONDS = 300 +MAX_FUTURE_SECONDS = 300 +SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") +MANIFEST_RE = re.compile(r"(?P[0-9]{4})-(?P[0-9a-f]{16})\.json\Z") +RECEIPT_RE = re.compile( + r"(?P[0-9]{4}-[0-9a-f]{16})" + r"\.(?Pfreetsa|digicert)\.tsr\Z" +) +STRICT_UTC_RE = re.compile( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:" + r"[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z\Z" +) +TIME_STAMP_RE = re.compile( + r"(?P[A-Z][a-z]{2})\s+" + r"(?P[0-9]{1,2})\s+" + r"(?P[0-9]{2}):(?P[0-9]{2}):" + r"(?P[0-9]{2})(?P\.[0-9]+)?\s+" + r"(?P[0-9]{4})\s+GMT\Z" +) + + +@dataclass(frozen=True) +class AnchorSpec: + filename: str + pem_sha256: str + policy_oid: str + signer_certificate_sha256: str + signer_spki_sha256: str + + +ANCHORS = { + "freetsa": AnchorSpec( + filename="freetsa-root-2016.pem", + pem_sha256=("2151b61137ffa86bf664691ba67e7da0b19f98c758e3d228d5d8ebf27e044438"), + policy_oid="1.2.3.4.1", + signer_certificate_sha256=( + "32e841a95cc1164101ffde41298ef2fc75c1c4372ef095e88a6bbd47dfb191fc" + ), + signer_spki_sha256=( + "fa02bd555e3e483d62b4e70be6218692068d2b0b0a7525db58dcbf2901cdb072" + ), + ), + "digicert": AnchorSpec( + filename="digicert-trusted-root-g4.pem", + pem_sha256=("ce7d6b44f5d510391be98c8d76b18709400a30cd87659bfebe1c6f97ff5181ee"), + policy_oid="2.16.840.1.114412.7.1", + signer_certificate_sha256=( + "4aa03fa22cd75c84c55c938f828e676b9caecab33fe36d269aa334f146110a33" + ), + signer_spki_sha256=( + "7abda95ed7301ac94bded350babc319903d0b4f16c4e7e39346dba5f9e992b72" + ), + ), +} + + +class ReleaseChainError(ValueError): + """The release journal is malformed, inconsistent, or untrusted.""" + + +@dataclass(frozen=True) +class GitEntry: + mode: str + object_type: str + object_id: str + path: str + + +@dataclass(frozen=True) +class ReleaseRecord: + path: pathlib.Path + raw: bytes + sha256: str + manifest: dict[str, Any] + receipt_paths: dict[str, pathlib.Path] + receipt_times: dict[str, datetime] + + @property + def release_index(self) -> int: + return int(self.manifest["releaseIndex"]) + + +@dataclass(frozen=True) +class ChainVerification: + releases: tuple[ReleaseRecord, ...] + + @property + def head(self) -> ReleaseRecord | None: + return self.releases[-1] if self.releases else None + + +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _fail_json_constant(value: str) -> None: + raise ReleaseChainError(f"manifest contains non-JSON number {value!r}") + + +def _object_without_duplicates( + pairs: list[tuple[str, Any]], +) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ReleaseChainError(f"manifest has duplicate key {key!r}") + result[key] = value + return result + + +def _exact_keys(value: Any, expected: set[str], label: str) -> dict[str, Any]: + if type(value) is not dict: + raise ReleaseChainError(f"{label} must be an object") + actual = set(value) + if actual != expected: + missing = sorted(expected - actual) + unknown = sorted(actual - expected) + raise ReleaseChainError( + f"{label} keys are not closed-world: missing={missing}, unknown={unknown}" + ) + return value + + +def _strict_int(value: Any, label: str, *, minimum: int = 0) -> int: + if type(value) is not int: + raise ReleaseChainError(f"{label} must be an integer, not a boolean") + if value < minimum: + raise ReleaseChainError(f"{label} must be >= {minimum}") + return value + + +def _strict_string(value: Any, label: str, *, nonempty: bool = True) -> str: + if type(value) is not str or (nonempty and not value): + suffix = " and non-empty" if nonempty else "" + raise ReleaseChainError(f"{label} must be a string{suffix}") + return value + + +def _sha256(value: Any, label: str) -> str: + if type(value) is not str or SHA256_RE.fullmatch(value) is None: + raise ReleaseChainError( + f"{label} must be exactly 64 lowercase hexadecimal characters" + ) + return value + + +def parse_created_at(value: Any, label: str = "createdAtUtc") -> datetime: + text = _strict_string(value, label) + if STRICT_UTC_RE.fullmatch(text) is None: + raise ReleaseChainError(f"{label} must be a strict UTC timestamp ending in Z") + try: + parsed = datetime.fromisoformat(text[:-1] + "+00:00") + except ValueError as exc: + raise ReleaseChainError(f"{label} is not a real UTC time: {text!r}") from exc + return parsed.astimezone(timezone.utc) + + +def validate_manifest_schema(manifest: Any) -> dict[str, Any]: + """Validate the closed-world ``thesis_ledger_release_v1`` schema.""" + + payload = _exact_keys( + manifest, + { + "schemaVersion", + "releaseIndex", + "previousManifestSha256", + "state", + "append", + "createdAtUtc", + "producer", + }, + "manifest", + ) + if payload["schemaVersion"] != SCHEMA_VERSION: + raise ReleaseChainError( + f"unsupported manifest schema {payload['schemaVersion']!r}" + ) + index = _strict_int(payload["releaseIndex"], "releaseIndex") + if index > MAX_RELEASE_INDEX: + raise ReleaseChainError( + f"releaseIndex {index} exceeds the four-digit filename limit" + ) + + previous = payload["previousManifestSha256"] + if index == 0: + if previous is not None: + raise ReleaseChainError("genesis previousManifestSha256 must be null") + else: + _sha256(previous, "previousManifestSha256") + + state = _exact_keys( + payload["state"], + { + "path", + "jsonlSha256", + "lineCount", + "immutablePrefixSha256", + }, + "state", + ) + if state["path"] != STATE_PATH: + raise ReleaseChainError(f"state.path must be exactly {STATE_PATH!r}") + _sha256(state["jsonlSha256"], "state.jsonlSha256") + _strict_int(state["lineCount"], "state.lineCount") + _sha256( + state["immutablePrefixSha256"], + "state.immutablePrefixSha256", + ) + + append = payload["append"] + if index == 0: + if append is not None: + raise ReleaseChainError("genesis append must be null") + else: + append_block = _exact_keys( + append, + { + "previousLineCount", + "appendedRowCount", + "appendedBytesSha256", + }, + "append", + ) + _strict_int( + append_block["previousLineCount"], + "append.previousLineCount", + ) + _strict_int( + append_block["appendedRowCount"], + "append.appendedRowCount", + minimum=1, + ) + _sha256( + append_block["appendedBytesSha256"], + "append.appendedBytesSha256", + ) + + parse_created_at(payload["createdAtUtc"]) + producer = _exact_keys(payload["producer"], {"repo", "branch"}, "producer") + _strict_string(producer["repo"], "producer.repo") + _strict_string(producer["branch"], "producer.branch") + return payload + + +def load_manifest(path: pathlib.Path) -> tuple[dict[str, Any], bytes, str]: + if path.is_symlink() or not path.is_file(): + raise ReleaseChainError(f"manifest is not a regular file: {path}") + raw = path.read_bytes() + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise ReleaseChainError(f"manifest is not UTF-8: {path}") from exc + try: + parsed = json.loads( + text, + object_pairs_hook=_object_without_duplicates, + parse_constant=_fail_json_constant, + ) + except json.JSONDecodeError as exc: + raise ReleaseChainError(f"manifest is not valid JSON: {path}: {exc}") from exc + payload = validate_manifest_schema(parsed) + expected = canonical_bytes(payload) + b"\n" + if raw != expected: + raise ReleaseChainError( + f"manifest bytes are not canonical JSON plus one newline: {path}" + ) + return payload, raw, sha256_bytes(raw) + + +def manifest_filename(index: int, raw: bytes) -> str: + _strict_int(index, "releaseIndex") + if index > MAX_RELEASE_INDEX: + raise ReleaseChainError( + f"releaseIndex {index} exceeds the four-digit filename limit" + ) + return f"{index:04d}-{sha256_bytes(raw)[:16]}.json" + + +def receipt_paths_for_manifest(path: pathlib.Path) -> dict[str, pathlib.Path]: + stem = path.stem + return {tsa: path.with_name(f"{stem}.{tsa}.tsr") for tsa in ANCHORS} + + +def _enumerate_manifest_files( + root: pathlib.Path, +) -> list[tuple[pathlib.Path, dict[str, pathlib.Path]]]: + directory = root / MANIFEST_RELATIVE + if not directory.exists(): + return [] + if directory.is_symlink() or not directory.is_dir(): + raise ReleaseChainError( + f"release manifest path is not a regular directory: {directory}" + ) + + manifests: dict[str, pathlib.Path] = {} + receipts: dict[str, dict[str, pathlib.Path]] = {} + for entry in directory.iterdir(): + if entry.is_symlink() or not entry.is_file(): + raise ReleaseChainError( + f"release manifest directory contains a non-regular entry: {entry}" + ) + manifest_match = MANIFEST_RE.fullmatch(entry.name) + if manifest_match is not None: + manifests[entry.stem] = entry + continue + receipt_match = RECEIPT_RE.fullmatch(entry.name) + if receipt_match is not None: + stem = receipt_match.group("stem") + tsa = receipt_match.group("tsa") + receipts.setdefault(stem, {})[tsa] = entry + continue + raise ReleaseChainError( + f"unknown file in closed release manifest directory: {entry.name}" + ) + + orphan_receipts = sorted(set(receipts) - set(manifests)) + if orphan_receipts: + raise ReleaseChainError( + f"orphan release receipts for manifest stems: {orphan_receipts}" + ) + result: list[tuple[pathlib.Path, dict[str, pathlib.Path]]] = [] + seen_indices: dict[int, str] = {} + for stem, path in manifests.items(): + match = MANIFEST_RE.fullmatch(path.name) + assert match is not None + index = int(match.group("index")) + if index in seen_indices: + raise ReleaseChainError( + f"duplicate release index {index}: {seen_indices[index]}, {path.name}" + ) + seen_indices[index] = path.name + actual_receipts = receipts.get(stem, {}) + if set(actual_receipts) != set(ANCHORS): + raise ReleaseChainError( + f"manifest {path.name} must have exactly freetsa and digicert " + f"receipts; found={sorted(actual_receipts)}" + ) + result.append((path, actual_receipts)) + return sorted( + result, + key=lambda item: int(MANIFEST_RE.fullmatch(item[0].name).group("index")), + ) + + +def _openssl_environment(empty_ca_dir: pathlib.Path) -> dict[str, str]: + environment = os.environ.copy() + environment.update( + { + "LC_ALL": "C", + "OPENSSL_CONF": "/dev/null", + "SSL_CERT_DIR": str(empty_ca_dir), + "SSL_CERT_FILE": "/dev/null", + } + ) + return environment + + +def _command_error(completed: subprocess.CompletedProcess[str]) -> str: + details = (completed.stderr or completed.stdout).strip() + return details[-1000:] if details else "no OpenSSL diagnostic" + + +def _parse_receipt_text(output: str, receipt: pathlib.Path) -> tuple[datetime, str]: + status_lines = [ + line.strip() for line in output.splitlines() if line.startswith("Status:") + ] + if status_lines != ["Status: Granted."]: + raise ReleaseChainError( + f"RFC 3161 receipt is not granted for {receipt}: {status_lines}" + ) + hash_lines = [ + line.split(":", 1)[1].strip() + for line in output.splitlines() + if line.startswith("Hash Algorithm:") + ] + if hash_lines != ["sha256"]: + raise ReleaseChainError( + f"RFC 3161 receipt does not use SHA-256 for {receipt}: {hash_lines}" + ) + policy_lines = [ + line.split(":", 1)[1].strip() + for line in output.splitlines() + if line.startswith("Policy OID:") + ] + if len(policy_lines) != 1: + raise ReleaseChainError( + f"RFC 3161 receipt has no unique policy OID for {receipt}" + ) + time_lines = [ + line.split(":", 1)[1].strip() + for line in output.splitlines() + if line.startswith("Time stamp:") + ] + if len(time_lines) != 1: + raise ReleaseChainError(f"RFC 3161 receipt has no unique genTime for {receipt}") + match = TIME_STAMP_RE.fullmatch(time_lines[0]) + if match is None: + raise ReleaseChainError( + f"unsupported RFC 3161 genTime for {receipt}: {time_lines[0]!r}" + ) + timestamp = ( + f"{match.group('month')} {match.group('day')} " + f"{match.group('hour')}:{match.group('minute')}:" + f"{match.group('second')} {match.group('year')} GMT" + ) + try: + parsed = datetime.strptime(timestamp, "%b %d %H:%M:%S %Y GMT").replace( + tzinfo=timezone.utc + ) + except ValueError as exc: + raise ReleaseChainError( + f"invalid RFC 3161 genTime for {receipt}: {timestamp!r}" + ) from exc + fraction = match.group("fraction") + if fraction: + parsed = parsed.replace(microsecond=int((fraction[1:] + "000000")[:6])) + return parsed, policy_lines[0] + + +def _openssl_binary( + arguments: list[str], + *, + environment: dict[str, str], + label: str, +) -> bytes: + try: + completed = subprocess.run( + ["openssl", *arguments], + check=False, + capture_output=True, + env=environment, + ) + except FileNotFoundError as exc: + raise ReleaseChainError( + "openssl is required for RFC 3161 verification" + ) from exc + if completed.returncode != 0: + diagnostic = (completed.stderr or completed.stdout).decode( + "utf-8", errors="replace" + ) + raise ReleaseChainError( + f"OpenSSL {label} failed (exit {completed.returncode}): " + f"{diagnostic.strip()[-1000:]}" + ) + return completed.stdout + + +def _verify_production_signer( + receipt: pathlib.Path, + anchor: pathlib.Path, + spec: AnchorSpec, + gen_time: datetime, + temporary: pathlib.Path, + environment: dict[str, str], +) -> None: + token = temporary / "token.der" + signer = temporary / "signer.pem" + content = temporary / "tst-info.der" + _openssl_binary( + [ + "ts", + "-reply", + "-config", + "/dev/null", + "-in", + str(receipt), + "-token_out", + "-out", + str(token), + ], + environment=environment, + label=f"token extraction for {receipt.name}", + ) + _openssl_binary( + [ + "cms", + "-verify", + "-inform", + "DER", + "-in", + str(token), + "-CAfile", + str(anchor), + "-no-CApath", + "-no-CAstore", + "-purpose", + "timestampsign", + "-attime", + str(int(gen_time.timestamp())), + "-signer", + str(signer), + "-out", + str(content), + ], + environment=environment, + label=f"signer extraction for {receipt.name}", + ) + certificate_der = _openssl_binary( + ["x509", "-in", str(signer), "-outform", "DER"], + environment=environment, + label=f"signer certificate decoding for {receipt.name}", + ) + public_key_pem = _openssl_binary( + ["x509", "-in", str(signer), "-pubkey", "-noout"], + environment=environment, + label=f"signer public-key extraction for {receipt.name}", + ) + public_key = temporary / "signer-public-key.pem" + public_key.write_bytes(public_key_pem) + public_key_der = _openssl_binary( + ["pkey", "-pubin", "-in", str(public_key), "-outform", "DER"], + environment=environment, + label=f"signer SPKI decoding for {receipt.name}", + ) + certificate_sha256 = sha256_bytes(certificate_der) + spki_sha256 = sha256_bytes(public_key_der) + if certificate_sha256 != spec.signer_certificate_sha256: + raise ReleaseChainError( + f"RFC 3161 signer certificate is not pinned for {receipt.name}: " + f"{certificate_sha256}" + ) + if spki_sha256 != spec.signer_spki_sha256: + raise ReleaseChainError( + f"RFC 3161 signer SPKI is not pinned for {receipt.name}: {spki_sha256}" + ) + + +def verify_receipt( + manifest_digest: str, + receipt: pathlib.Path, + tsa: str, + *, + anchor_dir: pathlib.Path, + enforce_production_pins: bool, + now: datetime | None = None, +) -> datetime: + """Cryptographically verify one receipt and return its signed genTime.""" + + if tsa not in ANCHORS: + raise ReleaseChainError(f"unknown TSA receipt kind {tsa!r}") + _sha256(manifest_digest, "manifest digest") + if receipt.is_symlink() or not receipt.is_file(): + raise ReleaseChainError(f"missing or non-regular RFC 3161 receipt: {receipt}") + spec = ANCHORS[tsa] + anchor = anchor_dir / spec.filename + if anchor.is_symlink() or not anchor.is_file(): + raise ReleaseChainError(f"missing or non-regular TSA anchor: {anchor}") + if enforce_production_pins: + anchor_digest = sha256_bytes(anchor.read_bytes()) + if anchor_digest != spec.pem_sha256: + raise ReleaseChainError( + f"production TSA anchor bytes are not code-pinned for {tsa}: " + f"{anchor_digest}" + ) + + with tempfile.TemporaryDirectory(prefix="thesis-release-tsa-") as name: + temporary = pathlib.Path(name) + empty_ca_dir = temporary / "empty-ca" + empty_ca_dir.mkdir() + environment = _openssl_environment(empty_ca_dir) + try: + text_result = subprocess.run( + [ + "openssl", + "ts", + "-reply", + "-config", + "/dev/null", + "-in", + str(receipt), + "-text", + ], + check=False, + capture_output=True, + text=True, + env=environment, + ) + except FileNotFoundError as exc: + raise ReleaseChainError( + "openssl is required for RFC 3161 verification" + ) from exc + if text_result.returncode != 0: + raise ReleaseChainError( + f"cannot inspect RFC 3161 receipt {receipt} " + f"(exit {text_result.returncode}): {_command_error(text_result)}" + ) + gen_time, policy_oid = _parse_receipt_text(text_result.stdout, receipt) + if enforce_production_pins and policy_oid != spec.policy_oid: + raise ReleaseChainError( + f"RFC 3161 policy is not pinned for {tsa}: {policy_oid!r}" + ) + current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) + if gen_time > current + timedelta(seconds=MAX_FUTURE_SECONDS): + raise ReleaseChainError( + f"RFC 3161 genTime {gen_time.isoformat()} for {receipt.name} " + f"postdates verifier time {current.isoformat()}" + ) + + verify_result = subprocess.run( + [ + "openssl", + "ts", + "-verify", + "-config", + "/dev/null", + "-digest", + manifest_digest, + "-in", + str(receipt), + "-CAfile", + str(anchor), + "-CApath", + str(empty_ca_dir), + "-attime", + str(int(gen_time.timestamp())), + ], + check=False, + capture_output=True, + text=True, + env=environment, + ) + if verify_result.returncode != 0: + raise ReleaseChainError( + f"RFC 3161 verification failed for {receipt.name} " + f"(exit {verify_result.returncode}): " + f"{_command_error(verify_result)}" + ) + if enforce_production_pins: + _verify_production_signer( + receipt, + anchor, + spec, + gen_time, + temporary, + environment, + ) + return gen_time + + +def jsonl_line_offsets(payload: bytes, label: str) -> list[int]: + """Return exact byte offsets after each non-empty LF-terminated row.""" + + try: + payload.decode("utf-8") + except UnicodeDecodeError as exc: + raise ReleaseChainError(f"{label} is not UTF-8") from exc + if not payload.endswith(b"\n"): + raise ReleaseChainError( + f"{label} must end with exactly one LF after its final JSONL row" + ) + rows = payload.split(b"\n") + if rows[-1] != b"": + raise AssertionError("split invariant") + rows = rows[:-1] + offsets = [0] + position = 0 + for number, row in enumerate(rows, start=1): + if not row.strip(): + raise ReleaseChainError(f"{label} row {number} is blank") + if row.endswith(b"\r"): + raise ReleaseChainError(f"{label} row {number} uses CRLF, not exact LF") + position += len(row) + 1 + offsets.append(position) + return offsets + + +def _regular_file_bytes(root: pathlib.Path, relative: pathlib.PurePosixPath) -> bytes: + path = root / relative + if path.is_symlink() or not path.is_file(): + raise ReleaseChainError( + f"required state file is missing or non-regular: {path}" + ) + return path.read_bytes() + + +def _verify_state_history( + records: list[ReleaseRecord], + root: pathlib.Path, + *, + require_head_current: bool, +) -> None: + ledger = _regular_file_bytes(root, LEDGER_RELATIVE) + prefix = _regular_file_bytes(root, PREFIX_RELATIVE) + offsets = jsonl_line_offsets(ledger, STATE_PATH) + total_lines = len(offsets) - 1 + prefix_digest = sha256_bytes(prefix) + + previous_line_count: int | None = None + for record in records: + state = record.manifest["state"] + line_count = int(state["lineCount"]) + if line_count > total_lines: + raise ReleaseChainError( + f"release {record.release_index} lineCount {line_count} exceeds " + f"working-tree line count {total_lines}" + ) + historical_bytes = ledger[: offsets[line_count]] + historical_digest = sha256_bytes(historical_bytes) + if historical_digest != state["jsonlSha256"]: + raise ReleaseChainError( + f"release {record.release_index} state.jsonlSha256 does not " + "match the exact historical JSONL prefix" + ) + if state["immutablePrefixSha256"] != prefix_digest: + raise ReleaseChainError( + f"release {record.release_index} immutablePrefixSha256 does " + "not match ledger/immutable_prefix.json" + ) + + if previous_line_count is not None: + append = record.manifest["append"] + assert isinstance(append, dict) + if line_count <= previous_line_count: + raise ReleaseChainError( + f"release {record.release_index} lineCount must strictly increase" + ) + if append["previousLineCount"] != previous_line_count: + raise ReleaseChainError( + f"release {record.release_index} append.previousLineCount " + "does not match the previous manifest" + ) + row_delta = line_count - previous_line_count + if append["appendedRowCount"] != row_delta: + raise ReleaseChainError( + f"release {record.release_index} appendedRowCount " + f"{append['appendedRowCount']} does not match line delta " + f"{row_delta}" + ) + suffix = ledger[offsets[previous_line_count] : offsets[line_count]] + suffix_digest = sha256_bytes(suffix) + if append["appendedBytesSha256"] != suffix_digest: + raise ReleaseChainError( + f"release {record.release_index} appendedBytesSha256 does " + "not match the exact byte suffix" + ) + previous_line_count = line_count + + if require_head_current: + head = records[-1] + if head.manifest["state"]["lineCount"] != total_lines: + raise ReleaseChainError( + f"HEAD release lineCount {head.manifest['state']['lineCount']} " + f"does not match working-tree line count {total_lines}" + ) + if head.manifest["state"]["jsonlSha256"] != sha256_bytes(ledger): + raise ReleaseChainError( + "HEAD release state.jsonlSha256 does not match working-tree bytes" + ) + + +def verify_release_chain( + root: pathlib.Path = ROOT, + *, + anchor_dir: pathlib.Path | None = None, + require_chain: bool = True, + verify_state: bool = True, + allow_pending_append: bool = False, + enforce_production_pins: bool | None = None, + clock_skew_seconds: int = DEFAULT_CLOCK_SKEW_SECONDS, + now: datetime | None = None, +) -> ChainVerification: + """Verify all manifests, receipts, links, chronology, and state bytes.""" + + root = root.resolve() + default_anchor_dir = root / "releases" / "anchors" + selected_anchors = (anchor_dir or default_anchor_dir).resolve() + if enforce_production_pins is None: + enforce_production_pins = selected_anchors == default_anchor_dir + if type(clock_skew_seconds) is not int or clock_skew_seconds < 0: + raise ReleaseChainError("clock_skew_seconds must be a non-negative integer") + + enumerated = _enumerate_manifest_files(root) + if not enumerated: + if require_chain: + raise ReleaseChainError("release chain is absent; genesis is required") + return ChainVerification(()) + + records: list[ReleaseRecord] = [] + previous_hash: str | None = None + previous_times: dict[str, datetime] | None = None + verification_now = now or datetime.now(timezone.utc) + for expected_index, (path, receipt_paths) in enumerate(enumerated): + manifest, raw, digest = load_manifest(path) + filename_match = MANIFEST_RE.fullmatch(path.name) + assert filename_match is not None + filename_index = int(filename_match.group("index")) + if filename_index != expected_index: + raise ReleaseChainError( + f"release indices are not contiguous from 0: expected " + f"{expected_index:04d}, found {filename_index:04d}" + ) + if manifest["releaseIndex"] != expected_index: + raise ReleaseChainError( + f"manifest releaseIndex {manifest['releaseIndex']} does not " + f"match filename index {expected_index}" + ) + if filename_match.group("digest") != digest[:16]: + raise ReleaseChainError( + f"manifest filename hash does not match exact file bytes: {path.name}" + ) + if manifest["previousManifestSha256"] != previous_hash: + raise ReleaseChainError( + f"release {expected_index} previousManifestSha256 does not " + "match the previous manifest file bytes" + ) + if records: + previous_line_count = records[-1].manifest["state"]["lineCount"] + line_count = manifest["state"]["lineCount"] + append = manifest["append"] + assert isinstance(append, dict) + if line_count <= previous_line_count: + raise ReleaseChainError( + f"release {expected_index} lineCount must strictly increase" + ) + if append["previousLineCount"] != previous_line_count: + raise ReleaseChainError( + f"release {expected_index} append.previousLineCount does " + "not match the previous manifest" + ) + row_delta = line_count - previous_line_count + if append["appendedRowCount"] != row_delta: + raise ReleaseChainError( + f"release {expected_index} appendedRowCount " + f"{append['appendedRowCount']} does not match line delta " + f"{row_delta}" + ) + + receipt_times = { + tsa: verify_receipt( + digest, + receipt_path, + tsa, + anchor_dir=selected_anchors, + enforce_production_pins=enforce_production_pins, + now=verification_now, + ) + for tsa, receipt_path in receipt_paths.items() + } + created_at = parse_created_at(manifest["createdAtUtc"]) + earliest_allowed = created_at - timedelta(seconds=clock_skew_seconds) + for tsa, gen_time in receipt_times.items(): + if gen_time < earliest_allowed: + raise ReleaseChainError( + f"release {expected_index} {tsa} genTime " + f"{gen_time.isoformat()} impossibly precedes createdAtUtc " + f"{created_at.isoformat()}" + ) + if previous_times is not None: + lower_bound = max(previous_times.values()) - timedelta( + seconds=clock_skew_seconds + ) + current_earliest = min(receipt_times.values()) + if current_earliest < lower_bound: + raise ReleaseChainError( + f"release {expected_index} receipt chronology regresses: " + f"earliest current genTime {current_earliest.isoformat()} " + f"precedes latest prior genTime " + f"{max(previous_times.values()).isoformat()} beyond " + f"{clock_skew_seconds}s skew" + ) + + records.append( + ReleaseRecord( + path=path, + raw=raw, + sha256=digest, + manifest=manifest, + receipt_paths=receipt_paths, + receipt_times=receipt_times, + ) + ) + previous_hash = digest + previous_times = receipt_times + + if type(allow_pending_append) is not bool: + raise ReleaseChainError("allow_pending_append must be a boolean") + if allow_pending_append and not verify_state: + raise ReleaseChainError( + "allow_pending_append requires historical state verification" + ) + if verify_state: + _verify_state_history( + records, + root, + require_head_current=not allow_pending_append, + ) + return ChainVerification(tuple(records)) + + +def _git_run( + root: pathlib.Path, + arguments: list[str], + *, + text: bool = False, +) -> subprocess.CompletedProcess[Any]: + try: + return subprocess.run( + ["git", *arguments], + cwd=root, + check=False, + capture_output=True, + text=text, + ) + except FileNotFoundError as exc: + raise ReleaseChainError("git is required for --base-ref verification") from exc + + +def resolve_base_commit(root: pathlib.Path, base_ref: str) -> str: + completed = _git_run( + root, + ["rev-parse", "--verify", "--end-of-options", f"{base_ref}^{{commit}}"], + text=True, + ) + if completed.returncode != 0: + raise ReleaseChainError( + f"cannot resolve base ref {base_ref!r} to a commit: " + f"{completed.stderr.strip()}" + ) + commit = completed.stdout.strip() + ancestor = _git_run(root, ["merge-base", "--is-ancestor", commit, "HEAD"]) + if ancestor.returncode != 0: + raise ReleaseChainError(f"base commit {commit} is not an ancestor of HEAD") + return commit + + +def git_tree_entries( + root: pathlib.Path, commit: str, pathspec: str +) -> dict[str, GitEntry]: + completed = _git_run( + root, + ["ls-tree", "-r", "-z", "--full-tree", commit, "--", pathspec], + ) + if completed.returncode != 0: + diagnostic = completed.stderr.decode("utf-8", errors="replace").strip() + raise ReleaseChainError( + f"cannot enumerate {pathspec} at base {commit}: {diagnostic}" + ) + entries: dict[str, GitEntry] = {} + for record in completed.stdout.split(b"\0"): + if not record: + continue + try: + metadata, raw_path = record.split(b"\t", 1) + mode, object_type, object_id = metadata.decode("ascii").split(" ") + path = raw_path.decode("utf-8") + except (ValueError, UnicodeDecodeError) as exc: + raise ReleaseChainError( + f"cannot parse git tree entry under {pathspec}" + ) from exc + if path in entries: + raise ReleaseChainError(f"duplicate git tree entry for {path}") + entries[path] = GitEntry(mode, object_type, object_id, path) + return entries + + +def git_blob_bytes(root: pathlib.Path, entry: GitEntry) -> bytes: + if entry.object_type != "blob": + raise ReleaseChainError( + f"base release entry is not a blob: {entry.path} ({entry.object_type})" + ) + completed = _git_run(root, ["cat-file", "blob", entry.object_id]) + if completed.returncode != 0: + diagnostic = completed.stderr.decode("utf-8", errors="replace").strip() + raise ReleaseChainError(f"cannot read base blob for {entry.path}: {diagnostic}") + return completed.stdout + + +def git_file_entry(root: pathlib.Path, commit: str, path: str) -> GitEntry: + entries = git_tree_entries(root, commit, path) + entry = entries.get(path) + if entry is None: + raise ReleaseChainError(f"required file {path} is absent at base {commit}") + return entry + + +def _working_release_files(root: pathlib.Path) -> dict[str, pathlib.Path]: + release_root = root / "releases" + if not release_root.exists(): + return {} + if release_root.is_symlink() or not release_root.is_dir(): + raise ReleaseChainError("releases must be a real directory, not a symlink") + files: dict[str, pathlib.Path] = {} + for path in release_root.rglob("*"): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + raise ReleaseChainError(f"release path is a symlink: {relative}") + if path.is_dir(): + continue + if not path.is_file(): + raise ReleaseChainError(f"release path is not regular: {relative}") + files[relative] = path + return files + + +def verify_release_history_immutable( + root: pathlib.Path, base_ref: str +) -> tuple[str, set[str], dict[str, GitEntry]]: + """Compare every base ``releases/`` file byte and mode to the candidate.""" + + root = root.resolve() + commit = resolve_base_commit(root, base_ref) + base_entries = git_tree_entries(root, commit, "releases") + current_files = _working_release_files(root) + for relative, entry in base_entries.items(): + if entry.mode not in {"100644", "100755"}: + raise ReleaseChainError( + f"base release entry has non-regular git mode {entry.mode}: {relative}" + ) + current = current_files.get(relative) + if current is None: + raise ReleaseChainError( + f"existing release file was deleted relative to {commit}: {relative}" + ) + candidate_mode = "100755" if current.stat().st_mode & 0o111 else "100644" + if candidate_mode != entry.mode: + raise ReleaseChainError( + f"existing release file mode changed relative to {commit}: " + f"{relative} ({entry.mode} -> {candidate_mode})" + ) + if current.read_bytes() != git_blob_bytes(root, entry): + raise ReleaseChainError( + f"existing release file bytes changed relative to {commit}: {relative}" + ) + return commit, set(current_files) - set(base_entries), base_entries + + +def materialize_base_tree( + root: pathlib.Path, + commit: str, + destination: pathlib.Path, + release_entries: dict[str, GitEntry], +) -> None: + entries = dict(release_entries) + for relative in (LEDGER_RELATIVE.as_posix(), PREFIX_RELATIVE.as_posix()): + entries[relative] = git_file_entry(root, commit, relative) + for relative, entry in entries.items(): + if entry.mode not in {"100644", "100755"}: + raise ReleaseChainError( + f"base tree entry has non-regular mode {entry.mode}: {relative}" + ) + output = destination / pathlib.PurePosixPath(relative) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(git_blob_bytes(root, entry)) + + +def verify_base_release_chain( + root: pathlib.Path, + commit: str, + release_entries: dict[str, GitEntry], + *, + anchor_dir: pathlib.Path | None = None, + enforce_production_pins: bool = True, + clock_skew_seconds: int = DEFAULT_CLOCK_SKEW_SECONDS, +) -> ChainVerification: + with tempfile.TemporaryDirectory(prefix="thesis-release-base-") as name: + base_root = pathlib.Path(name) + materialize_base_tree(root, commit, base_root, release_entries) + base_anchor_dir = anchor_dir or (base_root / "releases" / "anchors") + return verify_release_chain( + base_root, + anchor_dir=base_anchor_dir, + require_chain=True, + verify_state=True, + enforce_production_pins=enforce_production_pins, + clock_skew_seconds=clock_skew_seconds, + ) + + +def _format_time(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="verify the offline thesis-ledger release journal" + ) + parser.add_argument( + "--full", + action="store_true", + help="require a genesis-to-HEAD chain and exact working-tree state", + ) + parser.add_argument( + "--base-ref", + help="also reject any changed/deleted existing releases file vs this ref", + ) + parser.add_argument( + "--root", + type=pathlib.Path, + default=ROOT, + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--anchor-dir", + type=pathlib.Path, + help="override releases/anchors (intended for offline test fixtures)", + ) + parser.add_argument( + "--clock-skew-seconds", + type=int, + default=DEFAULT_CLOCK_SKEW_SECONDS, + ) + args = parser.parse_args() + + root = args.root.resolve() + anchor_dir = args.anchor_dir.resolve() if args.anchor_dir else None + enforce_pins = anchor_dir is None + try: + if args.base_ref: + verify_release_history_immutable(root, args.base_ref) + verification = verify_release_chain( + root, + anchor_dir=anchor_dir, + require_chain=args.full or bool(args.base_ref), + verify_state=True, + enforce_production_pins=enforce_pins, + clock_skew_seconds=args.clock_skew_seconds, + ) + except (OSError, ReleaseChainError) as exc: + print(f"release chain verification failed: {exc}", file=sys.stderr) + return 1 + if not verification.releases: + print("release chain absent (legacy pre-genesis state)") + return 0 + head = verification.releases[-1] + receipt_summary = ", ".join( + f"{tsa}={_format_time(value)}" + for tsa, value in sorted(head.receipt_times.items()) + ) + print( + f"release chain OK: {len(verification.releases)} releases, " + f"HEAD={head.path.name}, {receipt_summary}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/release_tsa/README.md b/tests/fixtures/release_tsa/README.md new file mode 100644 index 0000000..190febd --- /dev/null +++ b/tests/fixtures/release_tsa/README.md @@ -0,0 +1,44 @@ +# Local RFC 3161 test fixtures + +Everything in this directory is **TEST ONLY**. The names `freetsa` and +`digicert` identify the production verifier slot that each local fixture stands +in for; these certificates are not issued by, affiliated with, or trusted by +either public TSA. Their private keys are deliberately committed and public. +Never add these roots to a system or application trust store. + +`anchors/` is the verifier's trusted test anchor directory and intentionally +contains only the two independent local roots. `untrusted/root.pem` is kept +outside it so a receipt from the third signer exercises trust rejection. Each +receipt slot must remain mapped to its designated root: do not concatenate the +two roots into one permissive bundle for both suffixes, or one signer could fill +both nominally independent slots. Each signer certificate is directly issued by +its corresponding root and has critical +`CA:FALSE`, critical digital-signature key usage, and critical timestamping-only +extended key usage. Each root has critical `CA:TRUE,pathlen:0` and critical +certificate-signing/CRL-signing key usage. + +OpenSSL increments `tsa-serial` whenever it mints a response. Tests should copy +this fixture tree to a temporary directory and run `openssl ts -reply` there, +rather than modifying the checked-in serial templates. For example, from the +repository root: + +```sh +work="$(mktemp -d)" +cp -R tests/fixtures/release_tsa "$work/release_tsa" +printf '{"test":"manifest"}\n' > "$work/manifest.json" +openssl ts -query -data "$work/manifest.json" -sha256 -cert \ + -out "$work/request.tsq" +( + cd "$work/release_tsa/freetsa" + openssl ts -reply -config openssl-ts.cnf \ + -queryfile "$work/request.tsq" -out "$work/response.tsr" +) +digest="$(openssl dgst -sha256 -r "$work/manifest.json" | awk '{print $1}')" +openssl ts -verify -digest "$digest" -in "$work/response.tsr" \ + -CAfile "$work/release_tsa/anchors/freetsa-root-2016.pem" +``` + +Use the same commands with `digicert/` and +`anchors/digicert-trusted-root-g4.pem` for the second trusted response. A +response minted from `untrusted/` must fail verification against either trusted +anchor. diff --git a/tests/fixtures/release_tsa/anchors/digicert-trusted-root-g4.pem b/tests/fixtures/release_tsa/anchors/digicert-trusted-root-g4.pem new file mode 100644 index 0000000..257222d --- /dev/null +++ b/tests/fixtures/release_tsa/anchors/digicert-trusted-root-g4.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDwDCCAqigAwIBAgIUeq6eCBbwysDY/rjriipVT/ciUTUwDQYJKoZIhvcNAQEL +BQAwZTExMC8GA1UEAwwoVEVTVCBPTkxZIExvY2FsIERpZ2lDZXJ0IEZpeHR1cmUg +Um9vdCBDQTEwMC4GA1UECgwnU09MIEpvdXJuYWwgVGVzdCBGaXh0dXJlcyAtIE5P +VCBUUlVTVEVEMCAXDTI2MDcxMTEzMTY1OFoYDzIxMjYwNjE3MTMxNjU4WjBlMTEw +LwYDVQQDDChURVNUIE9OTFkgTG9jYWwgRGlnaUNlcnQgRml4dHVyZSBSb290IENB +MTAwLgYDVQQKDCdTT0wgSm91cm5hbCBUZXN0IEZpeHR1cmVzIC0gTk9UIFRSVVNU +RUQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC03iYpanrcih0GoD1T +nAgz1+POSDxY6Y5KvqugrsQsTPTOUE5vvE19XlSdHPIOxeY35mUeIeKA5EoO0KEl +wud4wccKcHyfD+t6INqysvXjjqRQiXGT69QToD5UO8+FNbeKkBBiSylGJdH//AXw +wEaS1m2A8p2IkmoU/vQ2plkilzpxk0RJCEpWP3awLTf92B/IFaYnpZFttKGn3NDn +oGIwiq/Xa0aIUXK6pZPtMi9le02IHMF/srq9L5fdQs8ppqz1fqyHbW7lnnZUFfJP +SwQY7vwAp23TRx3V5beLJWMSiBu2LSq1jTOwOQYRoWtBR4BttWOp6AEfdjFeC9XF +pdzjAgMBAAGjZjBkMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEG +MB0GA1UdDgQWBBSvuUJ4teB4AQ8Wm+b8FVioNBbhezAfBgNVHSMEGDAWgBSvuUJ4 +teB4AQ8Wm+b8FVioNBbhezANBgkqhkiG9w0BAQsFAAOCAQEALHA4mjRlNld1mdc/ +rFn02WrvlzyWtL849jc5msiTHzFKPk7Qlvq9wDWYyqlGKQM7jedVlxOK8wA7RUQR +op161qQAP6epxdHV0bQJK4k+C2423cZC19NoPcKriUjos3U1UEdaFD9TUAbqIn+s +CiVUPYAHylIU/hH8SvOPWJpGBw4i5PoSzuuBR/Ocd4/ZGxQz1KJ1uP+0/NmK/V1i +e9ffxMY8CGoFgzE0QixJ4J4YI/WTGVuGEsAyXz/grVChnkPVJkLaCAuSd7gYwvBY +n2yCbEbr6Xy3EsLecAP2oZ8Se/3O6urMbqPAHESmcJmIHUkD8TfgwY13PbTbB7Q3 +I9Pq1g== +-----END CERTIFICATE----- diff --git a/tests/fixtures/release_tsa/anchors/freetsa-root-2016.pem b/tests/fixtures/release_tsa/anchors/freetsa-root-2016.pem new file mode 100644 index 0000000..0bbbb8f --- /dev/null +++ b/tests/fixtures/release_tsa/anchors/freetsa-root-2016.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDvjCCAqagAwIBAgIUf0MkKE+lqmgYr+BOg4fZ2SuiNZIwDQYJKoZIhvcNAQEL +BQAwZDEwMC4GA1UEAwwnVEVTVCBPTkxZIExvY2FsIEZyZWVUU0EgRml4dHVyZSBS +b290IENBMTAwLgYDVQQKDCdTT0wgSm91cm5hbCBUZXN0IEZpeHR1cmVzIC0gTk9U +IFRSVVNURUQwIBcNMjYwNzExMTMxNjU4WhgPMjEyNjA2MTcxMzE2NThaMGQxMDAu +BgNVBAMMJ1RFU1QgT05MWSBMb2NhbCBGcmVlVFNBIEZpeHR1cmUgUm9vdCBDQTEw +MC4GA1UECgwnU09MIEpvdXJuYWwgVGVzdCBGaXh0dXJlcyAtIE5PVCBUUlVTVEVE +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsDhnSFmH1Jt70KyXQXtb +wywg15fgoVoFIXzniiy2XwraRXBC/ms0wnqsGr8mGfYrr/uPlVDX/5SnzQr93WQz +jd3QxTDKGjnwJJPiMkDJmet/cJm8cboN7BNK5f9r9CJV910ie8GwLouHlspCpBNh +4fjYip3A68wbzjX5otSCEyS8Pp0uZIBJKlYWOJJI5MOo3O0l1feXdz6EsISbiVp2 +PYaEv7MhGO+grSi01+ofP1ZyiRQKuhwzi4BpGnioiizv1rhKuxPMtdhXz750HKbB +jrFIgw3VklBJTIkksXBlco69fNlf0oGA0dYpFGtcw+lBOjJk+Jd2xiIo6OCuzZHu +RQIDAQABo2YwZDASBgNVHRMBAf8ECDAGAQH/AgEAMA4GA1UdDwEB/wQEAwIBBjAd +BgNVHQ4EFgQU+Ohqu5RRB2HtYEPKSLg/+gVhyJYwHwYDVR0jBBgwFoAU+Ohqu5RR +B2HtYEPKSLg/+gVhyJYwDQYJKoZIhvcNAQELBQADggEBAHN68Fzu+Unzer9QJhaf +N2SCot6Yrk3b5dJxHNK6bUa+5yj8ummwgJS+Lr4PXYIHLAgc6QIXJeDqv/PMrf7m ++5ICSqa51R8IJAsJXtXljrzcAkLP4kEDUM+d4rRMJT+PGquUn8CaF+vSD3y0TJOn +oPKDgH5tdix6Yi1M5sNAcyju7r+MQRF7cRTE27iXTo2l7fLqLCyyObunJ1pa5GIE +KZq0wUH1ABxEePGymsmLTGDoq9pG1UtD3uo1C3OlSJZKpLKmvEITF2HCLlh/42Mj +/fA84gzobtKcnAW9C7bgp2PNf4IrTr7m00wQ+BDzwYkJZCK1x9D4qMqGUslQ53Ic +V2o= +-----END CERTIFICATE----- diff --git a/tests/fixtures/release_tsa/digicert/openssl-ts.cnf b/tests/fixtures/release_tsa/digicert/openssl-ts.cnf new file mode 100644 index 0000000..e9557e4 --- /dev/null +++ b/tests/fixtures/release_tsa/digicert/openssl-ts.cnf @@ -0,0 +1,21 @@ +# TEST ONLY. This config uses a deliberately published local signing key. +[ tsa ] +default_tsa = tsa_config + +[ tsa_config ] +dir = . +serial = $dir/tsa-serial +crypto_device = builtin +signer_cert = $dir/signer.pem +certs = $dir/../anchors/digicert-trusted-root-g4.pem +signer_key = $dir/signer-key.pem +signer_digest = sha256 +default_policy = 1.3.6.1.4.1.55555.1.2 +other_policies = 1.3.6.1.4.1.55555.1.102 +digests = sha256 +accuracy = secs:1 +clock_precision_digits = 0 +ordering = no +tsa_name = yes +ess_cert_id_chain = no +ess_cert_id_alg = sha256 diff --git a/tests/fixtures/release_tsa/digicert/signer-key.pem b/tests/fixtures/release_tsa/digicert/signer-key.pem new file mode 100644 index 0000000..45beb99 --- /dev/null +++ b/tests/fixtures/release_tsa/digicert/signer-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDbZeBqFHXNZFjY +IZIxUQgIn0J5Uuhxm3WUCCIEpwCJMd3JuvZcbnDCI+9zqjnSfm6etXS3NlQksIQR +/8X9jW0p/8h9j70Us3hnxMH63kO0njpjxsLKWp5uip9qqHMJWlg9ft7uDCWCZ+Sd +B3Tr0QhcwIlpqTNrr4Sj46nouLExsoUaBbnneHrG6UXw9irHV8lbKBf2tCn8Q7jz +MGAQK4yQVFbRWeIGKd54t1vFyESN5UbhHuNOmQguXBCTdxLiz3/Ua68HtUWw9kva +iAedU+pbCCwmvEvokxZice7JH9GiX8X/ootwPM+YQLaQZJS096r+U5wsyon5lC59 +pCgzx6RJAgMBAAECggEAAMgEQ4VV6PNlekRz5OIRC/rQUxEPr2hVCwcUm+7aw1sz +8GJ5OyC0AKbEql4PqBkwnG/3/dO0iVbple/vTPUGCI3eBqMcrv2T9iMWE1Dppnq8 +qX7bbp4MjZyO4OOjA+fmg8SpED71YiBy7Fx1YaK3xIRZA5hCRclm1wXoQSlM/ujO +NYbmDR4+Y4dGqWwJbgScPNvGHkP8xwsbBIlqhaoBMGcr1xB42sikIBK72n+hXmXx +cmJhcehRbQ8s3yVEptTbWEBX1+Yew7PpSXbnJ3x9/WBejxGq2c8cjwNTbVccqHCm +DpOgCt3/fRR/rtuak0kkRqPuaiFetLIBvSDgqBxkCwKBgQD2hDVTrxfWjrl+ko2U +rk9UoDWJ+4zVvPUITETDNkmiIt6tvTF6cmF9VPR/9jFRzggxZdaTObEqA6zOwXgm +/suq03Mh5f6YoRU4OCCuiwFjfZf2gkTwOf9yQ6+gRX9Hs/NvbLTGOdDXUAPYHTHM +wiThHaAKbkSBqrbVkJI4+GlAfwKBgQDj1phALTUAm4R04qtqOSoOklmgR7KV9ynR +CWhZTg/Hu7w0H9H2Mio5O+zdQyVDCMTRXAcdIFE6rt6e4tgVKH/zuD+vDWZ8Qx41 +aIxUMjcR8o4zdJCNbuCBzCHNFSozCbEUVMFWfzK5fdDnMxTvEehLABnS4U/La6NI +7xNEX/C3NwKBgQDJpzigLfjIEJR4j5W1bCmExlFFgDqilSG0Gf+d2Ii/UtrpMHyK +94n8JkSjcFbD3lAhaHbwlB7yiXMQ5n5NF9yu7q2dqzBq1kovZOqHHTqTkid/BO0z +vZ3ScL/30SHfG7slCL2P8bd+ifu7C5azMpVMeRlzruPnViid5tdWNw6SLQKBgQCF +9LHnaaz0AnvKTUsDVUDI+HsBpaX2Ti+j26gieacFg/ePnfXQjoYMyJLnNIc/9nPh +s/3qvgmEOzRgrnblWb6BCR9ISNSs0rl8JU/8YNnID5hT4IDs5vfQ2Lg23+7DuzsF +/R7/QCIOtU6R4JK6TQkKr7B/MEG+FDcYtpfZZ58b2QKBgGBTPkIwh/PxSpRCBu+E +BnEecv0qY+MpV9g1OepHanx5sddheQRQbJPGYNPWOQNjkG1qF6KrNr4MDitoEEBI +W+kOehrryMCAvgtoo8Zy2RsVC44Tg3dBn5h4t8O5oFxy3WLyoR1UkTQV468WgCAt +zXwLAvEbN+EcIFkrohwECTQs +-----END PRIVATE KEY----- diff --git a/tests/fixtures/release_tsa/digicert/signer.pem b/tests/fixtures/release_tsa/digicert/signer.pem new file mode 100644 index 0000000..f7c13ea --- /dev/null +++ b/tests/fixtures/release_tsa/digicert/signer.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDvzCCAqegAwIBAgICIAEwDQYJKoZIhvcNAQELBQAwZTExMC8GA1UEAwwoVEVT +VCBPTkxZIExvY2FsIERpZ2lDZXJ0IEZpeHR1cmUgUm9vdCBDQTEwMC4GA1UECgwn +U09MIEpvdXJuYWwgVGVzdCBGaXh0dXJlcyAtIE5PVCBUUlVTVEVEMCAXDTI2MDcx +MTEzMTY1OFoYDzIxMjYwNjE3MTMxNjU4WjBkMTAwLgYDVQQDDCdURVNUIE9OTFkg +TG9jYWwgRGlnaUNlcnQgUkZDMzE2MSBTaWduZXIxMDAuBgNVBAoMJ1NPTCBKb3Vy +bmFsIFRlc3QgRml4dHVyZXMgLSBOT1QgVFJVU1RFRDCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBANtl4GoUdc1kWNghkjFRCAifQnlS6HGbdZQIIgSnAIkx +3cm69lxucMIj73OqOdJ+bp61dLc2VCSwhBH/xf2NbSn/yH2PvRSzeGfEwfreQ7Se +OmPGwspanm6Kn2qocwlaWD1+3u4MJYJn5J0HdOvRCFzAiWmpM2uvhKPjqei4sTGy +hRoFued4esbpRfD2KsdXyVsoF/a0KfxDuPMwYBArjJBUVtFZ4gYp3ni3W8XIRI3l +RuEe406ZCC5cEJN3EuLPf9Rrrwe1RbD2S9qIB51T6lsILCa8S+iTFmJx7skf0aJf +xf+ii3A8z5hAtpBklLT3qv5TnCzKifmULn2kKDPHpEkCAwEAAaN4MHYwDAYDVR0T +AQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwgw +HQYDVR0OBBYEFGwDOFi9Rp88vKWFiCkrs+ySORxSMB8GA1UdIwQYMBaAFK+5Qni1 +4HgBDxab5vwVWKg0FuF7MA0GCSqGSIb3DQEBCwUAA4IBAQAtZKEJBrOsBlq7x39b +MiVtY9Vdfrv6np0/pjEnXU88AV+dv7OxKsAFQdxyo9i1eB3WRefuxMlnYC6ghyxg +gqifJutRGsVouLfLO7hBojGlZ7DMG/0ydPTtAB/r8SYbhGmfZolCe9Zav6Cg7veo +ZZbiMu/PgLucX7XL7gSTFMiQYaSbl7xDAifznbWOHvc3NNSxRXIsdNm7Wahjge8H +OK/EvBAiJNWykxeY+TojUiv2H8+L8jGYo+4/LLYBNom9uHvRqoUw6DYGPPYAdg0Z +4O2MytRNHPrTf4ARwiguoxSkL2DUillvhswjymt8adO2ghg7A2ojTdcx7hqYFLBE +XW5V +-----END CERTIFICATE----- diff --git a/tests/fixtures/release_tsa/digicert/tsa-serial b/tests/fixtures/release_tsa/digicert/tsa-serial new file mode 100644 index 0000000..8a0f05e --- /dev/null +++ b/tests/fixtures/release_tsa/digicert/tsa-serial @@ -0,0 +1 @@ +01 diff --git a/tests/fixtures/release_tsa/freetsa/openssl-ts.cnf b/tests/fixtures/release_tsa/freetsa/openssl-ts.cnf new file mode 100644 index 0000000..71e6843 --- /dev/null +++ b/tests/fixtures/release_tsa/freetsa/openssl-ts.cnf @@ -0,0 +1,21 @@ +# TEST ONLY. This config uses a deliberately published local signing key. +[ tsa ] +default_tsa = tsa_config + +[ tsa_config ] +dir = . +serial = $dir/tsa-serial +crypto_device = builtin +signer_cert = $dir/signer.pem +certs = $dir/../anchors/freetsa-root-2016.pem +signer_key = $dir/signer-key.pem +signer_digest = sha256 +default_policy = 1.3.6.1.4.1.55555.1.1 +other_policies = 1.3.6.1.4.1.55555.1.101 +digests = sha256 +accuracy = secs:1 +clock_precision_digits = 0 +ordering = no +tsa_name = yes +ess_cert_id_chain = no +ess_cert_id_alg = sha256 diff --git a/tests/fixtures/release_tsa/freetsa/signer-key.pem b/tests/fixtures/release_tsa/freetsa/signer-key.pem new file mode 100644 index 0000000..3631b45 --- /dev/null +++ b/tests/fixtures/release_tsa/freetsa/signer-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDOalGCqqqE1Y/W +TMq/FIiHDmJD3lBBw9sNTPCxxyeQGwCpQJQsFSDA8Rb4P6ymw2psC9YVxGAJxOEU +afjAIvNZL+5aftYG/5vlQ6qWtNeaF299WJ/8HfO8WpxJwJJCtLJABje9regZLcSX +nk6zodz8cRVFVmcB5nElXh3YFqVF3KlPs7sES/TXGnDDoJBLPD2yun5ngmy/8Idl +s85fE4xJo5NpTamOKKtQwT/g+yYmb5FZcem5PTjK2O23cl+Ii+3kgMXTgvD1ZAP/ +EOtgNYkdfWFfsjWLpoVhT1JXn3cdGoKIkwcC7qv90//wykulO4DEnmrHcpniUN+I +/fW6E2jhAgMBAAECggEAV0gdr2L7N7AWYkeWc7X7BSDP7GLVDPoEZltia8oKsKS6 +Ytcr0HgeoXdQfyhtmRaIqadXn1yqP3dAtaEZziT5QX1DDIEVf2AWS5uRRqixgjbm +rdoLzE/eAIdQDt+e+RXvSaNXbp1ax0rTFkmafdqz1wr6M5eVdvg6X9KsS/NHb4Jc +X5r4TDqKDM2jfRTxgSl/AuMgjRX+a3BllzuqU8G/iesKPiTH/GSaeDg6PoEw436R +LM5bv6CZkZ0950tDV816z5szO4iQXDQbws9uDaBW3cK1Ism8kNQZ9eIenWQl0cpl +RnSBM5mqrs47e0O35Gp4Dcg5jd7LKs0B/3at2+bhAQKBgQDzH0nDJt59a6fyq6/R +TaEw3pLxbJUEPdV0EnTnG6SyTO/B3gBOEvphskADne44ugdy3eZbDDYJt5rfEcQ4 +I5i5NANercd026tXFsMsPSwslXlWPtlj3yfg8AmxvlaD5l6axN+elGgg1eaHLMt1 +hfTXZktgyrti0D57fBGXA5Qd7wKBgQDZWUnSQXdPvWiWPBMyqpW6PRO2gv8J5i5t +tkOeZn/Rmip3440nJaH0zs/+RN3Ibb3zg4fuUzIcSImXLZO/A9V5ALFnh1K6bjmi +8MXnqbPjWRKmRhq/GbeYhpeePQdLZi2AvF4zcTvFBIUGklcMJN7vNFhK0VTXNCO2 +tOYJ/wa2LwKBgFZ4IVb8YxN+j6w6rfymPJdSAjdFpDZu+5Ud32A3L7uOq5NrLvaV +v2Gx8RyYNhsM5wtOqthlHm8UU0vVWNGSr4XnXu90pUUFwAnoeaApXaW5v/8RuWXQ +/7D8DqaeCM0+yRUIwnP3WpWbHjOjlfWFoFBf/J0/XahGYoKu3N62Eh8XAoGAHojL +tS0bTOmIIusHq2uJo2NnPJHEsKhUx3b+ouc1d8XXx4YBU4mW6iSP1eHo4nqAYBCP +bSzk8lJCeimeAwzP1bMCOVAqkCRiFoEK3JPbwMnHs/JPWZ+MSnJGJKXgO3h8VgSg +m0uCIRiUc6rlbcBUTXimGsSzzZP2nHlng+n/EGMCgYBxS8y0q9ZwBj0wKIQgrUz6 +Kmm0fxH4FMnzuLfS1DNjmZU8Ykw6vTNvsIo7KGvUq2qhi087ACVun4aCVeNRtE2J +4JZa/AiLVYUXKkmr8ty3UPVNn9vRrDsshVOyWhJWzCEIcIzZ8Fji+kjDUVzD76Hf +j1DXB89ecMJitAxxr4ZoXA== +-----END PRIVATE KEY----- diff --git a/tests/fixtures/release_tsa/freetsa/signer.pem b/tests/fixtures/release_tsa/freetsa/signer.pem new file mode 100644 index 0000000..d16b22d --- /dev/null +++ b/tests/fixtures/release_tsa/freetsa/signer.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDvTCCAqWgAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwZDEwMC4GA1UEAwwnVEVT +VCBPTkxZIExvY2FsIEZyZWVUU0EgRml4dHVyZSBSb290IENBMTAwLgYDVQQKDCdT +T0wgSm91cm5hbCBUZXN0IEZpeHR1cmVzIC0gTk9UIFRSVVNURUQwIBcNMjYwNzEx +MTMxNjU4WhgPMjEyNjA2MTcxMzE2NThaMGMxLzAtBgNVBAMMJlRFU1QgT05MWSBM +b2NhbCBGcmVlVFNBIFJGQzMxNjEgU2lnbmVyMTAwLgYDVQQKDCdTT0wgSm91cm5h +bCBUZXN0IEZpeHR1cmVzIC0gTk9UIFRSVVNURUQwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQDOalGCqqqE1Y/WTMq/FIiHDmJD3lBBw9sNTPCxxyeQGwCp +QJQsFSDA8Rb4P6ymw2psC9YVxGAJxOEUafjAIvNZL+5aftYG/5vlQ6qWtNeaF299 +WJ/8HfO8WpxJwJJCtLJABje9regZLcSXnk6zodz8cRVFVmcB5nElXh3YFqVF3KlP +s7sES/TXGnDDoJBLPD2yun5ngmy/8Idls85fE4xJo5NpTamOKKtQwT/g+yYmb5FZ +cem5PTjK2O23cl+Ii+3kgMXTgvD1ZAP/EOtgNYkdfWFfsjWLpoVhT1JXn3cdGoKI +kwcC7qv90//wykulO4DEnmrHcpniUN+I/fW6E2jhAgMBAAGjeDB2MAwGA1UdEwEB +/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMIMB0G +A1UdDgQWBBRRjJ4M3DusdRxkGoRYdRgJebcw3TAfBgNVHSMEGDAWgBT46Gq7lFEH +Ye1gQ8pIuD/6BWHIljANBgkqhkiG9w0BAQsFAAOCAQEAjJjIUDkHCvNw6ozs+kyl +NPIC8mJfns60V2hcL6KjAHry+pVDbGns//E0n1fV7pusViGBgKXFiWB5zneaB82H +ss4GyPc8RuA92X4QdqQ+sgPQlm5YpdqOvdr1CNbFC95chuO1DV0JjPXpnohJC2qs +P+DvmS2mmW+dKu1Q/Q4KSERmHOe45I2USsYAQGAlaEJKrMnzkaqACXEB2XNrK1Qs +X8bBZCQz6YYUyr6ws1cYvFBNgeq3jvquyd1BaTXKLuHBOTMDB0MqyQ7f82+O/pOJ +eN8ei8HL04JLZ5PKt2xDfDfcC6/iXiurCWmQVkX7groStyIswanc2dO0lHA9OkFM +7w== +-----END CERTIFICATE----- diff --git a/tests/fixtures/release_tsa/freetsa/tsa-serial b/tests/fixtures/release_tsa/freetsa/tsa-serial new file mode 100644 index 0000000..8a0f05e --- /dev/null +++ b/tests/fixtures/release_tsa/freetsa/tsa-serial @@ -0,0 +1 @@ +01 diff --git a/tests/fixtures/release_tsa/untrusted/openssl-ts.cnf b/tests/fixtures/release_tsa/untrusted/openssl-ts.cnf new file mode 100644 index 0000000..8f841ec --- /dev/null +++ b/tests/fixtures/release_tsa/untrusted/openssl-ts.cnf @@ -0,0 +1,21 @@ +# TEST ONLY. This config uses a deliberately published local signing key. +[ tsa ] +default_tsa = tsa_config + +[ tsa_config ] +dir = . +serial = $dir/tsa-serial +crypto_device = builtin +signer_cert = $dir/signer.pem +certs = $dir/root.pem +signer_key = $dir/signer-key.pem +signer_digest = sha256 +default_policy = 1.3.6.1.4.1.55555.1.3 +other_policies = 1.3.6.1.4.1.55555.1.103 +digests = sha256 +accuracy = secs:1 +clock_precision_digits = 0 +ordering = no +tsa_name = yes +ess_cert_id_chain = no +ess_cert_id_alg = sha256 diff --git a/tests/fixtures/release_tsa/untrusted/root.pem b/tests/fixtures/release_tsa/untrusted/root.pem new file mode 100644 index 0000000..eb9549a --- /dev/null +++ b/tests/fixtures/release_tsa/untrusted/root.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDwjCCAqqgAwIBAgIUJtMNvXqFkTqgn37EGizKYWpuQQYwDQYJKoZIhvcNAQEL +BQAwZjEyMDAGA1UEAwwpVEVTVCBPTkxZIExvY2FsIFVudHJ1c3RlZCBGaXh0dXJl +IFJvb3QgQ0ExMDAuBgNVBAoMJ1NPTCBKb3VybmFsIFRlc3QgRml4dHVyZXMgLSBO +T1QgVFJVU1RFRDAgFw0yNjA3MTExMzE2NThaGA8yMTI2MDYxNzEzMTY1OFowZjEy +MDAGA1UEAwwpVEVTVCBPTkxZIExvY2FsIFVudHJ1c3RlZCBGaXh0dXJlIFJvb3Qg +Q0ExMDAuBgNVBAoMJ1NPTCBKb3VybmFsIFRlc3QgRml4dHVyZXMgLSBOT1QgVFJV +U1RFRDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOIDU7w8RTlo1b2o +POsuKS66rTLkUMdjiBYqu2UnYNe65WOHdRGKHMMeq0k7o94oJIFT6MtVurAeeqdi +oEm8e7UdsmRBkAeobFhk+7Bnoul17rlUVl19NtNhqEvVptF+JHnSilZFDtoEBZAu +8OWZG5kpMS31I/OKSro0kf0k4L1lgsrPXMZJ5aDxLpWk8DIgv2srtzwg50etTAt9 +/T/UwHs2XOnIfm6v2AWxmYejMD8hB/LdwZKow8vPVYS42QBJtkmLp09hK3sGFjKm +bK0BqWYAzmskagZxbCOvtMML6LlrB36krQOZHJ+PtDZa03K6K7GqG4uI+G7mj8CN +mdtgISMCAwEAAaNmMGQwEgYDVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMC +AQYwHQYDVR0OBBYEFPfke/Ix0sfURvjjeKRYu+FypkUuMB8GA1UdIwQYMBaAFPfk +e/Ix0sfURvjjeKRYu+FypkUuMA0GCSqGSIb3DQEBCwUAA4IBAQBuJ/MvGpqvWYEh +TiMWRxJhytX5nsfGzaFpvi2RhTjfLlMfC0xEkB4AXIqxjCxSE7cAz2NS9oPNA+qx +dqJvBsHA+egKOK5O+xyFUb/PLzkQCGrv6WhbkFK0Yy6OUuiQynpQZl3FEzIRvFE6 +rOb4dailpEvWfADXHZBYJeynvgY+lg/QXBqEox6xJ064LbVHdYCrt4AXP4hoNmov +Nta3zivUOSsXHFxBDBPWzTOy19POHWFw21dOQHP2bY8StYrYbQ/CHMOqZjLMCZvG +CkytmMWLtg1HOhdwxdrYRqhJRJhKIp/51e/lJtNS4+LLp5L4879ZC4014TPl6KC6 +5m4VK80P +-----END CERTIFICATE----- diff --git a/tests/fixtures/release_tsa/untrusted/signer-key.pem b/tests/fixtures/release_tsa/untrusted/signer-key.pem new file mode 100644 index 0000000..2be0678 --- /dev/null +++ b/tests/fixtures/release_tsa/untrusted/signer-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDJOrmzcfyi0pOS +zfYpwbTNehsNRVg3eUjdOQRkIc2CltpD6xDVzXZuUs30aY+v7+hkhh9e/USn9rze +5H+5pR07/E/bGrXcU+CtEbJsIAWqiP59gWj8zTI4W0GAtaHeycHns23eFadFadfw +e4EWZYkB9HT7m5PvKLjkOus9nYav+u6MVIco2agt09+GmsfEqbWah2kW5dVU+y+V +BfbSd4N22pts6IFkO7a+X+m+vAZ7EeciIRFXPvEtfL8YF8Bl8ziKsbrAiD2fsrPO +CtBwUGK7tM5ZV6SGmXoYVGF+FTuID4hsWNyZjFg2O8HarVcY/PhGtAYMxgVw0S9V +n33oCn7pAgMBAAECggEAGKP5Q1Ds558zdOGFiB1xH0wZbemU4M4YLwahVAWTna8N +MQ3itNuA6gd3srsOXjPZWB3YT5BhzNN40y5VuYzPZJ4DxUejB0S6GVc2aukM6JH3 +37Rt/668vIvoHHOhS5+W/+FHzc8G+Yf5ajglWXG7ciLWBE1Lsxhgc4oHbjcQ3BF9 +ZUj1UA9HfUfCXBhj6CckveQQ9kSaAbg2lr7hBmV0Hw/fvB8GISoFPwUIEAoj2EQ9 +nMjKwwnoqidyMGrZbGCSZzz68b1a63lkMQd48kwPeMvR7yMaNdQXZ1jR6gJqnZkK +bFmTHLWzxhqtldwbiNCSy7BDCCT+wdsFf3eRf112gQKBgQDvGOCnexPt5rHsjUgd +lPN7KmDLZ5sjn5XOdCoF0HcwmNtQfa1/bDn6xt6ct7ZqsHcXiF2AilI+UpDO2jtx +t66JucZN415mw/1uJHvJi7vwwHk0KIZ+MkjCJJ0QSqJRtDktUZeRz9kyfajN+bFF +DFwNZCmYy0fKM0nxgOlBNfxCOQKBgQDXdIak2qMhVHWB7IHID0SBzEj6L1qCCg1r +n/eK4tnuXWAVsWpW+iOpwB4GRTBsJrroET/lRyUr8l1lm3Rx7GDbnDVExi+HRbwc +BN48dfs0GW7f7RdqP1C2DlBVrCYsVtS6cSxNjdxY3QWMaLLIbQcjd7yVPvCxXzjZ +507A7KhiMQKBgBvwQkAPx7Ji5KOJlsWuJA1eVFUwVBaDjVPEiyyGwYayNd5f8SgU +k2nUVyrk5LHu0Nu1GYftodKaf0VMhumw1vpM9Pkg+mTzvedi5/wTEfD97KvFmIGi +ipH1CU1gCzuU3aGICTgGm5Ck0lbPOIGjAhamf53VExxzhc5si9UM498xAoGAYVB3 +dERwhCBixuEtKVCED+BWYfI7fhrOPvtM0sVty7UPLh0pO8nVi9mDdi+QFHlV2HXa +F1QQHQ5mUvtpF/+9K9QdV0fwtbewpDMqscJUSQ7nvmqXNd0NNn22mUudHk/F28Q8 +T3cwvE5wMCAmCFL+c22klwCLtIGe6wUH1CnQUDECgYAuIInh7rWFEkJTnUsEyRPa +5BmaacSB5G5v1yAFNPAJ7sIjnmikuVS1xbkJ/Rwfks3hKxty42eryI9SZs0PFQdf +c7+ydc4rXPiPVok8KG1H2R0cM0UjOpX3/1DwbxM2u2vNSTJsBtNXdLkCW824L0nF +rmTEJV9jL/Y1kLMZFa3Azg== +-----END PRIVATE KEY----- diff --git a/tests/fixtures/release_tsa/untrusted/signer.pem b/tests/fixtures/release_tsa/untrusted/signer.pem new file mode 100644 index 0000000..8c4a7b9 --- /dev/null +++ b/tests/fixtures/release_tsa/untrusted/signer.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDwTCCAqmgAwIBAgICMAEwDQYJKoZIhvcNAQELBQAwZjEyMDAGA1UEAwwpVEVT +VCBPTkxZIExvY2FsIFVudHJ1c3RlZCBGaXh0dXJlIFJvb3QgQ0ExMDAuBgNVBAoM +J1NPTCBKb3VybmFsIFRlc3QgRml4dHVyZXMgLSBOT1QgVFJVU1RFRDAgFw0yNjA3 +MTExMzE2NThaGA8yMTI2MDYxNzEzMTY1OFowZTExMC8GA1UEAwwoVEVTVCBPTkxZ +IExvY2FsIFVudHJ1c3RlZCBSRkMzMTYxIFNpZ25lcjEwMC4GA1UECgwnU09MIEpv +dXJuYWwgVGVzdCBGaXh0dXJlcyAtIE5PVCBUUlVTVEVEMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAyTq5s3H8otKTks32KcG0zXobDUVYN3lI3TkEZCHN +gpbaQ+sQ1c12blLN9GmPr+/oZIYfXv1Ep/a83uR/uaUdO/xP2xq13FPgrRGybCAF +qoj+fYFo/M0yOFtBgLWh3snB57Nt3hWnRWnX8HuBFmWJAfR0+5uT7yi45DrrPZ2G +r/rujFSHKNmoLdPfhprHxKm1modpFuXVVPsvlQX20neDdtqbbOiBZDu2vl/pvrwG +exHnIiERVz7xLXy/GBfAZfM4irG6wIg9n7KzzgrQcFBiu7TOWVekhpl6GFRhfhU7 +iA+IbFjcmYxYNjvB2q1XGPz4RrQGDMYFcNEvVZ996Ap+6QIDAQABo3gwdjAMBgNV +HRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDAWBgNVHSUBAf8EDDAKBggrBgEFBQcD +CDAdBgNVHQ4EFgQUay8SxhWqSyuFev1rSK3l0iCncW4wHwYDVR0jBBgwFoAU9+R7 +8jHSx9RG+ON4pFi74XKmRS4wDQYJKoZIhvcNAQELBQADggEBAGIRyqN/ykE1UcND +sg22iR+bbjtHasLbKeAJJjB4MMR5p3QD50hBGZ/ceCqCF7MIba5fi4bPTame9vEv +l1Tzswl6itgjE7QpVwMX2rqHQUqesShL2jVxK/9TqYUZfLGBp1JOcKkgJfioF5Us +h2kiU3Dd/13zEaqqLdojSl3b9IAsVGIAGcDBSnbTS+Jsnje5+4amkV0dAslOZYyV +IsNyu0Jbm1QZysQ9obPbqdW9TZfl7755aatlzXvtnuTaX0NP6aVRHA5z8hTzPqz3 +rUMblq8HmOksNL46ErucyFPtR8mB8EfvfNmQWtz8RmLtHvm2U0Wm6EEnjLNjGU8T ++MyZdLc= +-----END CERTIFICATE----- diff --git a/tests/fixtures/release_tsa/untrusted/tsa-serial b/tests/fixtures/release_tsa/untrusted/tsa-serial new file mode 100644 index 0000000..8a0f05e --- /dev/null +++ b/tests/fixtures/release_tsa/untrusted/tsa-serial @@ -0,0 +1 @@ +01 diff --git a/tests/test_policyengine_ledger.py b/tests/test_policyengine_ledger.py index 970e429..feaa3f8 100644 --- a/tests/test_policyengine_ledger.py +++ b/tests/test_policyengine_ledger.py @@ -146,7 +146,11 @@ def test_a_rewritten_prefix_line_is_detected(tmp_path): (ledger_dir / "immutable_prefix.json").write_text(PREFIX_PATH.read_text()) scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() - for name in ("check_thesis_facts_append.py", "canonical_json.py"): + for name in ( + "check_thesis_facts_append.py", + "canonical_json.py", + "verify_release_chain.py", + ): (scripts_dir / name).write_text((ROOT / "scripts" / name).read_text()) completed = subprocess.run( diff --git a/tests/test_release_chain.py b/tests/test_release_chain.py new file mode 100644 index 0000000..2ee1c5a --- /dev/null +++ b/tests/test_release_chain.py @@ -0,0 +1,857 @@ +"""Offline release-chain and append-gate tests. + +The committed fixture keys are deliberately public test credentials. Every +test copies the fixture tree before minting because OpenSSL advances each TSA's +serial file. No production TSA or network service is contacted. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +import shutil +import subprocess +import sys +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +TSA_FIXTURE = ROOT / "tests" / "fixtures" / "release_tsa" +SCRIPT_NAMES = ( + "canonical_json.py", + "check_thesis_facts_append.py", + "verify_release_chain.py", +) + +sys.path.insert(0, str(ROOT / "scripts")) + +from canonical_json import canonical_bytes # noqa: E402 +from check_thesis_facts_append import ( # noqa: E402 + expected_assertion_version_id, +) +from cut_release_manifest import ( # noqa: E402 + TSA_ENDPOINTS, + ReleaseCutError, + cut_release_manifest, +) +from verify_release_chain import ( # noqa: E402 + ReleaseChainError, + load_manifest, + manifest_filename, + validate_manifest_schema, +) + + +@dataclass(frozen=True) +class ReleaseEnvironment: + repo: Path + tsa: Path + + @property + def anchors(self) -> Path: + return self.tsa / "anchors" + + +def _run( + command: list[str], + *, + cwd: Path, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=cwd, + check=check, + capture_output=True, + text=True, + ) + + +def _git(repo: Path, *arguments: str) -> str: + return _run(["git", *arguments], cwd=repo).stdout.strip() + + +def _initialize_repo(repo: Path) -> str: + (repo / "ledger").mkdir(parents=True) + (repo / "scripts").mkdir() + for name in ("official_observations.jsonl", "immutable_prefix.json"): + shutil.copy2(ROOT / "ledger" / name, repo / "ledger" / name) + for name in SCRIPT_NAMES: + shutil.copy2(ROOT / "scripts" / name, repo / "scripts" / name) + _git(repo, "init", "-q") + _git(repo, "config", "user.name", "Release Chain Test") + _git(repo, "config", "user.email", "release-chain@example.invalid") + _git(repo, "add", ".") + _git(repo, "commit", "-qm", "pre-genesis base") + return _git(repo, "rev-parse", "HEAD") + + +def _copy_environment( + source: ReleaseEnvironment, destination: Path +) -> ReleaseEnvironment: + repo = destination / "repo" + tsa = destination / "release_tsa" + shutil.copytree(source.repo, repo) + shutil.copytree(source.tsa, tsa) + return ReleaseEnvironment(repo=repo, tsa=tsa) + + +def _sha256(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _line_count(payload: bytes) -> int: + assert payload.endswith(b"\n") + return len(payload.splitlines()) + + +def _created_at() -> str: + # Give OpenSSL a one-second margin so the receipt cannot precede the + # producer timestamp solely because the two clocks straddle a second. + value = datetime.now(timezone.utc) - timedelta(seconds=1) + return value.isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _release_manifest( + repo: Path, + *, + index: int, + previous_manifest: bytes | None = None, + previous_ledger: bytes | None = None, +) -> dict[str, Any]: + ledger = (repo / "ledger" / "official_observations.jsonl").read_bytes() + prefix = (repo / "ledger" / "immutable_prefix.json").read_bytes() + if index == 0: + assert previous_manifest is None + assert previous_ledger is None + append = None + previous_digest = None + else: + assert previous_manifest is not None + assert previous_ledger is not None + assert ledger.startswith(previous_ledger) + suffix = ledger[len(previous_ledger) :] + append = { + "previousLineCount": _line_count(previous_ledger), + "appendedRowCount": _line_count(ledger) - _line_count(previous_ledger), + "appendedBytesSha256": _sha256(suffix), + } + previous_digest = _sha256(previous_manifest) + return { + "schemaVersion": "thesis_ledger_release_v1", + "releaseIndex": index, + "previousManifestSha256": previous_digest, + "state": { + "path": "ledger/official_observations.jsonl", + "jsonlSha256": _sha256(ledger), + "lineCount": _line_count(ledger), + "immutablePrefixSha256": _sha256(prefix), + }, + "append": append, + "createdAtUtc": _created_at(), + "producer": { + "repo": "PolicyEngine/ledger", + "branch": "release-chain-test", + }, + } + + +def _mint_receipt( + payload: Path, + receipt: Path, + *, + tsa: Path, + signer: str, +) -> None: + request = tsa / f"{signer}-request.tsq" + _run( + [ + "openssl", + "ts", + "-query", + "-data", + str(payload), + "-sha256", + "-cert", + "-out", + str(request), + ], + cwd=tsa, + ) + _run( + [ + "openssl", + "ts", + "-reply", + "-config", + "openssl-ts.cnf", + "-queryfile", + str(request), + "-out", + str(receipt), + ], + cwd=tsa / signer, + ) + + +def _write_release( + environment: ReleaseEnvironment, + manifest: dict[str, Any], + *, + signers: dict[str, str] | None = None, + signed_payloads: dict[str, Path] | None = None, +) -> Path: + raw = canonical_bytes(manifest) + b"\n" + directory = environment.repo / "releases" / "manifests" + directory.mkdir(parents=True, exist_ok=True) + path = directory / manifest_filename(manifest["releaseIndex"], raw) + path.write_bytes(raw) + for slot in ("freetsa", "digicert"): + receipt = path.with_name(f"{path.stem}.{slot}.tsr") + payload = (signed_payloads or {}).get(slot, path) + signer = (signers or {}).get(slot, slot) + _mint_receipt(payload, receipt, tsa=environment.tsa, signer=signer) + return path + + +def _append_valid_row(repo: Path, identity: str) -> tuple[bytes, bytes]: + ledger_path = repo / "ledger" / "official_observations.jsonl" + before = ledger_path.read_bytes() + row = copy.deepcopy(json.loads(before.splitlines()[-1])) + row["source_record_id"] = identity + row["label"] = f"Release-chain fixture row {identity}" + row["value"] += 1 + row["assertionVersion"] = {"id": "", "supersedes": None} + row["assertionVersion"]["id"] = expected_assertion_version_id(row) + suffix = ( + json.dumps( + row, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + ledger_path.write_bytes(before + suffix) + return before, suffix + + +def _genesis(environment: ReleaseEnvironment) -> Path: + return _write_release( + environment, + _release_manifest(environment.repo, index=0), + ) + + +def _append_release(environment: ReleaseEnvironment, identity: str) -> Path: + manifest_directory = environment.repo / "releases" / "manifests" + genesis = next(manifest_directory.glob("0000-*.json")) + previous_manifest = genesis.read_bytes() + previous_ledger, _ = _append_valid_row(environment.repo, identity) + manifest = _release_manifest( + environment.repo, + index=1, + previous_manifest=previous_manifest, + previous_ledger=previous_ledger, + ) + return _write_release(environment, manifest) + + +def _run_gate( + environment: ReleaseEnvironment, + base_ref: str, +) -> subprocess.CompletedProcess[str]: + return _run( + [ + sys.executable, + str(environment.repo / "scripts" / "check_thesis_facts_append.py"), + "--base-ref", + base_ref, + "--release-anchor-dir", + str(environment.anchors), + ], + cwd=environment.repo, + check=False, + ) + + +def _run_verifier(environment: ReleaseEnvironment) -> subprocess.CompletedProcess[str]: + return _run( + [ + sys.executable, + str(environment.repo / "scripts" / "verify_release_chain.py"), + "--root", + str(environment.repo), + "--anchor-dir", + str(environment.anchors), + "--full", + ], + cwd=environment.repo, + check=False, + ) + + +def _local_timestamp_requester( + environment: ReleaseEnvironment, + *, + signer_overrides: dict[str, str] | None = None, +): + endpoint_slots = {endpoint: slot for slot, endpoint in TSA_ENDPOINTS.items()} + + def request(endpoint: str, query: bytes, _timeout_seconds: float) -> bytes: + slot = endpoint_slots[endpoint] + signer = (signer_overrides or {}).get(slot, slot) + request_path = environment.tsa / f"cutter-{slot}-request.tsq" + response_path = environment.tsa / f"cutter-{slot}-response.tsr" + request_path.write_bytes(query) + _run( + [ + "openssl", + "ts", + "-reply", + "-config", + "openssl-ts.cnf", + "-queryfile", + str(request_path), + "-out", + str(response_path), + ], + cwd=environment.tsa / signer, + ) + return response_path.read_bytes() + + return request + + +@pytest.fixture(scope="session") +def pregenesis_template(tmp_path_factory: pytest.TempPathFactory) -> ReleaseEnvironment: + root = tmp_path_factory.mktemp("release-pregenesis") + repo = root / "repo" + repo.mkdir() + tsa = root / "release_tsa" + shutil.copytree(TSA_FIXTURE, tsa) + _initialize_repo(repo) + return ReleaseEnvironment(repo=repo, tsa=tsa) + + +@pytest.fixture(scope="session") +def chain_template( + tmp_path_factory: pytest.TempPathFactory, + pregenesis_template: ReleaseEnvironment, +) -> ReleaseEnvironment: + environment = _copy_environment( + pregenesis_template, + tmp_path_factory.mktemp("release-genesis"), + ) + _genesis(environment) + _git(environment.repo, "add", ".") + _git(environment.repo, "commit", "-qm", "witness genesis") + return environment + + +@pytest.fixture(scope="session") +def full_chain_template( + tmp_path_factory: pytest.TempPathFactory, + chain_template: ReleaseEnvironment, +) -> ReleaseEnvironment: + environment = _copy_environment( + chain_template, + tmp_path_factory.mktemp("release-full-chain"), + ) + _append_release(environment, "release.fixture.full-chain") + _git(environment.repo, "add", ".") + _git(environment.repo, "commit", "-qm", "witness append") + return environment + + +@pytest.fixture +def pregenesis_environment( + tmp_path: Path, + pregenesis_template: ReleaseEnvironment, +) -> ReleaseEnvironment: + return _copy_environment(pregenesis_template, tmp_path) + + +@pytest.fixture +def chain_environment( + tmp_path: Path, + chain_template: ReleaseEnvironment, +) -> ReleaseEnvironment: + return _copy_environment(chain_template, tmp_path) + + +@pytest.fixture +def full_chain_environment( + tmp_path: Path, + full_chain_template: ReleaseEnvironment, +) -> ReleaseEnvironment: + return _copy_environment(full_chain_template, tmp_path) + + +def test_legacy_no_chain_append_is_accepted( + pregenesis_environment: ReleaseEnvironment, +): + base = _git(pregenesis_environment.repo, "rev-parse", "HEAD") + _append_valid_row( + pregenesis_environment.repo, + "release.fixture.legacy-pre-genesis", + ) + + completed = _run_gate(pregenesis_environment, base) + + assert completed.returncode == 0, completed.stderr + assert "release" not in completed.stdout + + +def test_genesis_proposal_is_accepted( + pregenesis_environment: ReleaseEnvironment, +): + base = _git(pregenesis_environment.repo, "rev-parse", "HEAD") + _genesis(pregenesis_environment) + + completed = _run_gate(pregenesis_environment, base) + + assert completed.returncode == 0, completed.stderr + assert "release 0" in completed.stdout + + +def test_correct_next_append_and_exact_suffix_are_accepted( + chain_environment: ReleaseEnvironment, +): + base = _git(chain_environment.repo, "rev-parse", "HEAD") + _append_release(chain_environment, "release.fixture.correct-next") + + completed = _run_gate(chain_environment, base) + + assert completed.returncode == 0, completed.stderr + assert "+1 appended vs base, release 1" in completed.stdout + + +def test_chain_base_rejects_append_without_manifest( + chain_environment: ReleaseEnvironment, +): + base = _git(chain_environment.repo, "rev-parse", "HEAD") + _append_valid_row(chain_environment.repo, "release.fixture.missing-manifest") + + completed = _run_gate(chain_environment, base) + + assert completed.returncode == 1 + assert "add exactly one manifest" in completed.stderr + + +def _candidate_manifest( + environment: ReleaseEnvironment, + identity: str, +) -> dict[str, Any]: + genesis = next((environment.repo / "releases" / "manifests").glob("0000-*.json")) + previous_ledger, _ = _append_valid_row(environment.repo, identity) + return _release_manifest( + environment.repo, + index=1, + previous_manifest=genesis.read_bytes(), + previous_ledger=previous_ledger, + ) + + +def test_chain_base_rejects_wrong_next_index( + chain_environment: ReleaseEnvironment, +): + base = _git(chain_environment.repo, "rev-parse", "HEAD") + manifest = _candidate_manifest(chain_environment, "release.fixture.wrong-index") + manifest["releaseIndex"] = 2 + _write_release(chain_environment, manifest) + + completed = _run_gate(chain_environment, base) + + assert completed.returncode == 1 + assert "proposal index must be 1, not 2" in completed.stderr + + +def test_chain_base_rejects_wrong_previous_hash( + chain_environment: ReleaseEnvironment, +): + base = _git(chain_environment.repo, "rev-parse", "HEAD") + manifest = _candidate_manifest( + chain_environment, + "release.fixture.wrong-previous", + ) + manifest["previousManifestSha256"] = "0" * 64 + _write_release(chain_environment, manifest) + + completed = _run_gate(chain_environment, base) + + assert completed.returncode == 1 + assert "previousManifestSha256" in completed.stderr + + +def test_chain_base_rejects_state_mismatch( + chain_environment: ReleaseEnvironment, +): + base = _git(chain_environment.repo, "rev-parse", "HEAD") + manifest = _candidate_manifest( + chain_environment, + "release.fixture.state-mismatch", + ) + manifest["state"]["jsonlSha256"] = "0" * 64 + _write_release(chain_environment, manifest) + + completed = _run_gate(chain_environment, base) + + assert completed.returncode == 1 + assert "state.jsonlSha256" in completed.stderr + + +@pytest.mark.parametrize( + ("field", "replacement", "diagnostic"), + [ + ("previousLineCount", 0, "previousLineCount"), + ("appendedRowCount", 2, "appendedRowCount"), + ("appendedBytesSha256", "0" * 64, "exact byte suffix"), + ], +) +def test_chain_base_rejects_append_block_mismatch( + chain_environment: ReleaseEnvironment, + field: str, + replacement: int | str, + diagnostic: str, +): + base = _git(chain_environment.repo, "rev-parse", "HEAD") + manifest = _candidate_manifest( + chain_environment, + f"release.fixture.append-mismatch.{field}", + ) + manifest["append"][field] = replacement + _write_release(chain_environment, manifest) + + completed = _run_gate(chain_environment, base) + + assert completed.returncode == 1 + assert diagnostic in completed.stderr + + +def test_chain_base_rejects_edited_historical_manifest( + chain_environment: ReleaseEnvironment, +): + base = _git(chain_environment.repo, "rev-parse", "HEAD") + genesis = next( + (chain_environment.repo / "releases" / "manifests").glob("0000-*.json") + ) + genesis.write_bytes(genesis.read_bytes() + b" ") + + completed = _run_gate(chain_environment, base) + + assert completed.returncode == 1 + assert "existing release file bytes changed" in completed.stderr + + +def test_chain_base_rejects_deleted_historical_receipt( + chain_environment: ReleaseEnvironment, +): + base = _git(chain_environment.repo, "rev-parse", "HEAD") + receipt = next( + (chain_environment.repo / "releases" / "manifests").glob("0000-*.freetsa.tsr") + ) + receipt.unlink() + + completed = _run_gate(chain_environment, base) + + assert completed.returncode == 1 + assert "existing release file was deleted" in completed.stderr + + +def test_chain_base_rejects_receipt_over_different_bytes( + chain_environment: ReleaseEnvironment, +): + base = _git(chain_environment.repo, "rev-parse", "HEAD") + manifest = _candidate_manifest( + chain_environment, + "release.fixture.different-imprint", + ) + other = chain_environment.tsa / "different-manifest.json" + other.write_bytes(b'{"different":true}\n') + _write_release( + chain_environment, + manifest, + signed_payloads={"freetsa": other}, + ) + + completed = _run_gate(chain_environment, base) + + assert completed.returncode == 1 + assert "RFC 3161 verification failed" in completed.stderr + + +def test_chain_base_rejects_untrusted_receipt( + chain_environment: ReleaseEnvironment, +): + base = _git(chain_environment.repo, "rev-parse", "HEAD") + manifest = _candidate_manifest( + chain_environment, + "release.fixture.untrusted-receipt", + ) + _write_release( + chain_environment, + manifest, + signers={"freetsa": "untrusted"}, + ) + + completed = _run_gate(chain_environment, base) + + assert completed.returncode == 1 + assert "RFC 3161 verification failed" in completed.stderr + + +def test_verifier_cli_full_chain_passes( + full_chain_environment: ReleaseEnvironment, +): + completed = _run_verifier(full_chain_environment) + + assert completed.returncode == 0, completed.stderr + assert "release chain OK: 2 releases" in completed.stdout + + +def test_cutter_no_tsa_writes_only_canonical_genesis( + pregenesis_environment: ReleaseEnvironment, +): + path = cut_release_manifest( + pregenesis_environment.repo, + repo="PolicyEngine/ledger", + branch="release-chain-test", + no_tsa=True, + anchor_dir=pregenesis_environment.anchors, + ) + + manifest, raw, digest = load_manifest(path) + + assert manifest["releaseIndex"] == 0 + assert manifest["append"] is None + assert path.name == manifest_filename(0, raw) + assert digest == _sha256(raw) + assert not path.with_name(f"{path.stem}.freetsa.tsr").exists() + assert not path.with_name(f"{path.stem}.digicert.tsr").exists() + + +def test_cutter_builds_and_verifies_exact_next_append( + chain_environment: ReleaseEnvironment, +): + _previous, suffix = _append_valid_row( + chain_environment.repo, + "release.fixture.cutter-next", + ) + + path = cut_release_manifest( + chain_environment.repo, + repo="PolicyEngine/ledger", + branch="release-chain-test", + anchor_dir=chain_environment.anchors, + requester=_local_timestamp_requester(chain_environment), + ) + + manifest, _raw, _digest = load_manifest(path) + completed = _run_verifier(chain_environment) + assert completed.returncode == 0, completed.stderr + assert manifest["releaseIndex"] == 1 + assert manifest["append"]["appendedRowCount"] == 1 + assert manifest["append"]["appendedBytesSha256"] == _sha256(suffix) + + +def test_cutter_rejects_wrong_second_tsa_without_partial_files( + pregenesis_environment: ReleaseEnvironment, +): + with pytest.raises((ReleaseChainError, ReleaseCutError)): + cut_release_manifest( + pregenesis_environment.repo, + repo="PolicyEngine/ledger", + branch="release-chain-test", + anchor_dir=pregenesis_environment.anchors, + requester=_local_timestamp_requester( + pregenesis_environment, + signer_overrides={"digicert": "freetsa"}, + ), + ) + + manifest_directory = ( + pregenesis_environment.repo / "releases" / "manifests" + ) + assert not manifest_directory.exists() or not any(manifest_directory.iterdir()) + + +@pytest.mark.parametrize( + "tamper", + [ + "historical-manifest", + "deleted-receipt", + "different-imprint", + "untrusted-receipt", + "swapped-receipt-slots", + "wrong-previous", + "state-mismatch", + "append-suffix-mismatch", + ], +) +def test_verifier_cli_catches_manifest_and_receipt_tampering( + full_chain_environment: ReleaseEnvironment, + tamper: str, +): + directory = full_chain_environment.repo / "releases" / "manifests" + genesis = next(directory.glob("0000-*.json")) + head = next(directory.glob("0001-*.json")) + if tamper == "historical-manifest": + genesis.write_bytes(genesis.read_bytes() + b" ") + elif tamper == "deleted-receipt": + head.with_name(f"{head.stem}.digicert.tsr").unlink() + elif tamper == "different-imprint": + other = full_chain_environment.tsa / "different.json" + other.write_bytes(b'{"different":true}\n') + _mint_receipt( + other, + head.with_name(f"{head.stem}.freetsa.tsr"), + tsa=full_chain_environment.tsa, + signer="freetsa", + ) + elif tamper == "untrusted-receipt": + _mint_receipt( + head, + head.with_name(f"{head.stem}.freetsa.tsr"), + tsa=full_chain_environment.tsa, + signer="untrusted", + ) + elif tamper == "swapped-receipt-slots": + freetsa = head.with_name(f"{head.stem}.freetsa.tsr") + digicert = head.with_name(f"{head.stem}.digicert.tsr") + freetsa_bytes = freetsa.read_bytes() + freetsa.write_bytes(digicert.read_bytes()) + digicert.write_bytes(freetsa_bytes) + else: + manifest = json.loads(head.read_text(encoding="utf-8")) + for path in ( + head, + head.with_name(f"{head.stem}.freetsa.tsr"), + head.with_name(f"{head.stem}.digicert.tsr"), + ): + path.unlink() + if tamper == "wrong-previous": + manifest["previousManifestSha256"] = "0" * 64 + elif tamper == "state-mismatch": + manifest["state"]["jsonlSha256"] = "0" * 64 + else: + manifest["append"]["appendedBytesSha256"] = "0" * 64 + _write_release(full_chain_environment, manifest) + + completed = _run_verifier(full_chain_environment) + + assert completed.returncode == 1 + assert "release chain verification failed" in completed.stderr + + +def test_receipt_slots_are_cryptographically_isolated( + full_chain_environment: ReleaseEnvironment, +): + directory = full_chain_environment.repo / "releases" / "manifests" + head = next(directory.glob("0001-*.json")) + freetsa = head.with_name(f"{head.stem}.freetsa.tsr") + digicert = head.with_name(f"{head.stem}.digicert.tsr") + freetsa_bytes = freetsa.read_bytes() + freetsa.write_bytes(digicert.read_bytes()) + digicert.write_bytes(freetsa_bytes) + + completed = _run_verifier(full_chain_environment) + + assert completed.returncode == 1 + assert "RFC 3161 verification failed" in completed.stderr + + +def _schema_manifest(index: int = 1) -> dict[str, Any]: + manifest = { + "schemaVersion": "thesis_ledger_release_v1", + "releaseIndex": index, + "previousManifestSha256": "1" * 64 if index else None, + "state": { + "path": "ledger/official_observations.jsonl", + "jsonlSha256": "2" * 64, + "lineCount": 2, + "immutablePrefixSha256": "3" * 64, + }, + "append": { + "previousLineCount": 1, + "appendedRowCount": 1, + "appendedBytesSha256": "4" * 64, + }, + "createdAtUtc": "2026-07-11T00:00:00Z", + "producer": {"repo": "PolicyEngine/ledger", "branch": "test"}, + } + if index == 0: + manifest["append"] = None + return manifest + + +@pytest.mark.parametrize( + ("location", "field"), + [ + ("manifest", "unexpected"), + ("state", "unexpected"), + ("append", "unexpected"), + ("producer", "unexpected"), + ], +) +def test_manifest_schema_is_closed_world(location: str, field: str): + manifest = _schema_manifest() + target = manifest if location == "manifest" else manifest[location] + target[field] = "candidate-controlled extension" + + with pytest.raises(ReleaseChainError, match="closed-world"): + validate_manifest_schema(manifest) + + +@pytest.mark.parametrize( + ("location", "field"), + [ + ("manifest", "releaseIndex"), + ("state", "lineCount"), + ("append", "previousLineCount"), + ("append", "appendedRowCount"), + ], +) +def test_manifest_counts_reject_boolean_values(location: str, field: str): + manifest = _schema_manifest() + target = manifest if location == "manifest" else manifest[location] + target[field] = True + + with pytest.raises(ReleaseChainError, match="not a boolean"): + validate_manifest_schema(manifest) + + +@pytest.mark.parametrize( + "render", + [ + lambda manifest: canonical_bytes(manifest), + lambda manifest: json.dumps(manifest, indent=2).encode("utf-8") + b"\n", + lambda manifest: canonical_bytes(manifest) + b"\n\n", + ], +) +def test_manifest_file_requires_exact_canonical_bytes( + tmp_path: Path, + render, +): + path = tmp_path / "manifest.json" + path.write_bytes(render(_schema_manifest())) + + with pytest.raises(ReleaseChainError, match="not canonical JSON"): + load_manifest(path) + + +def test_manifest_file_accepts_canonical_bytes_plus_one_newline(tmp_path: Path): + manifest = _schema_manifest() + raw = canonical_bytes(manifest) + b"\n" + path = tmp_path / "manifest.json" + path.write_bytes(raw) + + loaded, loaded_raw, digest = load_manifest(path) + + assert loaded == manifest + assert loaded_raw == raw + assert digest == _sha256(raw) diff --git a/tests/test_thesis_append_adversarial.py b/tests/test_thesis_append_adversarial.py index f1a3868..4dcf298 100644 --- a/tests/test_thesis_append_adversarial.py +++ b/tests/test_thesis_append_adversarial.py @@ -116,7 +116,11 @@ def _write_checker_fixture(path: Path, ledger_text: str, manifest: dict) -> None ) scripts_dir = path / "scripts" scripts_dir.mkdir(exist_ok=True) - for name in ("check_thesis_facts_append.py", "canonical_json.py"): + for name in ( + "check_thesis_facts_append.py", + "canonical_json.py", + "verify_release_chain.py", + ): (scripts_dir / name).write_text( (ROOT / "scripts" / name).read_text(encoding="utf-8"), encoding="utf-8", @@ -177,14 +181,20 @@ def test_base_check_rejects_an_existing_line_rewrite(): rewritten["value"] += 1 candidate = [*lines[:-1], _json_line(rewritten)] - with pytest.raises(AppendError, match="rewrites existing line 128"): + with pytest.raises( + AppendError, + match=rf"rewrites existing line {len(lines)}", + ): check_append_only("HEAD", candidate) def test_base_check_rejects_truncation(): lines = _read_lines() - with pytest.raises(AppendError, match="truncates the ledger: 128 -> 127"): + with pytest.raises( + AppendError, + match=rf"truncates the ledger: {len(lines)} -> {len(lines) - 1}", + ): check_append_only("HEAD", lines[:-1])