diff --git a/changelog.d/468.added.md b/changelog.d/468.added.md new file mode 100644 index 00000000..d0d2b22d --- /dev/null +++ b/changelog.d/468.added.md @@ -0,0 +1 @@ +Add versioned execution receipts that distinguish requested aliases from resolved runtime, package, ruleset, and population identities, and bind them into simulation TRACE run records. diff --git a/pyproject.toml b/pyproject.toml index c9f0b91c..f6665917 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "requests>=2.31.0", "psutil>=5.9.0", "packaging>=23.0", + "rfc8785==0.1.4", "google-cloud-storage>=3.1.0,<4.0.0", "diskcache>=5.6.3,<6.0.0", ] diff --git a/src/policyengine/__init__.py b/src/policyengine/__init__.py index 1c4fb0ae..5756706d 100644 --- a/src/policyengine/__init__.py +++ b/src/policyengine/__init__.py @@ -33,6 +33,28 @@ from policyengine import outputs as outputs from policyengine.core import Simulation as Simulation +from policyengine.execution import ( + EXECUTION_RECEIPT_CANONICALIZATION as EXECUTION_RECEIPT_CANONICALIZATION, +) +from policyengine.execution import ( + EXECUTION_RECEIPT_SCHEMA_VERSION as EXECUTION_RECEIPT_SCHEMA_VERSION, +) +from policyengine.execution import ArtifactIdentity as ArtifactIdentity +from policyengine.execution import ( + CertifiedReleaseManifest as CertifiedReleaseManifest, +) +from policyengine.execution import ExecutionReceipt as ExecutionReceipt +from policyengine.execution import PackageResolution as PackageResolution +from policyengine.execution import PackageVersion as PackageVersion +from policyengine.execution import ( + RequestedExecutionAliases as RequestedExecutionAliases, +) +from policyengine.execution import ( + ResolvedExecutionBundle as ResolvedExecutionBundle, +) +from policyengine.execution import RuntimeIdentity as RuntimeIdentity +from policyengine.execution import TraceReference as TraceReference +from policyengine.execution import canonical_content_hash as canonical_content_hash _SKIP_COUNTRY_IMPORTS = os.environ.get("POLICYENGINE_SKIP_COUNTRY_IMPORTS") == "1" @@ -46,4 +68,21 @@ else: # pragma: no cover uk = None # type: ignore[assignment] -__all__ = ["Simulation", "outputs", "uk", "us"] +__all__ = [ + "ArtifactIdentity", + "CertifiedReleaseManifest", + "EXECUTION_RECEIPT_CANONICALIZATION", + "EXECUTION_RECEIPT_SCHEMA_VERSION", + "ExecutionReceipt", + "PackageResolution", + "PackageVersion", + "RequestedExecutionAliases", + "ResolvedExecutionBundle", + "RuntimeIdentity", + "Simulation", + "TraceReference", + "canonical_content_hash", + "outputs", + "uk", + "us", +] diff --git a/src/policyengine/core/run_record.py b/src/policyengine/core/run_record.py index 13f46bf8..a69b0d22 100644 --- a/src/policyengine/core/run_record.py +++ b/src/policyengine/core/run_record.py @@ -25,6 +25,11 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Mapping, Optional, Union +from policyengine.execution import ( + ExecutionReceipt, + TraceReference, + canonical_content_hash, +) from policyengine.provenance.trace import ( build_simulation_trace_tro, canonical_json_bytes, @@ -186,6 +191,61 @@ def build_simulation_run_record_payloads( return {"reform": reform, "input": input_payload, "results": results_payload} +def _validate_execution_receipt_bindings( + receipt: ExecutionReceipt, + *, + simulation: Simulation, + bundle_tro: Mapping, + bundle_tro_url: Optional[str], + request_payload: Optional[Mapping], + results_payload: Mapping, +) -> None: + """Reject a receipt whose cross-references contradict this run record.""" + + if receipt.run_id is not None and receipt.run_id != str(simulation.id): + raise ValueError( + "Execution receipt run_id does not match the simulation being recorded." + ) + + if receipt.request_sha256 is not None: + if request_payload is None: + raise ValueError( + "Execution receipt has request_sha256 but no request_payload was " + "supplied to the run record." + ) + if receipt.request_sha256 != canonical_content_hash(request_payload): + raise ValueError( + "Execution receipt request_sha256 does not match request_payload." + ) + + if ( + receipt.result_sha256 is not None + and receipt.result_sha256 != canonical_content_hash(results_payload) + ): + raise ValueError( + "Execution receipt result_sha256 does not match results_payload." + ) + + claimed_trace = receipt.resolved.bundle_trace + if claimed_trace is None: + raise ValueError( + "Execution receipt is missing a bundle trace reference for bundle_tro." + ) + + actual_trace = TraceReference.from_trace_tro( + bundle_tro, + url=bundle_tro_url, + ) + if claimed_trace.composition_fingerprint != actual_trace.composition_fingerprint: + raise ValueError( + "Execution receipt bundle trace fingerprint does not match bundle_tro." + ) + if claimed_trace.sha256 is not None and claimed_trace.sha256 != actual_trace.sha256: + raise ValueError( + "Execution receipt bundle trace sha256 does not match bundle_tro." + ) + + def write_simulation_run_record( simulation: Simulation, directory: Union[str, Path], @@ -193,21 +253,24 @@ def write_simulation_run_record( bundle_tro: Optional[Mapping] = None, bundle_tro_url: Optional[str] = None, request_payload: Optional[Mapping] = None, + execution_receipt: Optional[ExecutionReceipt] = None, runtime_environment: Optional[Mapping[str, Any]] = None, created_at: Optional[str] = None, ) -> SimulationRunRecord: """Write a self-contained, offline-verifiable run record directory. - Files written: ``bundle.trace.tro.jsonld`` (the certified bundle - TRO), ``reform.json`` (when the simulation has a policy), - ``input.json``, ``results.json``, ``request.json`` (when an API - request payload is supplied), and ``run.trace.tro.jsonld`` binding - them all. Pass ``bundle_tro_url`` whenever a canonical published - location for the bundle TRO exists — it lets a verifier cross-check - the local copy against independently fetched bytes. - - ``created_at`` defaults to the simulation's creation time; pass an - explicit ISO 8601 string to make record bytes reproducible. + Files written: ``bundle.trace.tro.jsonld`` (the certified bundle TRO), + ``reform.json`` (when the simulation has a policy), ``input.json``, + ``results.json``, ``request.json`` (when an API request payload is supplied), + ``execution-receipt.json`` (when a receipt is supplied), and + ``run.trace.tro.jsonld`` binding them all. Pass ``bundle_tro_url`` whenever a + canonical published location for the bundle TRO exists — it lets a verifier + cross-check the local copy against independently fetched bytes. + + ``execution_receipt`` is written as ``execution-receipt.json`` and bound + into the run TRO as its runtime artifact. ``created_at`` defaults to the + simulation's creation time; pass an explicit ISO 8601 string to make + record bytes reproducible. """ if bundle_tro is None: if simulation.tax_benefit_model_version is None: @@ -219,6 +282,23 @@ def write_simulation_run_record( # Build payloads (and surface refusals) before touching the filesystem. payloads = build_simulation_run_record_payloads(simulation) + results_payload = payloads["results"] + if results_payload is None: # Defensive; the builder always returns results. + raise ValueError("Simulation run record results payload is missing.") + if execution_receipt is not None: + _validate_execution_receipt_bindings( + execution_receipt, + simulation=simulation, + bundle_tro=bundle_tro, + bundle_tro_url=bundle_tro_url, + request_payload=request_payload, + results_payload=results_payload, + ) + receipt_payload = ( + execution_receipt.model_dump(mode="json") + if execution_receipt is not None + else None + ) if created_at is None: created_at = simulation.created_at.isoformat() @@ -246,6 +326,10 @@ def write_simulation_run_record( path = directory / "request.json" path.write_bytes(canonical_json_bytes(request_payload)) paths["request"] = path + if receipt_payload is not None: + path = directory / "execution-receipt.json" + path.write_bytes(canonical_json_bytes(receipt_payload)) + paths["execution_receipt"] = path tro = build_simulation_trace_tro( bundle_tro=bundle_tro, @@ -253,6 +337,7 @@ def write_simulation_run_record( reform_payload=payloads["reform"], input_payload=payloads["input"], request_payload=request_payload, + runtime_payload=receipt_payload, runtime_environment=runtime_environment, simulation_id=simulation.id, created_at=created_at, @@ -260,6 +345,7 @@ def write_simulation_run_record( reform_location="reform.json", input_location="input.json", request_location="request.json", + runtime_location="execution-receipt.json", bundle_tro_location="bundle.trace.tro.jsonld", bundle_tro_url=bundle_tro_url, ) diff --git a/src/policyengine/execution.py b/src/policyengine/execution.py new file mode 100644 index 00000000..c4b6337e --- /dev/null +++ b/src/policyengine/execution.py @@ -0,0 +1,371 @@ +"""Versioned execution identities and receipts. + +The models in this module describe what a caller requested separately from +what actually executed. They are intentionally engine-neutral: a runtime can +be PolicyEngine Core, Axiom, or another engine, while ruleset and population +artifacts remain optional. + +Receipts snapshot release-manifest fields into strict schema-v1 DTOs and use +compact TRACE references. They do not replace +:attr:`policyengine.core.Simulation.release_bundle` or the full TRACE TRO +emitted by a simulation run record. +""" + +from __future__ import annotations + +import hashlib +from datetime import datetime +from typing import Any, Literal, Mapping, Optional, Union + +import rfc8785 +from pydantic import ( + BaseModel, + ConfigDict, + Field, + field_validator, + model_validator, +) + +from policyengine.provenance.manifest import CountryReleaseManifest +from policyengine.provenance.trace import ( + canonical_json_bytes, + extract_bundle_tro_reference, +) + +EXECUTION_RECEIPT_SCHEMA_VERSION: Literal[1] = 1 +EXECUTION_RECEIPT_CANONICALIZATION = "RFC8785-JCS" +SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class ExecutionContractModel(BaseModel): + """Strict base model for the public execution contract.""" + + model_config = ConfigDict(extra="forbid") + + +class PackageVersion(ExecutionContractModel): + """Strict package identity frozen by execution-receipt schema v1.""" + + name: str = Field(min_length=1) + version: str = Field(min_length=1) + sha256: Optional[str] = Field(default=None, pattern=SHA256_PATTERN) + wheel_url: Optional[str] = None + + +class DataPackageVersion(PackageVersion): + """Strict data-package identity in a certified release snapshot.""" + + repo_id: str = Field(min_length=1) + repo_type: str = Field(min_length=1) + release_manifest_path: str = Field(min_length=1) + release_manifest_revision: Optional[str] = None + + +class ArtifactPathReference(ExecutionContractModel): + """Strict dataset artifact reference in a certified release snapshot.""" + + path: str = Field(min_length=1) + revision: Optional[str] = None + sha256: Optional[str] = Field(default=None, pattern=SHA256_PATTERN) + metadata_sha256: Optional[str] = Field(default=None, pattern=SHA256_PATTERN) + repo_id: Optional[str] = None + repo_type: Optional[str] = None + + +class ArtifactPathTemplate(ExecutionContractModel): + """Strict region-dataset path template.""" + + path_template: str = Field(min_length=1) + + +class CertifiedDataArtifact(ExecutionContractModel): + """Certified population artifact attached to a release snapshot.""" + + data_package: Optional[PackageVersion] = None + dataset: str = Field(min_length=1) + uri: str = Field(min_length=1) + sha256: Optional[str] = Field(default=None, pattern=SHA256_PATTERN) + build_id: Optional[str] = None + + +class DataCertification(ExecutionContractModel): + """Strict certification statement for a release's population data.""" + + compatibility_basis: str = Field(min_length=1) + certified_for_model_version: str = Field(min_length=1) + data_build_id: Optional[str] = None + built_with_model_version: Optional[str] = None + built_with_model_git_sha: Optional[str] = None + data_build_fingerprint: Optional[str] = None + certified_by: Optional[str] = None + + +class CertifiedReleaseManifest(ExecutionContractModel): + """Strict, serialized snapshot of a certified country release. + + This receipt-owned DTO deliberately excludes the source model's private + ``source_sha256`` field. Freezing the nested shape prevents unknown future + manifest fields from being silently discarded by receipt v1 readers. + """ + + schema_version: int + bundle_id: Optional[str] = None + published_at: Optional[str] = None + country_id: str = Field(min_length=1) + policyengine_version: str = Field(min_length=1) + model_package: PackageVersion + data_package: DataPackageVersion + default_dataset: str = Field(min_length=1) + datasets: dict[str, ArtifactPathReference] + region_datasets: dict[str, ArtifactPathTemplate] + certified_data_artifact: Optional[CertifiedDataArtifact] = None + certification: Optional[DataCertification] = None + + @classmethod + def from_country_release_manifest( + cls, manifest: CountryReleaseManifest + ) -> CertifiedReleaseManifest: + """Copy the public serialized fields from an existing manifest.""" + + return cls.model_validate(manifest.model_dump(mode="json")) + + +class RequestedExecutionAliases(ExecutionContractModel): + """Caller-supplied selectors, which may be mutable aliases. + + Values such as ``latest`` and ``default`` belong here. The + :class:`ResolvedExecutionBundle` records the concrete identities selected + for the run. + """ + + engine: Optional[str] = None + bundle: Optional[str] = None + model: Optional[str] = None + data: Optional[str] = None + ruleset: Optional[str] = None + population: Optional[str] = None + numeric_mode: Optional[str] = None + + +class ArtifactIdentity(ExecutionContractModel): + """Resolved identity for a runtime, ruleset, or population artifact.""" + + name: str = Field(min_length=1) + version: Optional[str] = None + uri: Optional[str] = None + revision: Optional[str] = None + sha256: Optional[str] = Field(default=None, pattern=SHA256_PATTERN) + build_id: Optional[str] = None + + @model_validator(mode="after") + def _require_resolved_identity(self) -> ArtifactIdentity: + if not any((self.version, self.uri, self.revision, self.sha256, self.build_id)): + raise ValueError( + "A resolved artifact needs a version, URI, revision, sha256, " + "or build_id." + ) + return self + + +class RuntimeIdentity(ExecutionContractModel): + """Concrete identity of the engine that performed the calculation.""" + + name: str = Field(min_length=1) + version: str = Field(min_length=1) + git_sha: Optional[str] = None + artifact: Optional[ArtifactIdentity] = None + + +class PackageResolution(ExecutionContractModel): + """Actual and certified identities for one package role. + + ``actual`` is the package used by the process. ``certified`` is the + package selected by release metadata. Keeping both prevents an installed + version mismatch from being reported as though the certified package ran. + A resolution without ``actual`` is invalid: certification context alone + belongs in :class:`CertifiedReleaseManifest`, not in a field describing a + component that executed. + """ + + actual: PackageVersion + certified: Optional[PackageVersion] = None + + @classmethod + def from_release_bundle( + cls, + release_bundle: Mapping[str, Any], + component: Literal["model", "data"], + *, + actual: PackageVersion, + ) -> PackageResolution: + """Resolve one used package role from the existing dict API. + + The release bundle supplies only the certified selection. Callers must + pass ``actual`` explicitly when the component was loaded; this method + never assumes the certified package is what ran. Invoke this only for + components the execution actually used. + """ + + if component not in ("model", "data"): + raise ValueError("component must be 'model' or 'data'") + name = release_bundle.get(f"{component}_package") + version = release_bundle.get(f"{component}_version") + if (name is None) != (version is None): + raise ValueError( + f"release_bundle has an incomplete certified {component} package" + ) + certified = ( + PackageVersion(name=str(name), version=str(version)) + if name is not None + else None + ) + return cls(actual=actual, certified=certified) + + +class TraceReference(ExecutionContractModel): + """Compact reference to an existing TRACE Transparent Research Object.""" + + composition_fingerprint: str = Field(pattern=SHA256_PATTERN) + sha256: Optional[str] = Field(default=None, pattern=SHA256_PATTERN) + url: Optional[str] = None + name: Optional[str] = None + + @classmethod + def from_trace_tro( + cls, + tro: Mapping[str, Any], + *, + url: Optional[str] = None, + ) -> TraceReference: + """Build a compact reference with the existing TRACE extractor.""" + + reference = extract_bundle_tro_reference(tro) + return cls( + composition_fingerprint=reference["fingerprint"], + # TRACE serializes TROs with its established pretty canonical form; + # this digest identifies those exact published bytes rather than + # the semantic RFC 8785 request/result hash used by receipts. + sha256=hashlib.sha256(canonical_json_bytes(tro)).hexdigest(), + url=url or reference.get("self_url"), + name=reference.get("name"), + ) + + +class ResolvedExecutionBundle(ExecutionContractModel): + """Concrete runtime and artifacts used for an execution. + + ``model`` and ``data`` are absent when that package role was not part of + the run. In particular, a household calculation that did not load + population data should leave both ``data`` and ``population_artifact`` + unset, even when its selected release also certifies population data. + + ``certified_release`` freezes the public fields of a + :class:`CountryReleaseManifest` as release context. Its presence does not + by itself claim that every component in that release was accessed; used + components are the explicit fields above. + """ + + runtime: RuntimeIdentity + numeric_mode: str = Field(min_length=1) + model: Optional[PackageResolution] = None + data: Optional[PackageResolution] = None + ruleset_artifact: Optional[ArtifactIdentity] = None + population_artifact: Optional[ArtifactIdentity] = None + certified_release: Optional[CertifiedReleaseManifest] = None + bundle_trace: Optional[TraceReference] = None + + @field_validator("certified_release", mode="before") + @classmethod + def _copy_certified_release(cls, value: Any) -> Optional[CertifiedReleaseManifest]: + if value is None or isinstance(value, CertifiedReleaseManifest): + return value + if isinstance(value, CountryReleaseManifest): + return CertifiedReleaseManifest.from_country_release_manifest(value) + return CertifiedReleaseManifest.model_validate(value) + + @model_validator(mode="after") + def _certified_packages_match_release(self) -> ResolvedExecutionBundle: + if self.certified_release is None: + return self + expected = { + "model": self.certified_release.model_package, + "data": self.certified_release.data_package, + } + for component, resolution in (("model", self.model), ("data", self.data)): + if resolution is None or resolution.certified is None: + continue + package = resolution.certified + release_package = expected[component] + claimed_fields = package.model_dump(mode="json", exclude_none=True) + release_fields = release_package.model_dump(mode="json") + for field, claimed_value in claimed_fields.items(): + if claimed_value != release_fields.get(field): + raise ValueError( + f"Certified {component} package field {field!r} does not " + "match certified_release." + ) + return self + + +class ExecutionReceipt(ExecutionContractModel): + """Versioned, JSON-serializable receipt for one execution.""" + + schema_version: Literal[1] = EXECUTION_RECEIPT_SCHEMA_VERSION + requested: RequestedExecutionAliases = Field( + default_factory=RequestedExecutionAliases + ) + resolved: ResolvedExecutionBundle + run_id: Optional[str] = None + created_at: Optional[datetime] = None + request_sha256: Optional[str] = Field( + default=None, + pattern=SHA256_PATTERN, + description="SHA-256 of RFC 8785 JCS canonical request bytes.", + ) + result_sha256: Optional[str] = Field( + default=None, + pattern=SHA256_PATTERN, + description="SHA-256 of RFC 8785 JCS canonical result bytes.", + ) + + def content_hash(self) -> str: + """Return the receipt's canonical SHA-256 content hash.""" + + return canonical_content_hash(self) + + +def canonical_content_hash(value: Union[BaseModel, Mapping[str, Any]]) -> str: + """Hash semantic JSON content with RFC 8785 JCS canonicalization. + + Receipt schema v1 fixes ``RFC8785-JCS`` as its cross-language byte format. + The returned lowercase, 64-character digest is therefore portable across + Python, Rust, and TypeScript implementations. RFC 8785 rejects non-finite + numbers and non-string object keys rather than producing non-standard JSON. + + This differs intentionally from TRACE's established pretty canonical + serialization, which identifies exact TRO document bytes. + """ + + payload = value.model_dump(mode="json") if isinstance(value, BaseModel) else value + return hashlib.sha256(rfc8785.dumps(payload)).hexdigest() + + +__all__ = [ + "EXECUTION_RECEIPT_CANONICALIZATION", + "EXECUTION_RECEIPT_SCHEMA_VERSION", + "ArtifactIdentity", + "ArtifactPathReference", + "ArtifactPathTemplate", + "CertifiedDataArtifact", + "CertifiedReleaseManifest", + "DataCertification", + "DataPackageVersion", + "ExecutionReceipt", + "PackageResolution", + "PackageVersion", + "RequestedExecutionAliases", + "ResolvedExecutionBundle", + "RuntimeIdentity", + "TraceReference", + "canonical_content_hash", +] diff --git a/tests/fixtures/execution_fixtures.py b/tests/fixtures/execution_fixtures.py new file mode 100644 index 00000000..e1dcef68 --- /dev/null +++ b/tests/fixtures/execution_fixtures.py @@ -0,0 +1,73 @@ +"""Factories for execution-contract tests.""" + +from datetime import datetime, timezone + +from policyengine.execution import ( + ArtifactIdentity, + ExecutionReceipt, + PackageResolution, + PackageVersion, + RequestedExecutionAliases, + ResolvedExecutionBundle, + RuntimeIdentity, +) +from policyengine.provenance.manifest import get_release_manifest + +REQUEST_SHA256 = "1" * 64 +RESULT_SHA256 = "2" * 64 +POPULATION_SHA256 = "3" * 64 + + +def make_execution_receipt(*, run_id: str = "run-1") -> ExecutionReceipt: + release = get_release_manifest("us") + return ExecutionReceipt( + requested=RequestedExecutionAliases( + engine="default", + bundle="latest", + model="latest", + data="certified", + population="national", + numeric_mode="default", + ), + resolved=ResolvedExecutionBundle( + runtime=RuntimeIdentity( + name="policyengine-core", + version="3.30.0", + ), + numeric_mode="numpy-native", + model=PackageResolution( + actual=PackageVersion( + name="policyengine-us", + version="1.765.0", + ), + certified=PackageVersion.model_validate( + release.model_package.model_dump(mode="json") + ), + ), + data=PackageResolution( + actual=PackageVersion(name="populace-data", version="0.1.1"), + certified=PackageVersion( + name=release.data_package.name, + version=release.data_package.version, + sha256=release.data_package.sha256, + wheel_url=release.data_package.wheel_url, + ), + ), + population_artifact=ArtifactIdentity( + name=release.default_dataset, + uri=release.default_dataset_uri, + revision=release.data_package.release_manifest_revision, + sha256=POPULATION_SHA256, + build_id=( + release.certification.data_build_id + if release.certification is not None + else None + ), + ), + certified_release=release, + ), + run_id=run_id, + created_at=datetime(2026, 7, 9, 12, 0, tzinfo=timezone.utc), + request_sha256=REQUEST_SHA256, + result_sha256=RESULT_SHA256, + ) diff --git a/tests/test_execution.py b/tests/test_execution.py new file mode 100644 index 00000000..d487ee75 --- /dev/null +++ b/tests/test_execution.py @@ -0,0 +1,320 @@ +"""Public execution-contract and receipt tests.""" + +import hashlib +import json +import math + +import pytest +import rfc8785 +from jsonschema import Draft202012Validator +from pydantic import ValidationError + +from policyengine.execution import ( + ArtifactIdentity, + CertifiedReleaseManifest, + ExecutionReceipt, + PackageResolution, + PackageVersion, + ResolvedExecutionBundle, + RuntimeIdentity, + TraceReference, + canonical_content_hash, +) +from policyengine.provenance.manifest import get_release_manifest +from policyengine.provenance.trace import canonical_json_bytes + +from .fixtures.execution_fixtures import make_execution_receipt + + +class TestExecutionReceipt: + def test__given_aliases_and_resolved_versions__then_both_survive_json_round_trip( + self, + ): + # Given + receipt = make_execution_receipt() + + # When + payload = json.loads(receipt.model_dump_json()) + restored = ExecutionReceipt.model_validate(payload) + + # Then + assert payload["requested"]["model"] == "latest" + assert payload["resolved"]["model"]["actual"]["version"] == "1.765.0" + assert ( + payload["resolved"]["model"]["certified"]["version"] + == get_release_manifest("us").model_package.version + ) + assert restored.model_dump(mode="json") == payload + + def test__given_receipt__then_generated_json_schema_validates_payload(self): + # Given + receipt = make_execution_receipt() + schema = ExecutionReceipt.model_json_schema() + + # When + Draft202012Validator.check_schema(schema) + errors = list( + Draft202012Validator(schema).iter_errors(receipt.model_dump(mode="json")) + ) + + # Then + assert errors == [] + assert schema["properties"]["schema_version"]["const"] == 1 + release_properties = schema["$defs"]["CertifiedReleaseManifest"]["properties"] + assert "source_sha256" not in release_properties + + def test__given_unknown_nested_release_field__then_validation_fails(self): + # Given + payload = make_execution_receipt().model_dump(mode="json") + payload["resolved"]["certified_release"]["model_package"]["unknown"] = ( + "silently-lost" + ) + + # When / Then + with pytest.raises(ValidationError, match="Extra inputs are not permitted"): + ExecutionReceipt.model_validate(payload) + + def test__given_country_manifest__then_receipt_snapshot_round_trips_strictly(self): + # Given + release = get_release_manifest("us") + + # When + snapshot = CertifiedReleaseManifest.from_country_release_manifest(release) + restored = CertifiedReleaseManifest.model_validate( + snapshot.model_dump(mode="json") + ) + + # Then + assert restored == snapshot + assert "source_sha256" not in snapshot.model_dump(mode="json") + + def test__given_household_without_population_data__then_data_identity_is_absent( + self, + ): + # Given + release = get_release_manifest("us") + + # When + resolved = ResolvedExecutionBundle( + runtime=RuntimeIdentity(name="axiom", version="0.4.0"), + numeric_mode="exact-decimal", + ruleset_artifact=ArtifactIdentity( + name="us-federal-2026", + revision="rulespec-deadbeef", + sha256="a" * 64, + ), + certified_release=release, + ) + + # Then + assert resolved.data is None + assert resolved.population_artifact is None + + def test__given_certified_package_mismatch__then_validation_fails(self): + # Given + release = get_release_manifest("us") + + # When / Then + with pytest.raises(ValidationError, match="does not match"): + ResolvedExecutionBundle( + runtime=RuntimeIdentity(name="policyengine-core", version="3.30.0"), + numeric_mode="numpy-native", + model=PackageResolution( + actual=PackageVersion( + name="policyengine-us", + version="1.765.0", + ), + certified=PackageVersion( + name="policyengine-us", + version="0.0.1", + ), + ), + certified_release=release, + ) + + @pytest.mark.parametrize( + ("field", "mismatched_value"), + [ + ("sha256", "f" * 64), + ("wheel_url", "https://example.test/wrong-package.whl"), + ], + ) + def test__given_certified_package_metadata_mismatch__then_validation_fails( + self, + field, + mismatched_value, + ): + # Given + release = get_release_manifest("us") + certified = PackageVersion.model_validate( + release.model_package.model_dump(mode="json") + ).model_copy(update={field: mismatched_value}) + + # When / Then + with pytest.raises(ValidationError, match=field): + ResolvedExecutionBundle( + runtime=RuntimeIdentity(name="policyengine-core", version="3.30.0"), + numeric_mode="numpy-native", + model=PackageResolution( + actual=PackageVersion( + name="policyengine-us", + version="1.765.0", + ), + certified=certified, + ), + certified_release=release, + ) + + def test__given_wrong_schema_version__then_validation_fails(self): + # Given + payload = make_execution_receipt().model_dump(mode="json") + payload["schema_version"] = 2 + + # When / Then + with pytest.raises(ValidationError): + ExecutionReceipt.model_validate(payload) + + def test__given_no_resolved_numeric_mode__then_validation_fails(self): + # Given + payload = { + "runtime": { + "name": "policyengine-core", + "version": "3.30.0", + } + } + + # When / Then + with pytest.raises(ValidationError): + ResolvedExecutionBundle.model_validate(payload) + + +class TestPackageResolution: + def test__given_release_bundle__then_certified_and_actual_stay_distinct(self): + # Given + release_bundle = { + "model_package": "policyengine-us", + "model_version": "1.764.6", + "data_package": "populace-data", + "data_version": "0.1.0", + } + original = dict(release_bundle) + actual = PackageVersion(name="policyengine-us", version="1.765.0") + + # When + resolution = PackageResolution.from_release_bundle( + release_bundle, + "model", + actual=actual, + ) + + # Then + assert resolution.actual.version == "1.765.0" + assert resolution.certified.version == "1.764.6" + assert release_bundle == original + + def test__given_incomplete_release_bundle__then_validation_fails(self): + # Given + release_bundle = {"model_package": "policyengine-us"} + actual = PackageVersion(name="policyengine-us", version="1.765.0") + + # When / Then + with pytest.raises(ValueError, match="incomplete"): + PackageResolution.from_release_bundle( + release_bundle, + "model", + actual=actual, + ) + + def test__given_certification_without_actual_package__then_validation_fails(self): + # Given / When / Then + with pytest.raises(ValidationError, match="actual"): + PackageResolution( + certified=PackageVersion( + name="policyengine-us", + version="1.764.6", + ) + ) + + +class TestContentIdentity: + def test__given_same_mapping_in_different_order__then_hashes_match(self): + # Given + first = {"requested": {"model": "latest"}, "period": "2026"} + second = {"period": "2026", "requested": {"model": "latest"}} + + # When + first_hash = canonical_content_hash(first) + second_hash = canonical_content_hash(second) + + # Then + assert first_hash == second_hash + assert len(first_hash) == 64 + + @pytest.mark.parametrize( + ("value", "canonical_bytes"), + [ + ({"small": 1e-6}, b'{"small":0.000001}'), + ({"large": 1e30}, b'{"large":1e+30}'), + ({"unicode": "\u20ac\u00e9"}, '{"unicode":"\u20ac\u00e9"}'.encode()), + ], + ) + def test__given_cross_language_json_value__then_hash_uses_rfc8785( + self, value, canonical_bytes + ): + # When / Then + assert ( + canonical_content_hash(value) == hashlib.sha256(canonical_bytes).hexdigest() + ) + + def test__given_non_finite_number__then_hashing_refuses_non_json_value(self): + # Given / When / Then + with pytest.raises(rfc8785.FloatDomainError): + canonical_content_hash({"value": math.nan}) + + def test__given_resolved_runtime_changes__then_receipt_hash_changes(self): + # Given + receipt = make_execution_receipt() + changed_resolved = receipt.resolved.model_copy( + update={ + "runtime": RuntimeIdentity( + name="axiom-rules-engine", + version="0.4.0", + ) + } + ) + changed = receipt.model_copy(update={"resolved": changed_resolved}) + + # When + original_hash = receipt.content_hash() + changed_hash = changed.content_hash() + + # Then + assert original_hash != changed_hash + + def test__given_invalid_sha256__then_validation_fails(self): + # Given / When / Then + with pytest.raises(ValidationError): + ArtifactIdentity(name="ruleset", revision="abc", sha256="not-a-hash") + + def test__given_trace_tro__then_reference_uses_trace_fingerprint_and_hash(self): + # Given + tro = { + "@graph": [ + { + "@type": "trov:TransparentResearchObject", + "schema:name": "test bundle", + "pe:selfUrl": "https://example.test/bundle.trace.tro.jsonld", + "trov:hasComposition": { + "trov:hasFingerprint": {"trov:sha256": "b" * 64} + }, + } + ] + } + + # When + reference = TraceReference.from_trace_tro(tro) + + # Then + assert reference.composition_fingerprint == "b" * 64 + assert reference.sha256 == hashlib.sha256(canonical_json_bytes(tro)).hexdigest() + assert reference.url == "https://example.test/bundle.trace.tro.jsonld" diff --git a/tests/test_run_record.py b/tests/test_run_record.py index d5cecdb3..aa5f268f 100644 --- a/tests/test_run_record.py +++ b/tests/test_run_record.py @@ -30,6 +30,7 @@ write_simulation_run_record, ) from policyengine.core.simulation import Simulation +from policyengine.execution import TraceReference, canonical_content_hash from policyengine.provenance.manifest import ( get_data_release_manifest, get_release_manifest, @@ -40,11 +41,28 @@ ) from policyengine.provenance.verify import verify_trace_tro_path +from .fixtures.execution_fixtures import make_execution_receipt from .test_trace_tro import _fake_fetch_pypi, _us_data_release_manifest FIXED_CREATED_AT = "2026-06-12T00:00:00Z" +def _bound_execution_receipt(simulation, bundle_tro, request_payload): + payloads = build_simulation_run_record_payloads(simulation) + assert payloads["results"] is not None + receipt = make_execution_receipt(run_id=str(simulation.id)) + resolved = receipt.resolved.model_copy( + update={"bundle_trace": TraceReference.from_trace_tro(bundle_tro)} + ) + return receipt.model_copy( + update={ + "resolved": resolved, + "request_sha256": canonical_content_hash(request_payload), + "result_sha256": canonical_content_hash(payloads["results"]), + } + ) + + class _StubYearData(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) @@ -254,6 +272,147 @@ def test__given_record__then_verifier_passes_offline( assert report.fingerprint_status == "ok" assert report.ok + def test__given_execution_receipt__then_trace_binds_canonical_receipt( + self, simulation, bundle_tro, tmp_path + ): + # Given + request_payload = {"country_id": "us", "request": {"period": "2026"}} + receipt = _bound_execution_receipt( + simulation, + bundle_tro, + request_payload, + ) + + # When + record = write_simulation_run_record( + simulation, + tmp_path / "record", + bundle_tro=bundle_tro, + request_payload=request_payload, + execution_receipt=receipt, + created_at=FIXED_CREATED_AT, + ) + + # Then + receipt_path = record.paths["execution_receipt"] + assert receipt_path.name == "execution-receipt.json" + assert receipt_path.read_bytes() == canonical_json_bytes( + receipt.model_dump(mode="json") + ) + report = verify_trace_tro_path(record.paths["tro"]) + statuses = {check.artifact_id: check.status for check in report.artifacts} + assert statuses["runtime"] == "ok" + assert report.ok + + @pytest.mark.parametrize( + "mismatch", + ["run_id", "request_sha256", "result_sha256", "bundle_trace"], + ) + def test__given_receipt_cross_reference_mismatch__then_write_refuses_before_io( + self, simulation, bundle_tro, tmp_path, mismatch + ): + # Given + request_payload = {"country_id": "us", "request": {"period": "2026"}} + receipt = _bound_execution_receipt( + simulation, + bundle_tro, + request_payload, + ) + if mismatch == "bundle_trace": + assert receipt.resolved.bundle_trace is not None + receipt = receipt.model_copy( + update={ + "resolved": receipt.resolved.model_copy( + update={ + "bundle_trace": receipt.resolved.bundle_trace.model_copy( + update={"composition_fingerprint": "f" * 64} + ) + } + ) + } + ) + else: + receipt = receipt.model_copy(update={mismatch: "f" * 64}) + directory = tmp_path / "record" + + # When / Then + expected_message = "bundle trace" if mismatch == "bundle_trace" else mismatch + with pytest.raises(ValueError, match=expected_message): + write_simulation_run_record( + simulation, + directory, + bundle_tro=bundle_tro, + request_payload=request_payload, + execution_receipt=receipt, + created_at=FIXED_CREATED_AT, + ) + assert not directory.exists() + + def test__given_receipt_missing_bundle_trace__then_write_refuses_before_io( + self, simulation, bundle_tro, tmp_path + ): + # Given + request_payload = {"country_id": "us", "request": {"period": "2026"}} + receipt = _bound_execution_receipt( + simulation, + bundle_tro, + request_payload, + ) + receipt = receipt.model_copy( + update={ + "resolved": receipt.resolved.model_copy(update={"bundle_trace": None}) + } + ) + directory = tmp_path / "record" + + # When / Then + with pytest.raises(ValueError, match="missing a bundle trace"): + write_simulation_run_record( + simulation, + directory, + bundle_tro=bundle_tro, + request_payload=request_payload, + execution_receipt=receipt, + created_at=FIXED_CREATED_AT, + ) + assert not directory.exists() + + def test__given_receipt_bundle_trace_sha_mismatch__then_write_refuses_before_io( + self, simulation, bundle_tro, tmp_path + ): + # Given + request_payload = {"country_id": "us", "request": {"period": "2026"}} + receipt = _bound_execution_receipt( + simulation, + bundle_tro, + request_payload, + ) + assert receipt.resolved.bundle_trace is not None + receipt = receipt.model_copy( + update={ + "resolved": receipt.resolved.model_copy( + update={ + "bundle_trace": receipt.resolved.bundle_trace.model_copy( + update={"sha256": "f" * 64} + ) + } + ) + } + ) + directory = tmp_path / "record" + + # When / Then + with pytest.raises(ValueError, match="bundle trace sha256"): + write_simulation_run_record( + simulation, + directory, + bundle_tro=bundle_tro, + request_payload=request_payload, + execution_receipt=receipt, + created_at=FIXED_CREATED_AT, + ) + assert not directory.exists() + def test__given_tampered_results__then_verifier_reports_mismatch( self, simulation, bundle_tro, tmp_path ): diff --git a/uv.lock b/uv.lock index 556cf036..62fe1f4c 100644 --- a/uv.lock +++ b/uv.lock @@ -2820,7 +2820,7 @@ wheels = [ [[package]] name = "policyengine" -version = "4.20.1" +version = "4.20.3" source = { editable = "." } dependencies = [ { name = "diskcache" }, @@ -2835,6 +2835,7 @@ dependencies = [ { name = "psutil" }, { name = "pydantic" }, { name = "requests" }, + { name = "rfc8785" }, ] [package.optional-dependencies] @@ -2913,6 +2914,7 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.26.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, { name = "requests", specifier = ">=2.31.0" }, + { name = "rfc8785", specifier = "==0.1.4" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.0" }, { name = "towncrier", marker = "extra == 'dev'", specifier = ">=24.8.0" }, { name = "yaml-changelog", marker = "extra == 'dev'", specifier = ">=0.1.7" }, @@ -3659,6 +3661,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] +[[package]] +name = "rfc8785" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/2f/fa1d2e740c490191b572d33dbca5daa180cb423c24396b856f5886371d8b/rfc8785-0.1.4.tar.gz", hash = "sha256:e545841329fe0eee4f6a3b44e7034343100c12b4ec566dc06ca9735681deb4da", size = 14321, upload-time = "2024-09-27T16:33:31.206Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/78/119878110660b2ad709888c8a1614fce7e2fab39080ab960656dc8605bf6/rfc8785-0.1.4-py3-none-any.whl", hash = "sha256:520d690b448ecf0703691c76e1a34a24ddcd4fc5bc41d589cb7c58ec651bcd48", size = 9240, upload-time = "2024-09-27T16:33:29.683Z" }, +] + [[package]] name = "rpds-py" version = "0.27.1"