From 24d82541a8b3fb13c25d6d3500ad53c6de7b2a97 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 7 Jul 2026 07:14:52 -0400 Subject: [PATCH 1/2] Add Zenodo preservation mirroring for certified releases Port the reviewed-good deposit core from closed PR #405, retargeted to the bundle-metadata system that main adopted after that branch's base. - provenance/zenodo.py: deposit a release's certification record (bundle manifest, bundle TRACE TRO, data release manifest) on Zenodo, returning PreservationMirror entries. The licence gate refuses dataset bytes from private or gated source repos. Retargeted vs #405: read the shipped TRO from data/bundle/ (the deleted data/release_manifests/ path) and pass data_package.repo_type through to _dataset_location_from_uri instead of relying on the old "model" default. - cli.py: register the zenodo-mirror subcommand and its dispatch branch in the current parser/main structure. - trace.py: surface DataReleaseManifest.preservation_dois on the bundle TRO performance node as pe:preservationDoi (#439 code half; the old release-manifest pathway is not restored). - Tests: 15 mocked Zenodo/HF deposit tests and 2 TRO DOI-surfacing tests; all Zenodo/HF interaction is mocked, no live calls. - Re-add the preservation-mirroring docs section and a changelog fragment. Fixes #404 Co-Authored-By: Claude Fable 5 --- changelog.d/404.added.md | 1 + docs/release-bundles.md | 36 +++ src/policyengine/cli.py | 72 +++++ src/policyengine/provenance/trace.py | 8 + src/policyengine/provenance/zenodo.py | 446 ++++++++++++++++++++++++++ tests/test_trace_tro.py | 22 ++ tests/test_zenodo_mirror.py | 293 +++++++++++++++++ 7 files changed, 878 insertions(+) create mode 100644 changelog.d/404.added.md create mode 100644 src/policyengine/provenance/zenodo.py create mode 100644 tests/test_zenodo_mirror.py diff --git a/changelog.d/404.added.md b/changelog.d/404.added.md new file mode 100644 index 00000000..603944de --- /dev/null +++ b/changelog.d/404.added.md @@ -0,0 +1 @@ +Add `policyengine zenodo-mirror `, which deposits a certified release's certification record (bundle manifest, bundle TRACE TRO, data release manifest) on Zenodo for preservation and returns `PreservationMirror` entries, refusing dataset bytes from private source repos; the bundle TRO now surfaces a release's preservation DOI as `pe:preservationDoi` when the data release manifest records one. diff --git a/docs/release-bundles.md b/docs/release-bundles.md index 328f8e2c..bccab440 100644 --- a/docs/release-bundles.md +++ b/docs/release-bundles.md @@ -415,6 +415,42 @@ fail step 3. countries whose data release manifest is unreachable so a partial run does not block other countries. +### Preservation mirroring (Zenodo) + +Hugging Face hosts the primary artifacts but publishes no preservation +commitment — its DOIs are short URLs that are deleted with the repo. +Zenodo publishes a preservation policy and mints real DOIs, so every +certified release's *certification record* (the bundle manifest, the +bundle TRO, and the data release manifest) should also live there: + +```bash +ZENODO_TOKEN=... policyengine zenodo-mirror us # draft deposit +ZENODO_TOKEN=... policyengine zenodo-mirror us --publish # mint the DOI +policyengine zenodo-mirror us --sandbox # rehearsal +``` + +The deposit output includes `PreservationMirror` entries (host, URL, +DOI, sha256, timestamp) ready to merge into the data release manifest's +`preservation_mirrors`/`preservation_dois` fields, which the bundle TRO +then naturally carries on the next regeneration. + +Two deliberate safety properties: + +- **Licence gate.** `--include-dataset` deposits the dataset bytes too, + but only when the source Hugging Face repo is publicly readable. A + private source repo (the UK microdata, under UK Data Service licence) + is refused unconditionally — the certification record still mirrors, + because the manifests and TRO contain only hashes and version pins, + which is exactly what makes a future copy of restricted data + verifiable without redistributing it. +- **Deliberate publishing.** Deposits are drafts by default and carry + no licence unless `--license` is passed; minting the DOI is an + explicit `--publish`. + +Tokens come from `ZENODO_TOKEN` (create one under Zenodo → +Applications → Personal access tokens, with `deposit:write` and +`deposit:actions` scopes). + ### What TRACE does not replace TRACE is not the source of truth for compatibility policy. diff --git a/src/policyengine/cli.py b/src/policyengine/cli.py index 7bd47bdb..d412528a 100644 --- a/src/policyengine/cli.py +++ b/src/policyengine/cli.py @@ -6,6 +6,7 @@ - ``trace-tro-validate `` validate a TRO against the shipped schema - ``trace-tro-verify `` fetch and rehash every artifact a TRO claims - ``release-manifest `` print the bundled country manifest +- ``zenodo-mirror `` deposit the certification record on Zenodo See :mod:`policyengine.provenance.trace` and ``docs/release-bundles.md``. """ @@ -265,6 +266,43 @@ def _parser() -> argparse.ArgumentParser: "--manifest", help="Custom bundle manifest path or URL." ) + zenodo = subparsers.add_parser( + "zenodo-mirror", + help=( + "Deposit a certified release's certification record (bundle " + "manifest, bundle TRO, data release manifest) on Zenodo for " + "preservation. Draft by default; --publish mints the DOI." + ), + ) + zenodo.add_argument("country", help="Country id (e.g. us, uk).") + zenodo.add_argument( + "--publish", + action="store_true", + help="Publish the deposit and mint its DOI (default: leave a draft).", + ) + zenodo.add_argument( + "--include-dataset", + action="store_true", + help=( + "Also deposit the certified dataset bytes. Refused when the " + "source Hugging Face repo is not publicly readable." + ), + ) + zenodo.add_argument( + "--sandbox", + action="store_true", + help="Target sandbox.zenodo.org instead of zenodo.org.", + ) + zenodo.add_argument( + "--license", + dest="license_id", + default=None, + help=( + "Zenodo license identifier (e.g. cc-zero) for the deposit. Set " + "deliberately before publishing." + ), + ) + return parser @@ -430,6 +468,32 @@ def _emit_bundle_manifest(args: argparse.Namespace) -> int: return 0 +def _zenodo_mirror( + country_id: str, + *, + publish: bool, + include_dataset: bool, + sandbox: bool, + license_id: Optional[str], +) -> int: + from policyengine.provenance import zenodo + + base_url = zenodo.ZENODO_SANDBOX_API if sandbox else zenodo.ZENODO_API + try: + deposit = zenodo.mirror_release_to_zenodo( + country_id, + base_url=base_url, + publish=publish, + include_dataset=include_dataset, + license_id=license_id, + ) + except (zenodo.ZenodoDepositError, zenodo.PrivateSourceRepoError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + print(json.dumps(deposit.to_json(), indent=2, sort_keys=True)) + return 0 + + def main(argv: Optional[Sequence[str]] = None) -> int: args = _parser().parse_args(argv) if args.command == "trace-tro": @@ -440,6 +504,14 @@ def main(argv: Optional[Sequence[str]] = None) -> int: return _verify_tro(args.path, args.base_dir, args.skip) if args.command == "release-manifest": return _emit_release_manifest(args.country) + if args.command == "zenodo-mirror": + return _zenodo_mirror( + args.country, + publish=args.publish, + include_dataset=args.include_dataset, + sandbox=args.sandbox, + license_id=args.license_id, + ) if args.command == "bundle": if args.bundle_command == "install": return _install_bundle(args) diff --git a/src/policyengine/provenance/trace.py b/src/policyengine/provenance/trace.py index be314265..350c676f 100644 --- a/src/policyengine/provenance/trace.py +++ b/src/policyengine/provenance/trace.py @@ -463,6 +463,14 @@ def build_trace_tro_from_release_bundle( ) if data_release_manifest is None: performance["pe:dataReleaseManifestStatus"] = "unavailable" + elif data_release_manifest.preservation_dois: + # The release was mirrored to a DOI-minting preservation host + # (e.g. Zenodo). Surface the record-level DOI so a reader who + # has only this TRO can find and cite the preserved copy if the + # primary host ever becomes unavailable. + performance["pe:preservationDoi"] = list( + data_release_manifest.preservation_dois + ) tro_node = _assemble_tro_node( tro_name=f"policyengine {country_manifest.country_id} certified bundle TRO", diff --git a/src/policyengine/provenance/zenodo.py b/src/policyengine/provenance/zenodo.py new file mode 100644 index 00000000..94e1bca7 --- /dev/null +++ b/src/policyengine/provenance/zenodo.py @@ -0,0 +1,446 @@ +"""Zenodo mirroring of certified releases. + +Hugging Face hosts PolicyEngine's primary data artifacts, but it +publishes no preservation commitment: its DOIs are short URLs that die +with the repo. Zenodo publishes a preservation policy and mints real +DOIs, so a certified release mirrored there stays resolvable — and a +TRO citation can fall back to it — even if the primary host changes. + +What gets deposited is the *certification record* of a release: the +bundled country manifest, the bundle TRACE TRO, and the country data +release manifest. These are small, license-unencumbered metadata files +that pin every artifact by sha256, so a future reader can verify any +copy of the data they obtain. The dataset bytes themselves are only +deposited on explicit request (``include_dataset=True``) and only when +the source Hugging Face repo is publicly readable — a private source +repo (the UK microdata, under UK Data Service licence) is refused +unconditionally, because mirroring it would republish data we do not +have the right to redistribute. + +Deposits are created as drafts by default; ``publish=True`` publishes +and mints the DOI. Set deposit licensing deliberately before +publishing (the ``license`` argument, or Zenodo's web UI for drafts). +""" + +from __future__ import annotations + +import hashlib +import os +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Callable, Optional + +from .manifest import ( + CountryReleaseManifest, + DataReleaseManifest, + DataReleaseManifestUnavailableError, + PreservationMirror, + get_data_release_manifest, + get_release_manifest, +) +from .trace import canonical_json_bytes + +ZENODO_API = "https://zenodo.org/api" +ZENODO_SANDBOX_API = "https://sandbox.zenodo.org/api" + +_TIMEOUT_SECONDS = 120.0 + + +class ZenodoDepositError(RuntimeError): + """A Zenodo deposit step failed or was misconfigured.""" + + +class PrivateSourceRepoError(RuntimeError): + """Refused to mirror dataset bytes whose source repo is not public.""" + + +@dataclass(frozen=True) +class ZenodoDeposit: + """Outcome of mirroring one certified release to Zenodo.""" + + deposit_id: str + deposit_url: Optional[str] + doi: Optional[str] + concept_doi: Optional[str] + published: bool + mirrors: list[PreservationMirror] = field(default_factory=list) + + def to_json(self) -> dict[str, Any]: + return { + "deposit_id": self.deposit_id, + "deposit_url": self.deposit_url, + "doi": self.doi, + "concept_doi": self.concept_doi, + "published": self.published, + "mirrors": [mirror.model_dump(mode="json") for mirror in self.mirrors], + } + + +def deposition_metadata( + country_manifest: CountryReleaseManifest, + data_release_manifest: Optional[DataReleaseManifest], + *, + license_id: Optional[str] = None, +) -> dict[str, Any]: + """Zenodo deposition metadata describing a certified release. + + The description names the bundle, the data build, and the certified + dataset hash so the deposit is self-describing without downloading + any file. ``license_id`` is only applied when given — set it + deliberately, especially when dataset bytes are included. + """ + country_id = country_manifest.country_id + bundle_id = country_manifest.bundle_id or ( + f"{country_id}-{country_manifest.policyengine_version}" + ) + build_id = None + if data_release_manifest is not None and data_release_manifest.build is not None: + build_id = data_release_manifest.build.build_id + certified = country_manifest.certified_data_artifact + description_parts = [ + f"Certification record for the PolicyEngine {country_id} certified " + f"release bundle {bundle_id}.", + f"Model package: {country_manifest.model_package.name}==" + f"{country_manifest.model_package.version}.", + f"Data package: {country_manifest.data_package.name}==" + f"{country_manifest.data_package.version}.", + ] + if build_id is not None: + description_parts.append(f"Data build: {build_id}.") + if certified is not None and certified.sha256 is not None: + description_parts.append( + f"Certified dataset {certified.dataset} sha256 {certified.sha256}." + ) + description_parts.append( + "The bundled TRACE Transparent Research Object pins every artifact " + "by sha256; verify any copy with `policyengine trace-tro-verify`." + ) + + related_identifiers = [] + if certified is not None: + from .trace import _dataset_location_from_uri + + https_location = _dataset_location_from_uri( + certified.uri, + repo_type=country_manifest.data_package.repo_type, + ) + if https_location.startswith("https://"): + related_identifiers.append( + { + "identifier": https_location, + "relation": "isAlternateIdentifier", + } + ) + + metadata: dict[str, Any] = { + "title": ( + f"PolicyEngine {country_id} certified release {bundle_id} " + "(certification record)" + ), + "upload_type": "dataset", + "description": " ".join(description_parts), + "creators": [{"name": "PolicyEngine"}], + "version": country_manifest.policyengine_version, + "keywords": [ + "microsimulation", + "tax-benefit", + "replication", + "TRACE", + "provenance", + ], + "related_identifiers": related_identifiers, + } + if license_id is not None: + metadata["license"] = license_id + return metadata + + +def _bundled_tro_bytes(country_id: str) -> bytes: + from importlib.resources import files + + resource = files("policyengine").joinpath( + "data", "bundle", f"{country_id}.trace.tro.jsonld" + ) + try: + return resource.read_bytes() + except FileNotFoundError as exc: + raise ZenodoDepositError( + f"No bundled TRACE TRO for '{country_id}'. Run " + "scripts/generate_trace_tros.py before mirroring." + ) from exc + + +def _parse_hf_uri(uri: str) -> Optional[tuple[str, str]]: + """``hf://owner/repo/path@rev`` -> ``(repo_id, filename)``.""" + if not uri.startswith("hf://"): + return None + without_scheme = uri.removeprefix("hf://") + if "@" in without_scheme: + without_scheme, _ = without_scheme.rsplit("@", 1) + parts = without_scheme.split("/", 2) + if len(parts) != 3: + return None + return f"{parts[0]}/{parts[1]}", parts[2].rsplit("/", 1)[-1] + + +def _assert_source_repo_public(session: Any, repo_id: str) -> None: + """Refuse to proceed unless the Hub repo is openly readable. + + Checked against both repo types because the registry hosts datasets + in both. This is the licence gate: private *and gated* repos (UK Data + Service microdata) must never have their bytes re-deposited, + regardless of caller flags. + + A repo qualifies as openly readable only when the metadata endpoint + returns HTTP 200 *and* its ``gated`` flag is falsey. Hugging Face + enforces gating at the byte-download layer, so a gated repo's + metadata returns 200 while its files are access-walled — treating a + 200 alone as "public" would let gated bytes through. The ``gated`` + field is ``false`` for open repos and a truthy string + (``"auto"``/``"manual"``) when access is restricted. + + Definitive private/absent signals (401/403/404) refuse. Anything + else (429, 5xx, network error) means visibility could not be + confirmed: rather than risk depositing restricted data, refuse with + a retryable error instead of silently treating it as public. + """ + statuses: list[str] = [] + last_error: Optional[Exception] = None + for repo_type in ("datasets", "models"): + try: + response = session.get( + f"https://huggingface.co/api/{repo_type}/{repo_id}", + timeout=_TIMEOUT_SECONDS, + ) + except Exception as exc: # network failure: cannot confirm visibility + statuses.append(f"{repo_type}:error") + last_error = exc + continue + last_error = None + statuses.append(f"{repo_type}:{response.status_code}") + if response.status_code == 200: + try: + gated = response.json().get("gated", False) + except Exception: + gated = False + if gated: + raise PrivateSourceRepoError( + f"Source repo {repo_id} is gated (gated={gated!r}): its " + "files are access-walled even though metadata is public. " + "Refusing to mirror its dataset bytes; the certification " + "record (manifests and TRO) can still be mirrored without " + "the data." + ) + return + + if any(s.endswith((":401", ":403", ":404")) for s in statuses): + raise PrivateSourceRepoError( + f"Source repo {repo_id} is not publicly readable (private or " + f"not found; saw {statuses}). Refusing to mirror its dataset " + "bytes: redistribution rights cannot be assumed. The " + "certification record (manifests and TRO) can still be mirrored " + "without the data." + ) + raise ZenodoDepositError( + f"Could not verify that source repo {repo_id} is openly readable " + f"(HF returned {statuses}). Refusing to mirror dataset bytes rather " + "than risk depositing restricted data; retry, or omit " + "--include-dataset to mirror just the certification record." + ) from last_error + + +def _default_dataset_fetcher(session: Any) -> Callable[[str], bytes]: + def fetch(url: str) -> bytes: + response = session.get(url, timeout=_TIMEOUT_SECONDS) + if response.status_code != 200: + raise ZenodoDepositError( + f"Failed to download dataset from {url}: HTTP {response.status_code}" + ) + return response.content + + return fetch + + +def _expect(response: Any, expected: tuple[int, ...], step: str) -> dict[str, Any]: + if response.status_code not in expected: + raise ZenodoDepositError( + f"Zenodo {step} failed with HTTP {response.status_code}: " + f"{getattr(response, 'text', '')[:500]}" + ) + try: + return response.json() + except Exception: + return {} + + +def mirror_release_to_zenodo( + country_id: str, + *, + token: Optional[str] = None, + base_url: str = ZENODO_API, + session: Any = None, + publish: bool = False, + include_dataset: bool = False, + dataset_fetcher: Optional[Callable[[str], bytes]] = None, + data_release_manifest: Optional[DataReleaseManifest] = None, + license_id: Optional[str] = None, +) -> ZenodoDeposit: + """Deposit a certified release's certification record on Zenodo. + + Deposits the bundled country manifest, the bundle TRACE TRO, and + the country data release manifest (when available). With + ``include_dataset=True`` the certified dataset bytes are deposited + too — but only when their source Hugging Face repo is publicly + readable; :class:`PrivateSourceRepoError` is raised otherwise. + + The deposit is left as a draft unless ``publish=True``, which mints + the DOI recorded on every returned :class:`PreservationMirror`. + ``base_url=ZENODO_SANDBOX_API`` targets the Zenodo sandbox for + rehearsals. The token comes from the argument or ``ZENODO_TOKEN``. + """ + token = token or os.environ.get("ZENODO_TOKEN") + if not token: + raise ZenodoDepositError( + "No Zenodo token: pass token= or set ZENODO_TOKEN. Create one " + "at https://zenodo.org/account/settings/applications/ with " + "deposit:write and deposit:actions scopes." + ) + if session is None: + import requests + + session = requests.Session() + headers = {"Authorization": f"Bearer {token}"} + + country_manifest = get_release_manifest(country_id) + if data_release_manifest is None: + try: + data_release_manifest = get_data_release_manifest(country_id) + except DataReleaseManifestUnavailableError: + data_release_manifest = None + + files: list[tuple[str, bytes]] = [ + ( + f"{country_id}.bundle_manifest.json", + canonical_json_bytes(country_manifest.model_dump(mode="json")), + ), + (f"{country_id}.trace.tro.jsonld", _bundled_tro_bytes(country_id)), + ] + if data_release_manifest is not None: + files.append( + ( + f"{country_id}.data_release_manifest.json", + canonical_json_bytes(data_release_manifest.model_dump(mode="json")), + ) + ) + + if include_dataset: + certified = country_manifest.certified_data_artifact + if certified is None: + raise ZenodoDepositError( + f"Country manifest for '{country_id}' pins no certified " + "dataset artifact; nothing to include." + ) + parsed = _parse_hf_uri(certified.uri) + if parsed is None: + raise ZenodoDepositError( + f"Cannot parse certified dataset URI {certified.uri!r} as a " + "Hugging Face reference." + ) + repo_id, filename = parsed + _assert_source_repo_public(session, repo_id) + from .trace import _dataset_location_from_uri + + fetch = dataset_fetcher or _default_dataset_fetcher(session) + files.append( + ( + filename, + fetch( + _dataset_location_from_uri( + certified.uri, + repo_type=country_manifest.data_package.repo_type, + ) + ), + ) + ) + + created = _expect( + session.post( + f"{base_url}/deposit/depositions", + json={}, + headers=headers, + timeout=_TIMEOUT_SECONDS, + ), + (200, 201), + "deposition creation", + ) + deposit_id = str(created.get("id")) + links = created.get("links") or {} + bucket = links.get("bucket") + deposit_url = links.get("html") + if not bucket: + raise ZenodoDepositError("Zenodo deposition response has no bucket link.") + + for name, content in files: + _expect( + session.put( + f"{bucket}/{name}", + data=content, + headers=headers, + timeout=_TIMEOUT_SECONDS, + ), + (200, 201), + f"upload of {name}", + ) + + metadata = deposition_metadata( + country_manifest, data_release_manifest, license_id=license_id + ) + _expect( + session.put( + f"{base_url}/deposit/depositions/{deposit_id}", + json={"metadata": metadata}, + headers=headers, + timeout=_TIMEOUT_SECONDS, + ), + (200,), + "metadata update", + ) + + doi: Optional[str] = None + concept_doi: Optional[str] = None + record_url: Optional[str] = None + if publish: + published_payload = _expect( + session.post( + f"{base_url}/deposit/depositions/{deposit_id}/actions/publish", + headers=headers, + timeout=_TIMEOUT_SECONDS, + ), + (200, 202), + "publish", + ) + doi = published_payload.get("doi") + concept_doi = published_payload.get("conceptdoi") + record_url = (published_payload.get("links") or {}).get("record_html") + + deposited_at = datetime.now(timezone.utc).isoformat() + file_base = record_url or deposit_url or f"{base_url}/deposit/{deposit_id}" + mirrors = [ + PreservationMirror( + kind="zenodo", + url=f"{file_base}/files/{name}", + doi=doi, + sha256=hashlib.sha256(content).hexdigest(), + deposited_at=deposited_at, + ) + for name, content in files + ] + + return ZenodoDeposit( + deposit_id=deposit_id, + deposit_url=deposit_url, + doi=doi, + concept_doi=concept_doi, + published=publish, + mirrors=mirrors, + ) diff --git a/tests/test_trace_tro.py b/tests/test_trace_tro.py index 13ec377a..71ae1085 100644 --- a/tests/test_trace_tro.py +++ b/tests/test_trace_tro.py @@ -395,6 +395,28 @@ def test__given_certification__then_fields_are_machine_readable( == country_manifest.certification.data_build_id ) + def test__given_no_preservation_dois__then_performance_omits_the_field( + self, us_bundle_tro + ): + performance = us_bundle_tro["@graph"][0]["trov:hasPerformance"] + assert "pe:preservationDoi" not in performance + + def test__given_preservation_dois__then_performance_records_them(self, monkeypatch): + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + # Synthetic placeholder DOI: exercises propagation mechanics only. + # Real per-release preservation DOIs are recorded after merge as a + # per-release op (see #439); no live/stale DOI is embedded here. + manifest = _us_data_release_manifest().model_copy( + update={"preservation_dois": ["10.5281/zenodo.9999999"]} + ) + tro = build_trace_tro_from_release_bundle( + get_release_manifest("us"), + manifest, + fetch_pypi=_fake_fetch_pypi, + ) + performance = tro["@graph"][0]["trov:hasPerformance"] + assert performance["pe:preservationDoi"] == ["10.5281/zenodo.9999999"] + def test__given_github_actions_env__then_emitted_in_is_ci(self, monkeypatch): monkeypatch.setenv("GITHUB_ACTIONS", "true") monkeypatch.setenv("GITHUB_SERVER_URL", "https://github.com") diff --git a/tests/test_zenodo_mirror.py b/tests/test_zenodo_mirror.py new file mode 100644 index 00000000..e51feac4 --- /dev/null +++ b/tests/test_zenodo_mirror.py @@ -0,0 +1,293 @@ +"""Zenodo mirroring of certified releases. + +Hugging Face hosts the primary artifacts but publishes no preservation +commitment — its DOIs are deletable short URLs. These tests pin the +mirroring flow that deposits the certification record (bundle manifest, +bundle TRO, data release manifest, and optionally the dataset itself) +on Zenodo, which does publish one, and returns ``PreservationMirror`` +entries with real DOIs. + +License safety is load-bearing: a dataset file is only ever deposited +when its source Hugging Face repo is publicly readable. Private repos +(the UK microdata, under UK Data Service licence) are refused even when +the caller asks. +""" + +from __future__ import annotations + +import hashlib +import json + +import pytest + +from policyengine.cli import main +from policyengine.provenance.manifest import ( + get_data_release_manifest, + get_release_manifest, +) +from policyengine.provenance.zenodo import ( + ZENODO_SANDBOX_API, + PrivateSourceRepoError, + ZenodoDepositError, + deposition_metadata, + mirror_release_to_zenodo, +) + +from .test_trace_tro import _us_data_release_manifest + +BUCKET = "https://zenodo.example/api/files/bucket-1" + + +class _FakeResponse: + def __init__(self, status_code: int, payload: dict | None = None): + self.status_code = status_code + self._payload = payload or {} + self.text = json.dumps(self._payload) + + def json(self) -> dict: + return self._payload + + +class _FakeSession: + """Records every request; serves canned Zenodo + HF API responses.""" + + def __init__( + self, + *, + hf_public: bool = True, + hf_gated: bool = False, + hf_status: int = 401, + publish_doi: str = "10.5281/z.1", + ): + self.requests: list[tuple[str, str]] = [] + self.uploads: dict[str, bytes] = {} + self.metadata: dict | None = None + self.published = False + self.hf_public = hf_public + self.hf_gated = hf_gated + self.hf_status = hf_status + self.publish_doi = publish_doi + + def post(self, url, *, json=None, data=None, headers=None, timeout=None): + self.requests.append(("POST", url)) + if url.endswith("/deposit/depositions"): + return _FakeResponse( + 201, + { + "id": 4321, + "links": { + "bucket": BUCKET, + "html": "https://zenodo.example/deposit/4321", + }, + }, + ) + if url.endswith("/actions/publish"): + self.published = True + return _FakeResponse( + 202, + { + "doi": self.publish_doi, + "conceptdoi": "10.5281/z.concept", + "links": {"record_html": "https://zenodo.example/records/4321"}, + }, + ) + return _FakeResponse(404) + + def put(self, url, *, data=None, json=None, headers=None, timeout=None): + self.requests.append(("PUT", url)) + if url.startswith(BUCKET): + name = url.removeprefix(BUCKET + "/") + self.uploads[name] = data + return _FakeResponse(201, {"key": name}) + if "/deposit/depositions/" in url: + self.metadata = json["metadata"] + return _FakeResponse(200, {}) + return _FakeResponse(404) + + def get(self, url, *, headers=None, timeout=None): + self.requests.append(("GET", url)) + if "huggingface.co/api/" in url: + if not self.hf_public: + return _FakeResponse(self.hf_status) + return _FakeResponse(200, {"gated": "auto"} if self.hf_gated else {}) + if url.startswith("https://huggingface.co/datasets/"): + return _FakeResponse(200) + return _FakeResponse(404) + + +@pytest.fixture(autouse=True) +def _clear_manifest_caches(): + yield + get_release_manifest.cache_clear() + get_data_release_manifest.cache_clear() + + +@pytest.fixture +def fake_session(): + return _FakeSession() + + +def _mirror(session, **overrides): + kwargs = dict( + token="test-token", + session=session, + data_release_manifest=_us_data_release_manifest(), + ) + kwargs.update(overrides) + return mirror_release_to_zenodo("us", **kwargs) + + +class TestDeposit: + def test__given_release__then_certification_record_is_deposited(self, fake_session): + deposit = _mirror(fake_session) + assert set(fake_session.uploads) == { + "us.bundle_manifest.json", + "us.trace.tro.jsonld", + "us.data_release_manifest.json", + } + assert deposit.deposit_id == "4321" + # The TRO uploaded is the shipped artifact, byte-for-byte. + from importlib.resources import files + + shipped = ( + files("policyengine") + .joinpath("data", "bundle", "us.trace.tro.jsonld") + .read_bytes() + ) + assert fake_session.uploads["us.trace.tro.jsonld"] == shipped + + def test__given_default__then_dataset_bytes_are_not_deposited(self, fake_session): + _mirror(fake_session) + assert not any(name.endswith(".h5") for name in fake_session.uploads) + + def test__given_draft__then_mirrors_have_no_doi_and_not_published( + self, fake_session + ): + deposit = _mirror(fake_session, publish=False) + assert not fake_session.published + assert deposit.doi is None + assert all(mirror.doi is None for mirror in deposit.mirrors) + assert all(mirror.kind == "zenodo" for mirror in deposit.mirrors) + + def test__given_publish__then_doi_recorded_on_every_mirror(self, fake_session): + deposit = _mirror(fake_session, publish=True) + assert fake_session.published + assert deposit.doi == "10.5281/z.1" + assert deposit.concept_doi == "10.5281/z.concept" + assert all(mirror.doi == "10.5281/z.1" for mirror in deposit.mirrors) + + def test__given_uploads__then_mirror_sha256_matches_uploaded_bytes( + self, fake_session + ): + deposit = _mirror(fake_session) + by_name = {mirror.url.rsplit("/", 1)[-1]: mirror for mirror in deposit.mirrors} + for name, payload in fake_session.uploads.items(): + assert by_name[name].sha256 == hashlib.sha256(payload).hexdigest() + + def test__given_metadata__then_describes_certified_release(self, fake_session): + _mirror(fake_session) + metadata = fake_session.metadata + assert metadata["upload_type"] == "dataset" + assert "us" in metadata["title"] + assert metadata["creators"] == [{"name": "PolicyEngine"}] + assert any( + "huggingface.co" in related["identifier"] + for related in metadata["related_identifiers"] + ) + + def test__given_sandbox__then_requests_hit_sandbox_api(self): + session = _FakeSession() + _mirror(session, base_url=ZENODO_SANDBOX_API) + deposition_posts = [url for method, url in session.requests if method == "POST"] + assert all(url.startswith(ZENODO_SANDBOX_API) for url in deposition_posts) + + def test__given_no_token__then_raises(self, fake_session, monkeypatch): + monkeypatch.delenv("ZENODO_TOKEN", raising=False) + with pytest.raises(ZenodoDepositError, match="token"): + mirror_release_to_zenodo( + "us", + session=fake_session, + data_release_manifest=_us_data_release_manifest(), + ) + + def test__given_deposit_api_failure__then_raises_with_status(self): + class _FailingSession(_FakeSession): + def post(self, url, **kwargs): + return _FakeResponse(500, {"message": "boom"}) + + with pytest.raises(ZenodoDepositError, match="500"): + _mirror(_FailingSession()) + + +class TestDatasetInclusion: + def test__given_private_source_repo__then_refuses_even_when_asked(self): + session = _FakeSession(hf_public=False) + with pytest.raises(PrivateSourceRepoError, match="private"): + _mirror(session, include_dataset=True) + + def test__given_gated_source_repo__then_refuses_even_when_metadata_is_200(self): + # HF returns 200 for gated-repo metadata but access-walls the bytes; + # a 200 alone must not be treated as public. + session = _FakeSession(hf_gated=True) + with pytest.raises(PrivateSourceRepoError, match="gated"): + _mirror(session, include_dataset=True) + + def test__given_transient_hf_error__then_refuses_as_unverifiable_not_private(self): + # A 429/5xx means visibility is unknown; refuse with a retryable + # ZenodoDepositError rather than silently treating it as public. + session = _FakeSession(hf_public=False, hf_status=429) + with pytest.raises(ZenodoDepositError, match="verify"): + _mirror(session, include_dataset=True) + + def test__given_public_source_repo__then_dataset_bytes_deposited( + self, fake_session + ): + payload = b"dataset bytes" + deposit = _mirror( + fake_session, + include_dataset=True, + dataset_fetcher=lambda url: payload, + ) + assert fake_session.uploads["populace_us_2024.h5"] == payload + names = {mirror.url.rsplit("/", 1)[-1] for mirror in deposit.mirrors} + assert "populace_us_2024.h5" in names + + +class TestDepositionMetadata: + def test__given_manifests__then_version_and_description_pinned(self): + metadata = deposition_metadata( + get_release_manifest("us"), _us_data_release_manifest() + ) + assert metadata["version"] == get_release_manifest("us").policyengine_version + description = metadata["description"] + assert "populace-us-2024" in description + + +class TestCLI: + def test__given_cli__then_prints_mirrors_json(self, monkeypatch, capsys): + from policyengine.provenance import zenodo as zenodo_module + from policyengine.provenance.manifest import PreservationMirror + + def fake_mirror(country_id, **kwargs): + assert country_id == "us" + return zenodo_module.ZenodoDeposit( + deposit_id="1", + deposit_url="https://zenodo.example/deposit/1", + doi=None, + concept_doi=None, + published=False, + mirrors=[ + PreservationMirror( + kind="zenodo", + url="https://zenodo.example/deposit/1/files/us.trace.tro.jsonld", + ) + ], + ) + + monkeypatch.setattr(zenodo_module, "mirror_release_to_zenodo", fake_mirror) + monkeypatch.setenv("ZENODO_TOKEN", "t") + exit_code = main(["zenodo-mirror", "us"]) + captured = capsys.readouterr() + assert exit_code == 0 + payload = json.loads(captured.out) + assert payload["mirrors"][0]["kind"] == "zenodo" From 53127d6d8f4bce8ca82a0ccb900cd5cc9072eb75 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 7 Jul 2026 08:47:56 -0400 Subject: [PATCH 2/2] Harden Zenodo mirroring per adversarial review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix six issues found reviewing the deposit flow, each with a regression test (no live network; all Zenodo/HF interaction mocked). MAJOR: - Partial failure no longer orphans an unfindable draft. Steps after the deposition is created (upload, metadata, publish — e.g. a deposit:actions-scope 403) are wrapped so any ZenodoDepositError re-raises with the deposit id and URL and a "delete or resume" note. - _assert_source_repo_public no longer fails open. It now requires positive confirmation: a 200 with a JSON object reporting private=False and gated falsey. A non-JSON/non-object 200 (e.g. a Cloudflare page) falls through to the unverifiable refusal, and private=true is checked explicitly (an authenticated read of a private repo returns 200 {"private": true}). MINOR/NIT: - CLI catches ValueError so an unknown country id exits cleanly instead of dumping a traceback (get_release_manifest raises ValueError). - --include-dataset verifies fetched bytes against the certified sha256 and refuses on mismatch, naming both hashes, before depositing. - Draft mirror URLs are no longer fabricated /files/ paths off the edit page (they would 404). Drafts carry the deposit URL as a provisional handle; published records keep per-file record URLs. Docstring updated. - _bundled_tro_bytes catches KeyError as well as FileNotFoundError, so a zipimport loader's missing-resource error maps to ZenodoDepositError. Co-Authored-By: Claude Fable 5 --- src/policyengine/cli.py | 8 +- src/policyengine/provenance/zenodo.py | 183 +++++++++++++++++--------- tests/test_zenodo_mirror.py | 171 ++++++++++++++++++++++-- 3 files changed, 290 insertions(+), 72 deletions(-) diff --git a/src/policyengine/cli.py b/src/policyengine/cli.py index d412528a..1a754573 100644 --- a/src/policyengine/cli.py +++ b/src/policyengine/cli.py @@ -487,7 +487,13 @@ def _zenodo_mirror( include_dataset=include_dataset, license_id=license_id, ) - except (zenodo.ZenodoDepositError, zenodo.PrivateSourceRepoError) as exc: + except ( + zenodo.ZenodoDepositError, + zenodo.PrivateSourceRepoError, + ValueError, + ) as exc: + # ValueError covers an unknown country id (get_release_manifest + # raises it) so the CLI exits cleanly instead of dumping a traceback. print(f"error: {exc}", file=sys.stderr) return 1 print(json.dumps(deposit.to_json(), indent=2, sort_keys=True)) diff --git a/src/policyengine/provenance/zenodo.py b/src/policyengine/provenance/zenodo.py index 94e1bca7..97d7801f 100644 --- a/src/policyengine/provenance/zenodo.py +++ b/src/policyengine/provenance/zenodo.py @@ -163,7 +163,9 @@ def _bundled_tro_bytes(country_id: str) -> bytes: ) try: return resource.read_bytes() - except FileNotFoundError as exc: + except (FileNotFoundError, KeyError) as exc: + # FileNotFoundError from a filesystem loader; KeyError from a + # zipimport loader when the resource is absent from the archive. raise ZenodoDepositError( f"No bundled TRACE TRO for '{country_id}'. Run " "scripts/generate_trace_tros.py before mirroring." @@ -191,18 +193,28 @@ def _assert_source_repo_public(session: Any, repo_id: str) -> None: Service microdata) must never have their bytes re-deposited, regardless of caller flags. - A repo qualifies as openly readable only when the metadata endpoint - returns HTTP 200 *and* its ``gated`` flag is falsey. Hugging Face - enforces gating at the byte-download layer, so a gated repo's - metadata returns 200 while its files are access-walled — treating a - 200 alone as "public" would let gated bytes through. The ``gated`` - field is ``false`` for open repos and a truthy string - (``"auto"``/``"manual"``) when access is restricted. + This gate uses *positive confirmation only*: it proceeds solely when + the metadata endpoint returns HTTP 200 with a JSON object that + explicitly reports the repo as open — ``private`` is ``False`` and + ``gated`` is falsey. Any weaker signal fails closed. + + - A 200 whose body is not a JSON object (e.g. a Cloudflare challenge + or interstitial HTML page) is *not* confirmation and must not be + read as "public"; it falls through to the unverifiable refusal. + - ``gated`` is ``false`` for open repos and a truthy string + (``"auto"``/``"manual"``) when access is restricted. Hugging Face + enforces gating at the byte-download layer, so a gated repo's + metadata returns 200 while its files are access-walled. + - ``private`` is ``true`` for a private repo. An authenticated + session (one carrying a token) reads a private repo's metadata as + 200 ``{"private": true}``; checking ``gated`` alone would let those + bytes through, so ``private`` is checked explicitly. Definitive private/absent signals (401/403/404) refuse. Anything - else (429, 5xx, network error) means visibility could not be - confirmed: rather than risk depositing restricted data, refuse with - a retryable error instead of silently treating it as public. + else (429, 5xx, non-JSON 200, network error) means visibility could + not be confirmed: rather than risk depositing restricted data, + refuse with a retryable error instead of silently treating it as + public. """ statuses: list[str] = [] last_error: Optional[Exception] = None @@ -220,9 +232,17 @@ def _assert_source_repo_public(session: Any, repo_id: str) -> None: statuses.append(f"{repo_type}:{response.status_code}") if response.status_code == 200: try: - gated = response.json().get("gated", False) + payload = response.json() except Exception: - gated = False + # A 200 with a non-JSON body is not positive confirmation + # of a public repo. Record it as unverifiable and fall + # through rather than treating it as open. + statuses[-1] = f"{repo_type}:200-non-json" + continue + if not isinstance(payload, dict): + statuses[-1] = f"{repo_type}:200-non-object" + continue + gated = payload.get("gated", False) if gated: raise PrivateSourceRepoError( f"Source repo {repo_id} is gated (gated={gated!r}): its " @@ -231,6 +251,14 @@ def _assert_source_repo_public(session: Any, repo_id: str) -> None: "record (manifests and TRO) can still be mirrored without " "the data." ) + if payload.get("private", False): + raise PrivateSourceRepoError( + f"Source repo {repo_id} is private (private=true): an " + "authenticated session can read it, but redistribution " + "rights cannot be assumed. Refusing to mirror its dataset " + "bytes; the certification record (manifests and TRO) can " + "still be mirrored without the data." + ) return if any(s.endswith((":401", ":403", ":404")) for s in statuses): @@ -294,9 +322,16 @@ def mirror_release_to_zenodo( readable; :class:`PrivateSourceRepoError` is raised otherwise. The deposit is left as a draft unless ``publish=True``, which mints - the DOI recorded on every returned :class:`PreservationMirror`. + the DOI recorded on every returned :class:`PreservationMirror` and + gives each mirror a stable, dereferenceable per-file record URL. A + draft has no published per-file URL yet, so its mirrors carry the + draft deposit URL as a provisional handle until it is published. ``base_url=ZENODO_SANDBOX_API`` targets the Zenodo sandbox for rehearsals. The token comes from the argument or ``ZENODO_TOKEN``. + + If a step after the deposition is created fails, the raised + :class:`ZenodoDepositError` carries the deposit id and URL so the + orphaned draft can be deleted or resumed rather than lost. """ token = token or os.environ.get("ZENODO_TOKEN") if not token: @@ -351,17 +386,22 @@ def mirror_release_to_zenodo( from .trace import _dataset_location_from_uri fetch = dataset_fetcher or _default_dataset_fetcher(session) - files.append( - ( - filename, - fetch( - _dataset_location_from_uri( - certified.uri, - repo_type=country_manifest.data_package.repo_type, - ) - ), + dataset_bytes = fetch( + _dataset_location_from_uri( + certified.uri, + repo_type=country_manifest.data_package.repo_type, ) ) + if certified.sha256 is not None: + fetched_sha256 = hashlib.sha256(dataset_bytes).hexdigest() + if fetched_sha256 != certified.sha256: + raise ZenodoDepositError( + f"Fetched dataset bytes for '{filename}' hash " + f"{fetched_sha256}, but the certified artifact pins " + f"{certified.sha256}. Refusing to deposit dataset bytes " + "that do not match the certified release." + ) + files.append((filename, dataset_bytes)) created = _expect( session.post( @@ -377,58 +417,77 @@ def mirror_release_to_zenodo( links = created.get("links") or {} bucket = links.get("bucket") deposit_url = links.get("html") - if not bucket: - raise ZenodoDepositError("Zenodo deposition response has no bucket link.") - - for name, content in files: - _expect( - session.put( - f"{bucket}/{name}", - data=content, - headers=headers, - timeout=_TIMEOUT_SECONDS, - ), - (200, 201), - f"upload of {name}", - ) - - metadata = deposition_metadata( - country_manifest, data_release_manifest, license_id=license_id - ) - _expect( - session.put( - f"{base_url}/deposit/depositions/{deposit_id}", - json={"metadata": metadata}, - headers=headers, - timeout=_TIMEOUT_SECONDS, - ), - (200,), - "metadata update", - ) + deposit_ref = deposit_url or f"{base_url}/deposit/{deposit_id}" doi: Optional[str] = None concept_doi: Optional[str] = None record_url: Optional[str] = None - if publish: - published_payload = _expect( - session.post( - f"{base_url}/deposit/depositions/{deposit_id}/actions/publish", + # The deposition draft now exists on Zenodo. Any failure from here on + # (missing bucket, upload, metadata, publish — e.g. the common + # deposit:actions-scope 403 on publish) would otherwise raise without + # the deposit handle, orphaning a staged draft the caller cannot find; + # retries would pile up more. Re-raise with the id and URL plus a + # resolution note instead. + try: + if not bucket: + raise ZenodoDepositError("Zenodo deposition response has no bucket link.") + for name, content in files: + _expect( + session.put( + f"{bucket}/{name}", + data=content, + headers=headers, + timeout=_TIMEOUT_SECONDS, + ), + (200, 201), + f"upload of {name}", + ) + + metadata = deposition_metadata( + country_manifest, data_release_manifest, license_id=license_id + ) + _expect( + session.put( + f"{base_url}/deposit/depositions/{deposit_id}", + json={"metadata": metadata}, headers=headers, timeout=_TIMEOUT_SECONDS, ), - (200, 202), - "publish", + (200,), + "metadata update", ) - doi = published_payload.get("doi") - concept_doi = published_payload.get("conceptdoi") - record_url = (published_payload.get("links") or {}).get("record_html") + + if publish: + published_payload = _expect( + session.post( + f"{base_url}/deposit/depositions/{deposit_id}/actions/publish", + headers=headers, + timeout=_TIMEOUT_SECONDS, + ), + (200, 202), + "publish", + ) + doi = published_payload.get("doi") + concept_doi = published_payload.get("conceptdoi") + record_url = (published_payload.get("links") or {}).get("record_html") + except ZenodoDepositError as exc: + raise ZenodoDepositError( + f"{exc} A draft deposit was already created on Zenodo " + f"(deposit_id={deposit_id}, deposit_url={deposit_ref}); delete or " + "resume it manually before retrying, to avoid orphaned drafts." + ) from exc deposited_at = datetime.now(timezone.utc).isoformat() - file_base = record_url or deposit_url or f"{base_url}/deposit/{deposit_id}" mirrors = [ PreservationMirror( kind="zenodo", - url=f"{file_base}/files/{name}", + # A published record exposes stable, dereferenceable per-file + # download URLs. A draft has none yet, so every mirror points at + # the draft deposit itself as a provisional handle; the per-file + # URLs materialize on publish. + url=( + f"{record_url}/files/{name}" if record_url is not None else deposit_ref + ), doi=doi, sha256=hashlib.sha256(content).hexdigest(), deposited_at=deposited_at, diff --git a/tests/test_zenodo_mirror.py b/tests/test_zenodo_mirror.py index e51feac4..a894ee7d 100644 --- a/tests/test_zenodo_mirror.py +++ b/tests/test_zenodo_mirror.py @@ -39,12 +39,25 @@ class _FakeResponse: - def __init__(self, status_code: int, payload: dict | None = None): + def __init__( + self, + status_code: int, + payload: dict | None = None, + *, + raise_json: bool = False, + ): self.status_code = status_code self._payload = payload or {} - self.text = json.dumps(self._payload) + self._raise_json = raise_json + # A non-JSON body (e.g. a Cloudflare challenge page served with a + # 200) has text but no decodable JSON object. + self.text = ( + "challenge" if raise_json else json.dumps(self._payload) + ) def json(self) -> dict: + if self._raise_json: + raise ValueError("No JSON object could be decoded") return self._payload @@ -56,6 +69,8 @@ def __init__( *, hf_public: bool = True, hf_gated: bool = False, + hf_private: bool = False, + hf_nonjson: bool = False, hf_status: int = 401, publish_doi: str = "10.5281/z.1", ): @@ -65,6 +80,8 @@ def __init__( self.published = False self.hf_public = hf_public self.hf_gated = hf_gated + self.hf_private = hf_private + self.hf_nonjson = hf_nonjson self.hf_status = hf_status self.publish_doi = publish_doi @@ -109,7 +126,14 @@ def get(self, url, *, headers=None, timeout=None): if "huggingface.co/api/" in url: if not self.hf_public: return _FakeResponse(self.hf_status) - return _FakeResponse(200, {"gated": "auto"} if self.hf_gated else {}) + if self.hf_nonjson: + return _FakeResponse(200, raise_json=True) + payload: dict = {} + if self.hf_gated: + payload["gated"] = "auto" + if self.hf_private: + payload["private"] = True + return _FakeResponse(200, payload) if url.startswith("https://huggingface.co/datasets/"): return _FakeResponse(200) return _FakeResponse(404) @@ -180,9 +204,80 @@ def test__given_uploads__then_mirror_sha256_matches_uploaded_bytes( self, fake_session ): deposit = _mirror(fake_session) - by_name = {mirror.url.rsplit("/", 1)[-1]: mirror for mirror in deposit.mirrors} - for name, payload in fake_session.uploads.items(): - assert by_name[name].sha256 == hashlib.sha256(payload).hexdigest() + uploaded_hashes = { + hashlib.sha256(payload).hexdigest() + for payload in fake_session.uploads.values() + } + assert {mirror.sha256 for mirror in deposit.mirrors} == uploaded_hashes + + def test__given_draft__then_mirror_urls_are_provisional_deposit_handle( + self, fake_session + ): + # A draft has no published per-file URL, so every mirror points at + # the draft deposit itself as a provisional handle rather than a + # fabricated /files/ path that would 404. + deposit = _mirror(fake_session, publish=False) + assert all( + mirror.url == "https://zenodo.example/deposit/4321" + for mirror in deposit.mirrors + ) + assert all("/files/" not in mirror.url for mirror in deposit.mirrors) + + def test__given_publish__then_mirror_urls_are_per_file_record_urls( + self, fake_session + ): + # Publishing yields stable, dereferenceable per-file record URLs. + deposit = _mirror(fake_session, publish=True) + assert all( + mirror.url.startswith("https://zenodo.example/records/4321/files/") + for mirror in deposit.mirrors + ) + names = {mirror.url.rsplit("/", 1)[-1] for mirror in deposit.mirrors} + assert "us.trace.tro.jsonld" in names + + def test__given_post_creation_failure__then_error_names_the_orphaned_draft(self): + # Once the deposition is created, a later failure (here a metadata + # PUT 400) must re-raise with the deposit id and URL so the staged + # draft can be found and deleted rather than silently orphaned. + class _MetadataFailSession(_FakeSession): + def put(self, url, *, data=None, json=None, headers=None, timeout=None): + self.requests.append(("PUT", url)) + if url.startswith(BUCKET): + name = url.removeprefix(BUCKET + "/") + self.uploads[name] = data + return _FakeResponse(201, {"key": name}) + if "/deposit/depositions/" in url: + return _FakeResponse(400, {"message": "bad metadata"}) + return _FakeResponse(404) + + with pytest.raises(ZenodoDepositError) as exc_info: + _mirror(_MetadataFailSession()) + message = str(exc_info.value) + assert "4321" in message + assert "https://zenodo.example/deposit/4321" in message + assert "delete or resume" in message + + def test__given_zipimport_keyerror__then_raises_zenodo_error(self, monkeypatch): + # A zipimport loader raises KeyError (not FileNotFoundError) when a + # resource is absent from the archive; it must map to a + # ZenodoDepositError, not escape as a raw KeyError. + import importlib.resources as importlib_resources + + from policyengine.provenance import zenodo as zenodo_module + + class _MissingResource: + def read_bytes(self): + raise KeyError("resource not in zip archive") + + class _Traversable: + def joinpath(self, *parts): + return _MissingResource() + + monkeypatch.setattr( + importlib_resources, "files", lambda package: _Traversable() + ) + with pytest.raises(ZenodoDepositError, match="bundled TRACE TRO"): + zenodo_module._bundled_tro_bytes("us") def test__given_metadata__then_describes_certified_release(self, fake_session): _mirror(fake_session) @@ -239,18 +334,65 @@ def test__given_transient_hf_error__then_refuses_as_unverifiable_not_private(sel with pytest.raises(ZenodoDepositError, match="verify"): _mirror(session, include_dataset=True) + def test__given_non_json_200__then_refuses_as_unverifiable(self): + # A 200 with a non-JSON body (e.g. a Cloudflare challenge page) is + # not positive confirmation of a public repo; refuse rather than + # fail open by defaulting gated to False. + session = _FakeSession(hf_nonjson=True) + with pytest.raises(ZenodoDepositError, match="verify"): + _mirror(session, include_dataset=True) + + def test__given_private_flag_in_200_metadata__then_refuses(self): + # An authenticated session reads a private repo's metadata as + # 200 {"private": true, "gated": false}; the gate must check + # private, not only gated. + session = _FakeSession(hf_private=True) + with pytest.raises(PrivateSourceRepoError, match="private"): + _mirror(session, include_dataset=True) + def test__given_public_source_repo__then_dataset_bytes_deposited( - self, fake_session + self, fake_session, monkeypatch ): + from policyengine.provenance import zenodo as zenodo_module + payload = b"dataset bytes" + # The gate now verifies fetched bytes against the certified sha256, + # so pin this payload's hash on the manifest for the happy path. + base = get_release_manifest("us") + certified = base.certified_data_artifact.model_copy( + update={"sha256": hashlib.sha256(payload).hexdigest()} + ) + manifest = base.model_copy(update={"certified_data_artifact": certified}) + monkeypatch.setattr( + zenodo_module, "get_release_manifest", lambda country_id: manifest + ) deposit = _mirror( fake_session, include_dataset=True, dataset_fetcher=lambda url: payload, ) assert fake_session.uploads["populace_us_2024.h5"] == payload - names = {mirror.url.rsplit("/", 1)[-1] for mirror in deposit.mirrors} - assert "populace_us_2024.h5" in names + assert hashlib.sha256(payload).hexdigest() in { + mirror.sha256 for mirror in deposit.mirrors + } + + def test__given_tampered_dataset_bytes__then_refuses_on_sha256_mismatch( + self, fake_session + ): + # The public gate passes, but the fetched bytes do not match the + # certified sha256, so the deposit is refused before any upload and + # the error names both hashes. + certified_sha256 = get_release_manifest("us").certified_data_artifact.sha256 + with pytest.raises(ZenodoDepositError) as exc_info: + _mirror( + fake_session, + include_dataset=True, + dataset_fetcher=lambda url: b"tampered", + ) + message = str(exc_info.value) + assert hashlib.sha256(b"tampered").hexdigest() in message + assert certified_sha256 in message + assert not fake_session.uploads class TestDepositionMetadata: @@ -291,3 +433,14 @@ def fake_mirror(country_id, **kwargs): assert exit_code == 0 payload = json.loads(captured.out) assert payload["mirrors"][0]["kind"] == "zenodo" + + def test__given_unknown_country__then_clean_error_exit(self, monkeypatch, capsys): + # get_release_manifest raises ValueError for an unknown country id; + # the CLI must catch it and exit 1 with a clean message, not a + # traceback. Reaches the manifest lookup (no live network) with a + # token present. + monkeypatch.setenv("ZENODO_TOKEN", "t") + exit_code = main(["zenodo-mirror", "usa"]) + captured = capsys.readouterr() + assert exit_code == 1 + assert "usa" in captured.err