diff --git a/changelog.d/us-local-area-dataset-overlay.added.md b/changelog.d/us-local-area-dataset-overlay.added.md new file mode 100644 index 00000000..cecc50e2 --- /dev/null +++ b/changelog.d/us-local-area-dataset-overlay.added.md @@ -0,0 +1 @@ +Add named, non-default access to the US Populace local-area dataset (`populace_us_2024_acs_local`) through a `dataset_overlays` bundle-manifest section that resolves by name to its own pinned, sha-verified release revision, never changes default resolution, and survives re-certification. Includes a Build M US Populace certification runbook. diff --git a/docs/engineering/runbooks/build-m-us-populace-certification.md b/docs/engineering/runbooks/build-m-us-populace-certification.md new file mode 100644 index 00000000..dd0d628f --- /dev/null +++ b/docs/engineering/runbooks/build-m-us-populace-certification.md @@ -0,0 +1,145 @@ +# Build M US Populace certification runbook + +Fill-in-the-id runbook for certifying the next US Populace default +(`sparse-rmloss100` lineage: Build I → J → **M**) into the +`policyengine.py` bundle. It replays the exact steps that landed +[#470](https://github.com/PolicyEngine/policyengine.py/pull/470) (the Build J +certification), parameterized for the Build M release id. Read the +[data certification](../skills/data-certification.md) skill first for the +validation semantics; this file is the concrete checklist. + +## When to use + +A new US Populace `sparse-rmloss100` release has been published to +`policyengine/populace-us` and is ready to become the certified default. You +have the release id, the release manifest is reachable, and the model version +it was built with is known. + +## Prerequisites + +- A clean worktree branched from current `origin/main` + (`git fetch origin && git checkout -b certify-us-buildm origin/main`). +- Network access: certification fetches the release manifest and PyPI wheel + metadata and runs reachability `HEAD` checks against Hugging Face. +- `HUGGING_FACE_TOKEN` (or `HF_TOKEN`) exported — required to regenerate the + UK TRO in the `--include-tros` step and to run the UK data-release fetch in + the test suite. The US populace repo is public. + +## Fill in these three values + +``` +BUILD_M_RELEASE_ID = populace-us-2024-buildm-sparse-rmloss100--Z +MODEL_VERSION = 1.764.6 # policyengine-us Build M was built with +CURRENT_RELEASE_ID = populace-us-2024-buildj-sparse-rmloss100-75d5add-20260710T094201Z +``` + +`CURRENT_RELEASE_ID` is the outgoing default (Build J) — the string you are +replacing in the pinned test constants. `MODEL_VERSION` stays `1.764.6` unless +Build M was built against a newer `policyengine-us`; if it was, see step 4. + +## Step 1 — certify the release + +```bash +python scripts/certify_data_release.py \ + --country us \ + --data-producer populace \ + --model-version "$MODEL_VERSION" \ + --manifest-uri "hf://dataset/policyengine/populace-us@$BUILD_M_RELEASE_ID/releases/$BUILD_M_RELEASE_ID/release_manifest.json" +``` + +This one command: + +- rewrites `data_releases.us` in `src/policyengine/data/bundle/manifest.json` + from the Build M release manifest (default dataset, per-artifact + repo/revision/sha256 pins, certified artifact, certification block); +- runs `generate(check=False)`, which re-normalizes `manifest.json` and + updates `pyproject.toml` **only if the model pins moved**; +- writes the changelog fragment + `changelog.d/certify-us-$BUILD_M_RELEASE_ID.changed.md`. + +## Step 2 — regenerate the US TRO sidecar + +The certify step does not touch TRO sidecars. Rebind them so +`src/policyengine/data/bundle/us.trace.tro.jsonld` records the new +`manifest.json` sha256 (the `bundle_manifest` artifact hash is asserted by +`tests/test_certify_data_release.py::TestVendoredSidecarBinding`): + +```bash +HUGGING_FACE_TOKEN="$HUGGING_FACE_TOKEN" python scripts/bundle.py generate --include-tros +``` + +Only `us.trace.tro.jsonld` should change (UK regenerates identically because +only US was certified). If UK cannot be reached, the run writes a *limited* UK +TRO — do not commit a degraded `uk.trace.tro.jsonld`; `git checkout` it and +rerun with a valid token. + +## Step 3 — update the pinned test constants + +Three files hard-code the certified release id. Replace `CURRENT_RELEASE_ID` +with `BUILD_M_RELEASE_ID` in each (this is the whole of #470's test diff): + +- `tests/test_release_manifests.py` — `US_DATA_RELEASE_ID`. +- `tests/test_models.py` — the `us_latest.default_dataset_uri` `@` + assertion. +- `tests/test_us_regions.py` — the national `dataset_path` `@` assertion. + +## Step 4 — only if the model version changed + +Build J → M on the same `policyengine-us` needs nothing here (this is the +common case; #470 left `pyproject.toml` and the snapshots untouched). If Build +M was built against a newer `policyengine-us`: + +- `pyproject.toml` model pins are already rewritten by step 1's `generate`; + commit them. +- Update `US_MODEL_VERSION` and `US_BUILT_WITH_MODEL_VERSION` in + `tests/test_release_manifests.py`. +- Refresh the household snapshots: + `PE_UPDATE_SNAPSHOTS=1 pytest tests/test_household_calculator_snapshot.py` + and commit `tests/fixtures/household_calculator_snapshots/`. + +## Step 5 — verify the local-area overlay survived + +Certification rewrites only `data_releases.us`, so the non-default +`dataset_overlays.us.populace_us_2024_acs_local` entry (see +[Non-default dataset overlays](../../release-bundles.md#non-default-dataset-overlays)) +must still be present and resolvable. No manual re-add is needed — confirm it: + +```bash +python -c "import json; b=json.load(open('src/policyengine/data/bundle/manifest.json')); assert 'populace_us_2024_acs_local' in b['dataset_overlays']['us'], 'overlay lost'; print('overlay preserved')" +pytest tests/test_release_manifests.py -k "local_area or DatasetOverlays" -q +``` + +## Step 6 — check, format, lint, test + +```bash +python scripts/bundle.py check # must exit 0 +make format +make lint +make test # needs HUGGING_FACE_TOKEN for UK +``` + +## Step 7 — commit exactly these files + +Matches #470's file set (add `pyproject.toml` and the snapshot dir only when +step 4 applied): + +- `src/policyengine/data/bundle/manifest.json` +- `src/policyengine/data/bundle/us.trace.tro.jsonld` +- `changelog.d/certify-us-$BUILD_M_RELEASE_ID.changed.md` +- `tests/test_release_manifests.py` +- `tests/test_models.py` +- `tests/test_us_regions.py` + +## Step 8 — open the PR + +Follow [github-prs](../skills/github-prs.md): open/find the issue, put +`Fixes #ISSUE` first, push to the canonical repo, and open a **draft** PR. The +certify step already wrote the changelog fragment, so the changelog check +passes. + +## Verified against #470 + +Every step above is checked against the merged Build J certification (#470), +whose diff was exactly: `manifest.json`, `us.trace.tro.jsonld`, the three test +constants, and the auto-written changelog fragment — with `pyproject.toml` and +the snapshots untouched because `policyengine-us` stayed `1.764.6`. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 4e3f24b5..8c61e5ea 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -88,6 +88,13 @@ refused: a new data build or a published compatibility claim is required. Warnings (recorded, not blocking): artifacts without sha256, and the publisher-claim basis above. +## Release runbooks + +Concrete, fill-in-the-id runbooks that replay a specific certification live +under `docs/engineering/runbooks/`. See +`runbooks/build-m-us-populace-certification.md` for the next US Populace +`sparse-rmloss100` default. + ## Legacy paths Do not hand-edit bundle data releases for normal updates. Countries whose diff --git a/docs/microsim.md b/docs/microsim.md index f6927d93..1811459b 100644 --- a/docs/microsim.md +++ b/docs/microsim.md @@ -48,6 +48,50 @@ List datasets already known to the country: pe.us.load_datasets() # or pe.uk.load_datasets() ``` +### US local-area dataset + +Alongside the certified national default, the bundle registers a **non-default** +US dataset for finer geographic work: `populace_us_2024_acs_local`. It is a +Populace US 2024 build of roughly **1.6 million households** on an **ACS 2024 +multispine**, with each household **PUMA-assigned** to a 119th-Congress +congressional district, county, and state, and calibrated to **state +administrative totals and state and congressional-district population**. Its +release gate summary records **four reviewed limitations**, so read that gate +summary before relying on it. It ships in its own immutable release and is never +selected implicitly — you load it by name. + +Two-line load: + +```python +import policyengine as pe +sim = pe.us.managed_microsimulation(dataset="populace_us_2024_acs_local") +``` + +Or materialize it as a `PolicyEngineUSDataset` for `Simulation`: + +```python +datasets = pe.us.ensure_datasets(datasets=["populace_us_2024_acs_local"], years=[2024]) +dataset = datasets["populace_us_2024_acs_local_2024"] +``` + +Because this file carries PUMA-assigned district, county, and state identifiers +calibrated to state and congressional-district population, **state and +congressional-district breakdowns should filter this dataset** rather than the +national default. Filter it with the same `state_fips` / +`congressional_district_geoid` row filters used elsewhere (see +[Regional analysis](regions.md)): + +```python +from policyengine.core import Simulation +from policyengine.core.scoping_strategy import RowFilterStrategy + +ca = Simulation( + dataset=dataset, + tax_benefit_model_version=pe.us.model, + scoping_strategy=RowFilterStrategy(variable_name="state_fips", variable_value=6), +) +``` + ### UK private data and raw h5 access UK population data uses licensed Family Resources Survey inputs. The default diff --git a/docs/release-bundles.md b/docs/release-bundles.md index bccab440..17841211 100644 --- a/docs/release-bundles.md +++ b/docs/release-bundles.md @@ -141,6 +141,35 @@ default dataset. State and congressional-district regions are runtime row filters over that national dataset, so derived `states/*.h5` or `districts/*.h5` files are not vendored into `data_releases.us.datasets`. +### Non-default dataset overlays + +`certify_data_release` rewrites `data_releases.{country}` wholesale from the +certified release manifest, so that block only ever holds datasets from the +certified release. A staged artifact published in its own release — one that is +deliberately not the certified default — is registered instead under the +sibling `dataset_overlays.{country}` map: + +```json +"dataset_overlays": { + "us": { + "populace_us_2024_acs_local": { + "path": "populace_us_2024_acs_local.h5", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-buildl-acs-local-...", + "sha256": "..." + } + } +} +``` + +`get_release_manifest` merges overlays into the resolvable dataset registry, so +`resolve_dataset_reference` and `managed_microsimulation` load them by name at +their own pinned, sha-verified revision. Overlays are strictly additive: an +overlay may not shadow the certified default or any certified dataset, so +default resolution is untouched. Because certification only rewrites +`data_releases`, overlays survive re-certification without any manual +re-add step. + Earlier releases (policyengine 4.15.x–4.16.x) were certified through the `PolicyEngine/policyengine-bundles` archive flow; those bundles remain the historical record of their certifications. diff --git a/src/policyengine/data/bundle/manifest.json b/src/policyengine/data/bundle/manifest.json index b53f5232..dea9e49c 100644 --- a/src/policyengine/data/bundle/manifest.json +++ b/src/policyengine/data/bundle/manifest.json @@ -178,6 +178,16 @@ "version": "populace-us-2024-buildj-sparse-rmloss100-75d5add-20260710T094201Z" } }, + "dataset_overlays": { + "us": { + "populace_us_2024_acs_local": { + "path": "populace_us_2024_acs_local.h5", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-buildl-acs-local-36de5d9a-20260712T104640Z", + "sha256": "36de5d9aa69fa932cd42b1b9c08ef72b20176414118997de6240a588fab30a6b" + } + } + }, "extras": { "models": [ "policyengine-core", diff --git a/src/policyengine/data/bundle/us.trace.tro.jsonld b/src/policyengine/data/bundle/us.trace.tro.jsonld index 91f9b869..f55124cd 100644 --- a/src/policyengine/data/bundle/us.trace.tro.jsonld +++ b/src/policyengine/data/bundle/us.trace.tro.jsonld @@ -75,7 +75,7 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for us", "trov:mimeType": "application/json", - "trov:sha256": "af7a0e23bd5c705ef58fcfbfd905fc83e592bd76b97b1148b1166701690ca253" + "trov:sha256": "ed684abcc7fad65349022682572d159570c7a242a3381cd354786df880d51749" }, { "@id": "composition/1/artifact/data_release_manifest", @@ -102,7 +102,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "269b53486c6897fd527aa340d4b55b11aa3f15320c82cd5ff3eadb1cd0e460fa" + "trov:sha256": "b29142be3d790236769727e914cd8e97b035b87551c31119d0744f838d09cc65" } }, "trov:hasPerformance": { diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index b9a7d616..088676fd 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -251,6 +251,54 @@ def fetch_pypi_wheel_metadata(name: str, version: str) -> dict[str, Optional[str return {"sha256": None, "url": None} +DATASET_OVERLAYS_KEY = "dataset_overlays" + + +def _apply_dataset_overlays( + country_id: str, + release_payload: dict, + bundle: dict, +) -> dict: + """Merge hand-curated ``dataset_overlays`` into the certified datasets. + + ``data_releases.{country}`` is regenerated wholesale by + ``certify_data_release`` every time a release is certified, so a + certified dataset always originates from the certified release + manifest. ``dataset_overlays.{country}`` is a sibling, hand-maintained + registry of *additional* named datasets that resolution should expose + without ever certifying them as the default — for example a staged + artifact published in its own immutable release (see + ``docs/release-bundles.md``). Because certification only rewrites + ``data_releases``, overlays survive re-certification untouched. + + Overlays are strictly additive: an overlay may not shadow the certified + default dataset or any certified dataset entry, so it can never alter + default resolution. + """ + overlays = (bundle.get(DATASET_OVERLAYS_KEY) or {}).get(country_id) or {} + if not overlays: + return release_payload + + certified_datasets = dict(release_payload.get("datasets") or {}) + default_dataset = release_payload.get("default_dataset") + for overlay_name, overlay_reference in overlays.items(): + if overlay_name == default_dataset: + raise ValueError( + f"Dataset overlay '{overlay_name}' for country '{country_id}' " + "shadows the certified default dataset. Overlays must never " + "change default resolution." + ) + if overlay_name in certified_datasets: + raise ValueError( + f"Dataset overlay '{overlay_name}' for country '{country_id}' " + "collides with a certified dataset entry. Overlays must be " + "additive, not overrides." + ) + certified_datasets[overlay_name] = overlay_reference + + return {**release_payload, "datasets": certified_datasets} + + @lru_cache def get_release_manifest(country_id: str) -> CountryReleaseManifest: manifest_path = files("policyengine").joinpath("data", "bundle", "manifest.json") @@ -264,6 +312,7 @@ def get_release_manifest(country_id: str) -> CountryReleaseManifest: except KeyError as exc: raise ValueError(f"No bundled data release for country '{country_id}'") from exc + release_payload = _apply_dataset_overlays(country_id, release_payload, bundle) manifest = CountryReleaseManifest.model_validate(release_payload) manifest.source_sha256 = hashlib.sha256(source_bytes).hexdigest() return manifest diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index 5f53b58f..bef795c6 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -18,8 +18,10 @@ from policyengine.core.tax_benefit_model_version import TaxBenefitModelVersion from policyengine.provenance.manifest import ( ArtifactPathReference, + CountryReleaseManifest, DataCertification, DataReleaseManifestUnavailableError, + _apply_dataset_overlays, certify_data_release_compatibility, dataset_logical_name, get_data_release_manifest, @@ -55,6 +57,17 @@ US_RELEASE_MANIFEST_DATASET_URI = ( f"hf://policyengine/populace-us/populace_us_2024.h5@{US_DATA_RELEASE_REVISION}" ) +# Non-default local-area overlay: a staged Populace US artifact published in its +# own immutable release (Build L), loadable by name but never the default. +US_LOCAL_AREA_DATASET = "populace_us_2024_acs_local" +US_LOCAL_AREA_RELEASE_ID = "populace-us-2024-buildl-acs-local-36de5d9a-20260712T104640Z" +US_LOCAL_AREA_SHA256 = ( + "36de5d9aa69fa932cd42b1b9c08ef72b20176414118997de6240a588fab30a6b" +) +US_LOCAL_AREA_DATASET_URI = ( + "hf://policyengine/populace-us/populace_us_2024_acs_local.h5" + f"@{US_LOCAL_AREA_RELEASE_ID}" +) UK_MODEL_VERSION = "2.89.2" UK_BUILT_WITH_MODEL_VERSION = "2.89.2" UK_DATA_RELEASE_VERSION = "0.1.0" @@ -254,6 +267,47 @@ def test__given_no_dataset__then_managed_resolution_uses_certified_default(self) == get_release_manifest("us").default_dataset_uri ) + def test__given_local_area_overlay__then_default_resolution_is_unchanged(self): + """Regression: the non-default local-area overlay never changes defaults. + + The overlay adds a loadable name; it must never become the certified + default, and default resolution must keep pointing at the certified + national Populace artifact. + """ + manifest = get_release_manifest("us") + + assert manifest.default_dataset == "populace_us_2024" + assert manifest.default_dataset_uri == US_CERTIFIED_DATASET_URI + assert resolve_managed_dataset_reference("us") == US_CERTIFIED_DATASET_URI + assert resolve_managed_dataset_reference("us") != US_LOCAL_AREA_DATASET_URI + assert manifest.region_datasets["national"].path_template == ( + "populace_us_2024.h5" + ) + + def test__given_local_area_name__then_resolves_to_its_immutable_tag(self): + """The local-area overlay resolves to its own Build L release tag.""" + assert ( + resolve_dataset_reference("us", US_LOCAL_AREA_DATASET) + == US_LOCAL_AREA_DATASET_URI + ) + # It is a first-class managed dataset name, so managed resolution + # accepts it without allow_unmanaged. + assert ( + resolve_managed_dataset_reference("us", US_LOCAL_AREA_DATASET) + == US_LOCAL_AREA_DATASET_URI + ) + + def test__given_local_area_overlay__then_registered_with_sha_but_not_default(self): + manifest = get_release_manifest("us") + + reference = manifest.datasets[US_LOCAL_AREA_DATASET] + assert reference.path == "populace_us_2024_acs_local.h5" + assert reference.repo_id == "policyengine/populace-us" + assert reference.revision == US_LOCAL_AREA_RELEASE_ID + assert reference.sha256 == US_LOCAL_AREA_SHA256 + assert US_LOCAL_AREA_DATASET in resolve_default_datasets("us") + assert US_LOCAL_AREA_DATASET != manifest.default_dataset + def test__given_us_manifest__then_has_no_inherited_area_artifacts(self): manifest = get_release_manifest("us") @@ -1019,3 +1073,112 @@ def test__given_uk_unmanaged_dataset_uri__then_source_is_not_rewritten(self): assert microsim.policyengine_bundle["runtime_dataset_source"] == ( "/tmp/frs_2022_23.h5" ) + + +class TestDatasetOverlays: + """Unit tests for the ``dataset_overlays`` merge layer. + + ``dataset_overlays`` is the hand-maintained sibling of ``data_releases``. + ``certify_data_release`` rewrites ``data_releases.{country}`` wholesale, so + these tests pin the invariant that overlays are additive, survive + re-certification, and can never hijack default resolution. + """ + + def teardown_method(self): + get_release_manifest.cache_clear() + + def _recertified_us_payload(self) -> dict: + """A freshly certified ``data_releases.us`` payload (no overlay). + + This is the shape ``certify_data_release`` writes: it carries only the + certified release's own datasets and never the overlay. + """ + return { + "country_id": "us", + "policyengine_version": "9.9.9", + "model_package": {"name": "policyengine-us", "version": "1.999.0"}, + "data_package": { + "name": "populace-data", + "version": "0.1.0", + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + "release_manifest_revision": "populace-us-buildm", + }, + "default_dataset": "populace_us_2024", + "datasets": { + "populace_us_2024": { + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-buildm", + }, + }, + "region_datasets": {"national": {"path_template": "populace_us_2024.h5"}}, + } + + def _local_area_overlay(self) -> dict: + return { + US_LOCAL_AREA_DATASET: { + "path": "populace_us_2024_acs_local.h5", + "repo_id": "policyengine/populace-us", + "revision": US_LOCAL_AREA_RELEASE_ID, + "sha256": US_LOCAL_AREA_SHA256, + } + } + + def test__given_recertified_release__then_overlay_survives(self): + """The overlay lives outside ``data_releases``, so a re-certified + payload that never mentions it still resolves it by name.""" + payload = self._recertified_us_payload() + bundle = { + "data_releases": {"us": payload}, + "dataset_overlays": {"us": self._local_area_overlay()}, + } + + merged = _apply_dataset_overlays("us", payload, bundle) + manifest = CountryReleaseManifest.model_validate(merged) + + assert manifest.default_dataset == "populace_us_2024" + assert US_LOCAL_AREA_DATASET in manifest.datasets + reference = manifest.datasets[US_LOCAL_AREA_DATASET] + assert reference.revision == US_LOCAL_AREA_RELEASE_ID + assert reference.sha256 == US_LOCAL_AREA_SHA256 + + def test__given_no_overlays__then_payload_is_returned_unchanged(self): + payload = self._recertified_us_payload() + + assert _apply_dataset_overlays("us", payload, {"data_releases": {}}) is payload + + def test__given_overlay_shadowing_default__then_raises(self): + payload = self._recertified_us_payload() + bundle = { + "data_releases": {"us": payload}, + "dataset_overlays": { + "us": {"populace_us_2024": {"path": "other.h5"}}, + }, + } + + try: + _apply_dataset_overlays("us", payload, bundle) + except ValueError as error: + assert "shadows the certified default" in str(error) + else: + raise AssertionError("Expected overlay shadowing the default to fail") + + def test__given_overlay_colliding_with_certified_dataset__then_raises(self): + payload = self._recertified_us_payload() + payload["datasets"]["demographics"] = { + "path": "releases/populace-us-buildm/demographics.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-buildm", + } + bundle = { + "data_releases": {"us": payload}, + "dataset_overlays": {"us": {"demographics": {"path": "other.json"}}}, + } + + try: + _apply_dataset_overlays("us", payload, bundle) + except ValueError as error: + assert "collides with a certified dataset" in str(error) + else: + raise AssertionError("Expected overlay colliding with a dataset to fail")