Split candidate staging versions

.github/CONTRIBUTING.md

@@ -66,7 +66,7 @@ The PR is valid only if the head repository is `PolicyEngine/policyengine-us-dat
 Six workflow files in `.github/workflows/`:
 
 - `pr.yaml` — fork check, lint, uv.lock freshness, towncrier fragment check, unit tests, smoke test, independent docs build, and quality guards. Integration tests trigger when files in `policyengine_us_data/`, `modal_app/`, or `tests/integration/` change. ~2–3 min for the unit path.
-- `push.yaml` — on push to main: functional commits create the Towncrier version-bump commit; `Update package version` commits publish PyPI, verify the package version is visible, then dispatch the full Modal data build from that exact commit.
+- `push.yaml` — on push to main: functional commits create the Towncrier publication-candidate commit; `Update publication candidate` commits dispatch the full Modal data build from that exact commit. Candidate runs stage to Hugging Face only; PyPI publishing happens during final promotion.
 - `pipeline.yaml` — dispatch only, spawns the H5 generation pipeline on Modal with configurable GPU/epochs/workers.
 - `long_run_projection.yaml` — dispatch only, builds long-run CPS projection H5 files for explicit sampled years and can optionally upload them to a run-scoped Hugging Face staging prefix.
 - `local_area_publish.yaml` / `local_area_promote.yaml` — manual dispatch to build/stage local-area H5 files and promote a run-scoped US data release.

.github/bump_version.py

@@ -1,13 +1,24 @@
-"""Infer semver bump from towncrier fragment types and update version."""
+"""Infer release candidate scope from towncrier fragment types."""
 
+import json
 import re
 import sys
 from pathlib import Path
 
+from policyengine_us_data.utils.run_context import (
+    build_candidate_scope,
+    release_version_from_bump,
+)
+
+
+VERSION_RE = re.compile(r'^version\s*=\s*"([^"]+)"', re.MULTILINE)
+SEMVER_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)(?:rc(\d+))?$")
+PUBLICATION_SCOPE_PATH = Path(".github/publication_scope.json")
+
 
 def get_current_version(pyproject_path: Path) -> str:
     text = pyproject_path.read_text()
-    match = re.search(r'^version\s*=\s*"(\d+\.\d+\.\d+)"', text, re.MULTILINE)
+    match = VERSION_RE.search(text)
     if not match:
         print(
             "Could not find version in pyproject.toml",
@@ -39,24 +50,16 @@ def infer_bump(changelog_dir: Path) -> str:
 
 
 def bump_version(version: str, bump: str) -> str:
-    major, minor, patch = (int(x) for x in version.split("."))
-    if bump == "major":
-        return f"{major + 1}.0.0"
-    elif bump == "minor":
-        return f"{major}.{minor + 1}.0"
-    else:
-        return f"{major}.{minor}.{patch + 1}"
-
-
-def update_file(path: Path, old_version: str, new_version: str):
-    text = path.read_text()
-    updated = text.replace(
-        f'version = "{old_version}"',
-        f'version = "{new_version}"',
-    )
-    if updated != text:
-        path.write_text(updated)
-        print(f"  Updated {path}")
+    match = SEMVER_RE.match(version)
+    if not match:
+        print(f"Unsupported version format: {version}", file=sys.stderr)
+        sys.exit(1)
+    return release_version_from_bump(version, bump)
+
+
+def write_publication_scope(path: Path, payload: dict[str, str]) -> None:
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    print(f"  Updated {path}")
 
 
 def main():
@@ -66,11 +69,23 @@ def main():
 
     current = get_current_version(pyproject)
     bump = infer_bump(changelog_dir)
-    new = bump_version(current, bump)
-
-    print(f"Version: {current} -> {new} ({bump})")
-
-    update_file(pyproject, current, new)
+    would_release_as = bump_version(current, bump)
+    candidate_scope = build_candidate_scope(current, bump)
+
+    print(f"Base release version: {current}")
+    print(f"Candidate scope: {candidate_scope}")
+    print(f"Release bump: {bump}")
+    print(f"Would release as at build time: {would_release_as}")
+
+    write_publication_scope(
+        root / PUBLICATION_SCOPE_PATH,
+        {
+            "base_release_version": current,
+            "release_bump": bump,
+            "candidate_scope": candidate_scope,
+            "would_release_as_at_build_time": would_release_as,
+        },
+    )
 
 
 if __name__ == "__main__":

.github/scripts/dispatch_publication_pipeline.sh

@@ -14,10 +14,28 @@ if [[ -z "${SOURCE_SHA:-}" ]]; then
   exit 1
 fi
 
+if [[ -z "${CANDIDATE_VERSION:-}" ]]; then
+  echo "CANDIDATE_VERSION is required" >&2
+  exit 1
+fi
+
+if [[ -z "${BASE_RELEASE_VERSION:-}" ]]; then
+  echo "BASE_RELEASE_VERSION is required" >&2
+  exit 1
+fi
+
+if [[ -z "${RELEASE_BUMP:-}" ]]; then
+  echo "RELEASE_BUMP is required" >&2
+  exit 1
+fi
+
 gh workflow run "${workflow_file}" \
   --ref "${workflow_ref}" \
   -f run_id="${US_DATA_RUN_ID}" \
-  -f source_sha="${SOURCE_SHA}"
+  -f source_sha="${SOURCE_SHA}" \
+  -f candidate_version="${CANDIDATE_VERSION}" \
+  -f base_release_version="${BASE_RELEASE_VERSION}" \
+  -f release_bump="${RELEASE_BUMP}"
 
 if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
   {
@@ -26,6 +44,9 @@ if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
     echo "| Field | Value |"
     echo "|-------|-------|"
     echo "| Run ID | \`${US_DATA_RUN_ID}\` |"
+    echo "| Candidate scope | \`${CANDIDATE_VERSION}\` |"
+    echo "| Base release version | \`${BASE_RELEASE_VERSION}\` |"
+    echo "| Release bump | \`${RELEASE_BUMP}\` |"
     echo "| Source SHA | \`${SOURCE_SHA}\` |"
     echo "| Workflow | \`${workflow_file}\` |"
     echo "| Workflow ref | \`${workflow_ref}\` |"

.github/scripts/fetch_publication_scope.py

@@ -0,0 +1,50 @@
+"""Print one field from the generated publication candidate scope file."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+PUBLICATION_SCOPE_PATH = REPO_ROOT / ".github" / "publication_scope.json"
+VALID_FIELDS = frozenset(
+    {
+        "base_release_version",
+        "release_bump",
+        "candidate_scope",
+        "would_release_as_at_build_time",
+    }
+)
+
+
+def read_publication_scope(path: Path = PUBLICATION_SCOPE_PATH) -> dict[str, Any]:
+    if not path.exists():
+        raise FileNotFoundError(f"Missing publication scope file: {path}")
+    payload = json.loads(path.read_text())
+    missing = sorted(VALID_FIELDS.difference(payload))
+    if missing:
+        raise ValueError(
+            "Publication scope file is missing required field(s): " + ", ".join(missing)
+        )
+    return payload
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("field", choices=sorted(VALID_FIELDS))
+    args = parser.parse_args()
+
+    try:
+        value = read_publication_scope(PUBLICATION_SCOPE_PATH)[args.field]
+    except Exception as exc:
+        print(str(exc), file=sys.stderr)
+        sys.exit(1)
+    print(value)
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/fetch_release_version.py

@@ -0,0 +1,26 @@
+"""Print the stable release version corresponding to pyproject.toml."""
+
+from __future__ import annotations
+
+import re
+import sys
+import tomllib
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+VERSION_RE = re.compile(r"^(\d+\.\d+\.\d+)(?:rc\d+)?$")
+
+
+def main() -> None:
+    with (REPO_ROOT / "pyproject.toml").open("rb") as file:
+        version = tomllib.load(file)["project"]["version"]
+    match = VERSION_RE.match(version)
+    if not match:
+        print(f"Unsupported version format: {version}", file=sys.stderr)
+        sys.exit(1)
+    print(match.group(1))
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/finalize_package_version.py

@@ -0,0 +1,50 @@
+"""Rewrite pyproject.toml to the stable version selected at promotion time."""
+
+from __future__ import annotations
+
+import os
+import re
+import sys
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+VERSION_RE = re.compile(r'^(version\s*=\s*)"([^"]+)"', re.MULTILINE)
+PACKAGE_VERSION_RE = re.compile(r"^(\d+\.\d+\.\d+)(?:rc\d+)?$")
+
+
+def _release_version(candidate_version: str) -> str:
+    match = PACKAGE_VERSION_RE.match(candidate_version)
+    if not match:
+        raise ValueError(f"Unsupported package version: {candidate_version}")
+    return match.group(1)
+
+
+def _resolve_release_version(current_version: str) -> str:
+    release_version = os.environ.get("US_DATA_RELEASE_VERSION", "")
+    if not release_version:
+        return _release_version(current_version)
+    return _release_version(release_version)
+
+
+def main() -> None:
+    pyproject = REPO_ROOT / "pyproject.toml"
+    text = pyproject.read_text()
+    match = VERSION_RE.search(text)
+    if not match:
+        print("Could not find project version in pyproject.toml", file=sys.stderr)
+        sys.exit(1)
+
+    current_version = match.group(2)
+    release_version = _resolve_release_version(current_version)
+    if current_version == release_version:
+        print(f"pyproject.toml already uses final version {release_version}.")
+        return
+
+    updated = VERSION_RE.sub(rf'\1"{release_version}"', text, count=1)
+    pyproject.write_text(updated)
+    print(f"Finalized package version: {current_version} -> {release_version}")
+
+
+if __name__ == "__main__":
+    main()