From 14e735886ca932349037d99b18290d46cb72415b Mon Sep 17 00:00:00 2001 From: Danil Silantyev Date: Sat, 15 Aug 2026 04:53:38 +0500 Subject: [PATCH] feat(catalog): make tier requirements data, and hold every caller to them `--tier scheduled` needs a GitHub token, two external hosts and the tag refs. That was recorded once, as comments beside maintenance.yml's egress allow-list, and enforced nowhere. Three jobs invoked the tier; one granted what it needs. release.yml ran every tier with no token, no tags and an allow-list omitting both SDK hosts, so a release stopped in preflight on four capability failures -- before its own promotion gate, and long before the missing evidence manifest that is the known blocker. It had never been noticed because the graph had never run: at 0.13.3 `resolve` did not invoke validate_all at all, and the step that does landed ten days after the last release. The advisory tier now splits by what a check reaches for. CALENDAR reaches no further than the checkout; EXTERNAL needs a token or a named host. `--tier release` is core plus CALENDAR, so a release still refuses to ship an expired external fact without a publishing graph reaching the network to find out. It passes with no token and no reachable host. catalog/validation-tiers.yml records what each check needs and which job may run which tier. check_validation_tier_contract.py reads membership from validate_all, the requirement from the catalog, and the grant from the job. Discovery is fail-closed both ways: an undeclared caller is a finding, and so is a declared caller that no longer exists. Six mutations were each caught, including release.yml reverting to every tier. --- .github/workflows/release.yml | 29 ++- CHANGELOG.md | 28 +++ catalog/python-execution.yml | 5 +- catalog/validation-tiers.yml | 84 +++++++++ scripts/check_validation_tier_contract.py | 209 ++++++++++++++++++++++ scripts/validate_all.py | 35 +++- 6 files changed, 380 insertions(+), 10 deletions(-) create mode 100644 catalog/validation-tiers.yml create mode 100644 scripts/check_validation_tier_contract.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 53da3dd..a182d86 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,6 +62,12 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + # The preflight reconciles CHANGELOG headings against SemVer tags, and + # that check fails closed when it can see none -- a shallow checkout + # cannot tell "no tags" from "no tags fetched". A release of all things + # must not be the run that cannot see its own ledger. + fetch-depth: 0 + fetch-tags: true persist-credentials: false - name: Set up Python id: python @@ -88,14 +94,25 @@ jobs: "$PYTHON_PATH" -I -B -m venv --copies .venv uv pip install --python .venv/bin/python --require-hashes -r requirements-ci.txt - # A release is the one place where the full sweep is the right gate: an - # immutable artifact must not ship carrying an expired external fact or a - # runtime-coverage waiver that is already past due. Pull requests get the - # scoped tiers; a release gets everything. - - name: Full validation sweep + # An immutable artifact must not ship carrying an expired external fact or + # a runtime-coverage waiver already past due, so the release keeps the + # calendar checks. What it cannot keep is the rest of the advisory tier: + # `transitive-action-pins` and `anchor-contexts` fail closed without a + # GitHub token, and `flutter-pin` and `qt-pin` need two hosts this job's + # egress policy blocks. Running `validate_all.py --` here meant every + # release stopped in preflight on four capability failures -- before the + # promotion gate, and long before the missing evidence manifest (#157) that + # is the *known* blocker. The graph had never been exercised: at 0.13.3 + # `resolve` did not run validate_all at all. + # + # `--tier release` is core plus the calendar checks, and reaches no further + # than this checkout. `catalog/validation-tiers.yml` records why, and + # `check_validation_tier_contract.py` fails if this job ever invokes a tier + # it does not grant the capabilities for. + - name: Release validation preflight run: | .venv/bin/python -I -B scripts/check_python_syntax.py - .venv/bin/python -I -B scripts/check_python_execution_contract.py --launch validate_all.py -- + .venv/bin/python -I -B scripts/check_python_execution_contract.py --launch validate_all.py -- --tier release - name: Resolve and validate version id: v diff --git a/CHANGELOG.md b/CHANGELOG.md index 5092f3e..b9c69f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,34 @@ ## [Unreleased] +- Make a tier's requirements data, and hold every caller to them. `--tier + scheduled` needs a GitHub token, two external hosts and the tag refs. That was + written down once, as comments beside `maintenance.yml`'s egress allow-list, and + enforced nowhere. Three jobs invoked the tier and one granted what it needs. + + `release.yml` ran **every** tier with no token, no tags and an allow-list + omitting both SDK hosts, so a release stopped in preflight on four capability + failures — before its own promotion gate, and long before the missing evidence + manifest that is the *known* release blocker. Nothing had noticed because the + graph had never been exercised: at 0.13.3 `resolve` did not run `validate_all` + at all, and the step that does arrived ten days after the last release. + + The advisory tier splits by what a check reaches for. `CALENDAR` — + product-fact expiry, runtime-coverage waivers, the release ledger against tags, + documentation links — reaches no further than the checkout. `EXTERNAL` — the two + SDK pins and the two GitHub-API checks — does. A new `--tier release` is core + plus `CALENDAR`, so a release still refuses to ship an expired external fact + without a publishing graph reaching the network to find out. It passes with no + token and no reachable host, which is the point. + + `catalog/validation-tiers.yml` records what each check needs and which job may + run which tier. `check_validation_tier_contract.py` reads tier membership from + `validate_all` itself, the requirement from the catalog, and the grant from the + job — allow-list, step and job `env`, checkout inputs. Discovery is fail-closed + in both directions: an undeclared caller is a finding, and so is a declared + caller that no longer exists. Six mutations were each caught, including + `release.yml` reverting to every tier. + - Let the advisory sweep survive finding something. The first real run of `maintenance.yml` failed, and the reason was only visible from the run: GitHub runs `run:` steps as `bash -e {0}`, and `-e` arrives on the shell's own command diff --git a/catalog/python-execution.yml b/catalog/python-execution.yml index cd3527f..beebe87 100644 --- a/catalog/python-execution.yml +++ b/catalog/python-execution.yml @@ -2,7 +2,7 @@ "schema_version": 1, "python": { "major_minor": "3.13", - "subject_count": 58, + "subject_count": 59, "launcher": "scripts/check_python_execution_contract.py", "launcher_prefix": [".venv/bin/python", "-I", "-B", "scripts/check_python_execution_contract.py", "--launch"], "syntax_gate_prefix": [".venv/bin/python", "-I", "-B", "scripts/check_python_syntax.py"], @@ -122,6 +122,7 @@ "check_benchmark_contract.py", "check_cache_contract.py", "check_ci_tier_selection.py", "check_maintenance_report_contract.py", + "check_validation_tier_contract.py", "check_docs_links.py", "check_documented_commands.py", "check_flutter_pin.py", "check_qt_pin.py", "check_examples.py", @@ -162,6 +163,7 @@ "check_cache_contract.py": ["_strict_yaml", "_workflow_yaml"], "check_ci_tier_selection.py": ["_workflow_yaml", "check_python_execution_contract"], "check_maintenance_report_contract.py": ["_workflow_yaml", "check_python_execution_contract"], + "check_validation_tier_contract.py": ["_strict_yaml", "_workflow_yaml", "validate_all"], "check_documented_commands.py": ["_workflow_yaml"], "check_flutter_pin.py": ["_strict_yaml"], "check_qt_pin.py": ["_strict_yaml"], @@ -202,6 +204,7 @@ "check_benchmark_contract", "check_cache_contract", "check_ci_tier_selection", "check_maintenance_report_contract", + "check_validation_tier_contract", "check_docs_links", "check_documented_commands", "check_examples", "check_flutter_pin", "check_qt_pin", "check_gate_contract", diff --git a/catalog/validation-tiers.yml b/catalog/validation-tiers.yml new file mode 100644 index 0000000..f489183 --- /dev/null +++ b/catalog/validation-tiers.yml @@ -0,0 +1,84 @@ +# What each validator needs beyond the tree, and which job may invoke which tier. +# +# The tier split was written down in prose and enforced nowhere. `--tier +# scheduled` needs a GitHub token, two external hosts and the tag refs, and that +# requirement lived only in comments beside `maintenance.yml`'s egress +# allow-list. Three jobs invoked the tier; one of them granted what it needs. +# +# `release.yml` ran every tier with none of the four, so a release stopped in +# preflight on capability failures before its promotion gate -- and before the +# missing evidence manifest that is the known blocker. `ci.yml` fell back to the +# advisory tier when it could not resolve a change base, putting calendar and +# network work inside a required gate while skipping the blocking checks it was +# supposed to run. Both were invisible because neither path had ever executed. +# +# So the requirement becomes data, and `check_validation_tier_contract.py` +# holds every caller to it. Discovery is fail-closed: a job invoking +# `validate_all.py` that is not declared below is a finding, because a closed +# allowlist that only checks what it was told is exactly how the first version +# of this problem stayed invisible. +schema_version: 1 + +capabilities: + git_tags: >- + The repository's SemVer tag refs. `actions/checkout` fetches no tags by + default, and `fetch-depth: 0` alone still fetches none -- `fetch-tags: true` + is a separate input. A check that reconciles against tags fails closed + without them, because it cannot tell "no tags" from "no tags fetched". + github_token: >- + A GitHub API token in `GH_TOKEN` or `GITHUB_TOKEN`. Unauthenticated the API + allows 60 requests an hour, which these checks exhaust, so they report + themselves unverified rather than silently pass on a partial answer. + network:api.github.com: >- + The GitHub REST API, for resolving what a pinned third-party action itself + calls and for reading the live branch ruleset. + network:storage.googleapis.com: >- + Google's published Flutter release manifest, which the SDK pin is resolved + against. + network:download.qt.io: >- + The published Qt repository index, which the Qt pin is resolved against. + +# Checks needing something the checkout alone does not provide. A check absent +# from this map needs nothing. Every name here must be a registered check. +requirements: + release-ledger-tags: + - git_tags + flutter-pin: + - network:storage.googleapis.com + qt-pin: + - network:download.qt.io + transitive-action-pins: + - github_token + - network:api.github.com + anchor-contexts: + - github_token + - network:api.github.com + +# Every job that runs `validate_all.py`, and the tier it runs. An invocation not +# listed here fails the contract rather than inheriting whatever the job happens +# to grant. +invocations: + - workflow: .github/workflows/ci.yml + job: validate + tier: core + note: The blocking gate. Properties of the tree in hand, and nothing else. + - workflow: .github/workflows/ci.yml + job: validate-touched + tier: touched + note: >- + Blocking and scoped to the change. Must never invoke the advisory tier: + it does not contain these checks, and it does contain checks this job + grants nothing for. + - workflow: .github/workflows/maintenance.yml + job: sweep + tier: scheduled + note: >- + The only caller that grants the full advisory set, and the only one that + should. Failures here are maintenance debt, filed as one tracking issue. + - workflow: .github/workflows/release.yml + job: resolve + tier: release + note: >- + Core plus the calendar checks. An immutable artifact must not ship + carrying an expired external fact, but a publishing graph must not be + reaching the network to find out. diff --git a/scripts/check_validation_tier_contract.py b/scripts/check_validation_tier_contract.py new file mode 100644 index 0000000..7650471 --- /dev/null +++ b/scripts/check_validation_tier_contract.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""A tier is a promise about what its checks need. Every caller must keep it. + +`validate_all.py --tier scheduled` needs a GitHub token, two external hosts and +the tag refs. That requirement was written down once, as comments beside +`maintenance.yml`'s egress allow-list, and enforced nowhere. Three jobs invoked +the tier and one of them granted what it needs: + +* `release.yml` ran every tier with no token, no tags and an allow-list omitting + both SDK hosts, so a release stopped in preflight on four capability failures + -- before its own promotion gate, and long before the missing evidence + manifest that is the *known* release blocker. +* `ci.yml` fell back to the advisory tier whenever it could not resolve a change + base, which put calendar and network work inside a required gate while + skipping the blocking changed-path checks it was there to run. + +Neither was visible, because neither path had ever executed. So the requirement +becomes data in `catalog/validation-tiers.yml`, and this holds every caller to +it: tier membership is read from `validate_all` itself, the capability each +check needs is read from the catalog, and what a job grants is read from the job. + +Discovery is fail-closed in both directions. An invocation the catalog does not +declare is a finding, because a closed allowlist that only checks what it was +told is how the first version of this problem stayed invisible; and a declared +invocation that no longer exists is a finding too, because a contract naming +things that are gone stops being read. +""" +from __future__ import annotations + +import re +import sys +from typing import Any + +from ci_workflows_tools._strict_yaml import strict_load +from ci_workflows_tools._workflow_yaml import REPO_ROOT, load_yaml, workflow_files + +CONTRACT = REPO_ROOT / "catalog/validation-tiers.yml" +TOOL = "validate_all.py" +INVOCATION = re.compile(rf"--launch\s+{re.escape(TOOL)}\s+--(?P[^\n]*)") +TIER_FLAG = re.compile(r"--tier\s+(?P[a-z]+)") +NETWORK = "network:" + + +def _tiers() -> dict[str, list[str]]: + """Tier membership, read from the module that defines it. + + Imported here rather than at module scope: `validate_all` registers this + check, so a top-level import is a cycle. Reading the lists when they are + needed also means this can never disagree with what actually ran. + """ + from ci_workflows_tools import validate_all + + return { + "core": [name for name, _ in validate_all.CORE], + "touched": [name for name, _ in validate_all.TOUCHED], + "scheduled": [name for name, _ in validate_all.SCHEDULED], + "release": [name for name, _ in validate_all.RELEASE], + "all": [name for name, _ in (*validate_all.CORE, *validate_all.TOUCHED, + *validate_all.SCHEDULED)], + } + + +def _catalog_problems(contract: dict[str, Any], tiers: dict[str, list[str]]) -> list[str]: + problems: list[str] = [] + capabilities = contract.get("capabilities") or {} + requirements = contract.get("requirements") or {} + known_checks = {name for names in tiers.values() for name in names} + for check, needed in requirements.items(): + if check not in known_checks: + problems.append( + f"catalog/validation-tiers.yml: {check!r} needs {needed}, but no tier " + "registers a check by that name") + for capability in needed or []: + if capability not in capabilities: + problems.append( + f"catalog/validation-tiers.yml: {check!r} needs undeclared " + f"capability {capability!r}") + for capability, description in capabilities.items(): + if not str(description).strip(): + problems.append( + f"catalog/validation-tiers.yml: capability {capability!r} has no description") + if not any(capability in (needed or []) for needed in requirements.values()): + problems.append( + f"catalog/validation-tiers.yml: capability {capability!r} is declared " + "but no check needs it") + return problems + + +def _needed(tier: str, tiers: dict[str, list[str]], requirements: dict) -> set[str]: + return { + capability + for check in tiers.get(tier, []) + for capability in (requirements.get(check) or []) + } + + +def _discovered() -> dict[tuple[str, str], set[str]]: + """Every job that runs the tool, and the tiers it runs, read from the tree.""" + found: dict[tuple[str, str], set[str]] = {} + for path in workflow_files(): + relative = path.relative_to(REPO_ROOT).as_posix() + doc = load_yaml(path) + for job_id, job in (doc.get("jobs") or {}).items(): + if not isinstance(job, dict): + continue + for step in job.get("steps") or []: + if not isinstance(step, dict): + continue + for match in INVOCATION.finditer(str(step.get("run") or "")): + tier = TIER_FLAG.search(match.group("rest")) + found.setdefault((relative, str(job_id)), set()).add( + tier.group("tier") if tier else "all") + return found + + +def _granted(job: dict, step_env_names: set[str]) -> tuple[set[str], bool, bool]: + """What the job actually provides: token, reachable hosts, tag refs.""" + job_env = {str(k) for k in (job.get("env") or {})} + has_token = bool({"GH_TOKEN", "GITHUB_TOKEN"} & (step_env_names | job_env)) + hosts: set[str] = set() + blocked = False + tags = False + for step in job.get("steps") or []: + if not isinstance(step, dict): + continue + uses = str(step.get("uses") or "") + options = step.get("with") or {} + if "harden-runner" in uses: + if str(options.get("egress-policy") or "") == "block": + blocked = True + for entry in str(options.get("allowed-endpoints") or "").split(): + hosts.add(entry.rsplit(":", 1)[0]) + if "actions/checkout" in uses and options.get("fetch-tags") is True: + tags = True + return hosts, has_token, (tags, blocked) # type: ignore[return-value] + + +def check() -> list[str]: + contract = strict_load(CONTRACT) + tiers = _tiers() + problems = _catalog_problems(contract, tiers) + requirements = contract.get("requirements") or {} + declared = { + (str(entry["workflow"]), str(entry["job"])): str(entry["tier"]) + for entry in (contract.get("invocations") or []) + } + discovered = _discovered() + + for key in sorted(set(discovered) - set(declared)): + problems.append( + f"{key[0]}: job {key[1]!r} runs {TOOL} but is not declared in " + "catalog/validation-tiers.yml; every caller must state the tier it runs " + "so the capabilities it needs can be checked") + for key in sorted(set(declared) - set(discovered)): + problems.append( + f"catalog/validation-tiers.yml declares {key[0]} job {key[1]!r} as a " + f"{TOOL} caller, which it no longer is") + + for key, tiers_run in sorted(discovered.items()): + relative, job_id = key + expected = declared.get(key) + if expected is None: + continue + if tiers_run != {expected}: + problems.append( + f"{relative}: job {job_id!r} is declared as running the {expected!r} " + f"tier but runs {sorted(tiers_run)}") + doc = load_yaml(REPO_ROOT / relative) + job = (doc.get("jobs") or {}).get(job_id) or {} + step_env: set[str] = set() + for step in job.get("steps") or []: + if isinstance(step, dict) and INVOCATION.search(str(step.get("run") or "")): + step_env |= {str(k) for k in (step.get("env") or {})} + hosts, has_token, (tags, blocked) = _granted(job, step_env) + + for capability in sorted(_needed(expected, tiers, requirements)): + if capability == "github_token" and not has_token: + problems.append( + f"{relative}: job {job_id!r} runs the {expected!r} tier, which needs " + "a GitHub token, but neither the step nor the job sets GH_TOKEN or " + "GITHUB_TOKEN; those checks would fail closed on the environment") + elif capability == "git_tags" and not tags: + problems.append( + f"{relative}: job {job_id!r} runs the {expected!r} tier, which " + "reconciles against SemVer tags, but its checkout does not set " + "`fetch-tags: true`; `fetch-depth: 0` alone fetches no tags") + elif capability.startswith(NETWORK) and blocked: + host = capability[len(NETWORK):] + if host not in hosts: + problems.append( + f"{relative}: job {job_id!r} runs the {expected!r} tier, which " + f"reaches {host}, but the harden-runner allow-list does not " + "include it and the egress policy is `block`") + return problems + + +def main() -> int: + problems = check() + if problems: + print("check_validation_tier_contract: FAIL", file=sys.stderr) + for problem in problems: + print(f" - {problem}", file=sys.stderr) + return 1 + print("check_validation_tier_contract: OK") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_all.py b/scripts/validate_all.py index a449982..c213f5a 100644 --- a/scripts/validate_all.py +++ b/scripts/validate_all.py @@ -50,6 +50,7 @@ check_cache_contract, check_ci_tier_selection, check_maintenance_report_contract, + check_validation_tier_contract, check_docs_links, check_flutter_pin, check_documented_commands, @@ -119,6 +120,7 @@ ("cache-contract", check_cache_contract.check), ("ci-tier-selection", check_ci_tier_selection.check), ("maintenance-report", check_maintenance_report_contract.check), + ("validation-tiers", check_validation_tier_contract.check), ("actionlint-contract", check_actionlint_contract.check), ("actionlint-config", check_actionlint_config.check), ("documented-commands", check_documented_commands.check), @@ -151,18 +153,36 @@ ("runtime-coverage-touched", validate_runtime_coverage.check_for_paths), ] -# Advisory. Real work, but maintenance debt rather than a defect in a change. -SCHEDULED = [ +# Advisory, and reaching no further than the checkout. A release runs these: +# an immutable artifact must not ship carrying an expired external fact, and +# nothing here needs a credential or a third party to say so. `release-ledger-tags` +# needs the tag refs, which a release has by construction. +CALENDAR = [ ("product-facts-calendar", validate_product_facts.check), ("runtime-coverage-calendar", validate_runtime_coverage.check), ("release-ledger-tags", check_release_ledger.check_tags), ("docs-links", check_docs_links.check), +] + +# Advisory, and needing a GitHub token or a named external host. This is the +# split that matters: these fail closed when their capability is absent, so a +# caller that cannot grant them gets a report about its own environment rather +# than about the tree. `catalog/validation-tiers.yml` records what each needs and +# `check_validation_tier_contract.py` holds every caller to it. +EXTERNAL = [ ("flutter-pin", check_flutter_pin.check), ("qt-pin", check_qt_pin.check), ("transitive-action-pins", check_transitive_action_pins.check), ("anchor-contexts", check_anchor_contexts.check), ] +SCHEDULED = CALENDAR + EXTERNAL + +# What a release preflight runs. Deterministic properties of the tree, plus the +# freshness the release itself is answerable for -- and nothing that reaches the +# network from a publishing graph. +RELEASE = CORE + CALENDAR + def changed_paths(base: str | None, explicit: list[str]) -> set[str]: """Repository-relative paths this change touches.""" @@ -211,7 +231,7 @@ def run(label: str, problems: list[str]) -> bool: def main() -> int: parser = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0]) parser.add_argument( - "--tier", choices=["all", "core", "touched", "scheduled"], default="all", + "--tier", choices=["all", "core", "touched", "scheduled", "release"], default="all", help="which group to run (default: all)", ) parser.add_argument( @@ -225,6 +245,15 @@ def main() -> int: args = parser.parse_args() ok = True + if args.tier == "release": + for label, fn in RELEASE: + ok &= run(label, fn()) + if not ok: + print(f"\nvalidate_all ({args.tier}): FAIL", file=sys.stderr) + return 1 + print(f"validate_all ({args.tier}): OK") + return 0 + if args.tier in ("all", "core"): for label, fn in CORE: ok &= run(label, fn())