From 5d84067646cd57391cc9496290d64671748f6fa0 Mon Sep 17 00:00:00 2001 From: stacknil Date: Fri, 19 Jun 2026 12:21:25 +0800 Subject: [PATCH] docs(sbom): add policy decision examples --- .gitattributes | 2 + scripts/validate-reviewer-routes.py | 1 + tools/sbom-diff-and-risk/README.md | 659 +++++++++--------- .../docs/example-artifact-regeneration.md | 6 + .../docs/policy-decision-ci-cookbook.md | 4 + .../docs/policy-decision-explainability.md | 4 + .../sbom-diff-and-risk/docs/reviewer-path.md | 1 + .../examples/policy-decisions/README.md | 26 + .../examples/policy-decisions/fail.json | 34 + .../policy-decisions/needs-review.json | 18 + .../examples/policy-decisions/pass.json | 23 + .../examples/policy-decisions/warn.json | 34 + .../tests/test_policy_decision_examples.py | 77 ++ 13 files changed, 560 insertions(+), 329 deletions(-) create mode 100644 tools/sbom-diff-and-risk/examples/policy-decisions/README.md create mode 100644 tools/sbom-diff-and-risk/examples/policy-decisions/fail.json create mode 100644 tools/sbom-diff-and-risk/examples/policy-decisions/needs-review.json create mode 100644 tools/sbom-diff-and-risk/examples/policy-decisions/pass.json create mode 100644 tools/sbom-diff-and-risk/examples/policy-decisions/warn.json create mode 100644 tools/sbom-diff-and-risk/tests/test_policy_decision_examples.py diff --git a/.gitattributes b/.gitattributes index b2231e1..050bb8e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,6 @@ /.gitattributes text eol=lf .github/workflows/*.yml text eol=lf +*.json text eol=lf *.md text eol=lf +*.py text eol=lf scripts/validate-reviewer-routes.py text eol=lf diff --git a/scripts/validate-reviewer-routes.py b/scripts/validate-reviewer-routes.py index d6c0610..1304926 100644 --- a/scripts/validate-reviewer-routes.py +++ b/scripts/validate-reviewer-routes.py @@ -66,6 +66,7 @@ "tools/sbom-diff-and-risk/docs/reviewer-brief.md", "tools/sbom-diff-and-risk/docs/reviewer-evidence-pack.md", "tools/sbom-diff-and-risk/docs/verification.md", + "tools/sbom-diff-and-risk/examples/policy-decisions/README.md", "tools/sbom-diff-and-risk/examples/sample-report.json", "tools/sbom-diff-and-risk/examples/sample-summary.json", "tools/sbom-diff-and-risk/examples/sample-policy.json", diff --git a/tools/sbom-diff-and-risk/README.md b/tools/sbom-diff-and-risk/README.md index ee2e33a..195ac97 100644 --- a/tools/sbom-diff-and-risk/README.md +++ b/tools/sbom-diff-and-risk/README.md @@ -1,15 +1,15 @@ -# sbom-diff-and-risk - +# sbom-diff-and-risk + v0.9.0 is the policy JSON sidecar and consumer integration usability release. It adds optional `--policy-json PATH` output, checked-in policy sidecar examples, and copyable GitHub Actions consumer guidance. It keeps dependency analysis local and deterministic by default, preserves the completed TestPyPI dry-run story, and keeps production PyPI publishing intentionally deferred. - -`sbom-diff-and-risk` is a local, deterministic CLI for comparing two SBOMs or dependency manifests and producing JSON plus Markdown reports. - -It uses conservative heuristics for change intelligence. By default it does not resolve CVEs, does not act as a reputation oracle, and does not perform hidden network enrichment. - + +`sbom-diff-and-risk` is a local, deterministic CLI for comparing two SBOMs or dependency manifests and producing JSON plus Markdown reports. + +It uses conservative heuristics for change intelligence. By default it does not resolve CVEs, does not act as a reputation oracle, and does not perform hidden network enrichment. + ## Start Here For the clearest reviewer route, start with @@ -42,167 +42,167 @@ For regenerating checked-in local example outputs, see provenance, start with [Dependency provenance analysis](#dependency-provenance-analysis-opt-in) and [Dependency provenance reporting](#dependency-provenance-reporting). - -## Scope - -- Normalize two local inputs into a shared component schema. -- Diff components as `added`, `removed`, and `changed`. -- Apply conservative, heuristic risk buckets to newly added and changed components. -- Apply optional local policy enforcement over those findings. -- Produce machine-friendly JSON and reviewer-friendly Markdown reports. -- Stay fully local-file based by default. - -## v0.1 Internal Component Model - -The normalized schema is the core design choice for the project: - -- `name: str` -- `version: str | None` -- `ecosystem: str` -- `purl: str | None` -- `license_id: str | None` -- `supplier: str | None` -- `source_url: str | None` -- `bom_ref: str | None` -- `raw_type: str | None` -- `evidence: dict` - -Diff identity is intentionally conservative and uses this precedence: - -1. `purl` -2. `bom_ref` -3. `(ecosystem, name)` - -When a `purl` includes a version, the tool keeps the full value in `Component.purl` for auditability but uses the versionless package coordinate for identity so upgrades still diff as `changed`. - -## Non-goals - -- No vulnerability database integration in v0.1. -- No CVE, advisory, or exploit resolution in v0.1. -- No reputation scoring or malware verdicts. -- No hidden enrichment or implicit network access. -- No web UI. -- No packaged GitHub Marketplace Action. - -## Supported Formats - -- CycloneDX JSON -- SPDX JSON -- `requirements.txt` -- `pyproject.toml` via PEP 621 `[project]` metadata -- `pyproject.toml` dependency groups via PEP 735 `[dependency-groups]` with explicit selection - -## Risk Bucket Semantics - -The current heuristic buckets are: - -- `new_package` -- `major_upgrade` -- `version_change_unclassified` -- `unknown_license` -- `stale_package` -- `suspicious_source` -- `not_evaluated` - -Offline `stale_package` evaluation is intentionally deferred. When enrichment is disabled, the tool emits `not_evaluated` findings instead of guessing. - -## Output Formats - + +## Scope + +- Normalize two local inputs into a shared component schema. +- Diff components as `added`, `removed`, and `changed`. +- Apply conservative, heuristic risk buckets to newly added and changed components. +- Apply optional local policy enforcement over those findings. +- Produce machine-friendly JSON and reviewer-friendly Markdown reports. +- Stay fully local-file based by default. + +## v0.1 Internal Component Model + +The normalized schema is the core design choice for the project: + +- `name: str` +- `version: str | None` +- `ecosystem: str` +- `purl: str | None` +- `license_id: str | None` +- `supplier: str | None` +- `source_url: str | None` +- `bom_ref: str | None` +- `raw_type: str | None` +- `evidence: dict` + +Diff identity is intentionally conservative and uses this precedence: + +1. `purl` +2. `bom_ref` +3. `(ecosystem, name)` + +When a `purl` includes a version, the tool keeps the full value in `Component.purl` for auditability but uses the versionless package coordinate for identity so upgrades still diff as `changed`. + +## Non-goals + +- No vulnerability database integration in v0.1. +- No CVE, advisory, or exploit resolution in v0.1. +- No reputation scoring or malware verdicts. +- No hidden enrichment or implicit network access. +- No web UI. +- No packaged GitHub Marketplace Action. + +## Supported Formats + +- CycloneDX JSON +- SPDX JSON +- `requirements.txt` +- `pyproject.toml` via PEP 621 `[project]` metadata +- `pyproject.toml` dependency groups via PEP 735 `[dependency-groups]` with explicit selection + +## Risk Bucket Semantics + +The current heuristic buckets are: + +- `new_package` +- `major_upgrade` +- `version_change_unclassified` +- `unknown_license` +- `stale_package` +- `suspicious_source` +- `not_evaluated` + +Offline `stale_package` evaluation is intentionally deferred. When enrichment is disabled, the tool emits `not_evaluated` findings instead of guessing. + +## Output Formats + - `report.json` - `summary.json` when `--summary-json` is provided - `policy.json` when `--policy-json` is provided - `report.md` - `report.sarif` - -## Install - -```bash -python -m pip install -e .[dev] -``` - -## Usage - -Generate reports from the bundled CycloneDX example inputs: - -```bash -sbom-diff-risk compare \ - --before examples/cdx_before.json \ + +## Install + +```bash +python -m pip install -e .[dev] +``` + +## Usage + +Generate reports from the bundled CycloneDX example inputs: + +```bash +sbom-diff-risk compare \ + --before examples/cdx_before.json \ --after examples/cdx_after.json \ --format auto \ --out-json outputs/report.json \ --summary-json outputs/summary.json \ --out-md outputs/report.md ``` - -Generate reports from the `requirements.txt` examples: - -```bash -sbom-diff-risk compare \ - --before examples/requirements_before.txt \ - --after examples/requirements_after.txt \ - --format auto \ - --out-json outputs/requirements-report.json \ - --out-md outputs/requirements-report.md -``` - -Use explicit format flags when you do not want auto-detection: - -```bash -sbom-diff-risk compare \ - --before examples/spdx_before.json \ - --after examples/spdx_after.json \ - --before-format spdx-json \ - --after-format spdx-json \ - --out-json outputs/spdx-report.json \ - --out-md outputs/spdx-report.md -``` - -Generate reports from PEP 621 `pyproject.toml` examples: - -```bash -sbom-diff-risk compare \ - --before examples/pyproject_before.toml \ - --after examples/pyproject_after.toml \ - --format auto \ - --out-json outputs/pyproject-report.json \ - --out-md outputs/pyproject-report.md -``` - -Generate reports for a specific PEP 735 dependency group: - -```bash -sbom-diff-risk compare \ - --before examples/pyproject_groups_before.toml \ - --after examples/pyproject_groups_after.toml \ - --format pyproject-toml \ - --pyproject-group dev \ - --out-json outputs/pyproject-groups-report.json \ - --out-md outputs/pyproject-groups-report.md -``` - -## CLI Flags - -- `--before path` -- `--after path` -- `--format auto|cyclonedx-json|spdx-json|requirements-txt|pyproject-toml` -- `--before-format cyclonedx-json|spdx-json|requirements-txt|pyproject-toml` -- `--after-format cyclonedx-json|spdx-json|requirements-txt|pyproject-toml` + +Generate reports from the `requirements.txt` examples: + +```bash +sbom-diff-risk compare \ + --before examples/requirements_before.txt \ + --after examples/requirements_after.txt \ + --format auto \ + --out-json outputs/requirements-report.json \ + --out-md outputs/requirements-report.md +``` + +Use explicit format flags when you do not want auto-detection: + +```bash +sbom-diff-risk compare \ + --before examples/spdx_before.json \ + --after examples/spdx_after.json \ + --before-format spdx-json \ + --after-format spdx-json \ + --out-json outputs/spdx-report.json \ + --out-md outputs/spdx-report.md +``` + +Generate reports from PEP 621 `pyproject.toml` examples: + +```bash +sbom-diff-risk compare \ + --before examples/pyproject_before.toml \ + --after examples/pyproject_after.toml \ + --format auto \ + --out-json outputs/pyproject-report.json \ + --out-md outputs/pyproject-report.md +``` + +Generate reports for a specific PEP 735 dependency group: + +```bash +sbom-diff-risk compare \ + --before examples/pyproject_groups_before.toml \ + --after examples/pyproject_groups_after.toml \ + --format pyproject-toml \ + --pyproject-group dev \ + --out-json outputs/pyproject-groups-report.json \ + --out-md outputs/pyproject-groups-report.md +``` + +## CLI Flags + +- `--before path` +- `--after path` +- `--format auto|cyclonedx-json|spdx-json|requirements-txt|pyproject-toml` +- `--before-format cyclonedx-json|spdx-json|requirements-txt|pyproject-toml` +- `--after-format cyclonedx-json|spdx-json|requirements-txt|pyproject-toml` - `--pyproject-group name` - `--out-json path` - `--summary-json path` - `--policy-json path` - `--out-md path` - `--out-sarif path` -- `--policy path` -- `--fail-on rule[,rule...]` -- `--warn-on rule[,rule...]` -- `--strict` -- `--enrich-pypi` -- `--pypi-timeout seconds` -- `--enrich-scorecard` -- `--scorecard-timeout seconds` -- `--source-allowlist pypi.org,files.pythonhosted.org,github.com` - +- `--policy path` +- `--fail-on rule[,rule...]` +- `--warn-on rule[,rule...]` +- `--strict` +- `--enrich-pypi` +- `--pypi-timeout seconds` +- `--enrich-scorecard` +- `--scorecard-timeout seconds` +- `--source-allowlist pypi.org,files.pythonhosted.org,github.com` + Offline mode remains the default. No network access occurs unless `--enrich-pypi` or `--enrich-scorecard` is set explicitly. `--summary-json PATH` writes only the stable `report.json["summary"]` object for compact machine consumption. It uses the same summary schema as the full JSON report. @@ -220,97 +220,97 @@ The checked-in [examples/sample-policy.json](examples/sample-policy.json) artifact shows the standalone policy sidecar shape. ## Dependency Provenance Analysis (Opt-in) - -This section is about analyzing third-party package provenance signals. It is not about verifying the `sbom-diff-and-risk` tool's own release artifacts. - -PyPI provenance and integrity enrichment is explicit and additive in this PR: - -- only Python / PyPI packages are queried -- no hidden network access occurs in default mode -- enrichment results are captured as evidence and summarized in the reports -- per-component `evidence.provenance` records stable lookup fields such as `supported`, `lookup_performed`, and per-file attestation totals -- lack of attestation is treated as unavailable metadata, not as proof of compromise -- policy evaluation can use these signals explicitly when configured -- SARIF stays conservative and only emits selected high-signal provenance policy violations - -When enabled, the tool queries PyPI-facing release metadata plus file-level provenance data and records stable evidence fields under component `evidence.provenance`, along with run metadata under `metadata.enrichment` and the top-level trust-signal report fields in the JSON report. - -```bash -sbom-diff-risk compare \ - --before examples/requirements_before.txt \ - --after examples/requirements_after.txt \ - --enrich-pypi \ - --pypi-timeout 3 \ - --out-json outputs/report-enriched.json -``` - -## Dependency Provenance Reporting - -When provenance enrichment is enabled, the reports surface trust signals directly instead of burying them in component evidence: - -- JSON includes `provenance_summary`, `attestation_summary`, `enrichment_metadata`, `trust_signal_notes`, and `provenance_policy_impact` -- Markdown includes `Provenance summary`, `Attestation gaps`, `Policy impact for provenance-related rules`, and `Trust signal notes` -- core diff semantics do not change when enrichment is enabled -- SARIF maps only selected high-signal provenance decisions such as `provenance_required`, blocking `missing_attestation`, and blocking `unverified_provenance` -- provenance-related SARIF alerts prefer file-level locations that point to the relevant compared manifest or SBOM input - -Routine enrichment outcomes remain JSON and Markdown evidence for review. Non-blocking enrichment facts do not automatically become SARIF alerts. - -## Opt-in Scorecard Enrichment - -OpenSSF Scorecard enrichment is also explicit and advisory: - -- no Scorecard requests are made unless `--enrich-scorecard` is set -- lookups only occur when a component can be mapped to a repository with high confidence from explicit metadata -- repository registry pages and ambiguous URLs are treated as unmapped instead of inferred -- Scorecard results are auxiliary trust signals, not proof of safety -- Scorecard-only SARIF alerts are emitted only when policy explicitly turns a threshold breach into a violation - -```bash -sbom-diff-risk compare \ - --before examples/cdx_before.json \ - --after examples/cdx_after.json \ - --enrich-scorecard \ - --scorecard-timeout 3 \ - --out-json outputs/report-scorecard.json -``` - + +This section is about analyzing third-party package provenance signals. It is not about verifying the `sbom-diff-and-risk` tool's own release artifacts. + +PyPI provenance and integrity enrichment is explicit and additive in this PR: + +- only Python / PyPI packages are queried +- no hidden network access occurs in default mode +- enrichment results are captured as evidence and summarized in the reports +- per-component `evidence.provenance` records stable lookup fields such as `supported`, `lookup_performed`, and per-file attestation totals +- lack of attestation is treated as unavailable metadata, not as proof of compromise +- policy evaluation can use these signals explicitly when configured +- SARIF stays conservative and only emits selected high-signal provenance policy violations + +When enabled, the tool queries PyPI-facing release metadata plus file-level provenance data and records stable evidence fields under component `evidence.provenance`, along with run metadata under `metadata.enrichment` and the top-level trust-signal report fields in the JSON report. + +```bash +sbom-diff-risk compare \ + --before examples/requirements_before.txt \ + --after examples/requirements_after.txt \ + --enrich-pypi \ + --pypi-timeout 3 \ + --out-json outputs/report-enriched.json +``` + +## Dependency Provenance Reporting + +When provenance enrichment is enabled, the reports surface trust signals directly instead of burying them in component evidence: + +- JSON includes `provenance_summary`, `attestation_summary`, `enrichment_metadata`, `trust_signal_notes`, and `provenance_policy_impact` +- Markdown includes `Provenance summary`, `Attestation gaps`, `Policy impact for provenance-related rules`, and `Trust signal notes` +- core diff semantics do not change when enrichment is enabled +- SARIF maps only selected high-signal provenance decisions such as `provenance_required`, blocking `missing_attestation`, and blocking `unverified_provenance` +- provenance-related SARIF alerts prefer file-level locations that point to the relevant compared manifest or SBOM input + +Routine enrichment outcomes remain JSON and Markdown evidence for review. Non-blocking enrichment facts do not automatically become SARIF alerts. + +## Opt-in Scorecard Enrichment + +OpenSSF Scorecard enrichment is also explicit and advisory: + +- no Scorecard requests are made unless `--enrich-scorecard` is set +- lookups only occur when a component can be mapped to a repository with high confidence from explicit metadata +- repository registry pages and ambiguous URLs are treated as unmapped instead of inferred +- Scorecard results are auxiliary trust signals, not proof of safety +- Scorecard-only SARIF alerts are emitted only when policy explicitly turns a threshold breach into a violation + +```bash +sbom-diff-risk compare \ + --before examples/cdx_before.json \ + --after examples/cdx_after.json \ + --enrich-scorecard \ + --scorecard-timeout 3 \ + --out-json outputs/report-scorecard.json +``` + If you want policy gating, make it explicit with a v3 policy such as [policy-scorecard-minimal.yml](examples/policy-scorecard-minimal.yml), which sets `minimum_scorecard_score` and opts into the `scorecard_below_threshold` rule. - -Setting `minimum_scorecard_score` alone is advisory metadata for review. It only affects policy outcomes when `scorecard_below_threshold` is configured explicitly in `block_on`, `warn_on`, or `ignore_rules`. - -## Tool Provenance And Verification - + +Setting `minimum_scorecard_score` alone is advisory metadata for review. It only affects policy outcomes when `scorecard_below_threshold` is configured explicitly in `block_on`, `warn_on`, or `ignore_rules`. + +## Tool Provenance And Verification + This section is about verifying `sbom-diff-and-risk` itself. If you want the shortest path to the right verification instructions, start with [docs/verification.md](docs/verification.md). - -This repository also records provenance for `sbom-diff-and-risk` itself by generating GitHub artifact attestations for the wheel and source distribution produced by the `sbom-diff-and-risk-ci` workflow. - -- the attested files are the wheel and source distribution built by `python -m build` from `tools/sbom-diff-and-risk` + +This repository also records provenance for `sbom-diff-and-risk` itself by generating GitHub artifact attestations for the wheel and source distribution produced by the `sbom-diff-and-risk-ci` workflow. + +- the attested files are the wheel and source distribution built by `python -m build` from `tools/sbom-diff-and-risk` - the build files are uploaded together as the `sbom-diff-and-risk-dist` workflow artifact - version-tag runs also publish those same built files as GitHub Release assets for the matching tag - releases produced by the updated workflow include `sbom-diff-and-risk-SHA256SUMS.txt` for local SHA256 verification of downloaded wheel and source distribution files - only trusted non-PR runs publish the attestation -- consumers can verify workflow-built artifacts with `gh attestation verify` -- consumers can verify immutable releases and downloaded release assets with `gh release verify` and `gh release verify-asset` -- this complements the tool's analysis of third-party supply-chain inputs, but it does not replace that analysis - -Verification docs: - +- consumers can verify workflow-built artifacts with `gh attestation verify` +- consumers can verify immutable releases and downloaded release assets with `gh release verify` and `gh release verify-asset` +- this complements the tool's analysis of third-party supply-chain inputs, but it does not replace that analysis + +Verification docs: + - [docs/verification.md](docs/verification.md) for the quick decision guide - [docs/self-provenance.md](docs/self-provenance.md) for workflow-artifact attestation - [docs/release-provenance.md](docs/release-provenance.md) for release-asset verification and immutable release guidance - [docs/pypi-trusted-publishing-readiness.md](docs/pypi-trusted-publishing-readiness.md) for TestPyPI Trusted Publishing readiness and dry-run notes - [docs/pypi-production-publishing-decision.md](docs/pypi-production-publishing-decision.md) for the production PyPI decision gate, publisher identity, future workflow shape, and production prerequisites - -## Examples - + +## Examples + The [examples/](examples/) directory includes: - -- before/after inputs for CycloneDX JSON, SPDX JSON, `requirements.txt`, and `pyproject.toml` -- dependency-group examples at `examples/pyproject_groups_before.toml` and `examples/pyproject_groups_after.toml` -- example policies at `examples/policy-minimal.yml` and `examples/policy-strict.yml` -- provenance-aware policy examples at `examples/policy-provenance-minimal.yml` and `examples/policy-provenance-strict.yml` -- a Scorecard-aware policy example at `examples/policy-scorecard-minimal.yml` + +- before/after inputs for CycloneDX JSON, SPDX JSON, `requirements.txt`, and `pyproject.toml` +- dependency-group examples at `examples/pyproject_groups_before.toml` and `examples/pyproject_groups_after.toml` +- example policies at `examples/policy-minimal.yml` and `examples/policy-strict.yml` +- provenance-aware policy examples at `examples/policy-provenance-minimal.yml` and `examples/policy-provenance-strict.yml` +- a Scorecard-aware policy example at `examples/policy-scorecard-minimal.yml` - a sample pass JSON report at [sample-report.json](examples/sample-report.json) - a sample summary-only JSON artifact at [sample-summary.json](examples/sample-summary.json) - a consumer GitHub Actions workflow example at [github-actions-consumer.yml](examples/github-actions-consumer.yml) @@ -319,6 +319,7 @@ The [examples/](examples/) directory includes: - sample policy-warn reports at [sample-policy-warn-report.json](examples/sample-policy-warn-report.json) and [sample-policy-warn-report.md](examples/sample-policy-warn-report.md) - sample policy-fail reports at [sample-policy-fail-report.json](examples/sample-policy-fail-report.json) and [sample-policy-fail-report.md](examples/sample-policy-fail-report.md) - a sample policy-only sidecar at [sample-policy.json](examples/sample-policy.json) +- compact policy-decision examples at [examples/policy-decisions](examples/policy-decisions/README.md) - a sample SARIF export at [sample-sarif.sarif](examples/sample-sarif.sarif) - provenance-aware sample reports at [sample-provenance-report.json](examples/sample-provenance-report.json), [sample-provenance-report.md](examples/sample-provenance-report.md), and [sample-provenance-report.sarif](examples/sample-provenance-report.sarif) - Scorecard-aware sample reports at [sample-scorecard-report.json](examples/sample-scorecard-report.json), [sample-scorecard-report.md](examples/sample-scorecard-report.md), and [sample-scorecard-report.sarif](examples/sample-scorecard-report.sarif) @@ -334,60 +335,60 @@ python scripts/regenerate-example-artifacts.py --check Use `python scripts/regenerate-example-artifacts.py --list` and `--only SLUG` for focused checks such as `--only requirements`. - -## Enforcement Mode - -Policy enforcement is optional and deterministic. Exit codes are stable: - -- `0` = success / no blocking violations -- `1` = blocking policy violations -- `2` = usage, parse, policy, or runtime error - -Minimal policy enforcement example: - -```bash -sbom-diff-risk compare \ - --before examples/requirements_before.txt \ - --after examples/requirements_after.txt \ - --policy examples/policy-minimal.yml \ - --out-json outputs/report.json \ - --out-md outputs/report.md -``` - -Ad hoc enforcement without a policy file: - -```bash -sbom-diff-risk compare \ - --before examples/cdx_before.json \ - --after examples/cdx_after.json \ - --fail-on suspicious_source,unknown_license \ - --warn-on new_package \ - --out-json outputs/report.json \ - --out-md outputs/report.md -``` - -Failed runs still write reports on exit code `1`; stderr prints a concise blocking summary so CI logs are understandable without opening raw JSON. - -## SARIF Export - -SARIF export is intentionally conservative. The current renderer emits a GitHub-compatible SARIF 2.1.0 subset for: - -- `suspicious_source` -- `unknown_license` -- `major_upgrade` -- selected policy results such as `max_added_packages`, `allow_sources`, `provenance_required`, and blocking provenance violations like `missing_attestation` or `unverified_provenance` -- explicit Scorecard policy violations such as `scorecard_below_threshold` - -It does not turn every enrichment fact, diff, or informational heuristic into a code scanning alert. - -```bash -sbom-diff-risk compare \ - --before examples/sarif_before.json \ - --after examples/sarif_after.json \ - --policy examples/policy-strict.yml \ - --out-sarif outputs/report.sarif -``` - + +## Enforcement Mode + +Policy enforcement is optional and deterministic. Exit codes are stable: + +- `0` = success / no blocking violations +- `1` = blocking policy violations +- `2` = usage, parse, policy, or runtime error + +Minimal policy enforcement example: + +```bash +sbom-diff-risk compare \ + --before examples/requirements_before.txt \ + --after examples/requirements_after.txt \ + --policy examples/policy-minimal.yml \ + --out-json outputs/report.json \ + --out-md outputs/report.md +``` + +Ad hoc enforcement without a policy file: + +```bash +sbom-diff-risk compare \ + --before examples/cdx_before.json \ + --after examples/cdx_after.json \ + --fail-on suspicious_source,unknown_license \ + --warn-on new_package \ + --out-json outputs/report.json \ + --out-md outputs/report.md +``` + +Failed runs still write reports on exit code `1`; stderr prints a concise blocking summary so CI logs are understandable without opening raw JSON. + +## SARIF Export + +SARIF export is intentionally conservative. The current renderer emits a GitHub-compatible SARIF 2.1.0 subset for: + +- `suspicious_source` +- `unknown_license` +- `major_upgrade` +- selected policy results such as `max_added_packages`, `allow_sources`, `provenance_required`, and blocking provenance violations like `missing_attestation` or `unverified_provenance` +- explicit Scorecard policy violations such as `scorecard_below_threshold` + +It does not turn every enrichment fact, diff, or informational heuristic into a code scanning alert. + +```bash +sbom-diff-risk compare \ + --before examples/sarif_before.json \ + --after examples/sarif_after.json \ + --policy examples/policy-strict.yml \ + --out-sarif outputs/report.sarif +``` + For GitHub code scanning integration guidance and a minimal upload workflow, see [docs/github-code-scanning.md](docs/github-code-scanning.md). For the shortest path to the tool-verification docs, start with [docs/verification.md](docs/verification.md). @@ -399,49 +400,49 @@ For details on how version-tag releases publish those same build outputs as rele For TestPyPI Trusted Publishing readiness and the completed dry-run path, see [docs/pypi-trusted-publishing-readiness.md](docs/pypi-trusted-publishing-readiness.md). For the production PyPI decision gate, including the intended package name, first-version rule, publisher identity, future workflow shape, and provenance boundaries, see [docs/pypi-production-publishing-decision.md](docs/pypi-production-publishing-decision.md). - -## Parser Boundaries - + +## Parser Boundaries + Deterministic local mode intentionally supports a conservative subset of packaging syntax. The detailed matrix lives in [docs/parser-boundaries.md](docs/parser-boundaries.md). - -### requirements.txt subset - -| Syntax | Status | Notes | -| --- | --- | --- | -| Plain PEP 508 requirement entries | Supported | Names, specifiers, extras, and markers | -| Comments, blank lines, line continuations | Supported | Normalized locally without installer behavior | -| `-r`, `--requirement` | Unsupported | Include chains fail closed | -| `-c`, `--constraint` | Unsupported | Constraint files fail closed | -| Editable installs | Unsupported | `-e` and `--editable` are rejected | -| Direct URL, VCS, and local path refs | Unsupported | Includes `pkg @ https://...`, `git+...`, wheels, archives, and local paths | -| Index and source options | Unsupported | Includes `--index-url`, `--extra-index-url`, `--find-links`, and related flags | - -### pyproject.toml subset - -- default parsing supports PEP 621 `[project.dependencies]` and `[project.optional-dependencies]` -- dependency groups are supported through PEP 735 `[dependency-groups]` -- dependency groups must be selected explicitly with `--pyproject-group ` -- dependency groups are not treated as aliases for `[project.optional-dependencies]` -- tool-specific layouts such as Poetry, Hatch, and PDM remain out of scope in v0.2 - -## Limitations - -- default mode is local-file based only. -- PyPI provenance enrichment is opt-in only via `--enrich-pypi`; default runs stay offline. -- `generated_at` remains `null` to preserve deterministic report output. -- `stale_package` is not resolved offline. The report emits `not_evaluated` instead. -- provenance evidence is recorded for supported PyPI packages only; unsupported and failed lookups remain explicit evidence gaps. -- SARIF export intentionally covers only a conservative subset of findings in v0.2, including only selected high-signal provenance policy violations. -- Scorecard enrichment is opt-in only via `--enrich-scorecard`, uses only high-confidence repository mappings, and remains advisory unless policy explicitly gates it. -- No vulnerability database integration, CVE matching, or advisory enrichment. -- `requirements.txt` support intentionally covers a conservative subset: plain PEP 508 requirement entries, comments, extras, markers, and line continuations. -- `requirements.txt` intentionally rejects include/constraint directives, editable installs, direct URL/path refs, index/source options, and other pip-only install flags in deterministic mode. -- `pyproject.toml` support intentionally covers a conservative subset: PEP 621 `[project.dependencies]`, `[project.optional-dependencies]`, and explicit PEP 735 `[dependency-groups]` selection. -- `pyproject.toml` intentionally does not support tool-specific layouts such as Poetry, Hatch, or PDM sections in v0.2. -- Risk buckets are heuristics, not security verdicts. -- Runtime-generated `outputs/` artifacts are ignored; tracked examples live in `examples/`. -- Policy files are YAML-only in v0.2 and unknown rule ids fail closed. - -## Current Status - -The project now normalizes local CycloneDX JSON, SPDX JSON, `requirements.txt`, and conservative `pyproject.toml` inputs, including explicit PEP 735 dependency-group selection, into the shared component model, diffs them deterministically, and generates stable JSON/Markdown/SARIF reports with tests and optional policy enforcement. + +### requirements.txt subset + +| Syntax | Status | Notes | +| --- | --- | --- | +| Plain PEP 508 requirement entries | Supported | Names, specifiers, extras, and markers | +| Comments, blank lines, line continuations | Supported | Normalized locally without installer behavior | +| `-r`, `--requirement` | Unsupported | Include chains fail closed | +| `-c`, `--constraint` | Unsupported | Constraint files fail closed | +| Editable installs | Unsupported | `-e` and `--editable` are rejected | +| Direct URL, VCS, and local path refs | Unsupported | Includes `pkg @ https://...`, `git+...`, wheels, archives, and local paths | +| Index and source options | Unsupported | Includes `--index-url`, `--extra-index-url`, `--find-links`, and related flags | + +### pyproject.toml subset + +- default parsing supports PEP 621 `[project.dependencies]` and `[project.optional-dependencies]` +- dependency groups are supported through PEP 735 `[dependency-groups]` +- dependency groups must be selected explicitly with `--pyproject-group ` +- dependency groups are not treated as aliases for `[project.optional-dependencies]` +- tool-specific layouts such as Poetry, Hatch, and PDM remain out of scope in v0.2 + +## Limitations + +- default mode is local-file based only. +- PyPI provenance enrichment is opt-in only via `--enrich-pypi`; default runs stay offline. +- `generated_at` remains `null` to preserve deterministic report output. +- `stale_package` is not resolved offline. The report emits `not_evaluated` instead. +- provenance evidence is recorded for supported PyPI packages only; unsupported and failed lookups remain explicit evidence gaps. +- SARIF export intentionally covers only a conservative subset of findings in v0.2, including only selected high-signal provenance policy violations. +- Scorecard enrichment is opt-in only via `--enrich-scorecard`, uses only high-confidence repository mappings, and remains advisory unless policy explicitly gates it. +- No vulnerability database integration, CVE matching, or advisory enrichment. +- `requirements.txt` support intentionally covers a conservative subset: plain PEP 508 requirement entries, comments, extras, markers, and line continuations. +- `requirements.txt` intentionally rejects include/constraint directives, editable installs, direct URL/path refs, index/source options, and other pip-only install flags in deterministic mode. +- `pyproject.toml` support intentionally covers a conservative subset: PEP 621 `[project.dependencies]`, `[project.optional-dependencies]`, and explicit PEP 735 `[dependency-groups]` selection. +- `pyproject.toml` intentionally does not support tool-specific layouts such as Poetry, Hatch, or PDM sections in v0.2. +- Risk buckets are heuristics, not security verdicts. +- Runtime-generated `outputs/` artifacts are ignored; tracked examples live in `examples/`. +- Policy files are YAML-only in v0.2 and unknown rule ids fail closed. + +## Current Status + +The project now normalizes local CycloneDX JSON, SPDX JSON, `requirements.txt`, and conservative `pyproject.toml` inputs, including explicit PEP 735 dependency-group selection, into the shared component model, diffs them deterministically, and generates stable JSON/Markdown/SARIF reports with tests and optional policy enforcement. diff --git a/tools/sbom-diff-and-risk/docs/example-artifact-regeneration.md b/tools/sbom-diff-and-risk/docs/example-artifact-regeneration.md index 5b34397..245ea47 100644 --- a/tools/sbom-diff-and-risk/docs/example-artifact-regeneration.md +++ b/tools/sbom-diff-and-risk/docs/example-artifact-regeneration.md @@ -34,6 +34,12 @@ The strict-policy example intentionally exits with code `1` because it produces blocking local policy findings. The script treats that as expected while still capturing the generated reports. +The `examples/policy-decisions/` fixtures are hand-authored consumer examples +for `pass`, `warn`, `fail`, and `needs-review` review outcomes. They are +validated by tests, but they are not regenerated by this script because +`needs-review` is a consumer interpretation rather than a runtime +`summary.policy.status` value. + ## Check Mode Use `--check` to verify that generated output matches the checked-in artifacts diff --git a/tools/sbom-diff-and-risk/docs/policy-decision-ci-cookbook.md b/tools/sbom-diff-and-risk/docs/policy-decision-ci-cookbook.md index c203c08..b13bf32 100644 --- a/tools/sbom-diff-and-risk/docs/policy-decision-ci-cookbook.md +++ b/tools/sbom-diff-and-risk/docs/policy-decision-ci-cookbook.md @@ -25,6 +25,10 @@ decision metadata remains available for review. For a checked-in reference artifact generated from this path, see [sample-policy.json](../examples/sample-policy.json). +For compact consumer examples that distinguish `pass`, `warn`, `fail`, and +`needs-review` review outcomes, see +[examples/policy-decisions](../examples/policy-decisions/README.md). + For a full GitHub Actions consumer workflow example that captures `outputs/policy.json`, uploads it even when local policy fails, and then fails the job based on `summary.policy`, see diff --git a/tools/sbom-diff-and-risk/docs/policy-decision-explainability.md b/tools/sbom-diff-and-risk/docs/policy-decision-explainability.md index 4ebc98c..47550ac 100644 --- a/tools/sbom-diff-and-risk/docs/policy-decision-explainability.md +++ b/tools/sbom-diff-and-risk/docs/policy-decision-explainability.md @@ -84,6 +84,10 @@ Use `summary.policy` for compact counts and status. Use policy finding explanation fields when a reviewer needs to inspect why the status was `warn` or `fail`. +For small reviewer-facing decision fixtures covering `pass`, `warn`, `fail`, +and consumer-side `needs-review`, see +[examples/policy-decisions](../examples/policy-decisions/README.md). + ## Compatibility notes - The fields are additive JSON metadata for policy findings. diff --git a/tools/sbom-diff-and-risk/docs/reviewer-path.md b/tools/sbom-diff-and-risk/docs/reviewer-path.md index dc8f63b..4cb51f8 100644 --- a/tools/sbom-diff-and-risk/docs/reviewer-path.md +++ b/tools/sbom-diff-and-risk/docs/reviewer-path.md @@ -85,6 +85,7 @@ consumer workflow templates. They answer different review questions. | --- | --- | --- | --- | --- | | Default diff output | [sample-report.json](../examples/sample-report.json), [sample-summary.json](../examples/sample-summary.json), [sample-report.md](../examples/sample-report.md) | `scripts/regenerate-example-artifacts.py` with the CycloneDX example inputs | No network | Verify the default report, compact summary, and Markdown shapes. | | Policy output | [sample-policy-warn-report.json](../examples/sample-policy-warn-report.json), [sample-policy-fail-report.json](../examples/sample-policy-fail-report.json), [sample-policy.json](../examples/sample-policy.json) | `scripts/regenerate-example-artifacts.py` with local policy files | No network | Review warning, blocking, and policy-only sidecar semantics. | +| Policy decision examples | [policy-decisions](../examples/policy-decisions/README.md) | Checked-in consumer examples | No network | Review `pass`, `warn`, `fail`, and consumer-side `needs-review` interpretations without treating them as dependency safety verdicts. | | Requirements output | [sample-requirements-report.json](../examples/sample-requirements-report.json), [sample-requirements-report.md](../examples/sample-requirements-report.md) | `scripts/regenerate-example-artifacts.py --only requirements` | No network | Verify the `requirements.txt` parser path and report shape. | | Strict-policy SARIF | [sample-sarif.sarif](../examples/sample-sarif.sarif) | `scripts/regenerate-example-artifacts.py --only sarif` with normalized source root | No network | Inspect conservative code-scanning output for selected findings. | | PyPI provenance snapshot | [sample-provenance-report.json](../examples/sample-provenance-report.json), [sample-provenance-report.md](../examples/sample-provenance-report.md), [sample-provenance-report.sarif](../examples/sample-provenance-report.sarif) | Focused golden tests with constructed provenance evidence | No live lookup while testing | Review opt-in provenance rendering and policy semantics, not current PyPI package truth. | diff --git a/tools/sbom-diff-and-risk/examples/policy-decisions/README.md b/tools/sbom-diff-and-risk/examples/policy-decisions/README.md new file mode 100644 index 0000000..5169f02 --- /dev/null +++ b/tools/sbom-diff-and-risk/examples/policy-decisions/README.md @@ -0,0 +1,26 @@ +# Policy decision examples + +These examples show four reviewer-facing policy decision outcomes for CI +consumers. They are small interpretation fixtures, not generated report +artifacts. + +Use them when wiring a job summary, dashboard, or review note that needs to +separate local policy status from dependency safety claims. + +| Example | Meaning | +| --- | --- | +| [pass.json](pass.json) | Local policy ran and produced no blocking or warning findings. | +| [warn.json](warn.json) | Local policy ran and produced warning findings only. | +| [fail.json](fail.json) | Local policy ran and produced one or more blocking findings. | +| [needs-review.json](needs-review.json) | A consumer cannot make a pass, warn, or fail statement from the available policy surface and should route the change to human review. | + +`pass`, `warn`, and `fail` mirror the supported `summary.policy.status` values +emitted by `sbom-diff-risk`. `needs-review` is a consumer interpretation, not a +runtime `summary.policy.status` value. + +## Boundaries + +- These examples do not resolve CVEs. +- These examples do not prove that a dependency is safe or unsafe. +- These examples do not add network behavior. +- These examples do not imply production PyPI publishing. diff --git a/tools/sbom-diff-and-risk/examples/policy-decisions/fail.json b/tools/sbom-diff-and-risk/examples/policy-decisions/fail.json new file mode 100644 index 0000000..b4df6a0 --- /dev/null +++ b/tools/sbom-diff-and-risk/examples/policy-decisions/fail.json @@ -0,0 +1,34 @@ +{ + "example_type": "policy-decision-example", + "review_decision": "fail", + "decision_source": "summary.policy", + "summary": { + "policy": { + "status": "fail", + "blocking": 1, + "warning": 0, + "suppressed": 0 + } + }, + "policy_findings": [ + { + "rule_id": "max_added_packages", + "level": "block", + "decision_reason": "added_package_count_exceeded_threshold", + "policy_rule": "max_added_packages", + "severity_source": "block_on", + "matched_threshold": 0, + "observed_value": 1, + "component_name": null + } + ], + "reviewer_interpretation": { + "safe_statement": "Local policy ran and produced blocking findings.", + "next_action": "Treat the CI gate as failed until the local policy finding is resolved or the policy is intentionally changed.", + "non_claims": [ + "not a CVE result", + "not a dependency safety verdict", + "not a production PyPI release claim" + ] + } +} diff --git a/tools/sbom-diff-and-risk/examples/policy-decisions/needs-review.json b/tools/sbom-diff-and-risk/examples/policy-decisions/needs-review.json new file mode 100644 index 0000000..6d1fcd3 --- /dev/null +++ b/tools/sbom-diff-and-risk/examples/policy-decisions/needs-review.json @@ -0,0 +1,18 @@ +{ + "example_type": "policy-decision-example", + "review_decision": "needs-review", + "decision_source": "consumer_interpretation", + "summary": {}, + "policy_findings": [], + "human_review_reason": "No summary.policy status is present, so the consumer cannot honestly call the local policy result pass, warn, or fail.", + "reviewer_interpretation": { + "safe_statement": "The available policy surface is insufficient for an automated pass, warn, or fail statement.", + "next_action": "Route the change to human review or rerun with an explicit local policy.", + "non_claims": [ + "not a runtime summary.policy.status value", + "not a CVE result", + "not a dependency safety verdict", + "not a production PyPI release claim" + ] + } +} diff --git a/tools/sbom-diff-and-risk/examples/policy-decisions/pass.json b/tools/sbom-diff-and-risk/examples/policy-decisions/pass.json new file mode 100644 index 0000000..64c8c84 --- /dev/null +++ b/tools/sbom-diff-and-risk/examples/policy-decisions/pass.json @@ -0,0 +1,23 @@ +{ + "example_type": "policy-decision-example", + "review_decision": "pass", + "decision_source": "summary.policy", + "summary": { + "policy": { + "status": "pass", + "blocking": 0, + "warning": 0, + "suppressed": 0 + } + }, + "policy_findings": [], + "reviewer_interpretation": { + "safe_statement": "Local policy ran and produced no blocking or warning findings.", + "next_action": "Continue normal review of the underlying dependency diff.", + "non_claims": [ + "not a CVE result", + "not a dependency safety verdict", + "not a production PyPI release claim" + ] + } +} diff --git a/tools/sbom-diff-and-risk/examples/policy-decisions/warn.json b/tools/sbom-diff-and-risk/examples/policy-decisions/warn.json new file mode 100644 index 0000000..b963f47 --- /dev/null +++ b/tools/sbom-diff-and-risk/examples/policy-decisions/warn.json @@ -0,0 +1,34 @@ +{ + "example_type": "policy-decision-example", + "review_decision": "warn", + "decision_source": "summary.policy", + "summary": { + "policy": { + "status": "warn", + "blocking": 0, + "warning": 1, + "suppressed": 0 + } + }, + "policy_findings": [ + { + "rule_id": "new_package", + "level": "warn", + "decision_reason": "risk_finding_matched_policy_rule", + "policy_rule": "new_package", + "severity_source": "warn_on", + "matched_threshold": null, + "observed_value": "new_package", + "component_name": "urllib3" + } + ], + "reviewer_interpretation": { + "safe_statement": "Local policy ran and produced warning findings, but no blocking findings.", + "next_action": "Review the warning findings before treating the change as acceptable.", + "non_claims": [ + "not a CVE result", + "not a dependency safety verdict", + "not a production PyPI release claim" + ] + } +} diff --git a/tools/sbom-diff-and-risk/tests/test_policy_decision_examples.py b/tools/sbom-diff-and-risk/tests/test_policy_decision_examples.py new file mode 100644 index 0000000..4e445f5 --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_policy_decision_examples.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json +from pathlib import Path + + +EXAMPLES_DIR = Path(__file__).resolve().parents[1] / "examples" / "policy-decisions" +DECISION_FILES = { + "pass": "pass.json", + "warn": "warn.json", + "fail": "fail.json", + "needs-review": "needs-review.json", +} +RUNTIME_POLICY_STATUSES = {"pass", "warn", "fail"} +COMMON_NON_CLAIMS = { + "not a CVE result", + "not a dependency safety verdict", + "not a production PyPI release claim", +} + + +def test_policy_decision_examples_are_complete() -> None: + expected_files = {"README.md", *DECISION_FILES.values()} + actual_files = {path.name for path in EXAMPLES_DIR.iterdir() if path.is_file()} + + assert expected_files <= actual_files + + +def test_policy_decision_examples_have_expected_shape() -> None: + for decision, filename in DECISION_FILES.items(): + payload = _read_example(filename) + + assert payload["example_type"] == "policy-decision-example" + assert payload["review_decision"] == decision + interpretation = payload["reviewer_interpretation"] + assert interpretation["safe_statement"] + assert interpretation["next_action"] + assert COMMON_NON_CLAIMS <= set(interpretation["non_claims"]) + + +def test_pass_warn_fail_examples_mirror_runtime_policy_statuses() -> None: + for decision in sorted(RUNTIME_POLICY_STATUSES): + payload = _read_example(DECISION_FILES[decision]) + policy_summary = payload["summary"]["policy"] + + assert payload["decision_source"] == "summary.policy" + assert policy_summary["status"] == decision + + if decision == "pass": + assert policy_summary == { + "status": "pass", + "blocking": 0, + "warning": 0, + "suppressed": 0, + } + assert payload["policy_findings"] == [] + elif decision == "warn": + assert policy_summary["blocking"] == 0 + assert policy_summary["warning"] > 0 + assert all(finding["level"] == "warn" for finding in payload["policy_findings"]) + else: + assert policy_summary["blocking"] > 0 + assert any(finding["level"] == "block" for finding in payload["policy_findings"]) + + +def test_needs_review_is_consumer_interpretation_not_runtime_status() -> None: + payload = _read_example("needs-review.json") + + assert payload["review_decision"] == "needs-review" + assert payload["decision_source"] == "consumer_interpretation" + assert "policy" not in payload["summary"] + assert payload["review_decision"] not in RUNTIME_POLICY_STATUSES + assert "not a runtime summary.policy.status value" in payload["reviewer_interpretation"]["non_claims"] + + +def _read_example(filename: str) -> dict[str, object]: + return json.loads((EXAMPLES_DIR / filename).read_text(encoding="utf-8"))