diff --git a/.github/workflows/sbom-diff-and-risk-ci.yml b/.github/workflows/sbom-diff-and-risk-ci.yml index fab26ae..6c7df1f 100644 --- a/.github/workflows/sbom-diff-and-risk-ci.yml +++ b/.github/workflows/sbom-diff-and-risk-ci.yml @@ -196,6 +196,9 @@ jobs: if [[ "${RELEASE_TAG}" == *"rc"* ]]; then create_args+=(--prerelease --latest=false) edit_args+=(--prerelease) + else + create_args+=(--latest) + edit_args+=(--latest) fi if [ -f "${notes_file}" ]; then diff --git a/README.md b/README.md index d2b8705..b46416c 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # scientific-computing-toolkit -This repository is a portfolio space for scientific-computing infrastructure, -systems tooling, and supply-chain-security experiments that favor deterministic -behavior, auditable outputs, and clear release evidence. +This repository centers on a release-facing supply-chain review tool that +favors deterministic behavior, auditable outputs, and clear release evidence. +The scientific diagnostics projects remain bounded supporting material. ## Current Flagship Tool @@ -11,6 +11,14 @@ current flagship tool. It compares SBOMs and dependency manifests, produces JSON, Markdown, and SARIF review artifacts, supports local policy checks, and can optionally record PyPI provenance and OpenSSF Scorecard evidence. +### Supply-chain non-claims + +- not a CVE scanner +- not a malware scanner +- not a package safety verdict engine + +These boundaries apply to the CLI, its reports, and its policy decisions. + For the clearest reviewer route, start with the [`sbom-diff-and-risk` reviewer path](tools/sbom-diff-and-risk/docs/reviewer-path.md); it separates orientation, artifact inspection, local reproduction, and release @@ -72,7 +80,8 @@ Project: [`sbom-diff-and-risk`](tools/sbom-diff-and-risk/README.md) Status: -Released at `v1.0-rc.1` as a release candidate. +Main is the `v1.0.0` final candidate. The currently published release remains +`v1.0-rc.1` until the final tag is explicitly approved and created. What to review: Deterministic SBOM/dependency diffing, JSON/Markdown/SARIF output, local policy @@ -86,6 +95,7 @@ Useful entry points: - [Reviewer path](tools/sbom-diff-and-risk/docs/reviewer-path.md) - [Reviewer brief](tools/sbom-diff-and-risk/docs/reviewer-brief.md) - [Reviewer evidence pack](tools/sbom-diff-and-risk/docs/reviewer-evidence-pack.md) +- [v1.0.0 release notes draft][release-notes-v100] - [v1.0-rc.1 release notes][release-notes-v10rc1] - [Examples](tools/sbom-diff-and-risk/examples/) @@ -212,10 +222,13 @@ the review question: ## Status -- Current flagship release candidate: `sbom-diff-and-risk` `v1.0-rc.1` +- Current published flagship release candidate: `sbom-diff-and-risk` `v1.0-rc.1` - GitHub Release assets: available for `v1.0-rc.1` - GitHub Latest may still show `v0.9.0` because `v1.0-rc.1` is a release candidate. +- Next release policy: GitHub `v1.0.0` final, with production PyPI still deferred +- Main package metadata: `1.0.0` final candidate - TestPyPI Trusted Publishing dry-run: completed - Production PyPI publishing: intentionally deferred +[release-notes-v100]: tools/sbom-diff-and-risk/RELEASE_NOTES_v1.0.0.md [release-notes-v10rc1]: tools/sbom-diff-and-risk/RELEASE_NOTES_v1.0-rc.1.md diff --git a/scripts/validate-reviewer-routes.py b/scripts/validate-reviewer-routes.py index 1e23a85..fe72142 100644 --- a/scripts/validate-reviewer-routes.py +++ b/scripts/validate-reviewer-routes.py @@ -19,6 +19,7 @@ Path("tools/sbom-diff-and-risk/docs/report-schema.md"), Path("tools/sbom-diff-and-risk/docs/github-actions-consumer-example.md"), Path("tools/sbom-diff-and-risk/docs/policy-warning-reviewer-case.md"), + Path("tools/sbom-diff-and-risk/docs/release-provenance.md"), Path("tools/sbom-diff-and-risk/docs/reviewer-path.md"), Path("projects/precipitation-anomaly-diagnostics/docs/reviewer-path.md"), Path("projects/precipitation-anomaly-diagnostics-lab/docs/reviewer-path.md"), @@ -95,6 +96,11 @@ "tools/sbom-diff-and-risk/examples/policy-minimal.yml", "tools/sbom-diff-and-risk/examples/sample-policy-warn-report.json", "tools/sbom-diff-and-risk/examples/sample-policy-warn-report.md", + "tools/sbom-diff-and-risk/examples/sample-policy-warn-report.sarif", + }, + Path("tools/sbom-diff-and-risk/docs/release-provenance.md"): { + "tools/sbom-diff-and-risk/docs/pypi-production-publishing-decision.md", + "tools/sbom-diff-and-risk/docs/verification.md", }, Path("tools/sbom-diff-and-risk/docs/reviewer-path.md"): { ".github/workflows/reviewer-route-contract-ci.yml", @@ -109,6 +115,7 @@ "tools/sbom-diff-and-risk/examples/sample-report.json", "tools/sbom-diff-and-risk/examples/sample-summary.json", "tools/sbom-diff-and-risk/examples/sample-policy.json", + "tools/sbom-diff-and-risk/examples/sample-policy-warn-report.sarif", "tools/sbom-diff-and-risk/examples/sample-sarif.sarif", }, Path("projects/precipitation-anomaly-diagnostics/docs/reviewer-path.md"): { @@ -164,6 +171,9 @@ REQUIRED_TEXT = { Path("README.md"): ( "current flagship tool", + "not a CVE scanner", + "not a malware scanner", + "not a package safety verdict engine", "not part of the `sbom-diff-and-risk` release surface", "why the scientific-computing background helps", "GitHub Latest may still show `v0.9.0` because `v1.0-rc.1` is a release candidate.", @@ -223,8 +233,8 @@ "local_manifest_only", "sbom_present", "policy_matched", - "enrichment_recorded", "provenance_recorded", + "scorecard_recorded", "not a package safety verdict", "not a CVE result", ), @@ -248,10 +258,19 @@ "risk_finding_matched_policy_rule", "severity_source", "warn_on", + "sample-policy-warn-report.sarif", + "sdr.new_package", + "policy_level", "not a package safety verdict", "any CVE result", "any malware verdict", ), + Path("tools/sbom-diff-and-risk/docs/release-provenance.md"): ( + "publish `v1.0.0` as the stable GitHub Release contract", + "production PyPI publishing deferred", + "marks a final tag such as `v1.0.0` as GitHub Latest", + "workflow artifact attestation", + ), Path("tools/sbom-diff-and-risk/docs/reviewer-path.md"): ( "Artifact evidence map", "Reviewer route contract", diff --git a/tools/sbom-diff-and-risk/README.md b/tools/sbom-diff-and-risk/README.md index 5240ff8..10d5983 100644 --- a/tools/sbom-diff-and-risk/README.md +++ b/tools/sbom-diff-and-risk/README.md @@ -1,16 +1,20 @@ # sbom-diff-and-risk -v1.0-rc.1 is the Policy Evidence release candidate. It fixes the reviewer path -for policy decision examples, evidence-confidence labels, a one-page policy -warning case, and copyable GitHub Actions consumer guidance. It keeps -dependency analysis local and deterministic by default, preserves the completed -TestPyPI dry-run story, and keeps production PyPI publishing intentionally -deferred. +Main is the `v1.0.0` Policy Evidence final candidate. The currently published +release remains `v1.0-rc.1` until the final tag is explicitly approved and +created. The selected release policy is a stable GitHub `v1.0.0` release while +production PyPI publishing remains intentionally deferred. `sbom-diff-and-risk` is a local, deterministic CLI for comparing two SBOMs or dependency manifests and producing JSON plus Markdown reports. It uses conservative heuristics for change intelligence. By default it does not resolve CVEs, does not act as a reputation oracle, and does not perform hidden network enrichment. +## Supply-chain non-claims + +- not a CVE scanner +- not a malware scanner +- not a package safety verdict engine + ## Start Here For the clearest reviewer route, start with diff --git a/tools/sbom-diff-and-risk/RELEASE_NOTES_v1.0.0.md b/tools/sbom-diff-and-risk/RELEASE_NOTES_v1.0.0.md new file mode 100644 index 0000000..a9cce0b --- /dev/null +++ b/tools/sbom-diff-and-risk/RELEASE_NOTES_v1.0.0.md @@ -0,0 +1,58 @@ +# sbom-diff-and-risk v1.0.0 + +`v1.0.0` is the Policy Evidence release. + +The Python package metadata version for this tag is `1.0.0`. + +## Theme + +A stable, bounded release surface for dependency diff and local policy evidence. + +## Highlights + +- Provides minimal policy decision examples for `pass`, `warn`, `fail`, and + consumer-side `needs-review`. +- Stabilizes `evidence_confidence` as `local_manifest_only`, `sbom_present`, + `policy_matched`, `provenance_recorded`, or `scorecard_recorded`. +- Fixes one reviewer case from dependency diff to `new_package` policy warning + across JSON, Markdown, and SARIF artifacts. +- Records policy match metadata in direct SARIF risk results, including the + local policy level and rule ID. +- Keeps the three supply-chain non-claims visible: not a CVE scanner, not a + malware scanner, and not a package safety verdict engine. +- Keeps the minimal GitHub Actions consumer path for running policy, uploading + `policy.json`, and enforcing the tool's exit code. + +## Release policy + +- `v1.0.0` is the stable GitHub Release contract. +- GitHub Release assets are the distribution surface for this release. +- Production PyPI publishing remains intentionally deferred. +- The TestPyPI Trusted Publishing dry-run remains evidence of the test + publishing path only. + +## Compatibility and boundaries + +- Default analysis remains local-file based and deterministic. +- No default network enrichment was added. +- No CVE lookup, advisory resolution, malware scanning, or package safety + verdict was added. +- Policy warnings and failures remain bounded local policy decisions. +- `enrichment_recorded` was an rc label and is replaced by the more specific + `scorecard_recorded` label in the final contract. + +## Release evidence + +The tag-gated GitHub Actions workflow builds the wheel and source distribution, +generates a SHA256 checksum manifest, records workflow artifact attestations, +and publishes the same built files as GitHub Release assets. Final tags are +explicitly marked as GitHub Latest; rc tags remain prereleases. + +Expected assets: + +- `sbom_diff_and_risk-1.0.0-py3-none-any.whl` +- `sbom_diff_and_risk-1.0.0.tar.gz` +- `sbom-diff-and-risk-SHA256SUMS.txt` + +Use `docs/verification.md`, `docs/release-provenance.md`, and +`docs/self-provenance.md` for the correct verification path. diff --git a/tools/sbom-diff-and-risk/docs/example-artifact-regeneration.md b/tools/sbom-diff-and-risk/docs/example-artifact-regeneration.md index 245ea47..3711e75 100644 --- a/tools/sbom-diff-and-risk/docs/example-artifact-regeneration.md +++ b/tools/sbom-diff-and-risk/docs/example-artifact-regeneration.md @@ -69,7 +69,8 @@ python scripts/regenerate-example-artifacts.py --check --only requirements ## Boundaries The regeneration script covers no-network JSON, Markdown, summary, policy -sidecar, and strict-policy SARIF examples produced through the public CLI. +sidecar, policy-warning SARIF, and strict-policy SARIF examples produced +through the public CLI. It does not perform PyPI or Scorecard enrichment, does not call external services, and does not make dependency safety claims. Provenance-aware, diff --git a/tools/sbom-diff-and-risk/docs/github-actions-consumer-example.md b/tools/sbom-diff-and-risk/docs/github-actions-consumer-example.md index cbbebe4..a3cd432 100644 --- a/tools/sbom-diff-and-risk/docs/github-actions-consumer-example.md +++ b/tools/sbom-diff-and-risk/docs/github-actions-consumer-example.md @@ -12,6 +12,10 @@ Production PyPI publishing is intentionally deferred, so consumers should not install `sbom-diff-and-risk` from production PyPI. Use a GitHub Release asset or a local checkout instead. +The checked-in workflow pins the selected stable contract, `v1.0.0`. Before +that final release is published, use the currently available `v1.0-rc.1` +release asset instead. + ## Minimal policy workflow Replace the placeholder input and policy paths with files from the consumer @@ -47,15 +51,15 @@ jobs: GH_TOKEN: ${{ github.token }} run: | mkdir -p .tooling/sbom-diff-risk - gh release download v1.0-rc.1 \ + gh release download v1.0.0 \ --repo stacknil/scientific-computing-toolkit \ - --pattern "sbom_diff_and_risk-1.0rc1-py3-none-any.whl" \ + --pattern "sbom_diff_and_risk-1.0.0-py3-none-any.whl" \ --dir .tooling/sbom-diff-risk - name: Install sbom-diff-risk run: | python -m pip install \ - .tooling/sbom-diff-risk/sbom_diff_and_risk-1.0rc1-py3-none-any.whl + .tooling/sbom-diff-risk/sbom_diff_and_risk-1.0.0-py3-none-any.whl - name: Run dependency policy id: compare diff --git a/tools/sbom-diff-and-risk/docs/policy-warning-reviewer-case.md b/tools/sbom-diff-and-risk/docs/policy-warning-reviewer-case.md index 9d3b866..aade426 100644 --- a/tools/sbom-diff-and-risk/docs/policy-warning-reviewer-case.md +++ b/tools/sbom-diff-and-risk/docs/policy-warning-reviewer-case.md @@ -17,7 +17,8 @@ sbom-diff-risk compare ` --after examples/cdx_after.json ` --policy examples/policy-minimal.yml ` --out-json outputs/policy-warn-report.json ` - --out-md outputs/policy-warn-report.md + --out-md outputs/policy-warn-report.md ` + --out-sarif outputs/policy-warn-report.sarif ``` Reference inputs and policy: @@ -42,6 +43,7 @@ The checked-in reference outputs are: - [`examples/sample-policy-warn-report.json`](../examples/sample-policy-warn-report.json) - [`examples/sample-policy-warn-report.md`](../examples/sample-policy-warn-report.md) +- [`examples/sample-policy-warn-report.sarif`](../examples/sample-policy-warn-report.sarif) The fixed JSON facts for this case are: @@ -59,6 +61,16 @@ The fixed JSON facts for this case are: | `warning_findings[0].severity_source` | `warn_on` | | `warning_findings[0].observed_value` | `new_package` | +The fixed SARIF facts for the same decision are: + +| Field | Value | +| --- | --- | +| `runs[0].results[0].ruleId` | `sdr.new_package` | +| `runs[0].results[0].level` | `warning` | +| `runs[0].results[0].properties.component_name` | `urllib3` | +| `runs[0].results[0].properties.policy_matched` | `true` | +| `runs[0].results[0].properties.policy_level` | `warn` | + ## Explanation The after input contains `urllib3` `2.2.1`, which is not present in the before @@ -73,10 +85,12 @@ source of this policy warning. It receives `version_change_unclassified` and `not_evaluated` findings in offline mode, and the minimal policy does not warn or block on either rule. -## Fixed boundary +## Bounded decision This case proves that local policy matching can explain why a dependency diff -triggered a warning. +triggered a warning and can preserve that decision in Markdown and SARIF. The +bounded decision is: review the added dependency under local policy; do not +convert the warning into a security verdict. It does not prove: diff --git a/tools/sbom-diff-and-risk/docs/release-provenance.md b/tools/sbom-diff-and-risk/docs/release-provenance.md index 5097678..475c09a 100644 --- a/tools/sbom-diff-and-risk/docs/release-provenance.md +++ b/tools/sbom-diff-and-risk/docs/release-provenance.md @@ -9,6 +9,20 @@ This document is about the second surface: verifying a GitHub Release and a down This page is only about the `sbom-diff-and-risk` tool's own GitHub Releases. If you want the quick "which verification page do I need?" guide, start with [verification.md](verification.md). +## v1.0 release policy + +Chosen policy: publish `v1.0.0` as the stable GitHub Release contract while +keeping production PyPI publishing deferred. + +For `v1.0.0`, stability applies to the CLI, report schemas, policy decision +examples, evidence-confidence labels, reviewer case, and GitHub Release asset +verification path. It does not claim production PyPI availability. Until the +final tag is explicitly approved and created, `v1.0-rc.1` remains the current +published release candidate. + +The tag workflow marks rc tags as prereleases and not Latest. It explicitly +marks a final tag such as `v1.0.0` as GitHub Latest. + Release assets produced by the updated workflow also include a deterministic SHA256 checksum manifest named `sbom-diff-and-risk-SHA256SUMS.txt`. The manifest is written with filenames sorted in a stable order. It is not a separate provenance system; it is a local byte-integrity check that helps reviewers confirm downloaded wheel and source distribution files match the hashes published with the same GitHub Release. ## What the release workflow now does diff --git a/tools/sbom-diff-and-risk/docs/report-schema.md b/tools/sbom-diff-and-risk/docs/report-schema.md index 4fd20bb..f4f792d 100644 --- a/tools/sbom-diff-and-risk/docs/report-schema.md +++ b/tools/sbom-diff-and-risk/docs/report-schema.md @@ -130,8 +130,13 @@ not a CVE result. The same value appears at top level, in `summary`, and in | `local_manifest_only` | The report was produced from local manifest-style inputs without SBOM input, policy matches, or enrichment evidence. | | `sbom_present` | At least one input is an SBOM format such as CycloneDX JSON or SPDX JSON. | | `policy_matched` | Local policy evaluation produced at least one blocking, warning, or suppressed policy match. | -| `enrichment_recorded` | Optional non-provenance enrichment evidence, such as OpenSSF Scorecard evidence, was recorded for the report. | | `provenance_recorded` | PyPI provenance evidence or provenance-enrichment metadata was recorded for the report. | +| `scorecard_recorded` | OpenSSF Scorecard evidence or Scorecard-enrichment metadata was recorded for the report. | + +When both provenance and Scorecard evidence are recorded, the single summary +label is deterministically `scorecard_recorded`. This precedence keeps the +field stable; it does not rank package safety or claim that one evidence source +is universally stronger than another. `summary.policy` appears only when a policy is applied. Absence of `summary.policy` means policy was not used, not that policy evaluation failed. diff --git a/tools/sbom-diff-and-risk/docs/reviewer-brief.md b/tools/sbom-diff-and-risk/docs/reviewer-brief.md index 6c2e5dc..444117a 100644 --- a/tools/sbom-diff-and-risk/docs/reviewer-brief.md +++ b/tools/sbom-diff-and-risk/docs/reviewer-brief.md @@ -6,6 +6,9 @@ Current released version: `v1.0-rc.1` release candidate. +Main is the `v1.0.0` final candidate. The chosen release policy is a stable +GitHub final release while production PyPI remains deferred. + For the shortest ordered review route, use [reviewer-path.md](reviewer-path.md). @@ -32,7 +35,7 @@ Dependency review often needs evidence that is stable enough for code review, CI | How can a reviewer reproduce the core evidence? | [reviewer-evidence-pack.md](reviewer-evidence-pack.md) for demo, release, TestPyPI, and SARIF verification paths. | | What is the stable JSON shape? | [report-schema.md](report-schema.md) documents the machine-readable report structure and `summary` contract. | | How are policy findings explained? | [policy-decision-explainability.md](policy-decision-explainability.md) documents the policy decision metadata in JSON reports. | -| How does one dependency diff trigger a policy warning? | [policy-warning-reviewer-case.md](policy-warning-reviewer-case.md) fixes the input, output, explanation, and boundary for a `new_package` warning. | +| How does one dependency diff trigger a policy warning? | [policy-warning-reviewer-case.md](policy-warning-reviewer-case.md) fixes the input, JSON/Markdown/SARIF output, explanation, and boundary for a `new_package` warning. | | Can CI consume compact policy decisions? | [policy-decision-ci-cookbook.md](policy-decision-ci-cookbook.md), [examples/sample-policy.json](../examples/sample-policy.json), and [examples/github-actions-policy-consumer.yml](../examples/github-actions-policy-consumer.yml) show the policy sidecar path. | | Are default runs offline? | CLI docs, tests for no-enrichment behavior, and explicit enrichment flags. | | Can code scanning consume the output? | `docs/github-code-scanning.md` and `examples/sample-sarif.sarif`. | diff --git a/tools/sbom-diff-and-risk/docs/reviewer-evidence-pack.md b/tools/sbom-diff-and-risk/docs/reviewer-evidence-pack.md index 0653024..3ba685b 100644 --- a/tools/sbom-diff-and-risk/docs/reviewer-evidence-pack.md +++ b/tools/sbom-diff-and-risk/docs/reviewer-evidence-pack.md @@ -134,8 +134,8 @@ and suppressions. For a fixed one-page policy warning case, see [policy-warning-reviewer-case.md](policy-warning-reviewer-case.md). It traces the checked-in CycloneDX example pair and `policy-minimal.yml` from added -dependency to `new_package` warning without turning the warning into a package -safety verdict. +dependency to a `new_package` warning in JSON, Markdown, and SARIF without +turning the warning into a package safety verdict. For CI job-summary examples that consume policy decision metadata, see [policy-decision-ci-cookbook.md](policy-decision-ci-cookbook.md). @@ -255,6 +255,7 @@ The SARIF output is designed for GitHub code scanning consumption. Start with: The SARIF renderer intentionally emits a conservative subset: - selected heuristic findings such as suspicious source, unknown license, and major upgrade +- policy-matched `new_package` warnings with explicit policy metadata - explicit blocking policy decisions - selected provenance or Scorecard policy violations when policy turns them into findings diff --git a/tools/sbom-diff-and-risk/docs/reviewer-path.md b/tools/sbom-diff-and-risk/docs/reviewer-path.md index 1a4ee85..48874cb 100644 --- a/tools/sbom-diff-and-risk/docs/reviewer-path.md +++ b/tools/sbom-diff-and-risk/docs/reviewer-path.md @@ -57,7 +57,8 @@ Inspect the checked-in examples in this order: | 3 | [sample-policy.json](../examples/sample-policy.json) | Policy-only sidecar for CI consumers. | | 4 | [sample-report.md](../examples/sample-report.md) | Human-readable reviewer report. | | 5 | [sample-sarif.sarif](../examples/sample-sarif.sarif) | Conservative code-scanning output for selected high-signal findings. | -| 6 | [github-actions-policy-consumer.yml](../examples/github-actions-policy-consumer.yml) | Copyable consumer path for capturing policy JSON in GitHub Actions. | +| 6 | [sample-policy-warn-report.sarif](../examples/sample-policy-warn-report.sarif) | The fixed `new_package` policy warning represented as SARIF. | +| 7 | [github-actions-policy-consumer.yml](../examples/github-actions-policy-consumer.yml) | Copyable consumer path for capturing policy JSON in GitHub Actions. | Then read: @@ -202,6 +203,10 @@ Release status note: GitHub Latest may still show `v0.9.0` because `v1.0-rc.1` is a release candidate. Reviewers checking the current release candidate should open the explicit `v1.0-rc.1` release page. +Main is prepared as the `v1.0.0` final candidate. The selected policy is a +stable GitHub final release with production PyPI still deferred; this is not a +claim that the final tag already exists. + | Evidence surface | Use when | Read | | --- | --- | --- | | Verification decision guide | You need to choose the right release verification path. | [verification.md](verification.md) | diff --git a/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md b/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md index ca4bf77..bec859a 100644 --- a/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md +++ b/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md @@ -81,7 +81,7 @@ if ($newPackageCount -gt $maxNewPackages) { - `summary.enrichment` appears only when PyPI or Scorecard enrichment is used. - `summary.evidence_confidence` is always present and can be `local_manifest_only`, `sbom_present`, `policy_matched`, - `enrichment_recorded`, or `provenance_recorded`. + `provenance_recorded`, or `scorecard_recorded`. - `unchanged` is absent because unchanged components are not modeled. - Absence of `summary.policy` or `summary.enrichment` means the feature was not used, not that it failed. diff --git a/tools/sbom-diff-and-risk/examples/github-actions-policy-consumer.yml b/tools/sbom-diff-and-risk/examples/github-actions-policy-consumer.yml index 34e286b..491a87c 100644 --- a/tools/sbom-diff-and-risk/examples/github-actions-policy-consumer.yml +++ b/tools/sbom-diff-and-risk/examples/github-actions-policy-consumer.yml @@ -29,15 +29,15 @@ jobs: GH_TOKEN: ${{ github.token }} run: | mkdir -p .tooling/sbom-diff-risk - gh release download v1.0-rc.1 \ + gh release download v1.0.0 \ --repo stacknil/scientific-computing-toolkit \ - --pattern "sbom_diff_and_risk-1.0rc1-py3-none-any.whl" \ + --pattern "sbom_diff_and_risk-1.0.0-py3-none-any.whl" \ --dir .tooling/sbom-diff-risk - name: Install sbom-diff-risk run: | python -m pip install \ - .tooling/sbom-diff-risk/sbom_diff_and_risk-1.0rc1-py3-none-any.whl + .tooling/sbom-diff-risk/sbom_diff_and_risk-1.0.0-py3-none-any.whl - name: Run dependency policy id: compare diff --git a/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.sarif b/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.sarif new file mode 100644 index 0000000..ad993ea --- /dev/null +++ b/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.sarif @@ -0,0 +1,103 @@ +{ + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": "sbom-diff-risk", + "fullName": "sbom-diff-risk", + "version": "1.0.0", + "semanticVersion": "1.0.0", + "rules": [ + { + "id": "sdr.new_package", + "name": "new_package", + "shortDescription": { + "text": "Component is present only in the after input." + }, + "fullDescription": { + "text": "Component is present only in the after input." + }, + "defaultConfiguration": { + "level": "warning" + }, + "properties": { + "tags": [ + "supply-chain", + "sbom" + ] + } + } + ] + } + }, + "artifacts": [ + { + "location": { + "uri": "examples/cdx_before.json", + "uriBaseId": "%SRCROOT%" + } + }, + { + "location": { + "uri": "examples/cdx_after.json", + "uriBaseId": "%SRCROOT%" + } + } + ], + "properties": { + "sbom_diff_risk": { + "result_limit": 5000, + "total_candidate_results": 1, + "emitted_results": 1, + "omitted_results": 0, + "truncated": false, + "prioritization": "error results first, then warning, then note; direct mapped findings before policy-only checks; stable rule priority and component key tie-breakers.", + "warning": null + } + }, + "results": [ + { + "ruleId": "sdr.new_package", + "level": "warning", + "message": { + "text": "Policy warning: Component was not present in the before input." + }, + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "examples/cdx_after.json", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 1 + } + } + } + ], + "partialFingerprints": { + "ruleId": "sdr.new_package", + "componentKey": "purl:pkg:pypi/urllib3" + }, + "properties": { + "component_key": "purl:pkg:pypi/urllib3", + "component_name": "urllib3", + "finding_bucket": "new_package", + "policy_matched": true, + "policy_blocking": false, + "result_kind": "risk_finding", + "policy_rule_id": "new_package", + "policy_level": "warn" + } + } + ], + "originalUriBaseIds": { + "%SRCROOT%": { + "uri": "file:///__PROJECT_ROOT__/" + } + } + } + ] +} diff --git a/tools/sbom-diff-and-risk/examples/sample-provenance-report.sarif b/tools/sbom-diff-and-risk/examples/sample-provenance-report.sarif index 163ceb0..0e1b4ee 100644 --- a/tools/sbom-diff-and-risk/examples/sample-provenance-report.sarif +++ b/tools/sbom-diff-and-risk/examples/sample-provenance-report.sarif @@ -7,8 +7,8 @@ "driver": { "name": "sbom-diff-risk", "fullName": "sbom-diff-risk", - "version": "1.0rc1", - "semanticVersion": "1.0rc1", + "version": "1.0.0", + "semanticVersion": "1.0.0", "rules": [ { "id": "sdr.policy_violation.provenance_required", diff --git a/tools/sbom-diff-and-risk/examples/sample-sarif.sarif b/tools/sbom-diff-and-risk/examples/sample-sarif.sarif index 6dfc8d9..5497c36 100644 --- a/tools/sbom-diff-and-risk/examples/sample-sarif.sarif +++ b/tools/sbom-diff-and-risk/examples/sample-sarif.sarif @@ -7,8 +7,8 @@ "driver": { "name": "sbom-diff-risk", "fullName": "sbom-diff-risk", - "version": "1.0rc1", - "semanticVersion": "1.0rc1", + "version": "1.0.0", + "semanticVersion": "1.0.0", "rules": [ { "id": "sdr.major_upgrade", @@ -29,6 +29,25 @@ ] } }, + { + "id": "sdr.new_package", + "name": "new_package", + "shortDescription": { + "text": "Component is present only in the after input." + }, + "fullDescription": { + "text": "Component is present only in the after input." + }, + "defaultConfiguration": { + "level": "warning" + }, + "properties": { + "tags": [ + "supply-chain", + "sbom" + ] + } + }, { "id": "sdr.policy_violation.allow_sources", "name": "policy_violation.allow_sources", @@ -125,8 +144,8 @@ "properties": { "sbom_diff_risk": { "result_limit": 5000, - "total_candidate_results": 5, - "emitted_results": 5, + "total_candidate_results": 6, + "emitted_results": 6, "omitted_results": 0, "truncated": false, "prioritization": "error results first, then warning, then note; direct mapped findings before policy-only checks; stable rule priority and component key tie-breakers.", @@ -161,8 +180,11 @@ "component_key": "purl:pkg:pypi/mystery-lib", "component_name": "mystery-lib", "finding_bucket": "suspicious_source", + "policy_matched": true, "policy_blocking": true, "result_kind": "risk_finding", + "policy_rule_id": "suspicious_source", + "policy_level": "block", "blocking_rule_id": "suspicious_source" } }, @@ -193,8 +215,11 @@ "component_key": "purl:pkg:pypi/mystery-lib", "component_name": "mystery-lib", "finding_bucket": "unknown_license", + "policy_matched": true, "policy_blocking": true, "result_kind": "risk_finding", + "policy_rule_id": "unknown_license", + "policy_level": "block", "blocking_rule_id": "unknown_license" } }, @@ -260,9 +285,9 @@ }, { "ruleId": "sdr.major_upgrade", - "level": "note", + "level": "warning", "message": { - "text": "Version changed from 1.9.0 to 2.0.0 with a higher major version." + "text": "Policy warning: Version changed from 1.9.0 to 2.0.0 with a higher major version." }, "locations": [ { @@ -285,8 +310,45 @@ "component_key": "purl:pkg:pypi/requests", "component_name": "requests", "finding_bucket": "major_upgrade", + "policy_matched": true, + "policy_blocking": false, + "result_kind": "risk_finding", + "policy_rule_id": "major_upgrade", + "policy_level": "warn" + } + }, + { + "ruleId": "sdr.new_package", + "level": "warning", + "message": { + "text": "Policy warning: Component was not present in the before input." + }, + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "examples/sarif_after.json", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 1 + } + } + } + ], + "partialFingerprints": { + "ruleId": "sdr.new_package", + "componentKey": "purl:pkg:pypi/mystery-lib" + }, + "properties": { + "component_key": "purl:pkg:pypi/mystery-lib", + "component_name": "mystery-lib", + "finding_bucket": "new_package", + "policy_matched": true, "policy_blocking": false, - "result_kind": "risk_finding" + "result_kind": "risk_finding", + "policy_rule_id": "new_package", + "policy_level": "warn" } } ], diff --git a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json index b0747ce..f70e4df 100644 --- a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json @@ -12,7 +12,7 @@ "suspicious_source": 0, "not_evaluated": 0 }, - "evidence_confidence": "enrichment_recorded", + "evidence_confidence": "scorecard_recorded", "policy": { "status": "warn", "blocking": 0, @@ -32,7 +32,7 @@ } } }, - "evidence_confidence": "enrichment_recorded", + "evidence_confidence": "scorecard_recorded", "components": { "added": [ { @@ -575,7 +575,7 @@ }, "exit_code": 0 }, - "evidence_confidence": "enrichment_recorded", + "evidence_confidence": "scorecard_recorded", "enrichment": { "mode": "opt_in_scorecard", "pypi_enabled": false, diff --git a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md index c53bdae..928d4e7 100644 --- a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md +++ b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md @@ -6,7 +6,7 @@ - Added: 2 - Removed: 0 - Version changes: 1 -- Evidence confidence: enrichment_recorded +- Evidence confidence: scorecard_recorded ## Risk buckets - new_package: 2 diff --git a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.sarif b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.sarif index 5f6de7b..afe83cb 100644 --- a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.sarif +++ b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.sarif @@ -7,8 +7,8 @@ "driver": { "name": "sbom-diff-risk", "fullName": "sbom-diff-risk", - "version": "1.0rc1", - "semanticVersion": "1.0rc1", + "version": "1.0.0", + "semanticVersion": "1.0.0", "rules": [ { "id": "sdr.policy_violation.scorecard_below_threshold", diff --git a/tools/sbom-diff-and-risk/pyproject.toml b/tools/sbom-diff-and-risk/pyproject.toml index 86da491..760921c 100644 --- a/tools/sbom-diff-and-risk/pyproject.toml +++ b/tools/sbom-diff-and-risk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sbom-diff-and-risk" -version = "1.0rc1" +version = "1.0.0" description = "Deterministic SBOM diff CLI with heuristic risk reporting." readme = { file = "PYPI_DESCRIPTION.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/tools/sbom-diff-and-risk/scripts/regenerate-example-artifacts.py b/tools/sbom-diff-and-risk/scripts/regenerate-example-artifacts.py index 66b14e8..84141e6 100644 --- a/tools/sbom-diff-and-risk/scripts/regenerate-example-artifacts.py +++ b/tools/sbom-diff-and-risk/scripts/regenerate-example-artifacts.py @@ -41,7 +41,7 @@ class ExampleArtifactSet: ), ExampleArtifactSet( slug="policy-warn", - name="warn-only policy report", + name="warn-only policy JSON, Markdown, and SARIF report", base_args=( "--before", "examples/cdx_before.json", @@ -53,7 +53,9 @@ class ExampleArtifactSet: outputs=( ("--out-json", "sample-policy-warn-report.json"), ("--out-md", "sample-policy-warn-report.md"), + ("--out-sarif", "sample-policy-warn-report.sarif"), ), + normalize_sarif_srcroot=True, ), ExampleArtifactSet( slug="policy-fail", @@ -226,7 +228,9 @@ def _run_artifact_set(project_root: Path, output_root: Path, artifact_set: Examp f"expected {artifact_set.expected_exit_codes}: {detail}" ) if artifact_set.normalize_sarif_srcroot: - _normalize_sarif_srcroot(output_root / artifact_set.outputs[0][1]) + for flag, output_name in artifact_set.outputs: + if flag == "--out-sarif": + _normalize_sarif_srcroot(output_root / output_name) def _normalize_sarif_srcroot(path: Path) -> None: diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/__init__.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/__init__.py index a7ed799..8606a37 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/__init__.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/__init__.py @@ -2,4 +2,4 @@ __all__ = ["__version__"] -__version__ = "1.0rc1" +__version__ = "1.0.0" diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py index 96d2c47..b761290 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py @@ -9,12 +9,12 @@ def evidence_confidence_for_report(report: CompareReport) -> EvidenceConfidence: if report.metadata.evidence_confidence is not None: return report.metadata.evidence_confidence + if _has_scorecard_record(report): + return EvidenceConfidence.SCORECARD_RECORDED + if _has_provenance_record(report): return EvidenceConfidence.PROVENANCE_RECORDED - if _has_enrichment_record(report): - return EvidenceConfidence.ENRICHMENT_RECORDED - if _has_policy_match(report): return EvidenceConfidence.POLICY_MATCHED @@ -36,7 +36,7 @@ def _has_provenance_record(report: CompareReport) -> bool: return any(component.provenance is not None for component in _all_components(report)) -def _has_enrichment_record(report: CompareReport) -> bool: +def _has_scorecard_record(report: CompareReport) -> bool: metadata = report.metadata.enrichment if metadata.scorecard_enabled: return True diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py index 9776145..ec46473 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py @@ -22,8 +22,8 @@ class EvidenceConfidence(StrEnum): LOCAL_MANIFEST_ONLY = "local_manifest_only" SBOM_PRESENT = "sbom_present" POLICY_MATCHED = "policy_matched" - ENRICHMENT_RECORDED = "enrichment_recorded" PROVENANCE_RECORDED = "provenance_recorded" + SCORECARD_RECORDED = "scorecard_recorded" class ProvenanceStatus(StrEnum): diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/report_sarif.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/report_sarif.py index 0a4f90d..4f2d00b 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/report_sarif.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/report_sarif.py @@ -17,6 +17,7 @@ ) _SARIF_SUPPORTED_RISK_BUCKETS = { + RiskBucket.NEW_PACKAGE, RiskBucket.SUSPICIOUS_SOURCE, RiskBucket.UNKNOWN_LICENSE, RiskBucket.MAJOR_UPGRADE, @@ -44,12 +45,13 @@ "sdr.suspicious_source": 0, "sdr.unknown_license": 1, "sdr.major_upgrade": 2, - "sdr.policy_violation.provenance_required": 3, - "sdr.policy_violation.unverified_provenance": 4, - "sdr.policy_violation.missing_attestation": 5, - "sdr.policy_violation.allow_sources": 6, - "sdr.policy_violation.max_added_packages": 7, - "sdr.policy_violation.scorecard_below_threshold": 8, + "sdr.new_package": 3, + "sdr.policy_violation.provenance_required": 4, + "sdr.policy_violation.unverified_provenance": 5, + "sdr.policy_violation.missing_attestation": 6, + "sdr.policy_violation.allow_sources": 7, + "sdr.policy_violation.max_added_packages": 8, + "sdr.policy_violation.scorecard_below_threshold": 9, } @@ -111,7 +113,8 @@ def render_report_sarif_output( resolved_base_dir = base_dir.resolve() if base_dir is not None else None policy_evaluation = effective_policy_evaluation(report.metadata.policy_evaluation) - blocking_map = _blocking_violation_map(policy_evaluation.blocking_violations) + blocking_map = _policy_violation_map(policy_evaluation.blocking_violations) + warning_map = _policy_violation_map(policy_evaluation.warning_violations) provenance_required_levels = _provenance_required_levels(policy_evaluation) emitted_blocking_keys: set[tuple[str, str | None]] = set() @@ -125,11 +128,14 @@ def render_report_sarif_output( policy_rule_id = _policy_rule_id_for_bucket(finding.bucket) blocking_violation = blocking_map.get((policy_rule_id, finding.component_key)) + policy_violation = blocking_violation or warning_map.get((policy_rule_id, finding.component_key)) + if finding.bucket is RiskBucket.NEW_PACKAGE and policy_violation is None: + continue result = _risk_finding_to_result( finding, after_path=after_path, base_dir=resolved_base_dir, - blocking_violation=blocking_violation, + policy_violation=policy_violation, ) candidate_results.append(result) if blocking_violation is not None: @@ -223,16 +229,16 @@ def _risk_finding_to_result( *, after_path: Path, base_dir: Path | None, - blocking_violation: PolicyViolation | None, + policy_violation: PolicyViolation | None, ) -> dict[str, Any]: rule_id = sarif_rule_id_for_risk_bucket(finding.bucket) assert rule_id is not None result: dict[str, Any] = { "ruleId": rule_id, - "level": _risk_result_level(finding.bucket, blocking_violation), + "level": _risk_result_level(finding.bucket, policy_violation), "message": { - "text": _risk_result_message(finding, blocking_violation), + "text": _risk_result_message(finding, policy_violation), }, "locations": [_file_location(after_path, base_dir)], "partialFingerprints": { @@ -243,12 +249,16 @@ def _risk_finding_to_result( "component_key": finding.component_key, "component_name": finding.component.name, "finding_bucket": finding.bucket.value, - "policy_blocking": blocking_violation is not None, + "policy_matched": policy_violation is not None, + "policy_blocking": policy_violation is not None and policy_violation.level is PolicyLevel.BLOCK, "result_kind": "risk_finding", }, } - if blocking_violation is not None: - result["properties"]["blocking_rule_id"] = blocking_violation.rule_id + if policy_violation is not None: + result["properties"]["policy_rule_id"] = policy_violation.rule_id + result["properties"]["policy_level"] = policy_violation.level.value + if policy_violation.level is PolicyLevel.BLOCK: + result["properties"]["blocking_rule_id"] = policy_violation.rule_id return result @@ -281,15 +291,17 @@ def _policy_violation_to_result( } -def _risk_result_level(bucket: RiskBucket, blocking_violation: PolicyViolation | None) -> str: - if blocking_violation is not None: - return "error" +def _risk_result_level(bucket: RiskBucket, policy_violation: PolicyViolation | None) -> str: + if policy_violation is not None: + if policy_violation.level is PolicyLevel.BLOCK: + return "error" + return "warning" if bucket is RiskBucket.MAJOR_UPGRADE: return "note" return "warning" -def _risk_result_message(finding: RiskFinding, blocking_violation: PolicyViolation | None) -> str: +def _risk_result_message(finding: RiskFinding, policy_violation: PolicyViolation | None) -> str: component_label = _component_label(finding.component.name, finding.component.version) if finding.bucket is RiskBucket.UNKNOWN_LICENSE: base_message = f"{component_label} has missing or unknown license metadata." @@ -300,9 +312,11 @@ def _risk_result_message(finding: RiskFinding, blocking_violation: PolicyViolati else: base_message = finding.rationale - if blocking_violation is None: + if policy_violation is None: return base_message - return f"Blocked by policy: {base_message}" + if policy_violation.level is PolicyLevel.BLOCK: + return f"Blocked by policy: {base_message}" + return f"Policy warning: {base_message}" def _policy_result_message(violation: PolicyViolation) -> str: @@ -376,7 +390,7 @@ def _provenance_requirement_context(message: str) -> str | None: return normalized or None -def _blocking_violation_map(violations: list[PolicyViolation]) -> dict[tuple[str, str | None], PolicyViolation]: +def _policy_violation_map(violations: list[PolicyViolation]) -> dict[tuple[str, str | None], PolicyViolation]: return { (violation.rule_id, violation.component_key): violation for violation in violations diff --git a/tools/sbom-diff-and-risk/tests/test_ci_consumer_demo.py b/tools/sbom-diff-and-risk/tests/test_ci_consumer_demo.py index 6bf4b7c..ce89503 100644 --- a/tools/sbom-diff-and-risk/tests/test_ci_consumer_demo.py +++ b/tools/sbom-diff-and-risk/tests/test_ci_consumer_demo.py @@ -40,6 +40,8 @@ def test_policy_consumer_uploads_policy_json_before_enforcing_exit_code() -> Non assert "--out-md" not in text assert "--out-sarif" not in text assert "tee " not in text + assert "gh release download v1.0.0" in text + assert "sbom_diff_and_risk-1.0.0-py3-none-any.whl" in text run_index = step_names.index("Run dependency policy") upload_index = step_names.index("Upload policy JSON") diff --git a/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py b/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py index 7169d43..caab6ce 100644 --- a/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py +++ b/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py @@ -200,5 +200,5 @@ def build_report_metadata(self) -> ReportEnrichmentMetadata: assert payload["metadata"]["enrichment"]["mode"] == "opt_in_scorecard" assert payload["metadata"]["enrichment"]["scorecard_enabled"] is True assert payload["metadata"]["enrichment"]["scorecard_timeout_seconds"] == 4.25 - assert payload["evidence_confidence"] == "enrichment_recorded" + assert payload["evidence_confidence"] == "scorecard_recorded" assert payload["notes"][1] == "OpenSSF Scorecard enrichment was requested explicitly." diff --git a/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py b/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py index c52871a..3d680a9 100644 --- a/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py +++ b/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py @@ -224,7 +224,7 @@ def build_report_metadata(self) -> ReportEnrichmentMetadata: }, }, } - assert payload["evidence_confidence"] == "enrichment_recorded" + assert payload["evidence_confidence"] == "scorecard_recorded" assert "policy" not in payload diff --git a/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py b/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py index 4e43289..a93756b 100644 --- a/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py +++ b/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py @@ -18,8 +18,8 @@ def test_evidence_confidence_values_are_release_facing_labels() -> None: "local_manifest_only", "sbom_present", "policy_matched", - "enrichment_recorded", "provenance_recorded", + "scorecard_recorded", } @@ -68,7 +68,7 @@ def test_evidence_confidence_marks_provenance_recorded_when_pypi_enrichment_is_u assert evidence_confidence_for_report(report) is EvidenceConfidence.PROVENANCE_RECORDED -def test_evidence_confidence_marks_enrichment_recorded_when_scorecard_enrichment_is_used() -> None: +def test_evidence_confidence_marks_scorecard_recorded_when_scorecard_enrichment_is_used() -> None: report = _minimal_report( before_format="requirements-txt", after_format="requirements-txt", @@ -80,7 +80,24 @@ def test_evidence_confidence_marks_enrichment_recorded_when_scorecard_enrichment ), ) - assert evidence_confidence_for_report(report) is EvidenceConfidence.ENRICHMENT_RECORDED + assert evidence_confidence_for_report(report) is EvidenceConfidence.SCORECARD_RECORDED + + +def test_evidence_confidence_prefers_scorecard_when_both_enrichments_are_recorded() -> None: + report = _minimal_report( + before_format="requirements-txt", + after_format="requirements-txt", + enrichment=ReportEnrichmentMetadata( + mode="opt_in_pypi_and_scorecard", + pypi_enabled=True, + scorecard_enabled=True, + pypi_network_access_performed=True, + scorecard_network_access_performed=True, + network_access_performed=True, + ), + ) + + assert evidence_confidence_for_report(report) is EvidenceConfidence.SCORECARD_RECORDED def test_evidence_confidence_allows_explicit_recorded_override_for_constructed_snapshots() -> None: diff --git a/tools/sbom-diff-and-risk/tests/test_policy_warning_reviewer_case.py b/tools/sbom-diff-and-risk/tests/test_policy_warning_reviewer_case.py index 661eb5f..93c1f4b 100644 --- a/tools/sbom-diff-and-risk/tests/test_policy_warning_reviewer_case.py +++ b/tools/sbom-diff-and-risk/tests/test_policy_warning_reviewer_case.py @@ -7,11 +7,13 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1] DOC_PATH = PROJECT_ROOT / "docs" / "policy-warning-reviewer-case.md" WARN_REPORT = PROJECT_ROOT / "examples" / "sample-policy-warn-report.json" +WARN_SARIF = PROJECT_ROOT / "examples" / "sample-policy-warn-report.sarif" def test_policy_warning_reviewer_case_matches_warn_fixture() -> None: text = DOC_PATH.read_text(encoding="utf-8") payload = json.loads(WARN_REPORT.read_text(encoding="utf-8")) + sarif = json.loads(WARN_SARIF.read_text(encoding="utf-8")) warning = payload["warning_findings"][0] @@ -27,6 +29,13 @@ def test_policy_warning_reviewer_case_matches_warn_fixture() -> None: assert warning["decision_reason"] == "risk_finding_matched_policy_rule" assert warning["severity_source"] == "warn_on" + sarif_result = sarif["runs"][0]["results"][0] + assert sarif_result["ruleId"] == "sdr.new_package" + assert sarif_result["level"] == "warning" + assert sarif_result["properties"]["component_name"] == "urllib3" + assert sarif_result["properties"]["policy_matched"] is True + assert sarif_result["properties"]["policy_level"] == "warn" + for expected in ( "summary.policy.status", "policy_matched", @@ -34,6 +43,9 @@ def test_policy_warning_reviewer_case_matches_warn_fixture() -> None: "urllib3", "risk_finding_matched_policy_rule", "warn_on", + "sample-policy-warn-report.sarif", + "sdr.new_package", + "policy_level", "not a package safety verdict", "any CVE result", "any malware verdict", diff --git a/tools/sbom-diff-and-risk/tests/test_release_workflow.py b/tools/sbom-diff-and-risk/tests/test_release_workflow.py index 5ab1ea9..c49d1ee 100644 --- a/tools/sbom-diff-and-risk/tests/test_release_workflow.py +++ b/tools/sbom-diff-and-risk/tests/test_release_workflow.py @@ -8,6 +8,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3] WORKFLOW = REPO_ROOT / ".github" / "workflows" / "sbom-diff-and-risk-ci.yml" RELEASE_NOTES = REPO_ROOT / "tools" / "sbom-diff-and-risk" / "RELEASE_NOTES_v1.0-rc.1.md" +FINAL_RELEASE_NOTES = REPO_ROOT / "tools" / "sbom-diff-and-risk" / "RELEASE_NOTES_v1.0.0.md" def test_release_workflow_marks_rc_tags_as_prereleases() -> None: @@ -22,3 +23,14 @@ def test_release_workflow_marks_rc_tags_as_prereleases() -> None: assert "edit_args+=(--prerelease)" in publish_script assert "notes_args=(--notes-file" in publish_script assert "edit_args+=(--notes-file" in publish_script + + +def test_release_workflow_marks_final_tags_as_latest() -> None: + workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + publish_steps = workflow["jobs"]["publish-release-assets"]["steps"] + publish_script = publish_steps[-1]["run"] + + assert FINAL_RELEASE_NOTES.is_file() + assert "else" in publish_script + assert "create_args+=(--latest)" in publish_script + assert "edit_args+=(--latest)" in publish_script diff --git a/tools/sbom-diff-and-risk/tests/test_sarif.py b/tools/sbom-diff-and-risk/tests/test_sarif.py index 624d173..cde127f 100644 --- a/tools/sbom-diff-and-risk/tests/test_sarif.py +++ b/tools/sbom-diff-and-risk/tests/test_sarif.py @@ -34,6 +34,7 @@ def test_render_report_sarif_matches_golden() -> None: def test_sarif_rule_ids_are_stable() -> None: + assert sarif_rule_id_for_risk_bucket(RiskBucket.NEW_PACKAGE) == "sdr.new_package" assert sarif_rule_id_for_risk_bucket(RiskBucket.UNKNOWN_LICENSE) == "sdr.unknown_license" assert sarif_rule_id_for_risk_bucket(RiskBucket.SUSPICIOUS_SOURCE) == "sdr.suspicious_source" assert sarif_rule_id_for_risk_bucket(RiskBucket.MAJOR_UPGRADE) == "sdr.major_upgrade" @@ -42,6 +43,23 @@ def test_sarif_rule_ids_are_stable() -> None: assert sarif_rule_id_for_policy_violation("stale_package") is None +def test_new_package_sarif_requires_a_policy_match() -> None: + project_root = Path(__file__).resolve().parents[1] + report, before_path, after_path = _build_report("cdx_before.json", "cdx_after.json") + + payload = json.loads( + render_report_sarif( + report, + before_path=before_path, + after_path=after_path, + base_dir=project_root, + ) + ) + + rule_ids = {result["ruleId"] for result in payload["runs"][0]["results"]} + assert "sdr.new_package" not in rule_ids + + def test_sarif_structure_and_mapping_are_github_compatible() -> None: project_root = Path(__file__).resolve().parents[1] report, before_path, after_path = _build_report( @@ -63,6 +81,7 @@ def test_sarif_structure_and_mapping_are_github_compatible() -> None: rules = {rule["id"] for rule in run["tool"]["driver"]["rules"]} assert rules == { "sdr.major_upgrade", + "sdr.new_package", "sdr.policy_violation.allow_sources", "sdr.policy_violation.max_added_packages", "sdr.suspicious_source", @@ -76,6 +95,7 @@ def test_sarif_structure_and_mapping_are_github_compatible() -> None: "sdr.policy_violation.allow_sources", "sdr.policy_violation.max_added_packages", "sdr.major_upgrade", + "sdr.new_package", ] assert any(result["level"] == "error" for result in results) assert all(result["locations"][0]["physicalLocation"]["artifactLocation"]["uri"] == "examples/sarif_after.json" for result in results) @@ -108,9 +128,9 @@ def test_sarif_truncation_is_deterministic_and_recorded_in_metadata() -> None: assert first.content == second.content assert first.metadata.truncated is True - assert first.metadata.total_candidate_results == 5 + assert first.metadata.total_candidate_results == 6 assert first.metadata.emitted_results == 2 - assert first.metadata.omitted_results == 3 + assert first.metadata.omitted_results == 4 assert first.metadata.warning_message is not None payload = json.loads(first.content) diff --git a/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py b/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py index b720fc7..5bd5e59 100644 --- a/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py +++ b/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py @@ -48,7 +48,7 @@ def test_scorecard_report_json_summary_includes_enrichment_status() -> None: "warning": 1, "suppressed": 0, } - assert payload["summary"]["evidence_confidence"] == "enrichment_recorded" + assert payload["summary"]["evidence_confidence"] == "scorecard_recorded" assert payload["summary"]["enrichment"] == { "status": "used", "mode": "opt_in_scorecard", @@ -238,7 +238,7 @@ def _build_sample_scorecard_report() -> tuple[CompareReport, Path, Path]: strict=False, stub=False, policy_evaluation=policy_evaluation, - evidence_confidence=EvidenceConfidence.ENRICHMENT_RECORDED, + evidence_confidence=EvidenceConfidence.SCORECARD_RECORDED, enrichment=ReportEnrichmentMetadata( mode="opt_in_scorecard", scorecard_enabled=True,