From f495f90f8463c550d4100bcfd1d9b2889afaa6c0 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Tue, 20 Jan 2026 03:06:42 +0000 Subject: [PATCH 01/14] Move alpine/debian converters into converters dir --- vulnfeeds/cmd/{ => converters}/alpine/Dockerfile | 0 vulnfeeds/cmd/{ => converters}/alpine/alpine_secdb.go | 0 .../cmd/{ => converters}/alpine/fixtures/invalid_versions.txt | 0 vulnfeeds/cmd/{ => converters}/alpine/fixtures/valid_versions.txt | 0 vulnfeeds/cmd/{ => converters}/alpine/main.go | 0 vulnfeeds/cmd/{ => converters}/alpine/run_alpine_convert.sh | 0 vulnfeeds/cmd/{ => converters}/alpine/verify.go | 0 vulnfeeds/cmd/{ => converters}/alpine/verify_test.go | 0 vulnfeeds/cmd/{ => converters}/debian/Dockerfile | 0 vulnfeeds/cmd/{ => converters}/debian/debian_security_tracker.go | 0 vulnfeeds/cmd/{ => converters}/debian/main.go | 0 vulnfeeds/cmd/{ => converters}/debian/main_test.go | 0 vulnfeeds/cmd/{ => converters}/debian/run_debian_convert.sh | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename vulnfeeds/cmd/{ => converters}/alpine/Dockerfile (100%) rename vulnfeeds/cmd/{ => converters}/alpine/alpine_secdb.go (100%) rename vulnfeeds/cmd/{ => converters}/alpine/fixtures/invalid_versions.txt (100%) rename vulnfeeds/cmd/{ => converters}/alpine/fixtures/valid_versions.txt (100%) rename vulnfeeds/cmd/{ => converters}/alpine/main.go (100%) rename vulnfeeds/cmd/{ => converters}/alpine/run_alpine_convert.sh (100%) rename vulnfeeds/cmd/{ => converters}/alpine/verify.go (100%) rename vulnfeeds/cmd/{ => converters}/alpine/verify_test.go (100%) rename vulnfeeds/cmd/{ => converters}/debian/Dockerfile (100%) rename vulnfeeds/cmd/{ => converters}/debian/debian_security_tracker.go (100%) rename vulnfeeds/cmd/{ => converters}/debian/main.go (100%) rename vulnfeeds/cmd/{ => converters}/debian/main_test.go (100%) rename vulnfeeds/cmd/{ => converters}/debian/run_debian_convert.sh (100%) diff --git a/vulnfeeds/cmd/alpine/Dockerfile b/vulnfeeds/cmd/converters/alpine/Dockerfile similarity index 100% rename from vulnfeeds/cmd/alpine/Dockerfile rename to vulnfeeds/cmd/converters/alpine/Dockerfile diff --git a/vulnfeeds/cmd/alpine/alpine_secdb.go b/vulnfeeds/cmd/converters/alpine/alpine_secdb.go similarity index 100% rename from vulnfeeds/cmd/alpine/alpine_secdb.go rename to vulnfeeds/cmd/converters/alpine/alpine_secdb.go diff --git a/vulnfeeds/cmd/alpine/fixtures/invalid_versions.txt b/vulnfeeds/cmd/converters/alpine/fixtures/invalid_versions.txt similarity index 100% rename from vulnfeeds/cmd/alpine/fixtures/invalid_versions.txt rename to vulnfeeds/cmd/converters/alpine/fixtures/invalid_versions.txt diff --git a/vulnfeeds/cmd/alpine/fixtures/valid_versions.txt b/vulnfeeds/cmd/converters/alpine/fixtures/valid_versions.txt similarity index 100% rename from vulnfeeds/cmd/alpine/fixtures/valid_versions.txt rename to vulnfeeds/cmd/converters/alpine/fixtures/valid_versions.txt diff --git a/vulnfeeds/cmd/alpine/main.go b/vulnfeeds/cmd/converters/alpine/main.go similarity index 100% rename from vulnfeeds/cmd/alpine/main.go rename to vulnfeeds/cmd/converters/alpine/main.go diff --git a/vulnfeeds/cmd/alpine/run_alpine_convert.sh b/vulnfeeds/cmd/converters/alpine/run_alpine_convert.sh similarity index 100% rename from vulnfeeds/cmd/alpine/run_alpine_convert.sh rename to vulnfeeds/cmd/converters/alpine/run_alpine_convert.sh diff --git a/vulnfeeds/cmd/alpine/verify.go b/vulnfeeds/cmd/converters/alpine/verify.go similarity index 100% rename from vulnfeeds/cmd/alpine/verify.go rename to vulnfeeds/cmd/converters/alpine/verify.go diff --git a/vulnfeeds/cmd/alpine/verify_test.go b/vulnfeeds/cmd/converters/alpine/verify_test.go similarity index 100% rename from vulnfeeds/cmd/alpine/verify_test.go rename to vulnfeeds/cmd/converters/alpine/verify_test.go diff --git a/vulnfeeds/cmd/debian/Dockerfile b/vulnfeeds/cmd/converters/debian/Dockerfile similarity index 100% rename from vulnfeeds/cmd/debian/Dockerfile rename to vulnfeeds/cmd/converters/debian/Dockerfile diff --git a/vulnfeeds/cmd/debian/debian_security_tracker.go b/vulnfeeds/cmd/converters/debian/debian_security_tracker.go similarity index 100% rename from vulnfeeds/cmd/debian/debian_security_tracker.go rename to vulnfeeds/cmd/converters/debian/debian_security_tracker.go diff --git a/vulnfeeds/cmd/debian/main.go b/vulnfeeds/cmd/converters/debian/main.go similarity index 100% rename from vulnfeeds/cmd/debian/main.go rename to vulnfeeds/cmd/converters/debian/main.go diff --git a/vulnfeeds/cmd/debian/main_test.go b/vulnfeeds/cmd/converters/debian/main_test.go similarity index 100% rename from vulnfeeds/cmd/debian/main_test.go rename to vulnfeeds/cmd/converters/debian/main_test.go diff --git a/vulnfeeds/cmd/debian/run_debian_convert.sh b/vulnfeeds/cmd/converters/debian/run_debian_convert.sh similarity index 100% rename from vulnfeeds/cmd/debian/run_debian_convert.sh rename to vulnfeeds/cmd/converters/debian/run_debian_convert.sh From db52aad7cda22f949ae6fa9748547d4843a35888 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Tue, 20 Jan 2026 03:13:51 +0000 Subject: [PATCH 02/14] Make mirrors dir --- vulnfeeds/cmd/combine-to-osv/README.md | 4 ++-- vulnfeeds/cmd/{ => mirrors}/cpe-repo-gen/Dockerfile | 0 vulnfeeds/cmd/{ => mirrors}/cpe-repo-gen/README.md | 0 vulnfeeds/cmd/{ => mirrors}/cpe-repo-gen/build.sh | 0 vulnfeeds/cmd/{ => mirrors}/cpe-repo-gen/cpe-repo-gen_map.sh | 0 vulnfeeds/cmd/{ => mirrors}/cpe-repo-gen/main.go | 0 .../cmd/{ => mirrors}/debian-copyright-mirror/Dockerfile | 0 vulnfeeds/cmd/{ => mirrors}/debian-copyright-mirror/build.sh | 0 .../debian-copyright-mirror/debian-copyright-mirror.py | 0 .../debian-copyright-mirror/debian-copyright-mirror.sh | 0 vulnfeeds/cmd/{ => mirrors}/download-cves/Dockerfile | 0 vulnfeeds/cmd/{ => mirrors}/download-cves/main.go | 0 vulnfeeds/cmd/{ => mirrors}/download-cves/mirror_nvd.sh | 0 13 files changed, 2 insertions(+), 2 deletions(-) rename vulnfeeds/cmd/{ => mirrors}/cpe-repo-gen/Dockerfile (100%) rename vulnfeeds/cmd/{ => mirrors}/cpe-repo-gen/README.md (100%) rename vulnfeeds/cmd/{ => mirrors}/cpe-repo-gen/build.sh (100%) rename vulnfeeds/cmd/{ => mirrors}/cpe-repo-gen/cpe-repo-gen_map.sh (100%) rename vulnfeeds/cmd/{ => mirrors}/cpe-repo-gen/main.go (100%) rename vulnfeeds/cmd/{ => mirrors}/debian-copyright-mirror/Dockerfile (100%) rename vulnfeeds/cmd/{ => mirrors}/debian-copyright-mirror/build.sh (100%) rename vulnfeeds/cmd/{ => mirrors}/debian-copyright-mirror/debian-copyright-mirror.py (100%) rename vulnfeeds/cmd/{ => mirrors}/debian-copyright-mirror/debian-copyright-mirror.sh (100%) rename vulnfeeds/cmd/{ => mirrors}/download-cves/Dockerfile (100%) rename vulnfeeds/cmd/{ => mirrors}/download-cves/main.go (100%) rename vulnfeeds/cmd/{ => mirrors}/download-cves/mirror_nvd.sh (100%) diff --git a/vulnfeeds/cmd/combine-to-osv/README.md b/vulnfeeds/cmd/combine-to-osv/README.md index e9d1471ba0e..6d459f751c0 100644 --- a/vulnfeeds/cmd/combine-to-osv/README.md +++ b/vulnfeeds/cmd/combine-to-osv/README.md @@ -9,14 +9,14 @@ Combine [`PackageInfo`](https://github.com/google/osv.dev/blob/2c22e9534a521c6c6 To address the generation of CVE records from multiple disparate sources (all requiring a common record prefix): * Alpine, by [this code](../alpine) -* the NVD, by [this code](../nvd-cve-osv) +* the NVD, by [this code](../converters/cve/nvd-cve-osv) ## How See [`run_combine_to_osv_convert.sh`](run_combine_to_osv_convert.sh): * Reads from [`gs://cve-osv-conversion/parts`](https://storage.googleapis.com/cve-osv-conversion/index.html?prefix=parts/) -* Merges with CVE data from NVD (obtained from GCS mirror maintained by [`download-cves`](../download-cves/mirror_nvd.sh)) +* Merges with CVE data from NVD (obtained from GCS mirror maintained by [`download-cves`](../mirrors/download-cves/mirror_nvd.sh)) * Writes an OSV record to [`gs://cve-osv-conversion/osv-output`](https://storage.googleapis.com/cve-osv-conversion/index.html?prefix=osv-output/) * This is the import source for [`cve-osv`](https://github.com/google/osv.dev/blob/2c22e9534a521c6c6350275427f80e481065ca39/source.yaml#L96) * What gets written can be overridden by OSV records in [`gs://cve-osv-conversion/osv-output-overrides`](https://storage.googleapis.com/cve-osv-conversion/index.html?prefix=osv-output-overrides/) diff --git a/vulnfeeds/cmd/cpe-repo-gen/Dockerfile b/vulnfeeds/cmd/mirrors/cpe-repo-gen/Dockerfile similarity index 100% rename from vulnfeeds/cmd/cpe-repo-gen/Dockerfile rename to vulnfeeds/cmd/mirrors/cpe-repo-gen/Dockerfile diff --git a/vulnfeeds/cmd/cpe-repo-gen/README.md b/vulnfeeds/cmd/mirrors/cpe-repo-gen/README.md similarity index 100% rename from vulnfeeds/cmd/cpe-repo-gen/README.md rename to vulnfeeds/cmd/mirrors/cpe-repo-gen/README.md diff --git a/vulnfeeds/cmd/cpe-repo-gen/build.sh b/vulnfeeds/cmd/mirrors/cpe-repo-gen/build.sh similarity index 100% rename from vulnfeeds/cmd/cpe-repo-gen/build.sh rename to vulnfeeds/cmd/mirrors/cpe-repo-gen/build.sh diff --git a/vulnfeeds/cmd/cpe-repo-gen/cpe-repo-gen_map.sh b/vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh similarity index 100% rename from vulnfeeds/cmd/cpe-repo-gen/cpe-repo-gen_map.sh rename to vulnfeeds/cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh diff --git a/vulnfeeds/cmd/cpe-repo-gen/main.go b/vulnfeeds/cmd/mirrors/cpe-repo-gen/main.go similarity index 100% rename from vulnfeeds/cmd/cpe-repo-gen/main.go rename to vulnfeeds/cmd/mirrors/cpe-repo-gen/main.go diff --git a/vulnfeeds/cmd/debian-copyright-mirror/Dockerfile b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile similarity index 100% rename from vulnfeeds/cmd/debian-copyright-mirror/Dockerfile rename to vulnfeeds/cmd/mirrors/debian-copyright-mirror/Dockerfile diff --git a/vulnfeeds/cmd/debian-copyright-mirror/build.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh similarity index 100% rename from vulnfeeds/cmd/debian-copyright-mirror/build.sh rename to vulnfeeds/cmd/mirrors/debian-copyright-mirror/build.sh diff --git a/vulnfeeds/cmd/debian-copyright-mirror/debian-copyright-mirror.py b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py similarity index 100% rename from vulnfeeds/cmd/debian-copyright-mirror/debian-copyright-mirror.py rename to vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.py diff --git a/vulnfeeds/cmd/debian-copyright-mirror/debian-copyright-mirror.sh b/vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh similarity index 100% rename from vulnfeeds/cmd/debian-copyright-mirror/debian-copyright-mirror.sh rename to vulnfeeds/cmd/mirrors/debian-copyright-mirror/debian-copyright-mirror.sh diff --git a/vulnfeeds/cmd/download-cves/Dockerfile b/vulnfeeds/cmd/mirrors/download-cves/Dockerfile similarity index 100% rename from vulnfeeds/cmd/download-cves/Dockerfile rename to vulnfeeds/cmd/mirrors/download-cves/Dockerfile diff --git a/vulnfeeds/cmd/download-cves/main.go b/vulnfeeds/cmd/mirrors/download-cves/main.go similarity index 100% rename from vulnfeeds/cmd/download-cves/main.go rename to vulnfeeds/cmd/mirrors/download-cves/main.go diff --git a/vulnfeeds/cmd/download-cves/mirror_nvd.sh b/vulnfeeds/cmd/mirrors/download-cves/mirror_nvd.sh similarity index 100% rename from vulnfeeds/cmd/download-cves/mirror_nvd.sh rename to vulnfeeds/cmd/mirrors/download-cves/mirror_nvd.sh From e200103eaf8f02c3faed41af278b03aa527683ed Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Tue, 20 Jan 2026 03:15:03 +0000 Subject: [PATCH 03/14] Move nvd conversion --- vulnfeeds/cmd/{ => converters/cve}/nvd-cve-osv/Dockerfile | 0 vulnfeeds/cmd/{ => converters/cve}/nvd-cve-osv/README.md | 0 vulnfeeds/cmd/{ => converters/cve}/nvd-cve-osv/build.sh | 0 vulnfeeds/cmd/{ => converters/cve}/nvd-cve-osv/main.go | 0 .../{ => converters/cve}/nvd-cve-osv/run_cve_to_osv_generation.sh | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename vulnfeeds/cmd/{ => converters/cve}/nvd-cve-osv/Dockerfile (100%) rename vulnfeeds/cmd/{ => converters/cve}/nvd-cve-osv/README.md (100%) rename vulnfeeds/cmd/{ => converters/cve}/nvd-cve-osv/build.sh (100%) rename vulnfeeds/cmd/{ => converters/cve}/nvd-cve-osv/main.go (100%) rename vulnfeeds/cmd/{ => converters/cve}/nvd-cve-osv/run_cve_to_osv_generation.sh (100%) diff --git a/vulnfeeds/cmd/nvd-cve-osv/Dockerfile b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/Dockerfile similarity index 100% rename from vulnfeeds/cmd/nvd-cve-osv/Dockerfile rename to vulnfeeds/cmd/converters/cve/nvd-cve-osv/Dockerfile diff --git a/vulnfeeds/cmd/nvd-cve-osv/README.md b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/README.md similarity index 100% rename from vulnfeeds/cmd/nvd-cve-osv/README.md rename to vulnfeeds/cmd/converters/cve/nvd-cve-osv/README.md diff --git a/vulnfeeds/cmd/nvd-cve-osv/build.sh b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/build.sh similarity index 100% rename from vulnfeeds/cmd/nvd-cve-osv/build.sh rename to vulnfeeds/cmd/converters/cve/nvd-cve-osv/build.sh diff --git a/vulnfeeds/cmd/nvd-cve-osv/main.go b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go similarity index 100% rename from vulnfeeds/cmd/nvd-cve-osv/main.go rename to vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go diff --git a/vulnfeeds/cmd/nvd-cve-osv/run_cve_to_osv_generation.sh b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh similarity index 100% rename from vulnfeeds/cmd/nvd-cve-osv/run_cve_to_osv_generation.sh rename to vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh From 2ddd9206dac618f633cb47a9a369a45086d94b82 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Tue, 20 Jan 2026 03:17:27 +0000 Subject: [PATCH 04/14] move and rename cve5 converters --- .../cve/cve5/bulk-converter}/Dockerfile | 0 .../cve/cve5/bulk-converter}/cna_allowlist.txt | 0 .../cve/cve5/bulk-converter}/main.go | 0 .../cve/cve5/bulk-converter}/run-cvelist-converter.sh | 0 .../cve/cve5/single-converter}/main.go | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename vulnfeeds/cmd/{cve-bulk-converter => converters/cve/cve5/bulk-converter}/Dockerfile (100%) rename vulnfeeds/cmd/{cve-bulk-converter => converters/cve/cve5/bulk-converter}/cna_allowlist.txt (100%) rename vulnfeeds/cmd/{cve-bulk-converter => converters/cve/cve5/bulk-converter}/main.go (100%) rename vulnfeeds/cmd/{cve-bulk-converter => converters/cve/cve5/bulk-converter}/run-cvelist-converter.sh (100%) rename vulnfeeds/cmd/{cve-single-converter => converters/cve/cve5/single-converter}/main.go (100%) diff --git a/vulnfeeds/cmd/cve-bulk-converter/Dockerfile b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/Dockerfile similarity index 100% rename from vulnfeeds/cmd/cve-bulk-converter/Dockerfile rename to vulnfeeds/cmd/converters/cve/cve5/bulk-converter/Dockerfile diff --git a/vulnfeeds/cmd/cve-bulk-converter/cna_allowlist.txt b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/cna_allowlist.txt similarity index 100% rename from vulnfeeds/cmd/cve-bulk-converter/cna_allowlist.txt rename to vulnfeeds/cmd/converters/cve/cve5/bulk-converter/cna_allowlist.txt diff --git a/vulnfeeds/cmd/cve-bulk-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go similarity index 100% rename from vulnfeeds/cmd/cve-bulk-converter/main.go rename to vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go diff --git a/vulnfeeds/cmd/cve-bulk-converter/run-cvelist-converter.sh b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/run-cvelist-converter.sh similarity index 100% rename from vulnfeeds/cmd/cve-bulk-converter/run-cvelist-converter.sh rename to vulnfeeds/cmd/converters/cve/cve5/bulk-converter/run-cvelist-converter.sh diff --git a/vulnfeeds/cmd/cve-single-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go similarity index 100% rename from vulnfeeds/cmd/cve-single-converter/main.go rename to vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go From f71c2ed721d3a2a5818d73fef848b965538e6ba5 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Tue, 20 Jan 2026 03:21:44 +0000 Subject: [PATCH 05/14] Fix routing --- vulnfeeds/cmd/converters/cve/nvd-cve-osv/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/build.sh b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/build.sh index 9720c0dad53..5dc5ba89abb 100755 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/build.sh +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/build.sh @@ -21,5 +21,5 @@ cd ../../ docker build \ -t gcr.io/oss-vdb/nvd-cve-osv:latest \ - -f cmd/nvd-cve-osv/Dockerfile --pull . && \ + -f cmd/cve/nvd-cve-osv/Dockerfile --pull . && \ gcloud docker -- push gcr.io/oss-vdb/nvd-cve-osv:latest From f7130cd8943a392c4581a34d98dea70593de2110 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Tue, 20 Jan 2026 05:27:27 +0000 Subject: [PATCH 06/14] fix test path --- vulnfeeds/cmd/converters/debian/main_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnfeeds/cmd/converters/debian/main_test.go b/vulnfeeds/cmd/converters/debian/main_test.go index 8706a1b75c1..7757c2efe93 100644 --- a/vulnfeeds/cmd/converters/debian/main_test.go +++ b/vulnfeeds/cmd/converters/debian/main_test.go @@ -40,7 +40,7 @@ func sortAffected(affected []*osvschema.Affected) { func loadTestData(t *testing.T, cveName string) cves.Vulnerability { t.Helper() - fileName := fmt.Sprintf("../../test_data/nvdcve-2.0/%s.json", cveName) + fileName := fmt.Sprintf("../../../test_data/nvdcve-2.0/%s.json", cveName) file, err := os.Open(fileName) if err != nil { t.Fatalf("Failed to load test data from %q: %#v", fileName, err) @@ -65,7 +65,7 @@ func TestGenerateOSVFromDebianTracker(t *testing.T) { now := time.Date(2024, 7, 1, 0, 0, 0, 0, time.UTC) var trackerData DebianSecurityTrackerData - if err := json.Unmarshal(mustRead(t, "../../test_data/debian/debian_security_tracker_mock.json"), &trackerData); err != nil { + if err := json.Unmarshal(mustRead(t, "../../../test_data/debian/debian_security_tracker_mock.json"), &trackerData); err != nil { t.Fatalf("Failed to unmarshal test data: %v", err) } From f99caece8fa3998721f287e1639ee67bed3c736c Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Tue, 20 Jan 2026 05:45:11 +0000 Subject: [PATCH 07/14] Refactor duplicate use of CPE --- vulnfeeds/cves/versions.go | 4 ++-- vulnfeeds/cves/versions_test.go | 10 +++++----- vulnfeeds/models/types.go | 2 +- vulnfeeds/vulns/vulns.go | 1 - 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/vulnfeeds/cves/versions.go b/vulnfeeds/cves/versions.go index 163c9a229a3..38e80babe55 100644 --- a/vulnfeeds/cves/versions.go +++ b/vulnfeeds/cves/versions.go @@ -836,7 +836,7 @@ func RemoveQuoting(s string) (result string) { } // Parse a well-formed CPE string into a struct. -func ParseCPE(formattedString string) (*models.CPE, error) { +func ParseCPE(formattedString string) (*models.CPEString, error) { if !strings.HasPrefix(formattedString, "cpe:") { return nil, fmt.Errorf("%q does not have expected 'cpe:' prefix", formattedString) } @@ -847,7 +847,7 @@ func ParseCPE(formattedString string) (*models.CPE, error) { return nil, err } - return &models.CPE{ + return &models.CPEString{ CPEVersion: strings.Split(formattedString, ":")[1], Part: wfn.GetString("part"), Vendor: RemoveQuoting(wfn.GetString("vendor")), diff --git a/vulnfeeds/cves/versions_test.go b/vulnfeeds/cves/versions_test.go index 69e9bd9411f..09852d7716d 100644 --- a/vulnfeeds/cves/versions_test.go +++ b/vulnfeeds/cves/versions_test.go @@ -43,7 +43,7 @@ func TestParseCPE(t *testing.T) { tests := []struct { description string inputCPEString string - expectedCPEStruct *models.CPE + expectedCPEStruct *models.CPEString expectedOk bool }{ { @@ -67,7 +67,7 @@ func TestParseCPE(t *testing.T) { }, { description: "valid input (hardware)", inputCPEString: "cpe:2.3:h:intel:core_i3-1005g1:-:*:*:*:*:*:*:*", - expectedCPEStruct: &models.CPE{ + expectedCPEStruct: &models.CPEString{ CPEVersion: "2.3", Part: "h", Vendor: "intel", @@ -86,7 +86,7 @@ func TestParseCPE(t *testing.T) { { description: "valid input (software)", inputCPEString: "cpe:2.3:a:gitlab:gitlab:*:*:*:*:community:*:*:*", - expectedCPEStruct: &models.CPE{ + expectedCPEStruct: &models.CPEString{ CPEVersion: "2.3", Part: "a", Vendor: "gitlab", @@ -105,7 +105,7 @@ func TestParseCPE(t *testing.T) { { description: "valid input (software) with embedded colons", inputCPEString: "cpe:2.3:a:http\\:\\:daemon_project:http\\:\\:daemon:*:*:*:*:*:*:*:*", - expectedCPEStruct: &models.CPE{ + expectedCPEStruct: &models.CPEString{ CPEVersion: "2.3", Part: "a", Vendor: "http::daemon_project", @@ -124,7 +124,7 @@ func TestParseCPE(t *testing.T) { { description: "valid input (software) with escaped characters", inputCPEString: "cpe:2.3:a:bloodshed:dev-c\\+\\+:4.9.9.2:*:*:*:*:*:*:*", - expectedCPEStruct: &models.CPE{ + expectedCPEStruct: &models.CPEString{ CPEVersion: "2.3", Part: "a", Vendor: "bloodshed", diff --git a/vulnfeeds/models/types.go b/vulnfeeds/models/types.go index b48c3eefab8..54e5e27d166 100644 --- a/vulnfeeds/models/types.go +++ b/vulnfeeds/models/types.go @@ -219,7 +219,7 @@ func (vi *VersionInfo) Duplicated(candidate AffectedCommit) bool { return false } -type CPE struct { +type CPEString struct { CPEVersion string Part string Vendor string diff --git a/vulnfeeds/vulns/vulns.go b/vulnfeeds/vulns/vulns.go index 9cf1c6dd111..3f6eeb319b0 100644 --- a/vulnfeeds/vulns/vulns.go +++ b/vulnfeeds/vulns/vulns.go @@ -81,7 +81,6 @@ const ( Spaces // Contains space characters Empty // Contains no entry Filler // Has been determined to be a filler word - ) // AttachExtractedVersionInfo converts the models.VersionInfo struct to OSV GIT and ECOSYSTEM AffectedRanges and AffectedPackage. From e0b08d910d411c131b410824465bb5f900f1d524 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Tue, 20 Jan 2026 23:00:06 +0000 Subject: [PATCH 08/14] Move CVE5 and NVD CVE models into models dir, and renamed CVE -> NVDCVE for better clarity on which CVE format is in use --- vulnfeeds/cmd/combine-to-osv/main.go | 11 +- vulnfeeds/cmd/combine-to-osv/main_test.go | 4 +- vulnfeeds/cmd/converters/alpine/main.go | 5 +- .../cve/cve5/bulk-converter/main.go | 4 +- .../cve/cve5/single-converter/main.go | 4 +- .../cmd/converters/cve/nvd-cve-osv/main.go | 14 +- vulnfeeds/cmd/converters/debian/main.go | 5 +- vulnfeeds/cmd/converters/debian/main_test.go | 10 +- vulnfeeds/cmd/mirrors/download-cves/main.go | 8 +- vulnfeeds/cmd/pypi/main.go | 3 +- vulnfeeds/cvelist2osv/common.go | 7 +- vulnfeeds/cvelist2osv/converter.go | 45 +++--- vulnfeeds/cvelist2osv/converter_test.go | 142 +++++++++--------- vulnfeeds/cvelist2osv/default_extractor.go | 7 +- vulnfeeds/cvelist2osv/extraction.go | 6 +- vulnfeeds/cvelist2osv/linux_extractor.go | 12 +- vulnfeeds/cvelist2osv/strategies.go | 9 +- .../cvelist2osv/version_extraction_test.go | 51 ++++--- vulnfeeds/cves/versions.go | 14 +- vulnfeeds/cves/versions_test.go | 32 ++-- vulnfeeds/{cves => models}/cve.go | 2 +- vulnfeeds/{cves => models}/nvd2.go | 12 +- vulnfeeds/pypi/pypi.go | 9 +- vulnfeeds/vulns/vulns.go | 37 +++-- vulnfeeds/vulns/vulns_test.go | 25 ++- 25 files changed, 242 insertions(+), 236 deletions(-) rename vulnfeeds/{cves => models}/cve.go (99%) rename vulnfeeds/{cves => models}/nvd2.go (99%) diff --git a/vulnfeeds/cmd/combine-to-osv/main.go b/vulnfeeds/cmd/combine-to-osv/main.go index a44c3c93d50..a4c87f1a3cc 100644 --- a/vulnfeeds/cmd/combine-to-osv/main.go +++ b/vulnfeeds/cmd/combine-to-osv/main.go @@ -7,6 +7,7 @@ import ( "errors" "flag" "fmt" + "github.com/google/osv/vulnfeeds/models" "io/fs" "log/slog" "os" @@ -138,8 +139,8 @@ func listBucketObjects(bucketName string, prefix string) ([]string, error) { // The function returns a map of CVE IDs to their corresponding Vulnerability objects. // Files that are not ".json" files, directories, or files ending in ".metrics.json" are skipped. // The function will log warnings for files that fail to open or decode, and will terminate if it fails to walk the directory. -func loadOSV(osvPath string) map[cves.CVEID]*osvschema.Vulnerability { - allVulns := make(map[cves.CVEID]*osvschema.Vulnerability) +func loadOSV(osvPath string) map[models.CVEID]*osvschema.Vulnerability { + allVulns := make(map[models.CVEID]*osvschema.Vulnerability) logger.Info("Loading OSV records", slog.String("path", osvPath)) err := filepath.WalkDir(osvPath, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -161,7 +162,7 @@ func loadOSV(osvPath string) map[cves.CVEID]*osvschema.Vulnerability { logger.Error("Failed to decode, skipping", slog.String("file", path), slog.Any("err", decodeErr)) return nil } - allVulns[cves.CVEID(vuln.GetId())] = &vuln + allVulns[models.CVEID(vuln.GetId())] = &vuln return nil }) @@ -174,8 +175,8 @@ func loadOSV(osvPath string) map[cves.CVEID]*osvschema.Vulnerability { } // combineIntoOSV creates OSV entry by combining loaded CVEs from NVD and PackageInfo information from security advisories. -func combineIntoOSV(cve5osv map[cves.CVEID]*osvschema.Vulnerability, nvdosv map[cves.CVEID]*osvschema.Vulnerability, mandatoryCVEIDs []string) map[cves.CVEID]*osvschema.Vulnerability { - osvRecords := make(map[cves.CVEID]*osvschema.Vulnerability) +func combineIntoOSV(cve5osv map[models.CVEID]*osvschema.Vulnerability, nvdosv map[models.CVEID]*osvschema.Vulnerability, mandatoryCVEIDs []string) map[models.CVEID]*osvschema.Vulnerability { + osvRecords := make(map[models.CVEID]*osvschema.Vulnerability) // Iterate through CVEs from security advisories (cve5) as the base for cveID, cve5 := range cve5osv { diff --git a/vulnfeeds/cmd/combine-to-osv/main_test.go b/vulnfeeds/cmd/combine-to-osv/main_test.go index c60794efeb8..39cabcf36f6 100644 --- a/vulnfeeds/cmd/combine-to-osv/main_test.go +++ b/vulnfeeds/cmd/combine-to-osv/main_test.go @@ -1,6 +1,7 @@ package main import ( + "github.com/google/osv/vulnfeeds/models" "path/filepath" "sort" "testing" @@ -8,7 +9,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/google/osv/vulnfeeds/cves" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" @@ -35,7 +35,7 @@ func TestCombineIntoOSV(t *testing.T) { cve5osv := loadOSV(cve5Path) nvdosv := loadOSV(nvdPath) - nvdosvCopy := make(map[cves.CVEID]*osvschema.Vulnerability) + nvdosvCopy := make(map[models.CVEID]*osvschema.Vulnerability) for k, v := range nvdosv { nvdosvCopy[k] = v } diff --git a/vulnfeeds/cmd/converters/alpine/main.go b/vulnfeeds/cmd/converters/alpine/main.go index eb42b3932d8..adb57a3e3e8 100644 --- a/vulnfeeds/cmd/converters/alpine/main.go +++ b/vulnfeeds/cmd/converters/alpine/main.go @@ -15,7 +15,6 @@ import ( "strings" "time" - "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/upload" "github.com/google/osv/vulnfeeds/utility/logger" @@ -138,7 +137,7 @@ func getAlpineSecDBData() map[string][]VersionAndPkg { } // generateAlpineOSV generates the generic PackageInfo package from the information given by alpine advisory -func generateAlpineOSV(allAlpineSecDb map[string][]VersionAndPkg, allCVEs map[cves.CVEID]cves.Vulnerability) (osvVulnerabilities []*vulns.Vulnerability) { +func generateAlpineOSV(allAlpineSecDb map[string][]VersionAndPkg, allCVEs map[models.CVEID]models.Vulnerability) (osvVulnerabilities []*vulns.Vulnerability) { cveIDs := make([]string, 0, len(allAlpineSecDb)) for cveID := range allAlpineSecDb { cveIDs = append(cveIDs, cveID) @@ -157,7 +156,7 @@ func generateAlpineOSV(allAlpineSecDb map[string][]VersionAndPkg, allCVEs map[cv return verPkgs[i].Ver < verPkgs[j].Ver }) - cve, ok := allCVEs[cves.CVEID(cveID)] + cve, ok := allCVEs[models.CVEID(cveID)] var published time.Time var details string if ok { diff --git a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go index 12dd4ef1216..ca1ce1bee8e 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go +++ b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go @@ -15,7 +15,7 @@ import ( "time" "github.com/google/osv/vulnfeeds/cvelist2osv" - "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" ) @@ -103,7 +103,7 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string continue } - var cve cves.CVE5 + var cve models.CVE5 if err := json.Unmarshal(data, &cve); err != nil { logger.Info("Failed to unmarshal JSON", slog.String("path", path), slog.Any("err", err)) continue diff --git a/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go index 471e95fc161..14bfac1dbbb 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go +++ b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go @@ -8,7 +8,7 @@ import ( "os" "github.com/google/osv/vulnfeeds/cvelist2osv" - "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" ) @@ -28,7 +28,7 @@ func main() { logger.Fatal("Failed to open file", slog.Any("err", err)) } - var cve cves.CVE5 + var cve models.CVE5 if err = json.Unmarshal(data, &cve); err != nil { logger.Fatal("Failed to parse CVEList CVE JSON", slog.Any("err", err)) } diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go index 2878d2f6468..fadbeab6502 100644 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go @@ -59,11 +59,11 @@ var Metrics struct { CVEsForApplications int CVEsForKnownRepos int OSVRecordsGenerated int - Outcomes map[cves.CVEID]ConversionOutcome // Per-CVE-ID record of conversion result. + Outcomes map[models.CVEID]ConversionOutcome // Per-CVE-ID record of conversion result. } // Takes an NVD CVE record and outputs an OSV file in the specified directory. -func CVEToOSV(cve cves.CVE, repos []string, cache git.RepoTagsCache, directory string) error { +func CVEToOSV(cve models.NVDCVE, repos []string, cache git.RepoTagsCache, directory string) error { CPEs := cves.CPEs(cve) // The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy. maybeVendorName := "ENOCPE" @@ -156,7 +156,7 @@ func CVEToOSV(cve cves.CVE, repos []string, cache git.RepoTagsCache, directory s } // Takes an NVD CVE record and outputs a PackageInfo struct in a file in the specified directory. -func CVEToPackageInfo(cve cves.CVE, repos []string, cache git.RepoTagsCache, directory string) error { +func CVEToPackageInfo(cve models.NVDCVE, repos []string, cache git.RepoTagsCache, directory string) error { CPEs := cves.CPEs(cve) // The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy. maybeVendorName := "ENOCPE" @@ -268,7 +268,7 @@ func loadCPEDictionary(productToRepo *cves.VendorProductToRepoMap, f string) err } // Output a CSV summarizing per-CVE how it was handled. -func outputOutcomes(outcomes map[cves.CVEID]ConversionOutcome, reposForCVE map[cves.CVEID][]string, directory string) error { +func outputOutcomes(outcomes map[models.CVEID]ConversionOutcome, reposForCVE map[models.CVEID][]string, directory string) error { outcomesFile, err := os.Create(filepath.Join(directory, "outcomes.csv")) if err != nil { return err @@ -304,7 +304,7 @@ func main() { os.Exit(1) } - Metrics.Outcomes = make(map[cves.CVEID]ConversionOutcome) + Metrics.Outcomes = make(map[models.CVEID]ConversionOutcome) logger.InitGlobalLogger() @@ -313,7 +313,7 @@ func main() { logger.Fatal("Failed to open file", slog.Any("err", err)) // double check this is best practice output } - var parsed cves.CVEAPIJSON20Schema + var parsed models.CVEAPIJSON20Schema err = json.Unmarshal(data, &parsed) if err != nil { logger.Fatal("Failed to parse NVD CVE JSON", slog.Any("err", err)) @@ -329,7 +329,7 @@ func main() { logger.Info("VendorProductToRepoMap cache has entries preloaded", slog.Int("count", len(VPRepoCache))) } - ReposForCVE := make(map[cves.CVEID][]string) + ReposForCVE := make(map[models.CVEID][]string) for _, cve := range parsed.Vulnerabilities { refs := cve.CVE.References diff --git a/vulnfeeds/cmd/converters/debian/main.go b/vulnfeeds/cmd/converters/debian/main.go index 3dad1d64479..6634f88c796 100644 --- a/vulnfeeds/cmd/converters/debian/main.go +++ b/vulnfeeds/cmd/converters/debian/main.go @@ -14,7 +14,6 @@ import ( "strconv" "strings" - "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/faulttolerant" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/upload" @@ -75,7 +74,7 @@ func main() { } // generateOSVFromDebianTracker converts Debian Security Tracker entries to OSV format. -func generateOSVFromDebianTracker(debianData DebianSecurityTrackerData, debianReleaseMap map[string]string, allCVEs map[cves.CVEID]cves.Vulnerability) map[string]*vulns.Vulnerability { +func generateOSVFromDebianTracker(debianData DebianSecurityTrackerData, debianReleaseMap map[string]string, allCVEs map[models.CVEID]models.Vulnerability) map[string]*vulns.Vulnerability { logger.Info("Converting Debian Security Tracker data to OSV.") osvCves := make(map[string]*vulns.Vulnerability) @@ -107,7 +106,7 @@ func generateOSVFromDebianTracker(debianData DebianSecurityTrackerData, debianRe continue } v, ok := osvCves[cveID] - currentNVDCVE := allCVEs[cves.CVEID(cveID)] + currentNVDCVE := allCVEs[models.CVEID(cveID)] if !ok { v = &vulns.Vulnerability{ Vulnerability: &osvschema.Vulnerability{ diff --git a/vulnfeeds/cmd/converters/debian/main_test.go b/vulnfeeds/cmd/converters/debian/main_test.go index 7757c2efe93..ec0616af753 100644 --- a/vulnfeeds/cmd/converters/debian/main_test.go +++ b/vulnfeeds/cmd/converters/debian/main_test.go @@ -3,13 +3,13 @@ package main import ( "encoding/json" "fmt" + "github.com/google/osv/vulnfeeds/models" "os" "sort" "testing" "time" "github.com/google/go-cmp/cmp" - "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" @@ -38,14 +38,14 @@ func sortAffected(affected []*osvschema.Affected) { }) } -func loadTestData(t *testing.T, cveName string) cves.Vulnerability { +func loadTestData(t *testing.T, cveName string) models.Vulnerability { t.Helper() fileName := fmt.Sprintf("../../../test_data/nvdcve-2.0/%s.json", cveName) file, err := os.Open(fileName) if err != nil { t.Fatalf("Failed to load test data from %q: %#v", fileName, err) } - var nvdCves cves.CVEAPIJSON20Schema + var nvdCves models.CVEAPIJSON20Schema err = json.NewDecoder(file).Decode(&nvdCves) if err != nil { t.Fatalf("Failed to decode %q: %+v", fileName, err) @@ -57,7 +57,7 @@ func loadTestData(t *testing.T, cveName string) cves.Vulnerability { } t.Fatalf("test data doesn't contain %q", cveName) - return cves.Vulnerability{} + return models.Vulnerability{} } func TestGenerateOSVFromDebianTracker(t *testing.T) { @@ -77,7 +77,7 @@ func TestGenerateOSVFromDebianTracker(t *testing.T) { "bookworm": "12", "trixie": "13", } - cveStuff := map[cves.CVEID]cves.Vulnerability{ + cveStuff := map[models.CVEID]models.Vulnerability{ "CVE-2014-1424": loadTestData(t, "CVE-2014-1424"), "CVE-2017-6507": loadTestData(t, "CVE-2017-6507"), "CVE-2016-1585": loadTestData(t, "CVE-2016-1585"), diff --git a/vulnfeeds/cmd/mirrors/download-cves/main.go b/vulnfeeds/cmd/mirrors/download-cves/main.go index 4a507225966..de820ed2c74 100644 --- a/vulnfeeds/cmd/mirrors/download-cves/main.go +++ b/vulnfeeds/cmd/mirrors/download-cves/main.go @@ -8,6 +8,7 @@ import ( "errors" "flag" "fmt" + "github.com/google/osv/vulnfeeds/models" "io" "log/slog" "net/http" @@ -17,7 +18,6 @@ import ( "strconv" "time" - "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/sethvargo/go-retry" ) @@ -58,7 +58,7 @@ func main() { // Pages are offset based, this assumes the default (and maximum) page size of PageSize // Maintaining the recommended 6 seconds betweens calls is left to the caller. // See https://nvd.nist.gov/developers/vulnerabilities -func downloadCVE2FromAPIWithOffset(apiKey string, offset int) (page *cves.CVEAPIJSON20Schema, err error) { //nolint:unused +func downloadCVE2FromAPIWithOffset(apiKey string, offset int) (page *models.CVEAPIJSON20Schema, err error) { //nolint:unused client := &http.Client{} APIURL, err := url.Parse(NVDAPIEndpoint) if err != nil { @@ -125,8 +125,8 @@ func downloadCVE2FromAPI(apiKey string, cvePath string) { //nolint:unused logger.Fatal("Something went wrong when creating/opening file", slog.Any("err", err)) } defer file.Close() - var vulnerabilities []cves.Vulnerability - var page *cves.CVEAPIJSON20Schema + var vulnerabilities []models.Vulnerability + var page *models.CVEAPIJSON20Schema offset := 0 prevTotal := 0 for { diff --git a/vulnfeeds/cmd/pypi/main.go b/vulnfeeds/cmd/pypi/main.go index 10d9050aa20..2974199b89d 100644 --- a/vulnfeeds/cmd/pypi/main.go +++ b/vulnfeeds/cmd/pypi/main.go @@ -19,6 +19,7 @@ import ( "encoding/json" "flag" "fmt" + "github.com/google/osv/vulnfeeds/models" "io/fs" "log/slog" "net/http" @@ -115,7 +116,7 @@ func main() { if err != nil { logger.Fatal("Failed to open file", slog.Any("err", err)) } - var parsed cves.CVEAPIJSON20Schema + var parsed models.CVEAPIJSON20Schema err = json.Unmarshal(data, &parsed) if err != nil { logger.Fatal("Failed to parse NVD CVE JSON", slog.Any("err", err)) diff --git a/vulnfeeds/cvelist2osv/common.go b/vulnfeeds/cvelist2osv/common.go index da185c331c5..53a541c51fb 100644 --- a/vulnfeeds/cvelist2osv/common.go +++ b/vulnfeeds/cvelist2osv/common.go @@ -4,6 +4,7 @@ import ( "cmp" "encoding/json" "errors" + "github.com/google/osv/vulnfeeds/models" "log/slog" "strconv" "strings" @@ -82,7 +83,7 @@ func toVersionRangeType(s string) VersionRangeType { // resolveVersionToCommit is a helper to convert a version string to a commit hash. // It logs the outcome of the conversion attempt and returns an empty string on failure. -func resolveVersionToCommit(cveID cves.CVEID, version, versionType, repo string, normalizedTags map[string]git.NormalizedTag) string { +func resolveVersionToCommit(cveID models.CVEID, version, versionType, repo string, normalizedTags map[string]git.NormalizedTag) string { if version == "" { return "" } @@ -101,7 +102,7 @@ func resolveVersionToCommit(cveID cves.CVEID, version, versionType, repo string, // Takes a CVE ID string (for logging), VersionInfo with AffectedVersions and // typically no AffectedCommits and attempts to add AffectedCommits (including Fixed commits) where there aren't any. // Refuses to add the same commit to AffectedCommits more than once. -func gitVersionsToCommits(cveID cves.CVEID, versionRanges []*osvschema.Range, repos []string, metrics *ConversionMetrics, cache git.RepoTagsCache) (*osvschema.Affected, error) { +func gitVersionsToCommits(cveID models.CVEID, versionRanges []*osvschema.Range, repos []string, metrics *ConversionMetrics, cache git.RepoTagsCache) (*osvschema.Affected, error) { var newAff osvschema.Affected var newVersionRanges []*osvschema.Range unresolvedRanges := versionRanges @@ -193,7 +194,7 @@ func gitVersionsToCommits(cveID cves.CVEID, versionRanges []*osvschema.Range, re // findCPEVersionRanges extracts version ranges and CPE strings from the CNA's // CPE applicability statements in a CVE record. -func findCPEVersionRanges(cve cves.CVE5) (versionRanges []*osvschema.Range, cpes []string, err error) { +func findCPEVersionRanges(cve models.CVE5) (versionRanges []*osvschema.Range, cpes []string, err error) { // TODO(jesslowe): Add logic to also extract CPEs from the 'affected' field (e.g., CVE-2025-1110). for _, c := range cve.Containers.CNA.CPEApplicability { for _, node := range c.Nodes { diff --git a/vulnfeeds/cvelist2osv/converter.go b/vulnfeeds/cvelist2osv/converter.go index 3f643659d8c..4b33eb13cf4 100644 --- a/vulnfeeds/cvelist2osv/converter.go +++ b/vulnfeeds/cvelist2osv/converter.go @@ -14,6 +14,7 @@ import ( "time" "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/google/osv/vulnfeeds/vulns" @@ -28,7 +29,7 @@ const ( // ConversionMetrics holds the collected data about the conversion process for a single CVE. type ConversionMetrics struct { - CVEID cves.CVEID `json:"id"` // The CVE ID + CVEID models.CVEID `json:"id"` // The CVE ID CNA string `json:"cna"` // The CNA that assigned the CVE. Outcome ConversionOutcome `json:"outcome"` // The final outcome of the conversion (e.g., "Successful", "Failed"). Repos []string `json:"repos"` // A list of repositories extracted from the CVE's references. @@ -63,7 +64,7 @@ var RefTagDenyList = []string{ // extractConversionMetrics examines a CVE and its generated OSV references to populate // the ConversionMetrics struct with heuristics about the conversion process. // It captures the assigning CNA and counts the occurrences of each reference type. -func extractConversionMetrics(cve cves.CVE5, refs []*osvschema.Reference, metrics *ConversionMetrics) { +func extractConversionMetrics(cve models.CVE5, refs []*osvschema.Reference, metrics *ConversionMetrics) { // Capture the CNA for heuristic analysis. metrics.CNA = cve.Metadata.AssignerShortName // TODO(jesslowe): more CNA based analysis @@ -82,7 +83,7 @@ func extractConversionMetrics(cve cves.CVE5, refs []*osvschema.Reference, metric } // getCWEs extracts and adds CWE IDs from the CVE5 problem-types -func getCWEs(cna cves.CNA, metrics *ConversionMetrics) []string { +func getCWEs(cna models.CNA, metrics *ConversionMetrics) []string { var cwes []string for _, pt := range cna.ProblemTypes { @@ -106,30 +107,30 @@ func getCWEs(cna cves.CNA, metrics *ConversionMetrics) []string { return cwes } -// FromCVE5 creates a `vulns.Vulnerability` object from a `cves.CVE5` object. +// FromCVE5 creates a `vulns.Vulnerability` object from a `models.CVE5` object. // It populates the main fields of the OSV record, including ID, summary, details, // references, timestamps, severity, and version information. -func FromCVE5(cve cves.CVE5, refs []cves.Reference, metrics *ConversionMetrics, sourceLink string) *vulns.Vulnerability { +func FromCVE5(cve models.CVE5, refs []models.Reference, metrics *ConversionMetrics, sourceLink string) *vulns.Vulnerability { aliases, related := vulns.ExtractReferencedVulns(cve.Metadata.CVEID, cve.Metadata.CVEID, refs) v := vulns.Vulnerability{ Vulnerability: &osvschema.Vulnerability{ SchemaVersion: osvconstants.SchemaVersion, Id: string(cve.Metadata.CVEID), Summary: cve.Containers.CNA.Title, - Details: cves.EnglishDescription(cve.Containers.CNA.Descriptions), + Details: models.EnglishDescription(cve.Containers.CNA.Descriptions), Aliases: aliases, Related: related, References: vulns.ClassifyReferences(refs), }} - published, err := cves.ParseCVE5Timestamp(cve.Metadata.DatePublished) + published, err := models.ParseCVE5Timestamp(cve.Metadata.DatePublished) if err != nil { metrics.AddNote("[%s]: Published date failed to parse, setting time to now", cve.Metadata.CVEID) published = time.Now() } v.Published = timestamppb.New(published) - modified, err := cves.ParseCVE5Timestamp(cve.Metadata.DateUpdated) + modified, err := models.ParseCVE5Timestamp(cve.Metadata.DateUpdated) if err != nil { metrics.AddNote("[%s]: Modified date failed to parse, setting time to now", cve.Metadata.CVEID) modified = time.Now() @@ -165,7 +166,7 @@ func FromCVE5(cve cves.CVE5, refs []cves.Reference, metrics *ConversionMetrics, }) // Combine severity metrics from both CNA and ADP containers. - var severity []cves.Metrics + var severity []models.Metrics if len(cve.Containers.CNA.Metrics) != 0 { severity = append(severity, cve.Containers.CNA.Metrics...) } @@ -184,7 +185,7 @@ func FromCVE5(cve cves.CVE5, refs []cves.Reference, metrics *ConversionMetrics, } // CreateOSVFile creates the initial file for the OSV record. -func CreateOSVFile(id cves.CVEID, vulnDir string) (*os.File, error) { +func CreateOSVFile(id models.CVEID, vulnDir string) (*os.File, error) { outputFile := filepath.Join(vulnDir, string(id)+extension) f, err := os.Create(outputFile) @@ -199,7 +200,7 @@ func CreateOSVFile(id cves.CVEID, vulnDir string) (*os.File, error) { // CreateMetricsFile saves the collected conversion metrics to a JSON file. // This file provides data for analyzing the success and characteristics of the // conversion process for a given CVE. -func CreateMetricsFile(id cves.CVEID, vulnDir string) (*os.File, error) { +func CreateMetricsFile(id models.CVEID, vulnDir string) (*os.File, error) { metricsFile := filepath.Join(vulnDir, string(id)+".metrics.json") f, err := os.Create(metricsFile) if err != nil { @@ -229,15 +230,15 @@ func determineOutcome(metrics *ConversionMetrics) { // ConvertAndExportCVEToOSV is the main function for this file. It takes a CVE, // converts it into an OSV record, collects metrics, and writes both to disk. -func ConvertAndExportCVEToOSV(cve cves.CVE5, vulnSink io.Writer, metricsSink io.Writer, sourceLink string) error { +func ConvertAndExportCVEToOSV(cve models.CVE5, vulnSink io.Writer, metricsSink io.Writer, sourceLink string) error { cveID := cve.Metadata.CVEID cnaAssigner := cve.Metadata.AssignerShortName references := identifyPossibleURLs(cve) // Add NVD and computed source link to references - references = append(references, cves.Reference{URL: fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveID)}) + references = append(references, models.Reference{URL: fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveID)}) if sourceLink != "" { - references = append(references, cves.Reference{URL: sourceLink}) + references = append(references, models.Reference{URL: sourceLink}) } references = deduplicateRefs(references) @@ -281,7 +282,7 @@ func ConvertAndExportCVEToOSV(cve cves.CVE5, vulnSink io.Writer, metricsSink io. // identifyPossibleURLs extracts all URLs from a CVE object. // It searches for URLs in the CNA and ADP reference sections, as well as in // the 'collectionUrl' and 'repo' fields of the 'affected' entries. -func identifyPossibleURLs(cve cves.CVE5) []cves.Reference { +func identifyPossibleURLs(cve models.CVE5) []models.Reference { refs := cve.Containers.CNA.References for _, adp := range cve.Containers.ADP { @@ -292,15 +293,15 @@ func identifyPossibleURLs(cve cves.CVE5) []cves.Reference { for _, affected := range cve.Containers.CNA.Affected { if affected.CollectionURL != "" { - refs = append(refs, cves.Reference{URL: affected.CollectionURL}) + refs = append(refs, models.Reference{URL: affected.CollectionURL}) } if affected.Repo != "" { - refs = append(refs, cves.Reference{URL: affected.Repo}) + refs = append(refs, models.Reference{URL: affected.Repo}) } } // Filter out empty URLs from CNA references if any - filteredRefs := make([]cves.Reference, 0, len(refs)) + filteredRefs := make([]models.Reference, 0, len(refs)) for _, ref := range refs { if ref.URL != "" { filteredRefs = append(filteredRefs, ref) @@ -311,19 +312,19 @@ func identifyPossibleURLs(cve cves.CVE5) []cves.Reference { return refs } -func deduplicateRefs(refs []cves.Reference) []cves.Reference { +func deduplicateRefs(refs []models.Reference) []models.Reference { // Deduplicate references by URL. - slices.SortStableFunc(refs, func(a, b cves.Reference) int { + slices.SortStableFunc(refs, func(a, b models.Reference) int { return strings.Compare(a.URL, b.URL) }) - refs = slices.CompactFunc(refs, func(a, b cves.Reference) bool { + refs = slices.CompactFunc(refs, func(a, b models.Reference) bool { return a.URL == b.URL }) return refs } -func buildDBSpecific(cve cves.CVE5, metrics *ConversionMetrics, sourceLink string) map[string]any { +func buildDBSpecific(cve models.CVE5, metrics *ConversionMetrics, sourceLink string) map[string]any { dbSpecific := make(map[string]any) if sourceLink != "" { diff --git a/vulnfeeds/cvelist2osv/converter_test.go b/vulnfeeds/cvelist2osv/converter_test.go index 7ce0523525b..d0e5c5b1119 100644 --- a/vulnfeeds/cvelist2osv/converter_test.go +++ b/vulnfeeds/cvelist2osv/converter_test.go @@ -3,6 +3,7 @@ package cvelist2osv import ( "bytes" "encoding/json" + "github.com/google/osv/vulnfeeds/models" "os" "path/filepath" "sort" @@ -12,7 +13,6 @@ import ( "github.com/gkampitakis/go-snaps/snaps" "github.com/google/go-cmp/cmp" - "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvconstants" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -21,7 +21,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -func loadTestData(t *testing.T, cveName string) cves.CVE5 { +func loadTestData(t *testing.T, cveName string) models.CVE5 { t.Helper() prefix := strings.Split(cveName, "-")[2] prefixpath := prefix[:len(prefix)-3] + "xxx" @@ -30,14 +30,14 @@ func loadTestData(t *testing.T, cveName string) cves.CVE5 { return loadTestCVE(t, fileName) } -func loadTestCVE(t *testing.T, path string) cves.CVE5 { +func loadTestCVE(t *testing.T, path string) models.CVE5 { t.Helper() file, err := os.Open(path) if err != nil { t.Fatalf("Failed to load test data from %q: %v", path, err) } defer file.Close() - var cve cves.CVE5 + var cve models.CVE5 err = json.NewDecoder(file).Decode(&cve) if err != nil { t.Fatalf("Failed to decode %q: %+v", path, err) @@ -49,31 +49,31 @@ func loadTestCVE(t *testing.T, path string) cves.CVE5 { func TestIdentifyPossibleURLs(t *testing.T) { testCases := []struct { name string - cve cves.CVE5 - expectedRefs []cves.Reference + cve models.CVE5 + expectedRefs []models.Reference }{ { name: "simple case with duplicates", - cve: cves.CVE5{ + cve: models.CVE5{ Containers: struct { - CNA cves.CNA `json:"cna"` - ADP []cves.CNA `json:"adp,omitempty"` + CNA models.CNA `json:"cna"` + ADP []models.CNA `json:"adp,omitempty"` }{ - CNA: cves.CNA{ - References: []cves.Reference{ + CNA: models.CNA{ + References: []models.Reference{ {URL: "http://a.com"}, {URL: "http://b.com"}, }, - Affected: []cves.Affected{ + Affected: []models.Affected{ { CollectionURL: "http://d.com", Repo: "http://b.com", }, }, }, - ADP: []cves.CNA{ + ADP: []models.CNA{ { - References: []cves.Reference{ + References: []models.Reference{ {URL: "http://c.com"}, {URL: "http://a.com"}, }, @@ -81,7 +81,7 @@ func TestIdentifyPossibleURLs(t *testing.T) { }, }, }, - expectedRefs: []cves.Reference{ + expectedRefs: []models.Reference{ {URL: "http://a.com"}, {URL: "http://b.com"}, {URL: "http://c.com"}, @@ -92,53 +92,53 @@ func TestIdentifyPossibleURLs(t *testing.T) { }, { name: "no references and CNA refs is nil", - cve: cves.CVE5{ + cve: models.CVE5{ Containers: struct { - CNA cves.CNA `json:"cna"` - ADP []cves.CNA `json:"adp,omitempty"` + CNA models.CNA `json:"cna"` + ADP []models.CNA `json:"adp,omitempty"` }{ - CNA: cves.CNA{ + CNA: models.CNA{ References: nil, }, }, }, - expectedRefs: []cves.Reference{}, + expectedRefs: []models.Reference{}, }, { name: "no references and CNA refs is empty slice", - cve: cves.CVE5{ + cve: models.CVE5{ Containers: struct { - CNA cves.CNA `json:"cna"` - ADP []cves.CNA `json:"adp,omitempty"` + CNA models.CNA `json:"cna"` + ADP []models.CNA `json:"adp,omitempty"` }{ - CNA: cves.CNA{ - References: []cves.Reference{}, + CNA: models.CNA{ + References: []models.Reference{}, }, }, }, - expectedRefs: []cves.Reference{}, + expectedRefs: []models.Reference{}, }, { name: "empty url string", - cve: cves.CVE5{ + cve: models.CVE5{ Containers: struct { - CNA cves.CNA `json:"cna"` - ADP []cves.CNA `json:"adp,omitempty"` + CNA models.CNA `json:"cna"` + ADP []models.CNA `json:"adp,omitempty"` }{ - CNA: cves.CNA{ - Affected: []cves.Affected{ + CNA: models.CNA{ + Affected: []models.Affected{ { CollectionURL: "", }, }, - References: []cves.Reference{ + References: []models.Reference{ {URL: "http://a.com"}, {URL: ""}, }, }, }, }, - expectedRefs: []cves.Reference{ + expectedRefs: []models.Reference{ {URL: "http://a.com"}, }, }, @@ -155,36 +155,36 @@ func TestIdentifyPossibleURLs(t *testing.T) { } func TestFromCVE5(t *testing.T) { - cve1110Pub, _ := cves.ParseCVE5Timestamp("2025-05-22T14:02:31.385Z") - cve1110Mod, _ := cves.ParseCVE5Timestamp("2025-05-22T14:17:44.379Z") - cve21634Pub, _ := cves.ParseCVE5Timestamp("2024-01-03T22:46:03.585Z") - cve21634Mod, _ := cves.ParseCVE5Timestamp("2025-06-16T19:45:37.088Z") - cve21772Pub, _ := cves.ParseCVE5Timestamp("2025-02-27T02:18:19.528Z") - cve21772Mod, _ := cves.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") - cvePlaceholder, _ := cves.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") + cve1110Pub, _ := models.ParseCVE5Timestamp("2025-05-22T14:02:31.385Z") + cve1110Mod, _ := models.ParseCVE5Timestamp("2025-05-22T14:17:44.379Z") + cve21634Pub, _ := models.ParseCVE5Timestamp("2024-01-03T22:46:03.585Z") + cve21634Mod, _ := models.ParseCVE5Timestamp("2025-06-16T19:45:37.088Z") + cve21772Pub, _ := models.ParseCVE5Timestamp("2025-02-27T02:18:19.528Z") + cve21772Mod, _ := models.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") + cvePlaceholder, _ := models.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") testCases := []struct { name string - cve cves.CVE5 + cve models.CVE5 - refs []cves.Reference + refs []models.Reference expectedVuln *vulns.Vulnerability }{ { name: "disputed record", - cve: cves.CVE5{ - Metadata: cves.CVE5Metadata{ + cve: models.CVE5{ + Metadata: models.CVE5Metadata{ CVEID: "CVE-2025-9999", State: "PUBLISHED", DatePublished: "2025-05-04T07:20:46.575Z", DateUpdated: "2025-05-04T07:20:46.575Z", }, Containers: struct { - CNA cves.CNA `json:"cna"` - ADP []cves.CNA `json:"adp,omitempty"` + CNA models.CNA `json:"cna"` + ADP []models.CNA `json:"adp,omitempty"` }{ - CNA: cves.CNA{ + CNA: models.CNA{ Tags: []string{"disputed"}, - Descriptions: []cves.LangString{ + Descriptions: []models.LangString{ { Lang: "en", Value: "A disputed vulnerability.", @@ -193,7 +193,7 @@ func TestFromCVE5(t *testing.T) { }, }, }, - refs: []cves.Reference{}, + refs: []models.Reference{}, expectedVuln: &vulns.Vulnerability{ Vulnerability: &osvschema.Vulnerability{ Id: "CVE-2025-9999", @@ -213,7 +213,7 @@ func TestFromCVE5(t *testing.T) { { name: "CVE-2025-1110", cve: loadTestData(t, "CVE-2025-1110"), - refs: []cves.Reference{ + refs: []models.Reference{ {URL: "https://gitlab.com/gitlab-org/gitlab/-/issues/517693", Tags: []string{"issue-tracking", "permissions-required"}}, {URL: "https://hackerone.com/reports/2972576", Tags: []string{"technical-description", "exploit", "permissions-required"}}, }, @@ -256,7 +256,7 @@ func TestFromCVE5(t *testing.T) { { name: "CVE-2024-21634", cve: loadTestData(t, "CVE-2024-21634"), - refs: []cves.Reference{ + refs: []models.Reference{ {Tags: []string{"x_refsource_CONFIRM"}, URL: "https://github.com/amazon-ion/ion-java/security/advisories/GHSA-264p-99wq-f4j6"}, }, expectedVuln: &vulns.Vulnerability{ @@ -297,7 +297,7 @@ func TestFromCVE5(t *testing.T) { { name: "CVE-2025-21772", cve: loadTestData(t, "CVE-2025-21772"), - refs: []cves.Reference{ + refs: []models.Reference{ {URL: "https://git.kernel.org/stable/c/a3e77da9f843e4ab93917d30c314f0283e28c124"}, {URL: "https://git.kernel.org/stable/c/213ba5bd81b7e97ac6e6190b8f3bc6ba76123625"}, {URL: "https://git.kernel.org/stable/c/40a35d14f3c0dc72b689061ec72fc9b193f37d1f"}, @@ -392,24 +392,24 @@ func TestFromCVE5(t *testing.T) { } func TestConvertAndExportCVEToOSV(t *testing.T) { - cve1110Pub, _ := cves.ParseCVE5Timestamp("2025-05-22T14:02:31.385Z") - cve1110Mod, _ := cves.ParseCVE5Timestamp("2025-05-22T14:17:44.379Z") - cve21634Pub, _ := cves.ParseCVE5Timestamp("2024-01-03T22:46:03.585Z") - cve21634Mod, _ := cves.ParseCVE5Timestamp("2025-06-16T19:45:37.088Z") - cve21772Pub, _ := cves.ParseCVE5Timestamp("2025-02-27T02:18:19.528Z") - cve21772Mod, _ := cves.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") - cvePlaceholder, _ := cves.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") + cve1110Pub, _ := models.ParseCVE5Timestamp("2025-05-22T14:02:31.385Z") + cve1110Mod, _ := models.ParseCVE5Timestamp("2025-05-22T14:17:44.379Z") + cve21634Pub, _ := models.ParseCVE5Timestamp("2024-01-03T22:46:03.585Z") + cve21634Mod, _ := models.ParseCVE5Timestamp("2025-06-16T19:45:37.088Z") + cve21772Pub, _ := models.ParseCVE5Timestamp("2025-02-27T02:18:19.528Z") + cve21772Mod, _ := models.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") + cvePlaceholder, _ := models.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") testCases := []struct { name string - cve cves.CVE5 + cve models.CVE5 - refs []cves.Reference + refs []models.Reference expectedVuln *vulns.Vulnerability }{ { name: "disputed record", - cve: cves.CVE5{ - Metadata: cves.CVE5Metadata{ + cve: models.CVE5{ + Metadata: models.CVE5Metadata{ CVEID: "CVE-2025-9999", State: "PUBLISHED", DatePublished: "2025-05-04T07:20:46.575Z", @@ -417,12 +417,12 @@ func TestConvertAndExportCVEToOSV(t *testing.T) { AssignerShortName: "unknown", }, Containers: struct { - CNA cves.CNA `json:"cna"` - ADP []cves.CNA `json:"adp,omitempty"` + CNA models.CNA `json:"cna"` + ADP []models.CNA `json:"adp,omitempty"` }{ - CNA: cves.CNA{ + CNA: models.CNA{ Tags: []string{"disputed"}, - Descriptions: []cves.LangString{ + Descriptions: []models.LangString{ { Lang: "en", Value: "A disputed vulnerability.", @@ -431,7 +431,7 @@ func TestConvertAndExportCVEToOSV(t *testing.T) { }, }, }, - refs: []cves.Reference{}, + refs: []models.Reference{}, expectedVuln: &vulns.Vulnerability{ Vulnerability: &osvschema.Vulnerability{ Id: "CVE-2025-9999", @@ -452,7 +452,7 @@ func TestConvertAndExportCVEToOSV(t *testing.T) { { name: "CVE-2025-1110", cve: loadTestData(t, "CVE-2025-1110"), - refs: []cves.Reference{ + refs: []models.Reference{ {URL: "https://gitlab.com/gitlab-org/gitlab/-/issues/517693", Tags: []string{"issue-tracking", "permissions-required"}}, {URL: "https://hackerone.com/reports/2972576", Tags: []string{"technical-description", "exploit", "permissions-required"}}, }, @@ -497,7 +497,7 @@ func TestConvertAndExportCVEToOSV(t *testing.T) { { name: "CVE-2024-21634", cve: loadTestData(t, "CVE-2024-21634"), - refs: []cves.Reference{ + refs: []models.Reference{ {Tags: []string{"x_refsource_CONFIRM"}, URL: "https://github.com/amazon-ion/ion-java/security/advisories/GHSA-264p-99wq-f4j6"}, }, expectedVuln: &vulns.Vulnerability{ @@ -538,7 +538,7 @@ func TestConvertAndExportCVEToOSV(t *testing.T) { { name: "CVE-2025-21772", cve: loadTestData(t, "CVE-2025-21772"), - refs: []cves.Reference{ + refs: []models.Reference{ {URL: "https://git.kernel.org/stable/c/a3e77da9f843e4ab93917d30c314f0283e28c124"}, {URL: "https://git.kernel.org/stable/c/213ba5bd81b7e97ac6e6190b8f3bc6ba76123625"}, {URL: "https://git.kernel.org/stable/c/40a35d14f3c0dc72b689061ec72fc9b193f37d1f"}, diff --git a/vulnfeeds/cvelist2osv/default_extractor.go b/vulnfeeds/cvelist2osv/default_extractor.go index 8e497c7c175..b04d5436124 100644 --- a/vulnfeeds/cvelist2osv/default_extractor.go +++ b/vulnfeeds/cvelist2osv/default_extractor.go @@ -2,6 +2,7 @@ package cvelist2osv import ( "fmt" + "github.com/google/osv/vulnfeeds/models" "log/slog" "github.com/google/osv/vulnfeeds/cves" @@ -14,7 +15,7 @@ import ( // DefaultVersionExtractor provides the default version extraction logic. type DefaultVersionExtractor struct{} -func (d *DefaultVersionExtractor) handleAffected(affected []cves.Affected, metrics *ConversionMetrics) []*osvschema.Range { +func (d *DefaultVersionExtractor) handleAffected(affected []models.Affected, metrics *ConversionMetrics) []*osvschema.Range { var ranges []*osvschema.Range for _, cveAff := range affected { versionRanges, _ := d.FindNormalAffectedRanges(cveAff, metrics) @@ -30,7 +31,7 @@ func (d *DefaultVersionExtractor) handleAffected(affected []cves.Affected, metri } // ExtractVersions for DefaultVersionExtractor. -func (d *DefaultVersionExtractor) ExtractVersions(cve cves.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, repos []string) { +func (d *DefaultVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, repos []string) { gotVersions := false repoTagsCache := git.RepoTagsCache{} @@ -76,7 +77,7 @@ func (d *DefaultVersionExtractor) ExtractVersions(cve cves.CVE5, v *vulns.Vulner } } -func (d *DefaultVersionExtractor) FindNormalAffectedRanges(affected cves.Affected, metrics *ConversionMetrics) ([]*osvschema.Range, VersionRangeType) { +func (d *DefaultVersionExtractor) FindNormalAffectedRanges(affected models.Affected, metrics *ConversionMetrics) ([]*osvschema.Range, VersionRangeType) { versionTypesCount := make(map[VersionRangeType]int) var versionRanges []*osvschema.Range for _, vers := range affected.Versions { diff --git a/vulnfeeds/cvelist2osv/extraction.go b/vulnfeeds/cvelist2osv/extraction.go index fddc1dce961..13e17d3767d 100644 --- a/vulnfeeds/cvelist2osv/extraction.go +++ b/vulnfeeds/cvelist2osv/extraction.go @@ -1,15 +1,15 @@ package cvelist2osv import ( - "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" ) // VersionExtractor defines the interface for different version extraction strategies. type VersionExtractor interface { - ExtractVersions(cve cves.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, repos []string) - FindNormalAffectedRanges(affected cves.Affected, metrics *ConversionMetrics) ([]*osvschema.Range, VersionRangeType) + ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, repos []string) + FindNormalAffectedRanges(affected models.Affected, metrics *ConversionMetrics) ([]*osvschema.Range, VersionRangeType) } // GetVersionExtractor returns the appropriate VersionExtractor for a given CNA. diff --git a/vulnfeeds/cvelist2osv/linux_extractor.go b/vulnfeeds/cvelist2osv/linux_extractor.go index 266dd9be252..1eafb3938cb 100644 --- a/vulnfeeds/cvelist2osv/linux_extractor.go +++ b/vulnfeeds/cvelist2osv/linux_extractor.go @@ -6,6 +6,8 @@ import ( "strconv" "strings" + "github.com/google/osv/vulnfeeds/models" + "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvconstants" @@ -19,8 +21,8 @@ type LinuxVersionExtractor struct { var _ VersionExtractor = &LinuxVersionExtractor{} -// handleAffected takes an array of cves.Affected and handles how to extract them -func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected []cves.Affected, metrics *ConversionMetrics) bool { +// handleAffected takes an array of models.Affected and handles how to extract them +func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected []models.Affected, metrics *ConversionMetrics) bool { hasGit := false gotVersions := false for _, cveAff := range affected { @@ -49,7 +51,7 @@ func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected } // ExtractVersions for LinuxVersionExtractor. -func (l *LinuxVersionExtractor) ExtractVersions(cve cves.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, _ []string) { +func (l *LinuxVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, _ []string) { gotVersions := l.handleAffected(v, cve.Containers.CNA.Affected, metrics) if !gotVersions { @@ -88,7 +90,7 @@ func createLinuxAffected(versionRanges []*osvschema.Range, versionType VersionRa // of 'unaffected' versions. This is common in Linux kernel CVEs where a product is // considered affected by default, and only unaffected versions are listed. // It sorts the introduced and fixed versions to create chronological ranges. -func findInverseAffectedRanges(cveAff cves.Affected, metrics *ConversionMetrics) (ranges []*osvschema.Range, versType VersionRangeType) { +func findInverseAffectedRanges(cveAff models.Affected, metrics *ConversionMetrics) (ranges []*osvschema.Range, versType VersionRangeType) { var introduced []string fixed := make([]string, 0, len(cveAff.Versions)) for _, vers := range cveAff.Versions { @@ -149,7 +151,7 @@ func findInverseAffectedRanges(cveAff cves.Affected, metrics *ConversionMetrics) return nil, VersionRangeTypeUnknown } -func (l *LinuxVersionExtractor) FindNormalAffectedRanges(affected cves.Affected, metrics *ConversionMetrics) ([]*osvschema.Range, VersionRangeType) { +func (l *LinuxVersionExtractor) FindNormalAffectedRanges(affected models.Affected, metrics *ConversionMetrics) ([]*osvschema.Range, VersionRangeType) { versionTypesCount := make(map[VersionRangeType]int) var versionRanges []*osvschema.Range for _, vers := range affected.Versions { diff --git a/vulnfeeds/cvelist2osv/strategies.go b/vulnfeeds/cvelist2osv/strategies.go index f6f25b9b801..d3db99004fb 100644 --- a/vulnfeeds/cvelist2osv/strategies.go +++ b/vulnfeeds/cvelist2osv/strategies.go @@ -2,11 +2,12 @@ package cvelist2osv import ( "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" ) -func cpeVersionExtraction(cve cves.CVE5, metrics *ConversionMetrics) ([]*osvschema.Range, error) { +func cpeVersionExtraction(cve models.CVE5, metrics *ConversionMetrics) ([]*osvschema.Range, error) { cpeRanges, cpeStrings, err := findCPEVersionRanges(cve) if err == nil && len(cpeRanges) > 0 { metrics.VersionSources = append(metrics.VersionSources, VersionSourceCPE) @@ -21,9 +22,9 @@ func cpeVersionExtraction(cve cves.CVE5, metrics *ConversionMetrics) ([]*osvsche } // textVersionExtraction is a helper function for CPE and description extraction. -func textVersionExtraction(cve cves.CVE5, metrics *ConversionMetrics) []*osvschema.Range { +func textVersionExtraction(cve models.CVE5, metrics *ConversionMetrics) []*osvschema.Range { // As a last resort, try extracting versions from the description text. - versions, extractNotes := cves.ExtractVersionsFromText(nil, cves.EnglishDescription(cve.Containers.CNA.Descriptions)) + versions, extractNotes := cves.ExtractVersionsFromText(nil, models.EnglishDescription(cve.Containers.CNA.Descriptions)) for _, note := range extractNotes { metrics.AddNote("%s", note) } @@ -37,7 +38,7 @@ func textVersionExtraction(cve cves.CVE5, metrics *ConversionMetrics) []*osvsche } // initialNormalExtraction handles an expected case of version ranges in the affected field of CVE5 -func initialNormalExtraction(vers cves.Versions, metrics *ConversionMetrics, versionTypesCount map[VersionRangeType]int) ([]*osvschema.Range, VersionRangeType, bool) { +func initialNormalExtraction(vers models.Versions, metrics *ConversionMetrics, versionTypesCount map[VersionRangeType]int) ([]*osvschema.Range, VersionRangeType, bool) { if vers.Status != "affected" { return nil, VersionRangeTypeUnknown, true } diff --git a/vulnfeeds/cvelist2osv/version_extraction_test.go b/vulnfeeds/cvelist2osv/version_extraction_test.go index a1e87c8a4de..0015d548e67 100644 --- a/vulnfeeds/cvelist2osv/version_extraction_test.go +++ b/vulnfeeds/cvelist2osv/version_extraction_test.go @@ -1,6 +1,7 @@ package cvelist2osv import ( + "github.com/google/osv/vulnfeeds/models" "reflect" "sort" "testing" @@ -39,15 +40,15 @@ func TestToVersionRangeType(t *testing.T) { func TestFindNormalAffectedRanges(t *testing.T) { tests := []struct { name string - affected cves.Affected + affected models.Affected cnaAssigner string wantRanges []*osvschema.Range wantRangeType VersionRangeType }{ { name: "simple range", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "affected", Version: "1.0", @@ -63,8 +64,8 @@ func TestFindNormalAffectedRanges(t *testing.T) { }, { name: "single version fallback", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "affected", Version: "2.0", @@ -79,8 +80,8 @@ func TestFindNormalAffectedRanges(t *testing.T) { }, { name: "github range", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "affected", Version: ">= 2.0, < 2.5", @@ -94,8 +95,8 @@ func TestFindNormalAffectedRanges(t *testing.T) { }, { name: "git commit", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "affected", Version: "deadbeef", @@ -149,15 +150,15 @@ func TestCompareSemverLike(t *testing.T) { func TestFindInverseAffectedRanges(t *testing.T) { tests := []struct { name string - affected cves.Affected + affected models.Affected versionType VersionRangeType cnaAssigner string want []*osvschema.Range }{ { name: "linux with wildcard", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "affected", Version: "5.0", @@ -179,8 +180,8 @@ func TestFindInverseAffectedRanges(t *testing.T) { }, { name: "not linux", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "unaffected", Version: "1.0", @@ -195,8 +196,8 @@ func TestFindInverseAffectedRanges(t *testing.T) { }, { name: "linux no wildcard", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "affected", Version: "4.0", @@ -235,7 +236,7 @@ func TestFindInverseAffectedRanges(t *testing.T) { func TestRealWorldFindInverseAffectedRanges(t *testing.T) { testCases := []struct { name string - cve cves.CVE5 + cve models.CVE5 expectedRanges []*osvschema.Range }{ { @@ -265,7 +266,7 @@ func TestRealWorldFindInverseAffectedRanges(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - var affectedBlock cves.Affected + var affectedBlock models.Affected // Find the specific affected block with defaultStatus: "affected". for _, affected := range tc.cve.Containers.CNA.Affected { if affected.DefaultStatus == "affected" { @@ -324,13 +325,13 @@ func TestRealWorldFindInverseAffectedRanges(t *testing.T) { func TestGetVersionExtractor(t *testing.T) { testCases := []struct { name string - cve cves.CVE5 + cve models.CVE5 expectedType reflect.Type }{ { name: "Linux CVE", - cve: cves.CVE5{ - Metadata: cves.CVE5Metadata{ + cve: models.CVE5{ + Metadata: models.CVE5Metadata{ AssignerShortName: "Linux", }, }, @@ -338,8 +339,8 @@ func TestGetVersionExtractor(t *testing.T) { }, { name: "Default CVE", - cve: cves.CVE5{ - Metadata: cves.CVE5Metadata{ + cve: models.CVE5{ + Metadata: models.CVE5Metadata{ AssignerShortName: "Anything", }, }, @@ -347,7 +348,7 @@ func TestGetVersionExtractor(t *testing.T) { }, { name: "Empty provider", - cve: cves.CVE5{}, + cve: models.CVE5{}, expectedType: reflect.TypeOf(&DefaultVersionExtractor{}), }, } @@ -365,7 +366,7 @@ func TestGetVersionExtractor(t *testing.T) { func TestExtractVersions(t *testing.T) { testCases := []struct { name string - cve cves.CVE5 + cve models.CVE5 cnaAssigner string repos []string expectedAffected []*osvschema.Affected diff --git a/vulnfeeds/cves/versions.go b/vulnfeeds/cves/versions.go index 38e80babe55..b8755adfc0e 100644 --- a/vulnfeeds/cves/versions.go +++ b/vulnfeeds/cves/versions.go @@ -683,7 +683,7 @@ func deduplicateAffectedCommits(commits []models.AffectedCommit) []models.Affect return uniqueCommits } -func ExtractVersionInfo(cve CVE, validVersions []string, httpClient *http.Client) (v models.VersionInfo, notes []string) { +func ExtractVersionInfo(cve models.NVDCVE, validVersions []string, httpClient *http.Client) (v models.VersionInfo, notes []string) { for _, reference := range cve.References { // (Potentially faulty) Assumption: All viable Git commit reference links are fix commits. if commit, err := extractGitAffectedCommit(reference.URL, models.Fixed, httpClient); err == nil { @@ -782,7 +782,7 @@ func ExtractVersionInfo(cve CVE, validVersions []string, httpClient *http.Client } if !gotVersions { var extractNotes []string - v.AffectedVersions, extractNotes = ExtractVersionsFromText(validVersions, EnglishDescription(cve.Descriptions)) + v.AffectedVersions, extractNotes = ExtractVersionsFromText(validVersions, models.EnglishDescription(cve.Descriptions)) notes = append(notes, extractNotes...) if len(v.AffectedVersions) > 0 { logger.Info("Extracted versions from description", slog.String("cve", string(cve.ID)), slog.Any("versions", v.AffectedVersions)) @@ -815,7 +815,7 @@ func ExtractVersionInfo(cve CVE, validVersions []string, httpClient *http.Client return v, notes } -func CPEs(cve CVE) []string { +func CPEs(cve models.NVDCVE) []string { var cpes []string for _, config := range cve.Configurations { for _, node := range config.Nodes { @@ -870,7 +870,7 @@ func (vp *VendorProduct) UnmarshalText(text []byte) error { return nil } -func RefAcceptable(ref Reference, tagDenyList []string) bool { +func RefAcceptable(ref models.Reference, tagDenyList []string) bool { for _, deniedTag := range tagDenyList { if slices.Contains(ref.Tags, deniedTag) { return false @@ -923,7 +923,7 @@ func MaybeRemoveFromVPRepoCache(cache VendorProductToRepoMap, vp *VendorProduct, // Takes a CVE ID string (for logging), VersionInfo with AffectedVersions and // typically no AffectedCommits and attempts to add AffectedCommits (including Fixed commits) where there aren't any. // Refuses to add the same commit to AffectedCommits more than once. -func GitVersionsToCommits(cveID CVEID, versions models.VersionInfo, repos []string, cache git.RepoTagsCache) (v models.VersionInfo, e error) { +func GitVersionsToCommits(cveID models.CVEID, versions models.VersionInfo, repos []string, cache git.RepoTagsCache) (v models.VersionInfo, e error) { // versions is a VersionInfo with AffectedVersions and typically no AffectedCommits // v is a VersionInfo with AffectedCommits (containing Fixed commits) included v = versions @@ -1009,7 +1009,7 @@ func GitVersionsToCommits(cveID CVEID, versions models.VersionInfo, repos []stri // Examines the CVE references for a CVE and derives repos for it, optionally caching it. // TODO (jesslowe): refactor with below -func ReposFromReferences(cve string, cache VendorProductToRepoMap, vp *VendorProduct, refs []Reference, tagDenyList []string) (repos []string) { +func ReposFromReferences(cve string, cache VendorProductToRepoMap, vp *VendorProduct, refs []models.Reference, tagDenyList []string) (repos []string) { for _, ref := range refs { // If any of the denylist tags are in the ref's tag set, it's out of consideration. if !RefAcceptable(ref, tagDenyList) { @@ -1046,7 +1046,7 @@ func ReposFromReferences(cve string, cache VendorProductToRepoMap, vp *VendorPro } // Examines the CVE references for a CVE and derives repos for it, optionally caching it. -func ReposFromReferencesCVEList(cve string, refs []Reference, tagDenyList []string) (repos []string, notes []string) { +func ReposFromReferencesCVEList(cve string, refs []models.Reference, tagDenyList []string) (repos []string, notes []string) { for _, ref := range refs { // If any of the denylist tags are in the ref's tag set, it's out of consideration. if !RefAcceptable(ref, tagDenyList) { diff --git a/vulnfeeds/cves/versions_test.go b/vulnfeeds/cves/versions_test.go index 09852d7716d..9ace0f34afa 100644 --- a/vulnfeeds/cves/versions_test.go +++ b/vulnfeeds/cves/versions_test.go @@ -18,13 +18,13 @@ import ( "google.golang.org/protobuf/testing/protocmp" ) -func loadTestData2(cveName string) Vulnerability { +func loadTestData2(cveName string) models.Vulnerability { fileName := fmt.Sprintf("../test_data/nvdcve-2.0/%s.json", cveName) file, err := os.Open(fileName) if err != nil { log.Fatalf("Failed to load test data from %q", fileName) } - var nvdCves CVEAPIJSON20Schema + var nvdCves models.CVEAPIJSON20Schema err = json.NewDecoder(file).Decode(&nvdCves) if err != nil { log.Fatalf("Failed to decode %q: %+v", fileName, err) @@ -36,7 +36,7 @@ func loadTestData2(cveName string) Vulnerability { } log.Fatalf("test data doesn't contain %q", cveName) - return Vulnerability{} + return models.Vulnerability{} } func TestParseCPE(t *testing.T) { @@ -710,7 +710,7 @@ func TestExtractGitCommit(t *testing.T) { func TestExtractVersionInfo(t *testing.T) { tests := []struct { description string - inputCVEItem Vulnerability + inputCVEItem models.Vulnerability inputValidVersions []string expectedVersionInfo models.VersionInfo expectedNotes []string @@ -930,7 +930,7 @@ func TestExtractVersionInfo(t *testing.T) { func TestCPEs(t *testing.T) { tests := []struct { description string - inputCVEItem Vulnerability + inputCVEItem models.Vulnerability expectedCPEs []string }{ { @@ -1286,7 +1286,7 @@ func TestReposFromReferences(t *testing.T) { CVE string cache VendorProductToRepoMap vp *VendorProduct - refs []Reference + refs []models.Reference tagDenyList []string } tests := []struct { @@ -1300,7 +1300,7 @@ func TestReposFromReferences(t *testing.T) { CVE: "CVE-2023-0327", cache: nil, vp: &VendorProduct{"theradsystem_project", "theradsystem"}, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@vuldb.com", Tags: []string{"Patch", "Third Party Advisory"}, @@ -1316,7 +1316,7 @@ func TestReposFromReferences(t *testing.T) { CVE: "CVE-2025-0211", cache: nil, vp: &VendorProduct{"campcodes", "school_faculty_scheduling_system"}, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@vuldb.com", Tags: []string{"Exploit", "Third Party Advisory"}, @@ -1332,7 +1332,7 @@ func TestReposFromReferences(t *testing.T) { CVE: "CVE-2025-26519", cache: nil, vp: nil, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@mitre.org", Tags: nil, @@ -1349,7 +1349,7 @@ func TestReposFromReferences(t *testing.T) { CVE: "CVE-2016-10525", cache: nil, vp: nil, - refs: []Reference{ + refs: []models.Reference{ { Source: "support@hackerone.com", Tags: []string{"Patch", "Third Party Advisory"}, @@ -1376,7 +1376,7 @@ func TestReposFromReferencesCVEList(t *testing.T) { CVE string cache VendorProductToRepoMap vp *VendorProduct - refs []Reference + refs []models.Reference tagDenyList []string } tests := []struct { @@ -1390,7 +1390,7 @@ func TestReposFromReferencesCVEList(t *testing.T) { CVE: "CVE-2023-0327", cache: nil, vp: &VendorProduct{"theradsystem_project", "theradsystem"}, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@vuldb.com", Tags: []string{"Patch", "Third Party Advisory"}, @@ -1406,7 +1406,7 @@ func TestReposFromReferencesCVEList(t *testing.T) { CVE: "CVE-2025-0211", cache: nil, vp: &VendorProduct{"campcodes", "school_faculty_scheduling_system"}, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@vuldb.com", Tags: []string{"Exploit", "Third Party Advisory"}, @@ -1422,7 +1422,7 @@ func TestReposFromReferencesCVEList(t *testing.T) { CVE: "CVE-2025-26519", cache: nil, vp: nil, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@mitre.org", Tags: nil, @@ -1439,7 +1439,7 @@ func TestReposFromReferencesCVEList(t *testing.T) { CVE: "CVE-2016-10525", cache: nil, vp: nil, - refs: []Reference{ + refs: []models.Reference{ { Source: "support@hackerone.com", Tags: []string{"Patch", "Third Party Advisory"}, @@ -1455,7 +1455,7 @@ func TestReposFromReferencesCVEList(t *testing.T) { CVE: "CVE-2024-7790", cache: nil, vp: &VendorProduct{"Devikia", "DevikaAI"}, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@vuldb.com", Tags: []string{"Patch", "Third Party Advisory"}, diff --git a/vulnfeeds/cves/cve.go b/vulnfeeds/models/cve.go similarity index 99% rename from vulnfeeds/cves/cve.go rename to vulnfeeds/models/cve.go index da606e00fb7..75cab7e8a41 100644 --- a/vulnfeeds/cves/cve.go +++ b/vulnfeeds/models/cve.go @@ -13,7 +13,7 @@ // limitations under the License. // Package cves contains CVE-specific data structures. -package cves +package models import ( "strings" diff --git a/vulnfeeds/cves/nvd2.go b/vulnfeeds/models/nvd2.go similarity index 99% rename from vulnfeeds/cves/nvd2.go rename to vulnfeeds/models/nvd2.go index 95d5fd8b653..3b1038fbaa9 100644 --- a/vulnfeeds/cves/nvd2.go +++ b/vulnfeeds/models/nvd2.go @@ -17,7 +17,7 @@ // --capitalization JSON \ // cve_api_json_2.0.schema -package cves +package models import ( "encoding/json" @@ -109,7 +109,7 @@ type CVEAPIJSON20Schema struct { type CVEID string -type CVE struct { +type NVDCVE struct { // CISAActionDue corresponds to the JSON schema field "cisaActionDue". CISAActionDue *types.SerializableDate `json:"cisaActionDue,omitempty" mapstructure:"cisaActionDue,omitempty" yaml:"cisaActionDue,omitempty"` @@ -311,7 +311,7 @@ type LangString struct { } // UnmarshalJSON implements json.Unmarshaler. -func (j *CVE) UnmarshalJSON(b []byte) error { +func (j *NVDCVE) UnmarshalJSON(b []byte) error { var raw map[string]any if err := json.Unmarshal(b, &raw); err != nil { return err @@ -331,7 +331,7 @@ func (j *CVE) UnmarshalJSON(b []byte) error { if v, ok := raw["references"]; !ok || v == nil { return errors.New("field references in CveItem: required") } - type Plain CVE + type Plain NVDCVE var plain Plain if err := json.Unmarshal(b, &plain); err != nil { return err @@ -344,14 +344,14 @@ func (j *CVE) UnmarshalJSON(b []byte) error { // if len(plain.References) > 500 { // return fmt.Errorf("field %s length: must be <= %d", "references", 500) // } - *j = CVE(plain) + *j = NVDCVE(plain) return nil } // (hand generated), see https://github.com/omissis/go-jsonschema/issues/171 type Vulnerability struct { - CVE CVE `json:"cve" mapstructure:"cve" yaml:"cve"` + CVE NVDCVE `json:"cve" mapstructure:"cve" yaml:"cve"` } // CVSS subscore. diff --git a/vulnfeeds/pypi/pypi.go b/vulnfeeds/pypi/pypi.go index fe9b3541b3e..2294199d21d 100644 --- a/vulnfeeds/pypi/pypi.go +++ b/vulnfeeds/pypi/pypi.go @@ -17,6 +17,7 @@ package pypi import ( "encoding/json" + "github.com/google/osv/vulnfeeds/models" "log" "log/slog" "net/http" @@ -244,7 +245,7 @@ func New(pypiLinksPath string, pypiVersionsPath string) *PyPI { } } -func (p *PyPI) Matches(cve cves.CVE, falsePositives *triage.FalsePositives) []string { +func (p *PyPI) Matches(cve models.NVDCVE, falsePositives *triage.FalsePositives) []string { matches := []string{} for _, reference := range cve.References { // If there is a PyPI link, it must be a Python package. These take precedence. @@ -348,9 +349,9 @@ func (p *PyPI) packageExists(pkg string) bool { return result } -func (p *PyPI) finalPkgCheck(cve cves.CVE, pkg string, falsePositives *triage.FalsePositives) bool { +func (p *PyPI) finalPkgCheck(cve models.NVDCVE, pkg string, falsePositives *triage.FalsePositives) bool { // To avoid false positives, check that the pkg name is mentioned in the description. - desc := strings.ToLower(cves.EnglishDescription(cve.Descriptions)) + desc := strings.ToLower(models.EnglishDescription(cve.Descriptions)) pkgNameParts := strings.Split(pkg, "-") for _, part := range pkgNameParts { @@ -375,7 +376,7 @@ func (p *PyPI) finalPkgCheck(cve cves.CVE, pkg string, falsePositives *triage.Fa } // matchesPackage checks if a given reference link matches a PyPI package. -func (p *PyPI) matchesPackage(link string, cve cves.CVE, falsePositives *triage.FalsePositives) []string { +func (p *PyPI) matchesPackage(link string, cve models.NVDCVE, falsePositives *triage.FalsePositives) []string { pkgs := []string{} u, err := url.Parse(strings.ToLower(link)) if err != nil { diff --git a/vulnfeeds/vulns/vulns.go b/vulnfeeds/vulns/vulns.go index 3f6eeb319b0..058f3a16f36 100644 --- a/vulnfeeds/vulns/vulns.go +++ b/vulnfeeds/vulns/vulns.go @@ -34,7 +34,6 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "gopkg.in/yaml.v2" - "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility" "github.com/google/osv/vulnfeeds/utility/logger" @@ -324,7 +323,7 @@ func (v *Vulnerability) AddPkgInfo(pkgInfo PackageInfo) { // getBestSeverity finds the best CVSS severity vector from the provided metrics data. // It prioritizes newer CVSS versions and "Primary" sources. -func getBestSeverity(metricsData *cves.CVEItemMetrics) (string, string) { +func getBestSeverity(metricsData *models.CVEItemMetrics) (string, string) { // Define search passes. First pass for "Primary", second for any. for _, primaryOnly := range []bool{true, false} { // Inside each pass, prioritize v4.0 over v3.1 over v3.0. @@ -350,7 +349,7 @@ func getBestSeverity(metricsData *cves.CVEItemMetrics) (string, string) { // AddSeverity adds CVSS severity information to the OSV vulnerability object. // It uses the highest available CVSS score from the underlying CVE record. -func (v *Vulnerability) AddSeverity(metricsData *cves.CVEItemMetrics) { +func (v *Vulnerability) AddSeverity(metricsData *models.CVEItemMetrics) { bestVectorString, severityType := getBestSeverity(metricsData) if bestVectorString == "" { @@ -588,7 +587,7 @@ func ClassifyReferenceLink(link string, tag string) osvschema.Reference_Type { // ExtractReferencedVulns extracts other vulnerability IDs from a CVE's references // to place them into the aliases and related fields. -func ExtractReferencedVulns(id cves.CVEID, cveID cves.CVEID, references []cves.Reference) ([]string, []string) { +func ExtractReferencedVulns(id models.CVEID, cveID models.CVEID, references []models.Reference) ([]string, []string) { var aliases []string var related []string if id != cveID { @@ -663,7 +662,7 @@ func Unique[T comparable](s []T) []T { } // ClassifyReferences annotates reference links based on their tags or their shape. -func ClassifyReferences(refs []cves.Reference) []*osvschema.Reference { +func ClassifyReferences(refs []models.Reference) []*osvschema.Reference { var references []*osvschema.Reference refMap := make(map[string]map[osvschema.Reference_Type]bool) @@ -705,12 +704,12 @@ func ClassifyReferences(refs []cves.Reference) []*osvschema.Reference { // Leaves affected and version fields empty to be filled in later with AddPkgInfo // There are two id fields passed in as one of the users of this field (PyPi) sometimes has a different id than the CVEID // and the ExtractReferencedVulns function uses these in a check to add the other ID as an alias. -func FromNVDCVE(id cves.CVEID, cve cves.CVE) *Vulnerability { +func FromNVDCVE(id models.CVEID, cve models.NVDCVE) *Vulnerability { aliases, related := ExtractReferencedVulns(id, cve.ID, cve.References) v := &Vulnerability{ Vulnerability: &osvschema.Vulnerability{ Id: string(id), - Details: cves.EnglishDescription(cve.Descriptions), + Details: models.EnglishDescription(cve.Descriptions), Aliases: aliases, Related: related, Published: timestamppb.New(cve.Published.Time), @@ -723,9 +722,9 @@ func FromNVDCVE(id cves.CVEID, cve cves.CVE) *Vulnerability { return v } -// GetCPEs extracts CPE strings from a slice of cves.CPE. +// GetCPEs extracts CPE strings from a slice of models.CPE. // Returns array of CPE strings and array of notes. -func GetCPEs(cpeApplicability []cves.CPE) ([]string, []string) { +func GetCPEs(cpeApplicability []models.CPE) ([]string, []string) { var CPEs []string var notes []string for _, c := range cpeApplicability { @@ -796,13 +795,13 @@ func CheckQuality(text string) QualityCheck { } // LoadAllCVEs loads the downloaded CVE's from the NVD database into memory. -func LoadAllCVEs(cvePath string) map[cves.CVEID]cves.Vulnerability { +func LoadAllCVEs(cvePath string) map[models.CVEID]models.Vulnerability { dir, err := os.ReadDir(cvePath) if err != nil { logger.Fatal("Failed to read dir", slog.String("path", cvePath), slog.Any("err", err)) } - vulnsChan := make(chan cves.Vulnerability) + vulnsChan := make(chan models.Vulnerability) var wg sync.WaitGroup for _, entry := range dir { @@ -821,7 +820,7 @@ func LoadAllCVEs(cvePath string) map[cves.CVEID]cves.Vulnerability { } defer file.Close() - var nvdcve cves.CVEAPIJSON20Schema + var nvdcve models.CVEAPIJSON20Schema if err := json.NewDecoder(file).Decode(&nvdcve); err != nil { logger.Error("Failed to decode JSON", slog.String("file", filename), slog.Any("err", err)) return @@ -839,7 +838,7 @@ func LoadAllCVEs(cvePath string) map[cves.CVEID]cves.Vulnerability { close(vulnsChan) }() - result := make(map[cves.CVEID]cves.Vulnerability) + result := make(map[models.CVEID]models.Vulnerability) for item := range vulnsChan { result[item.CVE.ID] = item } @@ -847,7 +846,7 @@ func LoadAllCVEs(cvePath string) map[cves.CVEID]cves.Vulnerability { return result } -func FindSeverity(metricsData []cves.Metrics) *osvschema.Severity { +func FindSeverity(metricsData []models.Metrics) *osvschema.Severity { bestVectorString, severityType := getBestCVE5Severity(metricsData) if bestVectorString == "" { return nil @@ -859,14 +858,14 @@ func FindSeverity(metricsData []cves.Metrics) *osvschema.Severity { } } -func getBestCVE5Severity(metricsData []cves.Metrics) (string, osvschema.Severity_Type) { +func getBestCVE5Severity(metricsData []models.Metrics) (string, osvschema.Severity_Type) { checks := []struct { - getVectorString func(cves.Metrics) string + getVectorString func(models.Metrics) string severityType osvschema.Severity_Type }{ - {func(m cves.Metrics) string { return m.CVSSv4_0.VectorString }, osvschema.Severity_CVSS_V4}, - {func(m cves.Metrics) string { return m.CVSSv3_1.VectorString }, osvschema.Severity_CVSS_V3}, - {func(m cves.Metrics) string { return m.CVSSv3_0.VectorString }, osvschema.Severity_CVSS_V3}, + {func(m models.Metrics) string { return m.CVSSv4_0.VectorString }, osvschema.Severity_CVSS_V4}, + {func(m models.Metrics) string { return m.CVSSv3_1.VectorString }, osvschema.Severity_CVSS_V3}, + {func(m models.Metrics) string { return m.CVSSv3_0.VectorString }, osvschema.Severity_CVSS_V3}, } for _, check := range checks { diff --git a/vulnfeeds/vulns/vulns_test.go b/vulnfeeds/vulns/vulns_test.go index 8d9d57663c1..e5c84aa1586 100644 --- a/vulnfeeds/vulns/vulns_test.go +++ b/vulnfeeds/vulns/vulns_test.go @@ -13,7 +13,6 @@ import ( "slices" gocmp "github.com/google/go-cmp/cmp" - "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -81,11 +80,11 @@ func TestClassifyReferenceLink(t *testing.T) { func TestClassifyReferences(t *testing.T) { testcases := []struct { - refData []cves.Reference + refData []models.Reference references []*osvschema.Reference }{ { - refData: []cves.Reference{ + refData: []models.Reference{ { Source: "https://example.com", Tags: []string{"MISC"}, URL: "https://example.com", }, @@ -93,7 +92,7 @@ func TestClassifyReferences(t *testing.T) { references: []*osvschema.Reference{{Url: "https://example.com", Type: osvschema.Reference_WEB}}, }, { - refData: []cves.Reference{ + refData: []models.Reference{ { Source: "https://github.com/Netflix/lemur/issues/117", URL: "https://github.com/Netflix/lemur/issues/117", Tags: []string{"MISC", "Issue Tracking"}, }, @@ -101,7 +100,7 @@ func TestClassifyReferences(t *testing.T) { references: []*osvschema.Reference{{Url: "https://github.com/Netflix/lemur/issues/117", Type: osvschema.Reference_REPORT}}, }, { - refData: []cves.Reference{ + refData: []models.Reference{ { Source: "https://github.com/curl/curl/issues/9271", URL: "https://github.com/curl/curl/issues/9271", Tags: []string{"MISC", "Exploit", "Issue Tracking", "Third Party Advisory"}, }, @@ -113,7 +112,7 @@ func TestClassifyReferences(t *testing.T) { }, }, { - refData: []cves.Reference{ + refData: []models.Reference{ { Source: "https://gitlab.com/gitlab-org/gitlab/-/issues/517693", URL: "https://gitlab.com/gitlab-org/gitlab/-/issues/517693", Tags: []string{"issue-tracking", "permissions-required"}, }, @@ -123,7 +122,7 @@ func TestClassifyReferences(t *testing.T) { }, }, { - refData: []cves.Reference{ + refData: []models.Reference{ { Source: "https://security.gentoo.org/glsa/202307-01", URL: "https://security.gentoo.org/glsa/202307-01", Tags: []string{"vendor-advisory"}, }, @@ -133,7 +132,7 @@ func TestClassifyReferences(t *testing.T) { }, }, { - refData: []cves.Reference{ + refData: []models.Reference{ { Source: "http://www.openwall.com/lists/oss-security/2023/07/20/1", URL: "http://www.openwall.com/lists/oss-security/2023/07/20/1", Tags: []string{"mailing-list"}, }, @@ -154,13 +153,13 @@ func TestClassifyReferences(t *testing.T) { } } -func loadTestData2(cveName string) cves.Vulnerability { +func loadTestData2(cveName string) models.Vulnerability { fileName := fmt.Sprintf("../test_data/nvdcve-2.0/%s.json", cveName) file, err := os.Open(fileName) if err != nil { log.Fatalf("Failed to load test data from %q", fileName) } - var nvdCves cves.CVEAPIJSON20Schema + var nvdCves models.CVEAPIJSON20Schema err = json.NewDecoder(file).Decode(&nvdCves) if err != nil { log.Fatalf("Failed to decode %q: %+v", fileName, err) @@ -172,7 +171,7 @@ func loadTestData2(cveName string) cves.Vulnerability { } log.Fatalf("test data doesn't contain %q", cveName) - return cves.Vulnerability{} + return models.Vulnerability{} } func TestExtractAliases(t *testing.T) { @@ -198,7 +197,7 @@ func TestExtractAliases(t *testing.T) { func TestEnglishDescription(t *testing.T) { cveItem := loadTestData2("CVE-2022-36037") - description := cves.EnglishDescription(cveItem.CVE.Descriptions) + description := models.EnglishDescription(cveItem.CVE.Descriptions) expectedDescription := "kirby is a content management system (CMS) that adapts to many different projects and helps you build your own ideal interface. Cross-site scripting (XSS) is a type of vulnerability that allows execution of any kind of JavaScript code inside the Panel session of the same or other users. In the Panel, a harmful script can for example trigger requests to Kirby's API with the permissions of the victim. If bad actors gain access to your group of authenticated Panel users they can escalate their privileges via the Panel session of an admin user. Depending on your site, other JavaScript-powered attacks are possible. The multiselect field allows selection of tags from an autocompleted list. Unfortunately, the Panel in Kirby 3.5 used HTML rendering for the raw option value. This allowed **attackers with influence on the options source** to store HTML code. The browser of the victim who visited a page with manipulated multiselect options in the Panel will then have rendered this malicious HTML code when the victim opened the autocomplete dropdown. Users are *not* affected by this vulnerability if you don't use the multiselect field or don't use it with options that can be manipulated by attackers. The problem has been patched in Kirby 3.5.8.1." if description != expectedDescription { t.Errorf("Description not extracted, got %v, but expected %v", description, expectedDescription) @@ -407,7 +406,7 @@ func TestAddPkgInfo(t *testing.T) { func TestAddSeverity(t *testing.T) { tests := []struct { description string - inputCVE cves.Vulnerability + inputCVE models.Vulnerability expectedResult []*osvschema.Severity }{ { From 366145e7200eb3c3d7e92416ab3c532f3ccff988 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 21 Jan 2026 00:06:01 +0000 Subject: [PATCH 09/14] move ConversionOutcomes into models for NVD and CVE to share. --- .../cmd/converters/cve/nvd-cve-osv/main.go | 39 +++------- vulnfeeds/cvelist2osv/common.go | 29 +------- vulnfeeds/cvelist2osv/common_test.go | 3 +- vulnfeeds/cvelist2osv/converter.go | 71 +++---------------- vulnfeeds/cvelist2osv/converter_test.go | 2 +- vulnfeeds/cvelist2osv/default_extractor.go | 8 +-- vulnfeeds/cvelist2osv/extraction.go | 4 +- vulnfeeds/cvelist2osv/linux_extractor.go | 10 +-- vulnfeeds/cvelist2osv/strategies.go | 10 +-- .../cvelist2osv/version_extraction_test.go | 11 +-- 10 files changed, 46 insertions(+), 141 deletions(-) diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go index fadbeab6502..a1c8c4d6898 100644 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go @@ -22,31 +22,14 @@ import ( "github.com/google/osv/vulnfeeds/vulns" ) -type ConversionOutcome int - var ErrNoRanges = errors.New("no ranges") var ErrUnresolvedFix = errors.New("fixes not resolved to commits") -func (c ConversionOutcome) String() string { - return [...]string{"ConversionUnknown", "Successful", "Rejected", "NoSoftware", "NoRepos", "NoRanges", "FixUnresolvable"}[c] -} - const ( extension = ".json" ) -const ( - // Set of enums for categorizing conversion outcomes. - ConversionUnknown ConversionOutcome = iota // Shouldn't happen - Successful // It worked! - Rejected // The CVE was rejected - NoSoftware // The CVE had no CPEs relating to software (i.e. Operating Systems or Hardware). - NoRepos // The CPE Vendor/Product had no repositories derived for it. - NoRanges // No viable commit ranges could be calculated from the repository for the CVE's CPE(s). - FixUnresolvable // Partial resolution of versions, resulting in a false positive. -) - var ( jsonPath = flag.String("nvd-json", "", "Path to NVD CVE JSON to examine.") parsedCPEDictionary = flag.String("cpe-repos", "", "Path to JSON mapping of CPEs to repos generated by cpe-repo-gen") @@ -59,7 +42,7 @@ var Metrics struct { CVEsForApplications int CVEsForKnownRepos int OSVRecordsGenerated int - Outcomes map[models.CVEID]ConversionOutcome // Per-CVE-ID record of conversion result. + Outcomes map[models.CVEID]models.ConversionOutcome // Per-CVE-ID record of conversion result. } // Takes an NVD CVE record and outputs an OSV file in the specified directory. @@ -268,7 +251,7 @@ func loadCPEDictionary(productToRepo *cves.VendorProductToRepoMap, f string) err } // Output a CSV summarizing per-CVE how it was handled. -func outputOutcomes(outcomes map[models.CVEID]ConversionOutcome, reposForCVE map[models.CVEID][]string, directory string) error { +func outputOutcomes(outcomes map[models.CVEID]models.ConversionOutcome, reposForCVE map[models.CVEID][]string, directory string) error { outcomesFile, err := os.Create(filepath.Join(directory, "outcomes.csv")) if err != nil { return err @@ -304,7 +287,7 @@ func main() { os.Exit(1) } - Metrics.Outcomes = make(map[models.CVEID]ConversionOutcome) + Metrics.Outcomes = make(map[models.CVEID]models.ConversionOutcome) logger.InitGlobalLogger() @@ -339,7 +322,7 @@ func main() { if len(refs) == 0 && len(CPEs) == 0 { logger.Info("Skipping due to lack of CPEs and lack of references", slog.String("cve", string(CVEID))) // 100% of these in 2022 were rejected CVEs - Metrics.Outcomes[CVEID] = Rejected + Metrics.Outcomes[CVEID] = models.Rejected continue } @@ -361,7 +344,7 @@ func main() { CPE, err := cves.ParseCPE(CPEstr) if err != nil { logger.Warn("Failed to parse CPE", slog.String("cve", string(CVEID)), slog.String("cpe", CPEstr), slog.Any("err", err)) - Metrics.Outcomes[CVEID] = ConversionUnknown + Metrics.Outcomes[CVEID] = models.ConversionUnknown continue } @@ -386,7 +369,7 @@ func main() { if len(CPEs) > 0 && appCPECount == 0 { // This CVE is not for software (based on there being CPEs but not any application ones), skip. - Metrics.Outcomes[CVEID] = NoSoftware + Metrics.Outcomes[CVEID] = models.NoSoftware continue } @@ -441,7 +424,7 @@ func main() { if _, ok := ReposForCVE[CVEID]; !ok { // We have nothing useful to work with, so we'll assume it's out of scope logger.Info("Passing due to lack of viable repository", slog.String("cve", string(CVEID))) - Metrics.Outcomes[CVEID] = NoRepos + Metrics.Outcomes[CVEID] = models.NoRepos continue } @@ -460,19 +443,19 @@ func main() { if err != nil { logger.Warn("Failed to generate an OSV record", slog.String("cve", string(CVEID)), slog.Any("err", err)) if errors.Is(err, ErrNoRanges) { - Metrics.Outcomes[CVEID] = NoRanges + Metrics.Outcomes[CVEID] = models.NoRanges continue } if errors.Is(err, ErrUnresolvedFix) { - Metrics.Outcomes[CVEID] = FixUnresolvable + Metrics.Outcomes[CVEID] = models.FixUnresolvable continue } - Metrics.Outcomes[CVEID] = ConversionUnknown + Metrics.Outcomes[CVEID] = models.ConversionUnknown continue } Metrics.OSVRecordsGenerated++ - Metrics.Outcomes[CVEID] = Successful + Metrics.Outcomes[CVEID] = models.Successful } Metrics.TotalCVEs = len(parsed.Vulnerabilities) err = outputOutcomes(Metrics.Outcomes, ReposForCVE, *outDir) diff --git a/vulnfeeds/cvelist2osv/common.go b/vulnfeeds/cvelist2osv/common.go index 53a541c51fb..15e99c3af32 100644 --- a/vulnfeeds/cvelist2osv/common.go +++ b/vulnfeeds/cvelist2osv/common.go @@ -29,31 +29,6 @@ const ( VersionRangeTypeEcosystem ) -// VersionSource indicates the source of the extracted version information. -type VersionSource string - -const ( - VersionSourceNone VersionSource = "NOVERS" - VersionSourceAffected VersionSource = "CVEAFFVERS" - VersionSourceGit VersionSource = "GITVERS" - VersionSourceCPE VersionSource = "CPEVERS" - VersionSourceDescription VersionSource = "DESCRVERS" -) - -type ConversionOutcome int - -const ( - // Set of enums for categorizing conversion outcomes. - ConversionUnknown ConversionOutcome = iota // Shouldn't happen - Successful // It worked! - Rejected // The CVE was rejected - NoSoftware // The CVE had no CPEs relating to software (i.e. Operating Systems or Hardware). - NoRepos // The CPE Vendor/Product had no repositories derived for it. - NoCommitRanges // No viable commit ranges could be calculated from the repository for the CVE's CPE(s). - NoRanges // No version ranges could be extracted from the record. - FixUnresolvable // Partial resolution of versions, resulting in a false positive. -) - // String returns the string representation of a VersionRangeType. func (vrt VersionRangeType) String() string { switch vrt { @@ -102,7 +77,7 @@ func resolveVersionToCommit(cveID models.CVEID, version, versionType, repo strin // Takes a CVE ID string (for logging), VersionInfo with AffectedVersions and // typically no AffectedCommits and attempts to add AffectedCommits (including Fixed commits) where there aren't any. // Refuses to add the same commit to AffectedCommits more than once. -func gitVersionsToCommits(cveID models.CVEID, versionRanges []*osvschema.Range, repos []string, metrics *ConversionMetrics, cache git.RepoTagsCache) (*osvschema.Affected, error) { +func gitVersionsToCommits(cveID models.CVEID, versionRanges []*osvschema.Range, repos []string, metrics *models.ConversionMetrics, cache git.RepoTagsCache) (*osvschema.Affected, error) { var newAff osvschema.Affected var newVersionRanges []*osvschema.Range unresolvedRanges := versionRanges @@ -274,7 +249,7 @@ func compareSemverLike(a, b string) int { } // addAffected adds an osvschema.Affected to a vulnerability, ensuring that no duplicate ranges are added. -func addAffected(v *vulns.Vulnerability, aff *osvschema.Affected, metrics *ConversionMetrics) { +func addAffected(v *vulns.Vulnerability, aff *osvschema.Affected, metrics *models.ConversionMetrics) { allExistingRanges := make(map[string]struct{}) for _, existingAff := range v.Affected { for _, r := range existingAff.GetRanges() { diff --git a/vulnfeeds/cvelist2osv/common_test.go b/vulnfeeds/cvelist2osv/common_test.go index f291da87395..db9cd1798ef 100644 --- a/vulnfeeds/cvelist2osv/common_test.go +++ b/vulnfeeds/cvelist2osv/common_test.go @@ -1,6 +1,7 @@ package cvelist2osv import ( + "github.com/google/osv/vulnfeeds/models" "testing" "github.com/google/go-cmp/cmp" @@ -55,7 +56,7 @@ func TestAddAffected(t *testing.T) { }, }, } - metrics := &ConversionMetrics{} + metrics := &models.ConversionMetrics{} addAffected(v, aff, metrics) diff --git a/vulnfeeds/cvelist2osv/converter.go b/vulnfeeds/cvelist2osv/converter.go index 4b33eb13cf4..06026efd6cf 100644 --- a/vulnfeeds/cvelist2osv/converter.go +++ b/vulnfeeds/cvelist2osv/converter.go @@ -23,48 +23,10 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -const ( - extension = ".json" -) - -// ConversionMetrics holds the collected data about the conversion process for a single CVE. -type ConversionMetrics struct { - CVEID models.CVEID `json:"id"` // The CVE ID - CNA string `json:"cna"` // The CNA that assigned the CVE. - Outcome ConversionOutcome `json:"outcome"` // The final outcome of the conversion (e.g., "Successful", "Failed"). - Repos []string `json:"repos"` // A list of repositories extracted from the CVE's references. - RefTypesCount map[osvschema.Reference_Type]int `json:"ref_types_count"` // A count of each type of reference found. - VersionSources []VersionSource `json:"version_sources"` // A list of the ways the versions were extracted - Notes []string `json:"notes"` // A collection of notes and warnings generated during conversion. - CPEs []string `json:"cpes"` - UnresolvedRangesCount int `json:"unresolved_ranges_count"` - ResolvedRangesCount int `json:"resolved_ranges_count"` -} - -// AddNote adds a formatted note to the ConversionMetrics. -func (m *ConversionMetrics) AddNote(format string, a ...any) { - m.Notes = append(m.Notes, fmt.Sprintf(format, a...)) - logger.Debug(fmt.Sprintf(format, a...), slog.String("cna", m.CNA), slog.String("cve", string(m.CVEID))) -} - -// AddSource appends a source to the ConversionMetrics -func (m *ConversionMetrics) AddSource(source VersionSource) { - m.VersionSources = append(m.VersionSources, source) -} - -// RefTagDenyList contains reference tags that are often associated with unreliable or -// irrelevant repository URLs. References with these tags are currently ignored -// to avoid incorrect repository associations. -var RefTagDenyList = []string{ - // "Exploit", - // "Third Party Advisory", - "Broken Link", // Actively ignore these. -} - // extractConversionMetrics examines a CVE and its generated OSV references to populate // the ConversionMetrics struct with heuristics about the conversion process. // It captures the assigning CNA and counts the occurrences of each reference type. -func extractConversionMetrics(cve models.CVE5, refs []*osvschema.Reference, metrics *ConversionMetrics) { +func extractConversionMetrics(cve models.CVE5, refs []*osvschema.Reference, metrics *models.ConversionMetrics) { // Capture the CNA for heuristic analysis. metrics.CNA = cve.Metadata.AssignerShortName // TODO(jesslowe): more CNA based analysis @@ -83,7 +45,7 @@ func extractConversionMetrics(cve models.CVE5, refs []*osvschema.Reference, metr } // getCWEs extracts and adds CWE IDs from the CVE5 problem-types -func getCWEs(cna models.CNA, metrics *ConversionMetrics) []string { +func getCWEs(cna models.CNA, metrics *models.ConversionMetrics) []string { var cwes []string for _, pt := range cna.ProblemTypes { @@ -110,7 +72,7 @@ func getCWEs(cna models.CNA, metrics *ConversionMetrics) []string { // FromCVE5 creates a `vulns.Vulnerability` object from a `models.CVE5` object. // It populates the main fields of the OSV record, including ID, summary, details, // references, timestamps, severity, and version information. -func FromCVE5(cve models.CVE5, refs []models.Reference, metrics *ConversionMetrics, sourceLink string) *vulns.Vulnerability { +func FromCVE5(cve models.CVE5, refs []models.Reference, metrics *models.ConversionMetrics, sourceLink string) *vulns.Vulnerability { aliases, related := vulns.ExtractReferencedVulns(cve.Metadata.CVEID, cve.Metadata.CVEID, refs) v := vulns.Vulnerability{ Vulnerability: &osvschema.Vulnerability{ @@ -138,7 +100,7 @@ func FromCVE5(cve models.CVE5, refs []models.Reference, metrics *ConversionMetri v.Modified = timestamppb.New(modified) // Try to extract repository URLs from references. - repos, repoNotes := cves.ReposFromReferencesCVEList(string(cve.Metadata.CVEID), refs, RefTagDenyList) + repos, repoNotes := cves.ReposFromReferencesCVEList(string(cve.Metadata.CVEID), refs, models.RefTagDenyList) for _, note := range repoNotes { metrics.AddNote("%s", note) } @@ -186,7 +148,7 @@ func FromCVE5(cve models.CVE5, refs []models.Reference, metrics *ConversionMetri // CreateOSVFile creates the initial file for the OSV record. func CreateOSVFile(id models.CVEID, vulnDir string) (*os.File, error) { - outputFile := filepath.Join(vulnDir, string(id)+extension) + outputFile := filepath.Join(vulnDir, string(id)+models.Extension) f, err := os.Create(outputFile) if err != nil { @@ -211,23 +173,6 @@ func CreateMetricsFile(id models.CVEID, vulnDir string) (*os.File, error) { return f, nil } -func determineOutcome(metrics *ConversionMetrics) { - // check if we have affected ranges/versions. - if len(metrics.Repos) == 0 { - // Fix unlikely, as no repos to resolve - metrics.Outcome = NoRepos - return - } - - if metrics.ResolvedRangesCount > 0 { - metrics.Outcome = Successful - } else if metrics.UnresolvedRangesCount > 0 { - metrics.Outcome = NoCommitRanges - } else { - metrics.Outcome = NoRanges - } -} - // ConvertAndExportCVEToOSV is the main function for this file. It takes a CVE, // converts it into an OSV record, collects metrics, and writes both to disk. func ConvertAndExportCVEToOSV(cve models.CVE5, vulnSink io.Writer, metricsSink io.Writer, sourceLink string) error { @@ -243,7 +188,7 @@ func ConvertAndExportCVEToOSV(cve models.CVE5, vulnSink io.Writer, metricsSink i references = deduplicateRefs(references) - metrics := ConversionMetrics{CVEID: cveID, CNA: cnaAssigner, UnresolvedRangesCount: 0, ResolvedRangesCount: 0} + metrics := models.ConversionMetrics{CVEID: cveID, CNA: cnaAssigner, UnresolvedRangesCount: 0, ResolvedRangesCount: 0} // Create a base OSV record from the CVE. v := FromCVE5(cve, references, &metrics, sourceLink) @@ -257,7 +202,7 @@ func ConvertAndExportCVEToOSV(cve models.CVE5, vulnSink io.Writer, metricsSink i groupAffectedRanges(v.Affected) - determineOutcome(&metrics) + models.DetermineOutcome(&metrics) err := v.ToJSON(vulnSink) if err != nil { @@ -324,7 +269,7 @@ func deduplicateRefs(refs []models.Reference) []models.Reference { return refs } -func buildDBSpecific(cve models.CVE5, metrics *ConversionMetrics, sourceLink string) map[string]any { +func buildDBSpecific(cve models.CVE5, metrics *models.ConversionMetrics, sourceLink string) map[string]any { dbSpecific := make(map[string]any) if sourceLink != "" { diff --git a/vulnfeeds/cvelist2osv/converter_test.go b/vulnfeeds/cvelist2osv/converter_test.go index d0e5c5b1119..b4930099a1c 100644 --- a/vulnfeeds/cvelist2osv/converter_test.go +++ b/vulnfeeds/cvelist2osv/converter_test.go @@ -340,7 +340,7 @@ func TestFromCVE5(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - metrics := &ConversionMetrics{} + metrics := &models.ConversionMetrics{} vuln := FromCVE5(tc.cve, tc.refs, metrics, "") // Handle non-deterministic time.Now() diff --git a/vulnfeeds/cvelist2osv/default_extractor.go b/vulnfeeds/cvelist2osv/default_extractor.go index b04d5436124..e559c195f38 100644 --- a/vulnfeeds/cvelist2osv/default_extractor.go +++ b/vulnfeeds/cvelist2osv/default_extractor.go @@ -15,7 +15,7 @@ import ( // DefaultVersionExtractor provides the default version extraction logic. type DefaultVersionExtractor struct{} -func (d *DefaultVersionExtractor) handleAffected(affected []models.Affected, metrics *ConversionMetrics) []*osvschema.Range { +func (d *DefaultVersionExtractor) handleAffected(affected []models.Affected, metrics *models.ConversionMetrics) []*osvschema.Range { var ranges []*osvschema.Range for _, cveAff := range affected { versionRanges, _ := d.FindNormalAffectedRanges(cveAff, metrics) @@ -24,14 +24,14 @@ func (d *DefaultVersionExtractor) handleAffected(affected []models.Affected, met continue } ranges = append(ranges, versionRanges...) - metrics.AddSource(VersionSourceAffected) + metrics.AddSource(models.VersionSourceAffected) } return ranges } // ExtractVersions for DefaultVersionExtractor. -func (d *DefaultVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, repos []string) { +func (d *DefaultVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *models.ConversionMetrics, repos []string) { gotVersions := false repoTagsCache := git.RepoTagsCache{} @@ -77,7 +77,7 @@ func (d *DefaultVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vuln } } -func (d *DefaultVersionExtractor) FindNormalAffectedRanges(affected models.Affected, metrics *ConversionMetrics) ([]*osvschema.Range, VersionRangeType) { +func (d *DefaultVersionExtractor) FindNormalAffectedRanges(affected models.Affected, metrics *models.ConversionMetrics) ([]*osvschema.Range, VersionRangeType) { versionTypesCount := make(map[VersionRangeType]int) var versionRanges []*osvschema.Range for _, vers := range affected.Versions { diff --git a/vulnfeeds/cvelist2osv/extraction.go b/vulnfeeds/cvelist2osv/extraction.go index 13e17d3767d..88387753d95 100644 --- a/vulnfeeds/cvelist2osv/extraction.go +++ b/vulnfeeds/cvelist2osv/extraction.go @@ -8,8 +8,8 @@ import ( // VersionExtractor defines the interface for different version extraction strategies. type VersionExtractor interface { - ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, repos []string) - FindNormalAffectedRanges(affected models.Affected, metrics *ConversionMetrics) ([]*osvschema.Range, VersionRangeType) + ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *models.ConversionMetrics, repos []string) + FindNormalAffectedRanges(affected models.Affected, metrics *models.ConversionMetrics) ([]*osvschema.Range, VersionRangeType) } // GetVersionExtractor returns the appropriate VersionExtractor for a given CNA. diff --git a/vulnfeeds/cvelist2osv/linux_extractor.go b/vulnfeeds/cvelist2osv/linux_extractor.go index 1eafb3938cb..5584b76be55 100644 --- a/vulnfeeds/cvelist2osv/linux_extractor.go +++ b/vulnfeeds/cvelist2osv/linux_extractor.go @@ -22,7 +22,7 @@ type LinuxVersionExtractor struct { var _ VersionExtractor = &LinuxVersionExtractor{} // handleAffected takes an array of models.Affected and handles how to extract them -func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected []models.Affected, metrics *ConversionMetrics) bool { +func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected []models.Affected, metrics *models.ConversionMetrics) bool { hasGit := false gotVersions := false for _, cveAff := range affected { @@ -43,7 +43,7 @@ func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected hasGit = true } aff := createLinuxAffected(versionRanges, versionType, cveAff.Repo) - metrics.AddSource(VersionSourceAffected) + metrics.AddSource(models.VersionSourceAffected) addAffected(v, aff, metrics) } @@ -51,7 +51,7 @@ func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected } // ExtractVersions for LinuxVersionExtractor. -func (l *LinuxVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, _ []string) { +func (l *LinuxVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *models.ConversionMetrics, _ []string) { gotVersions := l.handleAffected(v, cve.Containers.CNA.Affected, metrics) if !gotVersions { @@ -90,7 +90,7 @@ func createLinuxAffected(versionRanges []*osvschema.Range, versionType VersionRa // of 'unaffected' versions. This is common in Linux kernel CVEs where a product is // considered affected by default, and only unaffected versions are listed. // It sorts the introduced and fixed versions to create chronological ranges. -func findInverseAffectedRanges(cveAff models.Affected, metrics *ConversionMetrics) (ranges []*osvschema.Range, versType VersionRangeType) { +func findInverseAffectedRanges(cveAff models.Affected, metrics *models.ConversionMetrics) (ranges []*osvschema.Range, versType VersionRangeType) { var introduced []string fixed := make([]string, 0, len(cveAff.Versions)) for _, vers := range cveAff.Versions { @@ -151,7 +151,7 @@ func findInverseAffectedRanges(cveAff models.Affected, metrics *ConversionMetric return nil, VersionRangeTypeUnknown } -func (l *LinuxVersionExtractor) FindNormalAffectedRanges(affected models.Affected, metrics *ConversionMetrics) ([]*osvschema.Range, VersionRangeType) { +func (l *LinuxVersionExtractor) FindNormalAffectedRanges(affected models.Affected, metrics *models.ConversionMetrics) ([]*osvschema.Range, VersionRangeType) { versionTypesCount := make(map[VersionRangeType]int) var versionRanges []*osvschema.Range for _, vers := range affected.Versions { diff --git a/vulnfeeds/cvelist2osv/strategies.go b/vulnfeeds/cvelist2osv/strategies.go index d3db99004fb..20cc1ae60be 100644 --- a/vulnfeeds/cvelist2osv/strategies.go +++ b/vulnfeeds/cvelist2osv/strategies.go @@ -7,10 +7,10 @@ import ( "github.com/ossf/osv-schema/bindings/go/osvschema" ) -func cpeVersionExtraction(cve models.CVE5, metrics *ConversionMetrics) ([]*osvschema.Range, error) { +func cpeVersionExtraction(cve models.CVE5, metrics *models.ConversionMetrics) ([]*osvschema.Range, error) { cpeRanges, cpeStrings, err := findCPEVersionRanges(cve) if err == nil && len(cpeRanges) > 0 { - metrics.VersionSources = append(metrics.VersionSources, VersionSourceCPE) + metrics.VersionSources = append(metrics.VersionSources, models.VersionSourceCPE) metrics.CPEs = vulns.Unique(cpeStrings) return cpeRanges, nil @@ -22,7 +22,7 @@ func cpeVersionExtraction(cve models.CVE5, metrics *ConversionMetrics) ([]*osvsc } // textVersionExtraction is a helper function for CPE and description extraction. -func textVersionExtraction(cve models.CVE5, metrics *ConversionMetrics) []*osvschema.Range { +func textVersionExtraction(cve models.CVE5, metrics *models.ConversionMetrics) []*osvschema.Range { // As a last resort, try extracting versions from the description text. versions, extractNotes := cves.ExtractVersionsFromText(nil, models.EnglishDescription(cve.Containers.CNA.Descriptions)) for _, note := range extractNotes { @@ -30,7 +30,7 @@ func textVersionExtraction(cve models.CVE5, metrics *ConversionMetrics) []*osvsc } if len(versions) > 0 { // NOTE: These versions are not currently saved due to the need for better validation. - metrics.VersionSources = append(metrics.VersionSources, VersionSourceDescription) + metrics.VersionSources = append(metrics.VersionSources, models.VersionSourceDescription) metrics.AddNote("Extracted versions from description but did not save them: %+v", versions) } @@ -38,7 +38,7 @@ func textVersionExtraction(cve models.CVE5, metrics *ConversionMetrics) []*osvsc } // initialNormalExtraction handles an expected case of version ranges in the affected field of CVE5 -func initialNormalExtraction(vers models.Versions, metrics *ConversionMetrics, versionTypesCount map[VersionRangeType]int) ([]*osvschema.Range, VersionRangeType, bool) { +func initialNormalExtraction(vers models.Versions, metrics *models.ConversionMetrics, versionTypesCount map[VersionRangeType]int) ([]*osvschema.Range, VersionRangeType, bool) { if vers.Status != "affected" { return nil, VersionRangeTypeUnknown, true } diff --git a/vulnfeeds/cvelist2osv/version_extraction_test.go b/vulnfeeds/cvelist2osv/version_extraction_test.go index 0015d548e67..46513f8314e 100644 --- a/vulnfeeds/cvelist2osv/version_extraction_test.go +++ b/vulnfeeds/cvelist2osv/version_extraction_test.go @@ -1,11 +1,12 @@ package cvelist2osv import ( - "github.com/google/osv/vulnfeeds/models" "reflect" "sort" "testing" + "github.com/google/osv/vulnfeeds/models" + "github.com/google/go-cmp/cmp" "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/vulns" @@ -114,7 +115,7 @@ func TestFindNormalAffectedRanges(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { versionExtractor := &DefaultVersionExtractor{} - gotRanges, gotRangeType := versionExtractor.FindNormalAffectedRanges(tt.affected, &ConversionMetrics{}) + gotRanges, gotRangeType := versionExtractor.FindNormalAffectedRanges(tt.affected, &models.ConversionMetrics{}) if diff := cmp.Diff(tt.wantRanges, gotRanges, protocmp.Transform()); diff != "" { t.Errorf("findNormalAffectedRanges() ranges mismatch (-want +got):\n%s", diff) } @@ -221,7 +222,7 @@ func TestFindInverseAffectedRanges(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - metrics := &ConversionMetrics{} + metrics := &models.ConversionMetrics{} gotRanges, gotVersionType := findInverseAffectedRanges(tt.affected, metrics) if diff := cmp.Diff(tt.want, gotRanges, protocmp.Transform()); diff != "" { t.Errorf("findInverseAffectedRanges() ranges mismatch (-want +got):\n%s", diff) @@ -280,7 +281,7 @@ func TestRealWorldFindInverseAffectedRanges(t *testing.T) { } // Run the function under test. - gotRanges, _ := findInverseAffectedRanges(affectedBlock, &ConversionMetrics{}) + gotRanges, _ := findInverseAffectedRanges(affectedBlock, &models.ConversionMetrics{}) // Sort slices for deterministic comparison. sort.Slice(gotRanges, func(i, j int) bool { @@ -551,7 +552,7 @@ func TestExtractVersions(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - metrics := &ConversionMetrics{} + metrics := &models.ConversionMetrics{} v := vulns.Vulnerability{ Vulnerability: &osvschema.Vulnerability{}, } From 8e9cc6d4be02b0933ccb2b8a48a438ab6b251d80 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 21 Jan 2026 03:41:44 +0000 Subject: [PATCH 10/14] Move functions that could be shared between converters --- .../cve/cve5/bulk-converter/main.go | 7 +- .../cve/cve5/single-converter/main.go | 5 +- .../cmd/converters/cve/nvd-cve-osv/main.go | 203 +--------------- vulnfeeds/conversion/common.go | 94 ++++++++ .../{cvelist2osv => conversion}/grouping.go | 6 +- .../grouping_test.go | 4 +- vulnfeeds/conversion/nvd/converter.go | 219 ++++++++++++++++++ vulnfeeds/cvelist2osv/common.go | 41 ---- vulnfeeds/cvelist2osv/common_test.go | 3 +- vulnfeeds/cvelist2osv/converter.go | 47 +--- vulnfeeds/cvelist2osv/default_extractor.go | 7 +- vulnfeeds/cvelist2osv/linux_extractor.go | 3 +- vulnfeeds/models/metrics.go | 93 ++++++++ 13 files changed, 432 insertions(+), 300 deletions(-) create mode 100644 vulnfeeds/conversion/common.go rename vulnfeeds/{cvelist2osv => conversion}/grouping.go (97%) rename vulnfeeds/{cvelist2osv => conversion}/grouping_test.go (99%) create mode 100644 vulnfeeds/conversion/nvd/converter.go create mode 100644 vulnfeeds/models/metrics.go diff --git a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go index ca1ce1bee8e..ecd2269aeb4 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go +++ b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go @@ -5,6 +5,7 @@ import ( _ "embed" "encoding/json" "flag" + "github.com/google/osv/vulnfeeds/conversion" "log/slog" "os" "path/filepath" @@ -34,7 +35,7 @@ func main() { flag.Parse() logger.InitGlobalLogger() - logger.Info("Commencing Linux CVE to OSV conversion run") + logger.Info("Commencing CVE to OSV conversion run") if err := os.MkdirAll(*localOutputDir, 0755); err != nil { logger.Fatal("Failed to create local output directory", slog.Any("err", err)) } @@ -115,8 +116,8 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string cveID := cve.Metadata.CVEID logger.Info("Processing "+string(cveID), slog.String("cve", string(cveID))) - osvFile, errCVE := cvelist2osv.CreateOSVFile(cveID, outDir) - metricsFile, errMetrics := cvelist2osv.CreateMetricsFile(cveID, outDir) + osvFile, errCVE := conversion.CreateOSVFile(cveID, outDir) + metricsFile, errMetrics := conversion.CreateMetricsFile(cveID, outDir) if errCVE != nil || errMetrics != nil { logger.Fatal("File failed to be created for CVE", slog.String("cve", string(cveID))) } diff --git a/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go index 14bfac1dbbb..105e01a5890 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go +++ b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go @@ -4,6 +4,7 @@ package main import ( "encoding/json" "flag" + "github.com/google/osv/vulnfeeds/conversion" "log/slog" "os" @@ -44,8 +45,8 @@ func main() { } // create the files - osvFile, errCVE := cvelist2osv.CreateOSVFile(cveID, outDir) - metricsFile, errMetrics := cvelist2osv.CreateMetricsFile(cveID, outDir) + osvFile, errCVE := conversion.CreateOSVFile(cveID, outDir) + metricsFile, errMetrics := conversion.CreateMetricsFile(cveID, outDir) if errCVE != nil || errMetrics != nil { logger.Fatal("File failed to be created for CVE", slog.String("cve", string(cveID))) } diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go index a1c8c4d6898..b498526d79c 100644 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go @@ -8,18 +8,17 @@ import ( "flag" "fmt" "log/slog" - "net/http" "os" "path/filepath" "strings" "slices" + "github.com/google/osv/vulnfeeds/conversion/nvd" "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/git" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" - "github.com/google/osv/vulnfeeds/vulns" ) var ErrNoRanges = errors.New("no ranges") @@ -45,202 +44,6 @@ var Metrics struct { Outcomes map[models.CVEID]models.ConversionOutcome // Per-CVE-ID record of conversion result. } -// Takes an NVD CVE record and outputs an OSV file in the specified directory. -func CVEToOSV(cve models.NVDCVE, repos []string, cache git.RepoTagsCache, directory string) error { - CPEs := cves.CPEs(cve) - // The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy. - maybeVendorName := "ENOCPE" - maybeProductName := "ENOCPE" - - if len(CPEs) > 0 { - CPE, err := cves.ParseCPE(CPEs[0]) // For naming the subdirectory used for output. - maybeVendorName = CPE.Vendor - maybeProductName = CPE.Product - if err != nil { - return fmt.Errorf("[%s]: Can't generate an OSV record without valid CPE data", cve.ID) - } - } - - v := vulns.FromNVDCVE(cve.ID, cve) - versions, notes := cves.ExtractVersionInfo(cve, nil, http.DefaultClient) - - if len(versions.AffectedVersions) != 0 { - var err error - // There are some AffectedVersions to try and resolve to AffectedCommits. - if len(repos) == 0 { - return fmt.Errorf("[%s]: No affected ranges for %q, and no repos to try and convert %+v to tags with", cve.ID, maybeProductName, versions.AffectedVersions) - } - logger.Info("Trying to convert version tags to commits", slog.String("cve", string(cve.ID)), slog.Any("versions", versions), slog.Any("repos", repos)) - versions, err = cves.GitVersionsToCommits(cve.ID, versions, repos, cache) - if err != nil { - return fmt.Errorf("[%s]: Failed to convert version tags to commits: %#w", cve.ID, err) - } - hasAnyFixedCommits := false - for _, repo := range repos { - if versions.HasFixedCommits(repo) { - hasAnyFixedCommits = true - break - } - } - - if versions.HasFixedVersions() && !hasAnyFixedCommits { - return fmt.Errorf("[%s]: Failed to convert fixed version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) - } - - hasAnyLastAffectedCommits := false - for _, repo := range repos { - if versions.HasLastAffectedCommits(repo) { - hasAnyLastAffectedCommits = true - break - } - } - - if versions.HasLastAffectedVersions() && !hasAnyLastAffectedCommits && !hasAnyFixedCommits { - return fmt.Errorf("[%s]: Failed to convert last_affected version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) - } - } - - slices.SortStableFunc(versions.AffectedCommits, models.AffectedCommitCompare) - - vulns.AttachExtractedVersionInfo(v, versions) - - if len(v.Affected) == 0 { - return fmt.Errorf("[%s]: No affected ranges detected for %q %w", cve.ID, maybeProductName, ErrNoRanges) - } - - vulnDir := filepath.Join(directory, maybeVendorName, maybeProductName) - err := os.MkdirAll(vulnDir, 0755) - if err != nil { - logger.Warn("Failed to create dir", slog.Any("err", err)) - return fmt.Errorf("failed to create dir: %w", err) - } - outputFile := filepath.Join(vulnDir, v.Id+extension) - notesFile := filepath.Join(vulnDir, v.Id+".notes") - f, err := os.Create(outputFile) - if err != nil { - logger.Warn("Failed to open for writing", slog.String("path", outputFile), slog.Any("err", err)) - return fmt.Errorf("failed to open %s for writing: %w", outputFile, err) - } - defer f.Close() - err = v.ToJSON(f) - if err != nil { - logger.Warn("Failed to write", slog.String("path", outputFile), slog.Any("err", err)) - return fmt.Errorf("failed to write %s: %w", outputFile, err) - } - logger.Info("Generated OSV record", slog.String("cve", string(cve.ID)), slog.String("product", maybeProductName)) - if len(notes) > 0 { - err = os.WriteFile(notesFile, []byte(strings.Join(notes, "\n")), 0600) - if err != nil { - logger.Warn("Failed to write", slog.String("cve", string(cve.ID)), slog.String("path", notesFile), slog.Any("err", err)) - } - } - - return nil -} - -// Takes an NVD CVE record and outputs a PackageInfo struct in a file in the specified directory. -func CVEToPackageInfo(cve models.NVDCVE, repos []string, cache git.RepoTagsCache, directory string) error { - CPEs := cves.CPEs(cve) - // The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy. - maybeVendorName := "ENOCPE" - maybeProductName := "ENOCPE" - - if len(CPEs) > 0 { - CPE, err := cves.ParseCPE(CPEs[0]) // For naming the subdirectory used for output. - maybeVendorName = CPE.Vendor - maybeProductName = CPE.Product - if err != nil { - return fmt.Errorf("[%s]: Can't generate an OSV record without valid CPE data", cve.ID) - } - } - - // more often than not, this yields a VersionInfo with AffectedVersions and no AffectedCommits. - versions, notes := cves.ExtractVersionInfo(cve, nil, http.DefaultClient) - - if len(versions.AffectedVersions) != 0 { - var err error - // There are some AffectedVersions to try and resolve to AffectedCommits. - if len(repos) == 0 { - return fmt.Errorf("[%s]: No affected ranges for %q, and no repos to try and convert %+v to tags with", cve.ID, maybeProductName, versions.AffectedVersions) - } - logger.Info("Trying to convert version tags to commits", slog.String("cve", string(cve.ID)), slog.Any("versions", versions), slog.Any("repos", repos)) - versions, err = cves.GitVersionsToCommits(cve.ID, versions, repos, cache) - if err != nil { - return fmt.Errorf("[%s]: Failed to convert version tags to commits: %#w", cve.ID, err) - } - } - - hasAnyFixedCommits := false - for _, repo := range repos { - if versions.HasFixedCommits(repo) { - hasAnyFixedCommits = true - } - } - - if versions.HasFixedVersions() && !hasAnyFixedCommits { - return fmt.Errorf("[%s]: Failed to convert fixed version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) - } - - hasAnyLastAffectedCommits := false - for _, repo := range repos { - if versions.HasLastAffectedCommits(repo) { - hasAnyLastAffectedCommits = true - } - } - - if versions.HasLastAffectedVersions() && !hasAnyLastAffectedCommits && !hasAnyFixedCommits { - return fmt.Errorf("[%s]: Failed to convert last_affected version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) - } - - if len(versions.AffectedCommits) == 0 { - return fmt.Errorf("[%s]: No affected commit ranges determined for %q %w", cve.ID, maybeProductName, ErrNoRanges) - } - - versions.AffectedVersions = nil // these have served their purpose and are not required in the resulting output. - - slices.SortStableFunc(versions.AffectedCommits, models.AffectedCommitCompare) - - var pkgInfos []vulns.PackageInfo - pi := vulns.PackageInfo{VersionInfo: versions} - pkgInfos = append(pkgInfos, pi) // combine-to-osv expects a serialised *array* of PackageInfo - - vulnDir := filepath.Join(directory, maybeVendorName, maybeProductName) - err := os.MkdirAll(vulnDir, 0755) - if err != nil { - logger.Warn("Failed to create dir", slog.Any("err", err)) - return fmt.Errorf("failed to create dir: %w", err) - } - - outputFile := filepath.Join(vulnDir, string(cve.ID)+".nvd"+extension) - notesFile := filepath.Join(vulnDir, string(cve.ID)+".nvd.notes") - f, err := os.Create(outputFile) - if err != nil { - logger.Warn("Failed to open for writing", slog.String("path", outputFile), slog.Any("err", err)) - return fmt.Errorf("failed to open %s for writing: %w", outputFile, err) - } - defer f.Close() - - encoder := json.NewEncoder(f) - encoder.SetIndent("", " ") - err = encoder.Encode(&pkgInfos) - - if err != nil { - logger.Warn("Failed to encode PackageInfo", slog.String("path", outputFile), slog.Any("err", err)) - return fmt.Errorf("failed to encode PackageInfo to %s: %w", outputFile, err) - } - - logger.Info("Generated PackageInfo record", slog.String("cve", string(cve.ID)), slog.String("product", maybeProductName)) - - if len(notes) > 0 { - err = os.WriteFile(notesFile, []byte(strings.Join(notes, "\n")), 0600) - if err != nil { - logger.Warn("Failed to write", slog.String("cve", string(cve.ID)), slog.String("path", notesFile), slog.Any("err", err)) - } - } - - return nil -} - func loadCPEDictionary(productToRepo *cves.VendorProductToRepoMap, f string) error { data, err := os.ReadFile(f) if err != nil { @@ -435,9 +238,9 @@ func main() { switch *outFormat { case "OSV": - err = CVEToOSV(cve.CVE, ReposForCVE[CVEID], RepoTagsCache, *outDir) + err = nvd.CVEToOSV(cve.CVE, ReposForCVE[CVEID], RepoTagsCache, *outDir) case "PackageInfo": - err = CVEToPackageInfo(cve.CVE, ReposForCVE[CVEID], RepoTagsCache, *outDir) + err = nvd.CVEToPackageInfo(cve.CVE, ReposForCVE[CVEID], RepoTagsCache, *outDir) } // Parse this error to determine which failure mode it was if err != nil { diff --git a/vulnfeeds/conversion/common.go b/vulnfeeds/conversion/common.go new file mode 100644 index 00000000000..7cb8fa48147 --- /dev/null +++ b/vulnfeeds/conversion/common.go @@ -0,0 +1,94 @@ +package conversion + +import ( + "encoding/json" + "github.com/google/osv/vulnfeeds/utility/logger" + "log/slog" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/google/osv/vulnfeeds/models" + "github.com/google/osv/vulnfeeds/vulns" + "github.com/ossf/osv-schema/bindings/go/osvschema" +) + +// AddAffected adds an osvschema.Affected to a vulnerability, ensuring that no duplicate ranges are added. +func AddAffected(v *vulns.Vulnerability, aff *osvschema.Affected, metrics *models.ConversionMetrics) { + allExistingRanges := make(map[string]struct{}) + for _, existingAff := range v.Affected { + for _, r := range existingAff.GetRanges() { + rangeBytes, err := json.Marshal(r) + if err == nil { + allExistingRanges[string(rangeBytes)] = struct{}{} + } + } + } + + uniqueRanges := []*osvschema.Range{} + for _, r := range aff.GetRanges() { + rangeBytes, err := json.Marshal(r) + if err != nil { + metrics.AddNote("Could not marshal range to check for duplicates, adding anyway: %+v", r) + uniqueRanges = append(uniqueRanges, r) + + continue + } + rangeStr := string(rangeBytes) + if _, exists := allExistingRanges[rangeStr]; !exists { + uniqueRanges = append(uniqueRanges, r) + allExistingRanges[rangeStr] = struct{}{} + } else { + metrics.AddNote("Skipping duplicate range: %+v", r) + } + } + + if len(uniqueRanges) > 0 { + newAff := &osvschema.Affected{ + Package: aff.GetPackage(), + Ranges: uniqueRanges, + DatabaseSpecific: aff.GetDatabaseSpecific(), + } + v.Affected = append(v.Affected, newAff) + } +} + +func DeduplicateRefs(refs []models.Reference) []models.Reference { + // Deduplicate references by URL. + slices.SortStableFunc(refs, func(a, b models.Reference) int { + return strings.Compare(a.URL, b.URL) + }) + refs = slices.CompactFunc(refs, func(a, b models.Reference) bool { + return a.URL == b.URL + }) + + return refs +} + +// CreateMetricsFile saves the collected conversion metrics to a JSON file. +// This file provides data for analyzing the success and characteristics of the +// conversion process for a given CVE. +func CreateMetricsFile(id models.CVEID, vulnDir string) (*os.File, error) { + metricsFile := filepath.Join(vulnDir, string(id)+".metrics.json") + f, err := os.Create(metricsFile) + if err != nil { + logger.Info("Failed to open for writing "+metricsFile, slog.String("cve", string(id)), slog.String("path", metricsFile), slog.Any("err", err)) + return nil, err + } + + return f, nil +} + +// CreateOSVFile creates the initial file for the OSV record. +func CreateOSVFile(id models.CVEID, vulnDir string) (*os.File, error) { + outputFile := filepath.Join(vulnDir, string(id)+models.Extension) + + f, err := os.Create(outputFile) + if err != nil { + logger.Info("Failed to open for writing "+outputFile, slog.String("cve", string(id)), slog.String("path", outputFile), slog.Any("err", err)) + return nil, err + } + + return f, err +} diff --git a/vulnfeeds/cvelist2osv/grouping.go b/vulnfeeds/conversion/grouping.go similarity index 97% rename from vulnfeeds/cvelist2osv/grouping.go rename to vulnfeeds/conversion/grouping.go index d5ff5cfa708..48f0b4546c9 100644 --- a/vulnfeeds/cvelist2osv/grouping.go +++ b/vulnfeeds/conversion/grouping.go @@ -1,4 +1,4 @@ -package cvelist2osv +package conversion import ( "fmt" @@ -11,11 +11,11 @@ import ( "google.golang.org/protobuf/types/known/structpb" ) -// groupAffectedRanges groups ranges that share the same introduced value, type, and repo. +// GroupAffectedRanges groups ranges that share the same introduced value, type, and repo. // This is because having multiple ranges with the same introduced value would act like an // OR condition, rather than AND. // This function modifies in-place -func groupAffectedRanges(affected []*osvschema.Affected) { +func GroupAffectedRanges(affected []*osvschema.Affected) { for _, aff := range affected { if len(aff.GetRanges()) <= 1 { continue diff --git a/vulnfeeds/cvelist2osv/grouping_test.go b/vulnfeeds/conversion/grouping_test.go similarity index 99% rename from vulnfeeds/cvelist2osv/grouping_test.go rename to vulnfeeds/conversion/grouping_test.go index 3d5876e4976..62d057dcc94 100644 --- a/vulnfeeds/cvelist2osv/grouping_test.go +++ b/vulnfeeds/conversion/grouping_test.go @@ -1,4 +1,4 @@ -package cvelist2osv +package conversion import ( "testing" @@ -423,7 +423,7 @@ func TestGroupAffectedRanges(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - groupAffectedRanges(tt.affected) + GroupAffectedRanges(tt.affected) if diff := cmp.Diff(tt.want, tt.affected, protocmp.Transform()); diff != "" { t.Errorf("groupAffectedRanges() mismatch (-want +got):\n%s", diff) } diff --git a/vulnfeeds/conversion/nvd/converter.go b/vulnfeeds/conversion/nvd/converter.go new file mode 100644 index 00000000000..3d95499c0bd --- /dev/null +++ b/vulnfeeds/conversion/nvd/converter.go @@ -0,0 +1,219 @@ +package nvd + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/git" + "github.com/google/osv/vulnfeeds/models" + "github.com/google/osv/vulnfeeds/utility/logger" + "github.com/google/osv/vulnfeeds/vulns" +) + +var ErrNoRanges = errors.New("no ranges") + +var ErrUnresolvedFix = errors.New("fixes not resolved to commits") + +// CVEToOSV Takes an NVD CVE record and outputs an OSV file in the specified directory. +func CVEToOSV(cve models.NVDCVE, repos []string, cache git.RepoTagsCache, directory string) error { + CPEs := cves.CPEs(cve) + // The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy. + maybeVendorName := "ENOCPE" + maybeProductName := "ENOCPE" + + if len(CPEs) > 0 { + CPE, err := cves.ParseCPE(CPEs[0]) // For naming the subdirectory used for output. + maybeVendorName = CPE.Vendor + maybeProductName = CPE.Product + if err != nil { + return fmt.Errorf("[%s]: Can't generate an OSV record without valid CPE data", cve.ID) + } + } + + v := vulns.FromNVDCVE(cve.ID, cve) + versions, notes := cves.ExtractVersionInfo(cve, nil, http.DefaultClient) + + if len(versions.AffectedVersions) != 0 { + var err error + // There are some AffectedVersions to try and resolve to AffectedCommits. + if len(repos) == 0 { + return fmt.Errorf("[%s]: No affected ranges for %q, and no repos to try and convert %+v to tags with", cve.ID, maybeProductName, versions.AffectedVersions) + } + logger.Info("Trying to convert version tags to commits", slog.String("cve", string(cve.ID)), slog.Any("versions", versions), slog.Any("repos", repos)) + versions, err = cves.GitVersionsToCommits(cve.ID, versions, repos, cache) + if err != nil { + return fmt.Errorf("[%s]: Failed to convert version tags to commits: %#w", cve.ID, err) + } + hasAnyFixedCommits := false + for _, repo := range repos { + if versions.HasFixedCommits(repo) { + hasAnyFixedCommits = true + break + } + } + + if versions.HasFixedVersions() && !hasAnyFixedCommits { + return fmt.Errorf("[%s]: Failed to convert fixed version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) + } + + hasAnyLastAffectedCommits := false + for _, repo := range repos { + if versions.HasLastAffectedCommits(repo) { + hasAnyLastAffectedCommits = true + break + } + } + + if versions.HasLastAffectedVersions() && !hasAnyLastAffectedCommits && !hasAnyFixedCommits { + return fmt.Errorf("[%s]: Failed to convert last_affected version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) + } + } + + slices.SortStableFunc(versions.AffectedCommits, models.AffectedCommitCompare) + + vulns.AttachExtractedVersionInfo(v, versions) + + if len(v.Affected) == 0 { + return fmt.Errorf("[%s]: No affected ranges detected for %q %w", cve.ID, maybeProductName, ErrNoRanges) + } + + vulnDir := filepath.Join(directory, maybeVendorName, maybeProductName) + err := os.MkdirAll(vulnDir, 0755) + if err != nil { + logger.Warn("Failed to create dir", slog.Any("err", err)) + return fmt.Errorf("failed to create dir: %w", err) + } + outputFile := filepath.Join(vulnDir, v.Id+models.Extension) + notesFile := filepath.Join(vulnDir, v.Id+".notes") + f, err := os.Create(outputFile) + if err != nil { + logger.Warn("Failed to open for writing", slog.String("path", outputFile), slog.Any("err", err)) + return fmt.Errorf("failed to open %s for writing: %w", outputFile, err) + } + defer f.Close() + err = v.ToJSON(f) + if err != nil { + logger.Warn("Failed to write", slog.String("path", outputFile), slog.Any("err", err)) + return fmt.Errorf("failed to write %s: %w", outputFile, err) + } + logger.Info("Generated OSV record", slog.String("cve", string(cve.ID)), slog.String("product", maybeProductName)) + if len(notes) > 0 { + err = os.WriteFile(notesFile, []byte(strings.Join(notes, "\n")), 0600) + if err != nil { + logger.Warn("Failed to write", slog.String("cve", string(cve.ID)), slog.String("path", notesFile), slog.Any("err", err)) + } + } + + return nil +} + +// CVEToPackageInfo takes an NVD CVE record and outputs a PackageInfo struct in a file in the specified directory. +func CVEToPackageInfo(cve models.NVDCVE, repos []string, cache git.RepoTagsCache, directory string) error { + CPEs := cves.CPEs(cve) + // The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy. + maybeVendorName := "ENOCPE" + maybeProductName := "ENOCPE" + + if len(CPEs) > 0 { + CPE, err := cves.ParseCPE(CPEs[0]) // For naming the subdirectory used for output. + maybeVendorName = CPE.Vendor + maybeProductName = CPE.Product + if err != nil { + return fmt.Errorf("[%s]: Can't generate an OSV record without valid CPE data", cve.ID) + } + } + + // more often than not, this yields a VersionInfo with AffectedVersions and no AffectedCommits. + versions, notes := cves.ExtractVersionInfo(cve, nil, http.DefaultClient) + + if len(versions.AffectedVersions) != 0 { + var err error + // There are some AffectedVersions to try and resolve to AffectedCommits. + if len(repos) == 0 { + return fmt.Errorf("[%s]: No affected ranges for %q, and no repos to try and convert %+v to tags with", cve.ID, maybeProductName, versions.AffectedVersions) + } + logger.Info("Trying to convert version tags to commits", slog.String("cve", string(cve.ID)), slog.Any("versions", versions), slog.Any("repos", repos)) + versions, err = cves.GitVersionsToCommits(cve.ID, versions, repos, cache) + if err != nil { + return fmt.Errorf("[%s]: Failed to convert version tags to commits: %#w", cve.ID, err) + } + } + + hasAnyFixedCommits := false + for _, repo := range repos { + if versions.HasFixedCommits(repo) { + hasAnyFixedCommits = true + } + } + + if versions.HasFixedVersions() && !hasAnyFixedCommits { + return fmt.Errorf("[%s]: Failed to convert fixed version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) + } + + hasAnyLastAffectedCommits := false + for _, repo := range repos { + if versions.HasLastAffectedCommits(repo) { + hasAnyLastAffectedCommits = true + } + } + + if versions.HasLastAffectedVersions() && !hasAnyLastAffectedCommits && !hasAnyFixedCommits { + return fmt.Errorf("[%s]: Failed to convert last_affected version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) + } + + if len(versions.AffectedCommits) == 0 { + return fmt.Errorf("[%s]: No affected commit ranges determined for %q %w", cve.ID, maybeProductName, ErrNoRanges) + } + + versions.AffectedVersions = nil // these have served their purpose and are not required in the resulting output. + + slices.SortStableFunc(versions.AffectedCommits, models.AffectedCommitCompare) + + var pkgInfos []vulns.PackageInfo + pi := vulns.PackageInfo{VersionInfo: versions} + pkgInfos = append(pkgInfos, pi) // combine-to-osv expects a serialised *array* of PackageInfo + + vulnDir := filepath.Join(directory, maybeVendorName, maybeProductName) + err := os.MkdirAll(vulnDir, 0755) + if err != nil { + logger.Warn("Failed to create dir", slog.Any("err", err)) + return fmt.Errorf("failed to create dir: %w", err) + } + + outputFile := filepath.Join(vulnDir, string(cve.ID)+".nvd"+models.Extension) + notesFile := filepath.Join(vulnDir, string(cve.ID)+".nvd.notes") + f, err := os.Create(outputFile) + if err != nil { + logger.Warn("Failed to open for writing", slog.String("path", outputFile), slog.Any("err", err)) + return fmt.Errorf("failed to open %s for writing: %w", outputFile, err) + } + defer f.Close() + + encoder := json.NewEncoder(f) + encoder.SetIndent("", " ") + err = encoder.Encode(&pkgInfos) + + if err != nil { + logger.Warn("Failed to encode PackageInfo", slog.String("path", outputFile), slog.Any("err", err)) + return fmt.Errorf("failed to encode PackageInfo to %s: %w", outputFile, err) + } + + logger.Info("Generated PackageInfo record", slog.String("cve", string(cve.ID)), slog.String("product", maybeProductName)) + + if len(notes) > 0 { + err = os.WriteFile(notesFile, []byte(strings.Join(notes, "\n")), 0600) + if err != nil { + logger.Warn("Failed to write", slog.String("cve", string(cve.ID)), slog.String("path", notesFile), slog.Any("err", err)) + } + } + + return nil +} diff --git a/vulnfeeds/cvelist2osv/common.go b/vulnfeeds/cvelist2osv/common.go index 15e99c3af32..954002ae1a2 100644 --- a/vulnfeeds/cvelist2osv/common.go +++ b/vulnfeeds/cvelist2osv/common.go @@ -2,7 +2,6 @@ package cvelist2osv import ( "cmp" - "encoding/json" "errors" "github.com/google/osv/vulnfeeds/models" "log/slog" @@ -247,43 +246,3 @@ func compareSemverLike(a, b string) int { // All extra parts were zero, so the versions are effectively equal. return 0 } - -// addAffected adds an osvschema.Affected to a vulnerability, ensuring that no duplicate ranges are added. -func addAffected(v *vulns.Vulnerability, aff *osvschema.Affected, metrics *models.ConversionMetrics) { - allExistingRanges := make(map[string]struct{}) - for _, existingAff := range v.Affected { - for _, r := range existingAff.GetRanges() { - rangeBytes, err := json.Marshal(r) - if err == nil { - allExistingRanges[string(rangeBytes)] = struct{}{} - } - } - } - - uniqueRanges := []*osvschema.Range{} - for _, r := range aff.GetRanges() { - rangeBytes, err := json.Marshal(r) - if err != nil { - metrics.AddNote("Could not marshal range to check for duplicates, adding anyway: %+v", r) - uniqueRanges = append(uniqueRanges, r) - - continue - } - rangeStr := string(rangeBytes) - if _, exists := allExistingRanges[rangeStr]; !exists { - uniqueRanges = append(uniqueRanges, r) - allExistingRanges[rangeStr] = struct{}{} - } else { - metrics.AddNote("Skipping duplicate range: %+v", r) - } - } - - if len(uniqueRanges) > 0 { - newAff := &osvschema.Affected{ - Package: aff.GetPackage(), - Ranges: uniqueRanges, - DatabaseSpecific: aff.GetDatabaseSpecific(), - } - v.Affected = append(v.Affected, newAff) - } -} diff --git a/vulnfeeds/cvelist2osv/common_test.go b/vulnfeeds/cvelist2osv/common_test.go index db9cd1798ef..fb3ff73585c 100644 --- a/vulnfeeds/cvelist2osv/common_test.go +++ b/vulnfeeds/cvelist2osv/common_test.go @@ -1,6 +1,7 @@ package cvelist2osv import ( + "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/models" "testing" @@ -58,7 +59,7 @@ func TestAddAffected(t *testing.T) { } metrics := &models.ConversionMetrics{} - addAffected(v, aff, metrics) + conversion.AddAffected(v, aff, metrics) expectedAffected := []*osvschema.Affected{ { diff --git a/vulnfeeds/cvelist2osv/converter.go b/vulnfeeds/cvelist2osv/converter.go index 06026efd6cf..7bfa990d43f 100644 --- a/vulnfeeds/cvelist2osv/converter.go +++ b/vulnfeeds/cvelist2osv/converter.go @@ -4,13 +4,11 @@ package cvelist2osv import ( "encoding/json" "fmt" + "github.com/google/osv/vulnfeeds/conversion" "io" "log/slog" - "os" - "path/filepath" "slices" "sort" - "strings" "time" "github.com/google/osv/vulnfeeds/cves" @@ -146,33 +144,6 @@ func FromCVE5(cve models.CVE5, refs []models.Reference, metrics *models.Conversi return &v } -// CreateOSVFile creates the initial file for the OSV record. -func CreateOSVFile(id models.CVEID, vulnDir string) (*os.File, error) { - outputFile := filepath.Join(vulnDir, string(id)+models.Extension) - - f, err := os.Create(outputFile) - if err != nil { - logger.Info("Failed to open for writing "+outputFile, slog.String("cve", string(id)), slog.String("path", outputFile), slog.Any("err", err)) - return nil, err - } - - return f, err -} - -// CreateMetricsFile saves the collected conversion metrics to a JSON file. -// This file provides data for analyzing the success and characteristics of the -// conversion process for a given CVE. -func CreateMetricsFile(id models.CVEID, vulnDir string) (*os.File, error) { - metricsFile := filepath.Join(vulnDir, string(id)+".metrics.json") - f, err := os.Create(metricsFile) - if err != nil { - logger.Info("Failed to open for writing "+metricsFile, slog.String("cve", string(id)), slog.String("path", metricsFile), slog.Any("err", err)) - return nil, err - } - - return f, nil -} - // ConvertAndExportCVEToOSV is the main function for this file. It takes a CVE, // converts it into an OSV record, collects metrics, and writes both to disk. func ConvertAndExportCVEToOSV(cve models.CVE5, vulnSink io.Writer, metricsSink io.Writer, sourceLink string) error { @@ -186,7 +157,7 @@ func ConvertAndExportCVEToOSV(cve models.CVE5, vulnSink io.Writer, metricsSink i references = append(references, models.Reference{URL: sourceLink}) } - references = deduplicateRefs(references) + references = conversion.DeduplicateRefs(references) metrics := models.ConversionMetrics{CVEID: cveID, CNA: cnaAssigner, UnresolvedRangesCount: 0, ResolvedRangesCount: 0} @@ -200,7 +171,7 @@ func ConvertAndExportCVEToOSV(cve models.CVE5, vulnSink io.Writer, metricsSink i versionExtractor := GetVersionExtractor(cve.Metadata.AssignerShortName) versionExtractor.ExtractVersions(cve, v, &metrics, metrics.Repos) - groupAffectedRanges(v.Affected) + conversion.GroupAffectedRanges(v.Affected) models.DetermineOutcome(&metrics) @@ -257,18 +228,6 @@ func identifyPossibleURLs(cve models.CVE5) []models.Reference { return refs } -func deduplicateRefs(refs []models.Reference) []models.Reference { - // Deduplicate references by URL. - slices.SortStableFunc(refs, func(a, b models.Reference) int { - return strings.Compare(a.URL, b.URL) - }) - refs = slices.CompactFunc(refs, func(a, b models.Reference) bool { - return a.URL == b.URL - }) - - return refs -} - func buildDBSpecific(cve models.CVE5, metrics *models.ConversionMetrics, sourceLink string) map[string]any { dbSpecific := make(map[string]any) diff --git a/vulnfeeds/cvelist2osv/default_extractor.go b/vulnfeeds/cvelist2osv/default_extractor.go index e559c195f38..ac05b5b2911 100644 --- a/vulnfeeds/cvelist2osv/default_extractor.go +++ b/vulnfeeds/cvelist2osv/default_extractor.go @@ -2,6 +2,7 @@ package cvelist2osv import ( "fmt" + "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/models" "log/slog" @@ -45,7 +46,7 @@ func (d *DefaultVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vuln } else { gotVersions = true } - addAffected(v, aff, metrics) + conversion.AddAffected(v, aff, metrics) } if !gotVersions { @@ -60,7 +61,7 @@ func (d *DefaultVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vuln gotVersions = true } - addAffected(v, aff, metrics) + conversion.AddAffected(v, aff, metrics) } } @@ -72,7 +73,7 @@ func (d *DefaultVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vuln if err != nil { logger.Error("Failed to convert git versions to commits", slog.Any("err", err)) } - addAffected(v, aff, metrics) + conversion.AddAffected(v, aff, metrics) } } } diff --git a/vulnfeeds/cvelist2osv/linux_extractor.go b/vulnfeeds/cvelist2osv/linux_extractor.go index 5584b76be55..5a9983e1840 100644 --- a/vulnfeeds/cvelist2osv/linux_extractor.go +++ b/vulnfeeds/cvelist2osv/linux_extractor.go @@ -2,6 +2,7 @@ package cvelist2osv import ( "fmt" + "github.com/google/osv/vulnfeeds/conversion" "slices" "strconv" "strings" @@ -44,7 +45,7 @@ func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected } aff := createLinuxAffected(versionRanges, versionType, cveAff.Repo) metrics.AddSource(models.VersionSourceAffected) - addAffected(v, aff, metrics) + conversion.AddAffected(v, aff, metrics) } return gotVersions diff --git a/vulnfeeds/models/metrics.go b/vulnfeeds/models/metrics.go new file mode 100644 index 00000000000..85d7c1db4cd --- /dev/null +++ b/vulnfeeds/models/metrics.go @@ -0,0 +1,93 @@ +package models + +import ( + "fmt" + "log/slog" + + "github.com/google/osv/vulnfeeds/utility/logger" + "github.com/ossf/osv-schema/bindings/go/osvschema" +) + +type ConversionOutcome int + +const ( + Extension = ".json" +) + +const ( + // Set of enums for categorizing conversion outcomes. + ConversionUnknown ConversionOutcome = iota // Shouldn't happen + Successful // It worked! + Rejected // The CVE was rejected + NoSoftware // The CVE had no CPEs relating to software (i.e. Operating Systems or Hardware). + NoRepos // The CPE Vendor/Product had no repositories derived for it. + NoCommitRanges // No viable commit ranges could be calculated from the repository for the CVE's CPE(s). + NoRanges // No version ranges could be extracted from the record. + FixUnresolvable // Partial resolution of versions, resulting in a false positive. +) + +// RefTagDenyList contains reference tags that are often associated with unreliable or +// irrelevant repository URLs. References with these tags are currently ignored +// to avoid incorrect repository associations. +var RefTagDenyList = []string{ + // "Exploit", + // "Third Party Advisory", + "Broken Link", // Actively ignore these. +} + +func (c ConversionOutcome) String() string { + return [...]string{"ConversionUnknown", "Successful", "Rejected", "NoSoftware", "NoRepos", "NoCommitRanges", "NoRanges", "FixUnresolvable"}[c] +} + +// ConversionMetrics holds the collected data about the conversion process for a single CVE. +type ConversionMetrics struct { + CVEID CVEID `json:"id"` // The CVE ID + CNA string `json:"cna"` // The CNA that assigned the CVE. + Outcome ConversionOutcome `json:"outcome"` // The final outcome of the conversion (e.g., "Successful", "Failed"). + Repos []string `json:"repos"` // A list of repositories extracted from the CVE's references. + RefTypesCount map[osvschema.Reference_Type]int `json:"ref_types_count"` // A count of each type of reference found. + VersionSources []VersionSource `json:"version_sources"` // A list of the ways the versions were extracted + Notes []string `json:"notes"` // A collection of notes and warnings generated during conversion. + CPEs []string `json:"cpes"` + UnresolvedRangesCount int `json:"unresolved_ranges_count"` + ResolvedRangesCount int `json:"resolved_ranges_count"` +} + +// AddNote adds a formatted note to the ConversionMetrics. +func (m *ConversionMetrics) AddNote(format string, a ...any) { + m.Notes = append(m.Notes, fmt.Sprintf(format, a...)) + logger.Debug(fmt.Sprintf(format, a...), slog.String("cna", m.CNA), slog.String("cve", string(m.CVEID))) +} + +// AddSource appends a source to the ConversionMetrics +func (m *ConversionMetrics) AddSource(source VersionSource) { + m.VersionSources = append(m.VersionSources, source) +} + +// VersionSource indicates the source of the extracted version information. +type VersionSource string + +const ( + VersionSourceNone VersionSource = "NOVERS" + VersionSourceAffected VersionSource = "CVEAFFVERS" + VersionSourceGit VersionSource = "GITVERS" + VersionSourceCPE VersionSource = "CPEVERS" + VersionSourceDescription VersionSource = "DESCRVERS" +) + +func DetermineOutcome(metrics *ConversionMetrics) { + // check if we have affected ranges/versions. + if len(metrics.Repos) == 0 { + // Fix unlikely, as no repos to resolve + metrics.Outcome = NoRepos + return + } + + if metrics.ResolvedRangesCount > 0 { + metrics.Outcome = Successful + } else if metrics.UnresolvedRangesCount > 0 { + metrics.Outcome = NoCommitRanges + } else { + metrics.Outcome = NoRanges + } +} From 3d193d7877e90df0999787449e5021833e3420e3 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 21 Jan 2026 03:56:42 +0000 Subject: [PATCH 11/14] Fix dockerfile routing --- vulnfeeds/cmd/converters/alpine/Dockerfile | 4 ++-- vulnfeeds/cmd/converters/cve/cve5/bulk-converter/Dockerfile | 4 ++-- vulnfeeds/cmd/converters/cve/nvd-cve-osv/Dockerfile | 4 ++-- vulnfeeds/cmd/converters/debian/Dockerfile | 4 ++-- vulnfeeds/cmd/mirrors/cpe-repo-gen/Dockerfile | 4 ++-- vulnfeeds/cmd/mirrors/download-cves/Dockerfile | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/vulnfeeds/cmd/converters/alpine/Dockerfile b/vulnfeeds/cmd/converters/alpine/Dockerfile index e760dbecc48..88202fd616c 100644 --- a/vulnfeeds/cmd/converters/alpine/Dockerfile +++ b/vulnfeeds/cmd/converters/alpine/Dockerfile @@ -22,13 +22,13 @@ COPY ./go.sum /src/go.sum RUN go mod download COPY ./ /src/ -RUN go build -o alpine-osv ./cmd/alpine/ +RUN go build -o alpine-osv ./cmd/converters/alpine/ FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:feca5d4cb9b422e124e6f28b8ed2e714160757eb383eaae712117c75f584aa2f WORKDIR /root/ COPY --from=GO_BUILD /src/alpine-osv ./ -COPY ./cmd/alpine/run_alpine_convert.sh ./ +COPY ./cmd/converters/alpine/run_alpine_convert.sh ./ ENTRYPOINT ["/root/run_alpine_convert.sh"] diff --git a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/Dockerfile b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/Dockerfile index 79b5dff1120..4efd790b895 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/Dockerfile +++ b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/Dockerfile @@ -23,13 +23,13 @@ RUN go mod download && go mod verify COPY ./ /src/ -RUN go build -o cve-bulk-converter ./cmd/cve-bulk-converter/ +RUN go build -o cve-bulk-converter ./cmd/converters/cve/cve5/bulk-converter/ FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:feca5d4cb9b422e124e6f28b8ed2e714160757eb383eaae712117c75f584aa2f RUN apk --no-cache add jq WORKDIR /root/ COPY --from=go_build /src/cve-bulk-converter ./ -COPY ./cmd/cve-bulk-converter/run-cvelist-converter.sh ./ +COPY ./cmd/converters/cve/cve5/bulk-converter/run-cvelist-converter.sh ./ ENTRYPOINT ["/root/run-cvelist-converter.sh"] diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/Dockerfile b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/Dockerfile index 34a9e8e4ad7..ddd1fec8ee8 100644 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/Dockerfile +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/Dockerfile @@ -20,12 +20,12 @@ COPY go.mod go.sum ./ RUN go mod download && go mod verify COPY . . -RUN CGO_ENABLED=0 go build -v -o /usr/local/bin ./cmd/nvd-cve-osv ./cmd/download-cves +RUN CGO_ENABLED=0 go build -v -o /usr/local/bin ./cmd/converters/cve/nvd-cve-osv ./cmd/mirrors/download-cves FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:feca5d4cb9b422e124e6f28b8ed2e714160757eb383eaae712117c75f584aa2f RUN apk --no-cache add jq COPY --from=GO_BUILD /usr/local/bin/ ./usr/local/bin/ -COPY --from=GO_BUILD /go/src/cmd/nvd-cve-osv/run_cve_to_osv_generation.sh ./usr/local/bin/ +COPY --from=GO_BUILD /go/src/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh ./usr/local/bin/ CMD ["/usr/local/bin/run_cve_to_osv_generation.sh"] diff --git a/vulnfeeds/cmd/converters/debian/Dockerfile b/vulnfeeds/cmd/converters/debian/Dockerfile index 31d08c85454..09f9954a506 100644 --- a/vulnfeeds/cmd/converters/debian/Dockerfile +++ b/vulnfeeds/cmd/converters/debian/Dockerfile @@ -22,14 +22,14 @@ COPY ./go.sum /src/go.sum RUN go mod download COPY ./ /src/ -RUN go build -o debian ./cmd/debian/ +RUN go build -o debian ./cmd/converters/debian/ FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:feca5d4cb9b422e124e6f28b8ed2e714160757eb383eaae712117c75f584aa2f WORKDIR /root/ COPY --from=GO_BUILD /src/debian ./ -COPY ./cmd/debian/run_debian_convert.sh ./ +COPY ./cmd/converters/debian/run_debian_convert.sh ./ RUN chmod 755 ./run_debian_convert.sh diff --git a/vulnfeeds/cmd/mirrors/cpe-repo-gen/Dockerfile b/vulnfeeds/cmd/mirrors/cpe-repo-gen/Dockerfile index c913d687c34..71a4fbec30c 100644 --- a/vulnfeeds/cmd/mirrors/cpe-repo-gen/Dockerfile +++ b/vulnfeeds/cmd/mirrors/cpe-repo-gen/Dockerfile @@ -22,13 +22,13 @@ COPY ./go.sum /src/go.sum RUN go mod download COPY ./ /src/ -RUN CGO_ENABLED=0 go build -o cpe-repo-gen ./cmd/cpe-repo-gen +RUN CGO_ENABLED=0 go build -o cpe-repo-gen ./cmd/mirrors/cpe-repo-gen FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:feca5d4cb9b422e124e6f28b8ed2e714160757eb383eaae712117c75f584aa2f RUN apk add --no-cache unzip COPY --from=GO_BUILD /src/cpe-repo-gen ./ -COPY ./cmd/cpe-repo-gen/cpe-repo-gen_map.sh ./ +COPY ./cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh ./ ENTRYPOINT ["/cpe-repo-gen_map.sh"] diff --git a/vulnfeeds/cmd/mirrors/download-cves/Dockerfile b/vulnfeeds/cmd/mirrors/download-cves/Dockerfile index 2476f82d04a..bcb2c6a2fbf 100644 --- a/vulnfeeds/cmd/mirrors/download-cves/Dockerfile +++ b/vulnfeeds/cmd/mirrors/download-cves/Dockerfile @@ -22,12 +22,12 @@ COPY ./go.sum /src/go.sum RUN go mod download COPY ./ /src/ -RUN go build -o download-cves ./cmd/download-cves/ +RUN go build -o download-cves ./cmd/mirrors/download-cves/ FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:feca5d4cb9b422e124e6f28b8ed2e714160757eb383eaae712117c75f584aa2f WORKDIR /usr/local/bin COPY --from=GO_BUILD /src/download-cves ./ -COPY ./cmd/download-cves/mirror_nvd.sh ./ +COPY ./cmd/mirrors/download-cves/mirror_nvd.sh ./ ENTRYPOINT ["/usr/local/bin/mirror_nvd.sh"] From 7dfa5f02f4703ae4c9e6760dcbc67f727e12d1b4 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Thu, 22 Jan 2026 04:48:55 +0000 Subject: [PATCH 12/14] fix lint --- vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go index d84689700a4..aa4fe012ce3 100644 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go @@ -8,7 +8,6 @@ import ( "flag" "fmt" "log/slog" - "net/http" "os" "path/filepath" "strings" @@ -108,6 +107,8 @@ func main() { VPRepoCache := make(cves.VendorProductToRepoMap) + ReposForCVE := make(map[models.CVEID][]string) + if *parsedCPEDictionary != "" { err = loadCPEDictionary(&VPRepoCache, *parsedCPEDictionary) if err != nil { @@ -116,7 +117,6 @@ func main() { logger.Info("VendorProductToRepoMap cache has entries preloaded", slog.Int("count", len(VPRepoCache))) } - for _, cve := range parsed.Vulnerabilities { refs := cve.CVE.References CPEs := cves.CPEs(cve.CVE) From ec8ef664588812f10d1a5dee6d2478ccfd44ee79 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Thu, 22 Jan 2026 21:48:37 +0000 Subject: [PATCH 13/14] Fix importing --- vulnfeeds/cmd/combine-to-osv/main.go | 5 ++--- vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go | 2 +- vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go | 2 +- vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go | 3 +-- vulnfeeds/cmd/mirrors/download-cves/main.go | 2 +- vulnfeeds/cmd/pypi/main.go | 2 +- vulnfeeds/conversion/common.go | 2 +- vulnfeeds/cvelist2osv/common.go | 2 +- vulnfeeds/cvelist2osv/common_test.go | 4 ++-- vulnfeeds/cvelist2osv/converter.go | 2 +- vulnfeeds/cvelist2osv/converter_test.go | 3 +-- vulnfeeds/cvelist2osv/default_extractor.go | 4 ++-- vulnfeeds/cvelist2osv/linux_extractor.go | 5 ++--- vulnfeeds/cvelist2osv/version_extraction_test.go | 3 +-- vulnfeeds/cves/versions_test.go | 3 +-- 15 files changed, 19 insertions(+), 25 deletions(-) diff --git a/vulnfeeds/cmd/combine-to-osv/main.go b/vulnfeeds/cmd/combine-to-osv/main.go index a4c87f1a3cc..ceae14c63a8 100644 --- a/vulnfeeds/cmd/combine-to-osv/main.go +++ b/vulnfeeds/cmd/combine-to-osv/main.go @@ -7,17 +7,16 @@ import ( "errors" "flag" "fmt" - "github.com/google/osv/vulnfeeds/models" "io/fs" "log/slog" "os" "path/filepath" - "strings" - "slices" + "strings" "cloud.google.com/go/storage" "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/upload" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/ossf/osv-schema/bindings/go/osvschema" diff --git a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go index ecd2269aeb4..6f2b7929908 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go +++ b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go @@ -5,7 +5,6 @@ import ( _ "embed" "encoding/json" "flag" - "github.com/google/osv/vulnfeeds/conversion" "log/slog" "os" "path/filepath" @@ -15,6 +14,7 @@ import ( "sync" "time" + "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/cvelist2osv" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" diff --git a/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go index 105e01a5890..f0190bd7ac5 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go +++ b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go @@ -4,10 +4,10 @@ package main import ( "encoding/json" "flag" - "github.com/google/osv/vulnfeeds/conversion" "log/slog" "os" + "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/cvelist2osv" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go index aa4fe012ce3..2e92d78a5c4 100644 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go @@ -10,9 +10,8 @@ import ( "log/slog" "os" "path/filepath" - "strings" - "slices" + "strings" "github.com/google/osv/vulnfeeds/conversion/nvd" "github.com/google/osv/vulnfeeds/cves" diff --git a/vulnfeeds/cmd/mirrors/download-cves/main.go b/vulnfeeds/cmd/mirrors/download-cves/main.go index de820ed2c74..8df5b63d87a 100644 --- a/vulnfeeds/cmd/mirrors/download-cves/main.go +++ b/vulnfeeds/cmd/mirrors/download-cves/main.go @@ -8,7 +8,6 @@ import ( "errors" "flag" "fmt" - "github.com/google/osv/vulnfeeds/models" "io" "log/slog" "net/http" @@ -18,6 +17,7 @@ import ( "strconv" "time" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/sethvargo/go-retry" ) diff --git a/vulnfeeds/cmd/pypi/main.go b/vulnfeeds/cmd/pypi/main.go index 2974199b89d..a20d2a1ec69 100644 --- a/vulnfeeds/cmd/pypi/main.go +++ b/vulnfeeds/cmd/pypi/main.go @@ -19,7 +19,6 @@ import ( "encoding/json" "flag" "fmt" - "github.com/google/osv/vulnfeeds/models" "io/fs" "log/slog" "net/http" @@ -28,6 +27,7 @@ import ( "strings" "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/pypi" "github.com/google/osv/vulnfeeds/triage" "github.com/google/osv/vulnfeeds/utility/logger" diff --git a/vulnfeeds/conversion/common.go b/vulnfeeds/conversion/common.go index 7cb8fa48147..f6e2c785e51 100644 --- a/vulnfeeds/conversion/common.go +++ b/vulnfeeds/conversion/common.go @@ -2,7 +2,6 @@ package conversion import ( "encoding/json" - "github.com/google/osv/vulnfeeds/utility/logger" "log/slog" "os" "path/filepath" @@ -10,6 +9,7 @@ import ( "strings" "github.com/google/osv/vulnfeeds/models" + "github.com/google/osv/vulnfeeds/utility/logger" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" ) diff --git a/vulnfeeds/cvelist2osv/common.go b/vulnfeeds/cvelist2osv/common.go index 954002ae1a2..a97ee193c6d 100644 --- a/vulnfeeds/cvelist2osv/common.go +++ b/vulnfeeds/cvelist2osv/common.go @@ -3,13 +3,13 @@ package cvelist2osv import ( "cmp" "errors" - "github.com/google/osv/vulnfeeds/models" "log/slog" "strconv" "strings" "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/git" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/google/osv/vulnfeeds/vulns" diff --git a/vulnfeeds/cvelist2osv/common_test.go b/vulnfeeds/cvelist2osv/common_test.go index fb3ff73585c..92fed895fe7 100644 --- a/vulnfeeds/cvelist2osv/common_test.go +++ b/vulnfeeds/cvelist2osv/common_test.go @@ -1,11 +1,11 @@ package cvelist2osv import ( - "github.com/google/osv/vulnfeeds/conversion" - "github.com/google/osv/vulnfeeds/models" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/osv/vulnfeeds/conversion" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" diff --git a/vulnfeeds/cvelist2osv/converter.go b/vulnfeeds/cvelist2osv/converter.go index 7bfa990d43f..77b5ca4d5bc 100644 --- a/vulnfeeds/cvelist2osv/converter.go +++ b/vulnfeeds/cvelist2osv/converter.go @@ -4,13 +4,13 @@ package cvelist2osv import ( "encoding/json" "fmt" - "github.com/google/osv/vulnfeeds/conversion" "io" "log/slog" "slices" "sort" "time" + "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility" diff --git a/vulnfeeds/cvelist2osv/converter_test.go b/vulnfeeds/cvelist2osv/converter_test.go index b4930099a1c..f27d513e0e8 100644 --- a/vulnfeeds/cvelist2osv/converter_test.go +++ b/vulnfeeds/cvelist2osv/converter_test.go @@ -3,7 +3,6 @@ package cvelist2osv import ( "bytes" "encoding/json" - "github.com/google/osv/vulnfeeds/models" "os" "path/filepath" "sort" @@ -11,8 +10,8 @@ import ( "testing" "github.com/gkampitakis/go-snaps/snaps" - "github.com/google/go-cmp/cmp" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvconstants" "github.com/ossf/osv-schema/bindings/go/osvschema" diff --git a/vulnfeeds/cvelist2osv/default_extractor.go b/vulnfeeds/cvelist2osv/default_extractor.go index ac05b5b2911..652dd215bba 100644 --- a/vulnfeeds/cvelist2osv/default_extractor.go +++ b/vulnfeeds/cvelist2osv/default_extractor.go @@ -2,12 +2,12 @@ package cvelist2osv import ( "fmt" - "github.com/google/osv/vulnfeeds/conversion" - "github.com/google/osv/vulnfeeds/models" "log/slog" + "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/git" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" diff --git a/vulnfeeds/cvelist2osv/linux_extractor.go b/vulnfeeds/cvelist2osv/linux_extractor.go index 5a9983e1840..4f2708f9077 100644 --- a/vulnfeeds/cvelist2osv/linux_extractor.go +++ b/vulnfeeds/cvelist2osv/linux_extractor.go @@ -2,14 +2,13 @@ package cvelist2osv import ( "fmt" - "github.com/google/osv/vulnfeeds/conversion" "slices" "strconv" "strings" - "github.com/google/osv/vulnfeeds/models" - + "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvconstants" "github.com/ossf/osv-schema/bindings/go/osvschema" diff --git a/vulnfeeds/cvelist2osv/version_extraction_test.go b/vulnfeeds/cvelist2osv/version_extraction_test.go index 46513f8314e..e1080681a0e 100644 --- a/vulnfeeds/cvelist2osv/version_extraction_test.go +++ b/vulnfeeds/cvelist2osv/version_extraction_test.go @@ -5,10 +5,9 @@ import ( "sort" "testing" - "github.com/google/osv/vulnfeeds/models" - "github.com/google/go-cmp/cmp" "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" diff --git a/vulnfeeds/cves/versions_test.go b/vulnfeeds/cves/versions_test.go index 9ace0f34afa..0cd6b30055e 100644 --- a/vulnfeeds/cves/versions_test.go +++ b/vulnfeeds/cves/versions_test.go @@ -6,11 +6,10 @@ import ( "log" "os" "reflect" + "slices" "testing" "time" - "slices" - "github.com/google/go-cmp/cmp" "github.com/google/osv/vulnfeeds/internal/testutils" "github.com/google/osv/vulnfeeds/models" From a7e8493c7406e64bf7d5c23980e8916bb2e38274 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Thu, 22 Jan 2026 23:31:32 +0000 Subject: [PATCH 14/14] fix lint (again) --- vulnfeeds/cmd/combine-to-osv/main_test.go | 2 +- vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go | 4 ---- vulnfeeds/cmd/converters/debian/main_test.go | 2 +- vulnfeeds/conversion/common.go | 2 ++ vulnfeeds/conversion/nvd/converter.go | 1 + vulnfeeds/cves/versions.go | 3 ++- vulnfeeds/models/cve.go | 2 +- vulnfeeds/pypi/pypi.go | 3 +-- 8 files changed, 9 insertions(+), 10 deletions(-) diff --git a/vulnfeeds/cmd/combine-to-osv/main_test.go b/vulnfeeds/cmd/combine-to-osv/main_test.go index 39cabcf36f6..6930851b895 100644 --- a/vulnfeeds/cmd/combine-to-osv/main_test.go +++ b/vulnfeeds/cmd/combine-to-osv/main_test.go @@ -1,7 +1,6 @@ package main import ( - "github.com/google/osv/vulnfeeds/models" "path/filepath" "sort" "testing" @@ -9,6 +8,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/osv/vulnfeeds/models" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go index 2e92d78a5c4..8ff417ce9ba 100644 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go @@ -24,10 +24,6 @@ var ErrNoRanges = errors.New("no ranges") var ErrUnresolvedFix = errors.New("fixes not resolved to commits") -const ( - extension = ".json" -) - var ( jsonPath = flag.String("nvd-json", "", "Path to NVD CVE JSON to examine.") parsedCPEDictionary = flag.String("cpe-repos", "", "Path to JSON mapping of CPEs to repos generated by cpe-repo-gen") diff --git a/vulnfeeds/cmd/converters/debian/main_test.go b/vulnfeeds/cmd/converters/debian/main_test.go index ec0616af753..6635ea5e324 100644 --- a/vulnfeeds/cmd/converters/debian/main_test.go +++ b/vulnfeeds/cmd/converters/debian/main_test.go @@ -3,13 +3,13 @@ package main import ( "encoding/json" "fmt" - "github.com/google/osv/vulnfeeds/models" "os" "sort" "testing" "time" "github.com/google/go-cmp/cmp" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" diff --git a/vulnfeeds/conversion/common.go b/vulnfeeds/conversion/common.go index f6e2c785e51..b306b1179b3 100644 --- a/vulnfeeds/conversion/common.go +++ b/vulnfeeds/conversion/common.go @@ -1,3 +1,5 @@ +// Package conversion implements common utilities for converting vulnerability data +// from various sources into the OSV schema. package conversion import ( diff --git a/vulnfeeds/conversion/nvd/converter.go b/vulnfeeds/conversion/nvd/converter.go index 3d95499c0bd..3fa39e1643e 100644 --- a/vulnfeeds/conversion/nvd/converter.go +++ b/vulnfeeds/conversion/nvd/converter.go @@ -1,3 +1,4 @@ +// Package nvd converts NVD CVEs to OSV format. package nvd import ( diff --git a/vulnfeeds/cves/versions.go b/vulnfeeds/cves/versions.go index b8755adfc0e..7923e2e52fe 100644 --- a/vulnfeeds/cves/versions.go +++ b/vulnfeeds/cves/versions.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// https://www.apache.org/licenses/LICENSE-2.0 +// https://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package cves provides utilities for working with CVEs and version information. package cves import ( diff --git a/vulnfeeds/models/cve.go b/vulnfeeds/models/cve.go index 75cab7e8a41..875896834dd 100644 --- a/vulnfeeds/models/cve.go +++ b/vulnfeeds/models/cve.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package cves contains CVE-specific data structures. +// Package models contains CVE-specific data structures. package models import ( diff --git a/vulnfeeds/pypi/pypi.go b/vulnfeeds/pypi/pypi.go index 2294199d21d..ed798350e1d 100644 --- a/vulnfeeds/pypi/pypi.go +++ b/vulnfeeds/pypi/pypi.go @@ -17,7 +17,6 @@ package pypi import ( "encoding/json" - "github.com/google/osv/vulnfeeds/models" "log" "log/slog" "net/http" @@ -28,8 +27,8 @@ import ( "strings" version "github.com/aquasecurity/go-pep440-version" - "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/triage" "github.com/google/osv/vulnfeeds/utility/logger" )