diff --git a/vulnfeeds/cmd/combine-to-osv/main.go b/vulnfeeds/cmd/combine-to-osv/main.go index a44c3c93d50..ceae14c63a8 100644 --- a/vulnfeeds/cmd/combine-to-osv/main.go +++ b/vulnfeeds/cmd/combine-to-osv/main.go @@ -11,12 +11,12 @@ import ( "log/slog" "os" "path/filepath" - "strings" - "slices" + "strings" "cloud.google.com/go/storage" "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/upload" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -138,8 +138,8 @@ func listBucketObjects(bucketName string, prefix string) ([]string, error) { // The function returns a map of CVE IDs to their corresponding Vulnerability objects. // Files that are not ".json" files, directories, or files ending in ".metrics.json" are skipped. // The function will log warnings for files that fail to open or decode, and will terminate if it fails to walk the directory. -func loadOSV(osvPath string) map[cves.CVEID]*osvschema.Vulnerability { - allVulns := make(map[cves.CVEID]*osvschema.Vulnerability) +func loadOSV(osvPath string) map[models.CVEID]*osvschema.Vulnerability { + allVulns := make(map[models.CVEID]*osvschema.Vulnerability) logger.Info("Loading OSV records", slog.String("path", osvPath)) err := filepath.WalkDir(osvPath, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -161,7 +161,7 @@ func loadOSV(osvPath string) map[cves.CVEID]*osvschema.Vulnerability { logger.Error("Failed to decode, skipping", slog.String("file", path), slog.Any("err", decodeErr)) return nil } - allVulns[cves.CVEID(vuln.GetId())] = &vuln + allVulns[models.CVEID(vuln.GetId())] = &vuln return nil }) @@ -174,8 +174,8 @@ func loadOSV(osvPath string) map[cves.CVEID]*osvschema.Vulnerability { } // combineIntoOSV creates OSV entry by combining loaded CVEs from NVD and PackageInfo information from security advisories. -func combineIntoOSV(cve5osv map[cves.CVEID]*osvschema.Vulnerability, nvdosv map[cves.CVEID]*osvschema.Vulnerability, mandatoryCVEIDs []string) map[cves.CVEID]*osvschema.Vulnerability { - osvRecords := make(map[cves.CVEID]*osvschema.Vulnerability) +func combineIntoOSV(cve5osv map[models.CVEID]*osvschema.Vulnerability, nvdosv map[models.CVEID]*osvschema.Vulnerability, mandatoryCVEIDs []string) map[models.CVEID]*osvschema.Vulnerability { + osvRecords := make(map[models.CVEID]*osvschema.Vulnerability) // Iterate through CVEs from security advisories (cve5) as the base for cveID, cve5 := range cve5osv { diff --git a/vulnfeeds/cmd/combine-to-osv/main_test.go b/vulnfeeds/cmd/combine-to-osv/main_test.go index c60794efeb8..6930851b895 100644 --- a/vulnfeeds/cmd/combine-to-osv/main_test.go +++ b/vulnfeeds/cmd/combine-to-osv/main_test.go @@ -8,7 +8,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" @@ -35,7 +35,7 @@ func TestCombineIntoOSV(t *testing.T) { cve5osv := loadOSV(cve5Path) nvdosv := loadOSV(nvdPath) - nvdosvCopy := make(map[cves.CVEID]*osvschema.Vulnerability) + nvdosvCopy := make(map[models.CVEID]*osvschema.Vulnerability) for k, v := range nvdosv { nvdosvCopy[k] = v } diff --git a/vulnfeeds/cmd/converters/alpine/main.go b/vulnfeeds/cmd/converters/alpine/main.go index eb42b3932d8..adb57a3e3e8 100644 --- a/vulnfeeds/cmd/converters/alpine/main.go +++ b/vulnfeeds/cmd/converters/alpine/main.go @@ -15,7 +15,6 @@ import ( "strings" "time" - "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/upload" "github.com/google/osv/vulnfeeds/utility/logger" @@ -138,7 +137,7 @@ func getAlpineSecDBData() map[string][]VersionAndPkg { } // generateAlpineOSV generates the generic PackageInfo package from the information given by alpine advisory -func generateAlpineOSV(allAlpineSecDb map[string][]VersionAndPkg, allCVEs map[cves.CVEID]cves.Vulnerability) (osvVulnerabilities []*vulns.Vulnerability) { +func generateAlpineOSV(allAlpineSecDb map[string][]VersionAndPkg, allCVEs map[models.CVEID]models.Vulnerability) (osvVulnerabilities []*vulns.Vulnerability) { cveIDs := make([]string, 0, len(allAlpineSecDb)) for cveID := range allAlpineSecDb { cveIDs = append(cveIDs, cveID) @@ -157,7 +156,7 @@ func generateAlpineOSV(allAlpineSecDb map[string][]VersionAndPkg, allCVEs map[cv return verPkgs[i].Ver < verPkgs[j].Ver }) - cve, ok := allCVEs[cves.CVEID(cveID)] + cve, ok := allCVEs[models.CVEID(cveID)] var published time.Time var details string if ok { diff --git a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go index 12dd4ef1216..6f2b7929908 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go +++ b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go @@ -14,8 +14,9 @@ import ( "sync" "time" + "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/cvelist2osv" - "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" ) @@ -34,7 +35,7 @@ func main() { flag.Parse() logger.InitGlobalLogger() - logger.Info("Commencing Linux CVE to OSV conversion run") + logger.Info("Commencing CVE to OSV conversion run") if err := os.MkdirAll(*localOutputDir, 0755); err != nil { logger.Fatal("Failed to create local output directory", slog.Any("err", err)) } @@ -103,7 +104,7 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string continue } - var cve cves.CVE5 + var cve models.CVE5 if err := json.Unmarshal(data, &cve); err != nil { logger.Info("Failed to unmarshal JSON", slog.String("path", path), slog.Any("err", err)) continue @@ -115,8 +116,8 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string cveID := cve.Metadata.CVEID logger.Info("Processing "+string(cveID), slog.String("cve", string(cveID))) - osvFile, errCVE := cvelist2osv.CreateOSVFile(cveID, outDir) - metricsFile, errMetrics := cvelist2osv.CreateMetricsFile(cveID, outDir) + osvFile, errCVE := conversion.CreateOSVFile(cveID, outDir) + metricsFile, errMetrics := conversion.CreateMetricsFile(cveID, outDir) if errCVE != nil || errMetrics != nil { logger.Fatal("File failed to be created for CVE", slog.String("cve", string(cveID))) } diff --git a/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go index 471e95fc161..f0190bd7ac5 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go +++ b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go @@ -7,8 +7,9 @@ import ( "log/slog" "os" + "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/cvelist2osv" - "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" ) @@ -28,7 +29,7 @@ func main() { logger.Fatal("Failed to open file", slog.Any("err", err)) } - var cve cves.CVE5 + var cve models.CVE5 if err = json.Unmarshal(data, &cve); err != nil { logger.Fatal("Failed to parse CVEList CVE JSON", slog.Any("err", err)) } @@ -44,8 +45,8 @@ func main() { } // create the files - osvFile, errCVE := cvelist2osv.CreateOSVFile(cveID, outDir) - metricsFile, errMetrics := cvelist2osv.CreateMetricsFile(cveID, outDir) + osvFile, errCVE := conversion.CreateOSVFile(cveID, outDir) + metricsFile, errMetrics := conversion.CreateMetricsFile(cveID, outDir) if errCVE != nil || errMetrics != nil { logger.Fatal("File failed to be created for CVE", slog.String("cve", string(cveID))) } diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go index 2878d2f6468..8ff417ce9ba 100644 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go @@ -8,45 +8,22 @@ import ( "flag" "fmt" "log/slog" - "net/http" "os" "path/filepath" - "strings" - "slices" + "strings" + "github.com/google/osv/vulnfeeds/conversion/nvd" "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/git" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" - "github.com/google/osv/vulnfeeds/vulns" ) -type ConversionOutcome int - var ErrNoRanges = errors.New("no ranges") var ErrUnresolvedFix = errors.New("fixes not resolved to commits") -func (c ConversionOutcome) String() string { - return [...]string{"ConversionUnknown", "Successful", "Rejected", "NoSoftware", "NoRepos", "NoRanges", "FixUnresolvable"}[c] -} - -const ( - extension = ".json" -) - -const ( - // Set of enums for categorizing conversion outcomes. - ConversionUnknown ConversionOutcome = iota // Shouldn't happen - Successful // It worked! - Rejected // The CVE was rejected - NoSoftware // The CVE had no CPEs relating to software (i.e. Operating Systems or Hardware). - NoRepos // The CPE Vendor/Product had no repositories derived for it. - NoRanges // No viable commit ranges could be calculated from the repository for the CVE's CPE(s). - FixUnresolvable // Partial resolution of versions, resulting in a false positive. -) - var ( jsonPath = flag.String("nvd-json", "", "Path to NVD CVE JSON to examine.") parsedCPEDictionary = flag.String("cpe-repos", "", "Path to JSON mapping of CPEs to repos generated by cpe-repo-gen") @@ -59,203 +36,7 @@ var Metrics struct { CVEsForApplications int CVEsForKnownRepos int OSVRecordsGenerated int - Outcomes map[cves.CVEID]ConversionOutcome // Per-CVE-ID record of conversion result. -} - -// Takes an NVD CVE record and outputs an OSV file in the specified directory. -func CVEToOSV(cve cves.CVE, repos []string, cache git.RepoTagsCache, directory string) error { - CPEs := cves.CPEs(cve) - // The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy. - maybeVendorName := "ENOCPE" - maybeProductName := "ENOCPE" - - if len(CPEs) > 0 { - CPE, err := cves.ParseCPE(CPEs[0]) // For naming the subdirectory used for output. - maybeVendorName = CPE.Vendor - maybeProductName = CPE.Product - if err != nil { - return fmt.Errorf("[%s]: Can't generate an OSV record without valid CPE data", cve.ID) - } - } - - v := vulns.FromNVDCVE(cve.ID, cve) - versions, notes := cves.ExtractVersionInfo(cve, nil, http.DefaultClient) - - if len(versions.AffectedVersions) != 0 { - var err error - // There are some AffectedVersions to try and resolve to AffectedCommits. - if len(repos) == 0 { - return fmt.Errorf("[%s]: No affected ranges for %q, and no repos to try and convert %+v to tags with", cve.ID, maybeProductName, versions.AffectedVersions) - } - logger.Info("Trying to convert version tags to commits", slog.String("cve", string(cve.ID)), slog.Any("versions", versions), slog.Any("repos", repos)) - versions, err = cves.GitVersionsToCommits(cve.ID, versions, repos, cache) - if err != nil { - return fmt.Errorf("[%s]: Failed to convert version tags to commits: %#w", cve.ID, err) - } - hasAnyFixedCommits := false - for _, repo := range repos { - if versions.HasFixedCommits(repo) { - hasAnyFixedCommits = true - break - } - } - - if versions.HasFixedVersions() && !hasAnyFixedCommits { - return fmt.Errorf("[%s]: Failed to convert fixed version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) - } - - hasAnyLastAffectedCommits := false - for _, repo := range repos { - if versions.HasLastAffectedCommits(repo) { - hasAnyLastAffectedCommits = true - break - } - } - - if versions.HasLastAffectedVersions() && !hasAnyLastAffectedCommits && !hasAnyFixedCommits { - return fmt.Errorf("[%s]: Failed to convert last_affected version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) - } - } - - slices.SortStableFunc(versions.AffectedCommits, models.AffectedCommitCompare) - - vulns.AttachExtractedVersionInfo(v, versions) - - if len(v.Affected) == 0 { - return fmt.Errorf("[%s]: No affected ranges detected for %q %w", cve.ID, maybeProductName, ErrNoRanges) - } - - vulnDir := filepath.Join(directory, maybeVendorName, maybeProductName) - err := os.MkdirAll(vulnDir, 0755) - if err != nil { - logger.Warn("Failed to create dir", slog.Any("err", err)) - return fmt.Errorf("failed to create dir: %w", err) - } - outputFile := filepath.Join(vulnDir, v.Id+extension) - notesFile := filepath.Join(vulnDir, v.Id+".notes") - f, err := os.Create(outputFile) - if err != nil { - logger.Warn("Failed to open for writing", slog.String("path", outputFile), slog.Any("err", err)) - return fmt.Errorf("failed to open %s for writing: %w", outputFile, err) - } - defer f.Close() - err = v.ToJSON(f) - if err != nil { - logger.Warn("Failed to write", slog.String("path", outputFile), slog.Any("err", err)) - return fmt.Errorf("failed to write %s: %w", outputFile, err) - } - logger.Info("Generated OSV record", slog.String("cve", string(cve.ID)), slog.String("product", maybeProductName)) - if len(notes) > 0 { - err = os.WriteFile(notesFile, []byte(strings.Join(notes, "\n")), 0600) - if err != nil { - logger.Warn("Failed to write", slog.String("cve", string(cve.ID)), slog.String("path", notesFile), slog.Any("err", err)) - } - } - - return nil -} - -// Takes an NVD CVE record and outputs a PackageInfo struct in a file in the specified directory. -func CVEToPackageInfo(cve cves.CVE, repos []string, cache git.RepoTagsCache, directory string) error { - CPEs := cves.CPEs(cve) - // The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy. - maybeVendorName := "ENOCPE" - maybeProductName := "ENOCPE" - - if len(CPEs) > 0 { - CPE, err := cves.ParseCPE(CPEs[0]) // For naming the subdirectory used for output. - maybeVendorName = CPE.Vendor - maybeProductName = CPE.Product - if err != nil { - return fmt.Errorf("[%s]: Can't generate an OSV record without valid CPE data", cve.ID) - } - } - - // more often than not, this yields a VersionInfo with AffectedVersions and no AffectedCommits. - versions, notes := cves.ExtractVersionInfo(cve, nil, http.DefaultClient) - - if len(versions.AffectedVersions) != 0 { - var err error - // There are some AffectedVersions to try and resolve to AffectedCommits. - if len(repos) == 0 { - return fmt.Errorf("[%s]: No affected ranges for %q, and no repos to try and convert %+v to tags with", cve.ID, maybeProductName, versions.AffectedVersions) - } - logger.Info("Trying to convert version tags to commits", slog.String("cve", string(cve.ID)), slog.Any("versions", versions), slog.Any("repos", repos)) - versions, err = cves.GitVersionsToCommits(cve.ID, versions, repos, cache) - if err != nil { - return fmt.Errorf("[%s]: Failed to convert version tags to commits: %#w", cve.ID, err) - } - } - - hasAnyFixedCommits := false - for _, repo := range repos { - if versions.HasFixedCommits(repo) { - hasAnyFixedCommits = true - } - } - - if versions.HasFixedVersions() && !hasAnyFixedCommits { - return fmt.Errorf("[%s]: Failed to convert fixed version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) - } - - hasAnyLastAffectedCommits := false - for _, repo := range repos { - if versions.HasLastAffectedCommits(repo) { - hasAnyLastAffectedCommits = true - } - } - - if versions.HasLastAffectedVersions() && !hasAnyLastAffectedCommits && !hasAnyFixedCommits { - return fmt.Errorf("[%s]: Failed to convert last_affected version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) - } - - if len(versions.AffectedCommits) == 0 { - return fmt.Errorf("[%s]: No affected commit ranges determined for %q %w", cve.ID, maybeProductName, ErrNoRanges) - } - - versions.AffectedVersions = nil // these have served their purpose and are not required in the resulting output. - - slices.SortStableFunc(versions.AffectedCommits, models.AffectedCommitCompare) - - var pkgInfos []vulns.PackageInfo - pi := vulns.PackageInfo{VersionInfo: versions} - pkgInfos = append(pkgInfos, pi) // combine-to-osv expects a serialised *array* of PackageInfo - - vulnDir := filepath.Join(directory, maybeVendorName, maybeProductName) - err := os.MkdirAll(vulnDir, 0755) - if err != nil { - logger.Warn("Failed to create dir", slog.Any("err", err)) - return fmt.Errorf("failed to create dir: %w", err) - } - - outputFile := filepath.Join(vulnDir, string(cve.ID)+".nvd"+extension) - notesFile := filepath.Join(vulnDir, string(cve.ID)+".nvd.notes") - f, err := os.Create(outputFile) - if err != nil { - logger.Warn("Failed to open for writing", slog.String("path", outputFile), slog.Any("err", err)) - return fmt.Errorf("failed to open %s for writing: %w", outputFile, err) - } - defer f.Close() - - encoder := json.NewEncoder(f) - encoder.SetIndent("", " ") - err = encoder.Encode(&pkgInfos) - - if err != nil { - logger.Warn("Failed to encode PackageInfo", slog.String("path", outputFile), slog.Any("err", err)) - return fmt.Errorf("failed to encode PackageInfo to %s: %w", outputFile, err) - } - - logger.Info("Generated PackageInfo record", slog.String("cve", string(cve.ID)), slog.String("product", maybeProductName)) - - if len(notes) > 0 { - err = os.WriteFile(notesFile, []byte(strings.Join(notes, "\n")), 0600) - if err != nil { - logger.Warn("Failed to write", slog.String("cve", string(cve.ID)), slog.String("path", notesFile), slog.Any("err", err)) - } - } - - return nil + Outcomes map[models.CVEID]models.ConversionOutcome // Per-CVE-ID record of conversion result. } func loadCPEDictionary(productToRepo *cves.VendorProductToRepoMap, f string) error { @@ -268,7 +49,7 @@ func loadCPEDictionary(productToRepo *cves.VendorProductToRepoMap, f string) err } // Output a CSV summarizing per-CVE how it was handled. -func outputOutcomes(outcomes map[cves.CVEID]ConversionOutcome, reposForCVE map[cves.CVEID][]string, directory string) error { +func outputOutcomes(outcomes map[models.CVEID]models.ConversionOutcome, reposForCVE map[models.CVEID][]string, directory string) error { outcomesFile, err := os.Create(filepath.Join(directory, "outcomes.csv")) if err != nil { return err @@ -304,7 +85,7 @@ func main() { os.Exit(1) } - Metrics.Outcomes = make(map[cves.CVEID]ConversionOutcome) + Metrics.Outcomes = make(map[models.CVEID]models.ConversionOutcome) logger.InitGlobalLogger() @@ -313,7 +94,7 @@ func main() { logger.Fatal("Failed to open file", slog.Any("err", err)) // double check this is best practice output } - var parsed cves.CVEAPIJSON20Schema + var parsed models.CVEAPIJSON20Schema err = json.Unmarshal(data, &parsed) if err != nil { logger.Fatal("Failed to parse NVD CVE JSON", slog.Any("err", err)) @@ -321,6 +102,8 @@ func main() { VPRepoCache := make(cves.VendorProductToRepoMap) + ReposForCVE := make(map[models.CVEID][]string) + if *parsedCPEDictionary != "" { err = loadCPEDictionary(&VPRepoCache, *parsedCPEDictionary) if err != nil { @@ -329,8 +112,6 @@ func main() { logger.Info("VendorProductToRepoMap cache has entries preloaded", slog.Int("count", len(VPRepoCache))) } - ReposForCVE := make(map[cves.CVEID][]string) - for _, cve := range parsed.Vulnerabilities { refs := cve.CVE.References CPEs := cves.CPEs(cve.CVE) @@ -339,7 +120,6 @@ func main() { if len(refs) == 0 && len(CPEs) == 0 { logger.Info("Skipping due to lack of CPEs and lack of references", slog.String("cve", string(CVEID))) // 100% of these in 2022 were rejected CVEs - Metrics.Outcomes[CVEID] = Rejected continue } @@ -361,7 +141,6 @@ func main() { CPE, err := cves.ParseCPE(CPEstr) if err != nil { logger.Warn("Failed to parse CPE", slog.String("cve", string(CVEID)), slog.String("cpe", CPEstr), slog.Any("err", err)) - Metrics.Outcomes[CVEID] = ConversionUnknown continue } @@ -386,7 +165,6 @@ func main() { if len(CPEs) > 0 && appCPECount == 0 { // This CVE is not for software (based on there being CPEs but not any application ones), skip. - Metrics.Outcomes[CVEID] = NoSoftware continue } @@ -441,7 +219,7 @@ func main() { if _, ok := ReposForCVE[CVEID]; !ok { // We have nothing useful to work with, so we'll assume it's out of scope logger.Info("Passing due to lack of viable repository", slog.String("cve", string(CVEID))) - Metrics.Outcomes[CVEID] = NoRepos + Metrics.Outcomes[CVEID] = models.NoRepos continue } @@ -452,27 +230,27 @@ func main() { switch *outFormat { case "OSV": - err = CVEToOSV(cve.CVE, ReposForCVE[CVEID], RepoTagsCache, *outDir) + err = nvd.CVEToOSV(cve.CVE, ReposForCVE[CVEID], RepoTagsCache, *outDir) case "PackageInfo": - err = CVEToPackageInfo(cve.CVE, ReposForCVE[CVEID], RepoTagsCache, *outDir) + err = nvd.CVEToPackageInfo(cve.CVE, ReposForCVE[CVEID], RepoTagsCache, *outDir) } // Parse this error to determine which failure mode it was if err != nil { logger.Warn("Failed to generate an OSV record", slog.String("cve", string(CVEID)), slog.Any("err", err)) if errors.Is(err, ErrNoRanges) { - Metrics.Outcomes[CVEID] = NoRanges + Metrics.Outcomes[CVEID] = models.NoRanges continue } if errors.Is(err, ErrUnresolvedFix) { - Metrics.Outcomes[CVEID] = FixUnresolvable + Metrics.Outcomes[CVEID] = models.FixUnresolvable continue } - Metrics.Outcomes[CVEID] = ConversionUnknown + Metrics.Outcomes[CVEID] = models.ConversionUnknown continue } Metrics.OSVRecordsGenerated++ - Metrics.Outcomes[CVEID] = Successful + Metrics.Outcomes[CVEID] = models.Successful } Metrics.TotalCVEs = len(parsed.Vulnerabilities) err = outputOutcomes(Metrics.Outcomes, ReposForCVE, *outDir) diff --git a/vulnfeeds/cmd/converters/debian/main.go b/vulnfeeds/cmd/converters/debian/main.go index 3dad1d64479..6634f88c796 100644 --- a/vulnfeeds/cmd/converters/debian/main.go +++ b/vulnfeeds/cmd/converters/debian/main.go @@ -14,7 +14,6 @@ import ( "strconv" "strings" - "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/faulttolerant" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/upload" @@ -75,7 +74,7 @@ func main() { } // generateOSVFromDebianTracker converts Debian Security Tracker entries to OSV format. -func generateOSVFromDebianTracker(debianData DebianSecurityTrackerData, debianReleaseMap map[string]string, allCVEs map[cves.CVEID]cves.Vulnerability) map[string]*vulns.Vulnerability { +func generateOSVFromDebianTracker(debianData DebianSecurityTrackerData, debianReleaseMap map[string]string, allCVEs map[models.CVEID]models.Vulnerability) map[string]*vulns.Vulnerability { logger.Info("Converting Debian Security Tracker data to OSV.") osvCves := make(map[string]*vulns.Vulnerability) @@ -107,7 +106,7 @@ func generateOSVFromDebianTracker(debianData DebianSecurityTrackerData, debianRe continue } v, ok := osvCves[cveID] - currentNVDCVE := allCVEs[cves.CVEID(cveID)] + currentNVDCVE := allCVEs[models.CVEID(cveID)] if !ok { v = &vulns.Vulnerability{ Vulnerability: &osvschema.Vulnerability{ diff --git a/vulnfeeds/cmd/converters/debian/main_test.go b/vulnfeeds/cmd/converters/debian/main_test.go index 7757c2efe93..6635ea5e324 100644 --- a/vulnfeeds/cmd/converters/debian/main_test.go +++ b/vulnfeeds/cmd/converters/debian/main_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/google/go-cmp/cmp" - "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" @@ -38,14 +38,14 @@ func sortAffected(affected []*osvschema.Affected) { }) } -func loadTestData(t *testing.T, cveName string) cves.Vulnerability { +func loadTestData(t *testing.T, cveName string) models.Vulnerability { t.Helper() fileName := fmt.Sprintf("../../../test_data/nvdcve-2.0/%s.json", cveName) file, err := os.Open(fileName) if err != nil { t.Fatalf("Failed to load test data from %q: %#v", fileName, err) } - var nvdCves cves.CVEAPIJSON20Schema + var nvdCves models.CVEAPIJSON20Schema err = json.NewDecoder(file).Decode(&nvdCves) if err != nil { t.Fatalf("Failed to decode %q: %+v", fileName, err) @@ -57,7 +57,7 @@ func loadTestData(t *testing.T, cveName string) cves.Vulnerability { } t.Fatalf("test data doesn't contain %q", cveName) - return cves.Vulnerability{} + return models.Vulnerability{} } func TestGenerateOSVFromDebianTracker(t *testing.T) { @@ -77,7 +77,7 @@ func TestGenerateOSVFromDebianTracker(t *testing.T) { "bookworm": "12", "trixie": "13", } - cveStuff := map[cves.CVEID]cves.Vulnerability{ + cveStuff := map[models.CVEID]models.Vulnerability{ "CVE-2014-1424": loadTestData(t, "CVE-2014-1424"), "CVE-2017-6507": loadTestData(t, "CVE-2017-6507"), "CVE-2016-1585": loadTestData(t, "CVE-2016-1585"), diff --git a/vulnfeeds/cmd/mirrors/download-cves/main.go b/vulnfeeds/cmd/mirrors/download-cves/main.go index 4a507225966..8df5b63d87a 100644 --- a/vulnfeeds/cmd/mirrors/download-cves/main.go +++ b/vulnfeeds/cmd/mirrors/download-cves/main.go @@ -17,7 +17,7 @@ import ( "strconv" "time" - "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/sethvargo/go-retry" ) @@ -58,7 +58,7 @@ func main() { // Pages are offset based, this assumes the default (and maximum) page size of PageSize // Maintaining the recommended 6 seconds betweens calls is left to the caller. // See https://nvd.nist.gov/developers/vulnerabilities -func downloadCVE2FromAPIWithOffset(apiKey string, offset int) (page *cves.CVEAPIJSON20Schema, err error) { //nolint:unused +func downloadCVE2FromAPIWithOffset(apiKey string, offset int) (page *models.CVEAPIJSON20Schema, err error) { //nolint:unused client := &http.Client{} APIURL, err := url.Parse(NVDAPIEndpoint) if err != nil { @@ -125,8 +125,8 @@ func downloadCVE2FromAPI(apiKey string, cvePath string) { //nolint:unused logger.Fatal("Something went wrong when creating/opening file", slog.Any("err", err)) } defer file.Close() - var vulnerabilities []cves.Vulnerability - var page *cves.CVEAPIJSON20Schema + var vulnerabilities []models.Vulnerability + var page *models.CVEAPIJSON20Schema offset := 0 prevTotal := 0 for { diff --git a/vulnfeeds/cmd/pypi/main.go b/vulnfeeds/cmd/pypi/main.go index 10d9050aa20..a20d2a1ec69 100644 --- a/vulnfeeds/cmd/pypi/main.go +++ b/vulnfeeds/cmd/pypi/main.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/pypi" "github.com/google/osv/vulnfeeds/triage" "github.com/google/osv/vulnfeeds/utility/logger" @@ -115,7 +116,7 @@ func main() { if err != nil { logger.Fatal("Failed to open file", slog.Any("err", err)) } - var parsed cves.CVEAPIJSON20Schema + var parsed models.CVEAPIJSON20Schema err = json.Unmarshal(data, &parsed) if err != nil { logger.Fatal("Failed to parse NVD CVE JSON", slog.Any("err", err)) diff --git a/vulnfeeds/conversion/common.go b/vulnfeeds/conversion/common.go new file mode 100644 index 00000000000..b306b1179b3 --- /dev/null +++ b/vulnfeeds/conversion/common.go @@ -0,0 +1,96 @@ +// Package conversion implements common utilities for converting vulnerability data +// from various sources into the OSV schema. +package conversion + +import ( + "encoding/json" + "log/slog" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/google/osv/vulnfeeds/models" + "github.com/google/osv/vulnfeeds/utility/logger" + "github.com/google/osv/vulnfeeds/vulns" + "github.com/ossf/osv-schema/bindings/go/osvschema" +) + +// AddAffected adds an osvschema.Affected to a vulnerability, ensuring that no duplicate ranges are added. +func AddAffected(v *vulns.Vulnerability, aff *osvschema.Affected, metrics *models.ConversionMetrics) { + allExistingRanges := make(map[string]struct{}) + for _, existingAff := range v.Affected { + for _, r := range existingAff.GetRanges() { + rangeBytes, err := json.Marshal(r) + if err == nil { + allExistingRanges[string(rangeBytes)] = struct{}{} + } + } + } + + uniqueRanges := []*osvschema.Range{} + for _, r := range aff.GetRanges() { + rangeBytes, err := json.Marshal(r) + if err != nil { + metrics.AddNote("Could not marshal range to check for duplicates, adding anyway: %+v", r) + uniqueRanges = append(uniqueRanges, r) + + continue + } + rangeStr := string(rangeBytes) + if _, exists := allExistingRanges[rangeStr]; !exists { + uniqueRanges = append(uniqueRanges, r) + allExistingRanges[rangeStr] = struct{}{} + } else { + metrics.AddNote("Skipping duplicate range: %+v", r) + } + } + + if len(uniqueRanges) > 0 { + newAff := &osvschema.Affected{ + Package: aff.GetPackage(), + Ranges: uniqueRanges, + DatabaseSpecific: aff.GetDatabaseSpecific(), + } + v.Affected = append(v.Affected, newAff) + } +} + +func DeduplicateRefs(refs []models.Reference) []models.Reference { + // Deduplicate references by URL. + slices.SortStableFunc(refs, func(a, b models.Reference) int { + return strings.Compare(a.URL, b.URL) + }) + refs = slices.CompactFunc(refs, func(a, b models.Reference) bool { + return a.URL == b.URL + }) + + return refs +} + +// CreateMetricsFile saves the collected conversion metrics to a JSON file. +// This file provides data for analyzing the success and characteristics of the +// conversion process for a given CVE. +func CreateMetricsFile(id models.CVEID, vulnDir string) (*os.File, error) { + metricsFile := filepath.Join(vulnDir, string(id)+".metrics.json") + f, err := os.Create(metricsFile) + if err != nil { + logger.Info("Failed to open for writing "+metricsFile, slog.String("cve", string(id)), slog.String("path", metricsFile), slog.Any("err", err)) + return nil, err + } + + return f, nil +} + +// CreateOSVFile creates the initial file for the OSV record. +func CreateOSVFile(id models.CVEID, vulnDir string) (*os.File, error) { + outputFile := filepath.Join(vulnDir, string(id)+models.Extension) + + f, err := os.Create(outputFile) + if err != nil { + logger.Info("Failed to open for writing "+outputFile, slog.String("cve", string(id)), slog.String("path", outputFile), slog.Any("err", err)) + return nil, err + } + + return f, err +} diff --git a/vulnfeeds/cvelist2osv/grouping.go b/vulnfeeds/conversion/grouping.go similarity index 97% rename from vulnfeeds/cvelist2osv/grouping.go rename to vulnfeeds/conversion/grouping.go index d5ff5cfa708..48f0b4546c9 100644 --- a/vulnfeeds/cvelist2osv/grouping.go +++ b/vulnfeeds/conversion/grouping.go @@ -1,4 +1,4 @@ -package cvelist2osv +package conversion import ( "fmt" @@ -11,11 +11,11 @@ import ( "google.golang.org/protobuf/types/known/structpb" ) -// groupAffectedRanges groups ranges that share the same introduced value, type, and repo. +// GroupAffectedRanges groups ranges that share the same introduced value, type, and repo. // This is because having multiple ranges with the same introduced value would act like an // OR condition, rather than AND. // This function modifies in-place -func groupAffectedRanges(affected []*osvschema.Affected) { +func GroupAffectedRanges(affected []*osvschema.Affected) { for _, aff := range affected { if len(aff.GetRanges()) <= 1 { continue diff --git a/vulnfeeds/cvelist2osv/grouping_test.go b/vulnfeeds/conversion/grouping_test.go similarity index 99% rename from vulnfeeds/cvelist2osv/grouping_test.go rename to vulnfeeds/conversion/grouping_test.go index 3d5876e4976..62d057dcc94 100644 --- a/vulnfeeds/cvelist2osv/grouping_test.go +++ b/vulnfeeds/conversion/grouping_test.go @@ -1,4 +1,4 @@ -package cvelist2osv +package conversion import ( "testing" @@ -423,7 +423,7 @@ func TestGroupAffectedRanges(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - groupAffectedRanges(tt.affected) + GroupAffectedRanges(tt.affected) if diff := cmp.Diff(tt.want, tt.affected, protocmp.Transform()); diff != "" { t.Errorf("groupAffectedRanges() mismatch (-want +got):\n%s", diff) } diff --git a/vulnfeeds/conversion/nvd/converter.go b/vulnfeeds/conversion/nvd/converter.go new file mode 100644 index 00000000000..3fa39e1643e --- /dev/null +++ b/vulnfeeds/conversion/nvd/converter.go @@ -0,0 +1,220 @@ +// Package nvd converts NVD CVEs to OSV format. +package nvd + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/git" + "github.com/google/osv/vulnfeeds/models" + "github.com/google/osv/vulnfeeds/utility/logger" + "github.com/google/osv/vulnfeeds/vulns" +) + +var ErrNoRanges = errors.New("no ranges") + +var ErrUnresolvedFix = errors.New("fixes not resolved to commits") + +// CVEToOSV Takes an NVD CVE record and outputs an OSV file in the specified directory. +func CVEToOSV(cve models.NVDCVE, repos []string, cache git.RepoTagsCache, directory string) error { + CPEs := cves.CPEs(cve) + // The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy. + maybeVendorName := "ENOCPE" + maybeProductName := "ENOCPE" + + if len(CPEs) > 0 { + CPE, err := cves.ParseCPE(CPEs[0]) // For naming the subdirectory used for output. + maybeVendorName = CPE.Vendor + maybeProductName = CPE.Product + if err != nil { + return fmt.Errorf("[%s]: Can't generate an OSV record without valid CPE data", cve.ID) + } + } + + v := vulns.FromNVDCVE(cve.ID, cve) + versions, notes := cves.ExtractVersionInfo(cve, nil, http.DefaultClient) + + if len(versions.AffectedVersions) != 0 { + var err error + // There are some AffectedVersions to try and resolve to AffectedCommits. + if len(repos) == 0 { + return fmt.Errorf("[%s]: No affected ranges for %q, and no repos to try and convert %+v to tags with", cve.ID, maybeProductName, versions.AffectedVersions) + } + logger.Info("Trying to convert version tags to commits", slog.String("cve", string(cve.ID)), slog.Any("versions", versions), slog.Any("repos", repos)) + versions, err = cves.GitVersionsToCommits(cve.ID, versions, repos, cache) + if err != nil { + return fmt.Errorf("[%s]: Failed to convert version tags to commits: %#w", cve.ID, err) + } + hasAnyFixedCommits := false + for _, repo := range repos { + if versions.HasFixedCommits(repo) { + hasAnyFixedCommits = true + break + } + } + + if versions.HasFixedVersions() && !hasAnyFixedCommits { + return fmt.Errorf("[%s]: Failed to convert fixed version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) + } + + hasAnyLastAffectedCommits := false + for _, repo := range repos { + if versions.HasLastAffectedCommits(repo) { + hasAnyLastAffectedCommits = true + break + } + } + + if versions.HasLastAffectedVersions() && !hasAnyLastAffectedCommits && !hasAnyFixedCommits { + return fmt.Errorf("[%s]: Failed to convert last_affected version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) + } + } + + slices.SortStableFunc(versions.AffectedCommits, models.AffectedCommitCompare) + + vulns.AttachExtractedVersionInfo(v, versions) + + if len(v.Affected) == 0 { + return fmt.Errorf("[%s]: No affected ranges detected for %q %w", cve.ID, maybeProductName, ErrNoRanges) + } + + vulnDir := filepath.Join(directory, maybeVendorName, maybeProductName) + err := os.MkdirAll(vulnDir, 0755) + if err != nil { + logger.Warn("Failed to create dir", slog.Any("err", err)) + return fmt.Errorf("failed to create dir: %w", err) + } + outputFile := filepath.Join(vulnDir, v.Id+models.Extension) + notesFile := filepath.Join(vulnDir, v.Id+".notes") + f, err := os.Create(outputFile) + if err != nil { + logger.Warn("Failed to open for writing", slog.String("path", outputFile), slog.Any("err", err)) + return fmt.Errorf("failed to open %s for writing: %w", outputFile, err) + } + defer f.Close() + err = v.ToJSON(f) + if err != nil { + logger.Warn("Failed to write", slog.String("path", outputFile), slog.Any("err", err)) + return fmt.Errorf("failed to write %s: %w", outputFile, err) + } + logger.Info("Generated OSV record", slog.String("cve", string(cve.ID)), slog.String("product", maybeProductName)) + if len(notes) > 0 { + err = os.WriteFile(notesFile, []byte(strings.Join(notes, "\n")), 0600) + if err != nil { + logger.Warn("Failed to write", slog.String("cve", string(cve.ID)), slog.String("path", notesFile), slog.Any("err", err)) + } + } + + return nil +} + +// CVEToPackageInfo takes an NVD CVE record and outputs a PackageInfo struct in a file in the specified directory. +func CVEToPackageInfo(cve models.NVDCVE, repos []string, cache git.RepoTagsCache, directory string) error { + CPEs := cves.CPEs(cve) + // The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy. + maybeVendorName := "ENOCPE" + maybeProductName := "ENOCPE" + + if len(CPEs) > 0 { + CPE, err := cves.ParseCPE(CPEs[0]) // For naming the subdirectory used for output. + maybeVendorName = CPE.Vendor + maybeProductName = CPE.Product + if err != nil { + return fmt.Errorf("[%s]: Can't generate an OSV record without valid CPE data", cve.ID) + } + } + + // more often than not, this yields a VersionInfo with AffectedVersions and no AffectedCommits. + versions, notes := cves.ExtractVersionInfo(cve, nil, http.DefaultClient) + + if len(versions.AffectedVersions) != 0 { + var err error + // There are some AffectedVersions to try and resolve to AffectedCommits. + if len(repos) == 0 { + return fmt.Errorf("[%s]: No affected ranges for %q, and no repos to try and convert %+v to tags with", cve.ID, maybeProductName, versions.AffectedVersions) + } + logger.Info("Trying to convert version tags to commits", slog.String("cve", string(cve.ID)), slog.Any("versions", versions), slog.Any("repos", repos)) + versions, err = cves.GitVersionsToCommits(cve.ID, versions, repos, cache) + if err != nil { + return fmt.Errorf("[%s]: Failed to convert version tags to commits: %#w", cve.ID, err) + } + } + + hasAnyFixedCommits := false + for _, repo := range repos { + if versions.HasFixedCommits(repo) { + hasAnyFixedCommits = true + } + } + + if versions.HasFixedVersions() && !hasAnyFixedCommits { + return fmt.Errorf("[%s]: Failed to convert fixed version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) + } + + hasAnyLastAffectedCommits := false + for _, repo := range repos { + if versions.HasLastAffectedCommits(repo) { + hasAnyLastAffectedCommits = true + } + } + + if versions.HasLastAffectedVersions() && !hasAnyLastAffectedCommits && !hasAnyFixedCommits { + return fmt.Errorf("[%s]: Failed to convert last_affected version tags to commits: %#v %w", cve.ID, versions, ErrUnresolvedFix) + } + + if len(versions.AffectedCommits) == 0 { + return fmt.Errorf("[%s]: No affected commit ranges determined for %q %w", cve.ID, maybeProductName, ErrNoRanges) + } + + versions.AffectedVersions = nil // these have served their purpose and are not required in the resulting output. + + slices.SortStableFunc(versions.AffectedCommits, models.AffectedCommitCompare) + + var pkgInfos []vulns.PackageInfo + pi := vulns.PackageInfo{VersionInfo: versions} + pkgInfos = append(pkgInfos, pi) // combine-to-osv expects a serialised *array* of PackageInfo + + vulnDir := filepath.Join(directory, maybeVendorName, maybeProductName) + err := os.MkdirAll(vulnDir, 0755) + if err != nil { + logger.Warn("Failed to create dir", slog.Any("err", err)) + return fmt.Errorf("failed to create dir: %w", err) + } + + outputFile := filepath.Join(vulnDir, string(cve.ID)+".nvd"+models.Extension) + notesFile := filepath.Join(vulnDir, string(cve.ID)+".nvd.notes") + f, err := os.Create(outputFile) + if err != nil { + logger.Warn("Failed to open for writing", slog.String("path", outputFile), slog.Any("err", err)) + return fmt.Errorf("failed to open %s for writing: %w", outputFile, err) + } + defer f.Close() + + encoder := json.NewEncoder(f) + encoder.SetIndent("", " ") + err = encoder.Encode(&pkgInfos) + + if err != nil { + logger.Warn("Failed to encode PackageInfo", slog.String("path", outputFile), slog.Any("err", err)) + return fmt.Errorf("failed to encode PackageInfo to %s: %w", outputFile, err) + } + + logger.Info("Generated PackageInfo record", slog.String("cve", string(cve.ID)), slog.String("product", maybeProductName)) + + if len(notes) > 0 { + err = os.WriteFile(notesFile, []byte(strings.Join(notes, "\n")), 0600) + if err != nil { + logger.Warn("Failed to write", slog.String("cve", string(cve.ID)), slog.String("path", notesFile), slog.Any("err", err)) + } + } + + return nil +} diff --git a/vulnfeeds/cvelist2osv/common.go b/vulnfeeds/cvelist2osv/common.go index da185c331c5..a97ee193c6d 100644 --- a/vulnfeeds/cvelist2osv/common.go +++ b/vulnfeeds/cvelist2osv/common.go @@ -2,7 +2,6 @@ package cvelist2osv import ( "cmp" - "encoding/json" "errors" "log/slog" "strconv" @@ -10,6 +9,7 @@ import ( "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/git" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/google/osv/vulnfeeds/vulns" @@ -28,31 +28,6 @@ const ( VersionRangeTypeEcosystem ) -// VersionSource indicates the source of the extracted version information. -type VersionSource string - -const ( - VersionSourceNone VersionSource = "NOVERS" - VersionSourceAffected VersionSource = "CVEAFFVERS" - VersionSourceGit VersionSource = "GITVERS" - VersionSourceCPE VersionSource = "CPEVERS" - VersionSourceDescription VersionSource = "DESCRVERS" -) - -type ConversionOutcome int - -const ( - // Set of enums for categorizing conversion outcomes. - ConversionUnknown ConversionOutcome = iota // Shouldn't happen - Successful // It worked! - Rejected // The CVE was rejected - NoSoftware // The CVE had no CPEs relating to software (i.e. Operating Systems or Hardware). - NoRepos // The CPE Vendor/Product had no repositories derived for it. - NoCommitRanges // No viable commit ranges could be calculated from the repository for the CVE's CPE(s). - NoRanges // No version ranges could be extracted from the record. - FixUnresolvable // Partial resolution of versions, resulting in a false positive. -) - // String returns the string representation of a VersionRangeType. func (vrt VersionRangeType) String() string { switch vrt { @@ -82,7 +57,7 @@ func toVersionRangeType(s string) VersionRangeType { // resolveVersionToCommit is a helper to convert a version string to a commit hash. // It logs the outcome of the conversion attempt and returns an empty string on failure. -func resolveVersionToCommit(cveID cves.CVEID, version, versionType, repo string, normalizedTags map[string]git.NormalizedTag) string { +func resolveVersionToCommit(cveID models.CVEID, version, versionType, repo string, normalizedTags map[string]git.NormalizedTag) string { if version == "" { return "" } @@ -101,7 +76,7 @@ func resolveVersionToCommit(cveID cves.CVEID, version, versionType, repo string, // Takes a CVE ID string (for logging), VersionInfo with AffectedVersions and // typically no AffectedCommits and attempts to add AffectedCommits (including Fixed commits) where there aren't any. // Refuses to add the same commit to AffectedCommits more than once. -func gitVersionsToCommits(cveID cves.CVEID, versionRanges []*osvschema.Range, repos []string, metrics *ConversionMetrics, cache git.RepoTagsCache) (*osvschema.Affected, error) { +func gitVersionsToCommits(cveID models.CVEID, versionRanges []*osvschema.Range, repos []string, metrics *models.ConversionMetrics, cache git.RepoTagsCache) (*osvschema.Affected, error) { var newAff osvschema.Affected var newVersionRanges []*osvschema.Range unresolvedRanges := versionRanges @@ -193,7 +168,7 @@ func gitVersionsToCommits(cveID cves.CVEID, versionRanges []*osvschema.Range, re // findCPEVersionRanges extracts version ranges and CPE strings from the CNA's // CPE applicability statements in a CVE record. -func findCPEVersionRanges(cve cves.CVE5) (versionRanges []*osvschema.Range, cpes []string, err error) { +func findCPEVersionRanges(cve models.CVE5) (versionRanges []*osvschema.Range, cpes []string, err error) { // TODO(jesslowe): Add logic to also extract CPEs from the 'affected' field (e.g., CVE-2025-1110). for _, c := range cve.Containers.CNA.CPEApplicability { for _, node := range c.Nodes { @@ -271,43 +246,3 @@ func compareSemverLike(a, b string) int { // All extra parts were zero, so the versions are effectively equal. return 0 } - -// addAffected adds an osvschema.Affected to a vulnerability, ensuring that no duplicate ranges are added. -func addAffected(v *vulns.Vulnerability, aff *osvschema.Affected, metrics *ConversionMetrics) { - allExistingRanges := make(map[string]struct{}) - for _, existingAff := range v.Affected { - for _, r := range existingAff.GetRanges() { - rangeBytes, err := json.Marshal(r) - if err == nil { - allExistingRanges[string(rangeBytes)] = struct{}{} - } - } - } - - uniqueRanges := []*osvschema.Range{} - for _, r := range aff.GetRanges() { - rangeBytes, err := json.Marshal(r) - if err != nil { - metrics.AddNote("Could not marshal range to check for duplicates, adding anyway: %+v", r) - uniqueRanges = append(uniqueRanges, r) - - continue - } - rangeStr := string(rangeBytes) - if _, exists := allExistingRanges[rangeStr]; !exists { - uniqueRanges = append(uniqueRanges, r) - allExistingRanges[rangeStr] = struct{}{} - } else { - metrics.AddNote("Skipping duplicate range: %+v", r) - } - } - - if len(uniqueRanges) > 0 { - newAff := &osvschema.Affected{ - Package: aff.GetPackage(), - Ranges: uniqueRanges, - DatabaseSpecific: aff.GetDatabaseSpecific(), - } - v.Affected = append(v.Affected, newAff) - } -} diff --git a/vulnfeeds/cvelist2osv/common_test.go b/vulnfeeds/cvelist2osv/common_test.go index f291da87395..92fed895fe7 100644 --- a/vulnfeeds/cvelist2osv/common_test.go +++ b/vulnfeeds/cvelist2osv/common_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/google/osv/vulnfeeds/conversion" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" @@ -55,9 +57,9 @@ func TestAddAffected(t *testing.T) { }, }, } - metrics := &ConversionMetrics{} + metrics := &models.ConversionMetrics{} - addAffected(v, aff, metrics) + conversion.AddAffected(v, aff, metrics) expectedAffected := []*osvschema.Affected{ { diff --git a/vulnfeeds/cvelist2osv/converter.go b/vulnfeeds/cvelist2osv/converter.go index 3f643659d8c..77b5ca4d5bc 100644 --- a/vulnfeeds/cvelist2osv/converter.go +++ b/vulnfeeds/cvelist2osv/converter.go @@ -6,14 +6,13 @@ import ( "fmt" "io" "log/slog" - "os" - "path/filepath" "slices" "sort" - "strings" "time" + "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/google/osv/vulnfeeds/vulns" @@ -22,48 +21,10 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -const ( - extension = ".json" -) - -// ConversionMetrics holds the collected data about the conversion process for a single CVE. -type ConversionMetrics struct { - CVEID cves.CVEID `json:"id"` // The CVE ID - CNA string `json:"cna"` // The CNA that assigned the CVE. - Outcome ConversionOutcome `json:"outcome"` // The final outcome of the conversion (e.g., "Successful", "Failed"). - Repos []string `json:"repos"` // A list of repositories extracted from the CVE's references. - RefTypesCount map[osvschema.Reference_Type]int `json:"ref_types_count"` // A count of each type of reference found. - VersionSources []VersionSource `json:"version_sources"` // A list of the ways the versions were extracted - Notes []string `json:"notes"` // A collection of notes and warnings generated during conversion. - CPEs []string `json:"cpes"` - UnresolvedRangesCount int `json:"unresolved_ranges_count"` - ResolvedRangesCount int `json:"resolved_ranges_count"` -} - -// AddNote adds a formatted note to the ConversionMetrics. -func (m *ConversionMetrics) AddNote(format string, a ...any) { - m.Notes = append(m.Notes, fmt.Sprintf(format, a...)) - logger.Debug(fmt.Sprintf(format, a...), slog.String("cna", m.CNA), slog.String("cve", string(m.CVEID))) -} - -// AddSource appends a source to the ConversionMetrics -func (m *ConversionMetrics) AddSource(source VersionSource) { - m.VersionSources = append(m.VersionSources, source) -} - -// RefTagDenyList contains reference tags that are often associated with unreliable or -// irrelevant repository URLs. References with these tags are currently ignored -// to avoid incorrect repository associations. -var RefTagDenyList = []string{ - // "Exploit", - // "Third Party Advisory", - "Broken Link", // Actively ignore these. -} - // extractConversionMetrics examines a CVE and its generated OSV references to populate // the ConversionMetrics struct with heuristics about the conversion process. // It captures the assigning CNA and counts the occurrences of each reference type. -func extractConversionMetrics(cve cves.CVE5, refs []*osvschema.Reference, metrics *ConversionMetrics) { +func extractConversionMetrics(cve models.CVE5, refs []*osvschema.Reference, metrics *models.ConversionMetrics) { // Capture the CNA for heuristic analysis. metrics.CNA = cve.Metadata.AssignerShortName // TODO(jesslowe): more CNA based analysis @@ -82,7 +43,7 @@ func extractConversionMetrics(cve cves.CVE5, refs []*osvschema.Reference, metric } // getCWEs extracts and adds CWE IDs from the CVE5 problem-types -func getCWEs(cna cves.CNA, metrics *ConversionMetrics) []string { +func getCWEs(cna models.CNA, metrics *models.ConversionMetrics) []string { var cwes []string for _, pt := range cna.ProblemTypes { @@ -106,30 +67,30 @@ func getCWEs(cna cves.CNA, metrics *ConversionMetrics) []string { return cwes } -// FromCVE5 creates a `vulns.Vulnerability` object from a `cves.CVE5` object. +// FromCVE5 creates a `vulns.Vulnerability` object from a `models.CVE5` object. // It populates the main fields of the OSV record, including ID, summary, details, // references, timestamps, severity, and version information. -func FromCVE5(cve cves.CVE5, refs []cves.Reference, metrics *ConversionMetrics, sourceLink string) *vulns.Vulnerability { +func FromCVE5(cve models.CVE5, refs []models.Reference, metrics *models.ConversionMetrics, sourceLink string) *vulns.Vulnerability { aliases, related := vulns.ExtractReferencedVulns(cve.Metadata.CVEID, cve.Metadata.CVEID, refs) v := vulns.Vulnerability{ Vulnerability: &osvschema.Vulnerability{ SchemaVersion: osvconstants.SchemaVersion, Id: string(cve.Metadata.CVEID), Summary: cve.Containers.CNA.Title, - Details: cves.EnglishDescription(cve.Containers.CNA.Descriptions), + Details: models.EnglishDescription(cve.Containers.CNA.Descriptions), Aliases: aliases, Related: related, References: vulns.ClassifyReferences(refs), }} - published, err := cves.ParseCVE5Timestamp(cve.Metadata.DatePublished) + published, err := models.ParseCVE5Timestamp(cve.Metadata.DatePublished) if err != nil { metrics.AddNote("[%s]: Published date failed to parse, setting time to now", cve.Metadata.CVEID) published = time.Now() } v.Published = timestamppb.New(published) - modified, err := cves.ParseCVE5Timestamp(cve.Metadata.DateUpdated) + modified, err := models.ParseCVE5Timestamp(cve.Metadata.DateUpdated) if err != nil { metrics.AddNote("[%s]: Modified date failed to parse, setting time to now", cve.Metadata.CVEID) modified = time.Now() @@ -137,7 +98,7 @@ func FromCVE5(cve cves.CVE5, refs []cves.Reference, metrics *ConversionMetrics, v.Modified = timestamppb.New(modified) // Try to extract repository URLs from references. - repos, repoNotes := cves.ReposFromReferencesCVEList(string(cve.Metadata.CVEID), refs, RefTagDenyList) + repos, repoNotes := cves.ReposFromReferencesCVEList(string(cve.Metadata.CVEID), refs, models.RefTagDenyList) for _, note := range repoNotes { metrics.AddNote("%s", note) } @@ -165,7 +126,7 @@ func FromCVE5(cve cves.CVE5, refs []cves.Reference, metrics *ConversionMetrics, }) // Combine severity metrics from both CNA and ADP containers. - var severity []cves.Metrics + var severity []models.Metrics if len(cve.Containers.CNA.Metrics) != 0 { severity = append(severity, cve.Containers.CNA.Metrics...) } @@ -183,66 +144,22 @@ func FromCVE5(cve cves.CVE5, refs []cves.Reference, metrics *ConversionMetrics, return &v } -// CreateOSVFile creates the initial file for the OSV record. -func CreateOSVFile(id cves.CVEID, vulnDir string) (*os.File, error) { - outputFile := filepath.Join(vulnDir, string(id)+extension) - - f, err := os.Create(outputFile) - if err != nil { - logger.Info("Failed to open for writing "+outputFile, slog.String("cve", string(id)), slog.String("path", outputFile), slog.Any("err", err)) - return nil, err - } - - return f, err -} - -// CreateMetricsFile saves the collected conversion metrics to a JSON file. -// This file provides data for analyzing the success and characteristics of the -// conversion process for a given CVE. -func CreateMetricsFile(id cves.CVEID, vulnDir string) (*os.File, error) { - metricsFile := filepath.Join(vulnDir, string(id)+".metrics.json") - f, err := os.Create(metricsFile) - if err != nil { - logger.Info("Failed to open for writing "+metricsFile, slog.String("cve", string(id)), slog.String("path", metricsFile), slog.Any("err", err)) - return nil, err - } - - return f, nil -} - -func determineOutcome(metrics *ConversionMetrics) { - // check if we have affected ranges/versions. - if len(metrics.Repos) == 0 { - // Fix unlikely, as no repos to resolve - metrics.Outcome = NoRepos - return - } - - if metrics.ResolvedRangesCount > 0 { - metrics.Outcome = Successful - } else if metrics.UnresolvedRangesCount > 0 { - metrics.Outcome = NoCommitRanges - } else { - metrics.Outcome = NoRanges - } -} - // ConvertAndExportCVEToOSV is the main function for this file. It takes a CVE, // converts it into an OSV record, collects metrics, and writes both to disk. -func ConvertAndExportCVEToOSV(cve cves.CVE5, vulnSink io.Writer, metricsSink io.Writer, sourceLink string) error { +func ConvertAndExportCVEToOSV(cve models.CVE5, vulnSink io.Writer, metricsSink io.Writer, sourceLink string) error { cveID := cve.Metadata.CVEID cnaAssigner := cve.Metadata.AssignerShortName references := identifyPossibleURLs(cve) // Add NVD and computed source link to references - references = append(references, cves.Reference{URL: fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveID)}) + references = append(references, models.Reference{URL: fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveID)}) if sourceLink != "" { - references = append(references, cves.Reference{URL: sourceLink}) + references = append(references, models.Reference{URL: sourceLink}) } - references = deduplicateRefs(references) + references = conversion.DeduplicateRefs(references) - metrics := ConversionMetrics{CVEID: cveID, CNA: cnaAssigner, UnresolvedRangesCount: 0, ResolvedRangesCount: 0} + metrics := models.ConversionMetrics{CVEID: cveID, CNA: cnaAssigner, UnresolvedRangesCount: 0, ResolvedRangesCount: 0} // Create a base OSV record from the CVE. v := FromCVE5(cve, references, &metrics, sourceLink) @@ -254,9 +171,9 @@ func ConvertAndExportCVEToOSV(cve cves.CVE5, vulnSink io.Writer, metricsSink io. versionExtractor := GetVersionExtractor(cve.Metadata.AssignerShortName) versionExtractor.ExtractVersions(cve, v, &metrics, metrics.Repos) - groupAffectedRanges(v.Affected) + conversion.GroupAffectedRanges(v.Affected) - determineOutcome(&metrics) + models.DetermineOutcome(&metrics) err := v.ToJSON(vulnSink) if err != nil { @@ -281,7 +198,7 @@ func ConvertAndExportCVEToOSV(cve cves.CVE5, vulnSink io.Writer, metricsSink io. // identifyPossibleURLs extracts all URLs from a CVE object. // It searches for URLs in the CNA and ADP reference sections, as well as in // the 'collectionUrl' and 'repo' fields of the 'affected' entries. -func identifyPossibleURLs(cve cves.CVE5) []cves.Reference { +func identifyPossibleURLs(cve models.CVE5) []models.Reference { refs := cve.Containers.CNA.References for _, adp := range cve.Containers.ADP { @@ -292,15 +209,15 @@ func identifyPossibleURLs(cve cves.CVE5) []cves.Reference { for _, affected := range cve.Containers.CNA.Affected { if affected.CollectionURL != "" { - refs = append(refs, cves.Reference{URL: affected.CollectionURL}) + refs = append(refs, models.Reference{URL: affected.CollectionURL}) } if affected.Repo != "" { - refs = append(refs, cves.Reference{URL: affected.Repo}) + refs = append(refs, models.Reference{URL: affected.Repo}) } } // Filter out empty URLs from CNA references if any - filteredRefs := make([]cves.Reference, 0, len(refs)) + filteredRefs := make([]models.Reference, 0, len(refs)) for _, ref := range refs { if ref.URL != "" { filteredRefs = append(filteredRefs, ref) @@ -311,19 +228,7 @@ func identifyPossibleURLs(cve cves.CVE5) []cves.Reference { return refs } -func deduplicateRefs(refs []cves.Reference) []cves.Reference { - // Deduplicate references by URL. - slices.SortStableFunc(refs, func(a, b cves.Reference) int { - return strings.Compare(a.URL, b.URL) - }) - refs = slices.CompactFunc(refs, func(a, b cves.Reference) bool { - return a.URL == b.URL - }) - - return refs -} - -func buildDBSpecific(cve cves.CVE5, metrics *ConversionMetrics, sourceLink string) map[string]any { +func buildDBSpecific(cve models.CVE5, metrics *models.ConversionMetrics, sourceLink string) map[string]any { dbSpecific := make(map[string]any) if sourceLink != "" { diff --git a/vulnfeeds/cvelist2osv/converter_test.go b/vulnfeeds/cvelist2osv/converter_test.go index 7ce0523525b..f27d513e0e8 100644 --- a/vulnfeeds/cvelist2osv/converter_test.go +++ b/vulnfeeds/cvelist2osv/converter_test.go @@ -10,9 +10,8 @@ import ( "testing" "github.com/gkampitakis/go-snaps/snaps" - "github.com/google/go-cmp/cmp" - "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvconstants" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -21,7 +20,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -func loadTestData(t *testing.T, cveName string) cves.CVE5 { +func loadTestData(t *testing.T, cveName string) models.CVE5 { t.Helper() prefix := strings.Split(cveName, "-")[2] prefixpath := prefix[:len(prefix)-3] + "xxx" @@ -30,14 +29,14 @@ func loadTestData(t *testing.T, cveName string) cves.CVE5 { return loadTestCVE(t, fileName) } -func loadTestCVE(t *testing.T, path string) cves.CVE5 { +func loadTestCVE(t *testing.T, path string) models.CVE5 { t.Helper() file, err := os.Open(path) if err != nil { t.Fatalf("Failed to load test data from %q: %v", path, err) } defer file.Close() - var cve cves.CVE5 + var cve models.CVE5 err = json.NewDecoder(file).Decode(&cve) if err != nil { t.Fatalf("Failed to decode %q: %+v", path, err) @@ -49,31 +48,31 @@ func loadTestCVE(t *testing.T, path string) cves.CVE5 { func TestIdentifyPossibleURLs(t *testing.T) { testCases := []struct { name string - cve cves.CVE5 - expectedRefs []cves.Reference + cve models.CVE5 + expectedRefs []models.Reference }{ { name: "simple case with duplicates", - cve: cves.CVE5{ + cve: models.CVE5{ Containers: struct { - CNA cves.CNA `json:"cna"` - ADP []cves.CNA `json:"adp,omitempty"` + CNA models.CNA `json:"cna"` + ADP []models.CNA `json:"adp,omitempty"` }{ - CNA: cves.CNA{ - References: []cves.Reference{ + CNA: models.CNA{ + References: []models.Reference{ {URL: "http://a.com"}, {URL: "http://b.com"}, }, - Affected: []cves.Affected{ + Affected: []models.Affected{ { CollectionURL: "http://d.com", Repo: "http://b.com", }, }, }, - ADP: []cves.CNA{ + ADP: []models.CNA{ { - References: []cves.Reference{ + References: []models.Reference{ {URL: "http://c.com"}, {URL: "http://a.com"}, }, @@ -81,7 +80,7 @@ func TestIdentifyPossibleURLs(t *testing.T) { }, }, }, - expectedRefs: []cves.Reference{ + expectedRefs: []models.Reference{ {URL: "http://a.com"}, {URL: "http://b.com"}, {URL: "http://c.com"}, @@ -92,53 +91,53 @@ func TestIdentifyPossibleURLs(t *testing.T) { }, { name: "no references and CNA refs is nil", - cve: cves.CVE5{ + cve: models.CVE5{ Containers: struct { - CNA cves.CNA `json:"cna"` - ADP []cves.CNA `json:"adp,omitempty"` + CNA models.CNA `json:"cna"` + ADP []models.CNA `json:"adp,omitempty"` }{ - CNA: cves.CNA{ + CNA: models.CNA{ References: nil, }, }, }, - expectedRefs: []cves.Reference{}, + expectedRefs: []models.Reference{}, }, { name: "no references and CNA refs is empty slice", - cve: cves.CVE5{ + cve: models.CVE5{ Containers: struct { - CNA cves.CNA `json:"cna"` - ADP []cves.CNA `json:"adp,omitempty"` + CNA models.CNA `json:"cna"` + ADP []models.CNA `json:"adp,omitempty"` }{ - CNA: cves.CNA{ - References: []cves.Reference{}, + CNA: models.CNA{ + References: []models.Reference{}, }, }, }, - expectedRefs: []cves.Reference{}, + expectedRefs: []models.Reference{}, }, { name: "empty url string", - cve: cves.CVE5{ + cve: models.CVE5{ Containers: struct { - CNA cves.CNA `json:"cna"` - ADP []cves.CNA `json:"adp,omitempty"` + CNA models.CNA `json:"cna"` + ADP []models.CNA `json:"adp,omitempty"` }{ - CNA: cves.CNA{ - Affected: []cves.Affected{ + CNA: models.CNA{ + Affected: []models.Affected{ { CollectionURL: "", }, }, - References: []cves.Reference{ + References: []models.Reference{ {URL: "http://a.com"}, {URL: ""}, }, }, }, }, - expectedRefs: []cves.Reference{ + expectedRefs: []models.Reference{ {URL: "http://a.com"}, }, }, @@ -155,36 +154,36 @@ func TestIdentifyPossibleURLs(t *testing.T) { } func TestFromCVE5(t *testing.T) { - cve1110Pub, _ := cves.ParseCVE5Timestamp("2025-05-22T14:02:31.385Z") - cve1110Mod, _ := cves.ParseCVE5Timestamp("2025-05-22T14:17:44.379Z") - cve21634Pub, _ := cves.ParseCVE5Timestamp("2024-01-03T22:46:03.585Z") - cve21634Mod, _ := cves.ParseCVE5Timestamp("2025-06-16T19:45:37.088Z") - cve21772Pub, _ := cves.ParseCVE5Timestamp("2025-02-27T02:18:19.528Z") - cve21772Mod, _ := cves.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") - cvePlaceholder, _ := cves.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") + cve1110Pub, _ := models.ParseCVE5Timestamp("2025-05-22T14:02:31.385Z") + cve1110Mod, _ := models.ParseCVE5Timestamp("2025-05-22T14:17:44.379Z") + cve21634Pub, _ := models.ParseCVE5Timestamp("2024-01-03T22:46:03.585Z") + cve21634Mod, _ := models.ParseCVE5Timestamp("2025-06-16T19:45:37.088Z") + cve21772Pub, _ := models.ParseCVE5Timestamp("2025-02-27T02:18:19.528Z") + cve21772Mod, _ := models.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") + cvePlaceholder, _ := models.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") testCases := []struct { name string - cve cves.CVE5 + cve models.CVE5 - refs []cves.Reference + refs []models.Reference expectedVuln *vulns.Vulnerability }{ { name: "disputed record", - cve: cves.CVE5{ - Metadata: cves.CVE5Metadata{ + cve: models.CVE5{ + Metadata: models.CVE5Metadata{ CVEID: "CVE-2025-9999", State: "PUBLISHED", DatePublished: "2025-05-04T07:20:46.575Z", DateUpdated: "2025-05-04T07:20:46.575Z", }, Containers: struct { - CNA cves.CNA `json:"cna"` - ADP []cves.CNA `json:"adp,omitempty"` + CNA models.CNA `json:"cna"` + ADP []models.CNA `json:"adp,omitempty"` }{ - CNA: cves.CNA{ + CNA: models.CNA{ Tags: []string{"disputed"}, - Descriptions: []cves.LangString{ + Descriptions: []models.LangString{ { Lang: "en", Value: "A disputed vulnerability.", @@ -193,7 +192,7 @@ func TestFromCVE5(t *testing.T) { }, }, }, - refs: []cves.Reference{}, + refs: []models.Reference{}, expectedVuln: &vulns.Vulnerability{ Vulnerability: &osvschema.Vulnerability{ Id: "CVE-2025-9999", @@ -213,7 +212,7 @@ func TestFromCVE5(t *testing.T) { { name: "CVE-2025-1110", cve: loadTestData(t, "CVE-2025-1110"), - refs: []cves.Reference{ + refs: []models.Reference{ {URL: "https://gitlab.com/gitlab-org/gitlab/-/issues/517693", Tags: []string{"issue-tracking", "permissions-required"}}, {URL: "https://hackerone.com/reports/2972576", Tags: []string{"technical-description", "exploit", "permissions-required"}}, }, @@ -256,7 +255,7 @@ func TestFromCVE5(t *testing.T) { { name: "CVE-2024-21634", cve: loadTestData(t, "CVE-2024-21634"), - refs: []cves.Reference{ + refs: []models.Reference{ {Tags: []string{"x_refsource_CONFIRM"}, URL: "https://github.com/amazon-ion/ion-java/security/advisories/GHSA-264p-99wq-f4j6"}, }, expectedVuln: &vulns.Vulnerability{ @@ -297,7 +296,7 @@ func TestFromCVE5(t *testing.T) { { name: "CVE-2025-21772", cve: loadTestData(t, "CVE-2025-21772"), - refs: []cves.Reference{ + refs: []models.Reference{ {URL: "https://git.kernel.org/stable/c/a3e77da9f843e4ab93917d30c314f0283e28c124"}, {URL: "https://git.kernel.org/stable/c/213ba5bd81b7e97ac6e6190b8f3bc6ba76123625"}, {URL: "https://git.kernel.org/stable/c/40a35d14f3c0dc72b689061ec72fc9b193f37d1f"}, @@ -340,7 +339,7 @@ func TestFromCVE5(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - metrics := &ConversionMetrics{} + metrics := &models.ConversionMetrics{} vuln := FromCVE5(tc.cve, tc.refs, metrics, "") // Handle non-deterministic time.Now() @@ -392,24 +391,24 @@ func TestFromCVE5(t *testing.T) { } func TestConvertAndExportCVEToOSV(t *testing.T) { - cve1110Pub, _ := cves.ParseCVE5Timestamp("2025-05-22T14:02:31.385Z") - cve1110Mod, _ := cves.ParseCVE5Timestamp("2025-05-22T14:17:44.379Z") - cve21634Pub, _ := cves.ParseCVE5Timestamp("2024-01-03T22:46:03.585Z") - cve21634Mod, _ := cves.ParseCVE5Timestamp("2025-06-16T19:45:37.088Z") - cve21772Pub, _ := cves.ParseCVE5Timestamp("2025-02-27T02:18:19.528Z") - cve21772Mod, _ := cves.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") - cvePlaceholder, _ := cves.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") + cve1110Pub, _ := models.ParseCVE5Timestamp("2025-05-22T14:02:31.385Z") + cve1110Mod, _ := models.ParseCVE5Timestamp("2025-05-22T14:17:44.379Z") + cve21634Pub, _ := models.ParseCVE5Timestamp("2024-01-03T22:46:03.585Z") + cve21634Mod, _ := models.ParseCVE5Timestamp("2025-06-16T19:45:37.088Z") + cve21772Pub, _ := models.ParseCVE5Timestamp("2025-02-27T02:18:19.528Z") + cve21772Mod, _ := models.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") + cvePlaceholder, _ := models.ParseCVE5Timestamp("2025-05-04T07:20:46.575Z") testCases := []struct { name string - cve cves.CVE5 + cve models.CVE5 - refs []cves.Reference + refs []models.Reference expectedVuln *vulns.Vulnerability }{ { name: "disputed record", - cve: cves.CVE5{ - Metadata: cves.CVE5Metadata{ + cve: models.CVE5{ + Metadata: models.CVE5Metadata{ CVEID: "CVE-2025-9999", State: "PUBLISHED", DatePublished: "2025-05-04T07:20:46.575Z", @@ -417,12 +416,12 @@ func TestConvertAndExportCVEToOSV(t *testing.T) { AssignerShortName: "unknown", }, Containers: struct { - CNA cves.CNA `json:"cna"` - ADP []cves.CNA `json:"adp,omitempty"` + CNA models.CNA `json:"cna"` + ADP []models.CNA `json:"adp,omitempty"` }{ - CNA: cves.CNA{ + CNA: models.CNA{ Tags: []string{"disputed"}, - Descriptions: []cves.LangString{ + Descriptions: []models.LangString{ { Lang: "en", Value: "A disputed vulnerability.", @@ -431,7 +430,7 @@ func TestConvertAndExportCVEToOSV(t *testing.T) { }, }, }, - refs: []cves.Reference{}, + refs: []models.Reference{}, expectedVuln: &vulns.Vulnerability{ Vulnerability: &osvschema.Vulnerability{ Id: "CVE-2025-9999", @@ -452,7 +451,7 @@ func TestConvertAndExportCVEToOSV(t *testing.T) { { name: "CVE-2025-1110", cve: loadTestData(t, "CVE-2025-1110"), - refs: []cves.Reference{ + refs: []models.Reference{ {URL: "https://gitlab.com/gitlab-org/gitlab/-/issues/517693", Tags: []string{"issue-tracking", "permissions-required"}}, {URL: "https://hackerone.com/reports/2972576", Tags: []string{"technical-description", "exploit", "permissions-required"}}, }, @@ -497,7 +496,7 @@ func TestConvertAndExportCVEToOSV(t *testing.T) { { name: "CVE-2024-21634", cve: loadTestData(t, "CVE-2024-21634"), - refs: []cves.Reference{ + refs: []models.Reference{ {Tags: []string{"x_refsource_CONFIRM"}, URL: "https://github.com/amazon-ion/ion-java/security/advisories/GHSA-264p-99wq-f4j6"}, }, expectedVuln: &vulns.Vulnerability{ @@ -538,7 +537,7 @@ func TestConvertAndExportCVEToOSV(t *testing.T) { { name: "CVE-2025-21772", cve: loadTestData(t, "CVE-2025-21772"), - refs: []cves.Reference{ + refs: []models.Reference{ {URL: "https://git.kernel.org/stable/c/a3e77da9f843e4ab93917d30c314f0283e28c124"}, {URL: "https://git.kernel.org/stable/c/213ba5bd81b7e97ac6e6190b8f3bc6ba76123625"}, {URL: "https://git.kernel.org/stable/c/40a35d14f3c0dc72b689061ec72fc9b193f37d1f"}, diff --git a/vulnfeeds/cvelist2osv/default_extractor.go b/vulnfeeds/cvelist2osv/default_extractor.go index 8e497c7c175..652dd215bba 100644 --- a/vulnfeeds/cvelist2osv/default_extractor.go +++ b/vulnfeeds/cvelist2osv/default_extractor.go @@ -4,8 +4,10 @@ import ( "fmt" "log/slog" + "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/git" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -14,7 +16,7 @@ import ( // DefaultVersionExtractor provides the default version extraction logic. type DefaultVersionExtractor struct{} -func (d *DefaultVersionExtractor) handleAffected(affected []cves.Affected, metrics *ConversionMetrics) []*osvschema.Range { +func (d *DefaultVersionExtractor) handleAffected(affected []models.Affected, metrics *models.ConversionMetrics) []*osvschema.Range { var ranges []*osvschema.Range for _, cveAff := range affected { versionRanges, _ := d.FindNormalAffectedRanges(cveAff, metrics) @@ -23,14 +25,14 @@ func (d *DefaultVersionExtractor) handleAffected(affected []cves.Affected, metri continue } ranges = append(ranges, versionRanges...) - metrics.AddSource(VersionSourceAffected) + metrics.AddSource(models.VersionSourceAffected) } return ranges } // ExtractVersions for DefaultVersionExtractor. -func (d *DefaultVersionExtractor) ExtractVersions(cve cves.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, repos []string) { +func (d *DefaultVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *models.ConversionMetrics, repos []string) { gotVersions := false repoTagsCache := git.RepoTagsCache{} @@ -44,7 +46,7 @@ func (d *DefaultVersionExtractor) ExtractVersions(cve cves.CVE5, v *vulns.Vulner } else { gotVersions = true } - addAffected(v, aff, metrics) + conversion.AddAffected(v, aff, metrics) } if !gotVersions { @@ -59,7 +61,7 @@ func (d *DefaultVersionExtractor) ExtractVersions(cve cves.CVE5, v *vulns.Vulner gotVersions = true } - addAffected(v, aff, metrics) + conversion.AddAffected(v, aff, metrics) } } @@ -71,12 +73,12 @@ func (d *DefaultVersionExtractor) ExtractVersions(cve cves.CVE5, v *vulns.Vulner if err != nil { logger.Error("Failed to convert git versions to commits", slog.Any("err", err)) } - addAffected(v, aff, metrics) + conversion.AddAffected(v, aff, metrics) } } } -func (d *DefaultVersionExtractor) FindNormalAffectedRanges(affected cves.Affected, metrics *ConversionMetrics) ([]*osvschema.Range, VersionRangeType) { +func (d *DefaultVersionExtractor) FindNormalAffectedRanges(affected models.Affected, metrics *models.ConversionMetrics) ([]*osvschema.Range, VersionRangeType) { versionTypesCount := make(map[VersionRangeType]int) var versionRanges []*osvschema.Range for _, vers := range affected.Versions { diff --git a/vulnfeeds/cvelist2osv/extraction.go b/vulnfeeds/cvelist2osv/extraction.go index fddc1dce961..88387753d95 100644 --- a/vulnfeeds/cvelist2osv/extraction.go +++ b/vulnfeeds/cvelist2osv/extraction.go @@ -1,15 +1,15 @@ package cvelist2osv import ( - "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" ) // VersionExtractor defines the interface for different version extraction strategies. type VersionExtractor interface { - ExtractVersions(cve cves.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, repos []string) - FindNormalAffectedRanges(affected cves.Affected, metrics *ConversionMetrics) ([]*osvschema.Range, VersionRangeType) + ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *models.ConversionMetrics, repos []string) + FindNormalAffectedRanges(affected models.Affected, metrics *models.ConversionMetrics) ([]*osvschema.Range, VersionRangeType) } // GetVersionExtractor returns the appropriate VersionExtractor for a given CNA. diff --git a/vulnfeeds/cvelist2osv/linux_extractor.go b/vulnfeeds/cvelist2osv/linux_extractor.go index 266dd9be252..4f2708f9077 100644 --- a/vulnfeeds/cvelist2osv/linux_extractor.go +++ b/vulnfeeds/cvelist2osv/linux_extractor.go @@ -6,7 +6,9 @@ import ( "strconv" "strings" + "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvconstants" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -19,8 +21,8 @@ type LinuxVersionExtractor struct { var _ VersionExtractor = &LinuxVersionExtractor{} -// handleAffected takes an array of cves.Affected and handles how to extract them -func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected []cves.Affected, metrics *ConversionMetrics) bool { +// handleAffected takes an array of models.Affected and handles how to extract them +func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected []models.Affected, metrics *models.ConversionMetrics) bool { hasGit := false gotVersions := false for _, cveAff := range affected { @@ -41,15 +43,15 @@ func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected hasGit = true } aff := createLinuxAffected(versionRanges, versionType, cveAff.Repo) - metrics.AddSource(VersionSourceAffected) - addAffected(v, aff, metrics) + metrics.AddSource(models.VersionSourceAffected) + conversion.AddAffected(v, aff, metrics) } return gotVersions } // ExtractVersions for LinuxVersionExtractor. -func (l *LinuxVersionExtractor) ExtractVersions(cve cves.CVE5, v *vulns.Vulnerability, metrics *ConversionMetrics, _ []string) { +func (l *LinuxVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *models.ConversionMetrics, _ []string) { gotVersions := l.handleAffected(v, cve.Containers.CNA.Affected, metrics) if !gotVersions { @@ -88,7 +90,7 @@ func createLinuxAffected(versionRanges []*osvschema.Range, versionType VersionRa // of 'unaffected' versions. This is common in Linux kernel CVEs where a product is // considered affected by default, and only unaffected versions are listed. // It sorts the introduced and fixed versions to create chronological ranges. -func findInverseAffectedRanges(cveAff cves.Affected, metrics *ConversionMetrics) (ranges []*osvschema.Range, versType VersionRangeType) { +func findInverseAffectedRanges(cveAff models.Affected, metrics *models.ConversionMetrics) (ranges []*osvschema.Range, versType VersionRangeType) { var introduced []string fixed := make([]string, 0, len(cveAff.Versions)) for _, vers := range cveAff.Versions { @@ -149,7 +151,7 @@ func findInverseAffectedRanges(cveAff cves.Affected, metrics *ConversionMetrics) return nil, VersionRangeTypeUnknown } -func (l *LinuxVersionExtractor) FindNormalAffectedRanges(affected cves.Affected, metrics *ConversionMetrics) ([]*osvschema.Range, VersionRangeType) { +func (l *LinuxVersionExtractor) FindNormalAffectedRanges(affected models.Affected, metrics *models.ConversionMetrics) ([]*osvschema.Range, VersionRangeType) { versionTypesCount := make(map[VersionRangeType]int) var versionRanges []*osvschema.Range for _, vers := range affected.Versions { diff --git a/vulnfeeds/cvelist2osv/strategies.go b/vulnfeeds/cvelist2osv/strategies.go index f6f25b9b801..20cc1ae60be 100644 --- a/vulnfeeds/cvelist2osv/strategies.go +++ b/vulnfeeds/cvelist2osv/strategies.go @@ -2,14 +2,15 @@ package cvelist2osv import ( "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" ) -func cpeVersionExtraction(cve cves.CVE5, metrics *ConversionMetrics) ([]*osvschema.Range, error) { +func cpeVersionExtraction(cve models.CVE5, metrics *models.ConversionMetrics) ([]*osvschema.Range, error) { cpeRanges, cpeStrings, err := findCPEVersionRanges(cve) if err == nil && len(cpeRanges) > 0 { - metrics.VersionSources = append(metrics.VersionSources, VersionSourceCPE) + metrics.VersionSources = append(metrics.VersionSources, models.VersionSourceCPE) metrics.CPEs = vulns.Unique(cpeStrings) return cpeRanges, nil @@ -21,15 +22,15 @@ func cpeVersionExtraction(cve cves.CVE5, metrics *ConversionMetrics) ([]*osvsche } // textVersionExtraction is a helper function for CPE and description extraction. -func textVersionExtraction(cve cves.CVE5, metrics *ConversionMetrics) []*osvschema.Range { +func textVersionExtraction(cve models.CVE5, metrics *models.ConversionMetrics) []*osvschema.Range { // As a last resort, try extracting versions from the description text. - versions, extractNotes := cves.ExtractVersionsFromText(nil, cves.EnglishDescription(cve.Containers.CNA.Descriptions)) + versions, extractNotes := cves.ExtractVersionsFromText(nil, models.EnglishDescription(cve.Containers.CNA.Descriptions)) for _, note := range extractNotes { metrics.AddNote("%s", note) } if len(versions) > 0 { // NOTE: These versions are not currently saved due to the need for better validation. - metrics.VersionSources = append(metrics.VersionSources, VersionSourceDescription) + metrics.VersionSources = append(metrics.VersionSources, models.VersionSourceDescription) metrics.AddNote("Extracted versions from description but did not save them: %+v", versions) } @@ -37,7 +38,7 @@ func textVersionExtraction(cve cves.CVE5, metrics *ConversionMetrics) []*osvsche } // initialNormalExtraction handles an expected case of version ranges in the affected field of CVE5 -func initialNormalExtraction(vers cves.Versions, metrics *ConversionMetrics, versionTypesCount map[VersionRangeType]int) ([]*osvschema.Range, VersionRangeType, bool) { +func initialNormalExtraction(vers models.Versions, metrics *models.ConversionMetrics, versionTypesCount map[VersionRangeType]int) ([]*osvschema.Range, VersionRangeType, bool) { if vers.Status != "affected" { return nil, VersionRangeTypeUnknown, true } diff --git a/vulnfeeds/cvelist2osv/version_extraction_test.go b/vulnfeeds/cvelist2osv/version_extraction_test.go index a1e87c8a4de..e1080681a0e 100644 --- a/vulnfeeds/cvelist2osv/version_extraction_test.go +++ b/vulnfeeds/cvelist2osv/version_extraction_test.go @@ -7,6 +7,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" @@ -39,15 +40,15 @@ func TestToVersionRangeType(t *testing.T) { func TestFindNormalAffectedRanges(t *testing.T) { tests := []struct { name string - affected cves.Affected + affected models.Affected cnaAssigner string wantRanges []*osvschema.Range wantRangeType VersionRangeType }{ { name: "simple range", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "affected", Version: "1.0", @@ -63,8 +64,8 @@ func TestFindNormalAffectedRanges(t *testing.T) { }, { name: "single version fallback", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "affected", Version: "2.0", @@ -79,8 +80,8 @@ func TestFindNormalAffectedRanges(t *testing.T) { }, { name: "github range", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "affected", Version: ">= 2.0, < 2.5", @@ -94,8 +95,8 @@ func TestFindNormalAffectedRanges(t *testing.T) { }, { name: "git commit", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "affected", Version: "deadbeef", @@ -113,7 +114,7 @@ func TestFindNormalAffectedRanges(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { versionExtractor := &DefaultVersionExtractor{} - gotRanges, gotRangeType := versionExtractor.FindNormalAffectedRanges(tt.affected, &ConversionMetrics{}) + gotRanges, gotRangeType := versionExtractor.FindNormalAffectedRanges(tt.affected, &models.ConversionMetrics{}) if diff := cmp.Diff(tt.wantRanges, gotRanges, protocmp.Transform()); diff != "" { t.Errorf("findNormalAffectedRanges() ranges mismatch (-want +got):\n%s", diff) } @@ -149,15 +150,15 @@ func TestCompareSemverLike(t *testing.T) { func TestFindInverseAffectedRanges(t *testing.T) { tests := []struct { name string - affected cves.Affected + affected models.Affected versionType VersionRangeType cnaAssigner string want []*osvschema.Range }{ { name: "linux with wildcard", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "affected", Version: "5.0", @@ -179,8 +180,8 @@ func TestFindInverseAffectedRanges(t *testing.T) { }, { name: "not linux", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "unaffected", Version: "1.0", @@ -195,8 +196,8 @@ func TestFindInverseAffectedRanges(t *testing.T) { }, { name: "linux no wildcard", - affected: cves.Affected{ - Versions: []cves.Versions{ + affected: models.Affected{ + Versions: []models.Versions{ { Status: "affected", Version: "4.0", @@ -220,7 +221,7 @@ func TestFindInverseAffectedRanges(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - metrics := &ConversionMetrics{} + metrics := &models.ConversionMetrics{} gotRanges, gotVersionType := findInverseAffectedRanges(tt.affected, metrics) if diff := cmp.Diff(tt.want, gotRanges, protocmp.Transform()); diff != "" { t.Errorf("findInverseAffectedRanges() ranges mismatch (-want +got):\n%s", diff) @@ -235,7 +236,7 @@ func TestFindInverseAffectedRanges(t *testing.T) { func TestRealWorldFindInverseAffectedRanges(t *testing.T) { testCases := []struct { name string - cve cves.CVE5 + cve models.CVE5 expectedRanges []*osvschema.Range }{ { @@ -265,7 +266,7 @@ func TestRealWorldFindInverseAffectedRanges(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - var affectedBlock cves.Affected + var affectedBlock models.Affected // Find the specific affected block with defaultStatus: "affected". for _, affected := range tc.cve.Containers.CNA.Affected { if affected.DefaultStatus == "affected" { @@ -279,7 +280,7 @@ func TestRealWorldFindInverseAffectedRanges(t *testing.T) { } // Run the function under test. - gotRanges, _ := findInverseAffectedRanges(affectedBlock, &ConversionMetrics{}) + gotRanges, _ := findInverseAffectedRanges(affectedBlock, &models.ConversionMetrics{}) // Sort slices for deterministic comparison. sort.Slice(gotRanges, func(i, j int) bool { @@ -324,13 +325,13 @@ func TestRealWorldFindInverseAffectedRanges(t *testing.T) { func TestGetVersionExtractor(t *testing.T) { testCases := []struct { name string - cve cves.CVE5 + cve models.CVE5 expectedType reflect.Type }{ { name: "Linux CVE", - cve: cves.CVE5{ - Metadata: cves.CVE5Metadata{ + cve: models.CVE5{ + Metadata: models.CVE5Metadata{ AssignerShortName: "Linux", }, }, @@ -338,8 +339,8 @@ func TestGetVersionExtractor(t *testing.T) { }, { name: "Default CVE", - cve: cves.CVE5{ - Metadata: cves.CVE5Metadata{ + cve: models.CVE5{ + Metadata: models.CVE5Metadata{ AssignerShortName: "Anything", }, }, @@ -347,7 +348,7 @@ func TestGetVersionExtractor(t *testing.T) { }, { name: "Empty provider", - cve: cves.CVE5{}, + cve: models.CVE5{}, expectedType: reflect.TypeOf(&DefaultVersionExtractor{}), }, } @@ -365,7 +366,7 @@ func TestGetVersionExtractor(t *testing.T) { func TestExtractVersions(t *testing.T) { testCases := []struct { name string - cve cves.CVE5 + cve models.CVE5 cnaAssigner string repos []string expectedAffected []*osvschema.Affected @@ -550,7 +551,7 @@ func TestExtractVersions(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - metrics := &ConversionMetrics{} + metrics := &models.ConversionMetrics{} v := vulns.Vulnerability{ Vulnerability: &osvschema.Vulnerability{}, } diff --git a/vulnfeeds/cves/versions.go b/vulnfeeds/cves/versions.go index 38e80babe55..7923e2e52fe 100644 --- a/vulnfeeds/cves/versions.go +++ b/vulnfeeds/cves/versions.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// https://www.apache.org/licenses/LICENSE-2.0 +// https://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package cves provides utilities for working with CVEs and version information. package cves import ( @@ -683,7 +684,7 @@ func deduplicateAffectedCommits(commits []models.AffectedCommit) []models.Affect return uniqueCommits } -func ExtractVersionInfo(cve CVE, validVersions []string, httpClient *http.Client) (v models.VersionInfo, notes []string) { +func ExtractVersionInfo(cve models.NVDCVE, validVersions []string, httpClient *http.Client) (v models.VersionInfo, notes []string) { for _, reference := range cve.References { // (Potentially faulty) Assumption: All viable Git commit reference links are fix commits. if commit, err := extractGitAffectedCommit(reference.URL, models.Fixed, httpClient); err == nil { @@ -782,7 +783,7 @@ func ExtractVersionInfo(cve CVE, validVersions []string, httpClient *http.Client } if !gotVersions { var extractNotes []string - v.AffectedVersions, extractNotes = ExtractVersionsFromText(validVersions, EnglishDescription(cve.Descriptions)) + v.AffectedVersions, extractNotes = ExtractVersionsFromText(validVersions, models.EnglishDescription(cve.Descriptions)) notes = append(notes, extractNotes...) if len(v.AffectedVersions) > 0 { logger.Info("Extracted versions from description", slog.String("cve", string(cve.ID)), slog.Any("versions", v.AffectedVersions)) @@ -815,7 +816,7 @@ func ExtractVersionInfo(cve CVE, validVersions []string, httpClient *http.Client return v, notes } -func CPEs(cve CVE) []string { +func CPEs(cve models.NVDCVE) []string { var cpes []string for _, config := range cve.Configurations { for _, node := range config.Nodes { @@ -870,7 +871,7 @@ func (vp *VendorProduct) UnmarshalText(text []byte) error { return nil } -func RefAcceptable(ref Reference, tagDenyList []string) bool { +func RefAcceptable(ref models.Reference, tagDenyList []string) bool { for _, deniedTag := range tagDenyList { if slices.Contains(ref.Tags, deniedTag) { return false @@ -923,7 +924,7 @@ func MaybeRemoveFromVPRepoCache(cache VendorProductToRepoMap, vp *VendorProduct, // Takes a CVE ID string (for logging), VersionInfo with AffectedVersions and // typically no AffectedCommits and attempts to add AffectedCommits (including Fixed commits) where there aren't any. // Refuses to add the same commit to AffectedCommits more than once. -func GitVersionsToCommits(cveID CVEID, versions models.VersionInfo, repos []string, cache git.RepoTagsCache) (v models.VersionInfo, e error) { +func GitVersionsToCommits(cveID models.CVEID, versions models.VersionInfo, repos []string, cache git.RepoTagsCache) (v models.VersionInfo, e error) { // versions is a VersionInfo with AffectedVersions and typically no AffectedCommits // v is a VersionInfo with AffectedCommits (containing Fixed commits) included v = versions @@ -1009,7 +1010,7 @@ func GitVersionsToCommits(cveID CVEID, versions models.VersionInfo, repos []stri // Examines the CVE references for a CVE and derives repos for it, optionally caching it. // TODO (jesslowe): refactor with below -func ReposFromReferences(cve string, cache VendorProductToRepoMap, vp *VendorProduct, refs []Reference, tagDenyList []string) (repos []string) { +func ReposFromReferences(cve string, cache VendorProductToRepoMap, vp *VendorProduct, refs []models.Reference, tagDenyList []string) (repos []string) { for _, ref := range refs { // If any of the denylist tags are in the ref's tag set, it's out of consideration. if !RefAcceptable(ref, tagDenyList) { @@ -1046,7 +1047,7 @@ func ReposFromReferences(cve string, cache VendorProductToRepoMap, vp *VendorPro } // Examines the CVE references for a CVE and derives repos for it, optionally caching it. -func ReposFromReferencesCVEList(cve string, refs []Reference, tagDenyList []string) (repos []string, notes []string) { +func ReposFromReferencesCVEList(cve string, refs []models.Reference, tagDenyList []string) (repos []string, notes []string) { for _, ref := range refs { // If any of the denylist tags are in the ref's tag set, it's out of consideration. if !RefAcceptable(ref, tagDenyList) { diff --git a/vulnfeeds/cves/versions_test.go b/vulnfeeds/cves/versions_test.go index 09852d7716d..0cd6b30055e 100644 --- a/vulnfeeds/cves/versions_test.go +++ b/vulnfeeds/cves/versions_test.go @@ -6,11 +6,10 @@ import ( "log" "os" "reflect" + "slices" "testing" "time" - "slices" - "github.com/google/go-cmp/cmp" "github.com/google/osv/vulnfeeds/internal/testutils" "github.com/google/osv/vulnfeeds/models" @@ -18,13 +17,13 @@ import ( "google.golang.org/protobuf/testing/protocmp" ) -func loadTestData2(cveName string) Vulnerability { +func loadTestData2(cveName string) models.Vulnerability { fileName := fmt.Sprintf("../test_data/nvdcve-2.0/%s.json", cveName) file, err := os.Open(fileName) if err != nil { log.Fatalf("Failed to load test data from %q", fileName) } - var nvdCves CVEAPIJSON20Schema + var nvdCves models.CVEAPIJSON20Schema err = json.NewDecoder(file).Decode(&nvdCves) if err != nil { log.Fatalf("Failed to decode %q: %+v", fileName, err) @@ -36,7 +35,7 @@ func loadTestData2(cveName string) Vulnerability { } log.Fatalf("test data doesn't contain %q", cveName) - return Vulnerability{} + return models.Vulnerability{} } func TestParseCPE(t *testing.T) { @@ -710,7 +709,7 @@ func TestExtractGitCommit(t *testing.T) { func TestExtractVersionInfo(t *testing.T) { tests := []struct { description string - inputCVEItem Vulnerability + inputCVEItem models.Vulnerability inputValidVersions []string expectedVersionInfo models.VersionInfo expectedNotes []string @@ -930,7 +929,7 @@ func TestExtractVersionInfo(t *testing.T) { func TestCPEs(t *testing.T) { tests := []struct { description string - inputCVEItem Vulnerability + inputCVEItem models.Vulnerability expectedCPEs []string }{ { @@ -1286,7 +1285,7 @@ func TestReposFromReferences(t *testing.T) { CVE string cache VendorProductToRepoMap vp *VendorProduct - refs []Reference + refs []models.Reference tagDenyList []string } tests := []struct { @@ -1300,7 +1299,7 @@ func TestReposFromReferences(t *testing.T) { CVE: "CVE-2023-0327", cache: nil, vp: &VendorProduct{"theradsystem_project", "theradsystem"}, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@vuldb.com", Tags: []string{"Patch", "Third Party Advisory"}, @@ -1316,7 +1315,7 @@ func TestReposFromReferences(t *testing.T) { CVE: "CVE-2025-0211", cache: nil, vp: &VendorProduct{"campcodes", "school_faculty_scheduling_system"}, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@vuldb.com", Tags: []string{"Exploit", "Third Party Advisory"}, @@ -1332,7 +1331,7 @@ func TestReposFromReferences(t *testing.T) { CVE: "CVE-2025-26519", cache: nil, vp: nil, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@mitre.org", Tags: nil, @@ -1349,7 +1348,7 @@ func TestReposFromReferences(t *testing.T) { CVE: "CVE-2016-10525", cache: nil, vp: nil, - refs: []Reference{ + refs: []models.Reference{ { Source: "support@hackerone.com", Tags: []string{"Patch", "Third Party Advisory"}, @@ -1376,7 +1375,7 @@ func TestReposFromReferencesCVEList(t *testing.T) { CVE string cache VendorProductToRepoMap vp *VendorProduct - refs []Reference + refs []models.Reference tagDenyList []string } tests := []struct { @@ -1390,7 +1389,7 @@ func TestReposFromReferencesCVEList(t *testing.T) { CVE: "CVE-2023-0327", cache: nil, vp: &VendorProduct{"theradsystem_project", "theradsystem"}, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@vuldb.com", Tags: []string{"Patch", "Third Party Advisory"}, @@ -1406,7 +1405,7 @@ func TestReposFromReferencesCVEList(t *testing.T) { CVE: "CVE-2025-0211", cache: nil, vp: &VendorProduct{"campcodes", "school_faculty_scheduling_system"}, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@vuldb.com", Tags: []string{"Exploit", "Third Party Advisory"}, @@ -1422,7 +1421,7 @@ func TestReposFromReferencesCVEList(t *testing.T) { CVE: "CVE-2025-26519", cache: nil, vp: nil, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@mitre.org", Tags: nil, @@ -1439,7 +1438,7 @@ func TestReposFromReferencesCVEList(t *testing.T) { CVE: "CVE-2016-10525", cache: nil, vp: nil, - refs: []Reference{ + refs: []models.Reference{ { Source: "support@hackerone.com", Tags: []string{"Patch", "Third Party Advisory"}, @@ -1455,7 +1454,7 @@ func TestReposFromReferencesCVEList(t *testing.T) { CVE: "CVE-2024-7790", cache: nil, vp: &VendorProduct{"Devikia", "DevikaAI"}, - refs: []Reference{ + refs: []models.Reference{ { Source: "cna@vuldb.com", Tags: []string{"Patch", "Third Party Advisory"}, diff --git a/vulnfeeds/cves/cve.go b/vulnfeeds/models/cve.go similarity index 98% rename from vulnfeeds/cves/cve.go rename to vulnfeeds/models/cve.go index da606e00fb7..875896834dd 100644 --- a/vulnfeeds/cves/cve.go +++ b/vulnfeeds/models/cve.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package cves contains CVE-specific data structures. -package cves +// Package models contains CVE-specific data structures. +package models import ( "strings" diff --git a/vulnfeeds/models/metrics.go b/vulnfeeds/models/metrics.go new file mode 100644 index 00000000000..85d7c1db4cd --- /dev/null +++ b/vulnfeeds/models/metrics.go @@ -0,0 +1,93 @@ +package models + +import ( + "fmt" + "log/slog" + + "github.com/google/osv/vulnfeeds/utility/logger" + "github.com/ossf/osv-schema/bindings/go/osvschema" +) + +type ConversionOutcome int + +const ( + Extension = ".json" +) + +const ( + // Set of enums for categorizing conversion outcomes. + ConversionUnknown ConversionOutcome = iota // Shouldn't happen + Successful // It worked! + Rejected // The CVE was rejected + NoSoftware // The CVE had no CPEs relating to software (i.e. Operating Systems or Hardware). + NoRepos // The CPE Vendor/Product had no repositories derived for it. + NoCommitRanges // No viable commit ranges could be calculated from the repository for the CVE's CPE(s). + NoRanges // No version ranges could be extracted from the record. + FixUnresolvable // Partial resolution of versions, resulting in a false positive. +) + +// RefTagDenyList contains reference tags that are often associated with unreliable or +// irrelevant repository URLs. References with these tags are currently ignored +// to avoid incorrect repository associations. +var RefTagDenyList = []string{ + // "Exploit", + // "Third Party Advisory", + "Broken Link", // Actively ignore these. +} + +func (c ConversionOutcome) String() string { + return [...]string{"ConversionUnknown", "Successful", "Rejected", "NoSoftware", "NoRepos", "NoCommitRanges", "NoRanges", "FixUnresolvable"}[c] +} + +// ConversionMetrics holds the collected data about the conversion process for a single CVE. +type ConversionMetrics struct { + CVEID CVEID `json:"id"` // The CVE ID + CNA string `json:"cna"` // The CNA that assigned the CVE. + Outcome ConversionOutcome `json:"outcome"` // The final outcome of the conversion (e.g., "Successful", "Failed"). + Repos []string `json:"repos"` // A list of repositories extracted from the CVE's references. + RefTypesCount map[osvschema.Reference_Type]int `json:"ref_types_count"` // A count of each type of reference found. + VersionSources []VersionSource `json:"version_sources"` // A list of the ways the versions were extracted + Notes []string `json:"notes"` // A collection of notes and warnings generated during conversion. + CPEs []string `json:"cpes"` + UnresolvedRangesCount int `json:"unresolved_ranges_count"` + ResolvedRangesCount int `json:"resolved_ranges_count"` +} + +// AddNote adds a formatted note to the ConversionMetrics. +func (m *ConversionMetrics) AddNote(format string, a ...any) { + m.Notes = append(m.Notes, fmt.Sprintf(format, a...)) + logger.Debug(fmt.Sprintf(format, a...), slog.String("cna", m.CNA), slog.String("cve", string(m.CVEID))) +} + +// AddSource appends a source to the ConversionMetrics +func (m *ConversionMetrics) AddSource(source VersionSource) { + m.VersionSources = append(m.VersionSources, source) +} + +// VersionSource indicates the source of the extracted version information. +type VersionSource string + +const ( + VersionSourceNone VersionSource = "NOVERS" + VersionSourceAffected VersionSource = "CVEAFFVERS" + VersionSourceGit VersionSource = "GITVERS" + VersionSourceCPE VersionSource = "CPEVERS" + VersionSourceDescription VersionSource = "DESCRVERS" +) + +func DetermineOutcome(metrics *ConversionMetrics) { + // check if we have affected ranges/versions. + if len(metrics.Repos) == 0 { + // Fix unlikely, as no repos to resolve + metrics.Outcome = NoRepos + return + } + + if metrics.ResolvedRangesCount > 0 { + metrics.Outcome = Successful + } else if metrics.UnresolvedRangesCount > 0 { + metrics.Outcome = NoCommitRanges + } else { + metrics.Outcome = NoRanges + } +} diff --git a/vulnfeeds/cves/nvd2.go b/vulnfeeds/models/nvd2.go similarity index 99% rename from vulnfeeds/cves/nvd2.go rename to vulnfeeds/models/nvd2.go index 95d5fd8b653..3b1038fbaa9 100644 --- a/vulnfeeds/cves/nvd2.go +++ b/vulnfeeds/models/nvd2.go @@ -17,7 +17,7 @@ // --capitalization JSON \ // cve_api_json_2.0.schema -package cves +package models import ( "encoding/json" @@ -109,7 +109,7 @@ type CVEAPIJSON20Schema struct { type CVEID string -type CVE struct { +type NVDCVE struct { // CISAActionDue corresponds to the JSON schema field "cisaActionDue". CISAActionDue *types.SerializableDate `json:"cisaActionDue,omitempty" mapstructure:"cisaActionDue,omitempty" yaml:"cisaActionDue,omitempty"` @@ -311,7 +311,7 @@ type LangString struct { } // UnmarshalJSON implements json.Unmarshaler. -func (j *CVE) UnmarshalJSON(b []byte) error { +func (j *NVDCVE) UnmarshalJSON(b []byte) error { var raw map[string]any if err := json.Unmarshal(b, &raw); err != nil { return err @@ -331,7 +331,7 @@ func (j *CVE) UnmarshalJSON(b []byte) error { if v, ok := raw["references"]; !ok || v == nil { return errors.New("field references in CveItem: required") } - type Plain CVE + type Plain NVDCVE var plain Plain if err := json.Unmarshal(b, &plain); err != nil { return err @@ -344,14 +344,14 @@ func (j *CVE) UnmarshalJSON(b []byte) error { // if len(plain.References) > 500 { // return fmt.Errorf("field %s length: must be <= %d", "references", 500) // } - *j = CVE(plain) + *j = NVDCVE(plain) return nil } // (hand generated), see https://github.com/omissis/go-jsonschema/issues/171 type Vulnerability struct { - CVE CVE `json:"cve" mapstructure:"cve" yaml:"cve"` + CVE NVDCVE `json:"cve" mapstructure:"cve" yaml:"cve"` } // CVSS subscore. diff --git a/vulnfeeds/pypi/pypi.go b/vulnfeeds/pypi/pypi.go index fe9b3541b3e..ed798350e1d 100644 --- a/vulnfeeds/pypi/pypi.go +++ b/vulnfeeds/pypi/pypi.go @@ -27,8 +27,8 @@ import ( "strings" version "github.com/aquasecurity/go-pep440-version" - "github.com/google/osv/vulnfeeds/cves" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/triage" "github.com/google/osv/vulnfeeds/utility/logger" ) @@ -244,7 +244,7 @@ func New(pypiLinksPath string, pypiVersionsPath string) *PyPI { } } -func (p *PyPI) Matches(cve cves.CVE, falsePositives *triage.FalsePositives) []string { +func (p *PyPI) Matches(cve models.NVDCVE, falsePositives *triage.FalsePositives) []string { matches := []string{} for _, reference := range cve.References { // If there is a PyPI link, it must be a Python package. These take precedence. @@ -348,9 +348,9 @@ func (p *PyPI) packageExists(pkg string) bool { return result } -func (p *PyPI) finalPkgCheck(cve cves.CVE, pkg string, falsePositives *triage.FalsePositives) bool { +func (p *PyPI) finalPkgCheck(cve models.NVDCVE, pkg string, falsePositives *triage.FalsePositives) bool { // To avoid false positives, check that the pkg name is mentioned in the description. - desc := strings.ToLower(cves.EnglishDescription(cve.Descriptions)) + desc := strings.ToLower(models.EnglishDescription(cve.Descriptions)) pkgNameParts := strings.Split(pkg, "-") for _, part := range pkgNameParts { @@ -375,7 +375,7 @@ func (p *PyPI) finalPkgCheck(cve cves.CVE, pkg string, falsePositives *triage.Fa } // matchesPackage checks if a given reference link matches a PyPI package. -func (p *PyPI) matchesPackage(link string, cve cves.CVE, falsePositives *triage.FalsePositives) []string { +func (p *PyPI) matchesPackage(link string, cve models.NVDCVE, falsePositives *triage.FalsePositives) []string { pkgs := []string{} u, err := url.Parse(strings.ToLower(link)) if err != nil { diff --git a/vulnfeeds/vulns/vulns.go b/vulnfeeds/vulns/vulns.go index 3f6eeb319b0..058f3a16f36 100644 --- a/vulnfeeds/vulns/vulns.go +++ b/vulnfeeds/vulns/vulns.go @@ -34,7 +34,6 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "gopkg.in/yaml.v2" - "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility" "github.com/google/osv/vulnfeeds/utility/logger" @@ -324,7 +323,7 @@ func (v *Vulnerability) AddPkgInfo(pkgInfo PackageInfo) { // getBestSeverity finds the best CVSS severity vector from the provided metrics data. // It prioritizes newer CVSS versions and "Primary" sources. -func getBestSeverity(metricsData *cves.CVEItemMetrics) (string, string) { +func getBestSeverity(metricsData *models.CVEItemMetrics) (string, string) { // Define search passes. First pass for "Primary", second for any. for _, primaryOnly := range []bool{true, false} { // Inside each pass, prioritize v4.0 over v3.1 over v3.0. @@ -350,7 +349,7 @@ func getBestSeverity(metricsData *cves.CVEItemMetrics) (string, string) { // AddSeverity adds CVSS severity information to the OSV vulnerability object. // It uses the highest available CVSS score from the underlying CVE record. -func (v *Vulnerability) AddSeverity(metricsData *cves.CVEItemMetrics) { +func (v *Vulnerability) AddSeverity(metricsData *models.CVEItemMetrics) { bestVectorString, severityType := getBestSeverity(metricsData) if bestVectorString == "" { @@ -588,7 +587,7 @@ func ClassifyReferenceLink(link string, tag string) osvschema.Reference_Type { // ExtractReferencedVulns extracts other vulnerability IDs from a CVE's references // to place them into the aliases and related fields. -func ExtractReferencedVulns(id cves.CVEID, cveID cves.CVEID, references []cves.Reference) ([]string, []string) { +func ExtractReferencedVulns(id models.CVEID, cveID models.CVEID, references []models.Reference) ([]string, []string) { var aliases []string var related []string if id != cveID { @@ -663,7 +662,7 @@ func Unique[T comparable](s []T) []T { } // ClassifyReferences annotates reference links based on their tags or their shape. -func ClassifyReferences(refs []cves.Reference) []*osvschema.Reference { +func ClassifyReferences(refs []models.Reference) []*osvschema.Reference { var references []*osvschema.Reference refMap := make(map[string]map[osvschema.Reference_Type]bool) @@ -705,12 +704,12 @@ func ClassifyReferences(refs []cves.Reference) []*osvschema.Reference { // Leaves affected and version fields empty to be filled in later with AddPkgInfo // There are two id fields passed in as one of the users of this field (PyPi) sometimes has a different id than the CVEID // and the ExtractReferencedVulns function uses these in a check to add the other ID as an alias. -func FromNVDCVE(id cves.CVEID, cve cves.CVE) *Vulnerability { +func FromNVDCVE(id models.CVEID, cve models.NVDCVE) *Vulnerability { aliases, related := ExtractReferencedVulns(id, cve.ID, cve.References) v := &Vulnerability{ Vulnerability: &osvschema.Vulnerability{ Id: string(id), - Details: cves.EnglishDescription(cve.Descriptions), + Details: models.EnglishDescription(cve.Descriptions), Aliases: aliases, Related: related, Published: timestamppb.New(cve.Published.Time), @@ -723,9 +722,9 @@ func FromNVDCVE(id cves.CVEID, cve cves.CVE) *Vulnerability { return v } -// GetCPEs extracts CPE strings from a slice of cves.CPE. +// GetCPEs extracts CPE strings from a slice of models.CPE. // Returns array of CPE strings and array of notes. -func GetCPEs(cpeApplicability []cves.CPE) ([]string, []string) { +func GetCPEs(cpeApplicability []models.CPE) ([]string, []string) { var CPEs []string var notes []string for _, c := range cpeApplicability { @@ -796,13 +795,13 @@ func CheckQuality(text string) QualityCheck { } // LoadAllCVEs loads the downloaded CVE's from the NVD database into memory. -func LoadAllCVEs(cvePath string) map[cves.CVEID]cves.Vulnerability { +func LoadAllCVEs(cvePath string) map[models.CVEID]models.Vulnerability { dir, err := os.ReadDir(cvePath) if err != nil { logger.Fatal("Failed to read dir", slog.String("path", cvePath), slog.Any("err", err)) } - vulnsChan := make(chan cves.Vulnerability) + vulnsChan := make(chan models.Vulnerability) var wg sync.WaitGroup for _, entry := range dir { @@ -821,7 +820,7 @@ func LoadAllCVEs(cvePath string) map[cves.CVEID]cves.Vulnerability { } defer file.Close() - var nvdcve cves.CVEAPIJSON20Schema + var nvdcve models.CVEAPIJSON20Schema if err := json.NewDecoder(file).Decode(&nvdcve); err != nil { logger.Error("Failed to decode JSON", slog.String("file", filename), slog.Any("err", err)) return @@ -839,7 +838,7 @@ func LoadAllCVEs(cvePath string) map[cves.CVEID]cves.Vulnerability { close(vulnsChan) }() - result := make(map[cves.CVEID]cves.Vulnerability) + result := make(map[models.CVEID]models.Vulnerability) for item := range vulnsChan { result[item.CVE.ID] = item } @@ -847,7 +846,7 @@ func LoadAllCVEs(cvePath string) map[cves.CVEID]cves.Vulnerability { return result } -func FindSeverity(metricsData []cves.Metrics) *osvschema.Severity { +func FindSeverity(metricsData []models.Metrics) *osvschema.Severity { bestVectorString, severityType := getBestCVE5Severity(metricsData) if bestVectorString == "" { return nil @@ -859,14 +858,14 @@ func FindSeverity(metricsData []cves.Metrics) *osvschema.Severity { } } -func getBestCVE5Severity(metricsData []cves.Metrics) (string, osvschema.Severity_Type) { +func getBestCVE5Severity(metricsData []models.Metrics) (string, osvschema.Severity_Type) { checks := []struct { - getVectorString func(cves.Metrics) string + getVectorString func(models.Metrics) string severityType osvschema.Severity_Type }{ - {func(m cves.Metrics) string { return m.CVSSv4_0.VectorString }, osvschema.Severity_CVSS_V4}, - {func(m cves.Metrics) string { return m.CVSSv3_1.VectorString }, osvschema.Severity_CVSS_V3}, - {func(m cves.Metrics) string { return m.CVSSv3_0.VectorString }, osvschema.Severity_CVSS_V3}, + {func(m models.Metrics) string { return m.CVSSv4_0.VectorString }, osvschema.Severity_CVSS_V4}, + {func(m models.Metrics) string { return m.CVSSv3_1.VectorString }, osvschema.Severity_CVSS_V3}, + {func(m models.Metrics) string { return m.CVSSv3_0.VectorString }, osvschema.Severity_CVSS_V3}, } for _, check := range checks { diff --git a/vulnfeeds/vulns/vulns_test.go b/vulnfeeds/vulns/vulns_test.go index 8d9d57663c1..e5c84aa1586 100644 --- a/vulnfeeds/vulns/vulns_test.go +++ b/vulnfeeds/vulns/vulns_test.go @@ -13,7 +13,6 @@ import ( "slices" gocmp "github.com/google/go-cmp/cmp" - "github.com/google/osv/vulnfeeds/cves" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -81,11 +80,11 @@ func TestClassifyReferenceLink(t *testing.T) { func TestClassifyReferences(t *testing.T) { testcases := []struct { - refData []cves.Reference + refData []models.Reference references []*osvschema.Reference }{ { - refData: []cves.Reference{ + refData: []models.Reference{ { Source: "https://example.com", Tags: []string{"MISC"}, URL: "https://example.com", }, @@ -93,7 +92,7 @@ func TestClassifyReferences(t *testing.T) { references: []*osvschema.Reference{{Url: "https://example.com", Type: osvschema.Reference_WEB}}, }, { - refData: []cves.Reference{ + refData: []models.Reference{ { Source: "https://github.com/Netflix/lemur/issues/117", URL: "https://github.com/Netflix/lemur/issues/117", Tags: []string{"MISC", "Issue Tracking"}, }, @@ -101,7 +100,7 @@ func TestClassifyReferences(t *testing.T) { references: []*osvschema.Reference{{Url: "https://github.com/Netflix/lemur/issues/117", Type: osvschema.Reference_REPORT}}, }, { - refData: []cves.Reference{ + refData: []models.Reference{ { Source: "https://github.com/curl/curl/issues/9271", URL: "https://github.com/curl/curl/issues/9271", Tags: []string{"MISC", "Exploit", "Issue Tracking", "Third Party Advisory"}, }, @@ -113,7 +112,7 @@ func TestClassifyReferences(t *testing.T) { }, }, { - refData: []cves.Reference{ + refData: []models.Reference{ { Source: "https://gitlab.com/gitlab-org/gitlab/-/issues/517693", URL: "https://gitlab.com/gitlab-org/gitlab/-/issues/517693", Tags: []string{"issue-tracking", "permissions-required"}, }, @@ -123,7 +122,7 @@ func TestClassifyReferences(t *testing.T) { }, }, { - refData: []cves.Reference{ + refData: []models.Reference{ { Source: "https://security.gentoo.org/glsa/202307-01", URL: "https://security.gentoo.org/glsa/202307-01", Tags: []string{"vendor-advisory"}, }, @@ -133,7 +132,7 @@ func TestClassifyReferences(t *testing.T) { }, }, { - refData: []cves.Reference{ + refData: []models.Reference{ { Source: "http://www.openwall.com/lists/oss-security/2023/07/20/1", URL: "http://www.openwall.com/lists/oss-security/2023/07/20/1", Tags: []string{"mailing-list"}, }, @@ -154,13 +153,13 @@ func TestClassifyReferences(t *testing.T) { } } -func loadTestData2(cveName string) cves.Vulnerability { +func loadTestData2(cveName string) models.Vulnerability { fileName := fmt.Sprintf("../test_data/nvdcve-2.0/%s.json", cveName) file, err := os.Open(fileName) if err != nil { log.Fatalf("Failed to load test data from %q", fileName) } - var nvdCves cves.CVEAPIJSON20Schema + var nvdCves models.CVEAPIJSON20Schema err = json.NewDecoder(file).Decode(&nvdCves) if err != nil { log.Fatalf("Failed to decode %q: %+v", fileName, err) @@ -172,7 +171,7 @@ func loadTestData2(cveName string) cves.Vulnerability { } log.Fatalf("test data doesn't contain %q", cveName) - return cves.Vulnerability{} + return models.Vulnerability{} } func TestExtractAliases(t *testing.T) { @@ -198,7 +197,7 @@ func TestExtractAliases(t *testing.T) { func TestEnglishDescription(t *testing.T) { cveItem := loadTestData2("CVE-2022-36037") - description := cves.EnglishDescription(cveItem.CVE.Descriptions) + description := models.EnglishDescription(cveItem.CVE.Descriptions) expectedDescription := "kirby is a content management system (CMS) that adapts to many different projects and helps you build your own ideal interface. Cross-site scripting (XSS) is a type of vulnerability that allows execution of any kind of JavaScript code inside the Panel session of the same or other users. In the Panel, a harmful script can for example trigger requests to Kirby's API with the permissions of the victim. If bad actors gain access to your group of authenticated Panel users they can escalate their privileges via the Panel session of an admin user. Depending on your site, other JavaScript-powered attacks are possible. The multiselect field allows selection of tags from an autocompleted list. Unfortunately, the Panel in Kirby 3.5 used HTML rendering for the raw option value. This allowed **attackers with influence on the options source** to store HTML code. The browser of the victim who visited a page with manipulated multiselect options in the Panel will then have rendered this malicious HTML code when the victim opened the autocomplete dropdown. Users are *not* affected by this vulnerability if you don't use the multiselect field or don't use it with options that can be manipulated by attackers. The problem has been patched in Kirby 3.5.8.1." if description != expectedDescription { t.Errorf("Description not extracted, got %v, but expected %v", description, expectedDescription) @@ -407,7 +406,7 @@ func TestAddPkgInfo(t *testing.T) { func TestAddSeverity(t *testing.T) { tests := []struct { description string - inputCVE cves.Vulnerability + inputCVE models.Vulnerability expectedResult []*osvschema.Severity }{ {