diff --git a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go index 8f01274d437..9ffa22f97af 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go +++ b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go @@ -26,6 +26,7 @@ var ( startYear = flag.String("start-year", "2022", "The first in scope year to process.") workers = flag.Int("workers", 30, "The number of concurrent workers to use for processing CVEs.") cnaAllowList = flag.String("cnas-allowlist", "", "A comma-separated list of CNAs to process. If not provided, defaults to cna_allowlist.txt.") + rejectFailed = flag.Bool("reject-failed", false, "If set, OSV records with a failed conversion outcome will not be generated.") ) //go:embed cna_allowlist.txt @@ -57,7 +58,7 @@ func main() { // Start the worker pool. for range *workers { wg.Add(1) - go worker(&wg, jobs, *localOutputDir, cnaList) + go worker(&wg, jobs, *localOutputDir, cnaList, *rejectFailed) } // Discover files and send them to the workers. @@ -76,6 +77,7 @@ func main() { logger.Info("Processing CVEs for year", slog.String("year", year)) err := filepath.Walk(yearDir, func(path string, info os.FileInfo, err error) error { if err != nil { + logger.Info("Error walking directory for year", slog.String("year", year), slog.Any("err", err)) return err } if !info.IsDir() && strings.HasSuffix(info.Name(), ".json") { @@ -95,7 +97,7 @@ func main() { } // worker is a function that processes CVE files from the jobs channel. -func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string) { +func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string, rejectFailed bool) { defer wg.Done() for path := range jobs { data, err := os.ReadFile(path) @@ -131,10 +133,17 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string } // Perform the conversion and export the results. - if err = cvelist2osv.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, sourceLink); err != nil { + metrics, err := cvelist2osv.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, sourceLink) + if err != nil { logger.Warn("Failed to generate an OSV record", slog.String("cve", string(cveID)), slog.Any("err", err)) } else { - logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName)) + if rejectFailed && metrics.Outcome != models.Successful { + logger.Info("Rejecting failed OSV record", slog.String("cve", string(cveID)), slog.String("outcome", metrics.Outcome.String())) + osvFile.Close() + os.Remove(osvFile.Name()) + } else { + logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName), slog.String("outcome", metrics.Outcome.String())) + } } metricsFile.Close() diff --git a/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go index f0190bd7ac5..682e7feb0a6 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go +++ b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go @@ -52,10 +52,10 @@ func main() { } // Perform the conversion and export the results. - if err = cvelist2osv.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, ""); err != nil { + if metrics, err := cvelist2osv.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, ""); err != nil { logger.Warn("Failed to generate an OSV record", slog.String("cve", string(cveID)), slog.Any("err", err)) } else { - logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName)) + logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName), slog.String("outcome", metrics.Outcome.String())) } metricsFile.Close() diff --git a/vulnfeeds/cvelist2osv/converter.go b/vulnfeeds/cvelist2osv/converter.go index de194e6516f..89faebba871 100644 --- a/vulnfeeds/cvelist2osv/converter.go +++ b/vulnfeeds/cvelist2osv/converter.go @@ -143,7 +143,7 @@ func FromCVE5(cve models.CVE5, refs []models.Reference, metrics *models.Conversi // ConvertAndExportCVEToOSV is the main function for this file. It takes a CVE, // converts it into an OSV record, collects metrics, and writes both to disk. -func ConvertAndExportCVEToOSV(cve models.CVE5, vulnSink io.Writer, metricsSink io.Writer, sourceLink string) error { +func ConvertAndExportCVEToOSV(cve models.CVE5, vulnSink io.Writer, metricsSink io.Writer, sourceLink string) (*models.ConversionMetrics, error) { cveID := cve.Metadata.CVEID cnaAssigner := cve.Metadata.AssignerShortName references := identifyPossibleURLs(cve) @@ -175,21 +175,21 @@ func ConvertAndExportCVEToOSV(cve models.CVE5, vulnSink io.Writer, metricsSink i err := v.ToJSON(vulnSink) if err != nil { logger.Info("Failed to write", slog.Any("err", err)) - return err + return &metrics, err } marshalledMetrics, err := json.MarshalIndent(&metrics, "", " ") if err != nil { logger.Info("Failed to marshal", slog.Any("err", err)) - return err + return &metrics, err } _, err = metricsSink.Write(marshalledMetrics) if err != nil { logger.Info("Failed to write", slog.Any("err", err)) - return err + return &metrics, err } - return nil + return &metrics, nil } // identifyPossibleURLs extracts all URLs from a CVE object. diff --git a/vulnfeeds/cvelist2osv/converter_test.go b/vulnfeeds/cvelist2osv/converter_test.go index f27d513e0e8..8e953cc4c59 100644 --- a/vulnfeeds/cvelist2osv/converter_test.go +++ b/vulnfeeds/cvelist2osv/converter_test.go @@ -582,7 +582,7 @@ func TestConvertAndExportCVEToOSV(t *testing.T) { t.Run(tc.name, func(t *testing.T) { vWriter := bytes.NewBuffer(nil) mWriter := bytes.NewBuffer(nil) - err := ConvertAndExportCVEToOSV(tc.cve, vWriter, mWriter, "") + _, err := ConvertAndExportCVEToOSV(tc.cve, vWriter, mWriter, "") if err != nil { t.Errorf("Unexpected error from ConvertAndExportCVEToOSV: %v", err) }