feat: create allowlist of accepted ecosystems per datasource

go/internal/database/datastore/internal/validate/validate.go

@@ -219,6 +219,7 @@ func writeRecords(ctx context.Context, client *datastore.Client) {
 		Extension:               ".yaml",
 		KeyPath:                 "key",
 		IgnoreGit:               false,
+		AcceptedEcosystems:      []string{"crates.io", "npm"},
 		DetectCherrypicks:       true,
 		ConsiderAllBranches:     true,
 		VersionsFromRepo:        true,

go/internal/database/datastore/internal/validate/validate.py

@@ -132,6 +132,7 @@ def main() -> int:
       extension='.json',
       key_path='vulnerability',
       ignore_git=False,
+      accepted_ecosystems=['Go', 'PyPI'],
       detect_cherrypicks=True,
       consider_all_branches=False,
       versions_from_repo=True,

go/internal/database/datastore/models.go

@@ -141,6 +141,7 @@ type SourceRepository struct {
 	Editable                bool                        `datastore:"editable"`
 	Extension               string                      `datastore:"extension"`
 	KeyPath                 string                      `datastore:"key_path"`
+	AcceptedEcosystems      []string                    `datastore:"accepted_ecosystems"`
 	IgnoreGit               bool                        `datastore:"ignore_git"`
 	DetectCherrypicks       bool                        `datastore:"detect_cherrypicks"`
 	ConsiderAllBranches     bool                        `datastore:"consider_all_branches"`

go/internal/database/datastore/source_repository.go

@@ -8,6 +8,7 @@ import (
 
 	"cloud.google.com/go/datastore"
 	"github.com/google/osv.dev/go/internal/models"
+	"github.com/ossf/osv-schema/bindings/go/osvconstants"
 	"google.golang.org/api/iterator"
 )
 
@@ -80,13 +81,18 @@ func (s *SourceRepositoryStore) All(ctx context.Context) iter.Seq2[*models.Sourc
 }
 
 func (sr *SourceRepository) toModel() *models.SourceRepository {
+	ecos := make([]osvconstants.Ecosystem, 0, len(sr.AcceptedEcosystems))
+	for _, e := range sr.AcceptedEcosystems {
+		ecos = append(ecos, osvconstants.Ecosystem(e))
+	}
 	msr := &models.SourceRepository{
-		Name:           sr.Name,
-		Type:           sr.Type,
-		Strictness:     sr.StrictValidation,
-		IgnorePatterns: sr.IgnorePatterns,
-		Extension:      sr.Extension,
-		KeyPath:        sr.KeyPath,
+		Name:               sr.Name,
+		Type:               sr.Type,
+		Strictness:         sr.StrictValidation,
+		IgnorePatterns:     sr.IgnorePatterns,
+		AcceptedEcosystems: ecos,
+		Extension:          sr.Extension,
+		KeyPath:            sr.KeyPath,
 		GitAnalysis: &models.GitAnalysisConfig{
 			IgnoreGit:           sr.IgnoreGit,
 			DetectCherrypicks:   sr.DetectCherrypicks,
@@ -126,16 +132,21 @@ func (sr *SourceRepository) toModel() *models.SourceRepository {
 }
 
 func newSourceRepositoryFromModel(r *models.SourceRepository) *SourceRepository {
+	ecos := make([]string, 0, len(r.AcceptedEcosystems))
+	for _, e := range r.AcceptedEcosystems {
+		ecos = append(ecos, string(e))
+	}
 	sr := &SourceRepository{
-		Name:             r.Name,
-		Type:             r.Type,
-		StrictValidation: r.Strictness,
-		IgnorePatterns:   r.IgnorePatterns,
-		Extension:        r.Extension,
-		KeyPath:          r.KeyPath,
-		Link:             r.Link,
-		HumanLink:        r.HumanLink,
-		DBPrefix:         r.IDPrefixes,
+		Name:               r.Name,
+		Type:               r.Type,
+		StrictValidation:   r.Strictness,
+		IgnorePatterns:     r.IgnorePatterns,
+		AcceptedEcosystems: ecos,
+		Extension:          r.Extension,
+		KeyPath:            r.KeyPath,
+		Link:               r.Link,
+		HumanLink:          r.HumanLink,
+		DBPrefix:           r.IDPrefixes,
 	}
 
 	if r.GitAnalysis != nil {

go/internal/models/source_repository.go

@@ -5,6 +5,8 @@ import (
 	"context"
 	"iter"
 	"time"
+
+	"github.com/ossf/osv-schema/bindings/go/osvconstants"
 )
 
 type SourceRepositoryStore interface {
@@ -53,6 +55,10 @@ type SourceRepository struct {
 	// Git Content Analysis (Applied to Git commit ranges found in vulnerabilities)
 	GitAnalysis *GitAnalysisConfig
 
+	// List of ecosystems that this source is allowed to import.
+	// A value of '*' means allow all ecosystems.
+	AcceptedEcosystems []osvconstants.Ecosystem
+
 	// HTTP link prefix to individual OSV source records.
 	Link string
 	// HTTP link prefix to individual vulnerability records for humans.

go/internal/worker/engine_test.go

@@ -112,7 +112,8 @@ func TestPipeline_EndToEnd_PortedFromPython(t *testing.T) {
 		Stores: Stores{
 			SourceRepo: mockSourceRepoStore{
 				repo: &models.SourceRepository{
-					Name: "source",
+					Name:               "source",
+					AcceptedEcosystems: []osvconstants.Ecosystem{"*"},
 					GitAnalysis: &models.GitAnalysisConfig{
 						IgnoreGit: false,
 					},

go/internal/worker/pipeline/filterecosystem/filterecosystem.go

@@ -4,11 +4,13 @@ package filterecosystem
 import (
 	"context"
 	"log/slog"
+	"slices"
 	"strings"
 
 	"github.com/google/osv.dev/go/internal/osvutil/schema"
 	"github.com/google/osv.dev/go/internal/worker/pipeline"
 	"github.com/google/osv.dev/go/logger"
+	"github.com/ossf/osv-schema/bindings/go/osvconstants"
 	"github.com/ossf/osv-schema/bindings/go/osvschema"
 	"google.golang.org/protobuf/proto"
 )
@@ -19,13 +21,17 @@ var _ pipeline.Enricher = (*Enricher)(nil)
 
 func (*Enricher) Enrich(ctx context.Context, vuln *osvschema.Vulnerability, params *pipeline.EnrichParams) error {
 	newAffected := make([]*osvschema.Affected, 0, len(vuln.GetAffected()))
+	acceptedEcos := params.SourceRepo.AcceptedEcosystems
+	allowAll := slices.Contains(acceptedEcos, "*")
 	for _, affected := range vuln.GetAffected() {
 		pkg := affected.GetPackage()
+		if pkg == nil {
+			continue
+		}
 		ecosystem := pkg.GetEcosystem()
 		ecoBase, _, _ := strings.Cut(ecosystem, ":")
 		shouldRemove := false
-		if params.SourceRepo.Name == "echo" && ecoBase != "Echo" {
-			// TODO(michaelkedar): Have a list of allowed ecosystems in the SourceRepo #5285
+		if !allowAll && !slices.Contains(acceptedEcos, osvconstants.Ecosystem(ecoBase)) {
 			shouldRemove = true
 		}
 		if !schema.IsKnownEcosystem(ecoBase) {

go/internal/worker/pipeline/filterecosystem/filterecosystem_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/osv.dev/go/internal/models"
 	"github.com/google/osv.dev/go/internal/worker/pipeline"
+	"github.com/ossf/osv-schema/bindings/go/osvconstants"
 	"github.com/ossf/osv-schema/bindings/go/osvschema"
 	"google.golang.org/protobuf/testing/protocmp"
 )
@@ -17,13 +18,13 @@ func TestEnricher_Enrich(t *testing.T) {
 
 	tests := []struct {
 		name             string
-		repoName         string
+		acceptedEcos     []osvconstants.Ecosystem
 		affected         []*osvschema.Affected
 		expectedAffected []*osvschema.Affected
 	}{
 		{
-			name:     "Keep valid ecosystem",
-			repoName: "all-allowed",
+			name:         "Keep valid ecosystem",
+			acceptedEcos: []osvconstants.Ecosystem{"*"},
 			affected: []*osvschema.Affected{
 				{
 					Package: &osvschema.Package{
@@ -42,8 +43,8 @@ func TestEnricher_Enrich(t *testing.T) {
 			},
 		},
 		{
-			name:     "Filter out invalid ecosystem",
-			repoName: "all-allowed",
+			name:         "Filter out invalid ecosystem",
+			acceptedEcos: []osvconstants.Ecosystem{"*"},
 			affected: []*osvschema.Affected{
 				{
 					Package: &osvschema.Package{
@@ -55,8 +56,8 @@ func TestEnricher_Enrich(t *testing.T) {
 			expectedAffected: []*osvschema.Affected{},
 		},
 		{
-			name:     "Filter out non-Echo for Echo repo",
-			repoName: "echo",
+			name:         "Filter out non-Echo for Echo repo",
+			acceptedEcos: []osvconstants.Ecosystem{osvconstants.EcosystemEcho},
 			affected: []*osvschema.Affected{
 				{
 					Package: &osvschema.Package{
@@ -68,8 +69,8 @@ func TestEnricher_Enrich(t *testing.T) {
 			expectedAffected: []*osvschema.Affected{},
 		},
 		{
-			name:     "Keep Echo for Echo repo",
-			repoName: "echo",
+			name:         "Keep Echo for Echo repo",
+			acceptedEcos: []osvconstants.Ecosystem{osvconstants.EcosystemEcho},
 			affected: []*osvschema.Affected{
 				{
 					Package: &osvschema.Package{
@@ -88,8 +89,8 @@ func TestEnricher_Enrich(t *testing.T) {
 			},
 		},
 		{
-			name:     "Preserve GIT ranges when filtering ecosystem",
-			repoName: "all-allowed",
+			name:         "Preserve GIT ranges when filtering ecosystem",
+			acceptedEcos: []osvconstants.Ecosystem{"*"},
 			affected: []*osvschema.Affected{
 				{
 					Package: &osvschema.Package{
@@ -128,8 +129,8 @@ func TestEnricher_Enrich(t *testing.T) {
 			},
 		},
 		{
-			name:     "Mixed valid and invalid ecosystems",
-			repoName: "all-allowed",
+			name:         "Mixed valid and invalid ecosystems",
+			acceptedEcos: []osvconstants.Ecosystem{"*"},
 			affected: []*osvschema.Affected{
 				{
 					Package: &osvschema.Package{
@@ -164,7 +165,7 @@ func TestEnricher_Enrich(t *testing.T) {
 
 			params := &pipeline.EnrichParams{
 				SourceRepo: &models.SourceRepository{
-					Name: tc.repoName,
+					AcceptedEcosystems: tc.acceptedEcos,
 				},
 			}
 

osv/models.py

@@ -1545,6 +1545,9 @@ class SourceRepository(ndb.Model):
   key_path: str = ndb.StringProperty()
   # If true, don't analyze any Git ranges.
   ignore_git: bool = ndb.BooleanProperty(default=False)
+  # List of ecosystems that this source is allowed to import.
+  # A value of '*' means allow all ecosystems.
+  accepted_ecosystems: list[str] = ndb.StringProperty(repeated=True)
   # Whether to detect cherypicks or not (slow for large repos).
   detect_cherrypicks: bool = ndb.BooleanProperty(default=True)
   # Whether to consider all branches when analyzing GIT ranges.