From f1e9c5ac1fcc4a6f2c7310517f4b1e83ca244260 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 25 Feb 2026 14:38:19 +0200 Subject: [PATCH 1/4] Fix null constraint violations in multiple v1 exploit collection pipelines Signed-off-by: ziad hany --- vulnerabilities/pipelines/enhance_with_exploitdb.py | 3 ++- vulnerabilities/pipelines/enhance_with_kev.py | 3 ++- vulnerabilities/pipelines/enhance_with_metasploit.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py index 4d2e966d9..74fc55b09 100644 --- a/vulnerabilities/pipelines/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -88,7 +88,8 @@ def add_vulnerability_exploit(row, logger): for raw_alias in aliases: try: if alias := Alias.objects.get(alias=raw_alias): - vulnerabilities.add(alias.vulnerability) + if alias.vulnerability: + vulnerabilities.add(alias.vulnerability) except Alias.DoesNotExist: continue diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index c9fc21a84..00fc72106 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -74,7 +74,8 @@ def add_vulnerability_exploit(kev_vul, logger): vulnerability = None try: if alias := Alias.objects.get(alias=cve_id): - vulnerability = alias.vulnerability + if alias.vulnerability: + vulnerability = alias.vulnerability except Alias.DoesNotExist: logger(f"No vulnerability found for aliases {cve_id}") return 0 diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py index a9b901400..12437fadc 100644 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -79,7 +79,8 @@ def add_vulnerability_exploit(record, logger): for ref in interesting_references: try: if alias := Alias.objects.get(alias=ref): - vulnerabilities.add(alias.vulnerability) + if alias.vulnerability: + vulnerabilities.add(alias.vulnerability) except Alias.DoesNotExist: continue From 1e208cff3cc6820ee848da7f7cc02dcc7150f56b Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 25 Feb 2026 15:43:09 +0200 Subject: [PATCH 2/4] Add a test, and update Kev pipeline Signed-off-by: ziad hany --- vulnerabilities/pipelines/enhance_with_kev.py | 4 ++++ .../pipelines/test_enhance_with_exploitdb.py | 15 +++++++++++++++ .../tests/pipelines/test_enhance_with_kev.py | 15 +++++++++++++++ .../pipelines/test_enhance_with_metasploit.py | 14 ++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index 00fc72106..b46daa789 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -80,6 +80,10 @@ def add_vulnerability_exploit(kev_vul, logger): logger(f"No vulnerability found for aliases {cve_id}") return 0 + if not vulnerability: + logger(f"No vulnerability found for aliases {cve_id}") + return 0 + Exploit.objects.update_or_create( vulnerability=vulnerability, data_source="KEV", diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py b/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py index f54dad55d..1b47591ba 100644 --- a/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py @@ -45,3 +45,18 @@ def test_exploit_db_improver(mock_get): # Run Exploit-DB Improver again when there are matching aliases. improver.execute() assert Exploit.objects.count() == 1 + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_invalid_exploit_db_improver(mock_get): + mock_response = Mock(status_code=200) + with open(TEST_DATA, "r") as f: + mock_response.text = f.read() + mock_get.return_value = mock_response + + improver = ExploitDBImproverPipeline() + Alias.objects.create(alias="CVE-2009-3699", vulnerability=None) + status, _ = improver.execute() + assert status == 0 + assert Exploit.objects.count() == 0 diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_kev.py b/vulnerabilities/tests/pipelines/test_enhance_with_kev.py index a93c16555..947d94a0a 100644 --- a/vulnerabilities/tests/pipelines/test_enhance_with_kev.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_kev.py @@ -45,3 +45,18 @@ def test_kev_improver(mock_get): # Run Kev Improver again when there are matching aliases. improver.execute() assert Exploit.objects.count() == 1 + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_invalid_kev_improver(mock_get): + mock_response = Mock(status_code=200) + mock_response.json.return_value = load_json(TEST_DATA) + mock_get.return_value = mock_response + + improver = VulnerabilityKevPipeline() + Alias.objects.create(alias="CVE-2021-38647", vulnerability=None) + + status, _ = improver.execute() + assert status == 0 + assert Exploit.objects.count() == 0 diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py b/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py index eea99e0ca..a4f4dae83 100644 --- a/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py @@ -42,3 +42,17 @@ def test_metasploit_improver(mock_get): # Run metasploit Improver again when there are matching aliases. improver.execute() assert Exploit.objects.count() == 1 + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_invalid_metasploit_improver(mock_get): + mock_response = Mock(status_code=200) + mock_response.json.return_value = load_json(TEST_DATA) + mock_get.return_value = mock_response + + Alias.objects.create(alias="CVE-2007-4387", vulnerability=None) # Alias without vulnerability + improver = MetasploitImproverPipeline() + status, _ = improver.execute() + assert status == 0 + assert Exploit.objects.count() == 0 From de3ecf6f76d73956e8b20702a1c93eca78ef321f Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 27 Feb 2026 22:02:40 +0200 Subject: [PATCH 3/4] Update (exploitdb, kev, metasploit ) pipelines to do single db query Signed-off-by: ziad hany --- .../pipelines/enhance_with_exploitdb.py | 20 ++++----- vulnerabilities/pipelines/enhance_with_kev.py | 44 +++++++++---------- .../pipelines/enhance_with_metasploit.py | 15 +++---- 3 files changed, 35 insertions(+), 44 deletions(-) diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py index 74fc55b09..70f7b4886 100644 --- a/vulnerabilities/pipelines/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -78,20 +78,16 @@ def add_exploit(self): def add_vulnerability_exploit(row, logger): - vulnerabilities = set() - aliases = row["codes"].split(";") if row["codes"] else [] if not aliases: return 0 - for raw_alias in aliases: - try: - if alias := Alias.objects.get(alias=raw_alias): - if alias.vulnerability: - vulnerabilities.add(alias.vulnerability) - except Alias.DoesNotExist: - continue + vulnerabilities = ( + Alias.objects.filter(alias__in=aliases, vulnerability__isnull=False) + .values_list("vulnerability_id", flat=True) + .distinct() + ) if not vulnerabilities: logger(f"No vulnerability found for aliases {aliases}") @@ -105,7 +101,7 @@ def add_vulnerability_exploit(row, logger): add_exploit_references(row["codes"], row["source_url"], row["file"], vulnerability, logger) try: Exploit.objects.update_or_create( - vulnerability=vulnerability, + vulnerability_id=vulnerability, data_source="Exploit-DB", defaults={ "date_added": date_added, @@ -126,7 +122,7 @@ def add_vulnerability_exploit(row, logger): return 1 -def add_exploit_references(ref_id, direct_url, path, vul, logger): +def add_exploit_references(ref_id, direct_url, path, vul_id, logger): url_map = { "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", "direct_url": direct_url, @@ -145,7 +141,7 @@ def add_exploit_references(ref_id, direct_url, path, vul, logger): if created: VulnerabilityRelatedReference.objects.get_or_create( - vulnerability=vul, + vulnerability_id=vul_id, reference=ref, ) diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index b46daa789..b9b0a84f4 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -71,31 +71,29 @@ def add_vulnerability_exploit(kev_vul, logger): if not cve_id: return 0 - vulnerability = None - try: - if alias := Alias.objects.get(alias=cve_id): - if alias.vulnerability: - vulnerability = alias.vulnerability - except Alias.DoesNotExist: - logger(f"No vulnerability found for aliases {cve_id}") - return 0 + vulnerabilities = ( + Alias.objects.filter(alias=cve_id, vulnerability__isnull=False) + .values_list("vulnerability", flat=True) + .distinct() + ) - if not vulnerability: + if not vulnerabilities: logger(f"No vulnerability found for aliases {cve_id}") return 0 - Exploit.objects.update_or_create( - vulnerability=vulnerability, - data_source="KEV", - defaults={ - "description": kev_vul["shortDescription"], - "date_added": kev_vul["dateAdded"], - "required_action": kev_vul["requiredAction"], - "due_date": kev_vul["dueDate"], - "notes": kev_vul["notes"], - "known_ransomware_campaign_use": True - if kev_vul["knownRansomwareCampaignUse"] == "Known" - else False, - }, - ) + for vulnerability in vulnerabilities: + Exploit.objects.update_or_create( + vulnerability_id=vulnerability, + data_source="KEV", + defaults={ + "description": kev_vul["shortDescription"], + "date_added": kev_vul["dateAdded"], + "required_action": kev_vul["requiredAction"], + "due_date": kev_vul["dueDate"], + "notes": kev_vul["notes"], + "known_ransomware_campaign_use": True + if kev_vul["knownRansomwareCampaignUse"] == "Known" + else False, + }, + ) return 1 diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py index 12437fadc..7e28160f9 100644 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -66,7 +66,6 @@ def add_vulnerability_exploits(self): def add_vulnerability_exploit(record, logger): - vulnerabilities = set() references = record.get("references", []) interesting_references = [ @@ -76,13 +75,11 @@ def add_vulnerability_exploit(record, logger): if not interesting_references: return 0 - for ref in interesting_references: - try: - if alias := Alias.objects.get(alias=ref): - if alias.vulnerability: - vulnerabilities.add(alias.vulnerability) - except Alias.DoesNotExist: - continue + vulnerabilities = ( + Alias.objects.filter(alias__in=interesting_references, vulnerability__isnull=False) + .values_list("vulnerability", flat=True) + .distinct() + ) if not vulnerabilities: logger(f"No vulnerability found for aliases {interesting_references}") @@ -108,7 +105,7 @@ def add_vulnerability_exploit(record, logger): for vulnerability in vulnerabilities: Exploit.objects.update_or_create( - vulnerability=vulnerability, + vulnerability_id=vulnerability, data_source="Metasploit", defaults={ "description": description, From af00938c0b9f4165cc6154f2362abd922af9de16 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 27 Feb 2026 22:07:31 +0200 Subject: [PATCH 4/4] Fix a vulnrichment importer test to correctly mock a CVSSv4 score Signed-off-by: ziad hany --- .../test_vulnrichment_importer_v2.py | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py index 1fb6f190e..f5c251e7f 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py @@ -8,15 +8,16 @@ # import json -from pathlib import Path from unittest.mock import MagicMock from unittest.mock import patch import pytest from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines.v2_importers.vulnrichment_importer import VulnrichImporterPipeline +from vulnerabilities.severity_systems import Cvssv4ScoringSystem @pytest.fixture @@ -58,8 +59,10 @@ def mock_pathlib(tmp_path): "metrics": [ { "cvssV4_0": { - "baseScore": 7.5, - "vectorString": "AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + "version": "4.0", + "baseScore": 5.3, + "baseSeverity": "MEDIUM", + "vectorString": "CVSS:4.0/AV:N/AC:L/AT:N/PR:L/UI:N/VC:L/VI:L/VA:L/SC:N/SI:N/SA:N", } } ], @@ -103,15 +106,20 @@ def test_collect_advisories(mock_pathlib, mock_vcs_response, mock_fetch_via_vcs) mock_parse.return_value = AdvisoryDataV2( advisory_id="CVE-2021-1234", summary="Sample PyPI vulnerability", - references=[{"url": "https://example.com"}], + references=[ReferenceV2(url="https://example.com")], affected_packages=[], weaknesses=[], url="https://example.com", severities=[ VulnerabilitySeverity( - system="cvssv4", - value=7.5, - scoring_elements="AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + system=Cvssv4ScoringSystem( + identifier="cvssv4", + name="CVSSv4 Base Score", + url="https://www.first.org/cvss/v4-0/", + notes="CVSSv4 base score and vector", + ), + value="5.3", + scoring_elements="CVSS:4.0/AV:N/AC:L/AT:N/PR:L/UI:N/VC:L/VI:L/VA:L/SC:N/SI:N/SA:N", ) ], ) @@ -126,6 +134,7 @@ def test_collect_advisories(mock_pathlib, mock_vcs_response, mock_fetch_via_vcs) assert advisory.advisory_id == "CVE-2021-1234" assert advisory.summary == "Sample PyPI vulnerability" assert advisory.url == "https://example.com" + assert len(advisory.severities) == 1 def test_clean_downloads(mock_vcs_response, mock_fetch_via_vcs): @@ -165,8 +174,10 @@ def test_parse_cve_advisory(mock_pathlib, mock_vcs_response, mock_fetch_via_vcs) "metrics": [ { "cvssV4_0": { - "baseScore": 7.5, - "vectorString": "AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + "version": "4.0", + "baseScore": 5.3, + "baseSeverity": "MEDIUM", + "vectorString": "CVSS:4.0/AV:N/AC:L/AT:N/PR:L/UI:N/VC:L/VI:L/VA:L/SC:N/SI:N/SA:N", } } ], @@ -185,7 +196,7 @@ def test_parse_cve_advisory(mock_pathlib, mock_vcs_response, mock_fetch_via_vcs) assert advisory.summary == "Sample PyPI vulnerability" assert advisory.url == advisory_url assert len(advisory.severities) == 1 - assert advisory.severities[0].value == 7.5 + assert advisory.severities[0].value == 5.3 def test_collect_advisories_with_invalid_json(mock_pathlib, mock_vcs_response, mock_fetch_via_vcs):