Migrate Gentoo importer to advisory V2

vulnerabilities/importers/__init__.py

@@ -50,6 +50,7 @@
 )
 from vulnerabilities.pipelines.v2_importers import epss_importer_v2
 from vulnerabilities.pipelines.v2_importers import fireeye_importer_v2
+from vulnerabilities.pipelines.v2_importers import gentoo_importer as gentoo_importer_v2
 from vulnerabilities.pipelines.v2_importers import github_osv_importer as github_osv_importer_v2
 from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2
 from vulnerabilities.pipelines.v2_importers import istio_importer as istio_importer_v2
@@ -89,6 +90,7 @@
         aosp_importer_v2.AospImporterPipeline,
         ruby_importer_v2.RubyImporterPipeline,
         epss_importer_v2.EPSSImporterPipeline,
+        gentoo_importer_v2.GentooImporterPipeline,
         mattermost_importer_v2.MattermostImporterPipeline,
         nvd_importer.NVDImporterPipeline,
         github_importer.GitHubAPIImporterPipeline,

vulnerabilities/importers/gentoo.py

@@ -6,8 +6,7 @@
 # See https://github.com/aboutcode-org/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-
-
+import logging
 import re
 import xml.etree.ElementTree as ET
 from pathlib import Path
@@ -17,12 +16,15 @@
 from univers.version_constraint import VersionConstraint
 from univers.version_range import EbuildVersionRange
 from univers.versions import GentooVersion
+from univers.versions import InvalidVersion
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 
+logger = logging.getLogger(__name__)
+
 
 class GentooImporter(Importer):
     repo_url = "git+https://anongit.gentoo.org/git/data/glsa.git"
@@ -104,14 +106,20 @@ def affected_and_safe_purls(affected_elem):
             safe_versions, affected_versions = GentooImporter.get_safe_and_affected_versions(pkg)
 
             for version in safe_versions:
-                constraints.append(
-                    VersionConstraint(version=GentooVersion(version), comparator="=").invert()
-                )
+                try:
+                    constraints.append(
+                        VersionConstraint(version=GentooVersion(version), comparator="=").invert()
+                    )
+                except InvalidVersion as e:
+                    logger.error(f"Invalid safe_version {version} - error: {e}")
 
             for version in affected_versions:
-                constraints.append(
-                    VersionConstraint(version=GentooVersion(version), comparator="=")
-                )
+                try:
+                    constraints.append(
+                        VersionConstraint(version=GentooVersion(version), comparator="=")
+                    )
+                except InvalidVersion as e:
+                    logger.error(f"Invalid affected_version {version} - error: {e}")
 
             if not constraints:
                 continue

vulnerabilities/pipelines/v2_importers/gentoo_importer.py

@@ -0,0 +1,193 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import re
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Iterable
+
+from fetchcode.vcs import fetch_via_vcs
+from packageurl import PackageURL
+from univers.version_constraint import VersionConstraint
+from univers.version_range import EbuildVersionRange
+from univers.versions import GentooVersion
+from univers.versions import InvalidVersion
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AffectedPackageV2
+from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.management.commands.commit_export import logger
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.severity_systems import GENERIC
+
+
+class GentooImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
+    repo_url = "git+https://anongit.gentoo.org/git/data/glsa.git"
+    spdx_license_expression = "CC-BY-SA-4.0"
+    # the license notice is at this url https://anongit.gentoo.org/ says:
+    # The contents of this document, unless otherwise expressly stated, are licensed
+    # under the [CC-BY-SA-4.0](https://creativecommons.org/licenses/by-sa/4.0/) license.
+    license_url = "https://creativecommons.org/licenses/by-sa/4.0/"
+    pipeline_id = "gentoo_importer_v2"
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.clone,
+            cls.collect_and_store_advisories,
+            cls.clean_downloads,
+        )
+
+    def clone(self):
+        self.log(f"Cloning `{self.repo_url}`")
+        self.vcs_response = fetch_via_vcs(self.repo_url)
+
+    def advisories_count(self):
+        advisory_dir = Path(self.vcs_response.dest_dir)
+        return sum(1 for _ in advisory_dir.rglob("*.xml"))
+
+    def collect_advisories(self) -> Iterable[AdvisoryData]:
+        base_path = Path(self.vcs_response.dest_dir)
+        for file_path in base_path.glob("**/*.xml"):
+            yield from self.process_file(file_path)
+
+    def process_file(self, file):
+        cves = []
+        summary = ""
+        xml_root = ET.parse(file).getroot()
+        id = xml_root.attrib.get("id")
+        glsa = "GLSA-" + id
+        vuln_references = [
+            ReferenceV2(
+                reference_id=glsa,
+                url=f"https://security.gentoo.org/glsa/{id}",
+            )
+        ]
+
+        severities = []
+        affected_packages = []
+        for child in xml_root:
+            if child.tag == "references":
+                cves = self.cves_from_reference(child)
+
+            if child.tag == "synopsis":
+                summary = child.text
+
+            if child.tag == "affected":
+                affected_packages = []
+                seen_packages = set()
+
+                for purl, constraint in get_affected_and_safe_purls(child):
+                    signature = (purl.to_string(), str(constraint))
+
+                    if signature not in seen_packages:
+                        seen_packages.add(signature)
+
+                        affected_package = AffectedPackageV2(
+                            package=purl,
+                            affected_version_range=EbuildVersionRange(constraints=[constraint]),
+                            fixed_version_range=None,
+                        )
+                        affected_packages.append(affected_package)
+
+            if child.tag == "impact":
+                severity_value = child.attrib.get("type")
+                if severity_value:
+                    severities.append(VulnerabilitySeverity(system=GENERIC, value=severity_value))
+
+        yield AdvisoryData(
+            advisory_id=glsa,
+            aliases=cves,
+            summary=summary,
+            references_v2=vuln_references,
+            severities=severities,
+            affected_packages=affected_packages,
+            url=f"https://security.gentoo.org/glsa/{id}"
+            if id
+            else "https://security.gentoo.org/glsa",
+            original_advisory_text=file,
+        )
+
+    def clean_downloads(self):
+        if self.vcs_response:
+            self.log("Removing cloned repository")
+            self.vcs_response.delete()
+
+    def on_failure(self):
+        self.clean_downloads()
+
+    @staticmethod
+    def cves_from_reference(reference):
+        cves = []
+        for child in reference:
+            txt = child.text.strip()
+            match = re.match(r"CVE-\d{4}-\d{4,}", txt)
+            if match:
+                cves.append(match.group())
+        return cves
+
+
+def extract_purls_and_constraints(pkg_name, pkg_ns, constraints, invert):
+    for comparator, version, slot_value in constraints:
+        qualifiers = {"slot": slot_value} if slot_value else {}
+        purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns, qualifiers=qualifiers)
+
+        try:
+            constraint = VersionConstraint(version=GentooVersion(version), comparator=comparator)
+
+            if invert:
+                constraint = constraint.invert()
+
+            yield purl, constraint
+        except InvalidVersion as e:
+            logger.error(f"InvalidVersion constraints version: {version} error:{e}")
+
+
+def get_affected_and_safe_purls(affected_elem):
+    for pkg in affected_elem:
+        name = pkg.attrib.get("name")
+        if not name:
+            continue
+        pkg_ns, _, pkg_name = name.rpartition("/")
+
+        safe_constraints, affected_constraints = get_safe_and_affected_constraints(pkg)
+
+        yield from extract_purls_and_constraints(
+            pkg_name, pkg_ns, affected_constraints, invert=False
+        )
+        yield from extract_purls_and_constraints(pkg_name, pkg_ns, safe_constraints, invert=True)
+
+
+def get_safe_and_affected_constraints(pkg):
+    # TODO : Revisit why we are skipping some versions in gentoo importer
+    skip_versions = {"1.3*", "7.3*", "7.4*"}
+    safe_versions = set()
+    affected_versions = set()
+    for info in pkg:
+        if info.text in skip_versions:
+            continue
+
+        # All possible values of info.attrib['range'] =
+        # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'}, out of
+        # which ('rle', 'rge', 'rgt') are ignored, because they compare
+        # 'release' not the 'version'.
+        range_value = info.attrib.get("range")
+        slot_value = info.attrib.get("slot")
+        comparator_dict = {"gt": ">", "lt": "<", "ge": ">=", "le": "<=", "eq": "="}
+        comparator = comparator_dict.get(range_value)
+        if not comparator:
+            continue
+
+        if info.tag == "unaffected":
+            safe_versions.add((comparator, info.text, slot_value))
+
+        elif info.tag == "vulnerable":
+            affected_versions.add((comparator, info.text, slot_value))
+    return safe_versions, affected_versions

vulnerabilities/tests/pipelines/v2_importers/test_gentoo_importer_v2.py

@@ -0,0 +1,41 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import json
+from pathlib import Path
+from unittest.mock import Mock
+from unittest.mock import patch
+
+import pytest
+
+from vulnerabilities.pipelines.v2_importers.gentoo_importer import GentooImporterPipeline
+from vulnerabilities.tests import util_tests
+
+TEST_DATA = Path(__file__).parent.parent.parent / "test_data" / "gentoo_v2"
+
+TEST_CVE_FILES = [
+    TEST_DATA / "glsa-201709-09.xml",
+    TEST_DATA / "glsa-202511-02.xml",
+    TEST_DATA / "glsa-202512-01.xml",
+]
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize("xml_file", TEST_CVE_FILES)
+def test_gentoo_advisories_per_file(xml_file):
+    pipeline = GentooImporterPipeline()
+    pipeline.vcs_response = Mock(dest_dir=TEST_DATA)
+
+    with patch.object(Path, "glob", return_value=[xml_file]):
+        results = [adv.to_dict() for adv in pipeline.collect_advisories()]
+
+    for adv in results:
+        adv["affected_packages"].sort(key=lambda x: json.dumps(x, sort_keys=True))
+
+    expected_file = xml_file.with_name(xml_file.stem + "-expected.json")
+    util_tests.check_results_against_json(results, expected_file)

vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json

@@ -0,0 +1,57 @@
+[
+  {
+    "advisory_id": "GLSA-201709-09",
+    "aliases": [
+      "CVE-2017-9800"
+    ],
+    "summary": "A command injection vulnerability in Subversion may allow remote\n    attackers to execute arbitrary code.",
+    "affected_packages": [
+      {
+        "package": {
+          "type": "ebuild",
+          "namespace": "dev-vcs",
+          "name": "subversion",
+          "version": "",
+          "qualifiers": "",
+          "subpath": ""
+        },
+        "affected_version_range": "vers:ebuild/0.1.1",
+        "fixed_version_range": null,
+        "introduced_by_commit_patches": [],
+        "fixed_by_commit_patches": []
+      },
+      {
+        "package": {
+          "type": "ebuild",
+          "namespace": "dev-vcs",
+          "name": "subversion",
+          "version": "",
+          "qualifiers": "",
+          "subpath": ""
+        },
+        "affected_version_range": "vers:ebuild/<1.9.7",
+        "fixed_version_range": null,
+        "introduced_by_commit_patches": [],
+        "fixed_by_commit_patches": []
+      }
+    ],
+    "references_v2": [
+      {
+        "reference_id": "GLSA-201709-09",
+        "reference_type": "",
+        "url": "https://security.gentoo.org/glsa/201709-09"
+      }
+    ],
+    "patches": [],
+    "severities": [
+      {
+        "system": "generic_textual",
+        "value": "normal",
+        "scoring_elements": ""
+      }
+    ],
+    "date_published": null,
+    "weaknesses": [],
+    "url": "https://security.gentoo.org/glsa/201709-09"
+  }
+]