packagedcode: add CargoDenyHandler to parse cargo deny.toml files

src/packagedcode/__init__.py

@@ -64,6 +64,7 @@
 
     cargo.CargoLockHandler,
     cargo.CargoTomlHandler,
+    cargo.CargoDenyHandler,
 
     chef.ChefMetadataJsonHandler,
     chef.ChefMetadataRbHandler,

src/packagedcode/cargo.py

@@ -427,3 +427,119 @@ def parse_person(person):
         email = email.strip('<> ')
 
     return name, email
+
+
+class CargoDenyHandler(models.NonAssemblableDatafileHandler):
+    datasource_id = 'cargo_deny'
+    path_patterns = ('*/deny.toml', '*deny.toml',)
+    default_package_type = 'cargo'
+    description = 'Cargo deny.toml license policy file'
+    documentation_url = 'https://embarkstudios.github.io/cargo-deny/'
+
+    @classmethod
+    def parse(cls, location, package_only=False):
+        try:
+            with open(location, "rb") as fp:
+                cargo_deny = tomllib.load(fp)
+        except (OSError, tomllib.TOMLDecodeError):
+            return
+
+        if not isinstance(cargo_deny, dict):
+            return
+
+        licenses = cargo_deny.get('licenses') or {}
+        if not isinstance(licenses, dict):
+            licenses = {}
+
+        allowed_licenses = licenses.get('allow') or []
+        if not isinstance(allowed_licenses, list):
+            allowed_licenses = [allowed_licenses]
+
+        denied_licenses = licenses.get('deny') or []
+        if not isinstance(denied_licenses, list):
+            denied_licenses = [denied_licenses]
+
+        clarify = licenses.get('clarify') or []
+        if not isinstance(clarify, list):
+            clarify = []
+
+        license_clarifications = []
+        for c in clarify:
+            if not isinstance(c, dict):
+                continue
+            license_clarifications.append({
+                'name': c.get('name'),
+                'expression': c.get('expression'),
+                'license-files': c.get('license-files', [])
+            })
+
+        exceptions = licenses.get('exceptions') or []
+        if not isinstance(exceptions, list):
+            exceptions = []
+
+        license_exceptions = []
+        for e in exceptions:
+            if not isinstance(e, dict):
+                continue
+            license_exceptions.append({
+                'name': e.get('name'),
+                'version': e.get('version'),
+                'allow': e.get('allow', [])
+            })
+
+        bans = cargo_deny.get('bans') or {}
+        if not isinstance(bans, dict):
+            bans = {}
+
+        ban_deny = bans.get('deny') or []
+        if not isinstance(ban_deny, list):
+            ban_deny = []
+
+        dependencies = []
+        for ban in ban_deny:
+            if isinstance(ban, str):
+                name = ban
+                version = '*'
+            elif isinstance(ban, dict):
+                name = ban.get('name')
+                version = ban.get('version', '*')
+            else:
+                continue
+
+            if not name:
+                continue
+
+            purl = PackageURL(type='cargo', name=name).to_string()
+            dependencies.append(
+                models.DependentPackage(
+                    purl=purl,
+                    extracted_requirement=version,
+                    scope='deny',
+                    is_runtime=False,
+                    is_optional=False,
+                )
+            )
+
+        advisories = cargo_deny.get('advisories') or {}
+        if not isinstance(advisories, dict):
+            advisories = {}
+
+        ignored_advisories = advisories.get('ignore') or []
+        if not isinstance(ignored_advisories, list):
+            ignored_advisories = [ignored_advisories]
+
+        extra_data = {
+            'allowed_licenses': allowed_licenses,
+            'denied_licenses': denied_licenses,
+            'license_clarifications': license_clarifications,
+            'license_exceptions': license_exceptions,
+            'ignored_advisories': ignored_advisories,
+        }
+
+        package_data = dict(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
+            dependencies=dependencies,
+            extra_data=extra_data,
+        )
+        yield models.PackageData.from_data(package_data, package_only)

tests/packagedcode/data/cargo/deny_toml/deny.toml

@@ -0,0 +1,20 @@
+[licenses]
+allow = ["MIT", "Apache-2.0"]
+deny = ["GPL-2.0"]
+
+[[licenses.clarify]]
+name = "ring"
+expression = "MIT AND ISC AND OpenSSL"
+
+[[licenses.exceptions]]
+allow = ["LicenseRef-special"]
+name = "special-crate"
+version = "*"
+
+[bans]
+deny = [
+    { name = "openssl", version = "*" }
+]
+
+[advisories]
+ignore = ["RUSTSEC-2020-0001"]

tests/packagedcode/data/cargo/deny_toml/egui-deny.toml

@@ -0,0 +1,109 @@
+# Copied from https://github.com/rerun-io/rerun_template
+#
+# https://github.com/EmbarkStudios/cargo-deny
+#
+# cargo-deny checks our dependency tree for copy-left licenses,
+# duplicate dependencies, and rustsec advisories (https://rustsec.org/advisories).
+#
+# Install: `cargo install cargo-deny`
+# Check: `cargo deny check`.
+
+
+# Note: running just `cargo deny check` without a `--target` can result in
+# false positives due to https://github.com/EmbarkStudios/cargo-deny/issues/324
+[graph]
+targets = [
+  { triple = "aarch64-apple-darwin" },
+  { triple = "i686-pc-windows-gnu" },
+  { triple = "i686-pc-windows-msvc" },
+  { triple = "i686-unknown-linux-gnu" },
+  { triple = "wasm32-unknown-unknown" },
+  { triple = "x86_64-apple-darwin" },
+  { triple = "x86_64-pc-windows-gnu" },
+  { triple = "x86_64-pc-windows-msvc" },
+  { triple = "x86_64-unknown-linux-gnu" },
+  { triple = "x86_64-unknown-linux-musl" },
+  { triple = "x86_64-unknown-redox" },
+]
+all-features = true
+
+
+[advisories]
+version = 2
+ignore = [
+  "RUSTSEC-2024-0320", # unmaintaines yaml-rust pulled in by syntect
+]
+
+[bans]
+multiple-versions = "deny"
+wildcards = "deny"
+deny = [
+  { name = "cmake", reason = "It has hurt me too much" },
+  { name = "openssl-sys", reason = "Use rustls" },
+  { name = "openssl", reason = "Use rustls" },
+]
+
+skip = [
+  { name = "base64" },         # Pretty small
+  { name = "bit-set" },        # wgpu's naga depends on 0.8, syntect's (used by egui_extras) fancy-regex depends on 0.5
+  { name = "bit-vec" },        # dependency of bit-set in turn, different between 0.6 and 0.5
+  { name = "bitflags" },       # old 1.0 version via glutin, png, spirv, …
+  { name = "cfg_aliases" },    # old version via wgpu
+  { name = "event-listener" }, # TODO(emilk): rustls pulls in two versions of this 😭
+  { name = "futures-lite" },   # old version via accesskit_unix and zbus
+  { name = "memoffset" },      # tiny dependency
+  { name = "ndk-sys" },        # old version via wgpu, winit uses newer version
+  { name = "quick-xml" },      # old version via wayland-scanner
+  { name = "redox_syscall" },  # old version via winit
+  { name = "time" },           # old version pulled in by unmaintianed crate 'chrono'
+  { name = "windows-core" },   # Chrono pulls in 0.51, accesskit uses 0.58.0
+  { name = "windows-sys" },    # glutin pulls in 0.52.0, accesskit pulls in 0.59.0, rfd pulls 0.48, webbrowser pulls 0.45.0 (via jni)
+]
+skip-tree = [
+  { name = "criterion" },     # dev-dependency
+  { name = "foreign-types" }, # small crate. Old version via core-graphics (winit).
+  { name = "rfd" },           # example dependency
+]
+
+
+[licenses]
+version = 2
+private = { ignore = true }
+confidence-threshold = 0.93 # We want really high confidence when inferring licenses from text
+allow = [
+  "Apache-2.0 WITH LLVM-exception", # https://spdx.org/licenses/LLVM-exception.html
+  "Apache-2.0",                     # https://tldrlegal.com/license/apache-license-2.0-(apache-2.0)
+  "BSD-2-Clause",                   # https://tldrlegal.com/license/bsd-2-clause-license-(freebsd)
+  "BSD-3-Clause",                   # https://tldrlegal.com/license/bsd-3-clause-license-(revised)
+  "BSL-1.0",                        # https://tldrlegal.com/license/boost-software-license-1.0-explained
+  "CC0-1.0",                        # https://creativecommons.org/publicdomain/zero/1.0/
+  "ISC",                            # https://www.tldrlegal.com/license/isc-license
+  "LicenseRef-UFL-1.0",             # no official SPDX, see https://github.com/emilk/egui/issues/2321
+  "MIT-0",                          # https://choosealicense.com/licenses/mit-0/
+  "MIT",                            # https://tldrlegal.com/license/mit-license
+  "MPL-2.0",                        # https://www.mozilla.org/en-US/MPL/2.0/FAQ/ - see Q11. Used by webpki-roots on Linux.
+  "OFL-1.1",                        # https://spdx.org/licenses/OFL-1.1.html
+  "OpenSSL",                        # https://www.openssl.org/source/license.html - used on Linux
+  "Unicode-DFS-2016",               # https://spdx.org/licenses/Unicode-DFS-2016.html
+  "Zlib",                           # https://tldrlegal.com/license/zlib-libpng-license-(zlib)
+]
+exceptions = []
+
+[[licenses.clarify]]
+name = "webpki"
+expression = "ISC"
+license-files = [{ path = "LICENSE", hash = 0x001c7e6c }]
+
+[[licenses.clarify]]
+name = "ring"
+expression = "MIT AND ISC AND OpenSSL"
+license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }]
+
+
+[sources]
+unknown-registry = "deny"
+unknown-git = "deny"
+
+allow-git = [
+  "https://github.com/rerun-io/kittest", # TODO(lucasmerlin): remove this once the kittest crate is published"
+]

tests/packagedcode/data/cargo/deny_toml/empty-deny.toml

@@ -0,0 +1 @@
+# empty deny.toml with no sections

tests/packagedcode/data/cargo/deny_toml/simple-deny.toml

@@ -0,0 +1,20 @@
+[licenses]
+allow = ["MIT", "Apache-2.0"]
+deny = ["GPL-2.0"]
+
+[[licenses.clarify]]
+name = "ring"
+expression = "MIT AND ISC AND OpenSSL"
+
+[[licenses.exceptions]]
+allow = ["LicenseRef-special"]
+name = "special-crate"
+version = "*"
+
+[bans]
+deny = [
+    { name = "openssl", version = "*" }
+]
+
+[advisories]
+ignore = ["RUSTSEC-2020-0001"]

tests/packagedcode/test_cargo_deny.py

@@ -0,0 +1,90 @@
+import os
+
+from packages_test_utils import PackageTester
+from packagedcode import cargo
+
+
+class TestCargoDeny(PackageTester):
+    test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
+
+    def test_cargo_deny_is_datafile(self):
+        test_file = self.get_test_loc('cargo/deny_toml/deny.toml')
+        assert cargo.CargoDenyHandler.is_datafile(test_file)
+
+    def test_parse_simple_deny_toml(self):
+        test_file = self.get_test_loc('cargo/deny_toml/simple-deny.toml')
+        packages = list(cargo.CargoDenyHandler.parse(test_file))
+        assert len(packages) == 1
+        package = packages[0]
+
+        extra = package.extra_data
+        assert 'MIT' in extra['allowed_licenses']
+        assert 'Apache-2.0' in extra['allowed_licenses']
+        assert 'GPL-2.0' in extra['denied_licenses']
+
+        clarifications = extra['license_clarifications']
+        ring_clarity = next((c for c in clarifications if c['name'] == 'ring'), None)
+        assert ring_clarity is not None
+        assert ring_clarity['expression'] == 'MIT AND ISC AND OpenSSL'
+
+        exceptions = extra['license_exceptions']
+        special_exc = next((e for e in exceptions if e['name'] == 'special-crate'), None)
+        assert special_exc is not None
+        assert special_exc['allow'] == ['LicenseRef-special']
+
+        assert 'RUSTSEC-2020-0001' in extra['ignored_advisories']
+
+        deps = package.dependencies
+        openssl_dep = next((d for d in deps if d.purl == 'pkg:cargo/openssl'), None)
+        assert openssl_dep is not None
+        assert openssl_dep.scope == 'deny'
+        assert openssl_dep.extracted_requirement == '*'
+
+    def test_parse_real_deny_toml(self):
+        test_file = self.get_test_loc('cargo/deny_toml/egui-deny.toml')
+        packages = list(cargo.CargoDenyHandler.parse(test_file))
+        assert len(packages) == 1
+        package = packages[0]
+
+        extra = package.extra_data
+        assert 'allowed_licenses' in extra
+        assert 'denied_licenses' in extra
+
+    def test_parse_empty_deny_toml(self):
+        test_file = self.get_test_loc('cargo/deny_toml/empty-deny.toml')
+        packages = list(cargo.CargoDenyHandler.parse(test_file))
+        assert len(packages) == 1
+        package = packages[0]
+
+        extra = package.extra_data
+        assert extra['allowed_licenses'] == []
+        assert extra['denied_licenses'] == []
+        assert extra['license_clarifications'] == []
+        assert extra['license_exceptions'] == []
+        assert extra['ignored_advisories'] == []
+        assert package.dependencies == []
+
+    def test_parse_missing_sections(self):
+        test_file = self.get_temp_file('missing-sections.toml')
+        with open(test_file, 'w') as f:
+            f.write('[licenses]\nallow = ["MIT"]\n')
+
+        packages = list(cargo.CargoDenyHandler.parse(test_file))
+        assert len(packages) == 1
+        package = packages[0]
+
+        extra = package.extra_data
+        assert extra['allowed_licenses'] == ['MIT']
+        assert extra['denied_licenses'] == []
+        assert extra['license_clarifications'] == []
+        assert extra['license_exceptions'] == []
+        assert extra['ignored_advisories'] == []
+        assert package.dependencies == []
+
+    def test_parse_invalid_toml_returns_no_package_data(self):
+        test_file = self.get_temp_file('invalid-deny.toml')
+        with open(test_file, 'w') as f:
+            f.write('not toml [')
+
+        packages = list(cargo.CargoDenyHandler.parse(test_file))
+        assert packages == []