diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index d3c48b6e25..c5803a73c6 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -64,6 +64,7 @@ cargo.CargoLockHandler, cargo.CargoTomlHandler, + cargo.CargoDenyHandler, chef.ChefMetadataJsonHandler, chef.ChefMetadataRbHandler, diff --git a/src/packagedcode/cargo.py b/src/packagedcode/cargo.py index 3b2d342d82..0ae08f3f1f 100644 --- a/src/packagedcode/cargo.py +++ b/src/packagedcode/cargo.py @@ -427,3 +427,119 @@ def parse_person(person): email = email.strip('<> ') return name, email + + +class CargoDenyHandler(models.NonAssemblableDatafileHandler): + datasource_id = 'cargo_deny' + path_patterns = ('*/deny.toml', '*deny.toml',) + default_package_type = 'cargo' + description = 'Cargo deny.toml license policy file' + documentation_url = 'https://embarkstudios.github.io/cargo-deny/' + + @classmethod + def parse(cls, location, package_only=False): + try: + with open(location, "rb") as fp: + cargo_deny = tomllib.load(fp) + except (OSError, tomllib.TOMLDecodeError): + return + + if not isinstance(cargo_deny, dict): + return + + licenses = cargo_deny.get('licenses') or {} + if not isinstance(licenses, dict): + licenses = {} + + allowed_licenses = licenses.get('allow') or [] + if not isinstance(allowed_licenses, list): + allowed_licenses = [allowed_licenses] + + denied_licenses = licenses.get('deny') or [] + if not isinstance(denied_licenses, list): + denied_licenses = [denied_licenses] + + clarify = licenses.get('clarify') or [] + if not isinstance(clarify, list): + clarify = [] + + license_clarifications = [] + for c in clarify: + if not isinstance(c, dict): + continue + license_clarifications.append({ + 'name': c.get('name'), + 'expression': c.get('expression'), + 'license-files': c.get('license-files', []) + }) + + exceptions = licenses.get('exceptions') or [] + if not isinstance(exceptions, list): + exceptions = [] + + license_exceptions = [] + for e in exceptions: + if not isinstance(e, dict): + continue + license_exceptions.append({ + 'name': e.get('name'), + 'version': e.get('version'), + 'allow': e.get('allow', []) + }) + + bans = cargo_deny.get('bans') or {} + if not isinstance(bans, dict): + bans = {} + + ban_deny = bans.get('deny') or [] + if not isinstance(ban_deny, list): + ban_deny = [] + + dependencies = [] + for ban in ban_deny: + if isinstance(ban, str): + name = ban + version = '*' + elif isinstance(ban, dict): + name = ban.get('name') + version = ban.get('version', '*') + else: + continue + + if not name: + continue + + purl = PackageURL(type='cargo', name=name).to_string() + dependencies.append( + models.DependentPackage( + purl=purl, + extracted_requirement=version, + scope='deny', + is_runtime=False, + is_optional=False, + ) + ) + + advisories = cargo_deny.get('advisories') or {} + if not isinstance(advisories, dict): + advisories = {} + + ignored_advisories = advisories.get('ignore') or [] + if not isinstance(ignored_advisories, list): + ignored_advisories = [ignored_advisories] + + extra_data = { + 'allowed_licenses': allowed_licenses, + 'denied_licenses': denied_licenses, + 'license_clarifications': license_clarifications, + 'license_exceptions': license_exceptions, + 'ignored_advisories': ignored_advisories, + } + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + dependencies=dependencies, + extra_data=extra_data, + ) + yield models.PackageData.from_data(package_data, package_only) diff --git a/tests/packagedcode/data/cargo/deny_toml/deny.toml b/tests/packagedcode/data/cargo/deny_toml/deny.toml new file mode 100644 index 0000000000..5a8dd9bad9 --- /dev/null +++ b/tests/packagedcode/data/cargo/deny_toml/deny.toml @@ -0,0 +1,20 @@ +[licenses] +allow = ["MIT", "Apache-2.0"] +deny = ["GPL-2.0"] + +[[licenses.clarify]] +name = "ring" +expression = "MIT AND ISC AND OpenSSL" + +[[licenses.exceptions]] +allow = ["LicenseRef-special"] +name = "special-crate" +version = "*" + +[bans] +deny = [ + { name = "openssl", version = "*" } +] + +[advisories] +ignore = ["RUSTSEC-2020-0001"] diff --git a/tests/packagedcode/data/cargo/deny_toml/egui-deny.toml b/tests/packagedcode/data/cargo/deny_toml/egui-deny.toml new file mode 100644 index 0000000000..2f8ac05198 --- /dev/null +++ b/tests/packagedcode/data/cargo/deny_toml/egui-deny.toml @@ -0,0 +1,109 @@ +# Copied from https://github.com/rerun-io/rerun_template +# +# https://github.com/EmbarkStudios/cargo-deny +# +# cargo-deny checks our dependency tree for copy-left licenses, +# duplicate dependencies, and rustsec advisories (https://rustsec.org/advisories). +# +# Install: `cargo install cargo-deny` +# Check: `cargo deny check`. + + +# Note: running just `cargo deny check` without a `--target` can result in +# false positives due to https://github.com/EmbarkStudios/cargo-deny/issues/324 +[graph] +targets = [ + { triple = "aarch64-apple-darwin" }, + { triple = "i686-pc-windows-gnu" }, + { triple = "i686-pc-windows-msvc" }, + { triple = "i686-unknown-linux-gnu" }, + { triple = "wasm32-unknown-unknown" }, + { triple = "x86_64-apple-darwin" }, + { triple = "x86_64-pc-windows-gnu" }, + { triple = "x86_64-pc-windows-msvc" }, + { triple = "x86_64-unknown-linux-gnu" }, + { triple = "x86_64-unknown-linux-musl" }, + { triple = "x86_64-unknown-redox" }, +] +all-features = true + + +[advisories] +version = 2 +ignore = [ + "RUSTSEC-2024-0320", # unmaintaines yaml-rust pulled in by syntect +] + +[bans] +multiple-versions = "deny" +wildcards = "deny" +deny = [ + { name = "cmake", reason = "It has hurt me too much" }, + { name = "openssl-sys", reason = "Use rustls" }, + { name = "openssl", reason = "Use rustls" }, +] + +skip = [ + { name = "base64" }, # Pretty small + { name = "bit-set" }, # wgpu's naga depends on 0.8, syntect's (used by egui_extras) fancy-regex depends on 0.5 + { name = "bit-vec" }, # dependency of bit-set in turn, different between 0.6 and 0.5 + { name = "bitflags" }, # old 1.0 version via glutin, png, spirv, … + { name = "cfg_aliases" }, # old version via wgpu + { name = "event-listener" }, # TODO(emilk): rustls pulls in two versions of this 😭 + { name = "futures-lite" }, # old version via accesskit_unix and zbus + { name = "memoffset" }, # tiny dependency + { name = "ndk-sys" }, # old version via wgpu, winit uses newer version + { name = "quick-xml" }, # old version via wayland-scanner + { name = "redox_syscall" }, # old version via winit + { name = "time" }, # old version pulled in by unmaintianed crate 'chrono' + { name = "windows-core" }, # Chrono pulls in 0.51, accesskit uses 0.58.0 + { name = "windows-sys" }, # glutin pulls in 0.52.0, accesskit pulls in 0.59.0, rfd pulls 0.48, webbrowser pulls 0.45.0 (via jni) +] +skip-tree = [ + { name = "criterion" }, # dev-dependency + { name = "foreign-types" }, # small crate. Old version via core-graphics (winit). + { name = "rfd" }, # example dependency +] + + +[licenses] +version = 2 +private = { ignore = true } +confidence-threshold = 0.93 # We want really high confidence when inferring licenses from text +allow = [ + "Apache-2.0 WITH LLVM-exception", # https://spdx.org/licenses/LLVM-exception.html + "Apache-2.0", # https://tldrlegal.com/license/apache-license-2.0-(apache-2.0) + "BSD-2-Clause", # https://tldrlegal.com/license/bsd-2-clause-license-(freebsd) + "BSD-3-Clause", # https://tldrlegal.com/license/bsd-3-clause-license-(revised) + "BSL-1.0", # https://tldrlegal.com/license/boost-software-license-1.0-explained + "CC0-1.0", # https://creativecommons.org/publicdomain/zero/1.0/ + "ISC", # https://www.tldrlegal.com/license/isc-license + "LicenseRef-UFL-1.0", # no official SPDX, see https://github.com/emilk/egui/issues/2321 + "MIT-0", # https://choosealicense.com/licenses/mit-0/ + "MIT", # https://tldrlegal.com/license/mit-license + "MPL-2.0", # https://www.mozilla.org/en-US/MPL/2.0/FAQ/ - see Q11. Used by webpki-roots on Linux. + "OFL-1.1", # https://spdx.org/licenses/OFL-1.1.html + "OpenSSL", # https://www.openssl.org/source/license.html - used on Linux + "Unicode-DFS-2016", # https://spdx.org/licenses/Unicode-DFS-2016.html + "Zlib", # https://tldrlegal.com/license/zlib-libpng-license-(zlib) +] +exceptions = [] + +[[licenses.clarify]] +name = "webpki" +expression = "ISC" +license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] + +[[licenses.clarify]] +name = "ring" +expression = "MIT AND ISC AND OpenSSL" +license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }] + + +[sources] +unknown-registry = "deny" +unknown-git = "deny" + +allow-git = [ + "https://github.com/rerun-io/kittest", # TODO(lucasmerlin): remove this once the kittest crate is published" +] diff --git a/tests/packagedcode/data/cargo/deny_toml/empty-deny.toml b/tests/packagedcode/data/cargo/deny_toml/empty-deny.toml new file mode 100644 index 0000000000..dab0e09ff2 --- /dev/null +++ b/tests/packagedcode/data/cargo/deny_toml/empty-deny.toml @@ -0,0 +1 @@ +# empty deny.toml with no sections diff --git a/tests/packagedcode/data/cargo/deny_toml/simple-deny.toml b/tests/packagedcode/data/cargo/deny_toml/simple-deny.toml new file mode 100644 index 0000000000..5a8dd9bad9 --- /dev/null +++ b/tests/packagedcode/data/cargo/deny_toml/simple-deny.toml @@ -0,0 +1,20 @@ +[licenses] +allow = ["MIT", "Apache-2.0"] +deny = ["GPL-2.0"] + +[[licenses.clarify]] +name = "ring" +expression = "MIT AND ISC AND OpenSSL" + +[[licenses.exceptions]] +allow = ["LicenseRef-special"] +name = "special-crate" +version = "*" + +[bans] +deny = [ + { name = "openssl", version = "*" } +] + +[advisories] +ignore = ["RUSTSEC-2020-0001"] diff --git a/tests/packagedcode/test_cargo_deny.py b/tests/packagedcode/test_cargo_deny.py new file mode 100644 index 0000000000..e79f5880ab --- /dev/null +++ b/tests/packagedcode/test_cargo_deny.py @@ -0,0 +1,90 @@ +import os + +from packages_test_utils import PackageTester +from packagedcode import cargo + + +class TestCargoDeny(PackageTester): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_cargo_deny_is_datafile(self): + test_file = self.get_test_loc('cargo/deny_toml/deny.toml') + assert cargo.CargoDenyHandler.is_datafile(test_file) + + def test_parse_simple_deny_toml(self): + test_file = self.get_test_loc('cargo/deny_toml/simple-deny.toml') + packages = list(cargo.CargoDenyHandler.parse(test_file)) + assert len(packages) == 1 + package = packages[0] + + extra = package.extra_data + assert 'MIT' in extra['allowed_licenses'] + assert 'Apache-2.0' in extra['allowed_licenses'] + assert 'GPL-2.0' in extra['denied_licenses'] + + clarifications = extra['license_clarifications'] + ring_clarity = next((c for c in clarifications if c['name'] == 'ring'), None) + assert ring_clarity is not None + assert ring_clarity['expression'] == 'MIT AND ISC AND OpenSSL' + + exceptions = extra['license_exceptions'] + special_exc = next((e for e in exceptions if e['name'] == 'special-crate'), None) + assert special_exc is not None + assert special_exc['allow'] == ['LicenseRef-special'] + + assert 'RUSTSEC-2020-0001' in extra['ignored_advisories'] + + deps = package.dependencies + openssl_dep = next((d for d in deps if d.purl == 'pkg:cargo/openssl'), None) + assert openssl_dep is not None + assert openssl_dep.scope == 'deny' + assert openssl_dep.extracted_requirement == '*' + + def test_parse_real_deny_toml(self): + test_file = self.get_test_loc('cargo/deny_toml/egui-deny.toml') + packages = list(cargo.CargoDenyHandler.parse(test_file)) + assert len(packages) == 1 + package = packages[0] + + extra = package.extra_data + assert 'allowed_licenses' in extra + assert 'denied_licenses' in extra + + def test_parse_empty_deny_toml(self): + test_file = self.get_test_loc('cargo/deny_toml/empty-deny.toml') + packages = list(cargo.CargoDenyHandler.parse(test_file)) + assert len(packages) == 1 + package = packages[0] + + extra = package.extra_data + assert extra['allowed_licenses'] == [] + assert extra['denied_licenses'] == [] + assert extra['license_clarifications'] == [] + assert extra['license_exceptions'] == [] + assert extra['ignored_advisories'] == [] + assert package.dependencies == [] + + def test_parse_missing_sections(self): + test_file = self.get_temp_file('missing-sections.toml') + with open(test_file, 'w') as f: + f.write('[licenses]\nallow = ["MIT"]\n') + + packages = list(cargo.CargoDenyHandler.parse(test_file)) + assert len(packages) == 1 + package = packages[0] + + extra = package.extra_data + assert extra['allowed_licenses'] == ['MIT'] + assert extra['denied_licenses'] == [] + assert extra['license_clarifications'] == [] + assert extra['license_exceptions'] == [] + assert extra['ignored_advisories'] == [] + assert package.dependencies == [] + + def test_parse_invalid_toml_returns_no_package_data(self): + test_file = self.get_temp_file('invalid-deny.toml') + with open(test_file, 'w') as f: + f.write('not toml [') + + packages = list(cargo.CargoDenyHandler.parse(test_file)) + assert packages == []