diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index d3c48b6e25..e7101446a8 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -49,75 +49,56 @@ # a handler classes MUST be added to this list to be active APPLICATION_PACKAGE_DATAFILE_HANDLERS = [ about.AboutFileHandler, - alpine.AlpineApkArchiveHandler, alpine.AlpineApkbuildHandler, - bower.BowerJsonHandler, - build_gradle.BuildGradleHandler, - build.AutotoolsConfigureHandler, build.BazelBuildHandler, + build.BazelModuleHandler, build.BuckMetadataBzlHandler, build.BuckPackageHandler, - cargo.CargoLockHandler, cargo.CargoTomlHandler, - chef.ChefMetadataJsonHandler, chef.ChefMetadataRbHandler, - cocoapods.PodspecHandler, cocoapods.PodspecJsonHandler, cocoapods.PodfileLockHandler, cocoapods.PodfileHandler, - conda.CondaMetaJsonHandler, conda.CondaMetaYamlHandler, conda.CondaYamlHandler, - conan.ConanFileHandler, conan.ConanDataHandler, - cran.CranDescriptionFileHandler, - debian_copyright.DebianCopyrightFileInPackageHandler, debian_copyright.StandaloneDebianCopyrightFileHandler, debian.DebianDscFileHandler, - debian.DebianControlFileInExtractedDebHandler, debian.DebianControlFileInSourceHandler, - debian.DebianDebPackageHandler, debian.DebianMd5sumFilelistInPackageHandler, - debian.DebianSourcePackageMetadataTarballHandler, debian.DebianSourcePackageTarballHandler, - distro.EtcOsReleaseHandler, - freebsd.CompactManifestHandler, - godeps.GodepsHandler, golang.GoModHandler, golang.GoSumHandler, - haxe.HaxelibJsonHandler, - maven.MavenPomXmlHandler, maven.MavenPomPropertiesHandler, maven.JavaJarManifestHandler, maven.JavaOSGiManifestHandler, - misc.AndroidAppArchiveHandler, misc.AndroidLibraryHandler, misc.AppleDmgHandler, - misc.Axis2MarArchiveHandler , - misc.Axis2MarModuleXmlHandler , + misc.Axis2MarArchiveHandler, + misc.Axis2MarModuleXmlHandler, misc.CabArchiveHandler, misc.ChromeExtensionHandler, - misc.CpanDistIniHandler , + misc.CpanDistIniHandler, misc.CpanMakefilePlHandler, misc.CpanManifestHandler, misc.CpanMetaJsonHandler, @@ -126,19 +107,14 @@ misc.IosAppIpaHandler, misc.IsoImageHandler, misc.IvyXmlHandler, - - misc.JavaEarAppXmlHandler , - misc.JavaEarHandler , - + misc.JavaEarAppXmlHandler, + misc.JavaEarHandler, # is this redundant with Jar manifest? misc.JavaJarHandler, - misc.JavaWarHandler, misc.JavaWarWebXmlHandler, - - misc.JBossSarHandler , - misc.JBossServiceXmlHandler , - + misc.JBossSarHandler, + misc.JBossServiceXmlHandler, misc.MeteorPackageHandler, misc.MozillaExtensionHandler, misc.NsisInstallerHandler, @@ -152,19 +128,14 @@ npm.PnpmShrinkwrapYamlHandler, npm.PnpmLockYamlHandler, npm.PnpmWorkspaceYamlHandler, - nuget.NugetNupkgHandler, nuget.NugetNuspecHandler, nuget.NugetPackagesLockHandler, - opam.OpamFileHandler, - phpcomposer.PhpComposerJsonHandler, phpcomposer.PhpComposerLockHandler, - pubspec.DartPubspecYamlHandler, pubspec.DartPubspecLockHandler, - pypi.PipfileHandler, pypi.PipfileLockHandler, pypi.PipRequirementsFileHandler, @@ -180,36 +151,26 @@ pypi.PythonSdistPkgInfoFile, pypi.PythonSetupPyHandler, pypi.SetupCfgHandler, - readme.ReadmeHandler, - rpm.RpmArchiveHandler, rpm.RpmSpecfileHandler, - rubygems.GemMetadataArchiveExtractedHandler, rubygems.GemArchiveHandler, - # the order of these handlers matter rubygems.GemfileInExtractedGemHandler, rubygems.GemfileHandler, - # the order of these handlers matter rubygems.GemfileLockInExtractedGemHandler, rubygems.GemfileLockHandler, - # the order of these handlers matter rubygems.GemspecInInstalledVendorBundleSpecificationsHandler, rubygems.GemspecInExtractedGemHandler, rubygems.GemspecHandler, - swift.SwiftManifestJsonHandler, swift.SwiftPackageResolvedHandler, swift.SwiftShowDependenciesDepLockHandler, - windows.MicrosoftUpdateManifestHandler, - win_pe.WindowsExecutableHandler, - # These are handlers for deplock generated files pypi.PipInspectDeplockHandler, ] @@ -221,18 +182,14 @@ SYSTEM_PACKAGE_DATAFILE_HANDLERS = [ alpine.AlpineInstalledDatabaseHandler, - debian_copyright.DebianCopyrightFileInPackageHandler, debian_copyright.DebianCopyrightFileInSourceHandler, - debian.DebianDistrolessInstalledDatabaseHandler, - debian.DebianInstalledFilelistHandler, debian.DebianInstalledMd5sumFilelistHandler, debian.DebianInstalledStatusDatabaseHandler, - rpm.RpmLicenseFilesHandler, - rpm.RpmMarinerContainerManifestHandler + rpm.RpmMarinerContainerManifestHandler, ] if on_linux: @@ -240,7 +197,6 @@ rpm.RpmInstalledBdbDatabaseHandler, rpm.RpmInstalledSqliteDatabaseHandler, rpm.RpmInstalledNdbDatabaseHandler, - win_reg.InstalledProgramFromDockerSoftwareDeltaHandler, win_reg.InstalledProgramFromDockerFilesSoftwareHandler, win_reg.InstalledProgramFromDockerUtilityvmSoftwareHandler, @@ -255,6 +211,7 @@ try: from go_inspector.binary import get_go_binary_handler + handler = get_go_binary_handler() PACKAGE_IN_COMPILED_DATAFILE_HANDLERS.append(handler) except ImportError: @@ -262,16 +219,20 @@ try: from rust_inspector.packages import get_rust_binary_handler + handler = get_rust_binary_handler() PACKAGE_IN_COMPILED_DATAFILE_HANDLERS.append(handler) except ImportError: pass ALL_DATAFILE_HANDLERS = ( - APPLICATION_PACKAGE_DATAFILE_HANDLERS + [ - p for p in SYSTEM_PACKAGE_DATAFILE_HANDLERS + APPLICATION_PACKAGE_DATAFILE_HANDLERS + + [ + p + for p in SYSTEM_PACKAGE_DATAFILE_HANDLERS if p not in APPLICATION_PACKAGE_DATAFILE_HANDLERS - ] + PACKAGE_IN_COMPILED_DATAFILE_HANDLERS + ] + + PACKAGE_IN_COMPILED_DATAFILE_HANDLERS ) # registry of all handler classes keyed by datasource_id @@ -294,6 +255,4 @@ def get_package_handler(package_data): return ppc -PACKAGE_DATA_CLASS_BY_DATASOURCE_ID = { - maven.MavenPackageData.datasource_id: maven.MavenPackageData -} +PACKAGE_DATA_CLASS_BY_DATASOURCE_ID = {maven.MavenPackageData.datasource_id: maven.MavenPackageData} diff --git a/src/packagedcode/build.py b/src/packagedcode/build.py index 8044f8c245..00b263c84c 100644 --- a/src/packagedcode/build.py +++ b/src/packagedcode/build.py @@ -11,6 +11,20 @@ import logging import ast from collections import defaultdict +import re + + +def _extract_starlark_kwarg(block, key): + """ + Extract a string value for a named keyword argument from a + Starlark (Bazel) function call body. + + Example block: 'name = "foo", version = "1.0"' + _extract_starlark_kwarg(block, 'name') -> 'foo' + """ + match = re.search(r"\b" + re.escape(key) + r'\s*=\s*["\']([^"\']+)["\']', block) + return match.group(1) if match else None + from commoncode import fileutils from packageurl import PackageURL @@ -28,7 +42,7 @@ Buck, Bazel, Pants, etc. """ -TRACE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False) +TRACE = os.environ.get("SCANCODE_DEBUG_PACKAGE", False) def logger_debug(*args): @@ -39,21 +53,23 @@ def logger_debug(*args): if TRACE: import sys + logging.basicConfig(stream=sys.stdout) logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug( - ' '.join(isinstance(a, str) and a or repr(a) for a in args) - ) + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) class AutotoolsConfigureHandler(models.NonAssemblableDatafileHandler): - datasource_id = 'autotools_configure' - path_patterns = ('*/configure', '*/configure.ac',) - default_package_type = 'autotools' - description = 'Autotools configure script' - documentation_url = 'https://www.gnu.org/software/automake/' + datasource_id = "autotools_configure" + path_patterns = ( + "*/configure", + "*/configure.ac", + ) + default_package_type = "autotools" + description = "Autotools configure script" + documentation_url = "https://www.gnu.org/software/automake/" @classmethod def parse(cls, location, package_only=False): @@ -77,8 +93,7 @@ def parse(cls, location, package_only=False): yield models.PackageData.from_data(package_data, package_only) - -def check_rule_name_ending(rule_name, starlark_rule_types=('binary', 'library')): +def check_rule_name_ending(rule_name, starlark_rule_types=("binary", "library")): """ Return True if `rule_name` ends with a rule type from `starlark_rule_types` Return False otherwise @@ -110,37 +125,38 @@ def assemble(cls, package_data, resource, codebase, package_adder): ) if TRACE: - logger_debug(f"BaseStarlarkManifestHandler.assemble: package_data: {package_data.to_dict()}") + logger_debug( + f"BaseStarlarkManifestHandler.assemble: package_data: {package_data.to_dict()}" + ) - package.license_detections, package.declared_license_expression = \ + package.license_detections, package.declared_license_expression = ( get_license_detections_and_expression( package=package_data, resource=resource, codebase=codebase, ) + ) if package.declared_license_expression: - package.declared_license_expression_spdx = str(build_spdx_license_expression( - license_expression=package.declared_license_expression, - licensing=get_cache().licensing, - )) + package.declared_license_expression_spdx = str( + build_spdx_license_expression( + license_expression=package.declared_license_expression, + licensing=get_cache().licensing, + ) + ) cls.assign_package_to_resources( - package=package, - resource=resource, - codebase=codebase, - package_adder=package_adder + package=package, resource=resource, codebase=codebase, package_adder=package_adder ) yield package - # we yield this as we do not want this further processed yield resource @classmethod def parse(cls, location, package_only=False): # Thanks to Starlark being a Python dialect, we can use `ast` to parse it - with open(location, 'rb') as f: + with open(location, "rb") as f: tree = ast.parse(f.read()) build_rules = defaultdict(list) @@ -170,8 +186,7 @@ def parse(cls, location, package_only=False): # We collect the elements of a list if the element is # not a function call args[arg_name] = [ - elt.value for elt in kw.value.elts - if not isinstance(elt, ast.Call) + elt.value for elt in kw.value.elts if not isinstance(elt, ast.Call) ] if args: build_rules[rule_name].append(args) @@ -179,13 +194,13 @@ def parse(cls, location, package_only=False): if build_rules: for rule_name, rule_instances_args in build_rules.items(): for args in rule_instances_args: - name = args.get('name') + name = args.get("name") # FIXME: we could still return partial package data if not name: continue - license_files = args.get('licenses') + license_files = args.get("licenses") if TRACE: logger_debug(f"build: parse: license_files: {license_files}") @@ -209,12 +224,14 @@ def parse(cls, location, package_only=False): package_data = dict( datasource_id=cls.datasource_id, type=cls.default_package_type, - name=fileutils.file_name(fileutils.parent_directory(location)) + name=fileutils.file_name(fileutils.parent_directory(location)), ) yield models.PackageData.from_data(package_data, package_only) @classmethod - def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name=None): + def assign_package_to_resources( + cls, package, resource, codebase, package_adder, skip_name=None + ): package_uid = package.package_uid if not package_uid: return @@ -258,8 +275,7 @@ def get_license_detections_and_expression(package, resource, codebase): if TRACE: logger_debug( - f"build: get_license_detections_and_expression:" - f"declared_licenses: {declared_licenses}" + f"build: get_license_detections_and_expression:declared_licenses: {declared_licenses}" ) logger_debug( f"build: get_license_detections_and_expression:" @@ -273,31 +289,28 @@ def get_license_detections_and_expression(package, resource, codebase): detections = detect_licenses(location=child.location) if TRACE: logger_debug( - f"build: get_license_detections_and_expression:" - f"detections: {detections}" + f"build: get_license_detections_and_expression:detections: {detections}" ) if not detections: - license_detections.append( - get_unknown_license_detection(declared_licenses) - ) + license_detections.append(get_unknown_license_detection(declared_licenses)) else: license_detections.extend(detections) - return get_mapping_and_expression_from_detections( - license_detections=license_detections - ) + return get_mapping_and_expression_from_detections(license_detections=license_detections) class BazelBuildHandler(BaseStarlarkManifestHandler): - datasource_id = 'bazel_build' - path_patterns = ('*/BUILD',) - default_package_type = 'bazel' - description = 'Bazel BUILD' - documentation_url = 'https://bazel.build/' + datasource_id = "bazel_build" + path_patterns = ("*/BUILD",) + default_package_type = "bazel" + description = "Bazel BUILD" + documentation_url = "https://bazel.build/" @classmethod - def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name='BUILD'): + def assign_package_to_resources( + cls, package, resource, codebase, package_adder, skip_name="BUILD" + ): return super().assign_package_to_resources( package=package, resource=resource, @@ -308,14 +321,16 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder, class BuckPackageHandler(BaseStarlarkManifestHandler): - datasource_id = 'buck_file' - path_patterns = ('*/BUCK',) - default_package_type = 'buck' - description = 'Buck file' - documentation_url = 'https://buck.build/' + datasource_id = "buck_file" + path_patterns = ("*/BUCK",) + default_package_type = "buck" + description = "Buck file" + documentation_url = "https://buck.build/" @classmethod - def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name='BUCK'): + def assign_package_to_resources( + cls, package, resource, codebase, package_adder, skip_name="BUCK" + ): return super().assign_package_to_resources( package=package, resource=resource, @@ -326,26 +341,26 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder, class BuckMetadataBzlHandler(BaseStarlarkManifestHandler): - datasource_id = 'buck_metadata' - path_patterns = ('*/METADATA.bzl',) - default_package_type = 'buck' - description = 'Buck metadata file' - documentation_url = 'https://buck.build/' + datasource_id = "buck_metadata" + path_patterns = ("*/METADATA.bzl",) + default_package_type = "buck" + description = "Buck metadata file" + documentation_url = "https://buck.build/" @classmethod def parse(cls, location, package_only=True): - with open(location, 'rb') as f: + with open(location, "rb") as f: tree = ast.parse(f.read()) metadata_fields = {} for statement in tree.body: - if not (hasattr(statement, 'targets') and isinstance(statement, ast.Assign)): + if not (hasattr(statement, "targets") and isinstance(statement, ast.Assign)): continue # We are looking for a dictionary assigned to the variable `METADATA` for target in statement.targets: - if not (target.id == 'METADATA' and isinstance(statement.value, ast.Dict)): + if not (target.id == "METADATA" and isinstance(statement.value, ast.Dict)): continue # Once we find the dictionary assignment, get and store its contents statement_keys = statement.value.keys @@ -365,61 +380,59 @@ def parse(cls, location, package_only=True): metadata_fields[key_name] = value parties = [] - maintainers = metadata_fields.get('maintainers', []) or [] + maintainers = metadata_fields.get("maintainers", []) or [] for maintainer in maintainers: parties.append( models.Party( type=models.party_org, name=maintainer, - role='maintainer', + role="maintainer", ) ) # TODO: Create function that determines package type from download URL, # then create a package of that package type from the metadata info - - if 'upstream_type' in metadata_fields: - package_type = metadata_fields['upstream_type'] - elif 'package_type' in metadata_fields: - package_type = metadata_fields['package_type'] + + if "upstream_type" in metadata_fields: + package_type = metadata_fields["upstream_type"] + elif "package_type" in metadata_fields: + package_type = metadata_fields["package_type"] else: package_type = cls.default_package_type - if 'licenses' in metadata_fields: - extracted_license_statement = metadata_fields['licenses'] + if "licenses" in metadata_fields: + extracted_license_statement = metadata_fields["licenses"] else: - extracted_license_statement = metadata_fields.get('license_expression') + extracted_license_statement = metadata_fields.get("license_expression") - if 'upstream_address' in metadata_fields: - homepage_url = metadata_fields['upstream_address'] + if "upstream_address" in metadata_fields: + homepage_url = metadata_fields["upstream_address"] else: - homepage_url = metadata_fields.get('homepage_url') - + homepage_url = metadata_fields.get("homepage_url") extra_data = {} - if 'vcs_commit_hash' in metadata_fields: - extra_data['vcs_commit_hash'] = metadata_fields['vcs_commit_hash'] - if 'upstream_hash' in metadata_fields: - extra_data['upstream_hash'] = metadata_fields['upstream_hash'] + if "vcs_commit_hash" in metadata_fields: + extra_data["vcs_commit_hash"] = metadata_fields["vcs_commit_hash"] + if "upstream_hash" in metadata_fields: + extra_data["upstream_hash"] = metadata_fields["upstream_hash"] package_data = dict( datasource_id=cls.datasource_id, type=package_type, - name=metadata_fields.get('name'), - version=metadata_fields.get('version'), + name=metadata_fields.get("name"), + version=metadata_fields.get("version"), extracted_license_statement=extracted_license_statement, parties=parties, homepage_url=homepage_url, - download_url=metadata_fields.get('download_url'), - vcs_url=metadata_fields.get('vcs_url'), - sha1=metadata_fields.get('download_archive_sha1'), - extra_data=extra_data + download_url=metadata_fields.get("download_url"), + vcs_url=metadata_fields.get("vcs_url"), + sha1=metadata_fields.get("download_archive_sha1"), + extra_data=extra_data, ) - if 'package_url' in metadata_fields: - package_data.update(PackageURL.from_string(metadata_fields['package_url']).to_dict()) - - yield models.PackageData.from_data(package_data, package_only=True) + if "package_url" in metadata_fields: + package_data.update(PackageURL.from_string(metadata_fields["package_url"]).to_dict()) + yield models.PackageData.from_data(package_data, package_only=True) @classmethod def assign_package_to_resources(cls, package, resource, codebase, package_adder): @@ -429,3 +442,71 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder) codebase=codebase, package_adder=package_adder, ) + + +class BazelModuleHandler(models.DatafileHandler): + """ + Handle Bazel MODULE.bazel module manifest files used by Bzlmod. + See: https://bazel.build/external/module + """ + + datasource_id = "bazel_module" + path_patterns = ("*/MODULE.bazel",) + default_package_type = "bazel" + description = "Bazel MODULE.bazel module manifest (Bzlmod)" + documentation_url = "https://bazel.build/external/module" + + @classmethod + def parse(cls, location, package_only=False): + with open(location, encoding="utf-8", errors="replace") as f: + content = f.read() + + # --- Extract module() declaration --- + name = None + version = None + module_match = re.search( + r"\bmodule\s*\(([^)]+)\)", + content, + re.DOTALL, + ) + if module_match: + block = module_match.group(1) + name = _extract_starlark_kwarg(block, "name") + version = _extract_starlark_kwarg(block, "version") + + # --- Extract bazel_dep() declarations --- + dependencies = [] + for dep_match in re.finditer( + r"\bbazel_dep\s*\(([^)]+)\)", + content, + re.DOTALL, + ): + block = dep_match.group(1) + dep_name = _extract_starlark_kwarg(block, "name") + dep_version = _extract_starlark_kwarg(block, "version") + is_dev = bool(re.search(r"\bdev_dependency\s*=\s*True\b", block)) + + if not dep_name: + continue + + purl = f"pkg:bazel/{dep_name}" + if dep_version: + purl = f"{purl}@{dep_version}" + + dependencies.append( + models.DependentPackage( + purl=purl, + extracted_requirement=dep_version, + scope="dev" if is_dev else "dependencies", + is_runtime=not is_dev, + is_optional=is_dev, + ) + ) + + yield models.PackageData( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + name=name, + version=version, + dependencies=dependencies, + ) diff --git a/tests/packagedcode/data/bazel_module/MODULE.bazel b/tests/packagedcode/data/bazel_module/MODULE.bazel new file mode 100644 index 0000000000..378b60b7fa --- /dev/null +++ b/tests/packagedcode/data/bazel_module/MODULE.bazel @@ -0,0 +1,11 @@ +# Sample Bazel module manifest +module( + name = "my_sample_project", + version = "0.5.0", + compatibility_level = 1, +) + +bazel_dep(name = "rules_python", version = "0.24.0") +bazel_dep(name = "rules_go", version = "0.41.0") +bazel_dep(name = "googletest", version = "1.14.0", dev_dependency = True) +bazel_dep(name = "abseil-cpp", version = "20230802.0", repo_name = "com_google_absl") diff --git a/tests/packagedcode/data/bazel_module/MODULE_no_version.bazel b/tests/packagedcode/data/bazel_module/MODULE_no_version.bazel new file mode 100644 index 0000000000..327ef2258e --- /dev/null +++ b/tests/packagedcode/data/bazel_module/MODULE_no_version.bazel @@ -0,0 +1,5 @@ +module( + name = "minimal_module", +) + +bazel_dep(name = "rules_python", version = "0.24.0") diff --git a/tests/packagedcode/test_build.py b/tests/packagedcode/test_build.py index 11b2c8cb5f..e32f33eab2 100644 --- a/tests/packagedcode/test_build.py +++ b/tests/packagedcode/test_build.py @@ -20,52 +20,51 @@ from scancode_config import REGEN_TEST_FIXTURES - class TestBuild(PackageTester): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data/build') + test_data_dir = os.path.join(os.path.dirname(__file__), "data/build") def test_end2end_scan_can_detect_bazel(self): - test_file = self.get_test_loc('bazel/end2end') - expected_file = self.get_test_loc('bazel/end2end-expected.json') - result_file = self.get_temp_file('results.json') - run_scan_click(['--package', test_file, '--json-pp', result_file]) + test_file = self.get_test_loc("bazel/end2end") + expected_file = self.get_test_loc("bazel/end2end-expected.json") + result_file = self.get_temp_file("results.json") + run_scan_click(["--package", test_file, "--json-pp", result_file]) check_json_scan(expected_file, result_file, regen=REGEN_TEST_FIXTURES) def test_end2end_scan_can_detect_buck(self): - test_file = self.get_test_loc('buck/end2end') - expected_file = self.get_test_loc('buck/end2end-expected.json') - result_file = self.get_temp_file('results.json') - run_scan_click(['--package', test_file, '--json-pp', result_file]) + test_file = self.get_test_loc("buck/end2end") + expected_file = self.get_test_loc("buck/end2end-expected.json") + result_file = self.get_temp_file("results.json") + run_scan_click(["--package", test_file, "--json-pp", result_file]) check_json_scan(expected_file, result_file, regen=REGEN_TEST_FIXTURES) def test_BazelPackage_parse(self): - test_file = self.get_test_loc('bazel/parse/BUILD') + test_file = self.get_test_loc("bazel/parse/BUILD") result_packages = build.BazelBuildHandler.parse(test_file) expected_packages = [ models.PackageData( - name='hello-greet', + name="hello-greet", type=build.BazelBuildHandler.default_package_type, datasource_id=build.BazelBuildHandler.datasource_id, ), models.PackageData( - name='hello-world', + name="hello-world", type=build.BazelBuildHandler.default_package_type, datasource_id=build.BazelBuildHandler.datasource_id, - ) + ), ] compare_package_results(expected_packages, result_packages) def test_BuckPackage_parse(self): - test_file = self.get_test_loc('buck/parse/BUCK') + test_file = self.get_test_loc("buck/parse/BUCK") result_packages = build.BuckPackageHandler.parse(test_file) expected_packages = [ models.PackageData( - name='app', + name="app", type=build.BuckPackageHandler.default_package_type, datasource_id=build.BuckPackageHandler.datasource_id, ), models.PackageData( - name='app2', + name="app2", type=build.BuckPackageHandler.default_package_type, datasource_id=build.BuckPackageHandler.datasource_id, ), @@ -73,81 +72,128 @@ def test_BuckPackage_parse(self): compare_package_results(expected_packages, result_packages) def test_BuckPackage_recognize_with_license(self): - test_file = self.get_test_loc('buck/parse/license/BUCK') - test_loc = self.get_test_loc('buck/parse/license/') + test_file = self.get_test_loc("buck/parse/license/BUCK") + test_loc = self.get_test_loc("buck/parse/license/") result_package = list(build.BuckPackageHandler.parse(test_file))[0] codebase = Codebase(test_loc) - resource = codebase.get_resource('license/BUCK') + resource = codebase.get_resource("license/BUCK") _detections, license_expression = build.get_license_detections_and_expression( result_package, resource, codebase ) - assert license_expression == 'apache-2.0' + assert license_expression == "apache-2.0" def test_MetadataBzl_parse(self): - test_file = self.get_test_loc('metadatabzl/METADATA.bzl') + test_file = self.get_test_loc("metadatabzl/METADATA.bzl") result_packages = build.BuckMetadataBzlHandler.parse(test_file, package_only=True) package_data = dict( datasource_id=build.BuckMetadataBzlHandler.datasource_id, - type='github', - name='example', - version='0.0.1', - extracted_license_statement=['BSD-3-Clause'], - parties=[ - models.Party( - type=models.party_org, - name='oss_foundation', - role='maintainer' - ) - ], - extra_data=dict(upstream_hash='deadbeef'), - homepage_url='https://github.com/example/example', + type="github", + name="example", + version="0.0.1", + extracted_license_statement=["BSD-3-Clause"], + parties=[models.Party(type=models.party_org, name="oss_foundation", role="maintainer")], + extra_data=dict(upstream_hash="deadbeef"), + homepage_url="https://github.com/example/example", ) - expected_packages = [models.PackageData.from_data(package_data=package_data, package_only=True)] + expected_packages = [ + models.PackageData.from_data(package_data=package_data, package_only=True) + ] compare_package_results(expected_packages, result_packages) - + def test_MetadataBzl_parse_with_package_url(self): - test_file = self.get_test_loc('metadatabzl/with-package-url/METADATA.bzl') + test_file = self.get_test_loc("metadatabzl/with-package-url/METADATA.bzl") result_packages = build.BuckMetadataBzlHandler.parse(test_file, package_only=True) package_data = dict( datasource_id=build.BuckMetadataBzlHandler.datasource_id, - name='animation', - namespace='androidx.compose.animation', - type='maven', - version='0.0.1', - extracted_license_statement=['BSD-3-Clause'], - parties=[ - models.Party( - type=models.party_org, - name='oss_foundation', - role='maintainer' - ) - ], - homepage_url='https://developer.android.com/jetpack/androidx/releases/compose-animation#0.0.1', + name="animation", + namespace="androidx.compose.animation", + type="maven", + version="0.0.1", + extracted_license_statement=["BSD-3-Clause"], + parties=[models.Party(type=models.party_org, name="oss_foundation", role="maintainer")], + homepage_url="https://developer.android.com/jetpack/androidx/releases/compose-animation#0.0.1", ) - expected_packages = [models.PackageData.from_data(package_data=package_data, package_only=True)] + expected_packages = [ + models.PackageData.from_data(package_data=package_data, package_only=True) + ] compare_package_results(expected_packages, result_packages) def test_MetadataBzl_recognize_new_format(self): - test_file = self.get_test_loc('metadatabzl/new-format/METADATA.bzl') + test_file = self.get_test_loc("metadatabzl/new-format/METADATA.bzl") result_packages = build.BuckMetadataBzlHandler.parse(test_file, package_only=True) package_data = dict( datasource_id=build.BuckMetadataBzlHandler.datasource_id, - type='github', - name='example/example', - version='0.0.1', - extracted_license_statement='BSD-3-Clause', - parties=[ - models.Party( - type=models.party_org, - name='example_org', - role='maintainer' - ) - ], - download_url='', - sha1='', - homepage_url='https://github.com/example/example', - vcs_url='https://github.com/example/example.git', - extra_data=dict(vcs_commit_hash="deadbeef") + type="github", + name="example/example", + version="0.0.1", + extracted_license_statement="BSD-3-Clause", + parties=[models.Party(type=models.party_org, name="example_org", role="maintainer")], + download_url="", + sha1="", + homepage_url="https://github.com/example/example", + vcs_url="https://github.com/example/example.git", + extra_data=dict(vcs_commit_hash="deadbeef"), ) - expected_packages = [models.PackageData.from_data(package_data=package_data, package_only=True)] + expected_packages = [ + models.PackageData.from_data(package_data=package_data, package_only=True) + ] compare_package_results(expected_packages, result_packages) + + +class TestBazelModuleHandler(PackageTester): + test_data_dir = os.path.join(os.path.dirname(__file__), "data") + + def test_parse_basic_module(self): + location = self.get_test_loc("bazel_module/MODULE.bazel") + results = list(build.BazelModuleHandler.parse(location)) + + assert len(results) == 1 + pkg = results[0] + + # Package identity + assert pkg.name == "my_sample_project" + assert pkg.version == "0.5.0" + assert pkg.type == "bazel" + assert pkg.datasource_id == "bazel_module" + + # Total deps: 3 runtime + 1 dev = 4 + assert len(pkg.dependencies) == 4 + + def test_parse_runtime_dependencies(self): + location = self.get_test_loc("bazel_module/MODULE.bazel") + results = list(build.BazelModuleHandler.parse(location)) + pkg = results[0] + + runtime_deps = [d for d in pkg.dependencies if d.scope == "dependencies"] + assert len(runtime_deps) == 3 + + dep_names = [d.purl for d in runtime_deps] + assert "pkg:bazel/rules_python@0.24.0" in dep_names + assert "pkg:bazel/rules_go@0.41.0" in dep_names + + def test_parse_dev_dependency(self): + location = self.get_test_loc("bazel_module/MODULE.bazel") + results = list(build.BazelModuleHandler.parse(location)) + pkg = results[0] + + dev_deps = [d for d in pkg.dependencies if d.scope == "dev"] + assert len(dev_deps) == 1 + assert dev_deps[0].purl == "pkg:bazel/googletest@1.14.0" + assert dev_deps[0].is_optional is True + assert dev_deps[0].is_runtime is False + + def test_parse_module_without_version(self): + location = self.get_test_loc("bazel_module/MODULE_no_version.bazel") + results = list(build.BazelModuleHandler.parse(location)) + + assert len(results) == 1 + pkg = results[0] + assert pkg.name == "minimal_module" + assert pkg.version is None + + def test_path_pattern_matches(self): + handler = build.BazelModuleHandler + assert handler.is_datafile("some/path/MODULE.bazel", _bare_filename=True) + assert not handler.is_datafile("some/path/notMODULE.bazel", _bare_filename=True) + assert not handler.is_datafile("some/path/WORKSPACE", _bare_filename=True) + assert not handler.is_datafile("some/path/BUILD", _bare_filename=True)