diff --git a/minecode/management/commands/federate_packages.py b/minecode/management/commands/federate_packages.py index f9567f50..41ed5b7f 100644 --- a/minecode/management/commands/federate_packages.py +++ b/minecode/management/commands/federate_packages.py @@ -15,7 +15,7 @@ from commoncode import fileutils from minecode.management import federatedcode from minecode.management.commands import VerboseCommand -from minecode_pipelines import pipes +from minecode import pipes from packagedb import models as packagedb_models diff --git a/minecode/management/commands/federate_sboms.py b/minecode/management/commands/federate_sboms.py index 3cb18916..9886629a 100644 --- a/minecode/management/commands/federate_sboms.py +++ b/minecode/management/commands/federate_sboms.py @@ -15,7 +15,7 @@ from commoncode import fileutils from minecode.management import federatedcode from minecode.management.commands import VerboseCommand -from minecode_pipelines import pipes +from minecode import pipes from packagedb import models as packagedb_models from packagedb import sbom diff --git a/minecode_pipelines/README.rst b/minecode/pipelines/README.rst similarity index 83% rename from minecode_pipelines/README.rst rename to minecode/pipelines/README.rst index 6f42f0cb..7ed16e61 100644 --- a/minecode_pipelines/README.rst +++ b/minecode/pipelines/README.rst @@ -1,7 +1,7 @@ -minecode-pipeline -=================== +minecode-pipelines +================== -minecode-pipeline is an add-on library working with scancode.io to define pipelines to mine +minecode-pipelines is an add-on library working with scancode.io to define pipelines to mine packageURLs and package metadata from ecosystem repositories and APIs. Installation @@ -10,8 +10,8 @@ Installation Requirements ############ -* install minecode-pipeline dependencies -* `pip install minecode-pipeline` +* install minecode-pipelines dependencies +* `pip install minecode-pipelines` Funding diff --git a/minecode_pipelines/pipelines/__init__.py b/minecode/pipelines/__init__.py similarity index 98% rename from minecode_pipelines/pipelines/__init__.py rename to minecode/pipelines/__init__.py index f1b5b1f9..15ea3ab3 100644 --- a/minecode_pipelines/pipelines/__init__.py +++ b/minecode/pipelines/__init__.py @@ -21,9 +21,9 @@ from scanpipe.pipelines import Pipeline from scanpipe.pipes import federatedcode -from minecode_pipelines import pipes -from minecode_pipelines.pipes import write_package_data_to_file -from minecode_pipelines.pipes import write_packageurls_to_file +from minecode import pipes +from minecode.pipes import write_package_data_to_file +from minecode.pipes import write_packageurls_to_file module_logger = logging.getLogger(__name__) diff --git a/minecode_pipelines/pipelines/mine_alpine.py b/minecode/pipelines/mine_alpine.py similarity index 94% rename from minecode_pipelines/pipelines/mine_alpine.py rename to minecode/pipelines/mine_alpine.py index 51bd8749..f71cdf91 100644 --- a/minecode_pipelines/pipelines/mine_alpine.py +++ b/minecode/pipelines/mine_alpine.py @@ -20,9 +20,9 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipelines import MineCodeBasePipeline +from minecode.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipes import alpine +from minecode.pipes import alpine class MineAlpine(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_apache.py b/minecode/pipelines/mine_apache.py similarity index 94% rename from minecode_pipelines/pipelines/mine_apache.py rename to minecode/pipelines/mine_apache.py index f51dda33..b5ff9191 100644 --- a/minecode_pipelines/pipelines/mine_apache.py +++ b/minecode/pipelines/mine_apache.py @@ -21,9 +21,9 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipes import apache -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode.pipes import apache +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls class MineApache(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_cargo.py b/minecode/pipelines/mine_cargo.py similarity index 96% rename from minecode_pipelines/pipelines/mine_cargo.py rename to minecode/pipelines/mine_cargo.py index fa556974..ebbb8582 100644 --- a/minecode_pipelines/pipelines/mine_cargo.py +++ b/minecode/pipelines/mine_cargo.py @@ -22,8 +22,8 @@ from pathlib import Path -from minecode_pipelines.pipes import cargo -from minecode_pipelines.pipelines import MineCodeBasePipeline +from minecode.pipes import cargo +from minecode.pipelines import MineCodeBasePipeline from scanpipe.pipes import federatedcode diff --git a/minecode_pipelines/pipelines/mine_composer.py b/minecode/pipelines/mine_composer.py similarity index 92% rename from minecode_pipelines/pipelines/mine_composer.py rename to minecode/pipelines/mine_composer.py index 5776d3aa..53b12630 100644 --- a/minecode_pipelines/pipelines/mine_composer.py +++ b/minecode/pipelines/mine_composer.py @@ -23,11 +23,11 @@ from datetime import datetime from scanpipe.pipes import federatedcode -from minecode_pipelines import pipes -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipes import composer -from minecode_pipelines.pipes.composer import mine_composer_packages, PACKAGE_BATCH_SIZE -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode import pipes +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipes import composer +from minecode.pipes.composer import mine_composer_packages, PACKAGE_BATCH_SIZE +from minecode.pipelines import _mine_and_publish_packageurls class MineComposer(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_conan.py b/minecode/pipelines/mine_conan.py similarity index 96% rename from minecode_pipelines/pipelines/mine_conan.py rename to minecode/pipelines/mine_conan.py index 1b94b749..619bb5ce 100644 --- a/minecode_pipelines/pipelines/mine_conan.py +++ b/minecode/pipelines/mine_conan.py @@ -21,8 +21,8 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. from pathlib import Path -from minecode_pipelines.pipes import conan -from minecode_pipelines.pipelines import MineCodeBasePipeline +from minecode.pipes import conan +from minecode.pipelines import MineCodeBasePipeline from scanpipe.pipes import federatedcode diff --git a/minecode_pipelines/pipelines/mine_cpan.py b/minecode/pipelines/mine_cpan.py similarity index 95% rename from minecode_pipelines/pipelines/mine_cpan.py rename to minecode/pipelines/mine_cpan.py index 813f090c..a9bda898 100644 --- a/minecode_pipelines/pipelines/mine_cpan.py +++ b/minecode/pipelines/mine_cpan.py @@ -21,8 +21,8 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipes import cpan -from minecode_pipelines.pipelines import MineCodeBasePipeline +from minecode.pipes import cpan +from minecode.pipelines import MineCodeBasePipeline class MineCpan(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_cran.py b/minecode/pipelines/mine_cran.py similarity index 93% rename from minecode_pipelines/pipelines/mine_cran.py rename to minecode/pipelines/mine_cran.py index 4de4a866..62779e51 100644 --- a/minecode_pipelines/pipelines/mine_cran.py +++ b/minecode/pipelines/mine_cran.py @@ -21,9 +21,9 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. import json -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipes import cran -from minecode_pipelines.pipes.cran import fetch_cran_db +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipes import cran +from minecode.pipes.cran import fetch_cran_db class MineCran(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_crates.py b/minecode/pipelines/mine_crates.py similarity index 95% rename from minecode_pipelines/pipelines/mine_crates.py rename to minecode/pipelines/mine_crates.py index 16a1e6bc..75ccdb80 100644 --- a/minecode_pipelines/pipelines/mine_crates.py +++ b/minecode/pipelines/mine_crates.py @@ -22,10 +22,10 @@ from scanpipe.pipes import federatedcode -from minecode_pipelines import pipes -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls -from minecode_pipelines.pipes import crates +from minecode import pipes +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls +from minecode.pipes import crates class MineCrates(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_debian.py b/minecode/pipelines/mine_debian.py similarity index 94% rename from minecode_pipelines/pipelines/mine_debian.py rename to minecode/pipelines/mine_debian.py index 54fb6969..e63af444 100644 --- a/minecode_pipelines/pipelines/mine_debian.py +++ b/minecode/pipelines/mine_debian.py @@ -22,10 +22,10 @@ from scanpipe.pipes import federatedcode -from minecode_pipelines import pipes -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls -from minecode_pipelines.pipes import debian +from minecode import pipes +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls +from minecode.pipes import debian class MineDebian(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_maven.py b/minecode/pipelines/mine_maven.py similarity index 98% rename from minecode_pipelines/pipelines/mine_maven.py rename to minecode/pipelines/mine_maven.py index da4bd4d2..dcb58ec6 100644 --- a/minecode_pipelines/pipelines/mine_maven.py +++ b/minecode/pipelines/mine_maven.py @@ -23,10 +23,10 @@ from aboutcode.pipeline import optional_step from scanpipe.pipes import federatedcode -from minecode_pipelines import pipes -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls -from minecode_pipelines.pipes import maven +from minecode import pipes +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls +from minecode.pipes import maven class MineMaven(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_nix.py b/minecode/pipelines/mine_nix.py similarity index 95% rename from minecode_pipelines/pipelines/mine_nix.py rename to minecode/pipelines/mine_nix.py index 4ebe5930..6429ecf9 100644 --- a/minecode_pipelines/pipelines/mine_nix.py +++ b/minecode/pipelines/mine_nix.py @@ -21,9 +21,9 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipes import nix -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode.pipes import nix +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls class MineNix(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_npm.py b/minecode/pipelines/mine_npm.py similarity index 95% rename from minecode_pipelines/pipelines/mine_npm.py rename to minecode/pipelines/mine_npm.py index 273b939c..57f49d49 100644 --- a/minecode_pipelines/pipelines/mine_npm.py +++ b/minecode/pipelines/mine_npm.py @@ -20,9 +20,9 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipes import npm -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode.pipes import npm +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls class MineNPM(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_nuget.py b/minecode/pipelines/mine_nuget.py similarity index 96% rename from minecode_pipelines/pipelines/mine_nuget.py rename to minecode/pipelines/mine_nuget.py index f3083e6e..61db7c79 100644 --- a/minecode_pipelines/pipelines/mine_nuget.py +++ b/minecode/pipelines/mine_nuget.py @@ -23,9 +23,9 @@ from pathlib import Path -from minecode_pipelines.pipes import nuget +from minecode.pipes import nuget -from minecode_pipelines.pipelines import MineCodeBasePipeline +from minecode.pipelines import MineCodeBasePipeline from scanpipe.pipes import federatedcode diff --git a/minecode_pipelines/pipelines/mine_pypi.py b/minecode/pipelines/mine_pypi.py similarity index 95% rename from minecode_pipelines/pipelines/mine_pypi.py rename to minecode/pipelines/mine_pypi.py index 6fd87972..08112c34 100644 --- a/minecode_pipelines/pipelines/mine_pypi.py +++ b/minecode/pipelines/mine_pypi.py @@ -20,9 +20,9 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipes import pypi -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode.pipes import pypi +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls class MinePypi(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_swift.py b/minecode/pipelines/mine_swift.py similarity index 92% rename from minecode_pipelines/pipelines/mine_swift.py rename to minecode/pipelines/mine_swift.py index 50f789db..d0c60b59 100644 --- a/minecode_pipelines/pipelines/mine_swift.py +++ b/minecode/pipelines/mine_swift.py @@ -22,9 +22,9 @@ from scanpipe.pipes import federatedcode -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipes.swift import mine_swift_packageurls -from minecode_pipelines.pipes.swift import load_swift_package_urls +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipes.swift import mine_swift_packageurls +from minecode.pipes.swift import load_swift_package_urls class MineSwift(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipes/__init__.py b/minecode/pipes/__init__.py similarity index 99% rename from minecode_pipelines/pipes/__init__.py rename to minecode/pipes/__init__.py index 4317f6a9..38832771 100644 --- a/minecode_pipelines/pipes/__init__.py +++ b/minecode/pipes/__init__.py @@ -16,7 +16,7 @@ import requests import saneyaml -from minecode_pipelines.utils import get_temp_file +from minecode.utils import get_temp_file # states: # note: a state is null when mining starts diff --git a/minecode_pipelines/pipes/alpine.py b/minecode/pipes/alpine.py similarity index 99% rename from minecode_pipelines/pipes/alpine.py rename to minecode/pipes/alpine.py index 9899e1d1..fbbaa9c1 100644 --- a/minecode_pipelines/pipes/alpine.py +++ b/minecode/pipes/alpine.py @@ -30,7 +30,7 @@ from scanpipe.pipes.fetch import fetch_http from scanpipe.pipes.scancode import extract_archives -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode.pipelines import _mine_and_publish_packageurls ALPINE_CHECKPOINT_PATH = "alpine/checkpoints.json" diff --git a/minecode_pipelines/pipes/apache.py b/minecode/pipes/apache.py similarity index 98% rename from minecode_pipelines/pipes/apache.py rename to minecode/pipes/apache.py index 6357fde8..9897ac4d 100644 --- a/minecode_pipelines/pipes/apache.py +++ b/minecode/pipes/apache.py @@ -9,11 +9,11 @@ from datetime import datetime, timezone -from minecode_pipelines.pipes import fetch_checkpoint_from_github -from minecode_pipelines.pipes import update_checkpoints_in_github -from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO +from minecode.pipes import fetch_checkpoint_from_github +from minecode.pipes import update_checkpoints_in_github +from minecode.pipes import MINECODE_PIPELINES_CONFIG_REPO -from minecode_pipelines.utils import get_temp_dir +from minecode.utils import get_temp_dir from packageurl import PackageURL diff --git a/minecode_pipelines/pipes/cargo.py b/minecode/pipes/cargo.py similarity index 100% rename from minecode_pipelines/pipes/cargo.py rename to minecode/pipes/cargo.py diff --git a/minecode_pipelines/pipes/composer.py b/minecode/pipes/composer.py similarity index 97% rename from minecode_pipelines/pipes/composer.py rename to minecode/pipes/composer.py index 06f50f86..e878e8bc 100644 --- a/minecode_pipelines/pipes/composer.py +++ b/minecode/pipes/composer.py @@ -21,11 +21,11 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. import json -from minecode_pipelines.utils import get_temp_file +from minecode.utils import get_temp_file from aboutcode.hashid import get_core_purl import requests from packageurl import PackageURL -from minecode_pipelines.utils import cycle_from_index, grouper +from minecode.utils import cycle_from_index, grouper PACKAGE_BATCH_SIZE = 100 diff --git a/minecode_pipelines/pipes/conan.py b/minecode/pipes/conan.py similarity index 100% rename from minecode_pipelines/pipes/conan.py rename to minecode/pipes/conan.py diff --git a/minecode_pipelines/miners/cpan.py b/minecode/pipes/cpan.py similarity index 58% rename from minecode_pipelines/miners/cpan.py rename to minecode/pipes/cpan.py index c006bf89..1c4d35b8 100644 --- a/minecode_pipelines/miners/cpan.py +++ b/minecode/pipes/cpan.py @@ -1,11 +1,24 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. # +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. import gzip import requests @@ -13,16 +26,18 @@ from bs4 import BeautifulSoup from packageurl import PackageURL -from minecode_pipelines.utils import get_temp_file - -""" -Visitors for cpan and cpan-like perl package repositories. -""" - +from minecode.utils import get_temp_file +from minecode.utils import grouper CPAN_REPO = "https://www.cpan.org/" CPAN_TYPE = "cpan" +# If True, show full details on fetching packageURL for +# a package name present in the index +LOG_PACKAGEURL_DETAILS = False + +PACKAGE_BATCH_SIZE = 500 + def get_cpan_packages(cpan_repo=CPAN_REPO, logger=None): """ @@ -157,3 +172,62 @@ def get_cpan_packageurls(name, path_prefix, logger=None): packageurls.append(purl.to_string()) return packageurls + + +def mine_cpan_packages(logger=None): + if logger: + logger("Getting packages from cpan index") + + package_path_by_name = get_cpan_packages(cpan_repo=CPAN_REPO, logger=logger) + + if logger: + packages_count = len(package_path_by_name.keys()) + logger(f"Mined {packages_count} packages from cpan index") + + return package_path_by_name + + +def mine_and_publish_cpan_packageurls(package_path_by_name, logger=None): + if not package_path_by_name: + return + + for package_batch in grouper(n=PACKAGE_BATCH_SIZE, iterable=package_path_by_name.keys()): + packages_mined = [] + + if logger and LOG_PACKAGEURL_DETAILS: + logger("Starting package mining for a batch of packages") + + for package_name in package_batch: + if not package_name or package_name in packages_mined: + continue + + # fetch packageURLs for package + if logger and LOG_PACKAGEURL_DETAILS: + logger(f"getting packageURLs for package: {package_name}") + + path_prefix = package_path_by_name.get(package_name) + if not path_prefix: + continue + + packageurls = get_cpan_packageurls( + name=package_name, + path_prefix=path_prefix, + logger=logger, + ) + if not packageurls: + if logger and LOG_PACKAGEURL_DETAILS: + logger(f"Package versions not present for package: {package_name}") + + # We don't want to try fetching versions for these again + packages_mined.append(package_name) + continue + + # get repo and path for package + base_purl = PackageURL(type=CPAN_TYPE, name=package_name).to_string() + if logger and LOG_PACKAGEURL_DETAILS: + logger(f"fetched packageURLs for package: {base_purl}") + purls_string = " ".join(packageurls) + logger(f"packageURLs: {purls_string}") + + packages_mined.append(package_name) + yield base_purl, packageurls, [] diff --git a/minecode_pipelines/pipes/cran.py b/minecode/pipes/cran.py similarity index 100% rename from minecode_pipelines/pipes/cran.py rename to minecode/pipes/cran.py diff --git a/minecode_pipelines/pipes/crates.py b/minecode/pipes/crates.py similarity index 100% rename from minecode_pipelines/pipes/crates.py rename to minecode/pipes/crates.py diff --git a/minecode_pipelines/pipes/debian.py b/minecode/pipes/debian.py similarity index 99% rename from minecode_pipelines/pipes/debian.py rename to minecode/pipes/debian.py index d2ccc665..7c010d6b 100644 --- a/minecode_pipelines/pipes/debian.py +++ b/minecode/pipes/debian.py @@ -32,7 +32,7 @@ from packageurl import PackageURL from scanpipe.pipes.fetch import fetch_http -from minecode_pipelines.pipes import ls +from minecode.pipes import ls DEBIAN_LSLR_URL = "http://ftp.debian.org/debian/ls-lR.gz" diff --git a/minecode_pipelines/pipes/java_stream.py b/minecode/pipes/java_stream.py similarity index 100% rename from minecode_pipelines/pipes/java_stream.py rename to minecode/pipes/java_stream.py diff --git a/minecode_pipelines/pipes/java_stream.py.ABOUT b/minecode/pipes/java_stream.py.ABOUT similarity index 100% rename from minecode_pipelines/pipes/java_stream.py.ABOUT rename to minecode/pipes/java_stream.py.ABOUT diff --git a/minecode_pipelines/pipes/java_stream.py.LICENSE b/minecode/pipes/java_stream.py.LICENSE similarity index 100% rename from minecode_pipelines/pipes/java_stream.py.LICENSE rename to minecode/pipes/java_stream.py.LICENSE diff --git a/minecode_pipelines/pipes/ls.py b/minecode/pipes/ls.py similarity index 100% rename from minecode_pipelines/pipes/ls.py rename to minecode/pipes/ls.py diff --git a/minecode_pipelines/pipes/maven.py b/minecode/pipes/maven.py similarity index 99% rename from minecode_pipelines/pipes/maven.py rename to minecode/pipes/maven.py index 30c4991f..062b64f0 100644 --- a/minecode_pipelines/pipes/maven.py +++ b/minecode/pipes/maven.py @@ -27,7 +27,7 @@ from packageurl import PackageURL -from minecode_pipelines.pipes import java_stream +from minecode.pipes import java_stream TRACE = False TRACE_DEEP = False diff --git a/minecode_pipelines/pipes/nix.py b/minecode/pipes/nix.py similarity index 63% rename from minecode_pipelines/pipes/nix.py rename to minecode/pipes/nix.py index 90d187be..12eb5f65 100644 --- a/minecode_pipelines/pipes/nix.py +++ b/minecode/pipes/nix.py @@ -20,38 +20,35 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import json +import requests +import time from datetime import datetime - -from minecode_pipelines.pipes import fetch_checkpoint_from_github -from minecode_pipelines.pipes import update_checkpoints_in_github -from minecode_pipelines.pipes import update_checkpoints_file_in_github -from minecode_pipelines.pipes import get_mined_packages_from_checkpoint -from minecode_pipelines.pipes import update_mined_packages_in_checkpoint -from minecode_pipelines.pipes import update_checkpoint_state -from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO -from minecode_pipelines.pipes import INITIAL_SYNC_STATE -from minecode_pipelines.pipes import PERIODIC_SYNC_STATE -from minecode_pipelines.pipes import write_packages_json -from minecode_pipelines.pipes import compress_packages_file -from minecode_pipelines.pipes import decompress_packages_file -from minecode_pipelines.pipes import fetch_checkpoint_by_git - -from minecode_pipelines.miners.nix import get_all_nix_packages -from minecode_pipelines.miners.nix import get_all_nix_packages_name -from minecode_pipelines.miners.nix import load_nix_packages -from minecode_pipelines.miners.nix import get_nix_packageurls -from minecode_pipelines.miners.nix import yield_nix_package_data -from minecode_pipelines.miners.nix import NIX_TYPE - - -from minecode_pipelines.utils import get_temp_dir - +import brotli from packageurl import PackageURL - from scanpipe.pipes.federatedcode import clone_repository from scanpipe.pipes.federatedcode import delete_local_clone +from minecode.pipes import fetch_checkpoint_from_github +from minecode.pipes import update_checkpoints_in_github +from minecode.pipes import update_checkpoints_file_in_github +from minecode.pipes import get_mined_packages_from_checkpoint +from minecode.pipes import update_mined_packages_in_checkpoint +from minecode.pipes import update_checkpoint_state +from minecode.pipes import MINECODE_PIPELINES_CONFIG_REPO +from minecode.pipes import INITIAL_SYNC_STATE +from minecode.pipes import PERIODIC_SYNC_STATE +from minecode.pipes import write_packages_json +from minecode.pipes import compress_packages_file +from minecode.pipes import decompress_packages_file +from minecode.pipes import fetch_checkpoint_by_git +from minecode.utils import get_temp_dir + +NIX_TYPE = "nix" +NIXPKGS = "nixpkgs" +DELAY_MULTIPLIER = 2 +MAX_RETRIES = 3 PACKAGE_FILE_NAME = "NixPackages.json" COMPRESSED_PACKAGE_FILE_NAME = "NixPackages.json.gz" @@ -64,6 +61,127 @@ CHANNEL_URL = "https://channels.nixos.org/nixos-unstable/packages.json.br" +def get_all_nix_packages(channel_url, logger=None): + """Get all Nix packages""" + if logger: + logger(f"Fetching Nix packages from: {channel_url}") + response = requests.get(channel_url) + response.raise_for_status() + + try: + decompressed_data = brotli.decompress(response.content) + data = json.loads(decompressed_data) + except brotli.error: + data = response.json() + + packages_dict = data.get("packages", {}) + return packages_dict + + +def get_all_nix_packages_name(packages_dict, logger=None): + """Get all Nix packages name""" + all_package_names = [] + packages = list(packages_dict.keys()) + for attr_path in packages: + all_package_names.append(attr_path) + + return {"packages": all_package_names} + + +def load_nix_packages(packages_file): + with open(packages_file) as f: + packages_data = json.load(f) + + return packages_data.get("packages", []) + + +def get_nix_packageurls(name, packages_dict, logger=None): + packageurls = [] + data = [] + # Get all the version of a package from the following API: + # https://search.devbox.sh/v2/pkg?name={name} + url = f"https://search.devbox.sh/v2/pkg?name={name}" + + for attempt in range(MAX_RETRIES): + try: + response = requests.get(url) + response.raise_for_status() + data = response.json() + break + except requests.HTTPError as e: + if response.status_code == 429: + retry_after = response.headers.get("Retry-After") + if retry_after: + wait_time = int(retry_after) + else: + wait_time = DELAY_MULTIPLIER**attempt + if logger: + logger(f"Rate limited (429). Waiting {wait_time}s before retry...") + time.sleep(wait_time) + continue + elif response.status_code == 404: + # The package_dict contains the package's current version + # devbox.sh may have only index the top + # level, all others deeper level may not be indexed + # TODO: This can only get the current version. + version = get_version_from_package_dict(name, packages_dict) + if version: + purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name, version=version) + else: + purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name) + packageurls.append(purl.to_string()) + return packageurls + else: + if logger: + logger(f"Request failed: {e}") + logger(f"HTTP error {response.status_code}: {response.text}") + return packageurls + except requests.RequestException as e: + if logger: + logger(f"Request failed: {e}") + return packageurls + except ValueError as e: + if logger: + logger(f"JSON parsing failed: {e}") + return packageurls + if not data: + # No data collected after retries + if logger: + logger(f"Max retries reached. Skipping package: {name}") + return packageurls + + releases = data.get("releases", []) + for release in releases: + version = release.get("version") + if version: + purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name, version=version) + purl_string = purl.to_string() + if purl_string not in packageurls: + packageurls.append(purl_string) + + return packageurls + + +def get_version_from_package_dict(name, package_dict): + """Get the version data from the package_dict""" + package_info = package_dict.get(name) + if not package_info: + return None + return package_info.get("version") + + +def yield_nix_package_data(name, packageurls=[]): + for purl in packageurls: + package_url = PackageURL.from_string(purl) + package_data_url = ( + f"https://search.devbox.sh/v2/resolve?name={name}&version={package_url.version}" + ) + response = requests.get(package_data_url) + if not response.ok: + continue + yield purl, response.json() + + def mine_nix_packages(logger=None): """ Mine nix package names from nixos.org and save to checkpoints, diff --git a/minecode_pipelines/pipes/npm.py b/minecode/pipes/npm.py similarity index 64% rename from minecode_pipelines/pipes/npm.py rename to minecode/pipes/npm.py index e5cbb084..a5ac5978 100644 --- a/minecode_pipelines/pipes/npm.py +++ b/minecode/pipes/npm.py @@ -20,39 +20,28 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import json from datetime import datetime -from minecode_pipelines.pipes import fetch_checkpoint_from_github -from minecode_pipelines.pipes import update_checkpoints_in_github -from minecode_pipelines.pipes import update_checkpoints_file_in_github -from minecode_pipelines.pipes import get_mined_packages_from_checkpoint -from minecode_pipelines.pipes import update_mined_packages_in_checkpoint -from minecode_pipelines.pipes import update_checkpoint_state -from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO -from minecode_pipelines.pipes import INITIAL_SYNC_STATE -from minecode_pipelines.pipes import PERIODIC_SYNC_STATE -from minecode_pipelines.pipes import write_packages_json -from minecode_pipelines.pipes import compress_packages_file -from minecode_pipelines.pipes import decompress_packages_file -from minecode_pipelines.pipes import fetch_checkpoint_by_git - - -from minecode_pipelines.miners.npm import get_npm_packages -from minecode_pipelines.miners.npm import get_updated_npm_packages -from minecode_pipelines.miners.npm import get_current_last_seq -from minecode_pipelines.miners.npm import load_npm_packages -from minecode_pipelines.miners.npm import get_npm_packageurls -from minecode_pipelines.miners.npm import yield_npm_package_data -from minecode_pipelines.miners.npm import NPM_REPLICATE_REPO -from minecode_pipelines.miners.npm import NPM_TYPE - -from minecode_pipelines.utils import get_temp_dir - +import requests from packageurl import PackageURL - from scanpipe.pipes.federatedcode import clone_repository from scanpipe.pipes.federatedcode import delete_local_clone +from minecode.pipes import fetch_checkpoint_from_github +from minecode.pipes import update_checkpoints_in_github +from minecode.pipes import update_checkpoints_file_in_github +from minecode.pipes import get_mined_packages_from_checkpoint +from minecode.pipes import update_mined_packages_in_checkpoint +from minecode.pipes import update_checkpoint_state +from minecode.pipes import MINECODE_PIPELINES_CONFIG_REPO +from minecode.pipes import INITIAL_SYNC_STATE +from minecode.pipes import PERIODIC_SYNC_STATE +from minecode.pipes import write_packages_json +from minecode.pipes import compress_packages_file +from minecode.pipes import decompress_packages_file +from minecode.pipes import fetch_checkpoint_by_git +from minecode.utils import get_temp_dir PACKAGE_FILE_NAME = "NPMPackages.json" COMPRESSED_PACKAGE_FILE_NAME = "NPMPackages.json.gz" @@ -62,6 +51,157 @@ NPM_PACKAGES_CHECKPOINT_PATH = "npm/packages_checkpoint.json" PACKAGE_BATCH_SIZE = 700 +""" +Visitors for Npmjs and npmjs-like javascript package repositories. + +We have this hierarchy in npm replicate and registry index: + npm projects replicate.npmjs.com (paginated JSON) -> versions at registry.npmjs.org (JSON) -> download urls + +See https://github.com/orgs/community/discussions/152515 for information on +the latest replicate.npmjs.com API. + +https://replicate.npmjs.com/_all_docs +This NPMJS replicate API serves as an index to get all npm packages and their revision IDs +in paginated queries. + +https://replicate.npmjs.com/_changes +This NPMJS replicate API serves as a CHANGELOG of npm packages with update sequences which +can be fetched in paginated queries. + +https://registry.npmjs.org/{namespace/name} +For each npm package, a JSON containing details including the list of all releases +and archives, their URLs, and some metadata for each release. + +https://registry.npmjs.org/{namespace/name}/{version} +For each release, a JSON contains details for the released version and all the +downloads available for this release. +""" + +NPM_REPLICATE_REPO = "https://replicate.npmjs.com/" +NPM_REGISTRY_REPO = "https://registry.npmjs.org/" +NPM_TYPE = "npm" +NPM_REPLICATE_BATCH_SIZE = 10000 + + +def get_package_names_last_key(package_data): + names = [package.get("id") for package in package_data.get("rows")] + last_key = package_data.get("rows")[-1].get("key") + return names, last_key + + +def get_package_names_last_seq(package_data): + names = [package.get("id") for package in package_data.get("results")] + last_seq = package_data.get("last_seq") + return names, last_seq + + +def get_current_last_seq(replicate_url=NPM_REPLICATE_REPO): + npm_replicate_latest_changes = replicate_url + "_changes?descending=True" + response = requests.get(npm_replicate_latest_changes) + if not response.ok: + return + + package_data = response.json() + _package_names, last_seq = get_package_names_last_seq(package_data) + return last_seq + + +def get_updated_npm_packages(last_seq, replicate_url=NPM_REPLICATE_REPO, logger=None): + all_package_names = [] + i = 0 + + while True: + if logger: + logger(f"Processing iteration: {i}: changes after seq: {last_seq}") + + npm_replicate_changes = ( + replicate_url + "_changes?" + f"limit={NPM_REPLICATE_BATCH_SIZE}" + f"&since={last_seq}" + ) + response = requests.get(npm_replicate_changes) + if not response.ok: + return all_package_names + + package_data = response.json() + package_names, last_seq = get_package_names_last_seq(package_data) + all_package_names.extend(package_names) + + # We have fetched the last set of changes if True + if len(package_names) < NPM_REPLICATE_BATCH_SIZE: + break + + i += 1 + + return {"packages": all_package_names}, last_seq + + +def get_npm_packages(replicate_url=NPM_REPLICATE_REPO, logger=None): + all_package_names = [] + + npm_replicate_all = replicate_url + "_all_docs?" + f"limit={NPM_REPLICATE_BATCH_SIZE}" + response = requests.get(npm_replicate_all) + if not response.ok: + return all_package_names + + package_data = response.json() + package_names, last_key = get_package_names_last_key(package_data) + all_package_names.extend(package_names) + + total_rows = package_data.get("total_rows") + iterations = int(total_rows / NPM_REPLICATE_BATCH_SIZE) + 1 + + for i in range(iterations): + npm_replicate_from_id = npm_replicate_all + f'&start_key="{last_key}"' + if logger: + logger(f"Processing iteration: {i}: {npm_replicate_from_id}") + + response = requests.get(npm_replicate_from_id) + if not response.ok: + raise Exception(npm_replicate_from_id, response.text) + + package_data = response.json() + package_names, last_key = get_package_names_last_key(package_data) + all_package_names.extend(package_names) + + return {"packages": all_package_names} + + +def get_npm_packageurls(name, npm_repo=NPM_REGISTRY_REPO): + packageurls = [] + + project_index_api_url = npm_repo + name + response = requests.get(project_index_api_url) + if not response.ok: + return packageurls + + project_data = response.json() + versions = project_data.get("versions") or [] + for version in versions: + purl = PackageURL( + type=NPM_TYPE, + name=name, + version=version, + ) + packageurls.append(purl.to_string()) + + return packageurls + + +def yield_npm_package_data(name, packageurls=[]): + for purl in packageurls or get_npm_packageurls(name): + package_url = PackageURL.from_string(purl) + package_data_url = NPM_REGISTRY_REPO + name + "/" + package_url.version + response = requests.get(package_data_url) + if not response.ok: + continue + yield purl, response.json() + + +def load_npm_packages(packages_file): + with open(packages_file) as f: + packages_data = json.load(f) + + return packages_data.get("packages", []) + def mine_npm_packages(logger=None): """ diff --git a/minecode_pipelines/pipes/nuget.py b/minecode/pipes/nuget.py similarity index 100% rename from minecode_pipelines/pipes/nuget.py rename to minecode/pipes/nuget.py diff --git a/minecode_pipelines/pipes/pypi.py b/minecode/pipes/pypi.py similarity index 74% rename from minecode_pipelines/pipes/pypi.py rename to minecode/pipes/pypi.py index 756e0ecb..d11999bb 100644 --- a/minecode_pipelines/pipes/pypi.py +++ b/minecode/pipes/pypi.py @@ -20,44 +20,128 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import json from datetime import datetime -from minecode_pipelines.pipes import fetch_checkpoint_from_github -from minecode_pipelines.pipes import update_checkpoints_in_github -from minecode_pipelines.pipes import update_checkpoint_state -from minecode_pipelines.pipes import get_mined_packages_from_checkpoint -from minecode_pipelines.pipes import update_mined_packages_in_checkpoint -from minecode_pipelines.pipes import get_packages_file_from_checkpoint -from minecode_pipelines.pipes import write_packages_json -from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO -from minecode_pipelines.pipes import INITIAL_SYNC_STATE -from minecode_pipelines.pipes import PERIODIC_SYNC_STATE - - -from minecode_pipelines.miners.pypi import get_pypi_packages -from minecode_pipelines.miners.pypi import get_pypi_packageurls -from minecode_pipelines.miners.pypi import load_pypi_packages -from minecode_pipelines.miners.pypi import yield_pypi_package_data -from minecode_pipelines.miners.pypi import PYPI_SIMPLE_REPO - - -from minecode_pipelines.miners.pypi import PYPI_TYPE -from minecode_pipelines.utils import get_temp_dir - -from scanpipe.pipes.federatedcode import delete_local_clone +import requests from packageurl import PackageURL from scanpipe.pipes.federatedcode import clone_repository +from scanpipe.pipes.federatedcode import delete_local_clone + +from minecode.pipes import INITIAL_SYNC_STATE +from minecode.pipes import MINECODE_PIPELINES_CONFIG_REPO +from minecode.pipes import PERIODIC_SYNC_STATE +from minecode.pipes import fetch_checkpoint_from_github +from minecode.pipes import get_mined_packages_from_checkpoint +from minecode.pipes import get_packages_file_from_checkpoint +from minecode.pipes import update_checkpoints_in_github +from minecode.pipes import update_checkpoint_state +from minecode.pipes import update_mined_packages_in_checkpoint +from minecode.pipes import write_packages_json +from minecode.utils import get_temp_dir # If True, show full details on fetching packageURL for # a package name present in the index LOG_PACKAGEURL_DETAILS = False - PACKAGE_FILE_NAME = "PypiPackages.json" PYPI_SIMPLE_CHECKPOINT_PATH = "pypi/simple_index/" + PACKAGE_FILE_NAME PYPI_CHECKPOINT_PATH = "pypi/checkpoints.json" PYPI_PACKAGES_CHECKPOINT_PATH = "pypi/packages_checkpoint.json" +""" +Visitors for Pypi and Pypi-like Python package repositories. + +We have this hierarchy in Pypi simple/ index: + pypi projects (JSON/HTML) -> project versions (JSON/HTML) -> download urls + +https://pypi.org/simple/ +Pypi serves a main index via JSON/HTML API that contains a list of package names +and some info on when a package was updated by releasing a new version. +See https://docs.pypi.org/api/index-api/ for more details. +This index also has a list of versions and download URLs of all +uploaded/available package archives and some basic metadata. + +https://pypi.org/pypi/{name}/json +For each package, a JSON contains details including the list of all releases +and archives, their URLs, and some metadata for each release. +For each release, a JSON contains details for the released version and all the +downloads available for this release. +""" + + +pypi_json_headers = {"Accept": "application/vnd.pypi.simple.v1+json"} + + +PYPI_SIMPLE_REPO = "https://pypi.org/simple" +PYPI_METADATA_REPO = "https://pypi.org/pypi" +PYPI_TYPE = "pypi" + + +def get_pypi_packages(pypi_repo, logger=None): + response = requests.get(pypi_repo, headers=pypi_json_headers) + if not response.ok: + return + + return response.json() + + +def get_pypi_package_versions(name): + versions = [] + + project_index_api_url = PYPI_SIMPLE_REPO + "/" + name + response = requests.get(project_index_api_url, headers=pypi_json_headers) + if not response.ok: + return versions + + project_data = response.json() + versions = project_data.get("versions", []) + return versions + + +def get_pypi_packageurls(name): + packageurls = [] + + for version in get_pypi_package_versions(name=name): + purl = PackageURL( + type=PYPI_TYPE, + name=name, + version=version, + ) + packageurls.append(purl.to_string()) + + return packageurls + + +def yield_pypi_package_data(name, packageurls=[]): + for purl in packageurls or get_pypi_packageurls(name): + package_url = PackageURL.from_string(purl) + package_data_url = ( + PYPI_METADATA_REPO + "/" + name + "/" + package_url.version + "/" + "json" + ) + response = requests.get(package_data_url, headers=pypi_json_headers) + if not response.ok: + continue + yield purl, response.json() + + +def load_pypi_packages(packages_file): + with open(packages_file) as f: + packages_data = json.load(f) + + last_serial = packages_data.get("meta").get("_last-serial") + packages = packages_data.get("projects") + + return last_serial, packages + + +def get_last_serial_from_packages(packages_file): + with open(packages_file) as f: + packages_data = json.load(f) + + last_serial = packages_data.get("meta").get("_last-serial") + return last_serial + def mine_pypi_packages(logger=None): """ diff --git a/minecode_pipelines/pipes/swift.py b/minecode/pipes/swift.py similarity index 100% rename from minecode_pipelines/pipes/swift.py rename to minecode/pipes/swift.py diff --git a/minecode_pipelines/tests/data/alpine/APKINDEX b/minecode/tests/data/alpine/APKINDEX similarity index 100% rename from minecode_pipelines/tests/data/alpine/APKINDEX rename to minecode/tests/data/alpine/APKINDEX diff --git a/minecode_pipelines/tests/data/alpine/expected_packages.json b/minecode/tests/data/alpine/expected_packages.json similarity index 100% rename from minecode_pipelines/tests/data/alpine/expected_packages.json rename to minecode/tests/data/alpine/expected_packages.json diff --git a/minecode_pipelines/tests/data/cargo/c5store b/minecode/tests/data/cargo/c5store similarity index 100% rename from minecode_pipelines/tests/data/cargo/c5store rename to minecode/tests/data/cargo/c5store diff --git a/minecode_pipelines/tests/data/cargo/c5store-expected.yaml b/minecode/tests/data/cargo/c5store-expected.yaml similarity index 100% rename from minecode_pipelines/tests/data/cargo/c5store-expected.yaml rename to minecode/tests/data/cargo/c5store-expected.yaml diff --git a/minecode_pipelines/tests/data/composer/expected_output.json b/minecode/tests/data/composer/expected_output.json similarity index 100% rename from minecode_pipelines/tests/data/composer/expected_output.json rename to minecode/tests/data/composer/expected_output.json diff --git a/minecode_pipelines/tests/data/composer/package_details.json b/minecode/tests/data/composer/package_details.json similarity index 100% rename from minecode_pipelines/tests/data/composer/package_details.json rename to minecode/tests/data/composer/package_details.json diff --git a/minecode_pipelines/tests/data/composer/packages_list.json b/minecode/tests/data/composer/packages_list.json similarity index 100% rename from minecode_pipelines/tests/data/composer/packages_list.json rename to minecode/tests/data/composer/packages_list.json diff --git a/minecode_pipelines/tests/data/conan/cairo/cairo-config.yml b/minecode/tests/data/conan/cairo/cairo-config.yml similarity index 100% rename from minecode_pipelines/tests/data/conan/cairo/cairo-config.yml rename to minecode/tests/data/conan/cairo/cairo-config.yml diff --git a/minecode_pipelines/tests/data/conan/expected-cairo-purls.yml b/minecode/tests/data/conan/expected-cairo-purls.yml similarity index 100% rename from minecode_pipelines/tests/data/conan/expected-cairo-purls.yml rename to minecode/tests/data/conan/expected-cairo-purls.yml diff --git a/minecode_pipelines/tests/data/cran/cran_db.json b/minecode/tests/data/cran/cran_db.json similarity index 100% rename from minecode_pipelines/tests/data/cran/cran_db.json rename to minecode/tests/data/cran/cran_db.json diff --git a/minecode_pipelines/tests/data/cran/expected_abbreviate.yaml b/minecode/tests/data/cran/expected_abbreviate.yaml similarity index 100% rename from minecode_pipelines/tests/data/cran/expected_abbreviate.yaml rename to minecode/tests/data/cran/expected_abbreviate.yaml diff --git a/minecode_pipelines/tests/data/cran/expected_abc.data.yaml b/minecode/tests/data/cran/expected_abc.data.yaml similarity index 100% rename from minecode_pipelines/tests/data/cran/expected_abc.data.yaml rename to minecode/tests/data/cran/expected_abc.data.yaml diff --git a/minecode_pipelines/tests/data/cran/expected_abc.yaml b/minecode/tests/data/cran/expected_abc.yaml similarity index 100% rename from minecode_pipelines/tests/data/cran/expected_abc.yaml rename to minecode/tests/data/cran/expected_abc.yaml diff --git a/minecode_pipelines/tests/data/directories/find-ls b/minecode/tests/data/directories/find-ls similarity index 100% rename from minecode_pipelines/tests/data/directories/find-ls rename to minecode/tests/data/directories/find-ls diff --git a/minecode_pipelines/tests/data/directories/find-ls-apache-start b/minecode/tests/data/directories/find-ls-apache-start similarity index 100% rename from minecode_pipelines/tests/data/directories/find-ls-apache-start rename to minecode/tests/data/directories/find-ls-apache-start diff --git a/minecode_pipelines/tests/data/directories/find-ls-apache-start-expected.json b/minecode/tests/data/directories/find-ls-apache-start-expected.json similarity index 100% rename from minecode_pipelines/tests/data/directories/find-ls-apache-start-expected.json rename to minecode/tests/data/directories/find-ls-apache-start-expected.json diff --git a/minecode_pipelines/tests/data/directories/find-ls-expected.json b/minecode/tests/data/directories/find-ls-expected.json similarity index 100% rename from minecode_pipelines/tests/data/directories/find-ls-expected.json rename to minecode/tests/data/directories/find-ls-expected.json diff --git a/minecode_pipelines/tests/data/directories/ls-lr b/minecode/tests/data/directories/ls-lr similarity index 100% rename from minecode_pipelines/tests/data/directories/ls-lr rename to minecode/tests/data/directories/ls-lr diff --git a/minecode_pipelines/tests/data/directories/ls-lr-expected.json b/minecode/tests/data/directories/ls-lr-expected.json similarity index 100% rename from minecode_pipelines/tests/data/directories/ls-lr-expected.json rename to minecode/tests/data/directories/ls-lr-expected.json diff --git a/minecode_pipelines/tests/data/directories/ls-lr-ubuntu b/minecode/tests/data/directories/ls-lr-ubuntu similarity index 100% rename from minecode_pipelines/tests/data/directories/ls-lr-ubuntu rename to minecode/tests/data/directories/ls-lr-ubuntu diff --git a/minecode_pipelines/tests/data/directories/ls-lr-ubuntu-expected.json b/minecode/tests/data/directories/ls-lr-ubuntu-expected.json similarity index 100% rename from minecode_pipelines/tests/data/directories/ls-lr-ubuntu-expected.json rename to minecode/tests/data/directories/ls-lr-ubuntu-expected.json diff --git a/minecode_pipelines/tests/data/maven/index/buggy/expected_artifacts-defaults.json b/minecode/tests/data/maven/index/buggy/expected_artifacts-defaults.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/buggy/expected_artifacts-defaults.json rename to minecode/tests/data/maven/index/buggy/expected_artifacts-defaults.json diff --git a/minecode_pipelines/tests/data/maven/index/buggy/expected_artifacts.json b/minecode/tests/data/maven/index/buggy/expected_artifacts.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/buggy/expected_artifacts.json rename to minecode/tests/data/maven/index/buggy/expected_artifacts.json diff --git a/minecode_pipelines/tests/data/maven/index/buggy/expected_entries.json b/minecode/tests/data/maven/index/buggy/expected_entries.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/buggy/expected_entries.json rename to minecode/tests/data/maven/index/buggy/expected_entries.json diff --git a/minecode_pipelines/tests/data/maven/index/buggy/expected_uris.json b/minecode/tests/data/maven/index/buggy/expected_uris.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/buggy/expected_uris.json rename to minecode/tests/data/maven/index/buggy/expected_uris.json diff --git a/minecode_pipelines/tests/data/maven/index/buggy/expected_visited_uris.json b/minecode/tests/data/maven/index/buggy/expected_visited_uris.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/buggy/expected_visited_uris.json rename to minecode/tests/data/maven/index/buggy/expected_visited_uris.json diff --git a/minecode_pipelines/tests/data/maven/index/buggy/nexus-maven-repository-index.gz b/minecode/tests/data/maven/index/buggy/nexus-maven-repository-index.gz similarity index 100% rename from minecode_pipelines/tests/data/maven/index/buggy/nexus-maven-repository-index.gz rename to minecode/tests/data/maven/index/buggy/nexus-maven-repository-index.gz diff --git a/minecode_pipelines/tests/data/maven/index/expected_artifacts-all-worthy.json b/minecode/tests/data/maven/index/expected_artifacts-all-worthy.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/expected_artifacts-all-worthy.json rename to minecode/tests/data/maven/index/expected_artifacts-all-worthy.json diff --git a/minecode_pipelines/tests/data/maven/index/expected_artifacts-defaults.json b/minecode/tests/data/maven/index/expected_artifacts-defaults.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/expected_artifacts-defaults.json rename to minecode/tests/data/maven/index/expected_artifacts-defaults.json diff --git a/minecode_pipelines/tests/data/maven/index/expected_artifacts.json b/minecode/tests/data/maven/index/expected_artifacts.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/expected_artifacts.json rename to minecode/tests/data/maven/index/expected_artifacts.json diff --git a/minecode_pipelines/tests/data/maven/index/expected_entries.json b/minecode/tests/data/maven/index/expected_entries.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/expected_entries.json rename to minecode/tests/data/maven/index/expected_entries.json diff --git a/minecode_pipelines/tests/data/maven/index/expected_uris.json b/minecode/tests/data/maven/index/expected_uris.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/expected_uris.json rename to minecode/tests/data/maven/index/expected_uris.json diff --git a/minecode_pipelines/tests/data/maven/index/increment/expected_artifacts-defaults.json b/minecode/tests/data/maven/index/increment/expected_artifacts-defaults.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/expected_artifacts-defaults.json rename to minecode/tests/data/maven/index/increment/expected_artifacts-defaults.json diff --git a/minecode_pipelines/tests/data/maven/index/increment/expected_artifacts.json b/minecode/tests/data/maven/index/increment/expected_artifacts.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/expected_artifacts.json rename to minecode/tests/data/maven/index/increment/expected_artifacts.json diff --git a/minecode_pipelines/tests/data/maven/index/increment/expected_entries.json b/minecode/tests/data/maven/index/increment/expected_entries.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/expected_entries.json rename to minecode/tests/data/maven/index/increment/expected_entries.json diff --git a/minecode_pipelines/tests/data/maven/index/increment/expected_properties_uris.json b/minecode/tests/data/maven/index/increment/expected_properties_uris.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/expected_properties_uris.json rename to minecode/tests/data/maven/index/increment/expected_properties_uris.json diff --git a/minecode_pipelines/tests/data/maven/index/increment/expected_uris.json b/minecode/tests/data/maven/index/increment/expected_uris.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/expected_uris.json rename to minecode/tests/data/maven/index/increment/expected_uris.json diff --git a/minecode_pipelines/tests/data/maven/index/increment/nexus-maven-repository-index.445.gz b/minecode/tests/data/maven/index/increment/nexus-maven-repository-index.445.gz similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/nexus-maven-repository-index.445.gz rename to minecode/tests/data/maven/index/increment/nexus-maven-repository-index.445.gz diff --git a/minecode_pipelines/tests/data/maven/index/increment/nexus-maven-repository-index.properties b/minecode/tests/data/maven/index/increment/nexus-maven-repository-index.properties similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/nexus-maven-repository-index.properties rename to minecode/tests/data/maven/index/increment/nexus-maven-repository-index.properties diff --git a/minecode_pipelines/tests/data/maven/index/increment2/expected_mini_package.json b/minecode/tests/data/maven/index/increment2/expected_mini_package.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment2/expected_mini_package.json rename to minecode/tests/data/maven/index/increment2/expected_mini_package.json diff --git a/minecode_pipelines/tests/data/maven/index/increment2/expected_uris.json b/minecode/tests/data/maven/index/increment2/expected_uris.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment2/expected_uris.json rename to minecode/tests/data/maven/index/increment2/expected_uris.json diff --git a/minecode_pipelines/tests/data/maven/index/increment2/nexus-maven-repository-index.457.gz b/minecode/tests/data/maven/index/increment2/nexus-maven-repository-index.457.gz similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment2/nexus-maven-repository-index.457.gz rename to minecode/tests/data/maven/index/increment2/nexus-maven-repository-index.457.gz diff --git a/minecode_pipelines/tests/data/maven/index/nexus-maven-repository-index.gz b/minecode/tests/data/maven/index/nexus-maven-repository-index.gz similarity index 100% rename from minecode_pipelines/tests/data/maven/index/nexus-maven-repository-index.gz rename to minecode/tests/data/maven/index/nexus-maven-repository-index.gz diff --git a/minecode_pipelines/tests/data/nuget/catalog/pages/page10897.json b/minecode/tests/data/nuget/catalog/pages/page10897.json similarity index 100% rename from minecode_pipelines/tests/data/nuget/catalog/pages/page10897.json rename to minecode/tests/data/nuget/catalog/pages/page10897.json diff --git a/minecode_pipelines/tests/data/swift/commits_tags1.txt b/minecode/tests/data/swift/commits_tags1.txt similarity index 100% rename from minecode_pipelines/tests/data/swift/commits_tags1.txt rename to minecode/tests/data/swift/commits_tags1.txt diff --git a/minecode_pipelines/tests/data/swift/commits_tags2.txt b/minecode/tests/data/swift/commits_tags2.txt similarity index 100% rename from minecode_pipelines/tests/data/swift/commits_tags2.txt rename to minecode/tests/data/swift/commits_tags2.txt diff --git a/minecode_pipelines/tests/data/swift/commits_tags3.txt b/minecode/tests/data/swift/commits_tags3.txt similarity index 100% rename from minecode_pipelines/tests/data/swift/commits_tags3.txt rename to minecode/tests/data/swift/commits_tags3.txt diff --git a/minecode_pipelines/tests/data/swift/commits_tags4.txt b/minecode/tests/data/swift/commits_tags4.txt similarity index 100% rename from minecode_pipelines/tests/data/swift/commits_tags4.txt rename to minecode/tests/data/swift/commits_tags4.txt diff --git a/minecode_pipelines/tests/data/swift/expected1.yaml b/minecode/tests/data/swift/expected1.yaml similarity index 100% rename from minecode_pipelines/tests/data/swift/expected1.yaml rename to minecode/tests/data/swift/expected1.yaml diff --git a/minecode_pipelines/tests/data/swift/expected2.yaml b/minecode/tests/data/swift/expected2.yaml similarity index 100% rename from minecode_pipelines/tests/data/swift/expected2.yaml rename to minecode/tests/data/swift/expected2.yaml diff --git a/minecode_pipelines/tests/data/swift/expected3.yaml b/minecode/tests/data/swift/expected3.yaml similarity index 100% rename from minecode_pipelines/tests/data/swift/expected3.yaml rename to minecode/tests/data/swift/expected3.yaml diff --git a/minecode_pipelines/tests/data/swift/expected4.yaml b/minecode/tests/data/swift/expected4.yaml similarity index 100% rename from minecode_pipelines/tests/data/swift/expected4.yaml rename to minecode/tests/data/swift/expected4.yaml diff --git a/minecode_pipelines/tests/pipes/__init__.py b/minecode/tests/pipes/__init__.py similarity index 100% rename from minecode_pipelines/tests/pipes/__init__.py rename to minecode/tests/pipes/__init__.py diff --git a/minecode_pipelines/tests/pipes/test_alpine.py b/minecode/tests/pipes/test_alpine.py similarity index 96% rename from minecode_pipelines/tests/pipes/test_alpine.py rename to minecode/tests/pipes/test_alpine.py index ff0b91b3..8886eceb 100644 --- a/minecode_pipelines/tests/pipes/test_alpine.py +++ b/minecode/tests/pipes/test_alpine.py @@ -12,7 +12,7 @@ from commoncode.testcase import check_against_expected_json_file from commoncode.testcase import FileBasedTesting -from minecode_pipelines.pipes import alpine +from minecode.pipes import alpine class AlpineMapperTest(FileBasedTesting): diff --git a/minecode_pipelines/tests/pipes/test_apache.py b/minecode/tests/pipes/test_apache.py similarity index 99% rename from minecode_pipelines/tests/pipes/test_apache.py rename to minecode/tests/pipes/test_apache.py index ad440fc9..06d06acf 100644 --- a/minecode_pipelines/tests/pipes/test_apache.py +++ b/minecode/tests/pipes/test_apache.py @@ -11,7 +11,7 @@ from commoncode.testcase import FileBasedTesting -from minecode_pipelines.pipes import apache +from minecode.pipes import apache class ApacheMiscTest(FileBasedTesting): diff --git a/minecode_pipelines/tests/pipes/test_cargo.py b/minecode/tests/pipes/test_cargo.py similarity index 94% rename from minecode_pipelines/tests/pipes/test_cargo.py rename to minecode/tests/pipes/test_cargo.py index 777fda74..e482373e 100644 --- a/minecode_pipelines/tests/pipes/test_cargo.py +++ b/minecode/tests/pipes/test_cargo.py @@ -14,7 +14,7 @@ import saneyaml from unittest import TestCase -from minecode_pipelines.pipes.cargo import get_cargo_packages +from minecode.pipes.cargo import get_cargo_packages DATA_DIR = Path(__file__).parent.parent / "data" / "cargo" diff --git a/minecode_pipelines/tests/pipes/test_composer.py b/minecode/tests/pipes/test_composer.py similarity index 90% rename from minecode_pipelines/tests/pipes/test_composer.py rename to minecode/tests/pipes/test_composer.py index a60211e4..986ffdd2 100644 --- a/minecode_pipelines/tests/pipes/test_composer.py +++ b/minecode/tests/pipes/test_composer.py @@ -12,9 +12,9 @@ from unittest.mock import patch, MagicMock from django.test import SimpleTestCase -from minecode_pipelines.pipes.composer import get_composer_packages -from minecode_pipelines.pipes.composer import load_composer_packages -from minecode_pipelines.pipes.composer import get_composer_purl +from minecode.pipes.composer import get_composer_packages +from minecode.pipes.composer import load_composer_packages +from minecode.pipes.composer import get_composer_purl DATA_DIR = Path(__file__).parent.parent / "data" / "composer" diff --git a/minecode_pipelines/tests/pipes/test_conan.py b/minecode/tests/pipes/test_conan.py similarity index 94% rename from minecode_pipelines/tests/pipes/test_conan.py rename to minecode/tests/pipes/test_conan.py index 38dd96d4..f025d039 100644 --- a/minecode_pipelines/tests/pipes/test_conan.py +++ b/minecode/tests/pipes/test_conan.py @@ -13,7 +13,7 @@ from django.test import TestCase -from minecode_pipelines.pipes.conan import get_conan_packages +from minecode.pipes.conan import get_conan_packages DATA_DIR = Path(__file__).parent.parent / "data" / "conan" diff --git a/minecode_pipelines/tests/pipes/test_cran.py b/minecode/tests/pipes/test_cran.py similarity index 96% rename from minecode_pipelines/tests/pipes/test_cran.py rename to minecode/tests/pipes/test_cran.py index 53dfb075..264800ea 100644 --- a/minecode_pipelines/tests/pipes/test_cran.py +++ b/minecode/tests/pipes/test_cran.py @@ -11,7 +11,7 @@ from pathlib import Path from unittest import TestCase -from minecode_pipelines.pipes.cran import mine_cran_packageurls +from minecode.pipes.cran import mine_cran_packageurls DATA_DIR = Path(__file__).parent.parent / "data" / "cran" diff --git a/minecode_pipelines/tests/pipes/test_ls.py b/minecode/tests/pipes/test_ls.py similarity index 98% rename from minecode_pipelines/tests/pipes/test_ls.py rename to minecode/tests/pipes/test_ls.py index bc87e8ab..fcb3fffe 100644 --- a/minecode_pipelines/tests/pipes/test_ls.py +++ b/minecode/tests/pipes/test_ls.py @@ -13,7 +13,7 @@ from commoncode.testcase import check_against_expected_json_file from commoncode.testcase import FileBasedTesting -from minecode_pipelines.pipes import ls +from minecode.pipes import ls class ParseDirectoryListingTest(FileBasedTesting): diff --git a/minecode_pipelines/tests/pipes/test_maven.py b/minecode/tests/pipes/test_maven.py similarity index 99% rename from minecode_pipelines/tests/pipes/test_maven.py rename to minecode/tests/pipes/test_maven.py index 199c8322..a962cfae 100644 --- a/minecode_pipelines/tests/pipes/test_maven.py +++ b/minecode/tests/pipes/test_maven.py @@ -12,7 +12,7 @@ from commoncode.testcase import check_against_expected_json_file from commoncode.testcase import FileBasedTesting -from minecode_pipelines.pipes import maven +from minecode.pipes import maven class MavenMiscTest(FileBasedTesting): diff --git a/minecode_pipelines/tests/pipes/test_nuget.py b/minecode/tests/pipes/test_nuget.py similarity index 94% rename from minecode_pipelines/tests/pipes/test_nuget.py rename to minecode/tests/pipes/test_nuget.py index 77e80c93..972b3f25 100644 --- a/minecode_pipelines/tests/pipes/test_nuget.py +++ b/minecode/tests/pipes/test_nuget.py @@ -11,8 +11,8 @@ from pathlib import Path from unittest import TestCase -from minecode_pipelines.pipes import nuget -from minecode_pipelines.tests.pipes import TestLogger +from minecode.pipes import nuget +from minecode.tests.pipes import TestLogger TEST_DIR = Path(__file__).parent.parent / "data" / "nuget" diff --git a/minecode_pipelines/tests/pipes/test_pipes.py b/minecode/tests/pipes/test_pipes.py similarity index 97% rename from minecode_pipelines/tests/pipes/test_pipes.py rename to minecode/tests/pipes/test_pipes.py index 1c8161bc..369e5520 100644 --- a/minecode_pipelines/tests/pipes/test_pipes.py +++ b/minecode/tests/pipes/test_pipes.py @@ -11,7 +11,7 @@ from unittest import TestCase from git import Repo -from minecode_pipelines.pipes import get_commit_at_distance_ahead +from minecode.pipes import get_commit_at_distance_ahead class GetCommitAtDistanceAheadIntegrationTests(TestCase): diff --git a/minecode_pipelines/tests/pipes/test_pypi.py b/minecode/tests/pipes/test_pypi.py similarity index 100% rename from minecode_pipelines/tests/pipes/test_pypi.py rename to minecode/tests/pipes/test_pypi.py diff --git a/minecode_pipelines/tests/pipes/test_swift.py b/minecode/tests/pipes/test_swift.py similarity index 98% rename from minecode_pipelines/tests/pipes/test_swift.py rename to minecode/tests/pipes/test_swift.py index f21cb67b..9f977bca 100644 --- a/minecode_pipelines/tests/pipes/test_swift.py +++ b/minecode/tests/pipes/test_swift.py @@ -10,7 +10,7 @@ from pathlib import Path from unittest import TestCase import saneyaml -from minecode_pipelines.pipes.swift import ( +from minecode.pipes.swift import ( get_tags_and_commits_from_git_output, generate_package_urls, ) diff --git a/minecode/utils.py b/minecode/utils.py index 16ac1513..3e45b753 100644 --- a/minecode/utils.py +++ b/minecode/utils.py @@ -13,6 +13,7 @@ import os import tempfile import uuid +from itertools import zip_longest from django.conf import settings from django.core import signing @@ -404,3 +405,19 @@ def get_webhook_url(view_name, user_uuid): site_url = settings.SITE_URL.rstrip("/") webhook_url = site_url + target_url return webhook_url + + +def grouper(n, iterable, padvalue=None): + """ + Produce batches of length `n` for an `iterable`. + https://docs.python.org/3.10/library/itertools.html#itertools-recipes + + #TODO: replace with `itertools.batched` added in python3.12 + """ + return zip_longest(*[iter(iterable)] * n, fillvalue=padvalue) + + +def cycle_from_index(lst, x): + """Return list rotated so it starts from index x.""" + x = x % len(lst) # handle cases where x is out of bounds + return lst[x:] + lst[:x] diff --git a/minecode_pipelines/__init__.py b/minecode_pipelines/__init__.py deleted file mode 100644 index 5b0bba7a..00000000 --- a/minecode_pipelines/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -VERSION = "0.1.1" diff --git a/minecode_pipelines/miners/__init__.py b/minecode_pipelines/miners/__init__.py deleted file mode 100644 index e1521118..00000000 --- a/minecode_pipelines/miners/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# diff --git a/minecode_pipelines/miners/nix.py b/minecode_pipelines/miners/nix.py deleted file mode 100644 index 6a3b77d8..00000000 --- a/minecode_pipelines/miners/nix.py +++ /dev/null @@ -1,143 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import brotli -import json -import requests -import time - -from packageurl import PackageURL - - -NIX_TYPE = "nix" -NIXPKGS = "nixpkgs" -DELAY_MULTIPLIER = 2 -MAX_RETRIES = 3 - - -def get_all_nix_packages(channel_url, logger=None): - """Get all Nix packages""" - if logger: - logger(f"Fetching Nix packages from: {channel_url}") - response = requests.get(channel_url) - response.raise_for_status() - - try: - decompressed_data = brotli.decompress(response.content) - data = json.loads(decompressed_data) - except brotli.error: - data = response.json() - - packages_dict = data.get("packages", {}) - return packages_dict - - -def get_all_nix_packages_name(packages_dict, logger=None): - """Get all Nix packages name""" - all_package_names = [] - packages = list(packages_dict.keys()) - for attr_path in packages: - all_package_names.append(attr_path) - - return {"packages": all_package_names} - - -def load_nix_packages(packages_file): - with open(packages_file) as f: - packages_data = json.load(f) - - return packages_data.get("packages", []) - - -def get_nix_packageurls(name, packages_dict, logger=None): - packageurls = [] - data = [] - # Get all the version of a package from the following API: - # https://search.devbox.sh/v2/pkg?name={name} - url = f"https://search.devbox.sh/v2/pkg?name={name}" - - for attempt in range(MAX_RETRIES): - try: - response = requests.get(url) - response.raise_for_status() - data = response.json() - break - except requests.HTTPError as e: - if response.status_code == 429: - retry_after = response.headers.get("Retry-After") - if retry_after: - wait_time = int(retry_after) - else: - wait_time = DELAY_MULTIPLIER**attempt - if logger: - logger(f"Rate limited (429). Waiting {wait_time}s before retry...") - time.sleep(wait_time) - continue - elif response.status_code == 404: - # The package_dict contains the package's current version - # devbox.sh may have only index the top - # level, all others deeper level may not be indexed - # TODO: This can only get the current version. - version = get_version_from_package_dict(name, packages_dict) - if version: - purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name, version=version) - else: - purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name) - packageurls.append(purl.to_string()) - return packageurls - else: - if logger: - logger(f"Request failed: {e}") - logger(f"HTTP error {response.status_code}: {response.text}") - return packageurls - except requests.RequestException as e: - if logger: - logger(f"Request failed: {e}") - return packageurls - except ValueError as e: - if logger: - logger(f"JSON parsing failed: {e}") - return packageurls - if not data: - # No data collected after retries - if logger: - logger(f"Max retries reached. Skipping package: {name}") - return packageurls - - releases = data.get("releases", []) - for release in releases: - version = release.get("version") - if version: - purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name, version=version) - purl_string = purl.to_string() - if purl_string not in packageurls: - packageurls.append(purl_string) - - return packageurls - - -def get_version_from_package_dict(name, package_dict): - """Get the version data from the package_dict""" - package_info = package_dict.get(name) - if not package_info: - return None - return package_info.get("version") - - -def yield_nix_package_data(name, packageurls=[]): - for purl in packageurls: - package_url = PackageURL.from_string(purl) - package_data_url = ( - f"https://search.devbox.sh/v2/resolve?name={name}&version={package_url.version}" - ) - response = requests.get(package_data_url) - if not response.ok: - continue - yield purl, response.json() diff --git a/minecode_pipelines/miners/npm.py b/minecode_pipelines/miners/npm.py deleted file mode 100644 index 783feff4..00000000 --- a/minecode_pipelines/miners/npm.py +++ /dev/null @@ -1,167 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import json -import requests - -from packageurl import PackageURL - - -""" -Visitors for Npmjs and npmjs-like javascript package repositories. - -We have this hierarchy in npm replicate and registry index: - npm projects replicate.npmjs.com (paginated JSON) -> versions at registry.npmjs.org (JSON) -> download urls - -See https://github.com/orgs/community/discussions/152515 for information on -the latest replicate.npmjs.com API. - -https://replicate.npmjs.com/_all_docs -This NPMJS replicate API serves as an index to get all npm packages and their revision IDs -in paginated queries. - -https://replicate.npmjs.com/_changes -This NPMJS replicate API serves as a CHANGELOG of npm packages with update sequences which -can be fetched in paginated queries. - -https://registry.npmjs.org/{namespace/name} -For each npm package, a JSON containing details including the list of all releases -and archives, their URLs, and some metadata for each release. - -https://registry.npmjs.org/{namespace/name}/{version} -For each release, a JSON contains details for the released version and all the -downloads available for this release. -""" - - -NPM_REPLICATE_REPO = "https://replicate.npmjs.com/" -NPM_REGISTRY_REPO = "https://registry.npmjs.org/" -NPM_TYPE = "npm" -NPM_REPLICATE_BATCH_SIZE = 10000 - - -def get_package_names_last_key(package_data): - names = [package.get("id") for package in package_data.get("rows")] - last_key = package_data.get("rows")[-1].get("key") - return names, last_key - - -def get_package_names_last_seq(package_data): - names = [package.get("id") for package in package_data.get("results")] - last_seq = package_data.get("last_seq") - return names, last_seq - - -def get_current_last_seq(replicate_url=NPM_REPLICATE_REPO): - npm_replicate_latest_changes = replicate_url + "_changes?descending=True" - response = requests.get(npm_replicate_latest_changes) - if not response.ok: - return - - package_data = response.json() - _package_names, last_seq = get_package_names_last_seq(package_data) - return last_seq - - -def get_updated_npm_packages(last_seq, replicate_url=NPM_REPLICATE_REPO, logger=None): - all_package_names = [] - i = 0 - - while True: - if logger: - logger(f"Processing iteration: {i}: changes after seq: {last_seq}") - - npm_replicate_changes = ( - replicate_url + "_changes?" + f"limit={NPM_REPLICATE_BATCH_SIZE}" + f"&since={last_seq}" - ) - response = requests.get(npm_replicate_changes) - if not response.ok: - return all_package_names - - package_data = response.json() - package_names, last_seq = get_package_names_last_seq(package_data) - all_package_names.extend(package_names) - - # We have fetched the last set of changes if True - if len(package_names) < NPM_REPLICATE_BATCH_SIZE: - break - - i += 1 - - return {"packages": all_package_names}, last_seq - - -def get_npm_packages(replicate_url=NPM_REPLICATE_REPO, logger=None): - all_package_names = [] - - npm_replicate_all = replicate_url + "_all_docs?" + f"limit={NPM_REPLICATE_BATCH_SIZE}" - response = requests.get(npm_replicate_all) - if not response.ok: - return all_package_names - - package_data = response.json() - package_names, last_key = get_package_names_last_key(package_data) - all_package_names.extend(package_names) - - total_rows = package_data.get("total_rows") - iterations = int(total_rows / NPM_REPLICATE_BATCH_SIZE) + 1 - - for i in range(iterations): - npm_replicate_from_id = npm_replicate_all + f'&start_key="{last_key}"' - if logger: - logger(f"Processing iteration: {i}: {npm_replicate_from_id}") - - response = requests.get(npm_replicate_from_id) - if not response.ok: - raise Exception(npm_replicate_from_id, response.text) - - package_data = response.json() - package_names, last_key = get_package_names_last_key(package_data) - all_package_names.extend(package_names) - - return {"packages": all_package_names} - - -def get_npm_packageurls(name, npm_repo=NPM_REGISTRY_REPO): - packageurls = [] - - project_index_api_url = npm_repo + name - response = requests.get(project_index_api_url) - if not response.ok: - return packageurls - - project_data = response.json() - versions = project_data.get("versions") or [] - for version in versions: - purl = PackageURL( - type=NPM_TYPE, - name=name, - version=version, - ) - packageurls.append(purl.to_string()) - - return packageurls - - -def yield_npm_package_data(name, packageurls=[]): - for purl in packageurls or get_npm_packageurls(name): - package_url = PackageURL.from_string(purl) - package_data_url = NPM_REGISTRY_REPO + name + "/" + package_url.version - response = requests.get(package_data_url) - if not response.ok: - continue - yield purl, response.json() - - -def load_npm_packages(packages_file): - with open(packages_file) as f: - packages_data = json.load(f) - - return packages_data.get("packages", []) diff --git a/minecode_pipelines/miners/pypi.py b/minecode_pipelines/miners/pypi.py deleted file mode 100644 index 00f6cb6c..00000000 --- a/minecode_pipelines/miners/pypi.py +++ /dev/null @@ -1,107 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import json -import requests - -from packageurl import PackageURL - -""" -Visitors for Pypi and Pypi-like Python package repositories. - -We have this hierarchy in Pypi simple/ index: - pypi projects (JSON/HTML) -> project versions (JSON/HTML) -> download urls - -https://pypi.org/simple/ -Pypi serves a main index via JSON/HTML API that contains a list of package names -and some info on when a package was updated by releasing a new version. -See https://docs.pypi.org/api/index-api/ for more details. -This index also has a list of versions and download URLs of all -uploaded/available package archives and some basic metadata. - -https://pypi.org/pypi/{name}/json -For each package, a JSON contains details including the list of all releases -and archives, their URLs, and some metadata for each release. -For each release, a JSON contains details for the released version and all the -downloads available for this release. -""" - - -pypi_json_headers = {"Accept": "application/vnd.pypi.simple.v1+json"} - - -PYPI_SIMPLE_REPO = "https://pypi.org/simple" -PYPI_METADATA_REPO = "https://pypi.org/pypi" -PYPI_TYPE = "pypi" - - -def get_pypi_packages(pypi_repo, logger=None): - response = requests.get(pypi_repo, headers=pypi_json_headers) - if not response.ok: - return - - return response.json() - - -def get_pypi_package_versions(name): - versions = [] - - project_index_api_url = PYPI_SIMPLE_REPO + "/" + name - response = requests.get(project_index_api_url, headers=pypi_json_headers) - if not response.ok: - return versions - - project_data = response.json() - versions = project_data.get("versions", []) - return versions - - -def get_pypi_packageurls(name): - packageurls = [] - - for version in get_pypi_package_versions(name=name): - purl = PackageURL( - type=PYPI_TYPE, - name=name, - version=version, - ) - packageurls.append(purl.to_string()) - - return packageurls - - -def yield_pypi_package_data(name, packageurls=[]): - for purl in packageurls or get_pypi_packageurls(name): - package_url = PackageURL.from_string(purl) - package_data_url = ( - PYPI_METADATA_REPO + "/" + name + "/" + package_url.version + "/" + "json" - ) - response = requests.get(package_data_url, headers=pypi_json_headers) - if not response.ok: - continue - yield purl, response.json() - - -def load_pypi_packages(packages_file): - with open(packages_file) as f: - packages_data = json.load(f) - - last_serial = packages_data.get("meta").get("_last-serial") - packages = packages_data.get("projects") - - return last_serial, packages - - -def get_last_serial_from_packages(packages_file): - with open(packages_file) as f: - packages_data = json.load(f) - - last_serial = packages_data.get("meta").get("_last-serial") - return last_serial diff --git a/minecode_pipelines/pipes/cpan.py b/minecode_pipelines/pipes/cpan.py deleted file mode 100644 index 9f02707b..00000000 --- a/minecode_pipelines/pipes/cpan.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# http://nexb.com and https://github.com/aboutcode-org/scancode.io -# The ScanCode.io software is licensed under the Apache License version 2.0. -# Data generated with ScanCode.io is provided as-is without warranties. -# ScanCode is a trademark of nexB Inc. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode.io should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# -# ScanCode.io is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/aboutcode-org/scancode.io for support and download. - -from minecode_pipelines.miners.cpan import get_cpan_packages -from minecode_pipelines.miners.cpan import get_cpan_packageurls -from minecode_pipelines.miners.cpan import CPAN_REPO - -from minecode_pipelines.miners.cpan import CPAN_TYPE -from minecode_pipelines.utils import grouper - -from packageurl import PackageURL - -# If True, show full details on fetching packageURL for -# a package name present in the index -LOG_PACKAGEURL_DETAILS = False - -PACKAGE_BATCH_SIZE = 500 - - -def mine_cpan_packages(logger=None): - if logger: - logger("Getting packages from cpan index") - - package_path_by_name = get_cpan_packages(cpan_repo=CPAN_REPO, logger=logger) - - if logger: - packages_count = len(package_path_by_name.keys()) - logger(f"Mined {packages_count} packages from cpan index") - - return package_path_by_name - - -def mine_and_publish_cpan_packageurls(package_path_by_name, logger=None): - if not package_path_by_name: - return - - for package_batch in grouper(n=PACKAGE_BATCH_SIZE, iterable=package_path_by_name.keys()): - packages_mined = [] - - if logger and LOG_PACKAGEURL_DETAILS: - logger("Starting package mining for a batch of packages") - - for package_name in package_batch: - if not package_name or package_name in packages_mined: - continue - - # fetch packageURLs for package - if logger and LOG_PACKAGEURL_DETAILS: - logger(f"getting packageURLs for package: {package_name}") - - path_prefix = package_path_by_name.get(package_name) - if not path_prefix: - continue - - packageurls = get_cpan_packageurls( - name=package_name, - path_prefix=path_prefix, - logger=logger, - ) - if not packageurls: - if logger and LOG_PACKAGEURL_DETAILS: - logger(f"Package versions not present for package: {package_name}") - - # We don't want to try fetching versions for these again - packages_mined.append(package_name) - continue - - # get repo and path for package - base_purl = PackageURL(type=CPAN_TYPE, name=package_name).to_string() - if logger and LOG_PACKAGEURL_DETAILS: - logger(f"fetched packageURLs for package: {base_purl}") - purls_string = " ".join(packageurls) - logger(f"packageURLs: {purls_string}") - - packages_mined.append(package_name) - yield base_purl, packageurls, [] diff --git a/minecode_pipelines/tests/__init__.py b/minecode_pipelines/tests/__init__.py deleted file mode 100644 index e1521118..00000000 --- a/minecode_pipelines/tests/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# diff --git a/minecode_pipelines/utils.py b/minecode_pipelines/utils.py deleted file mode 100644 index 29bf9f8c..00000000 --- a/minecode_pipelines/utils.py +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import tempfile -import os -from commoncode.fileutils import create_dir - -from itertools import zip_longest - - -def grouper(n, iterable, padvalue=None): - """ - Produce batches of length `n` for an `iterable`. - https://docs.python.org/3.10/library/itertools.html#itertools-recipes - - #TODO: replace with `itertools.batched` added in python3.12 - """ - return zip_longest(*[iter(iterable)] * n, fillvalue=padvalue) - - -def system_temp_dir(temp_dir=os.getenv("MINECODE_TMP")): - """Return the global temp directory..""" - if not temp_dir: - temp_dir = os.path.join(tempfile.gettempdir(), "minecode") - create_dir(temp_dir) - return temp_dir - - -def get_temp_dir(base_dir="", prefix=""): - """ - Return the path to base a new unique temporary directory, created under - the system-wide `system_temp_dir` temp directory and as a subdir of the - base_dir path, a path relative to the `system_temp_dir`. - """ - if base_dir: - base_dir = os.path.join(system_temp_dir(), base_dir) - create_dir(base_dir) - else: - base_dir = system_temp_dir() - return tempfile.mkdtemp(prefix=prefix, dir=base_dir) - - -def get_temp_file(file_name="data", extension=".file", dir_name=""): - """ - Return a file path string to a new, unique and non-existing - temporary file that can safely be created without a risk of name - collision. - """ - if extension and not extension.startswith("."): - extension = "." + extension - - file_name = file_name + extension - # create a new temp dir each time - temp_dir = get_temp_dir(dir_name) - location = os.path.join(temp_dir, file_name) - return location - - -def cycle_from_index(lst, x): - """Return list rotated so it starts from index x.""" - x = x % len(lst) # handle cases where x is out of bounds - return lst[x:] + lst[:x] diff --git a/pyproject-minecode_pipelines.toml b/pyproject-minecode_pipelines.toml index 31ba9bea..395b8bbd 100644 --- a/pyproject-minecode_pipelines.toml +++ b/pyproject-minecode_pipelines.toml @@ -4,9 +4,9 @@ build-backend = "flot.buildapi" [project] name = "minecode_pipelines" -version = "1.0.1" +version = "2.0.0" description = "A library for mining packageURLs and package metadata from ecosystem repositories." -readme = "minecode_pipelines/README.rst" +readme = "README-minecode_pipelines.rst" license = { text = "Apache-2.0" } requires-python = ">=3.10" @@ -26,7 +26,6 @@ keywords = [ "data", ] - classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", @@ -51,24 +50,24 @@ dependencies = [ urls = { Homepage = "https://github.com/aboutcode-org/purldb" } [project.entry-points."scancodeio_pipelines"] -mine_pypi = "minecode_pipelines.pipelines.mine_pypi:MinePypi" -mine_npm = "minecode_pipelines.pipelines.mine_npm:MineNPM" -mine_maven = "minecode_pipelines.pipelines.mine_maven:MineMaven" -mine_cargo = "minecode_pipelines.pipelines.mine_cargo:MineCargo" -mine_debian = "minecode_pipelines.pipelines.mine_debian:MineDebian" -mine_nuget = "minecode_pipelines.pipelines.mine_nuget:MineNuGet" -mine_alpine = "minecode_pipelines.pipelines.mine_alpine:MineAlpine" -mine_conan = "minecode_pipelines.pipelines.mine_conan:MineConan" -mine_cpan = "minecode_pipelines.pipelines.mine_cpan:MineCpan" -mine_cran = "minecode_pipelines.pipelines.mine_cran:MineCran" -mine_swift = "minecode_pipelines.pipelines.mine_swift:MineSwift" -mine_composer = "minecode_pipelines.pipelines.mine_composer:MineComposer" -mine_nix = "minecode_pipelines.pipelines.mine_nix:MineNix" -mine_crates = "minecode_pipelines.pipelines.mine_crates:MineCrates" -mine_apache = "minecode_pipelines.pipelines.mine_apache:MineApache" +mine_pypi = "minecode.pipelines.mine_pypi:MinePypi" +mine_npm = "minecode.pipelines.mine_npm:MineNPM" +mine_maven = "minecode.pipelines.mine_maven:MineMaven" +mine_cargo = "minecode.pipelines.mine_cargo:MineCargo" +mine_debian = "minecode.pipelines.mine_debian:MineDebian" +mine_nuget = "minecode.pipelines.mine_nuget:MineNuGet" +mine_alpine = "minecode.pipelines.mine_alpine:MineAlpine" +mine_conan = "minecode.pipelines.mine_conan:MineConan" +mine_cpan = "minecode.pipelines.mine_cpan:MineCpan" +mine_cran = "minecode.pipelines.mine_cran:MineCran" +mine_swift = "minecode.pipelines.mine_swift:MineSwift" +mine_composer = "minecode.pipelines.mine_composer:MineComposer" +mine_nix = "minecode.pipelines.mine_nix:MineNix" +mine_crates = "minecode.pipelines.mine_crates:MineCrates" +mine_apache = "minecode.pipelines.mine_apache:MineApache" [tool.bumpversion] -current_version = "1.0.1" +current_version = "2.0.0" allow_dirty = true files = [ @@ -77,7 +76,9 @@ files = [ [tool.flot] includes = [ - "minecode_pipelines/**/*", + "minecode/pipelines/*", + "minecode/pipes/*", + "minecode/utils.py", ] excludes = [ @@ -91,8 +92,8 @@ excludes = [ "**/.ve", "**/*.bak", "**/.ipynb_checkpoints/*", - "minecode_pipelines/tests/**/*", - "minecode_pipelines/notebooks/*", + "minecode/tests/**/*", + "minecode/notebooks/*", ] metadata_files = ["apache-2.0.LICENSE", "NOTICE"] diff --git a/setup.cfg b/setup.cfg index 308aec8b..9e59c196 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,6 +57,7 @@ install_requires = rubymarshal == 1.0.3 samecode == 0.5.1 urlpy == 0.5 + brotli == 1.2.0 setup_requires = setuptools_scm[toml] >= 4 @@ -90,3 +91,18 @@ console_scripts = scancodeio_pipelines = matching = matchcode.pipelines.matching:Matching d2d = scanpipe.pipelines.deploy_to_develop:DeployToDevelop + mine_pypi = minecode.pipelines.mine_pypi:MinePypi + mine_npm = minecode.pipelines.mine_npm:MineNPM + mine_maven = minecode.pipelines.mine_maven:MineMaven + mine_cargo = minecode.pipelines.mine_cargo:MineCargo + mine_debian = minecode.pipelines.mine_debian:MineDebian + mine_nuget = minecode.pipelines.mine_nuget:MineNuGet + mine_alpine = minecode.pipelines.mine_alpine:MineAlpine + mine_conan = minecode.pipelines.mine_conan:MineConan + mine_cpan = minecode.pipelines.mine_cpan:MineCpan + mine_cran = minecode.pipelines.mine_cran:MineCran + mine_swift = minecode.pipelines.mine_swift:MineSwift + mine_composer = minecode.pipelines.mine_composer:MineComposer + mine_nix = minecode.pipelines.mine_nix:MineNix + mine_crates = minecode.pipelines.mine_crates:MineCrates + mine_apache = minecode.pipelines.mine_apache:MineApache