Merge minecode pipelines

minecode/management/commands/federate_packages.py

@@ -15,7 +15,7 @@
 from commoncode import fileutils
 from minecode.management import federatedcode
 from minecode.management.commands import VerboseCommand
-from minecode_pipelines import pipes
+from minecode import pipes
 from packagedb import models as packagedb_models
 
 

minecode/management/commands/federate_sboms.py

@@ -15,7 +15,7 @@
 from commoncode import fileutils
 from minecode.management import federatedcode
 from minecode.management.commands import VerboseCommand
-from minecode_pipelines import pipes
+from minecode import pipes
 from packagedb import models as packagedb_models
 from packagedb import sbom
 

minecode_pipelines/README.rst → minecode/pipelines/README.rst

@@ -1,7 +1,7 @@
-minecode-pipeline
-===================
+minecode-pipelines
+==================
 
-minecode-pipeline is an add-on library working with scancode.io to define pipelines to mine
+minecode-pipelines is an add-on library working with scancode.io to define pipelines to mine
 packageURLs and package metadata from ecosystem repositories and APIs.
 
 Installation
@@ -10,8 +10,8 @@ Installation
 Requirements
 ############
 
-* install minecode-pipeline dependencies
-* `pip install minecode-pipeline`
+* install minecode-pipelines dependencies
+* `pip install minecode-pipelines`
 
 
 Funding

minecode_pipelines/pipelines/__init__.py → minecode/pipelines/__init__.py

@@ -21,9 +21,9 @@
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import federatedcode
 
-from minecode_pipelines import pipes
-from minecode_pipelines.pipes import write_package_data_to_file
-from minecode_pipelines.pipes import write_packageurls_to_file
+from minecode import pipes
+from minecode.pipes import write_package_data_to_file
+from minecode.pipes import write_packageurls_to_file
 
 module_logger = logging.getLogger(__name__)
 

minecode_pipelines/pipelines/mine_alpine.py → minecode/pipelines/mine_alpine.py

@@ -20,9 +20,9 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from minecode_pipelines.pipelines import MineCodeBasePipeline
+from minecode.pipelines import MineCodeBasePipeline
 
-from minecode_pipelines.pipes import alpine
+from minecode.pipes import alpine
 
 
 class MineAlpine(MineCodeBasePipeline):

minecode_pipelines/pipelines/mine_apache.py → minecode/pipelines/mine_apache.py

@@ -21,9 +21,9 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 
-from minecode_pipelines.pipes import apache
-from minecode_pipelines.pipelines import MineCodeBasePipeline
-from minecode_pipelines.pipelines import _mine_and_publish_packageurls
+from minecode.pipes import apache
+from minecode.pipelines import MineCodeBasePipeline
+from minecode.pipelines import _mine_and_publish_packageurls
 
 
 class MineApache(MineCodeBasePipeline):

minecode_pipelines/pipelines/mine_cargo.py → minecode/pipelines/mine_cargo.py

@@ -22,8 +22,8 @@
 
 from pathlib import Path
 
-from minecode_pipelines.pipes import cargo
-from minecode_pipelines.pipelines import MineCodeBasePipeline
+from minecode.pipes import cargo
+from minecode.pipelines import MineCodeBasePipeline
 from scanpipe.pipes import federatedcode
 
 

minecode_pipelines/pipelines/mine_composer.py → minecode/pipelines/mine_composer.py

@@ -23,11 +23,11 @@
 from datetime import datetime
 from scanpipe.pipes import federatedcode
 
-from minecode_pipelines import pipes
-from minecode_pipelines.pipelines import MineCodeBasePipeline
-from minecode_pipelines.pipes import composer
-from minecode_pipelines.pipes.composer import mine_composer_packages, PACKAGE_BATCH_SIZE
-from minecode_pipelines.pipelines import _mine_and_publish_packageurls
+from minecode import pipes
+from minecode.pipelines import MineCodeBasePipeline
+from minecode.pipes import composer
+from minecode.pipes.composer import mine_composer_packages, PACKAGE_BATCH_SIZE
+from minecode.pipelines import _mine_and_publish_packageurls
 
 
 class MineComposer(MineCodeBasePipeline):

minecode_pipelines/pipelines/mine_conan.py → minecode/pipelines/mine_conan.py

@@ -21,8 +21,8 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 from pathlib import Path
-from minecode_pipelines.pipes import conan
-from minecode_pipelines.pipelines import MineCodeBasePipeline
+from minecode.pipes import conan
+from minecode.pipelines import MineCodeBasePipeline
 from scanpipe.pipes import federatedcode
 
 

minecode_pipelines/pipelines/mine_cpan.py → minecode/pipelines/mine_cpan.py

@@ -21,8 +21,8 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 
-from minecode_pipelines.pipes import cpan
-from minecode_pipelines.pipelines import MineCodeBasePipeline
+from minecode.pipes import cpan
+from minecode.pipelines import MineCodeBasePipeline
 
 
 class MineCpan(MineCodeBasePipeline):

minecode_pipelines/pipelines/mine_cran.py → minecode/pipelines/mine_cran.py

@@ -21,9 +21,9 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 import json
-from minecode_pipelines.pipelines import MineCodeBasePipeline
-from minecode_pipelines.pipes import cran
-from minecode_pipelines.pipes.cran import fetch_cran_db
+from minecode.pipelines import MineCodeBasePipeline
+from minecode.pipes import cran
+from minecode.pipes.cran import fetch_cran_db
 
 
 class MineCran(MineCodeBasePipeline):

minecode_pipelines/pipelines/mine_crates.py → minecode/pipelines/mine_crates.py

@@ -22,10 +22,10 @@
 
 from scanpipe.pipes import federatedcode
 
-from minecode_pipelines import pipes
-from minecode_pipelines.pipelines import MineCodeBasePipeline
-from minecode_pipelines.pipelines import _mine_and_publish_packageurls
-from minecode_pipelines.pipes import crates
+from minecode import pipes
+from minecode.pipelines import MineCodeBasePipeline
+from minecode.pipelines import _mine_and_publish_packageurls
+from minecode.pipes import crates
 
 
 class MineCrates(MineCodeBasePipeline):

minecode_pipelines/pipelines/mine_debian.py → minecode/pipelines/mine_debian.py

@@ -22,10 +22,10 @@
 
 from scanpipe.pipes import federatedcode
 
-from minecode_pipelines import pipes
-from minecode_pipelines.pipelines import MineCodeBasePipeline
-from minecode_pipelines.pipelines import _mine_and_publish_packageurls
-from minecode_pipelines.pipes import debian
+from minecode import pipes
+from minecode.pipelines import MineCodeBasePipeline
+from minecode.pipelines import _mine_and_publish_packageurls
+from minecode.pipes import debian
 
 
 class MineDebian(MineCodeBasePipeline):

minecode_pipelines/pipelines/mine_maven.py → minecode/pipelines/mine_maven.py

@@ -23,10 +23,10 @@
 from aboutcode.pipeline import optional_step
 from scanpipe.pipes import federatedcode
 
-from minecode_pipelines import pipes
-from minecode_pipelines.pipelines import MineCodeBasePipeline
-from minecode_pipelines.pipelines import _mine_and_publish_packageurls
-from minecode_pipelines.pipes import maven
+from minecode import pipes
+from minecode.pipelines import MineCodeBasePipeline
+from minecode.pipelines import _mine_and_publish_packageurls
+from minecode.pipes import maven
 
 
 class MineMaven(MineCodeBasePipeline):

minecode_pipelines/pipelines/mine_nix.py → minecode/pipelines/mine_nix.py

@@ -21,9 +21,9 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 
-from minecode_pipelines.pipes import nix
-from minecode_pipelines.pipelines import MineCodeBasePipeline
-from minecode_pipelines.pipelines import _mine_and_publish_packageurls
+from minecode.pipes import nix
+from minecode.pipelines import MineCodeBasePipeline
+from minecode.pipelines import _mine_and_publish_packageurls
 
 
 class MineNix(MineCodeBasePipeline):

minecode_pipelines/pipelines/mine_npm.py → minecode/pipelines/mine_npm.py

@@ -20,9 +20,9 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from minecode_pipelines.pipes import npm
-from minecode_pipelines.pipelines import MineCodeBasePipeline
-from minecode_pipelines.pipelines import _mine_and_publish_packageurls
+from minecode.pipes import npm
+from minecode.pipelines import MineCodeBasePipeline
+from minecode.pipelines import _mine_and_publish_packageurls
 
 
 class MineNPM(MineCodeBasePipeline):

minecode_pipelines/pipelines/mine_nuget.py → minecode/pipelines/mine_nuget.py

@@ -23,9 +23,9 @@
 
 from pathlib import Path
 
-from minecode_pipelines.pipes import nuget
+from minecode.pipes import nuget
 
-from minecode_pipelines.pipelines import MineCodeBasePipeline
+from minecode.pipelines import MineCodeBasePipeline
 from scanpipe.pipes import federatedcode
 
 

minecode_pipelines/pipelines/mine_pypi.py → minecode/pipelines/mine_pypi.py

@@ -20,9 +20,9 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from minecode_pipelines.pipes import pypi
-from minecode_pipelines.pipelines import MineCodeBasePipeline
-from minecode_pipelines.pipelines import _mine_and_publish_packageurls
+from minecode.pipes import pypi
+from minecode.pipelines import MineCodeBasePipeline
+from minecode.pipelines import _mine_and_publish_packageurls
 
 
 class MinePypi(MineCodeBasePipeline):

minecode_pipelines/pipelines/mine_swift.py → minecode/pipelines/mine_swift.py

@@ -22,9 +22,9 @@
 
 from scanpipe.pipes import federatedcode
 
-from minecode_pipelines.pipelines import MineCodeBasePipeline
-from minecode_pipelines.pipes.swift import mine_swift_packageurls
-from minecode_pipelines.pipes.swift import load_swift_package_urls
+from minecode.pipelines import MineCodeBasePipeline
+from minecode.pipes.swift import mine_swift_packageurls
+from minecode.pipes.swift import load_swift_package_urls
 
 
 class MineSwift(MineCodeBasePipeline):

minecode_pipelines/pipes/__init__.py → minecode/pipes/__init__.py

@@ -16,7 +16,7 @@
 import requests
 import saneyaml
 
-from minecode_pipelines.utils import get_temp_file
+from minecode.utils import get_temp_file
 
 # states:
 # note: a state is null when mining starts

minecode_pipelines/pipes/alpine.py → minecode/pipes/alpine.py

@@ -30,7 +30,7 @@
 from scanpipe.pipes.fetch import fetch_http
 from scanpipe.pipes.scancode import extract_archives
 
-from minecode_pipelines.pipelines import _mine_and_publish_packageurls
+from minecode.pipelines import _mine_and_publish_packageurls
 
 ALPINE_CHECKPOINT_PATH = "alpine/checkpoints.json"
 

minecode_pipelines/pipes/apache.py → minecode/pipes/apache.py

@@ -9,11 +9,11 @@
 
 from datetime import datetime, timezone
 
-from minecode_pipelines.pipes import fetch_checkpoint_from_github
-from minecode_pipelines.pipes import update_checkpoints_in_github
-from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO
+from minecode.pipes import fetch_checkpoint_from_github
+from minecode.pipes import update_checkpoints_in_github
+from minecode.pipes import MINECODE_PIPELINES_CONFIG_REPO
 
-from minecode_pipelines.utils import get_temp_dir
+from minecode.utils import get_temp_dir
 
 from packageurl import PackageURL
 

minecode_pipelines/pipes/composer.py → minecode/pipes/composer.py

@@ -21,11 +21,11 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 import json
-from minecode_pipelines.utils import get_temp_file
+from minecode.utils import get_temp_file
 from aboutcode.hashid import get_core_purl
 import requests
 from packageurl import PackageURL
-from minecode_pipelines.utils import cycle_from_index, grouper
+from minecode.utils import cycle_from_index, grouper
 
 PACKAGE_BATCH_SIZE = 100