Refine settings handling

* Do not hide settings module imports with settings variable
* Update requirements for pydantic and pydantic_settings
* Restore tests on Python 3.8
* Regenerate tests expectations
* Remove code formatting changes and minor typing changes
* Improve support for defaults and index_urls

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

CHANGELOG.rst

@@ -1,6 +1,38 @@
 Changelog
 =========
 
+v0.13.0
+-----------
+
+- Speed up downloads with asyncio
+
+- New settings featuring environment variables and .env file to store settings and defaults.
+
+  - This also changes the CACHE_THIRDPARTY_DIR environment variable: it used to default first
+    to ".cache/python_inspector" and if not writable, it would fallback to home
+    "~/.cache/python_inspector".  The new behavior is to only use the "~/.cache/python_inspector"
+    in the home directory. You can configure this directory to any other value.
+
+  - Another change is that pypi.org is no longer systematically added as an index URL for
+    resolution. Instead the list of configured index URLs is used, and only defaults to pypi.org
+    if not provided.
+
+  - Another change is that it is possible to only use the provided or configured index URLs
+    and skip other URLs from requirements that are not in these configured URLs.
+
+  - Calling utils_pypi.download_sdist or utils_pypi.download_wheel requires a non-empty list
+    of PypiSimpleRepository.
+
+  - python_inspector.utils_pypi.Distribution.download_url is now a method, not a property
+
+- Drop support for running on Python 3.8. You can still resolve dependencies for Python 3.8.
+  The default command line tool Python version used for resolution is now 3.9.
+
+- Add support for the latest Python and OS versions.
+
+- Merge latest skeleton and adopt ruff for code formatting.
+
+
 v0.12.1
 -----------
 

requirements.txt

@@ -14,7 +14,8 @@ packaging==21.3
 packvers==21.5
 pip-requirements-parser==32.0.1
 pkginfo2==30.0.0
-pydantic_settings >= 2.6.1
+pydantic_settings == 2.8.1
+pydantic == 2.11.2
 pyparsing==3.0.9
 PyYAML==6.0
 requests==2.28.1

setup.cfg

@@ -69,6 +69,9 @@ install_requires =
     toml >= 0.10.0
     mock >= 3.0.5
     packvers >= 21.5
+    pydantic >= 2.10.0
+    pydantic_settings >= 2.8.0
+
 [options.packages.find]
 where = src
 

src/python_inspector/__init__.py

@@ -7,12 +7,9 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-from pydantic import ValidationError
-
-from python_inspector.settings import Settings
+from python_inspector import settings
 
 # Initialize global settings
-try:
-    settings = Settings()
-except ValidationError as e:
-    print(e)
+pyinspector_settings = settings.Settings()
+
+settings.create_cache_directory(pyinspector_settings.CACHE_THIRDPARTY_DIR)

src/python_inspector/api.py

@@ -27,7 +27,7 @@
 from _packagedcode.pypi import PythonSetupPyHandler
 from _packagedcode.pypi import can_process_dependent_package
 from python_inspector import dependencies
-from python_inspector import settings
+from python_inspector import pyinspector_settings as settings
 from python_inspector import utils
 from python_inspector import utils_pypi
 from python_inspector.package_data import get_pypi_data_from_purl
@@ -64,8 +64,7 @@ def to_dict(self, generic_paths=False):
             # clean file paths
             for file in files:
                 path = file["path"]
-                file["path"] = utils.remove_test_data_dir_variable_prefix(
-                    path=path)
+                file["path"] = utils.remove_test_data_dir_variable_prefix(path=path)
         return {
             "files": files,
             "packages": [package for package in self.packages],
@@ -102,19 +101,19 @@ def resolve_dependencies(
     linux OS.
 
     Download from the provided PyPI simple index_urls INDEX(s) URLs defaulting
-    to PyPI.org
+    to PyPI.org or a configured setting.
     """
 
     if not operating_system:
-        raise Exception("No operating system provided.")
+        raise Exception(f"No operating system provided.")
     if operating_system not in PLATFORMS_BY_OS:
         raise ValueError(
             f"Invalid operating system: {operating_system}. "
             f"Must be one of: {', '.join(PLATFORMS_BY_OS.keys())}"
         )
 
     if not python_version:
-        raise Exception("No python version provided.")
+        raise Exception(f"No python version provided.")
     if python_version not in valid_python_versions:
         raise ValueError(
             f"Invalid python version: {python_version}. "
@@ -149,22 +148,16 @@ def resolve_dependencies(
 
     # requirements
     for req_file in requirement_files:
-        deps = dependencies.get_dependencies_from_requirements(
-            requirements_file=req_file)
-        for extra_data in dependencies.get_extra_data_from_requirements(
-            requirements_file=req_file
-        ):
-            index_urls = (
-                *index_urls, *tuple(extra_data.get("extra_index_urls") or []))
-            index_urls = (
-                *index_urls, *tuple(extra_data.get("index_url") or []))
+        deps = dependencies.get_dependencies_from_requirements(requirements_file=req_file)
+        for extra_data in dependencies.get_extra_data_from_requirements(requirements_file=req_file):
+            index_urls = (*index_urls, *tuple(extra_data.get("extra_index_urls") or []))
+            index_urls = (*index_urls, *tuple(extra_data.get("index_url") or []))
         direct_dependencies.extend(deps)
         package_data = [
             pkg_data.to_dict() for pkg_data in PipRequirementsFileHandler.parse(location=req_file)
         ]
         if generic_paths:
-            req_file = utils.remove_test_data_dir_variable_prefix(
-                path=req_file)
+            req_file = utils.remove_test_data_dir_variable_prefix(path=req_file)
 
         files.append(
             dict(
@@ -217,15 +210,13 @@ def resolve_dependencies(
                     files=[setup_py_file],
                     analyze_setup_py_insecurely=analyze_setup_py_insecurely,
                 )
-                setup_py_file_deps = list(
-                    get_dependent_packages_from_reqs(reqs))
+                setup_py_file_deps = list(get_dependent_packages_from_reqs(reqs))
                 direct_dependencies.extend(setup_py_file_deps)
 
         package_data.dependencies = setup_py_file_deps
         file_package_data = [package_data.to_dict()]
         if generic_paths:
-            setup_py_file = utils.remove_test_data_dir_variable_prefix(
-                path=setup_py_file)
+            setup_py_file = utils.remove_test_data_dir_variable_prefix(path=setup_py_file)
         files.append(
             dict(
                 type="file",
@@ -254,29 +245,32 @@ def resolve_dependencies(
     if verbose:
         printer(f"environment: {environment}")
 
-    repos = []
+    repos_by_url = {}
     if not use_pypi_json_api:
         # Collect PyPI repos
+        use_only_confed = settings.USE_ONLY_CONFIGURED_INDEX_URLS
         for index_url in index_urls:
             index_url = index_url.strip("/")
-            if index_url in settings.INDEX_URL:
-                repos.append(PypiSimpleRepository(index_url))
-            else:
-                credentials = None
-                if parsed_netrc:
-                    login, password = utils.get_netrc_auth(
-                        index_url, parsed_netrc)
-                    credentials = (
-                        dict(login=login,
-                             password=password) if login and password else None
-                    )
-                repo = PypiSimpleRepository(
-                    index_url=index_url,
-                    use_cached_index=use_cached_index,
-                    credentials=credentials,
-                )
-                repos.append(repo)
+            if use_only_confed and index_url not in settings.INDEX_URL:
+                if verbose:
+                    printer(f"Skipping index URL unknown in settings: {index_url!r}")
+                continue
+            if index_url in repos_by_url:
+                continue
+
+            credentials = None
+            if parsed_netrc:
+                login, password = utils.get_netrc_auth(index_url, parsed_netrc)
+                if login and password:
+                    credentials = dict(login=login, password=password)
+            repo = utils_pypi.PypiSimpleRepository(
+                index_url=index_url,
+                use_cached_index=use_cached_index,
+                credentials=credentials,
+            )
+            repos_by_url[index_url] = repo
 
+    repos = repos_by_url.values()
     if verbose:
         printer("repos:")
         for repo in repos:
@@ -363,8 +357,8 @@ def resolve(
 
 def get_resolved_dependencies(
     requirements: List[Requirement],
-    environment: Environment,
-    repos: Sequence[PypiSimpleRepository] = tuple(),
+    environment: Environment = None,
+    repos: Sequence[utils_pypi.PypiSimpleRepository] = tuple(),
     as_tree: bool = False,
     max_rounds: int = 200000,
     pdt_output: bool = False,
@@ -379,7 +373,6 @@ def get_resolved_dependencies(
     Used the provided ``repos`` list of PypiSimpleRepository.
     If empty, use instead the PyPI.org JSON API exclusively instead
     """
-
     resolver = Resolver(
         provider=PythonInputProvider(
             environment=environment,
@@ -389,12 +382,8 @@ def get_resolved_dependencies(
         ),
         reporter=BaseReporter(),
     )
-
-    resolver_results = resolver.resolve(
-        requirements=requirements, max_rounds=max_rounds)
-
+    resolver_results = resolver.resolve(requirements=requirements, max_rounds=max_rounds)
     package_list = get_package_list(results=resolver_results)
-
     if pdt_output:
         return (format_pdt_tree(resolver_results), package_list)
     return (

src/python_inspector/cli_utils.py

@@ -21,8 +21,7 @@ class FileOptionType(click.File):
 
     def convert(self, value, param, ctx):
         known_opts = set(
-            chain.from_iterable(
-                p.opts for p in ctx.command.params if isinstance(p, click.Option))
+            chain.from_iterable(p.opts for p in ctx.command.params if isinstance(p, click.Option))
         )
         if value in known_opts:
             self.fail(

src/python_inspector/package_data.py

@@ -9,7 +9,6 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-from collections.abc import Generator
 from typing import List
 
 from packageurl import PackageURL
@@ -26,10 +25,7 @@
 
 
 def get_pypi_data_from_purl(
-    purl: str,
-    environment: Environment,
-    repos: list[PypiSimpleRepository],
-    prefer_source: bool,
+    purl: str, environment: Environment, repos: List[PypiSimpleRepository], prefer_source: bool
 ) -> PackageData:
     """
     Generate `Package` object from the `purl` string of pypi type
@@ -40,9 +36,9 @@ def get_pypi_data_from_purl(
     ``prefer_source`` is a boolean value to prefer source distribution over wheel,
     if no source distribution is available then wheel is used
     """
-    packageurl: PackageURL = PackageURL.from_string(purl)
-    name = packageurl.name
-    version = packageurl.version
+    purl = PackageURL.from_string(purl)
+    name = purl.name
+    version = purl.version
     if not version:
         raise Exception("Version is not specified in the purl")
     base_path = "https://pypi.org/pypi"
@@ -57,14 +53,12 @@ def get_pypi_data_from_purl(
     project_urls = info.get("project_urls") or {}
     code_view_url = get_pypi_codeview_url(project_urls)
     bug_tracking_url = get_pypi_bugtracker_url(project_urls)
-    python_version = get_python_version_from_env_tag(
-        python_version=environment.python_version
-    )
+    python_version = get_python_version_from_env_tag(python_version=environment.python_version)
     valid_distribution_urls = []
 
     valid_distribution_urls.append(
         get_sdist_download_url(
-            purl=packageurl,
+            purl=purl,
             repos=repos,
             python_version=python_version,
         )
@@ -75,7 +69,7 @@ def get_pypi_data_from_purl(
     if not valid_distribution_urls or not prefer_source:
         wheel_urls = list(
             get_wheel_download_urls(
-                purl=packageurl,
+                purl=purl,
                 repos=repos,
                 environment=environment,
                 python_version=python_version,
@@ -113,7 +107,7 @@ def get_pypi_data_from_purl(
                 maintainer_key="maintainer",
                 maintainer_email_key="maintainer_email",
             ),
-            **packageurl.to_dict(),
+            **purl.to_dict(),
         )
 
 
@@ -149,7 +143,7 @@ def get_wheel_download_urls(
     repos: List[PypiSimpleRepository],
     environment: Environment,
     python_version: str,
-) -> Generator[str, None, None]:
+) -> List[str]:
     """
     Return a list of download urls for the given purl.
     """
@@ -161,7 +155,7 @@ def get_wheel_download_urls(
             environment=environment,
             python_version=python_version,
         ):
-            yield wheel.download_url
+            yield wheel.download_url(repo)
 
 
 def get_sdist_download_url(
@@ -178,4 +172,4 @@ def get_sdist_download_url(
             python_version=python_version,
         )
         if sdist:
-            return sdist.download_url
+            return sdist.download_url(repo)