From 4589536b1c60c532b74a0d2f7e8defac591427d0 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Tue, 24 Mar 2026 16:13:37 +0100 Subject: [PATCH] feat(python): add pyproject.toml provider support Add support for pyproject.toml manifest files in the Java client library, enabling analysis of Python projects using Poetry, uv, or PEP 621-compliant tools. - New PythonPyprojectProvider that parses TOML dependency sections and generates temporary requirements.txt for dependency tree resolution - Refactor shared Python infrastructure into PythonProvider base class - Support PEP 621 [project.dependencies] and Poetry [tool.poetry.dependencies] (production deps only) - Convert Poetry version operators (^ and ~) to PEP 440 ranges - Handle bare versions, pre-release suffixes, and ignore patterns - Add license resolution and identification support - Update README with pyproject.toml documentation - Comprehensive unit tests for parsing, conversion, and edge cases Jira-Issue: TC-3851 Assisted-by: Claude Code Co-Authored-By: Claude Opus 4.6 --- README.md | 31 +- .../providers/PythonPipProvider.java | 273 +--------------- .../trustifyda/providers/PythonProvider.java | 295 ++++++++++++++++++ .../providers/PythonPyprojectProvider.java | 187 +++++++++++ .../guacsec/trustifyda/tools/Ecosystem.java | 2 + .../Python_Pyproject_Provider_Test.java | 193 ++++++++++++ .../pip_pyproject_toml_ignore/pyproject.toml | 13 + .../pyproject.toml | 13 + .../pip_pyproject_toml_poetry/pyproject.toml | 12 + 9 files changed, 756 insertions(+), 263 deletions(-) create mode 100644 src/main/java/io/github/guacsec/trustifyda/providers/PythonProvider.java create mode 100644 src/main/java/io/github/guacsec/trustifyda/providers/PythonPyprojectProvider.java create mode 100644 src/test/java/io/github/guacsec/trustifyda/providers/Python_Pyproject_Provider_Test.java create mode 100644 src/test/resources/tst_manifests/pip/pip_pyproject_toml_ignore/pyproject.toml create mode 100644 src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml create mode 100644 src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry/pyproject.toml diff --git a/README.md b/README.md index 4dde423c..6281af21 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ public class TrustifyExample {
  • Java - Maven
  • JavaScript - Npm
  • Golang - Go Modules
    • -
  • Python - pip Installer
    • +
  • Python - pip Installer (requirements.txt, pyproject.toml)
  • Gradle - Gradle Installation
  • Rust - Cargo
    • @@ -264,7 +264,7 @@ require (
  • -Python pip users can add in requirement text a comment with #trustify-da-ignore(or # trustify-da-ignore) to the right of the same artifact to be ignored, for example: +Python pip users can add in requirements.txt a comment with #trustify-da-ignore(or # trustify-da-ignore) to the right of the same artifact to be ignored, for example: ```properties anyio==3.6.2 @@ -297,6 +297,20 @@ zipp==3.6.0 ```
    • +
  • +Python pyproject.toml users can add a comment with #trustify-da-ignore next to a dependency in pyproject.toml: + +```toml +[project] +name = "my-project" +dependencies = [ + "requests>=2.28.1", + "flask>=2.0", # trustify-da-ignore + "click>=8.0", +] +``` +
    • +
  • Gradle users can add in build.gradle a comment with //trustify-da-ignore next to the package to be ignored: ```build.gradle @@ -538,7 +552,7 @@ System.setProperty("TRUSTIFY_DA_MVN_LOCAL_REPO", "/home/user/custom-maven-repo") ##### Background In Python pip and in golang go modules package managers ( especially in Python pip) , There is a big chance that for a certain manifest and a given package inside it, the client machine environment has different version installed/resolved -for that package, which can lead to perform the analysis on the installed packages' versions , instead on the declared versions ( in manifests - that is requirements.txt/go.mod ), and this +for that package, which can lead to perform the analysis on the installed packages' versions , instead on the declared versions ( in manifests - that is requirements.txt/pyproject.toml/go.mod ), and this can cause a confusion for the user in the client consuming the API and leads to inconsistent output ( in THE manifest there is version X For a given Package `A` , and in the analysis report there is another version for the same package `A` - Y). ##### Usage @@ -579,16 +593,18 @@ TRUSTIFY_DA_GO_MVS_LOGIC_ENABLED=false #### Python Support +Python support works with both `requirements.txt` and `pyproject.toml` manifest files. The `pyproject.toml` provider scans production dependencies from PEP 621 `[project.dependencies]` and Poetry `[tool.poetry.dependencies]` sections. Optional dependencies (`[project.optional-dependencies]`) and Poetry group dependencies (`[tool.poetry.group.*.dependencies]`) are intentionally excluded to focus on runtime dependencies. + By default, Python support assumes that the package is installed using the pip/pip3 binary on the system PATH, or of the customized Binaries passed to environment variables. If the package is not installed , then an error will be thrown. -There is an experimental feature of installing the requirement.txt on a virtual env(only python3 or later is supported for this feature) - in this case, +There is an experimental feature of installing the dependencies on a virtual env(only python3 or later is supported for this feature) - in this case, it's important to pass in a path to python3 binary as `TRUSTIFY_DA_PYTHON3_PATH` or instead make sure that python3 is on the system path. -in such case, You can use that feature by setting environment variable `TRUSTIFY_DA_PYTHON_VIRTUAL_ENV` to true +in such case, You can use that feature by setting environment variable `TRUSTIFY_DA_PYTHON_VIRTUAL_ENV` to true ##### "Best Efforts Installation" Since Python pip packages are very sensitive/picky regarding python version changes( every small range of versions is only tailored for a certain python version), I'm introducing this feature, that -tries to install all packages in requirements.txt onto created virtual environment while **disregarding** versions declared for packages in requirements.txt +tries to install all packages in the manifest onto created virtual environment while **disregarding** versions declared for packages This increasing the chances and the probability a lot that the automatic installation will succeed. ##### Usage @@ -693,6 +709,9 @@ java -jar trustify-da-java-client-cli.jar stack /path/to/build.gradle --html # Component analysis with JSON output (default) java -jar trustify-da-java-client-cli.jar component /path/to/requirements.txt +# Component analysis for pyproject.toml +java -jar trustify-da-java-client-cli.jar component /path/to/pyproject.toml + # Component analysis with summary java -jar trustify-da-java-client-cli.jar component /path/to/go.mod --summary diff --git a/src/main/java/io/github/guacsec/trustifyda/providers/PythonPipProvider.java b/src/main/java/io/github/guacsec/trustifyda/providers/PythonPipProvider.java index 9d6ce7ef..ab44ceac 100644 --- a/src/main/java/io/github/guacsec/trustifyda/providers/PythonPipProvider.java +++ b/src/main/java/io/github/guacsec/trustifyda/providers/PythonPipProvider.java @@ -16,196 +16,42 @@ */ package io.github.guacsec.trustifyda.providers; -import static io.github.guacsec.trustifyda.impl.ExhortApi.debugLoggingIsNeeded; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; -import io.github.guacsec.trustifyda.Api; -import io.github.guacsec.trustifyda.Provider; -import io.github.guacsec.trustifyda.license.LicenseUtils; -import io.github.guacsec.trustifyda.logging.LoggersFactory; -import io.github.guacsec.trustifyda.sbom.Sbom; -import io.github.guacsec.trustifyda.sbom.SbomFactory; -import io.github.guacsec.trustifyda.tools.Ecosystem; -import io.github.guacsec.trustifyda.tools.Operations; -import io.github.guacsec.trustifyda.utils.Environment; -import io.github.guacsec.trustifyda.utils.IgnorePatternDetector; import io.github.guacsec.trustifyda.utils.PythonControllerBase; -import io.github.guacsec.trustifyda.utils.PythonControllerRealEnv; -import io.github.guacsec.trustifyda.utils.PythonControllerVirtualEnv; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; -import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.logging.Logger; import java.util.stream.Collectors; -public final class PythonPipProvider extends Provider { - - private static final Logger log = LoggersFactory.getLogger(PythonPipProvider.class.getName()); - private static final String DEFAULT_PIP_ROOT_COMPONENT_NAME = "default-pip-root"; - private static final String DEFAULT_PIP_ROOT_COMPONENT_VERSION = "0.0.0"; - - public void setPythonController(PythonControllerBase pythonController) { - this.pythonController = pythonController; - } - - private PythonControllerBase pythonController; +public final class PythonPipProvider extends PythonProvider { public PythonPipProvider(Path manifest) { - super(Ecosystem.Type.PYTHON, manifest); + super(manifest); } @Override - public String readLicenseFromManifest() { - return LicenseUtils.readLicenseFile(manifest); + protected Path getRequirementsPath() { + return manifest; } @Override - public Content provideStack() throws IOException { - PythonControllerBase pythonController = getPythonController(); - List> dependencies = - pythonController.getDependencies(manifest.toString(), true); - printDependenciesTree(dependencies); - Sbom sbom = SbomFactory.newInstance(Sbom.BelongingCondition.PURL, "sensitive"); - sbom.addRoot( - toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME, DEFAULT_PIP_ROOT_COMPONENT_VERSION), - readLicenseFromManifest()); - for (Map component : dependencies) { - addAllDependencies(sbom.getRoot(), component, sbom); - } - byte[] requirementsFile = Files.readAllBytes(manifest); - handleIgnoredDependencies(new String(requirementsFile), sbom); - return new Content( - sbom.getAsJsonString().getBytes(StandardCharsets.UTF_8), Api.CYCLONEDX_MEDIA_TYPE); - } - - private void addAllDependencies(PackageURL source, Map component, Sbom sbom) { - - PackageURL packageURL = - toPurl((String) component.get("name"), (String) component.get("version")); - sbom.addDependency(source, packageURL, null); - - List> directDeps = - (List>) component.get("dependencies"); - if (directDeps != null) { - for (Map dep : directDeps) { - addAllDependencies(packageURL, dep, sbom); - } - } + protected void cleanupRequirementsPath(Path requirementsPath) { + // No cleanup needed — the manifest is the requirements file itself. } @Override - public Content provideComponent() throws IOException { - PythonControllerBase pythonController = getPythonController(); - List> dependencies = - pythonController.getDependencies(manifest.toString(), false); - printDependenciesTree(dependencies); - Sbom sbom = SbomFactory.newInstance(); - sbom.addRoot( - toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME, DEFAULT_PIP_ROOT_COMPONENT_VERSION), - readLicenseFromManifest()); - dependencies.forEach( - (component) -> - sbom.addDependency( - sbom.getRoot(), - toPurl((String) component.get("name"), (String) component.get("version")), - null)); - - var manifestContent = Files.readString(manifest); - handleIgnoredDependencies(manifestContent, sbom); - return new Content( - sbom.getAsJsonString().getBytes(StandardCharsets.UTF_8), Api.CYCLONEDX_MEDIA_TYPE); + protected Set getIgnoredDependencies(String manifestContent) { + String[] lines = manifestContent.split(System.lineSeparator()); + return Arrays.stream(lines) + .filter(this::containsIgnorePattern) + .map(PythonPipProvider::extractDepFull) + .map(this::splitToNameVersion) + .map(dep -> toPurl(dep[0], dep[1])) + .collect(Collectors.toSet()); } - private void printDependenciesTree(List> dependencies) - throws JsonProcessingException { - if (debugLoggingIsNeeded()) { - String pythonControllerTree = - objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(dependencies); - log.info( - String.format( - "Python Generated Dependency Tree in Json Format: %s %s %s", - System.lineSeparator(), pythonControllerTree, System.lineSeparator())); - } - } - - private void handleIgnoredDependencies(String manifestContent, Sbom sbom) { - Set ignoredDeps = getIgnoredDependencies(manifestContent); - Set ignoredDepsVersions = - ignoredDeps.stream() - .filter(dep -> !dep.getVersion().trim().equals("*")) - .map(PackageURL::getCoordinates) - .collect(Collectors.toSet()); - Set ignoredDepsNoVersions = - ignoredDeps.stream() - .filter(dep -> dep.getVersion().trim().equals("*")) - .map(PackageURL::getCoordinates) - .collect(Collectors.toSet()); - - // filter out by name only from sbom all exhortignore dependencies that their version will be - // resolved by pip. - sbom.setBelongingCriteriaBinaryAlgorithm(Sbom.BelongingCondition.NAME); - sbom.filterIgnoredDeps(ignoredDepsNoVersions); - boolean matchManifestVersions = Environment.getBoolean(PROP_MATCH_MANIFEST_VERSIONS, true); - // filter out by purl from sbom all exhortignore dependencies that their version hardcoded in - // requirements.txt - - // in case all versions in manifest matching installed versions of packages in environment. - if (matchManifestVersions) { - sbom.setBelongingCriteriaBinaryAlgorithm(Sbom.BelongingCondition.PURL); - sbom.filterIgnoredDeps(ignoredDepsVersions); - } else { - // in case version mismatch is possible (MATCH_MANIFEST_VERSIONS=false) , need to parse the - // name of package - // from the purl, and remove the package name from sbom according to name only - Set deps = - ignoredDepsVersions.stream() - .map( - purlString -> { - try { - return new PackageURL(purlString).getName(); - } catch (MalformedPackageURLException e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toSet()); - sbom.setBelongingCriteriaBinaryAlgorithm(Sbom.BelongingCondition.NAME); - sbom.filterIgnoredDeps(deps); - } - } - - /** - * Checks if a text line contains a Python pip ignore pattern. Handles both '#exhortignore' and - * '#trustify-da-ignore' with optional spacing. - * - * @param line the line to check - * @return true if the line contains a Python pip ignore pattern - */ - private boolean containsPythonIgnorePattern(String line) { - return line.contains("#" + IgnorePatternDetector.IGNORE_PATTERN) - || line.contains("# " + IgnorePatternDetector.IGNORE_PATTERN) - || line.contains("#" + IgnorePatternDetector.LEGACY_IGNORE_PATTERN) - || line.contains("# " + IgnorePatternDetector.LEGACY_IGNORE_PATTERN); - } - - private Set getIgnoredDependencies(String requirementsDeps) { - - String[] requirementsLines = requirementsDeps.split(System.lineSeparator()); - Set collected = - Arrays.stream(requirementsLines) - .filter(this::containsPythonIgnorePattern) - .map(PythonPipProvider::extractDepFull) - .map(this::splitToNameVersion) - .map(dep -> toPurl(dep[0], dep[1])) - // .map(packageURL -> packageURL.getCoordinates()) - .collect(Collectors.toSet()); - - return collected; + private static String extractDepFull(String requirementLine) { + return requirementLine.substring(0, requirementLine.indexOf("#")).trim(); } private String[] splitToNameVersion(String nameVersion) { @@ -219,91 +65,4 @@ private String[] splitToNameVersion(String nameVersion) { } return result; } - - private static String extractDepFull(String requirementLine) { - return requirementLine.substring(0, requirementLine.indexOf("#")).trim(); - } - - private PackageURL toPurl(String name, String version) { - - try { - return new PackageURL(Ecosystem.Type.PYTHON.getType(), null, name, version, null, null); - } catch (MalformedPackageURLException e) { - throw new RuntimeException(e); - } - } - - private PythonControllerBase getPythonController() { - String pythonPipBinaries; - boolean useVirtualPythonEnv; - if (!Environment.get(PythonControllerBase.PROP_TRUSTIFY_DA_PIP_SHOW, "").trim().isEmpty() - && !Environment.get(PythonControllerBase.PROP_TRUSTIFY_DA_PIP_FREEZE, "") - .trim() - .isEmpty()) { - pythonPipBinaries = "python;;pip"; - useVirtualPythonEnv = false; - } else { - pythonPipBinaries = getExecutable("python", "--version"); - useVirtualPythonEnv = - Environment.getBoolean(PythonControllerBase.PROP_TRUSTIFY_DA_PYTHON_VIRTUAL_ENV, false); - } - - String[] parts = pythonPipBinaries.split(";;"); - var python = parts[0]; - var pip = parts[1]; - PythonControllerBase pythonController; - if (this.pythonController == null) { - if (useVirtualPythonEnv) { - pythonController = new PythonControllerVirtualEnv(python); - } else { - pythonController = new PythonControllerRealEnv(python, pip); - } - } else { - pythonController = this.pythonController; - } - return pythonController; - } - - private String getExecutable(String command, String args) { - String python = Operations.getCustomPathOrElse("python3"); - String pip = Operations.getCustomPathOrElse("pip3"); - try { - Operations.runProcess(python, args); - Operations.runProcess(pip, args); - } catch (Exception e) { - python = Operations.getCustomPathOrElse("python"); - pip = Operations.getCustomPathOrElse("pip"); - try { - Process process = new ProcessBuilder(command, args).redirectErrorStream(true).start(); - int exitCode = process.waitFor(); - if (exitCode != 0) { - throw new IOException( - "Python executable found, but it exited with error code " + exitCode); - } - } catch (IOException | InterruptedException ex) { - throw new RuntimeException( - String.format( - "Unable to find or run Python executable '%s'. Please ensure Python is installed" - + " and available in your PATH.", - command), - ex); - } - - try { - Process process = new ProcessBuilder("pip", args).redirectErrorStream(true).start(); - int exitCode = process.waitFor(); - if (exitCode != 0) { - throw new IOException("Pip executable found, but it exited with error code " + exitCode); - } - } catch (IOException | InterruptedException ex) { - throw new RuntimeException( - String.format( - "Unable to find or run Pip executable '%s'. Please ensure Pip is installed and" - + " available in your PATH.", - command), - ex); - } - } - return String.format("%s;;%s", python, pip); - } } diff --git a/src/main/java/io/github/guacsec/trustifyda/providers/PythonProvider.java b/src/main/java/io/github/guacsec/trustifyda/providers/PythonProvider.java new file mode 100644 index 00000000..cfd6d47e --- /dev/null +++ b/src/main/java/io/github/guacsec/trustifyda/providers/PythonProvider.java @@ -0,0 +1,295 @@ +/* + * Copyright 2023-2025 Trustify Dependency Analytics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.guacsec.trustifyda.providers; + +import static io.github.guacsec.trustifyda.impl.ExhortApi.debugLoggingIsNeeded; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import io.github.guacsec.trustifyda.Api; +import io.github.guacsec.trustifyda.Provider; +import io.github.guacsec.trustifyda.license.LicenseUtils; +import io.github.guacsec.trustifyda.logging.LoggersFactory; +import io.github.guacsec.trustifyda.sbom.Sbom; +import io.github.guacsec.trustifyda.sbom.SbomFactory; +import io.github.guacsec.trustifyda.tools.Ecosystem; +import io.github.guacsec.trustifyda.tools.Operations; +import io.github.guacsec.trustifyda.utils.Environment; +import io.github.guacsec.trustifyda.utils.IgnorePatternDetector; +import io.github.guacsec.trustifyda.utils.PythonControllerBase; +import io.github.guacsec.trustifyda.utils.PythonControllerRealEnv; +import io.github.guacsec.trustifyda.utils.PythonControllerVirtualEnv; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Abstract base class for Python providers. Encapsulates shared Python infrastructure including + * controller resolution, executable discovery, SBOM construction, and ignore-pattern handling. + */ +public abstract class PythonProvider extends Provider { + + private static final Logger log = LoggersFactory.getLogger(PythonProvider.class.getName()); + static final String DEFAULT_PIP_ROOT_COMPONENT_NAME = "default-pip-root"; + static final String DEFAULT_PIP_ROOT_COMPONENT_VERSION = "0.0.0"; + + private PythonControllerBase pythonController; + + protected PythonProvider(Path manifest) { + super(Ecosystem.Type.PYTHON, manifest); + } + + public void setPythonController(PythonControllerBase pythonController) { + this.pythonController = pythonController; + } + + @Override + public String readLicenseFromManifest() { + return LicenseUtils.readLicenseFile(manifest); + } + + /** + * Returns the path to a requirements-format file that the {@link PythonControllerBase} can + * consume. For requirements.txt this is the manifest itself; for pyproject.toml a temporary file + * is generated. + */ + protected abstract Path getRequirementsPath() throws IOException; + + /** Clean up any temporary files created by {@link #getRequirementsPath()}. */ + protected abstract void cleanupRequirementsPath(Path requirementsPath) throws IOException; + + /** Parse ignored dependencies from the raw manifest content. */ + protected abstract Set getIgnoredDependencies(String manifestContent); + + @Override + public Content provideStack() throws IOException { + Path requirementsPath = getRequirementsPath(); + try { + PythonControllerBase controller = getPythonController(); + List> dependencies = + controller.getDependencies(requirementsPath.toString(), true); + printDependenciesTree(dependencies); + Sbom sbom = SbomFactory.newInstance(Sbom.BelongingCondition.PURL, "sensitive"); + sbom.addRoot( + toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME, DEFAULT_PIP_ROOT_COMPONENT_VERSION), + readLicenseFromManifest()); + for (Map component : dependencies) { + addAllDependencies(sbom.getRoot(), component, sbom); + } + String manifestContent = Files.readString(manifest); + handleIgnoredDependencies(manifestContent, sbom); + return new Content( + sbom.getAsJsonString().getBytes(StandardCharsets.UTF_8), Api.CYCLONEDX_MEDIA_TYPE); + } finally { + try { + cleanupRequirementsPath(requirementsPath); + } catch (IOException e) { + log.warning("Failed to clean up temporary requirements file: " + e.getMessage()); + } + } + } + + @Override + public Content provideComponent() throws IOException { + Path requirementsPath = getRequirementsPath(); + try { + PythonControllerBase controller = getPythonController(); + List> dependencies = + controller.getDependencies(requirementsPath.toString(), false); + printDependenciesTree(dependencies); + Sbom sbom = SbomFactory.newInstance(); + sbom.addRoot( + toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME, DEFAULT_PIP_ROOT_COMPONENT_VERSION), + readLicenseFromManifest()); + dependencies.forEach( + (component) -> + sbom.addDependency( + sbom.getRoot(), + toPurl((String) component.get("name"), (String) component.get("version")), + null)); + String manifestContent = Files.readString(manifest); + handleIgnoredDependencies(manifestContent, sbom); + return new Content( + sbom.getAsJsonString().getBytes(StandardCharsets.UTF_8), Api.CYCLONEDX_MEDIA_TYPE); + } finally { + try { + cleanupRequirementsPath(requirementsPath); + } catch (IOException e) { + log.warning("Failed to clean up temporary requirements file: " + e.getMessage()); + } + } + } + + @SuppressWarnings("unchecked") + private void addAllDependencies(PackageURL source, Map component, Sbom sbom) { + PackageURL packageURL = + toPurl((String) component.get("name"), (String) component.get("version")); + sbom.addDependency(source, packageURL, null); + + List> directDeps = + (List>) component.get("dependencies"); + if (directDeps != null) { + for (Map dep : directDeps) { + addAllDependencies(packageURL, dep, sbom); + } + } + } + + private void printDependenciesTree(List> dependencies) + throws JsonProcessingException { + if (debugLoggingIsNeeded()) { + String pythonControllerTree = + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(dependencies); + log.info( + String.format( + "Python Generated Dependency Tree in Json Format: %s %s %s", + System.lineSeparator(), pythonControllerTree, System.lineSeparator())); + } + } + + private void handleIgnoredDependencies(String manifestContent, Sbom sbom) { + Set ignoredDeps = getIgnoredDependencies(manifestContent); + Set ignoredDepsVersions = + ignoredDeps.stream() + .filter(dep -> !dep.getVersion().trim().equals("*")) + .map(PackageURL::getCoordinates) + .collect(Collectors.toSet()); + Set ignoredDepsNoVersions = + ignoredDeps.stream() + .filter(dep -> dep.getVersion().trim().equals("*")) + .map(PackageURL::getName) + .collect(Collectors.toSet()); + + sbom.setBelongingCriteriaBinaryAlgorithm(Sbom.BelongingCondition.NAME); + sbom.filterIgnoredDeps(ignoredDepsNoVersions); + boolean matchManifestVersions = Environment.getBoolean(PROP_MATCH_MANIFEST_VERSIONS, true); + if (matchManifestVersions) { + sbom.setBelongingCriteriaBinaryAlgorithm(Sbom.BelongingCondition.PURL); + sbom.filterIgnoredDeps(ignoredDepsVersions); + } else { + Set deps = + ignoredDepsVersions.stream() + .map( + purlString -> { + try { + return new PackageURL(purlString).getName(); + } catch (MalformedPackageURLException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toSet()); + sbom.setBelongingCriteriaBinaryAlgorithm(Sbom.BelongingCondition.NAME); + sbom.filterIgnoredDeps(deps); + } + } + + protected boolean containsIgnorePattern(String line) { + return line.contains("#" + IgnorePatternDetector.IGNORE_PATTERN) + || line.contains("# " + IgnorePatternDetector.IGNORE_PATTERN) + || line.contains("#" + IgnorePatternDetector.LEGACY_IGNORE_PATTERN) + || line.contains("# " + IgnorePatternDetector.LEGACY_IGNORE_PATTERN); + } + + protected PackageURL toPurl(String name, String version) { + try { + return new PackageURL(Ecosystem.Type.PYTHON.getType(), null, name, version, null, null); + } catch (MalformedPackageURLException e) { + throw new RuntimeException(e); + } + } + + protected PythonControllerBase getPythonController() { + String pythonPipBinaries; + boolean useVirtualPythonEnv; + if (!Environment.get(PythonControllerBase.PROP_TRUSTIFY_DA_PIP_SHOW, "").trim().isEmpty() + && !Environment.get(PythonControllerBase.PROP_TRUSTIFY_DA_PIP_FREEZE, "") + .trim() + .isEmpty()) { + pythonPipBinaries = "python;;pip"; + useVirtualPythonEnv = false; + } else { + pythonPipBinaries = getExecutable("python", "--version"); + useVirtualPythonEnv = + Environment.getBoolean(PythonControllerBase.PROP_TRUSTIFY_DA_PYTHON_VIRTUAL_ENV, false); + } + + String[] parts = pythonPipBinaries.split(";;"); + var python = parts[0]; + var pip = parts[1]; + PythonControllerBase controller; + if (this.pythonController == null) { + if (useVirtualPythonEnv) { + controller = new PythonControllerVirtualEnv(python); + } else { + controller = new PythonControllerRealEnv(python, pip); + } + } else { + controller = this.pythonController; + } + return controller; + } + + private String getExecutable(String command, String args) { + String python = Operations.getCustomPathOrElse("python3"); + String pip = Operations.getCustomPathOrElse("pip3"); + try { + Operations.runProcess(python, args); + Operations.runProcess(pip, args); + } catch (Exception e) { + python = Operations.getCustomPathOrElse("python"); + pip = Operations.getCustomPathOrElse("pip"); + try { + Process process = new ProcessBuilder(command, args).redirectErrorStream(true).start(); + int exitCode = process.waitFor(); + if (exitCode != 0) { + throw new IOException( + "Python executable found, but it exited with error code " + exitCode); + } + } catch (IOException | InterruptedException ex) { + throw new RuntimeException( + String.format( + "Unable to find or run Python executable '%s'. Please ensure Python is installed" + + " and available in your PATH.", + command), + ex); + } + + try { + Process process = new ProcessBuilder("pip", args).redirectErrorStream(true).start(); + int exitCode = process.waitFor(); + if (exitCode != 0) { + throw new IOException("Pip executable found, but it exited with error code " + exitCode); + } + } catch (IOException | InterruptedException ex) { + throw new RuntimeException( + String.format( + "Unable to find or run Pip executable '%s'. Please ensure Pip is installed and" + + " available in your PATH.", + command), + ex); + } + } + return String.format("%s;;%s", python, pip); + } +} diff --git a/src/main/java/io/github/guacsec/trustifyda/providers/PythonPyprojectProvider.java b/src/main/java/io/github/guacsec/trustifyda/providers/PythonPyprojectProvider.java new file mode 100644 index 00000000..057e4b68 --- /dev/null +++ b/src/main/java/io/github/guacsec/trustifyda/providers/PythonPyprojectProvider.java @@ -0,0 +1,187 @@ +/* + * Copyright 2023-2025 Trustify Dependency Analytics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.guacsec.trustifyda.providers; + +import com.github.packageurl.PackageURL; +import io.github.guacsec.trustifyda.utils.PythonControllerBase; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.tomlj.Toml; +import org.tomlj.TomlArray; +import org.tomlj.TomlParseResult; +import org.tomlj.TomlTable; + +public final class PythonPyprojectProvider extends PythonProvider { + + private Set collectedIgnoredDeps; + + public PythonPyprojectProvider(Path manifest) { + super(manifest); + } + + @Override + protected Path getRequirementsPath() throws IOException { + List depStrings = parseDependencyStrings(); + Path tmpDir = Files.createTempDirectory("trustify_da_pyproject_"); + Path tmpFile = Files.createFile(tmpDir.resolve("requirements.txt")); + Files.write(tmpFile, depStrings); + return tmpFile; + } + + @Override + protected void cleanupRequirementsPath(Path requirementsPath) throws IOException { + Files.deleteIfExists(requirementsPath); + Files.deleteIfExists(requirementsPath.getParent()); + } + + @Override + protected Set getIgnoredDependencies(String manifestContent) { + if (collectedIgnoredDeps == null) { + return Set.of(); + } + return collectedIgnoredDeps.stream() + .map( + dep -> { + String name = PythonControllerBase.getDependencyName(dep); + return toPurl(name, "*"); + }) + .collect(Collectors.toSet()); + } + + List parseDependencyStrings() throws IOException { + TomlParseResult toml = Toml.parse(manifest); + if (toml.hasErrors()) { + throw new IOException("Invalid pyproject.toml format: " + toml.errors().get(0).getMessage()); + } + + List rawLines = Files.readAllLines(manifest); + collectedIgnoredDeps = new HashSet<>(); + List deps = new ArrayList<>(); + + // [project.dependencies] - PEP 621 + TomlArray projectDeps = toml.getArray("project.dependencies"); + if (projectDeps != null) { + for (int i = 0; i < projectDeps.size(); i++) { + String dep = projectDeps.getString(i); + deps.add(dep); + checkIgnored(rawLines, dep, dep); + } + } + + // [tool.poetry.dependencies] - production only + TomlTable poetryDeps = toml.getTable("tool.poetry.dependencies"); + if (poetryDeps != null) { + for (String name : poetryDeps.keySet()) { + if (!"python".equalsIgnoreCase(name)) { + deps.add(poetryDepToRequirement(name, poetryDeps, name)); + checkIgnored(rawLines, name, name); + } + } + } + + return deps; + } + + private void checkIgnored(List rawLines, String searchToken, String depIdentifier) { + for (String line : rawLines) { + if (line.contains(searchToken) && containsIgnorePattern(line)) { + collectedIgnoredDeps.add(depIdentifier); + break; + } + } + } + + /** + * Converts a Poetry dependency entry to a pip-compatible requirement string. Poetry uses {@code + * ^} and {@code ~} operators which are not PEP 440, so they must be converted to PEP 440 ranges. + */ + static String poetryDepToRequirement(String name, TomlTable table, String key) { + String version = null; + if (table.isString(key)) { + version = table.getString(key); + } else if (table.isTable(key)) { + TomlTable depTable = table.getTable(key); + if (depTable != null) { + version = depTable.getString("version"); + } + } + if (version == null || version.isEmpty() || "*".equals(version)) { + return name; + } + return name + convertPoetryVersion(version); + } + + /** + * Converts a Poetry version constraint to PEP 440 format. + * + *
      + *
    • {@code ^X.Y.Z} → {@code >=X.Y.Z,<(X+1).0.0} (when X > 0) + *
    • {@code ^0.Y.Z} → {@code >=0.Y.Z,<0.(Y+1).0} (when Y > 0) + *
    • {@code ^0.0.Z} → {@code >=0.0.Z,<0.0.(Z+1)} + *
    • {@code ~X.Y.Z} → {@code >=X.Y.Z,PEP 440 operators ({@code >=}, {@code ==}, etc.) are passed through unchanged + *
    + */ + static String convertPoetryVersion(String version) { + if (version.startsWith("^")) { + return convertCaret(version.substring(1)); + } + if (version.startsWith("~") && !version.startsWith("~=")) { + return convertTilde(version.substring(1)); + } + if (version.matches("^\\d.*")) { + return "==" + version; + } + // Already PEP 440 compatible (>=, ==, ~=, !=, etc.) + return version; + } + + private static int parseNumericPart(String part) { + return Integer.parseInt(part.replaceAll("[^0-9].*", "")); + } + + private static String convertCaret(String ver) { + String[] parts = ver.split("\\."); + int major = parseNumericPart(parts[0]); + int minor = parts.length > 1 ? parseNumericPart(parts[1]) : 0; + int patch = parts.length > 2 ? parseNumericPart(parts[2]) : 0; + String fullVer = major + "." + minor + "." + patch; + + if (major > 0) { + return ">=" + fullVer + ",<" + (major + 1) + ".0.0"; + } + if (minor > 0) { + return ">=" + fullVer + ",<0." + (minor + 1) + ".0"; + } + return ">=" + fullVer + ",<0.0." + (patch + 1); + } + + private static String convertTilde(String ver) { + String[] parts = ver.split("\\."); + int major = parseNumericPart(parts[0]); + int minor = parts.length > 1 ? parseNumericPart(parts[1]) : 0; + int patch = parts.length > 2 ? parseNumericPart(parts[2]) : 0; + String fullVer = major + "." + minor + "." + patch; + return ">=" + fullVer + ",<" + major + "." + (minor + 1) + ".0"; + } +} diff --git a/src/main/java/io/github/guacsec/trustifyda/tools/Ecosystem.java b/src/main/java/io/github/guacsec/trustifyda/tools/Ecosystem.java index 698cd78f..16650f79 100644 --- a/src/main/java/io/github/guacsec/trustifyda/tools/Ecosystem.java +++ b/src/main/java/io/github/guacsec/trustifyda/tools/Ecosystem.java @@ -23,6 +23,7 @@ import io.github.guacsec.trustifyda.providers.JavaMavenProvider; import io.github.guacsec.trustifyda.providers.JavaScriptProviderFactory; import io.github.guacsec.trustifyda.providers.PythonPipProvider; +import io.github.guacsec.trustifyda.providers.PythonPyprojectProvider; import java.nio.file.Path; /** Utility class used for instantiating providers. * */ @@ -85,6 +86,7 @@ private static Provider resolveProvider(final Path manifestPath) { case "package.json" -> JavaScriptProviderFactory.create(manifestPath); case "go.mod" -> new GoModulesProvider(manifestPath); case "requirements.txt" -> new PythonPipProvider(manifestPath); + case "pyproject.toml" -> new PythonPyprojectProvider(manifestPath); case "build.gradle", "build.gradle.kts" -> new GradleProvider(manifestPath); case "Cargo.toml" -> new CargoProvider(manifestPath); default -> diff --git a/src/test/java/io/github/guacsec/trustifyda/providers/Python_Pyproject_Provider_Test.java b/src/test/java/io/github/guacsec/trustifyda/providers/Python_Pyproject_Provider_Test.java new file mode 100644 index 00000000..5e71562d --- /dev/null +++ b/src/test/java/io/github/guacsec/trustifyda/providers/Python_Pyproject_Provider_Test.java @@ -0,0 +1,193 @@ +/* + * Copyright 2023-2025 Trustify Dependency Analytics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.guacsec.trustifyda.providers; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; + +import io.github.guacsec.trustifyda.Api; +import io.github.guacsec.trustifyda.ExhortTest; +import io.github.guacsec.trustifyda.tools.Ecosystem; +import io.github.guacsec.trustifyda.utils.PythonControllerTestEnv; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +class Python_Pyproject_Provider_Test extends ExhortTest { + + @Test + void test_ecosystem_resolves_pyproject_toml() { + var provider = + Ecosystem.getProvider( + Path.of( + "src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml")); + assertThat(provider).isInstanceOf(PythonPyprojectProvider.class); + assertThat(provider.ecosystem).isEqualTo(Ecosystem.Type.PYTHON); + } + + @Test + void test_parse_pep621_dependencies() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml"); + var provider = new PythonPyprojectProvider(pyprojectPath); + List deps = provider.parseDependencyStrings(); + assertThat(deps).contains("anyio==3.6.2", "flask==2.0.3", "requests==2.25.1"); + } + + @Test + void test_parse_pep621_excludes_optional_dependencies() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml"); + var provider = new PythonPyprojectProvider(pyprojectPath); + List deps = provider.parseDependencyStrings(); + assertThat(deps).doesNotContain("click==8.0.4"); + } + + @Test + void test_parse_poetry_dependencies_converts_to_pep440() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry/pyproject.toml"); + var provider = new PythonPyprojectProvider(pyprojectPath); + List deps = provider.parseDependencyStrings(); + assertThat(deps) + .contains("anyio>=3.6.2,<4.0.0", "flask>=2.0.3,<3.0.0", "requests>=2.25.1,<3.0.0"); + } + + @Test + void test_parse_poetry_excludes_python() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry/pyproject.toml"); + var provider = new PythonPyprojectProvider(pyprojectPath); + List deps = provider.parseDependencyStrings(); + assertThat(deps).doesNotContain("python"); + } + + @Test + void test_parse_poetry_excludes_dev_group_dependencies() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry/pyproject.toml"); + var provider = new PythonPyprojectProvider(pyprojectPath); + List deps = provider.parseDependencyStrings(); + assertThat(deps).doesNotContain("click", "click>=8.0.4,<9.0.0"); + } + + @Test + void test_convert_caret_major() { + assertThat(PythonPyprojectProvider.convertPoetryVersion("^3.6.2")).isEqualTo(">=3.6.2,<4.0.0"); + } + + @Test + void test_convert_caret_zero_major() { + assertThat(PythonPyprojectProvider.convertPoetryVersion("^0.5.1")).isEqualTo(">=0.5.1,<0.6.0"); + } + + @Test + void test_convert_caret_zero_zero() { + assertThat(PythonPyprojectProvider.convertPoetryVersion("^0.0.3")).isEqualTo(">=0.0.3,<0.0.4"); + } + + @Test + void test_convert_caret_two_parts() { + assertThat(PythonPyprojectProvider.convertPoetryVersion("^2.0")).isEqualTo(">=2.0.0,<3.0.0"); + } + + @Test + void test_convert_tilde() { + assertThat(PythonPyprojectProvider.convertPoetryVersion("~1.2.3")).isEqualTo(">=1.2.3,<1.3.0"); + } + + @Test + void test_convert_tilde_two_parts() { + assertThat(PythonPyprojectProvider.convertPoetryVersion("~1.2")).isEqualTo(">=1.2.0,<1.3.0"); + } + + @Test + void test_pep440_passthrough() { + assertThat(PythonPyprojectProvider.convertPoetryVersion(">=2.0")).isEqualTo(">=2.0"); + assertThat(PythonPyprojectProvider.convertPoetryVersion("==1.0.0")).isEqualTo("==1.0.0"); + assertThat(PythonPyprojectProvider.convertPoetryVersion("~=1.4")).isEqualTo("~=1.4"); + } + + @Test + void test_convert_bare_version_prepends_equals() { + assertThat(PythonPyprojectProvider.convertPoetryVersion("1.2.3")).isEqualTo("==1.2.3"); + assertThat(PythonPyprojectProvider.convertPoetryVersion("2.0")).isEqualTo("==2.0"); + } + + @Test + void test_convert_caret_prerelease_does_not_crash() { + assertThatNoException() + .isThrownBy(() -> PythonPyprojectProvider.convertPoetryVersion("^1.2.3b1")); + assertThat(PythonPyprojectProvider.convertPoetryVersion("^1.2.3b1")) + .isEqualTo(">=1.2.3,<2.0.0"); + } + + @Test + void test_convert_tilde_prerelease_does_not_crash() { + assertThatNoException() + .isThrownBy(() -> PythonPyprojectProvider.convertPoetryVersion("~1.2.3rc1")); + assertThat(PythonPyprojectProvider.convertPoetryVersion("~1.2.3rc1")) + .isEqualTo(">=1.2.3,<1.3.0"); + } + + @Test + void test_ignored_deps_collected_during_parsing() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_ignore/pyproject.toml"); + var provider = new PythonPyprojectProvider(pyprojectPath); + provider.parseDependencyStrings(); + String manifestContent = Files.readString(pyprojectPath); + var ignored = provider.getIgnoredDependencies(manifestContent); + Set ignoredNames = + ignored.stream().map(purl -> purl.getName()).collect(Collectors.toSet()); + assertThat(ignoredNames).contains("flask"); + assertThat(ignoredNames).doesNotContain("anyio", "requests"); + } + + @Test + void test_provideComponent_generates_correct_media_type() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml"); + var tmpDir = Files.createTempDirectory("trustify_da_test_"); + var tmpFile = Files.createFile(tmpDir.resolve("pyproject.toml")); + Files.copy(pyprojectPath, tmpFile, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + var provider = new PythonPyprojectProvider(tmpFile); + var mockController = + new PythonControllerTestEnv( + io.github.guacsec.trustifyda.tools.Operations.getCustomPathOrElse("python3"), + io.github.guacsec.trustifyda.tools.Operations.getCustomPathOrElse("pip3")); + provider.setPythonController(mockController); + try { + var content = provider.provideComponent(); + assertThat(content.type).isEqualTo(Api.CYCLONEDX_MEDIA_TYPE); + String sbomJson = new String(content.buffer); + assertThat(sbomJson).contains("CycloneDX"); + assertThat(sbomJson).contains("pkg:pypi/"); + } catch (RuntimeException e) { + Assumptions.assumeTrue( + false, "Skipping: Python/pip environment not usable - " + e.getMessage()); + } finally { + Files.deleteIfExists(tmpFile); + Files.deleteIfExists(tmpDir); + } + } +} diff --git a/src/test/resources/tst_manifests/pip/pip_pyproject_toml_ignore/pyproject.toml b/src/test/resources/tst_manifests/pip/pip_pyproject_toml_ignore/pyproject.toml new file mode 100644 index 00000000..2c3abc88 --- /dev/null +++ b/src/test/resources/tst_manifests/pip/pip_pyproject_toml_ignore/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "test-project" +version = "0.1.0" +dependencies = [ + "anyio==3.6.2", + "flask==2.0.3", # exhortignore + "requests==2.25.1", +] + +[project.optional-dependencies] +dev = [ + "click==8.0.4", +] diff --git a/src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml b/src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml new file mode 100644 index 00000000..0f01e703 --- /dev/null +++ b/src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "test-project" +version = "0.1.0" +dependencies = [ + "anyio==3.6.2", + "flask==2.0.3", + "requests==2.25.1", +] + +[project.optional-dependencies] +dev = [ + "click==8.0.4", +] diff --git a/src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry/pyproject.toml b/src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry/pyproject.toml new file mode 100644 index 00000000..fa86a74a --- /dev/null +++ b/src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry/pyproject.toml @@ -0,0 +1,12 @@ +[tool.poetry] +name = "test-project" +version = "0.1.0" + +[tool.poetry.dependencies] +python = "^3.9" +anyio = "^3.6.2" +flask = "^2.0.3" +requests = "^2.25.1" + +[tool.poetry.group.dev.dependencies] +click = "^8.0.4"