From b75988312dfba64cf9e552694d92399f02887a5d Mon Sep 17 00:00:00 2001 From: ziad hany Date: Mon, 27 Jul 2026 22:08:22 +0300 Subject: [PATCH 1/6] Add initial support for the npm-health ScanGrimoireLab pipeline Signed-off-by: ziad hany --- pyproject.toml | 1 + scancodeio/settings.py | 21 +++ scanpipe/pipelines/metrics_model.py | 75 ++++++++++ scanpipe/pipelines/scan_repo_grimoirelab.py | 133 ++++++++++++++++++ .../tests/pipes/test_scan_repo_grimoirelab.py | 9 ++ 5 files changed, 239 insertions(+) create mode 100644 scanpipe/pipelines/metrics_model.py create mode 100644 scanpipe/pipelines/scan_repo_grimoirelab.py create mode 100644 scanpipe/tests/pipes/test_scan_repo_grimoirelab.py diff --git a/pyproject.toml b/pyproject.toml index fe51c049a1..2a9849cdd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -171,6 +171,7 @@ resolve_dependencies = "scanpipe.pipelines.resolve_dependencies:ResolveDependenc scan_codebase = "scanpipe.pipelines.scan_codebase:ScanCodebase" scan_for_virus = "scanpipe.pipelines.scan_for_virus:ScanForVirus" scan_single_package = "scanpipe.pipelines.scan_single_package:ScanSinglePackage" +scan_repo_grimoirelab = "scanpipe.pipelines.scan_repo_grimoirelab:ScanGrimoirelab" [tool.setuptools.packages.find] where = ["."] diff --git a/scancodeio/settings.py b/scancodeio/settings.py index f869892140..9b5e72788d 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -362,3 +362,24 @@ FEDERATEDCODE_GIT_SERVICE_TOKEN = env.str("FEDERATEDCODE_GIT_SERVICE_TOKEN", default="") FEDERATEDCODE_GIT_SERVICE_NAME = env.str("FEDERATEDCODE_GIT_SERVICE_NAME", default="") FEDERATEDCODE_GIT_SERVICE_EMAIL = env.str("FEDERATEDCODE_GIT_SERVICE_EMAIL", default="") + +# GrimoireLab integration + +GRIMOIRELAB_METRICS_EXECUTABLE = env.str("GRIMOIRELAB_METRICS_EXECUTABLE", default="") +GRIMOIRELAB_URL = env.str("GRIMOIRELAB_URL", default="") +GRIMOIRELAB_USERNAME = env.str("GRIMOIRELAB_USERNAME", default="") +GRIMOIRELAB_PASSWORD = env.str("GRIMOIRELAB_PASSWORD", default="") +GRIMOIRELAB_OPENSEARCH_URL = env.str("GRIMOIRELAB_OPENSEARCH_URL", default="") +GRIMOIRELAB_OPENSEARCH_INDEX = env.str("GRIMOIRELAB_OPENSEARCH_INDEX", default="") +GRIMOIRELAB_OPENSEARCH_USERNAME = env.str("GRIMOIRELAB_OPENSEARCH_USERNAME", default="") +GRIMOIRELAB_OPENSEARCH_PASSWORD = env.str("GRIMOIRELAB_OPENSEARCH_PASSWORD", default="") +GRIMOIRELAB_FROM_DATE = env.str("GRIMOIRELAB_FROM_DATE", default="") +GRIMOIRELAB_TO_DATE = env.str("GRIMOIRELAB_TO_DATE", default="") +GRIMOIRELAB_REPOSITORY_TIMEOUT = env.str("GRIMOIRELAB_REPOSITORY_TIMEOUT", default="") +GRIMOIRELAB_CODE_FILE_PATTERN = env.str("GRIMOIRELAB_CODE_FILE_PATTERN", default="") +GRIMOIRELAB_BINARY_FILE_PATTERN = env.str("GRIMOIRELAB_BINARY_FILE_PATTERN", default="") +GRIMOIRELAB_PONY_THRESHOLD = env.str("GRIMOIRELAB_PONY_THRESHOLD", default="") +GRIMOIRELAB_ELEPHANT_THRESHOLD = env.str("GRIMOIRELAB_ELEPHANT_THRESHOLD", default="") +GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS = env.str( + "GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS", default="" +) diff --git a/scanpipe/pipelines/metrics_model.py b/scanpipe/pipelines/metrics_model.py new file mode 100644 index 0000000000..7aefb6756b --- /dev/null +++ b/scanpipe/pipelines/metrics_model.py @@ -0,0 +1,75 @@ +# +# Copyright (C) AboutCode +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import math + +# These coefficients were calculated with the notebooks and data available +# at https://github.com/aboutcode-org/healthycode/blob/main/model/npm/README.md + + +class npmModel: + # We have dropped the low-impact metrics, those with a coefficient close to 0 + COEFFICIENTS = { + "elephant_factor": -1.635941, + "coefficient_of_variation": -1.404157, + "total_contributors": -0.991894, + "days_since_last_commit": 0.865738, + "contributor_growth_rate": 0.435875, + "commits_over_periods_rate": -0.410393, + "total_commits": -0.330035, + "message_size_mean": -0.320026, + "found_file_license": 0.266483, + } + + # Model Intercept + Z = -0.549873845969752 + + def __init__(self): + self.coefficients = self.COEFFICIENTS.copy() + self.z = self.Z + + def calculate_score(self, metrics: dict[str, float]) -> float: + """ + Calculates the probability of a repository being 'Unhealthy' based on + the pruned logistic regression model metrics. + + Parameters + ---------- + metrics (dict): Dictionary containing the project feature names and values. + + Returns + ------- + float: Probability score between 0.0 (Healthy) and 1.0 (Unhealthy). + + """ + z = self.z + + # Calculate the linear combination (log-odds) + for metric, coef in self.coefficients.items(): + # FIXME. We set by default 0 if a metric is missing. Is this safe? + value = metrics.get(metric, 0.0) + z += coef * value + + # Apply the Sigmoid function to get the final probability + try: + probability = 1 / (1 + math.exp(-z)) + except OverflowError: + # Safeguard against extreme values of z + # FIXME Is this correct? + probability = 0.0 if z < 0 else 1.0 + + return probability diff --git a/scanpipe/pipelines/scan_repo_grimoirelab.py b/scanpipe/pipelines/scan_repo_grimoirelab.py new file mode 100644 index 0000000000..f293b3a96a --- /dev/null +++ b/scanpipe/pipelines/scan_repo_grimoirelab.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import json +import subprocess + +from scancodeio.settings import GRIMOIRELAB_BINARY_FILE_PATTERN +from scancodeio.settings import GRIMOIRELAB_CODE_FILE_PATTERN +from scancodeio.settings import GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS +from scancodeio.settings import GRIMOIRELAB_ELEPHANT_THRESHOLD +from scancodeio.settings import GRIMOIRELAB_FROM_DATE +from scancodeio.settings import GRIMOIRELAB_METRICS_EXECUTABLE +from scancodeio.settings import GRIMOIRELAB_OPENSEARCH_INDEX +from scancodeio.settings import GRIMOIRELAB_OPENSEARCH_PASSWORD +from scancodeio.settings import GRIMOIRELAB_OPENSEARCH_URL +from scancodeio.settings import GRIMOIRELAB_OPENSEARCH_USERNAME +from scancodeio.settings import GRIMOIRELAB_PASSWORD +from scancodeio.settings import GRIMOIRELAB_PONY_THRESHOLD +from scancodeio.settings import GRIMOIRELAB_REPOSITORY_TIMEOUT +from scancodeio.settings import GRIMOIRELAB_TO_DATE +from scancodeio.settings import GRIMOIRELAB_URL +from scancodeio.settings import GRIMOIRELAB_USERNAME +from scanpipe.pipelines import Pipeline +from scanpipe.pipelines.metrics_model import npmModel + + +class ScanGrimoirelab(Pipeline): + results_url = "/project/{slug}/resources/?extra_data=grimoire_data" + + @classmethod + def steps(cls): + return ( + cls.collect_grimoire_metric, + cls.compute_and_store_metric_score, + ) + + def collect_grimoire_metric(self): + metrics_output_path = self.project.get_output_file_path("metrics", "json") + repo_url = "https://github.com/aboutcode-org/fetchcode.git" + + cmd = [ + GRIMOIRELAB_METRICS_EXECUTABLE, + repo_url, + "--grimoirelab-url", + GRIMOIRELAB_URL, + "--grimoirelab-user", + GRIMOIRELAB_USERNAME, + "--grimoirelab-password", + GRIMOIRELAB_PASSWORD, + "--opensearch-url", + GRIMOIRELAB_OPENSEARCH_URL, + "--opensearch-index", + GRIMOIRELAB_OPENSEARCH_INDEX, + "--opensearch-user", + GRIMOIRELAB_OPENSEARCH_USERNAME, + "--opensearch-password", + GRIMOIRELAB_OPENSEARCH_PASSWORD, + "--from-date", + GRIMOIRELAB_FROM_DATE, + "--to-date", + GRIMOIRELAB_TO_DATE, + "--repository-timeout", + GRIMOIRELAB_REPOSITORY_TIMEOUT, + "--code-file-pattern", + GRIMOIRELAB_CODE_FILE_PATTERN, + "--binary-file-pattern", + GRIMOIRELAB_BINARY_FILE_PATTERN, + "--pony-threshold", + GRIMOIRELAB_PONY_THRESHOLD, + "--elephant-threshold", + GRIMOIRELAB_ELEPHANT_THRESHOLD, + "--dev-categories-thresholds", + *GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS, + "--output", + str(metrics_output_path), + ] + + try: + subprocess.run( + cmd, + capture_output=True, + text=True, + check=True, + ) + self.log(f"Metrics successfully saved to {metrics_output_path}") + with open(metrics_output_path) as f: + self.metrics = json.load(f) + + except subprocess.CalledProcessError as e: + self.log(f"failed with exit code {e.returncode}") + raise + except FileNotFoundError: + self.log( + "Error: 'grimoirelab-metrics' command not found. Is it installed and on your PATH?" + ) + raise + + def compute_and_store_metric_score(self): + model = npmModel() + probability = model.calculate_score(self.metrics) + status = "Healthy" if probability >= 0.5 else "Unhealthy" + + self.log(f"Repository Health: {status}, Probability: {probability:.2%}") + score_data = { + "status": status, + "probability": probability, + "metrics": self.metrics, + } + + score_output_path = self.project.get_output_file_path("results", "json") + with open(score_output_path, "w") as f: + json.dump(score_data, f, indent=2) + + return score_data diff --git a/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py b/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py new file mode 100644 index 0000000000..273df9b8b9 --- /dev/null +++ b/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py @@ -0,0 +1,9 @@ +from django.test import TestCase + + +class ScanGrimoirelabTest(TestCase): + def test_collect_grimoire_metric(self): + raise NotImplementedError + + def test_compute_and_store_metric_score(self): + raise NotImplementedError From cb46931a6ad717fb753f96204bd81686a4019132 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 4 Aug 2026 01:59:18 +0300 Subject: [PATCH 2/6] Remove metrics_model.py Add a test for ScanRepoGrimoirelabTest Remove hardcoded env variable Signed-off-by: ziad hany --- pyproject.toml | 2 +- scancodeio/settings.py | 2 +- scanpipe/pipelines/metrics_model.py | 75 --------- scanpipe/pipelines/scan_repo_grimoirelab.py | 142 +++++++----------- .../tests/pipes/test_scan_repo_grimoirelab.py | 47 +++++- 5 files changed, 102 insertions(+), 166 deletions(-) delete mode 100644 scanpipe/pipelines/metrics_model.py diff --git a/pyproject.toml b/pyproject.toml index 2a9849cdd1..327745a19c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -171,7 +171,7 @@ resolve_dependencies = "scanpipe.pipelines.resolve_dependencies:ResolveDependenc scan_codebase = "scanpipe.pipelines.scan_codebase:ScanCodebase" scan_for_virus = "scanpipe.pipelines.scan_for_virus:ScanForVirus" scan_single_package = "scanpipe.pipelines.scan_single_package:ScanSinglePackage" -scan_repo_grimoirelab = "scanpipe.pipelines.scan_repo_grimoirelab:ScanGrimoirelab" +scan_repo_grimoirelab = "scanpipe.pipelines.scan_repo_grimoirelab:ScanRepoGrimoirelab" [tool.setuptools.packages.find] where = ["."] diff --git a/scancodeio/settings.py b/scancodeio/settings.py index 9b5e72788d..482d7cc60f 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -382,4 +382,4 @@ GRIMOIRELAB_ELEPHANT_THRESHOLD = env.str("GRIMOIRELAB_ELEPHANT_THRESHOLD", default="") GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS = env.str( "GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS", default="" -) +).split(",") diff --git a/scanpipe/pipelines/metrics_model.py b/scanpipe/pipelines/metrics_model.py deleted file mode 100644 index 7aefb6756b..0000000000 --- a/scanpipe/pipelines/metrics_model.py +++ /dev/null @@ -1,75 +0,0 @@ -# -# Copyright (C) AboutCode -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -import math - -# These coefficients were calculated with the notebooks and data available -# at https://github.com/aboutcode-org/healthycode/blob/main/model/npm/README.md - - -class npmModel: - # We have dropped the low-impact metrics, those with a coefficient close to 0 - COEFFICIENTS = { - "elephant_factor": -1.635941, - "coefficient_of_variation": -1.404157, - "total_contributors": -0.991894, - "days_since_last_commit": 0.865738, - "contributor_growth_rate": 0.435875, - "commits_over_periods_rate": -0.410393, - "total_commits": -0.330035, - "message_size_mean": -0.320026, - "found_file_license": 0.266483, - } - - # Model Intercept - Z = -0.549873845969752 - - def __init__(self): - self.coefficients = self.COEFFICIENTS.copy() - self.z = self.Z - - def calculate_score(self, metrics: dict[str, float]) -> float: - """ - Calculates the probability of a repository being 'Unhealthy' based on - the pruned logistic regression model metrics. - - Parameters - ---------- - metrics (dict): Dictionary containing the project feature names and values. - - Returns - ------- - float: Probability score between 0.0 (Healthy) and 1.0 (Unhealthy). - - """ - z = self.z - - # Calculate the linear combination (log-odds) - for metric, coef in self.coefficients.items(): - # FIXME. We set by default 0 if a metric is missing. Is this safe? - value = metrics.get(metric, 0.0) - z += coef * value - - # Apply the Sigmoid function to get the final probability - try: - probability = 1 / (1 + math.exp(-z)) - except OverflowError: - # Safeguard against extreme values of z - # FIXME Is this correct? - probability = 0.0 if z < 0 else 1.0 - - return probability diff --git a/scanpipe/pipelines/scan_repo_grimoirelab.py b/scanpipe/pipelines/scan_repo_grimoirelab.py index f293b3a96a..798674dde3 100644 --- a/scanpipe/pipelines/scan_repo_grimoirelab.py +++ b/scanpipe/pipelines/scan_repo_grimoirelab.py @@ -20,8 +20,8 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -import json import subprocess +from datetime import date from scancodeio.settings import GRIMOIRELAB_BINARY_FILE_PATTERN from scancodeio.settings import GRIMOIRELAB_CODE_FILE_PATTERN @@ -36,98 +36,72 @@ from scancodeio.settings import GRIMOIRELAB_PASSWORD from scancodeio.settings import GRIMOIRELAB_PONY_THRESHOLD from scancodeio.settings import GRIMOIRELAB_REPOSITORY_TIMEOUT -from scancodeio.settings import GRIMOIRELAB_TO_DATE from scancodeio.settings import GRIMOIRELAB_URL from scancodeio.settings import GRIMOIRELAB_USERNAME from scanpipe.pipelines import Pipeline -from scanpipe.pipelines.metrics_model import npmModel +from scanpipe.pipes import run_command_safely -class ScanGrimoirelab(Pipeline): +class ScanRepoGrimoirelab(Pipeline): + """ + Run a GrimoireLab scan on a specific Git repository URL to extract its metrics and health score. + """ + results_url = "/project/{slug}/resources/?extra_data=grimoire_data" @classmethod def steps(cls): - return ( - cls.collect_grimoire_metric, - cls.compute_and_store_metric_score, - ) - - def collect_grimoire_metric(self): - metrics_output_path = self.project.get_output_file_path("metrics", "json") - repo_url = "https://github.com/aboutcode-org/fetchcode.git" - - cmd = [ - GRIMOIRELAB_METRICS_EXECUTABLE, - repo_url, - "--grimoirelab-url", - GRIMOIRELAB_URL, - "--grimoirelab-user", - GRIMOIRELAB_USERNAME, - "--grimoirelab-password", - GRIMOIRELAB_PASSWORD, - "--opensearch-url", - GRIMOIRELAB_OPENSEARCH_URL, - "--opensearch-index", - GRIMOIRELAB_OPENSEARCH_INDEX, - "--opensearch-user", - GRIMOIRELAB_OPENSEARCH_USERNAME, - "--opensearch-password", - GRIMOIRELAB_OPENSEARCH_PASSWORD, - "--from-date", - GRIMOIRELAB_FROM_DATE, - "--to-date", - GRIMOIRELAB_TO_DATE, - "--repository-timeout", - GRIMOIRELAB_REPOSITORY_TIMEOUT, - "--code-file-pattern", - GRIMOIRELAB_CODE_FILE_PATTERN, - "--binary-file-pattern", - GRIMOIRELAB_BINARY_FILE_PATTERN, - "--pony-threshold", - GRIMOIRELAB_PONY_THRESHOLD, - "--elephant-threshold", - GRIMOIRELAB_ELEPHANT_THRESHOLD, - "--dev-categories-thresholds", - *GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS, - "--output", - str(metrics_output_path), - ] - - try: - subprocess.run( - cmd, - capture_output=True, - text=True, - check=True, - ) - self.log(f"Metrics successfully saved to {metrics_output_path}") - with open(metrics_output_path) as f: - self.metrics = json.load(f) - - except subprocess.CalledProcessError as e: - self.log(f"failed with exit code {e.returncode}") - raise - except FileNotFoundError: - self.log( - "Error: 'grimoirelab-metrics' command not found. Is it installed and on your PATH?" - ) - raise - - def compute_and_store_metric_score(self): - model = npmModel() - probability = model.calculate_score(self.metrics) - status = "Healthy" if probability >= 0.5 else "Unhealthy" + return (cls.collect_and_store_grimoire_metric,) - self.log(f"Repository Health: {status}, Probability: {probability:.2%}") - score_data = { - "status": status, - "probability": probability, - "metrics": self.metrics, - } + def collect_and_store_grimoire_metric(self): + for input_source in self.project.input_sources: + repo_url = input_source["download_url"] + metrics_output_path = self.project.get_output_file_path("metrics", "json") + grimoirelab_to_date = date.today().isoformat() - score_output_path = self.project.get_output_file_path("results", "json") - with open(score_output_path, "w") as f: - json.dump(score_data, f, indent=2) + command_args = [ + GRIMOIRELAB_METRICS_EXECUTABLE, + repo_url, + "--grimoirelab-url", + GRIMOIRELAB_URL, + "--grimoirelab-user", + GRIMOIRELAB_USERNAME, + "--grimoirelab-password", + GRIMOIRELAB_PASSWORD, + "--opensearch-url", + GRIMOIRELAB_OPENSEARCH_URL, + "--opensearch-index", + GRIMOIRELAB_OPENSEARCH_INDEX, + "--opensearch-user", + GRIMOIRELAB_OPENSEARCH_USERNAME, + "--opensearch-password", + GRIMOIRELAB_OPENSEARCH_PASSWORD, + "--from-date", + GRIMOIRELAB_FROM_DATE, + "--to-date", + grimoirelab_to_date, + "--repository-timeout", + GRIMOIRELAB_REPOSITORY_TIMEOUT, + "--code-file-pattern", + GRIMOIRELAB_CODE_FILE_PATTERN, + "--binary-file-pattern", + GRIMOIRELAB_BINARY_FILE_PATTERN, + "--pony-threshold", + GRIMOIRELAB_PONY_THRESHOLD, + "--elephant-threshold", + GRIMOIRELAB_ELEPHANT_THRESHOLD, + "--dev-categories-thresholds", + *GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS, + "--output", + str(metrics_output_path), + ] - return score_data + try: + run_command_safely(command_args=command_args) + self.log(f"Metrics successfully saved to {metrics_output_path}") + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"grimoirelab-metrics pipeline failed {e.returncode} for {repo_url}" + ) + except subprocess.TimeoutExpired: + raise RuntimeError("grimoirelab-metrics pipeline timed out") diff --git a/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py b/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py index 273df9b8b9..2c62b43f84 100644 --- a/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py +++ b/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py @@ -1,9 +1,46 @@ +import subprocess +from unittest.mock import MagicMock +from unittest.mock import patch + from django.test import TestCase +from scanpipe.pipelines.scan_repo_grimoirelab import ScanRepoGrimoirelab + + +class ScanRepoGrimoirelabTest(TestCase): + def setUp(self): + mock_run = MagicMock() + self.pipeline = ScanRepoGrimoirelab(mock_run) + + self.pipeline.project = MagicMock() + self.pipeline.project.input_sources = [ + {"download_url": "https://github.com/example/repo.git"} + ] + self.pipeline.project.get_output_file_path.return_value = ( + "/tmp/project/metrics.json" + ) + self.pipeline.log = MagicMock() + + @patch("scanpipe.pipelines.scan_repo_grimoirelab.run_command_safely") + def test_collect_and_store_grimoire_metric_called_process_error( + self, mock_run_command + ): + """Test handling of a non-zero exit code (CalledProcessError).""" + mock_run_command.side_effect = subprocess.CalledProcessError( + returncode=1, cmd=["grimoirelab-metrics"] + ) + + expected_msg = "grimoirelab-metrics pipeline failed 1 for https://github.com/example/repo.git" + with self.assertRaisesMessage(RuntimeError, expected_msg): + self.pipeline.collect_and_store_grimoire_metric() -class ScanGrimoirelabTest(TestCase): - def test_collect_grimoire_metric(self): - raise NotImplementedError + @patch("scanpipe.pipelines.scan_repo_grimoirelab.run_command_safely") + def test_collect_and_store_grimoire_metric_timeout(self, mock_run_command): + """Test handling of a command execution timeout.""" + mock_run_command.side_effect = subprocess.TimeoutExpired( + cmd=["grimoirelab-metrics"], timeout=300 + ) - def test_compute_and_store_metric_score(self): - raise NotImplementedError + expected_msg = "grimoirelab-metrics pipeline timed out" + with self.assertRaisesMessage(RuntimeError, expected_msg): + self.pipeline.collect_and_store_grimoire_metric() From c1075d4cc549ae507e7d764e10c6a8eb908a6649 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 4 Aug 2026 15:01:13 +0300 Subject: [PATCH 3/6] Fix Ruff format Signed-off-by: ziad hany --- scanpipe/pipelines/scan_repo_grimoirelab.py | 4 +--- scanpipe/tests/pipes/test_scan_repo_grimoirelab.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/scanpipe/pipelines/scan_repo_grimoirelab.py b/scanpipe/pipelines/scan_repo_grimoirelab.py index 798674dde3..a8af4b8ada 100644 --- a/scanpipe/pipelines/scan_repo_grimoirelab.py +++ b/scanpipe/pipelines/scan_repo_grimoirelab.py @@ -43,9 +43,7 @@ class ScanRepoGrimoirelab(Pipeline): - """ - Run a GrimoireLab scan on a specific Git repository URL to extract its metrics and health score. - """ + """Run a GrimoireLab scan to extract repository metrics and health score.""" results_url = "/project/{slug}/resources/?extra_data=grimoire_data" diff --git a/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py b/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py index 2c62b43f84..abf3549c26 100644 --- a/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py +++ b/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py @@ -16,9 +16,7 @@ def setUp(self): self.pipeline.project.input_sources = [ {"download_url": "https://github.com/example/repo.git"} ] - self.pipeline.project.get_output_file_path.return_value = ( - "/tmp/project/metrics.json" - ) + self.pipeline.project.get_output_file_path.return_value = "metrics.json" self.pipeline.log = MagicMock() @patch("scanpipe.pipelines.scan_repo_grimoirelab.run_command_safely") From 5a893d1f36361ad20002b49931b3f83f5087f328 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 4 Aug 2026 19:55:20 +0300 Subject: [PATCH 4/6] Update grimoirelab and format output Validate VCS URL Add a test Update Grimoirelab to use the minimal setting Signed-off-by: ziad hany --- scancodeio/settings.py | 10 -- scanpipe/pipelines/scan_repo_grimoirelab.py | 156 +++++++++++------- .../data/grimorielab/expected-metrics.json | 38 +++++ scanpipe/tests/data/grimorielab/metrics.json | 65 ++++++++ .../tests/pipes/test_scan_repo_grimoirelab.py | 103 ++++++++++-- 5 files changed, 291 insertions(+), 81 deletions(-) create mode 100644 scanpipe/tests/data/grimorielab/expected-metrics.json create mode 100644 scanpipe/tests/data/grimorielab/metrics.json diff --git a/scancodeio/settings.py b/scancodeio/settings.py index 482d7cc60f..a57cc95d17 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -373,13 +373,3 @@ GRIMOIRELAB_OPENSEARCH_INDEX = env.str("GRIMOIRELAB_OPENSEARCH_INDEX", default="") GRIMOIRELAB_OPENSEARCH_USERNAME = env.str("GRIMOIRELAB_OPENSEARCH_USERNAME", default="") GRIMOIRELAB_OPENSEARCH_PASSWORD = env.str("GRIMOIRELAB_OPENSEARCH_PASSWORD", default="") -GRIMOIRELAB_FROM_DATE = env.str("GRIMOIRELAB_FROM_DATE", default="") -GRIMOIRELAB_TO_DATE = env.str("GRIMOIRELAB_TO_DATE", default="") -GRIMOIRELAB_REPOSITORY_TIMEOUT = env.str("GRIMOIRELAB_REPOSITORY_TIMEOUT", default="") -GRIMOIRELAB_CODE_FILE_PATTERN = env.str("GRIMOIRELAB_CODE_FILE_PATTERN", default="") -GRIMOIRELAB_BINARY_FILE_PATTERN = env.str("GRIMOIRELAB_BINARY_FILE_PATTERN", default="") -GRIMOIRELAB_PONY_THRESHOLD = env.str("GRIMOIRELAB_PONY_THRESHOLD", default="") -GRIMOIRELAB_ELEPHANT_THRESHOLD = env.str("GRIMOIRELAB_ELEPHANT_THRESHOLD", default="") -GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS = env.str( - "GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS", default="" -).split(",") diff --git a/scanpipe/pipelines/scan_repo_grimoirelab.py b/scanpipe/pipelines/scan_repo_grimoirelab.py index a8af4b8ada..e2cb48ae82 100644 --- a/scanpipe/pipelines/scan_repo_grimoirelab.py +++ b/scanpipe/pipelines/scan_repo_grimoirelab.py @@ -19,23 +19,16 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. - +import json import subprocess -from datetime import date +import urllib.parse -from scancodeio.settings import GRIMOIRELAB_BINARY_FILE_PATTERN -from scancodeio.settings import GRIMOIRELAB_CODE_FILE_PATTERN -from scancodeio.settings import GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS -from scancodeio.settings import GRIMOIRELAB_ELEPHANT_THRESHOLD -from scancodeio.settings import GRIMOIRELAB_FROM_DATE from scancodeio.settings import GRIMOIRELAB_METRICS_EXECUTABLE from scancodeio.settings import GRIMOIRELAB_OPENSEARCH_INDEX from scancodeio.settings import GRIMOIRELAB_OPENSEARCH_PASSWORD from scancodeio.settings import GRIMOIRELAB_OPENSEARCH_URL from scancodeio.settings import GRIMOIRELAB_OPENSEARCH_USERNAME from scancodeio.settings import GRIMOIRELAB_PASSWORD -from scancodeio.settings import GRIMOIRELAB_PONY_THRESHOLD -from scancodeio.settings import GRIMOIRELAB_REPOSITORY_TIMEOUT from scancodeio.settings import GRIMOIRELAB_URL from scancodeio.settings import GRIMOIRELAB_USERNAME from scanpipe.pipelines import Pipeline @@ -49,57 +42,98 @@ class ScanRepoGrimoirelab(Pipeline): @classmethod def steps(cls): - return (cls.collect_and_store_grimoire_metric,) + return ( + cls.collect_and_store_grimoire_metric, + cls.format_metrics_output, + ) def collect_and_store_grimoire_metric(self): - for input_source in self.project.input_sources: - repo_url = input_source["download_url"] - metrics_output_path = self.project.get_output_file_path("metrics", "json") - grimoirelab_to_date = date.today().isoformat() - - command_args = [ - GRIMOIRELAB_METRICS_EXECUTABLE, - repo_url, - "--grimoirelab-url", - GRIMOIRELAB_URL, - "--grimoirelab-user", - GRIMOIRELAB_USERNAME, - "--grimoirelab-password", - GRIMOIRELAB_PASSWORD, - "--opensearch-url", - GRIMOIRELAB_OPENSEARCH_URL, - "--opensearch-index", - GRIMOIRELAB_OPENSEARCH_INDEX, - "--opensearch-user", - GRIMOIRELAB_OPENSEARCH_USERNAME, - "--opensearch-password", - GRIMOIRELAB_OPENSEARCH_PASSWORD, - "--from-date", - GRIMOIRELAB_FROM_DATE, - "--to-date", - grimoirelab_to_date, - "--repository-timeout", - GRIMOIRELAB_REPOSITORY_TIMEOUT, - "--code-file-pattern", - GRIMOIRELAB_CODE_FILE_PATTERN, - "--binary-file-pattern", - GRIMOIRELAB_BINARY_FILE_PATTERN, - "--pony-threshold", - GRIMOIRELAB_PONY_THRESHOLD, - "--elephant-threshold", - GRIMOIRELAB_ELEPHANT_THRESHOLD, - "--dev-categories-thresholds", - *GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS, - "--output", - str(metrics_output_path), - ] - - try: - run_command_safely(command_args=command_args) - self.log(f"Metrics successfully saved to {metrics_output_path}") - except subprocess.CalledProcessError as e: - raise RuntimeError( - f"grimoirelab-metrics pipeline failed {e.returncode} for {repo_url}" - ) - except subprocess.TimeoutExpired: - raise RuntimeError("grimoirelab-metrics pipeline timed out") + """ + Run the grimoirelab-metrics command against the input source. + Save the generated metrics JSON to the project output directory. + """ + if len(self.project.input_sources) != 1: + raise ValueError("Expected exactly one input source") + + repo_url = self.project.input_sources[0]["download_url"] + if not is_valid_vcs_url(repo_url): + raise ValueError( + "Invalid input source: the pipeline accepts only a valid repository URL" + ) + + self.metrics_output_path = self.project.get_output_file_path("metrics", "json") + command_args = [ + GRIMOIRELAB_METRICS_EXECUTABLE, + repo_url, + "--grimoirelab-url", + GRIMOIRELAB_URL, + "--grimoirelab-user", + GRIMOIRELAB_USERNAME, + "--grimoirelab-password", + GRIMOIRELAB_PASSWORD, + "--opensearch-url", + GRIMOIRELAB_OPENSEARCH_URL, + "--opensearch-index", + GRIMOIRELAB_OPENSEARCH_INDEX, + "--opensearch-user", + GRIMOIRELAB_OPENSEARCH_USERNAME, + "--opensearch-password", + GRIMOIRELAB_OPENSEARCH_PASSWORD, + "--output", + str(self.metrics_output_path), + ] + + try: + run_command_safely(command_args=command_args) + self.log("GrimoireLab metrics pipeline completed successfully") + except subprocess.SubprocessError: + raise RuntimeError("Grimoirelab-metrics pipeline failed") + except FileNotFoundError: + raise FileNotFoundError( + "Grimoirelab-metrics not found. " + "Please ensure grimoirelab-metrics is correctly configured." + ) + + def format_metrics_output(self): + """ + Format the GrimoireLab metrics output by extracting the repository URL, + score, and metrics from the generated JSON and overwriting it with a + simplified structure. + """ + with open(self.metrics_output_path) as f: + data = json.load(f) + + package = list(data["packages"].values())[0] + + repository = package["repository"] + score = package["score"] + metrics = package["metrics"] + + result = { + "repository": repository, + "npm_health_score": score, + "metrics": metrics, + } + + with open(self.metrics_output_path, "w") as f: + json.dump(result, f) + + +def is_valid_vcs_url(url): + """Determine whether the URL is a valid VCS repository URL.""" + if not isinstance(url, str) or not url: + return False + + if any(char.isspace() for char in url): + return False + + forbidden_chars = ["|", ";", "&", "`", "$(", ">", "<", "&&", "||"] + if any(char in forbidden_chars for char in url): + return False + + parsed = urllib.parse.urlparse(url) + valid_schemes = {"https", "git", "ssh", "git+https", "git+ssh"} + if parsed.scheme in valid_schemes and parsed.netloc: + return True + + return False diff --git a/scanpipe/tests/data/grimorielab/expected-metrics.json b/scanpipe/tests/data/grimorielab/expected-metrics.json new file mode 100644 index 0000000000..f3d89fd501 --- /dev/null +++ b/scanpipe/tests/data/grimorielab/expected-metrics.json @@ -0,0 +1,38 @@ +{ + "metrics": { + "total_commits": 260, + "total_contributors": 33, + "total_organizations": 16, + "pony_factor": 2, + "elephant_factor": 1, + "recent_organizations": 4, + "recent_contributors": 6, + "recent_commits": 69, + "contributor_growth": 7, + "contributor_growth_rate": 0.4666666666666667, + "active_branches": 4, + "days_since_last_commit": 0, + "casual_regular_contributors_rate": 0.65, + "returning_contributors": 4, + "commits_over_periods_rate": 0.2653846153846154, + "coefficient_of_variation": 0.745479473205275, + "file_types_code": 143, + "file_types_binary": 0, + "file_types_other": 2124, + "commit_size_added_lines": 38313, + "commit_size_removed_lines": 62036, + "message_size_total": 35352, + "message_size_mean": 135.96923076923076, + "message_size_median": 88, + "developer_categories_core": 1, + "developer_categories_regular": 19, + "developer_categories_casual": 13, + "commits_per_week": 1.4387351778656126, + "commits_per_month": 6.16600790513834, + "commits_per_year": 75.0197628458498, + "found_file_license": 1, + "found_file_adopters": 1 + }, + "repository": "https://github.com/chaoss/grimoirelab.git", + "npm_health_score": 2.361341589536065e-72 +} \ No newline at end of file diff --git a/scanpipe/tests/data/grimorielab/metrics.json b/scanpipe/tests/data/grimorielab/metrics.json new file mode 100644 index 0000000000..9f1baa1d2f --- /dev/null +++ b/scanpipe/tests/data/grimorielab/metrics.json @@ -0,0 +1,65 @@ +{ + "packages": { + "SPDXRef-Package-grimoirelab": { + "metrics": { + "total_commits": 260, + "total_contributors": 33, + "total_organizations": 16, + "pony_factor": 2, + "elephant_factor": 1, + "recent_organizations": 4, + "recent_contributors": 6, + "recent_commits": 69, + "contributor_growth": 7, + "contributor_growth_rate": 0.4666666666666667, + "active_branches": 4, + "days_since_last_commit": 0, + "casual_regular_contributors_rate": 0.65, + "returning_contributors": 4, + "commits_over_periods_rate": 0.2653846153846154, + "coefficient_of_variation": 0.745479473205275, + "file_types_code": 143, + "file_types_binary": 0, + "file_types_other": 2124, + "commit_size_added_lines": 38313, + "commit_size_removed_lines": 62036, + "message_size_total": 35352, + "message_size_mean": 135.96923076923076, + "message_size_median": 88, + "developer_categories_core": 1, + "developer_categories_regular": 19, + "developer_categories_casual": 13, + "commits_per_week": 1.4387351778656126, + "commits_per_month": 6.16600790513834, + "commits_per_year": 75.0197628458498, + "found_file_license": 1, + "found_file_adopters": 1 + }, + "metadata": { + "first_commit": "fc8754a69b50d1bb6a7a30fafc4fe69bda6e3a10", + "last_commit": "c51f5deb930371f512394af3d95a1a27383a14c2", + "first_commit_date": "2023-01-10T09:54:25+01:00", + "last_commit_date": "2026-06-18T09:38:47+02:00" + }, + "repository": "https://github.com/chaoss/grimoirelab.git", + "score": 2.361341589536065e-72 + } + }, + "metadata": { + "version": "0.1.0", + "started_at": "2026-08-01T02:02:39.612433+00:00", + "finished_at": "2026-08-01T02:02:40.794247+00:00", + "configuration": { + "from_date": "2023-01-01T00:00:00", + "to_date": "2026-06-19T00:00:00", + "code_file_pattern": "\\.py$|\\.js$", + "binary_file_pattern": "\\.exe$|\\.tar$", + "pony_threshold": 0.5, + "elephant_threshold": 0.5, + "dev_categories_thresholds": [ + 0.8, + 0.95 + ] + } + } +} \ No newline at end of file diff --git a/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py b/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py index abf3549c26..2ba957a061 100644 --- a/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py +++ b/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py @@ -1,17 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +import json +import shutil import subprocess +import tempfile +from pathlib import Path from unittest.mock import MagicMock from unittest.mock import patch from django.test import TestCase from scanpipe.pipelines.scan_repo_grimoirelab import ScanRepoGrimoirelab +from scanpipe.pipelines.scan_repo_grimoirelab import is_valid_vcs_url class ScanRepoGrimoirelabTest(TestCase): + data = Path(__file__).parent.parent / "data" / "grimorielab" + def setUp(self): mock_run = MagicMock() self.pipeline = ScanRepoGrimoirelab(mock_run) - self.pipeline.project = MagicMock() self.pipeline.project.input_sources = [ {"download_url": "https://github.com/example/repo.git"} @@ -28,17 +56,72 @@ def test_collect_and_store_grimoire_metric_called_process_error( returncode=1, cmd=["grimoirelab-metrics"] ) - expected_msg = "grimoirelab-metrics pipeline failed 1 for https://github.com/example/repo.git" + expected_msg = "Grimoirelab-metrics pipeline failed" with self.assertRaisesMessage(RuntimeError, expected_msg): self.pipeline.collect_and_store_grimoire_metric() - @patch("scanpipe.pipelines.scan_repo_grimoirelab.run_command_safely") - def test_collect_and_store_grimoire_metric_timeout(self, mock_run_command): - """Test handling of a command execution timeout.""" - mock_run_command.side_effect = subprocess.TimeoutExpired( - cmd=["grimoirelab-metrics"], timeout=300 - ) + def test_invalid_input_sources(self): + """Test handling of invalid number of input sources.""" + self.pipeline.project.input_sources = [] - expected_msg = "grimoirelab-metrics pipeline timed out" - with self.assertRaisesMessage(RuntimeError, expected_msg): + expected_msg = "Expected exactly one input source" + with self.assertRaisesMessage(Exception, expected_msg): + self.pipeline.collect_and_store_grimoire_metric() + + self.pipeline.project.input_sources = [ + {"download_url": "https://github.com/example/repo1.git"}, + {"download_url": "https://github.com/example/repo2.git"}, + ] + + expected_msg = "Expected exactly one input source" + with self.assertRaisesMessage(Exception, expected_msg): self.pipeline.collect_and_store_grimoire_metric() + + def test_format_metrics_output(self): + """Test formatting the GrimoireLab metrics output using fixture files.""" + temp_dir = tempfile.mkdtemp() + + input_file = self.data / "metrics.json" + expected_file = self.data / "expected-metrics.json" + + temp_metrics_path = Path(temp_dir) / "metrics.json" + temp_metrics_path.write_text(input_file.read_text()) + + self.pipeline.metrics_output_path = temp_metrics_path + self.pipeline.format_metrics_output() + + with open(temp_metrics_path) as f: + result = json.load(f) + + with open(expected_file) as f: + expected_result = json.load(f) + + self.assertEqual(expected_result, result) + shutil.rmtree(temp_dir) + + def test_is_valid_vcs_url(self): + """Test VCS repository URL validation.""" + test_cases = [ + # Valid URLs + ("https://github.com/example/repo.git", True), + ("git://github.com/example/repo.git", True), + ("ssh://git@github.com/example/repo.git", True), + ("git+https://github.com/example/repo.git", True), + ("git+ssh://git@github.com/example/repo.git", True), + # Invalid URLs + ("git@github.com:example/repo.git", False), + ("user@localhost:path/to/repo.git", False), + ("hg@bitbucket.org:owner/repo", False), + ("https://github.com/repo.git; rm -rf /", False), + ("https://github.com/repo.git | ls -la", False), + ("https://github.com/repo.git & whoami", False), + ("git@github.com:repo.git\nrm -rf /", False), + ("https://github.com/ repo.git", False), + (" https://github.com/repo.git", False), + ("", False), + (None, False), + ] + + for url, expected in test_cases: + with self.subTest(url=url): + self.assertEqual(is_valid_vcs_url(url), expected) From b949169df3bdb218017c939f839b9ff7e3438a68 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 6 Aug 2026 16:43:31 +0300 Subject: [PATCH 5/6] Fix VCS URL validation Signed-off-by: ziad hany --- scanpipe/pipelines/scan_repo_grimoirelab.py | 4 ++-- scanpipe/tests/pipes/test_scan_repo_grimoirelab.py | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scanpipe/pipelines/scan_repo_grimoirelab.py b/scanpipe/pipelines/scan_repo_grimoirelab.py index e2cb48ae82..55d209bb58 100644 --- a/scanpipe/pipelines/scan_repo_grimoirelab.py +++ b/scanpipe/pipelines/scan_repo_grimoirelab.py @@ -128,11 +128,11 @@ def is_valid_vcs_url(url): return False forbidden_chars = ["|", ";", "&", "`", "$(", ">", "<", "&&", "||"] - if any(char in forbidden_chars for char in url): + if any(char in url for char in forbidden_chars): return False parsed = urllib.parse.urlparse(url) - valid_schemes = {"https", "git", "ssh", "git+https", "git+ssh"} + valid_schemes = {"https", "http", "git"} if parsed.scheme in valid_schemes and parsed.netloc: return True diff --git a/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py b/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py index 2ba957a061..1a68105c5c 100644 --- a/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py +++ b/scanpipe/tests/pipes/test_scan_repo_grimoirelab.py @@ -105,11 +105,7 @@ def test_is_valid_vcs_url(self): # Valid URLs ("https://github.com/example/repo.git", True), ("git://github.com/example/repo.git", True), - ("ssh://git@github.com/example/repo.git", True), - ("git+https://github.com/example/repo.git", True), - ("git+ssh://git@github.com/example/repo.git", True), # Invalid URLs - ("git@github.com:example/repo.git", False), ("user@localhost:path/to/repo.git", False), ("hg@bitbucket.org:owner/repo", False), ("https://github.com/repo.git; rm -rf /", False), @@ -118,6 +114,9 @@ def test_is_valid_vcs_url(self): ("git@github.com:repo.git\nrm -rf /", False), ("https://github.com/ repo.git", False), (" https://github.com/repo.git", False), + ("https://", False), + ("https:///repo.git", False), + ("https://?foo=bar", False), ("", False), (None, False), ] From d6b39f342fb5b6fbd0db3969c2f8b245eef765d8 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 6 Aug 2026 16:55:46 +0300 Subject: [PATCH 6/6] Don't download the input Replace git:// with https:// Signed-off-by: ziad hany --- scanpipe/pipelines/scan_repo_grimoirelab.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scanpipe/pipelines/scan_repo_grimoirelab.py b/scanpipe/pipelines/scan_repo_grimoirelab.py index 55d209bb58..eee1bcf451 100644 --- a/scanpipe/pipelines/scan_repo_grimoirelab.py +++ b/scanpipe/pipelines/scan_repo_grimoirelab.py @@ -39,6 +39,7 @@ class ScanRepoGrimoirelab(Pipeline): """Run a GrimoireLab scan to extract repository metrics and health score.""" results_url = "/project/{slug}/resources/?extra_data=grimoire_data" + download_inputs = False @classmethod def steps(cls): @@ -61,6 +62,7 @@ def collect_and_store_grimoire_metric(self): "Invalid input source: the pipeline accepts only a valid repository URL" ) + repo_url = repo_url.replace("git://", "https://") self.metrics_output_path = self.project.get_output_file_path("metrics", "json") command_args = [ GRIMOIRELAB_METRICS_EXECUTABLE,