From aca11660cdcbf9b485af1b95502a3bc136fa39c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Pereira?= Date: Thu, 17 Apr 2025 16:56:09 +0200 Subject: [PATCH 1/3] Change fetchAllParallel implementation to ignore Gitlab response headers that are not always present Instead, it fetches while there is something to fetch. --- .../src/main/java/org/gitlab4j/api/Pager.java | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java b/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java index 6c40dc2ed..a715983bd 100644 --- a/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java +++ b/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java @@ -300,7 +300,7 @@ public List page(int pageNumber) { currentPage = 1; } - if (pageNumber > totalPages && pageNumber > kaminariNextPage) { + if (pageNumber > totalPages && pageNumber > kaminariNextPage && !api.gitLabApi.isDefaultPageFetchParallel()) { throw new NoSuchElementException(); } else if (pageNumber < 1) { throw new NoSuchElementException(); @@ -388,22 +388,34 @@ private List fetchAllParallel() throws GitLabApiException { throw new IllegalStateException("no parallel task executor set, cannot fetch pages in parallel"); } - List>> tasks = new ArrayList<>(); - for (int i = 1; i <= totalPages; i++) { - final int pageNumber = i; - tasks.add(() -> page(pageNumber)); - } - try { - List> results = taskExecutor.execute(tasks); - for (List items : results) { - allItems.addAll(items); + int taskNr = 1; + boolean allPagesFetched = false; + + while (!allPagesFetched) { + List>> tasks = new ArrayList<>(); + + while(tasks.size() < 100) { + final int pageNumber = taskNr++; + tasks.add(() -> page(pageNumber)); + } + try { + List> results = taskExecutor.execute(tasks); + for (List items : results) { + if (items.isEmpty()) { + allPagesFetched = true; + break; + } else { + allItems.addAll(items); + } + } + } catch (GitLabApiException ge) { + throw ge; + } catch (Exception e) { + throw new GitLabApiException(e); } - return allItems; - } catch (GitLabApiException ge) { - throw ge; - } catch (Exception e) { - throw new GitLabApiException(e); } + + return allItems; } private List fetchAllSynchronously() { From 5915997fe3377f0e61d40c3c7e556a66d417a778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Pereira?= Date: Tue, 22 Apr 2025 15:56:33 +0200 Subject: [PATCH 2/3] Fix java Spotless violations --- gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java b/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java index a715983bd..5965683eb 100644 --- a/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java +++ b/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java @@ -394,7 +394,7 @@ private List fetchAllParallel() throws GitLabApiException { while (!allPagesFetched) { List>> tasks = new ArrayList<>(); - while(tasks.size() < 100) { + while (tasks.size() < 100) { final int pageNumber = taskNr++; tasks.add(() -> page(pageNumber)); } From 5b27856bffae3bedaee743ae7419e49bd797deff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Pereira?= Date: Fri, 25 Apr 2025 08:19:16 +0200 Subject: [PATCH 3/3] Optimize number of parallel tasks and make stopping condition more robust --- .../src/main/java/org/gitlab4j/api/Pager.java | 11 ++++++++--- .../java/org/gitlab4j/api/ParallelTaskExecutor.java | 2 ++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java b/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java index 5965683eb..eed833479 100644 --- a/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java +++ b/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java @@ -388,13 +388,18 @@ private List fetchAllParallel() throws GitLabApiException { throw new IllegalStateException("no parallel task executor set, cannot fetch pages in parallel"); } + int maxTasks = taskExecutor.getParallelCount(); + if (totalPages != -1) { + maxTasks = Math.min(totalPages, taskExecutor.getParallelCount()); + } + int taskNr = 1; boolean allPagesFetched = false; while (!allPagesFetched) { - List>> tasks = new ArrayList<>(); + List>> tasks = new ArrayList<>(maxTasks); - while (tasks.size() < 100) { + while (tasks.size() < maxTasks) { final int pageNumber = taskNr++; tasks.add(() -> page(pageNumber)); } @@ -403,9 +408,9 @@ private List fetchAllParallel() throws GitLabApiException { for (List items : results) { if (items.isEmpty()) { allPagesFetched = true; - break; } else { allItems.addAll(items); + allPagesFetched = items.size() < itemsPerPage; } } } catch (GitLabApiException ge) { diff --git a/gitlab4j-api/src/main/java/org/gitlab4j/api/ParallelTaskExecutor.java b/gitlab4j-api/src/main/java/org/gitlab4j/api/ParallelTaskExecutor.java index fc51d4b9b..605ea7135 100644 --- a/gitlab4j-api/src/main/java/org/gitlab4j/api/ParallelTaskExecutor.java +++ b/gitlab4j-api/src/main/java/org/gitlab4j/api/ParallelTaskExecutor.java @@ -5,4 +5,6 @@ public interface ParallelTaskExecutor { public List execute(List> tasks) throws Exception; + + public int getParallelCount(); }