diff --git a/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java b/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java index 6c40dc2ed..eed833479 100644 --- a/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java +++ b/gitlab4j-api/src/main/java/org/gitlab4j/api/Pager.java @@ -300,7 +300,7 @@ public List page(int pageNumber) { currentPage = 1; } - if (pageNumber > totalPages && pageNumber > kaminariNextPage) { + if (pageNumber > totalPages && pageNumber > kaminariNextPage && !api.gitLabApi.isDefaultPageFetchParallel()) { throw new NoSuchElementException(); } else if (pageNumber < 1) { throw new NoSuchElementException(); @@ -388,22 +388,39 @@ private List fetchAllParallel() throws GitLabApiException { throw new IllegalStateException("no parallel task executor set, cannot fetch pages in parallel"); } - List>> tasks = new ArrayList<>(); - for (int i = 1; i <= totalPages; i++) { - final int pageNumber = i; - tasks.add(() -> page(pageNumber)); + int maxTasks = taskExecutor.getParallelCount(); + if (totalPages != -1) { + maxTasks = Math.min(totalPages, taskExecutor.getParallelCount()); } - try { - List> results = taskExecutor.execute(tasks); - for (List items : results) { - allItems.addAll(items); + + int taskNr = 1; + boolean allPagesFetched = false; + + while (!allPagesFetched) { + List>> tasks = new ArrayList<>(maxTasks); + + while (tasks.size() < maxTasks) { + final int pageNumber = taskNr++; + tasks.add(() -> page(pageNumber)); + } + try { + List> results = taskExecutor.execute(tasks); + for (List items : results) { + if (items.isEmpty()) { + allPagesFetched = true; + } else { + allItems.addAll(items); + allPagesFetched = items.size() < itemsPerPage; + } + } + } catch (GitLabApiException ge) { + throw ge; + } catch (Exception e) { + throw new GitLabApiException(e); } - return allItems; - } catch (GitLabApiException ge) { - throw ge; - } catch (Exception e) { - throw new GitLabApiException(e); } + + return allItems; } private List fetchAllSynchronously() { diff --git a/gitlab4j-api/src/main/java/org/gitlab4j/api/ParallelTaskExecutor.java b/gitlab4j-api/src/main/java/org/gitlab4j/api/ParallelTaskExecutor.java index fc51d4b9b..605ea7135 100644 --- a/gitlab4j-api/src/main/java/org/gitlab4j/api/ParallelTaskExecutor.java +++ b/gitlab4j-api/src/main/java/org/gitlab4j/api/ParallelTaskExecutor.java @@ -5,4 +5,6 @@ public interface ParallelTaskExecutor { public List execute(List> tasks) throws Exception; + + public int getParallelCount(); }